From 29791aa3cb6813df2b023a6c8704157585d1e611 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 12:16:42 +0800 Subject: [PATCH 01/37] add loader dry run Signed-off-by: Lenson --- cmd/loader.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmd/loader.go b/cmd/loader.go index 1dc95e64..037bfe0e 100644 --- a/cmd/loader.go +++ b/cmd/loader.go @@ -53,6 +53,7 @@ var ( verbosity = flag.String("verbosity", "info", "Logging verbosity - choose from [info, debug, trace]") iatGeneration = flag.Bool("iatGeneration", false, "Generate IATs only or run invocations as well") iatFromFile = flag.Bool("generated", false, "True if iats were already generated") + dryRun = flag.Bool("dryRun", false, "Dry run mode - do not deploy functions or generate invocations") ) func init() { @@ -206,6 +207,10 @@ func runTraceMode(cfg *config.LoaderConfiguration, readIATFromFile bool, writeIA Functions: functions, }) + if *dryRun { + return + } + log.Infof("Using %s as a service YAML specification file.\n", experimentDriver.Configuration.YAMLPath) experimentDriver.GenerateSpecification() From d278eff76cc0f7f0b74324b89f376da5e48060b7 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 12:17:07 +0800 Subject: [PATCH 02/37] add loader dry run documentation Signed-off-by: Lenson --- docs/loader.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/loader.md b/docs/loader.md index 4ee1bf9a..9f2d9f22 100644 --- a/docs/loader.md +++ b/docs/loader.md @@ -119,6 +119,8 @@ $ go run cmd/loader.go --config cmd/config_knative_trace.json Additionally, one can specify log verbosity argument as `--verbosity [info, debug, trace]`. The default value is `info`. +To execute in a dry run mode without generating any load, set the `--dry-run` flag to `true`. This is useful for testing and validating configurations without executing actual requests. + For to configure the workload for load generator, please refer to `docs/configuration.md`. There are a couple of constants that should not be exposed to the users. They can be examined and changed From f23e1aa346f7e533e59b32b9b29cbfe8001500b5 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 23 Nov 2024 15:04:49 +0800 Subject: [PATCH 03/37] add cpu limit validation Signed-off-by: Lenson --- cmd/loader.go | 4 ++++ pkg/common/constants.go | 8 ++++++++ pkg/common/validators.go | 12 ++++++++++++ 3 files changed, 24 insertions(+) create mode 100644 pkg/common/validators.go diff --git a/cmd/loader.go b/cmd/loader.go index 037bfe0e..2e39c304 100644 --- a/cmd/loader.go +++ b/cmd/loader.go @@ -107,6 +107,10 @@ func main() { log.Fatal("Unsupported platform!") } + if cfg.Platform == "Knative" || cfg.Platform == "Knative-RPS" { + common.CheckCPULimit(cfg.CPULimit) + } + if !strings.HasSuffix(cfg.Platform, "-RPS") { runTraceMode(&cfg, *iatFromFile, *iatGeneration) } else { diff --git a/pkg/common/constants.go b/pkg/common/constants.go index 20d1e27c..bf455791 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -97,3 +97,11 @@ const ( AwsRegion = "us-east-1" AwsTraceFuncRepositoryName = "invitro_trace_function_aws" ) + +// CPULimits +const ( + CPULimit1vCPU string = "1vCPU" + CPULimitGCP string = "GCP" +) + +var ValidCPULimits = []string{CPULimit1vCPU, CPULimitGCP} diff --git a/pkg/common/validators.go b/pkg/common/validators.go new file mode 100644 index 00000000..7429b7be --- /dev/null +++ b/pkg/common/validators.go @@ -0,0 +1,12 @@ +package common + +import ( + "log" + "slices" +) + +func CheckCPULimit(cpuLimit string) { + if !slices.Contains(ValidCPULimits, cpuLimit) { + log.Fatal("Invalid CPU Limit ", cpuLimit) + } +} \ No newline at end of file From 4cbe15146ab12d1570959f173187ad52727c6d75 Mon Sep 17 00:00:00 2001 From: Lenson Date: Tue, 24 Dec 2024 11:15:49 +0800 Subject: [PATCH 04/37] add early return for dry run mode in runRPSMode function Signed-off-by: Lenson --- cmd/loader.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/loader.go b/cmd/loader.go index 2e39c304..8c2d9509 100644 --- a/cmd/loader.go +++ b/cmd/loader.go @@ -27,11 +27,12 @@ package main import ( "flag" "fmt" - "github.com/vhive-serverless/loader/pkg/generator" "os" "strings" "time" + "github.com/vhive-serverless/loader/pkg/generator" + "golang.org/x/exp/slices" "github.com/vhive-serverless/loader/pkg/common" @@ -53,7 +54,7 @@ var ( verbosity = flag.String("verbosity", "info", "Logging verbosity - choose from [info, debug, trace]") iatGeneration = flag.Bool("iatGeneration", false, "Generate IATs only or run invocations as well") iatFromFile = flag.Bool("generated", false, "True if iats were already generated") - dryRun = flag.Bool("dryRun", false, "Dry run mode - do not deploy functions or generate invocations") + dryRun = flag.Bool("dryRun", false, "Dry run mode - do not deploy functions or generate invocations") ) func init() { @@ -211,6 +212,7 @@ func runTraceMode(cfg *config.LoaderConfiguration, readIATFromFile bool, writeIA Functions: functions, }) + // Skip experiments execution during dry run mode if *dryRun { return } @@ -243,6 +245,11 @@ func runRPSMode(cfg *config.LoaderConfiguration, readIATFromFile bool, writeIATs Functions: generator.CreateRPSFunctions(cfg, warmFunction, warmStartCount, coldFunctions, coldStartCount), }) + // Skip experiments execution during dry run mode + if *dryRun { + return + } + experimentDriver.ReadOrWriteFileSpecification(writeIATsToFile, readIATFromFile) experimentDriver.RunExperiment() } From 7f71bedec2ee54c00ea9fe8b01217afce095b8c0 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 12:36:43 +0800 Subject: [PATCH 05/37] add multi-loader config Signed-off-by: Lenson --- pkg/config/parser.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pkg/config/parser.go b/pkg/config/parser.go index 060849be..a9cde170 100644 --- a/pkg/config/parser.go +++ b/pkg/config/parser.go @@ -85,6 +85,38 @@ type LoaderConfiguration struct { Depth int `json:"Depth"` } +type MutliLoaderConfiguration struct { + Experiments []LoaderExperiment `json:"Experiments"` + BaseConfigPath string `json:"BaseConfigPath"` + // Optional + PreScript string `json:"PreScript"` + PostScript string `json:"PostScript"` + MasterNode string `json:"MasterNode"` + AutoScalerNode string `json:"AutoScalerNode"` + ActivatorNode string `json:"ActivatorNode"` + LoaderNode string `json:"LoaderNode"` + WorkerNodes []string `json:"WorkerNodes"` + Metrics []string `json:"Metrics"` +} + +type LoaderExperiment struct { + Name string `json:"Name"` + Config map[string]interface{} `json:"Config"` + // A combination of format and values or just dir should be specified + TracesDir string `json:"TracesDir"` + + TracesFormat string `json:"TracesFormat"` + TraceValues []interface{} `json:"TraceValues"` + + // Optional + OutputDir string `json:"OutputDir"` + Verbosity string `json:"Verbosity"` + IatGeneration bool `json:"IatGeneration"` + Generated bool `json:"Generated"` + PreScript string `json:"PreScript"` + PostScript string `json:"PostScript"` +} + func ReadConfigurationFile(path string) LoaderConfiguration { byteValue, err := os.ReadFile(path) if err != nil { From 0b83f5b3d4a0b514cdb90aa50ac9bacf646d3353 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 12:37:14 +0800 Subject: [PATCH 06/37] add multi-loader config reader Signed-off-by: Lenson --- pkg/config/parser.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/config/parser.go b/pkg/config/parser.go index a9cde170..bb64434f 100644 --- a/pkg/config/parser.go +++ b/pkg/config/parser.go @@ -149,3 +149,18 @@ func ReadFailureConfiguration(path string) *FailureConfiguration { return &config } + +func ReadMultiLoaderConfigurationFile(path string) MutliLoaderConfiguration { + byteValue, err := os.ReadFile(path) + if err != nil { + log.Fatal(err) + } + + var config MutliLoaderConfiguration + err = json.Unmarshal(byteValue, &config) + if err != nil { + log.Fatal(err) + } + + return config +} From 6061f5d8afd9ccaab6ed46a3be91fbfaf6581f38 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 12:49:44 +0800 Subject: [PATCH 07/37] add multi loader base Signed-off-by: Lenson add multi loader base Signed-off-by: Lenson --- tools/multi_loader/multi_loader.go | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 tools/multi_loader/multi_loader.go diff --git a/tools/multi_loader/multi_loader.go b/tools/multi_loader/multi_loader.go new file mode 100644 index 00000000..bab6517e --- /dev/null +++ b/tools/multi_loader/multi_loader.go @@ -0,0 +1,39 @@ +package main + +import ( + "flag" + "os" + "time" + + log "github.com/sirupsen/logrus" +) + +var ( + verbosity = flag.String("verbosity", "info", "Logging verbosity - choose from [info, debug, trace]") +) + +func init() { + flag.Parse() + initLogger() +} + +func initLogger() { + log.SetFormatter(&log.TextFormatter{ + TimestampFormat: time.StampMilli, + FullTimestamp: true, + }) + log.SetOutput(os.Stdout) + + switch *verbosity { + case "debug": + log.SetLevel(log.DebugLevel) + case "trace": + log.SetLevel(log.TraceLevel) + default: + log.SetLevel(log.InfoLevel) + } +} + +func main() { + // Run multi loader +} From 54ec2c11eb3c51461cc4afba95c17bdf47e11c99 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 12:55:04 +0800 Subject: [PATCH 08/37] add node group struct Signed-off-by: Lenson --- pkg/common/node_types.go | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 pkg/common/node_types.go diff --git a/pkg/common/node_types.go b/pkg/common/node_types.go new file mode 100644 index 00000000..05890592 --- /dev/null +++ b/pkg/common/node_types.go @@ -0,0 +1,9 @@ +package common + +type NodeGroup struct { + MasterNode string + AutoScalerNode string + ActivatorNode string + LoaderNode string + WorkerNodes []string +} \ No newline at end of file From affe39a3cf784ae981dc19eb6103a640a3f92116 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 13:07:43 +0800 Subject: [PATCH 09/37] add multi loader runner Signed-off-by: Lenson --- .../runner/multi_loader_runner.go | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tools/multi_loader/runner/multi_loader_runner.go diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go new file mode 100644 index 00000000..fcc75e0b --- /dev/null +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -0,0 +1,31 @@ +package runner + +import ( + "github.com/vhive-serverless/loader/pkg/common" + "github.com/vhive-serverless/loader/pkg/config" +) + +type MultiLoaderRunner struct { + MultiLoaderConfig config.MutliLoaderConfiguration + NodeGroup common.NodeGroup + DryRunSuccess bool + Verbosity string + IatGeneration bool + Generated bool + DryRun bool + Platform string +} + +// init multi loader runner +func NewMultiLoaderRunner(configPath string, verbosity string, iatGeneration bool, generated bool) (*MultiLoaderRunner, error) { + multiLoaderConfig := config.ReadMultiLoaderConfigurationFile(configPath) + + return &MultiLoaderRunner{ + MultiLoaderConfig: multiLoaderConfig, + DryRunSuccess: true, + Verbosity: verbosity, + IatGeneration: iatGeneration, + Generated: generated, + DryRun: false, + }, nil +} \ No newline at end of file From 6c734c3150f6b98fbd517a5d684308b8893c4ae3 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 13:21:48 +0800 Subject: [PATCH 10/37] refactor multi loader config Signed-off-by: Lenson --- pkg/common/config_types.go | 33 +++++++++++++++++ pkg/config/parser.go | 37 ++----------------- .../runner/multi_loader_runner.go | 2 +- 3 files changed, 37 insertions(+), 35 deletions(-) create mode 100644 pkg/common/config_types.go diff --git a/pkg/common/config_types.go b/pkg/common/config_types.go new file mode 100644 index 00000000..55929e93 --- /dev/null +++ b/pkg/common/config_types.go @@ -0,0 +1,33 @@ +package common + +type MutliLoaderConfiguration struct { + Experiments []LoaderExperiment `json:"Experiments"` + BaseConfigPath string `json:"BaseConfigPath"` + // Optional + PreScript string `json:"PreScript"` + PostScript string `json:"PostScript"` + MasterNode string `json:"MasterNode"` + AutoScalerNode string `json:"AutoScalerNode"` + ActivatorNode string `json:"ActivatorNode"` + LoaderNode string `json:"LoaderNode"` + WorkerNodes []string `json:"WorkerNodes"` + Metrics []string `json:"Metrics"` +} + +type LoaderExperiment struct { + Name string `json:"Name"` + Config map[string]interface{} `json:"Config"` + // A combination of format and values or just dir should be specified + TracesDir string `json:"TracesDir"` + + TracesFormat string `json:"TracesFormat"` + TraceValues []interface{} `json:"TraceValues"` + + // Optional + OutputDir string `json:"OutputDir"` + Verbosity string `json:"Verbosity"` + IatGeneration bool `json:"IatGeneration"` + Generated bool `json:"Generated"` + PreScript string `json:"PreScript"` + PostScript string `json:"PostScript"` +} \ No newline at end of file diff --git a/pkg/config/parser.go b/pkg/config/parser.go index bb64434f..bc8b2819 100644 --- a/pkg/config/parser.go +++ b/pkg/config/parser.go @@ -29,6 +29,7 @@ import ( "os" log "github.com/sirupsen/logrus" + "github.com/vhive-serverless/loader/pkg/common" ) type FailureConfiguration struct { @@ -85,38 +86,6 @@ type LoaderConfiguration struct { Depth int `json:"Depth"` } -type MutliLoaderConfiguration struct { - Experiments []LoaderExperiment `json:"Experiments"` - BaseConfigPath string `json:"BaseConfigPath"` - // Optional - PreScript string `json:"PreScript"` - PostScript string `json:"PostScript"` - MasterNode string `json:"MasterNode"` - AutoScalerNode string `json:"AutoScalerNode"` - ActivatorNode string `json:"ActivatorNode"` - LoaderNode string `json:"LoaderNode"` - WorkerNodes []string `json:"WorkerNodes"` - Metrics []string `json:"Metrics"` -} - -type LoaderExperiment struct { - Name string `json:"Name"` - Config map[string]interface{} `json:"Config"` - // A combination of format and values or just dir should be specified - TracesDir string `json:"TracesDir"` - - TracesFormat string `json:"TracesFormat"` - TraceValues []interface{} `json:"TraceValues"` - - // Optional - OutputDir string `json:"OutputDir"` - Verbosity string `json:"Verbosity"` - IatGeneration bool `json:"IatGeneration"` - Generated bool `json:"Generated"` - PreScript string `json:"PreScript"` - PostScript string `json:"PostScript"` -} - func ReadConfigurationFile(path string) LoaderConfiguration { byteValue, err := os.ReadFile(path) if err != nil { @@ -150,13 +119,13 @@ func ReadFailureConfiguration(path string) *FailureConfiguration { return &config } -func ReadMultiLoaderConfigurationFile(path string) MutliLoaderConfiguration { +func ReadMultiLoaderConfigurationFile(path string) common.MutliLoaderConfiguration { byteValue, err := os.ReadFile(path) if err != nil { log.Fatal(err) } - var config MutliLoaderConfiguration + var config common.MutliLoaderConfiguration err = json.Unmarshal(byteValue, &config) if err != nil { log.Fatal(err) diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index fcc75e0b..0d5ac75b 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -6,7 +6,7 @@ import ( ) type MultiLoaderRunner struct { - MultiLoaderConfig config.MutliLoaderConfiguration + MultiLoaderConfig common.MutliLoaderConfiguration NodeGroup common.NodeGroup DryRunSuccess bool Verbosity string From 56bee814e41ddab22b57abab76da7d43b8dcde3f Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 13:31:36 +0800 Subject: [PATCH 11/37] add multi loader config validators Signed-off-by: Lenson --- pkg/common/constants.go | 15 ++- pkg/common/validators.go | 102 +++++++++++++++++- .../runner/multi_loader_runner.go | 3 + 3 files changed, 111 insertions(+), 9 deletions(-) diff --git a/pkg/common/constants.go b/pkg/common/constants.go index bf455791..c32a3516 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -98,10 +98,17 @@ const ( AwsTraceFuncRepositoryName = "invitro_trace_function_aws" ) -// CPULimits +// Multi-loader Constants const ( - CPULimit1vCPU string = "1vCPU" - CPULimitGCP string = "GCP" + TraceFormatString = "{}" ) -var ValidCPULimits = []string{CPULimit1vCPU, CPULimitGCP} +// Multi-loader possible collectable metrics +const ( + Activator string = "activator" + AutoScaler string = "autoscaler" + TOP string = "top" + Prometheus string = "prometheus" +) + +var ValidCollectableMetrics = []string{Activator, AutoScaler, TOP, Prometheus} diff --git a/pkg/common/validators.go b/pkg/common/validators.go index 7429b7be..29dbe27f 100644 --- a/pkg/common/validators.go +++ b/pkg/common/validators.go @@ -1,12 +1,104 @@ package common import ( - "log" + "bytes" + "net" + "os" + "os/exec" + "path" "slices" + "strings" + + log "github.com/sirupsen/logrus" ) -func CheckCPULimit(cpuLimit string) { - if !slices.Contains(ValidCPULimits, cpuLimit) { - log.Fatal("Invalid CPU Limit ", cpuLimit) +func CheckNode(node string) { + if !IsValidIP(node) { + log.Fatal("Invalid IP address for node ", node) + } + cmd := exec.Command("ssh", "-oStrictHostKeyChecking=no", "-p", "22", node, "exit") + // -oStrictHostKeyChecking=no -p 22 + out, err := cmd.CombinedOutput() + if bytes.Contains(out, []byte("Permission denied")) || err != nil { + log.Error(string(out)) + log.Fatal("Failed to connect to node ", node) + } +} + +func CheckPath(path string) { + if (path) == "" { + return + } + _, err := os.Stat(path) + if err != nil { + log.Fatal(err) + } +} + +func IsValidIP(ip string) bool { + parsedIP := net.ParseIP(ip) + return parsedIP != nil +} + +// Check general multi-loader configuration that applies to all platforms +func CheckMultiLoaderConfig(multiLoaderConfig MutliLoaderConfiguration) { + log.Info("Checking multi-loader configuration") + // Check if all paths are valid + CheckPath(multiLoaderConfig.BaseConfigPath) + // Check each experiments + if len(multiLoaderConfig.Experiments) == 0 { + log.Fatal("No experiments found in configuration file") + } + for _, experiment := range multiLoaderConfig.Experiments { + // Check trace directory + // if configs does not have TracePath or OutputPathPreix, either TracesDir or (TracesFormat and TraceValues) should be defined along with OutputDir + if experiment.TracesDir == "" && (experiment.TracesFormat == "" || len(experiment.TraceValues) == 0) { + if _, ok := experiment.Config["TracePath"]; !ok { + log.Fatal("Missing one of TracesDir, TracesFormat & TraceValues, Config.TracePath in multi_loader_config ", experiment.Name) + } + } + if experiment.TracesFormat != "" { + // check if trace format contains TRACE_FORMAT_STRING + if !strings.Contains(experiment.TracesFormat, TraceFormatString) { + log.Fatal("Invalid TracesFormat in multi_loader_config ", experiment.Name, ". Missing ", TraceFormatString, " in format") + } + } + if experiment.OutputDir == "" { + if _, ok := experiment.Config["OutputPathPrefix"]; !ok { + log.Warn("Missing one of OutputDir or Config.OutputPathPrefix in multi_loader_config ", experiment.Name) + // set default output directory + experiment.OutputDir = path.Join("data", "out", experiment.Name) + log.Warn("Setting default output directory to ", experiment.OutputDir) + } + } + } + log.Info("All experiments configs are valid") +} + +// check platform specific multi-loader configuration +func CheckMultiLoaderPlatformSpecificConfig(multiLoaderConfig MutliLoaderConfiguration, nodeGroup NodeGroup, platform string) { + log.Info("Checking platform specific multi-loader configuration") + // For knative platform, there is an additional check for nodes + if platform == "Knative" { + // Check if metrics are valid + for _, metric := range multiLoaderConfig.Metrics { + CheckCollectableMetrics(metric) + } + // Check nodes + CheckNode(nodeGroup.MasterNode) + CheckNode(nodeGroup.AutoScalerNode) + CheckNode(nodeGroup.ActivatorNode) + CheckNode(nodeGroup.LoaderNode) + for _, node := range nodeGroup.WorkerNodes { + CheckNode(node) + } + log.Info("Nodes are reachable") + } + log.Info("All platform specific configs are valid") +} + +func CheckCollectableMetrics(metrics string) { + if !slices.Contains(ValidCollectableMetrics, metrics) { + log.Fatal("Invalid metrics ", metrics) } -} \ No newline at end of file +} diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index 0d5ac75b..c8b66e3b 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -20,6 +20,9 @@ type MultiLoaderRunner struct { func NewMultiLoaderRunner(configPath string, verbosity string, iatGeneration bool, generated bool) (*MultiLoaderRunner, error) { multiLoaderConfig := config.ReadMultiLoaderConfigurationFile(configPath) + // validate configuration + common.CheckMultiLoaderConfig(multiLoaderConfig) + return &MultiLoaderRunner{ MultiLoaderConfig: multiLoaderConfig, DryRunSuccess: true, From 9cd536fc3b127bff039111ae12f103aaee9b8584 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 13:38:08 +0800 Subject: [PATCH 12/37] add knative specific config enricher Signed-off-by: Lenson add additional knative platform type Signed-off-by: Lenson --- pkg/common/utilities.go | 57 ++++++++++++++++- pkg/common/validators.go | 2 +- .../runner/multi_loader_runner.go | 63 ++++++++++++++++++- 3 files changed, 118 insertions(+), 4 deletions(-) diff --git a/pkg/common/utilities.go b/pkg/common/utilities.go index c836fa83..75837417 100644 --- a/pkg/common/utilities.go +++ b/pkg/common/utilities.go @@ -25,9 +25,11 @@ package common import ( + "fmt" "hash/fnv" "log" "math/rand" + "os/exec" "strconv" "strings" ) @@ -146,4 +148,57 @@ func GetName(function *Function) int { log.Fatal(err) } return functionId -} \ No newline at end of file +} + +func DetermineWorkerNodes() []string { + cmd := exec.Command("sh", "-c", "kubectl get nodes --show-labels --no-headers -o wide | grep nodetype=worker | awk '{print $6}'") + out, err := cmd.CombinedOutput() + if err != nil { + log.Fatal(err) + } + workerNodes := strings.Split(strings.Trim(string(out), " \n"), "\n") + for i := range workerNodes { + workerNodes[i] = strings.TrimSpace(workerNodes[i]) + } + return workerNodes +} + +func DetermineMasterNode() string { + cmd := exec.Command("sh", "-c", "kubectl get nodes --show-labels --no-headers -o wide | grep nodetype=master | awk '{print $6}'") + out, err := cmd.CombinedOutput() + if err != nil { + log.Fatal(err) + } + return strings.Trim(string(out), " \n") +} + +func DetermineLoaderNode() string { + cmd := exec.Command("sh", "-c", "kubectl get nodes --show-labels --no-headers -o wide | grep nodetype=monitoring | awk '{print $6}'") + out, err := cmd.CombinedOutput() + if err != nil { + log.Fatal(err) + } + return strings.Trim(string(out), " \n") +} + +func DetermineOtherNodes(podNamePrefix string) string { + // Get the pod alias + cmdPodName := exec.Command("sh", "-c", fmt.Sprintf("kubectl get pods -n knative-serving --no-headers | grep %s- | awk '{print $1}'", podNamePrefix)) + out, err := cmdPodName.CombinedOutput() + + if err != nil { + log.Fatal("Error getting", podNamePrefix, "pod name:", err) + } + + // Get the private ip using the pod alias + podName := strings.Trim(string(out), "\n") + cmdNodeIP := exec.Command("sh", "-c", fmt.Sprintf("kubectl get pod %s -n knative-serving -o=jsonpath='{.status.hostIP}'", podName)) + out, err = cmdNodeIP.CombinedOutput() + + if err != nil { + log.Fatal("Error getting", cmdNodeIP, "node IP:", err) + } + + nodeIp := strings.Split(string(out), "\n")[0] + return strings.Trim(nodeIp, " ") +} diff --git a/pkg/common/validators.go b/pkg/common/validators.go index 29dbe27f..0f192715 100644 --- a/pkg/common/validators.go +++ b/pkg/common/validators.go @@ -79,7 +79,7 @@ func CheckMultiLoaderConfig(multiLoaderConfig MutliLoaderConfiguration) { func CheckMultiLoaderPlatformSpecificConfig(multiLoaderConfig MutliLoaderConfiguration, nodeGroup NodeGroup, platform string) { log.Info("Checking platform specific multi-loader configuration") // For knative platform, there is an additional check for nodes - if platform == "Knative" { + if platform == "Knative" || platform == "Knative-RPS" { // Check if metrics are valid for _, metric := range multiLoaderConfig.Metrics { CheckCollectableMetrics(metric) diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index c8b66e3b..e3484f39 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -1,8 +1,13 @@ package runner import ( + "encoding/json" + "os" + "github.com/vhive-serverless/loader/pkg/common" "github.com/vhive-serverless/loader/pkg/config" + + log "github.com/sirupsen/logrus" ) type MultiLoaderRunner struct { @@ -22,13 +27,67 @@ func NewMultiLoaderRunner(configPath string, verbosity string, iatGeneration boo // validate configuration common.CheckMultiLoaderConfig(multiLoaderConfig) + + // determine platform + platform := determinePlatform(multiLoaderConfig) - return &MultiLoaderRunner{ + runner := MultiLoaderRunner{ MultiLoaderConfig: multiLoaderConfig, DryRunSuccess: true, Verbosity: verbosity, IatGeneration: iatGeneration, Generated: generated, DryRun: false, - }, nil + Platform: platform, + } + + // For knative platform, help to determine and validate nodes in cluster + if platform == "Knative" || platform == "Knative-RPS" { + nodeGroup := determineNodes(multiLoaderConfig) + // add to runner + runner.NodeGroup = nodeGroup + common.CheckMultiLoaderPlatformSpecificConfig(multiLoaderConfig, nodeGroup, platform) + } + + return &runner, nil +} + +func determinePlatform(multiLoaderConfig common.MutliLoaderConfiguration) string { + // Determine platform + baseConfigByteValue, err := os.ReadFile(multiLoaderConfig.BaseConfigPath) + if err != nil { + log.Fatal(err) + } + var loaderConfig config.LoaderConfiguration + // Unmarshal base configuration + if err = json.Unmarshal(baseConfigByteValue, &loaderConfig); err != nil { + log.Fatal(err) + } + return loaderConfig.Platform +} + +func determineNodes(multiLoaderConfig common.MutliLoaderConfiguration) common.NodeGroup { + var nodeGroup common.NodeGroup + nodeGroup.MasterNode = multiLoaderConfig.MasterNode + nodeGroup.AutoScalerNode = multiLoaderConfig.AutoScalerNode + nodeGroup.ActivatorNode = multiLoaderConfig.ActivatorNode + nodeGroup.LoaderNode = multiLoaderConfig.LoaderNode + nodeGroup.WorkerNodes = multiLoaderConfig.WorkerNodes + + if len(nodeGroup.WorkerNodes) == 0 { + nodeGroup.WorkerNodes = common.DetermineWorkerNodes() + } + if nodeGroup.MasterNode == "" { + nodeGroup.MasterNode = common.DetermineMasterNode() + } + if nodeGroup.LoaderNode == "" { + nodeGroup.LoaderNode = common.DetermineLoaderNode() + } + if nodeGroup.AutoScalerNode == "" { + nodeGroup.AutoScalerNode = common.DetermineOtherNodes("autoscaler") + } + if nodeGroup.ActivatorNode == "" { + nodeGroup.ActivatorNode = common.DetermineOtherNodes("activator") + } + return nodeGroup } \ No newline at end of file From 4a67925c0d68c9f147b8e8b04cad7af3939ae088 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 13:41:56 +0800 Subject: [PATCH 13/37] add base runner entry point Signed-off-by: Lenson --- pkg/common/utilities.go | 14 ++++++++++++++ tools/multi_loader/runner/multi_loader_runner.go | 15 +++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/pkg/common/utilities.go b/pkg/common/utilities.go index 75837417..55f4a30b 100644 --- a/pkg/common/utilities.go +++ b/pkg/common/utilities.go @@ -32,6 +32,8 @@ import ( "os/exec" "strconv" "strings" + + logger "github.com/sirupsen/logrus" ) type Pair struct { @@ -202,3 +204,15 @@ func DetermineOtherNodes(podNamePrefix string) string { nodeIp := strings.Split(string(out), "\n")[0] return strings.Trim(nodeIp, " ") } + +func RunScript(command string) { + if command == "" { + return + } + logger.Info("Running command ", command) + cmd, err := exec.Command("/bin/sh", command).Output() + if err != nil { + log.Fatal(err) + } + logger.Info(string(cmd)) +} diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index e3484f39..9e3d7eab 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -90,4 +90,19 @@ func determineNodes(multiLoaderConfig common.MutliLoaderConfiguration) common.No nodeGroup.ActivatorNode = common.DetermineOtherNodes("activator") } return nodeGroup +} + +func (d *MultiLoaderRunner) run(){ + // Run global prescript + common.RunScript(d.MultiLoaderConfig.PreScript) + // Iterate over experiments and run them + for _, experiment := range d.MultiLoaderConfig.Experiments { + log.Info("Setting up experiment: ", experiment.Name) + // Run pre script + common.RunScript(experiment.PreScript) + // Run post script + common.RunScript(experiment.PostScript) + } + // Run global postscript + common.RunScript(d.MultiLoaderConfig.PostScript) } \ No newline at end of file From c87ff3857accb519820c4f5fbc0f003d45323dc8 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 13:46:21 +0800 Subject: [PATCH 14/37] refactor multi loader config Signed-off-by: Lenson update multi loader config struct Signed-off-by: Lenson --- pkg/common/config_types.go | 18 ++++------ pkg/common/validators.go | 34 +++++++++---------- pkg/config/parser.go | 4 +-- .../runner/multi_loader_runner.go | 16 ++++----- 4 files changed, 33 insertions(+), 39 deletions(-) diff --git a/pkg/common/config_types.go b/pkg/common/config_types.go index 55929e93..4cbe0f3e 100644 --- a/pkg/common/config_types.go +++ b/pkg/common/config_types.go @@ -1,20 +1,14 @@ package common -type MutliLoaderConfiguration struct { - Experiments []LoaderExperiment `json:"Experiments"` - BaseConfigPath string `json:"BaseConfigPath"` +type MultiLoaderConfiguration struct { + Studies []LoaderStudy `json:"Studies"` + BaseConfigPath string `json:"BaseConfigPath"` // Optional - PreScript string `json:"PreScript"` - PostScript string `json:"PostScript"` - MasterNode string `json:"MasterNode"` - AutoScalerNode string `json:"AutoScalerNode"` - ActivatorNode string `json:"ActivatorNode"` - LoaderNode string `json:"LoaderNode"` - WorkerNodes []string `json:"WorkerNodes"` - Metrics []string `json:"Metrics"` + PreScript string `json:"PreScript"` + PostScript string `json:"PostScript"` } -type LoaderExperiment struct { +type LoaderStudy struct { Name string `json:"Name"` Config map[string]interface{} `json:"Config"` // A combination of format and values or just dir should be specified diff --git a/pkg/common/validators.go b/pkg/common/validators.go index 0f192715..0182172f 100644 --- a/pkg/common/validators.go +++ b/pkg/common/validators.go @@ -41,34 +41,34 @@ func IsValidIP(ip string) bool { } // Check general multi-loader configuration that applies to all platforms -func CheckMultiLoaderConfig(multiLoaderConfig MutliLoaderConfiguration) { +func CheckMultiLoaderConfig(multiLoaderConfig MultiLoaderConfiguration) { log.Info("Checking multi-loader configuration") // Check if all paths are valid CheckPath(multiLoaderConfig.BaseConfigPath) - // Check each experiments - if len(multiLoaderConfig.Experiments) == 0 { - log.Fatal("No experiments found in configuration file") + // Check each study + if len(multiLoaderConfig.Studies) == 0 { + log.Fatal("No study found in configuration file") } - for _, experiment := range multiLoaderConfig.Experiments { + for _, study := range multiLoaderConfig.Studies { // Check trace directory // if configs does not have TracePath or OutputPathPreix, either TracesDir or (TracesFormat and TraceValues) should be defined along with OutputDir - if experiment.TracesDir == "" && (experiment.TracesFormat == "" || len(experiment.TraceValues) == 0) { - if _, ok := experiment.Config["TracePath"]; !ok { - log.Fatal("Missing one of TracesDir, TracesFormat & TraceValues, Config.TracePath in multi_loader_config ", experiment.Name) + if study.TracesDir == "" && (study.TracesFormat == "" || len(study.TraceValues) == 0) { + if _, ok := study.Config["TracePath"]; !ok { + log.Fatal("Missing one of TracesDir, TracesFormat & TraceValues, Config.TracePath in multi_loader_config ", study.Name) } } - if experiment.TracesFormat != "" { + if study.TracesFormat != "" { // check if trace format contains TRACE_FORMAT_STRING - if !strings.Contains(experiment.TracesFormat, TraceFormatString) { - log.Fatal("Invalid TracesFormat in multi_loader_config ", experiment.Name, ". Missing ", TraceFormatString, " in format") + if !strings.Contains(study.TracesFormat, TraceFormatString) { + log.Fatal("Invalid TracesFormat in multi_loader_config ", study.Name, ". Missing ", TraceFormatString, " in format") } } - if experiment.OutputDir == "" { - if _, ok := experiment.Config["OutputPathPrefix"]; !ok { - log.Warn("Missing one of OutputDir or Config.OutputPathPrefix in multi_loader_config ", experiment.Name) + if study.OutputDir == "" { + if _, ok := study.Config["OutputPathPrefix"]; !ok { + log.Warn("Missing one of OutputDir or Config.OutputPathPrefix in multi_loader_config ", study.Name) // set default output directory - experiment.OutputDir = path.Join("data", "out", experiment.Name) - log.Warn("Setting default output directory to ", experiment.OutputDir) + study.OutputDir = path.Join("data", "out", study.Name) + log.Warn("Setting default output directory to ", study.OutputDir) } } } @@ -76,7 +76,7 @@ func CheckMultiLoaderConfig(multiLoaderConfig MutliLoaderConfiguration) { } // check platform specific multi-loader configuration -func CheckMultiLoaderPlatformSpecificConfig(multiLoaderConfig MutliLoaderConfiguration, nodeGroup NodeGroup, platform string) { +func CheckMultiLoaderPlatformSpecificConfig(multiLoaderConfig MultiLoaderConfiguration, nodeGroup NodeGroup, platform string) { log.Info("Checking platform specific multi-loader configuration") // For knative platform, there is an additional check for nodes if platform == "Knative" || platform == "Knative-RPS" { diff --git a/pkg/config/parser.go b/pkg/config/parser.go index bc8b2819..fb95d1c7 100644 --- a/pkg/config/parser.go +++ b/pkg/config/parser.go @@ -119,13 +119,13 @@ func ReadFailureConfiguration(path string) *FailureConfiguration { return &config } -func ReadMultiLoaderConfigurationFile(path string) common.MutliLoaderConfiguration { +func ReadMultiLoaderConfigurationFile(path string) common.MultiLoaderConfiguration { byteValue, err := os.ReadFile(path) if err != nil { log.Fatal(err) } - var config common.MutliLoaderConfiguration + var config common.MultiLoaderConfiguration err = json.Unmarshal(byteValue, &config) if err != nil { log.Fatal(err) diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index 9e3d7eab..82b5c723 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -11,7 +11,7 @@ import ( ) type MultiLoaderRunner struct { - MultiLoaderConfig common.MutliLoaderConfiguration + MultiLoaderConfig common.MultiLoaderConfiguration NodeGroup common.NodeGroup DryRunSuccess bool Verbosity string @@ -52,7 +52,7 @@ func NewMultiLoaderRunner(configPath string, verbosity string, iatGeneration boo return &runner, nil } -func determinePlatform(multiLoaderConfig common.MutliLoaderConfiguration) string { +func determinePlatform(multiLoaderConfig common.MultiLoaderConfiguration) string { // Determine platform baseConfigByteValue, err := os.ReadFile(multiLoaderConfig.BaseConfigPath) if err != nil { @@ -66,7 +66,7 @@ func determinePlatform(multiLoaderConfig common.MutliLoaderConfiguration) string return loaderConfig.Platform } -func determineNodes(multiLoaderConfig common.MutliLoaderConfiguration) common.NodeGroup { +func determineNodes(multiLoaderConfig common.MultiLoaderConfiguration) common.NodeGroup { var nodeGroup common.NodeGroup nodeGroup.MasterNode = multiLoaderConfig.MasterNode nodeGroup.AutoScalerNode = multiLoaderConfig.AutoScalerNode @@ -95,13 +95,13 @@ func determineNodes(multiLoaderConfig common.MutliLoaderConfiguration) common.No func (d *MultiLoaderRunner) run(){ // Run global prescript common.RunScript(d.MultiLoaderConfig.PreScript) - // Iterate over experiments and run them - for _, experiment := range d.MultiLoaderConfig.Experiments { - log.Info("Setting up experiment: ", experiment.Name) + // Iterate over studies and run them + for _, study := range d.MultiLoaderConfig.Studies { + log.Info("Setting up experiment: ", study.Name) // Run pre script - common.RunScript(experiment.PreScript) + common.RunScript(study.PreScript) // Run post script - common.RunScript(experiment.PostScript) + common.RunScript(study.PostScript) } // Run global postscript common.RunScript(d.MultiLoaderConfig.PostScript) From 51468c502bb126bb1b8b5bdcaa865379f512d6a0 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 14:04:20 +0800 Subject: [PATCH 15/37] update unpack study doc Signed-off-by: Lenson add unpack study Signed-off-by: Lenson --- pkg/common/utilities.go | 11 ++ .../runner/multi_loader_runner.go | 111 ++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/pkg/common/utilities.go b/pkg/common/utilities.go index 55f4a30b..99b88001 100644 --- a/pkg/common/utilities.go +++ b/pkg/common/utilities.go @@ -25,6 +25,7 @@ package common import ( + "encoding/json" "fmt" "hash/fnv" "log" @@ -152,6 +153,16 @@ func GetName(function *Function) int { return functionId } +func DeepCopy[T any](a T) (T, error) { + var b T + byt, err := json.Marshal(a) + if err != nil { + return b, err + } + err = json.Unmarshal(byt, &b) + return b, err +} + func DetermineWorkerNodes() []string { cmd := exec.Command("sh", "-c", "kubectl get nodes --show-labels --no-headers -o wide | grep nodetype=worker | awk '{print $6}'") out, err := cmd.CombinedOutput() diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index 82b5c723..e8dd21ab 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -2,7 +2,11 @@ package runner import ( "encoding/json" + "fmt" "os" + "path" + "strings" + "time" "github.com/vhive-serverless/loader/pkg/common" "github.com/vhive-serverless/loader/pkg/config" @@ -10,6 +14,10 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + TIME_FORMAT = "Jan_02_1504" +) + type MultiLoaderRunner struct { MultiLoaderConfig common.MultiLoaderConfiguration NodeGroup common.NodeGroup @@ -100,9 +108,112 @@ func (d *MultiLoaderRunner) run(){ log.Info("Setting up experiment: ", study.Name) // Run pre script common.RunScript(study.PreScript) + + // Unpack study to sparse experiments + sparseExperiments := d.unpackStudy(study) + // Run post script common.RunScript(study.PostScript) } // Run global postscript common.RunScript(d.MultiLoaderConfig.PostScript) +} + +/** +* As a study can have multiple experiments, this function will unpack the study +* but first by duplicating the study to multiple studies with different values +* in the config field. Those values will override the base loader config later +*/ +func (d *MultiLoaderRunner) unpackStudy(experiment common.LoaderStudy) []common.LoaderStudy { + log.Info("Unpacking experiment ", experiment.Name) + var experiments []common.LoaderStudy + + // If user specified a trace directory + if experiment.TracesDir != "" { + experiments = d.unpackFromTraceDir(experiment) + // User Define trace format and values instead of directory + } else if experiment.TracesFormat != "" && len(experiment.TraceValues) > 0 { + experiments = d.unpackFromTraceValues(experiment) + } else { + // Theres only one experiment in the study + experiments = d.unpackSingleExperiment(experiment) + } + + return experiments +} + +func (d *MultiLoaderRunner) unpackFromTraceDir(study common.LoaderStudy) []common.LoaderStudy { + var experiments []common.LoaderStudy + files, err := os.ReadDir(study.TracesDir) + if err != nil { + log.Fatal(err) + } + + for _, file := range files { + newExperiment := d.createNewStudy(study, file.Name()) + experiments = append(experiments, newExperiment) + } + return experiments +} + +func (d *MultiLoaderRunner) unpackFromTraceValues(study common.LoaderStudy) []common.LoaderStudy { + var experiments []common.LoaderStudy + for _, traceValue := range study.TraceValues { + tracePath := strings.Replace(study.TracesFormat, common.TraceFormatString, fmt.Sprintf("%v", traceValue), -1) + fileName := path.Base(tracePath) + newExperiment := d.createNewStudy(study, fileName) + newExperiment.Config["TracePath"] = tracePath + newExperiment.Name += "_" + fileName + experiments = append(experiments, newExperiment) + } + return experiments +} + +func (d *MultiLoaderRunner) unpackSingleExperiment(study common.LoaderStudy) []common.LoaderStudy { + var experiments []common.LoaderStudy + pathDir := "" + if study.Config["OutputPathPrefix"] != nil { + pathDir = path.Dir(study.Config["OutputPathPrefix"].(string)) + } else { + pathDir = study.OutputDir + } + study.OutputDir = pathDir + newExperiment := d.createNewStudy(study, study.Name) + experiments = append(experiments, newExperiment) + return experiments +} + +func (d *MultiLoaderRunner) createNewStudy(study common.LoaderStudy, fileName string) common.LoaderStudy { + newStudy, err := common.DeepCopy(study) + if err != nil { + log.Fatal(err) + } + + dryRunAdditionalPath := "" + if d.DryRun { + dryRunAdditionalPath = "dry_run" + } + newStudy.Config["OutputPathPrefix"] = path.Join( + study.OutputDir, + study.Name, + dryRunAdditionalPath, + time.Now().Format(TIME_FORMAT)+"_"+fileName, + fileName, + ) + d.addCommandFlags(newStudy) + return newStudy +} + +func (d *MultiLoaderRunner) addCommandFlags(study common.LoaderStudy) { + // Add flags to experiment config + if study.Verbosity == "" { + study.Verbosity = d.Verbosity + } + if !study.IatGeneration { + study.IatGeneration = d.IatGeneration + } + if !study.Generated { + study.Generated = d.Generated + } + } \ No newline at end of file From 3c92d842fd1961fadf72f2fc946f775e532e5fe0 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 14:15:22 +0800 Subject: [PATCH 16/37] add prepare experiment Signed-off-by: Lenson update experiment config temp path Signed-off-by: Lenson --- .../runner/multi_loader_runner.go | 68 ++++++++++++++++++- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index e8dd21ab..90647acf 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -16,6 +16,8 @@ import ( const ( TIME_FORMAT = "Jan_02_1504" + EXPERIMENT_TEMP_CONFIG_PATH = "tools/multi_loader/current_running_config.json" + NUM_OF_RETRIES = 2 ) type MultiLoaderRunner struct { @@ -109,9 +111,15 @@ func (d *MultiLoaderRunner) run(){ // Run pre script common.RunScript(study.PreScript) - // Unpack study to sparse experiments + // Unpack study to a list of studies with different loader configs sparseExperiments := d.unpackStudy(study) + // Iterate over sparse experiments, prepare and run + for _, experiment := range sparseExperiments { + // Prepare experiment: merge with base config, create output dir and write merged config to temp file + d.prepareExperiment(experiment) + } + // Run post script common.RunScript(study.PostScript) } @@ -128,10 +136,10 @@ func (d *MultiLoaderRunner) unpackStudy(experiment common.LoaderStudy) []common. log.Info("Unpacking experiment ", experiment.Name) var experiments []common.LoaderStudy - // If user specified a trace directory + // if user specified a trace directory if experiment.TracesDir != "" { experiments = d.unpackFromTraceDir(experiment) - // User Define trace format and values instead of directory + // user define trace format and values instead of directory } else if experiment.TracesFormat != "" && len(experiment.TraceValues) > 0 { experiments = d.unpackFromTraceValues(experiment) } else { @@ -215,5 +223,59 @@ func (d *MultiLoaderRunner) addCommandFlags(study common.LoaderStudy) { if !study.Generated { study.Generated = d.Generated } +} + +/** +* Prepare experiment by merging with base config, creating output directory and writing experiment config to temp file +*/ +func (d *MultiLoaderRunner) prepareExperiment(experiment common.LoaderStudy) { + log.Info("Preparing ", experiment.Name) + // Merge base configs with experiment configs + experimentConfig := d.mergeConfigurations(d.MultiLoaderConfig.BaseConfigPath, experiment) + + // Create output directory + outputDir := path.Dir(experimentConfig.OutputPathPrefix) + + if err := os.MkdirAll(outputDir, 0755); err != nil { + log.Fatal(err) + } + // Write experiment configs to temp file + d.writeExperimentConfigToTempFile(experimentConfig, EXPERIMENT_TEMP_CONFIG_PATH) +} + +/** +* Merge base configs with partial loader configs +*/ +func (d *MultiLoaderRunner) mergeConfigurations(baseConfigPath string, experiment common.LoaderStudy) config.LoaderConfiguration { + // Read base configuration + baseConfigByteValue, err := os.ReadFile(baseConfigPath) + if err != nil { + log.Fatal(err) + } + log.Debug("Experiment configuration ", experiment.Config) + var mergedConfig config.LoaderConfiguration + // Unmarshal base configuration + if err = json.Unmarshal(baseConfigByteValue, &mergedConfig); err != nil { + log.Fatal(err) + } + + log.Debug("Base configuration ", mergedConfig) + + // merge experiment config onto base config + experimentConfigBytes, _ := json.Marshal(experiment.Config) + if err = json.Unmarshal(experimentConfigBytes, &mergedConfig); err != nil { + log.Fatal(err) + } + log.Debug("Merged configuration ", mergedConfig) + + return mergedConfig +} + +func (d *MultiLoaderRunner) writeExperimentConfigToTempFile(experimentConfig config.LoaderConfiguration, fileWritePath string) { + experimentConfigBytes, _ := json.Marshal(experimentConfig) + err := os.WriteFile(fileWritePath, experimentConfigBytes, 0644) + if err != nil { + log.Fatal(err) + } } \ No newline at end of file From fae06989518f36b4442635be907292c746b7dfa1 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 14:23:51 +0800 Subject: [PATCH 17/37] add run loader function Signed-off-by: Lenson update log parser Signed-off-by: Lenson update log parser Signed-off-by: Lenson update log parser Signed-off-by: Lenson --- pkg/common/utilities.go | 16 +++ .../runner/multi_loader_runner.go | 116 +++++++++++++++++- 2 files changed, 130 insertions(+), 2 deletions(-) diff --git a/pkg/common/utilities.go b/pkg/common/utilities.go index 99b88001..a9eab56d 100644 --- a/pkg/common/utilities.go +++ b/pkg/common/utilities.go @@ -227,3 +227,19 @@ func RunScript(command string) { } logger.Info(string(cmd)) } + +func ParseLogType(logString string) string { + logTypeArr := strings.Split(logString, "level=") + if len(logTypeArr) > 1 { + return strings.Split(logTypeArr[1], " ")[0] + } + return "info" +} + +func ParseLogMessage(logString string) string { + message := strings.Split(logString, "msg=") + if len(message) > 1 { + return message[1][1 : len(message[1])-1] + } + return logString +} diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index 90647acf..62effc43 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -1,10 +1,14 @@ package runner import ( + "bufio" "encoding/json" "fmt" + "io" "os" + "os/exec" "path" + "strconv" "strings" "time" @@ -15,6 +19,7 @@ import ( ) const ( + LOADER_PATH = "cmd/loader.go" TIME_FORMAT = "Jan_02_1504" EXPERIMENT_TEMP_CONFIG_PATH = "tools/multi_loader/current_running_config.json" NUM_OF_RETRIES = 2 @@ -118,8 +123,15 @@ func (d *MultiLoaderRunner) run(){ for _, experiment := range sparseExperiments { // Prepare experiment: merge with base config, create output dir and write merged config to temp file d.prepareExperiment(experiment) - } + err := d.runExperiment(experiment) + + // Check if should continue this study + if err != nil { + log.Info("Experiment failed: ", experiment.Name, ". Skipping remaining experiments in study...") + break + } + } // Run post script common.RunScript(study.PostScript) } @@ -278,4 +290,104 @@ func (d *MultiLoaderRunner) writeExperimentConfigToTempFile(experimentConfig con if err != nil { log.Fatal(err) } -} \ No newline at end of file +} + +func (d *MultiLoaderRunner) runExperiment(experiment common.LoaderStudy) error { + log.Info("Running ", experiment.Name) + log.Debug("Experiment configuration ", experiment.Config) + + // Create the log file + logFilePath := path.Join(path.Dir(experiment.Config["OutputPathPrefix"].(string)), "loader.log") + logFile, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + log.Fatal(err) + } + defer logFile.Close() + + for i := 0; i < NUM_OF_RETRIES; i++ { + // Run loader.go with experiment configs + if err := d.executeLoaderCommand(experiment, logFile); err != nil { + log.Error(err) + log.Error("Experiment failed: ", experiment.Name) + logFile.WriteString("Experiment failed: " + experiment.Name + ". Error: " + err.Error() + "\n") + if i == 0 && !d.DryRun { + log.Info("Retrying experiment ", experiment.Name) + logFile.WriteString("==================================RETRYING==================================\n") + experiment.Verbosity = "debug" + } else{ + // Experiment failed set dry run flag to false + d.DryRunSuccess = false + log.Error("Check log file for more information: ", logFilePath) + // should not continue with experiment + return err + } + continue + }else{ + break + } + } + log.Info("Completed ", experiment.Name) + return nil +} + +func (d *MultiLoaderRunner) executeLoaderCommand(experiment common.LoaderStudy, logFile *os.File) error { + cmd := exec.Command("go", "run", LOADER_PATH, + "--config="+EXPERIMENT_TEMP_CONFIG_PATH, + "--verbosity="+experiment.Verbosity, + "--iatGeneration="+strconv.FormatBool(experiment.IatGeneration), + "--generated="+strconv.FormatBool(experiment.Generated), + "--dryRun="+strconv.FormatBool(d.DryRun)) + + stdout, _ := cmd.StdoutPipe() + stderr, _ := cmd.StderrPipe() + + if err := cmd.Start(); err != nil { + return err + } + + go d.logLoaderStdOutput(stdout, logFile) + go d.logLoaderStdError(stderr, logFile) + + return cmd.Wait() +} + +func (d *MultiLoaderRunner) logLoaderStdOutput(stdPipe io.ReadCloser, logFile *os.File) { + scanner := bufio.NewScanner(stdPipe) + for scanner.Scan() { + m := scanner.Text() + // write to log file + logFile.WriteString(m + "\n") + + // Log key information + if m == "" { + continue + } + logType := common.ParseLogType(m) + message := common.ParseLogMessage(m) + + switch logType { + case "debug": + log.Debug(message) + case "trace": + log.Trace(message) + default: + if strings.Contains(message, "Number of successful invocations:") || strings.Contains(message, "Number of failed invocations:") { + log.Info(strings.ReplaceAll(strings.ReplaceAll(message, "\\t", " ",), "\\n", "")) + } + } + } +} + +func (d *MultiLoaderRunner) logLoaderStdError(stdPipe io.ReadCloser, logFile *os.File) { + scanner := bufio.NewScanner(stdPipe) + for scanner.Scan() { + m := scanner.Text() + // write to log file + logFile.WriteString(m + "\n") + + if m == "" { + continue + } + log.Error(m) + } +} From a9c016559c83b5e725d4e9834594bfd7f83b843d Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 14:29:01 +0800 Subject: [PATCH 18/37] add clean up function Signed-off-by: Lenson --- tools/multi_loader/runner/multi_loader_runner.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index 62effc43..b75d2a36 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -126,6 +126,9 @@ func (d *MultiLoaderRunner) run(){ err := d.runExperiment(experiment) + // Perform cleanup + d.performCleanup() + // Check if should continue this study if err != nil { log.Info("Experiment failed: ", experiment.Name, ". Skipping remaining experiments in study...") @@ -391,3 +394,14 @@ func (d *MultiLoaderRunner) logLoaderStdError(stdPipe io.ReadCloser, logFile *os log.Error(m) } } + +func (d *MultiLoaderRunner) performCleanup() { + log.Info("Runnning Cleanup") + // Run make clean + if err := exec.Command("make", "clean").Run(); err != nil { + log.Error(err) + } + log.Info("Cleanup completed") + // Remove temp file + os.Remove(EXPERIMENT_TEMP_CONFIG_PATH) +} From 9f4dfad61b2f8039b2d35f7292edfaa091033c0f Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 14:30:22 +0800 Subject: [PATCH 19/37] add logs to indicate run status Signed-off-by: Lenson --- tools/multi_loader/runner/multi_loader_runner.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index b75d2a36..c62b372b 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -121,6 +121,9 @@ func (d *MultiLoaderRunner) run(){ // Iterate over sparse experiments, prepare and run for _, experiment := range sparseExperiments { + if d.DryRun{ + log.Info("Dry Running: ", experiment.Name) + } // Prepare experiment: merge with base config, create output dir and write merged config to temp file d.prepareExperiment(experiment) @@ -137,6 +140,9 @@ func (d *MultiLoaderRunner) run(){ } // Run post script common.RunScript(study.PostScript) + if len(sparseExperiments) > 1 && !d.DryRun{ + log.Info("All experiments for ", study.Name, " completed") + } } // Run global postscript common.RunScript(d.MultiLoaderConfig.PostScript) From e198e28be02f407cff6b696f1be6c20030ba71e1 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 14:32:15 +0800 Subject: [PATCH 20/37] expose entry points for multi loader runner Signed-off-by: Lenson --- tools/multi_loader/runner/multi_loader_runner.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index c62b372b..8a1b5581 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -107,6 +107,18 @@ func determineNodes(multiLoaderConfig common.MultiLoaderConfiguration) common.No return nodeGroup } +func (d *MultiLoaderRunner) RunDryRun() { + log.Info("Running dry run") + d.DryRun = true + d.run() +} + +func (d *MultiLoaderRunner) RunActual() { + log.Info("Running actual experiments") + d.DryRun = false + d.run() +} + func (d *MultiLoaderRunner) run(){ // Run global prescript common.RunScript(d.MultiLoaderConfig.PreScript) From 1471a8799aa5f27f0c1d913ce178e936bc952a95 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 14:35:24 +0800 Subject: [PATCH 21/37] add multi loader runner execution Signed-off-by: Lenson update default multi loader config path Signed-off-by: Lenson --- tools/multi_loader/multi_loader.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tools/multi_loader/multi_loader.go b/tools/multi_loader/multi_loader.go index bab6517e..a801ce68 100644 --- a/tools/multi_loader/multi_loader.go +++ b/tools/multi_loader/multi_loader.go @@ -6,10 +6,14 @@ import ( "time" log "github.com/sirupsen/logrus" + "github.com/vhive-serverless/loader/tools/multi_loader/runner" ) var ( + multiLoaderConfigPath = flag.String("multiLoaderConfig", "tools/multi_loader/multi_loader_config.json", "Path to multi loader configuration file") verbosity = flag.String("verbosity", "info", "Logging verbosity - choose from [info, debug, trace]") + iatGeneration = flag.Bool("iatGeneration", false, "Generate iats only and skip invocations") + generated = flag.Bool("generated", false, "If iats were already generated") ) func init() { @@ -35,5 +39,24 @@ func initLogger() { } func main() { - // Run multi loader + log.Info("Starting multiloader") + // Create multi loader driver + multiLoaderDriver, err := runner.NewMultiLoaderRunner(*multiLoaderConfigPath, *verbosity, *iatGeneration, *generated) + if err != nil { + log.Fatalf("Failed to create multi loader driver: %v", err) + } + // Dry run + multiLoaderDriver.RunDryRun() + + // Check if dry run was successful + if !multiLoaderDriver.DryRunSuccess { + log.Fatal("Dry run failed. Exiting...") + } + + // Actual run + log.Info("Running experiments") + multiLoaderDriver.RunActual() + + // Finish + log.Info("All experiments completed") } From a447865e3cbaae9276e30a3f7fc7f0a0f4999864 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 14:38:03 +0800 Subject: [PATCH 22/37] add basic config Signed-off-by: Lenson update basic config Signed-off-by: Lenson update basic config Signed-off-by: Lenson add basic configs Signed-off-by: Lenson update base config Signed-off-by: Lenson --- tools/multi_loader/base_loader_config.json | 28 +++++++++++++++++++++ tools/multi_loader/multi_loader_config.json | 22 ++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 tools/multi_loader/base_loader_config.json create mode 100644 tools/multi_loader/multi_loader_config.json diff --git a/tools/multi_loader/base_loader_config.json b/tools/multi_loader/base_loader_config.json new file mode 100644 index 00000000..beae7786 --- /dev/null +++ b/tools/multi_loader/base_loader_config.json @@ -0,0 +1,28 @@ +{ + "Seed": 42, + + "Platform": "Knative", + "InvokeProtocol" : "grpc", + "YAMLSelector": "container", + "EndpointPort": 80, + + "BusyLoopOnSandboxStartup": false, + + "TracePath": "data/traces/example", + "Granularity": "minute", + "OutputPathPrefix": "data/out/experiment", + "IATDistribution": "exponential", + "CPULimit": "1vCPU", + "ExperimentDuration": 5, + "WarmupDuration": 0, + + "IsPartiallyPanic": false, + "EnableZipkinTracing": false, + "EnableMetricsScrapping": false, + "MetricScrapingPeriodSeconds": 15, + "AutoscalingMetric": "concurrency", + + "GRPCConnectionTimeoutSeconds": 15, + "GRPCFunctionTimeoutSeconds": 900, + "DAGMode": false +} diff --git a/tools/multi_loader/multi_loader_config.json b/tools/multi_loader/multi_loader_config.json new file mode 100644 index 00000000..bbe65974 --- /dev/null +++ b/tools/multi_loader/multi_loader_config.json @@ -0,0 +1,22 @@ +{ + "Studies": [ + { + "Name": "experiment1", + "Config": { + "ExperimentDuration": 1 + }, + "IatGeneration": false, + "Generated": false, + "Verbosity": "info", + "TracesDir": "", + "TracesFormat": "data/traces/multi-exp-test/example_{}_test", + "TraceValues": [1], + "OutputDir": "data/out/multi-test", + "PreScript": "", + "PostScript": "" + } + ], + "PreScript": "", + "PostScript": "", + "BaseConfigPath": "tools/multi_loader/base_loader_config.json" +} \ No newline at end of file From 7081f3610e0d563018f4bf395902d10453d7d276 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 16:42:46 +0800 Subject: [PATCH 23/37] add cpu limit validator Signed-off-by: Lenson --- pkg/common/constants.go | 8 ++++++++ pkg/common/validators.go | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/pkg/common/constants.go b/pkg/common/constants.go index c32a3516..d8f6e280 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -112,3 +112,11 @@ const ( ) var ValidCollectableMetrics = []string{Activator, AutoScaler, TOP, Prometheus} + +// CPULimits +const ( + CPULimit1vCPU string = "1vCPU" + CPULimitGCP string = "GCP" +) + +var ValidCPULimits = []string{CPULimit1vCPU, CPULimitGCP} diff --git a/pkg/common/validators.go b/pkg/common/validators.go index 0182172f..b9800fbb 100644 --- a/pkg/common/validators.go +++ b/pkg/common/validators.go @@ -102,3 +102,9 @@ func CheckCollectableMetrics(metrics string) { log.Fatal("Invalid metrics ", metrics) } } + +func CheckCPULimit(cpuLimit string) { + if !slices.Contains(ValidCPULimits, cpuLimit) { + log.Fatal("Invalid CPU Limit ", cpuLimit) + } +} From 932f7cb8b4f8f5a28a71f2f55fb6b01e0b2b5316 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 17:13:09 +0800 Subject: [PATCH 24/37] remove extra knative feature Signed-off-by: Lenson remove knative extra features Signed-off-by: Lenson --- pkg/common/utilities.go | 54 ------------------- pkg/common/validators.go | 22 -------- .../runner/multi_loader_runner.go | 34 ------------ 3 files changed, 110 deletions(-) diff --git a/pkg/common/utilities.go b/pkg/common/utilities.go index a9eab56d..a841ece5 100644 --- a/pkg/common/utilities.go +++ b/pkg/common/utilities.go @@ -26,7 +26,6 @@ package common import ( "encoding/json" - "fmt" "hash/fnv" "log" "math/rand" @@ -163,59 +162,6 @@ func DeepCopy[T any](a T) (T, error) { return b, err } -func DetermineWorkerNodes() []string { - cmd := exec.Command("sh", "-c", "kubectl get nodes --show-labels --no-headers -o wide | grep nodetype=worker | awk '{print $6}'") - out, err := cmd.CombinedOutput() - if err != nil { - log.Fatal(err) - } - workerNodes := strings.Split(strings.Trim(string(out), " \n"), "\n") - for i := range workerNodes { - workerNodes[i] = strings.TrimSpace(workerNodes[i]) - } - return workerNodes -} - -func DetermineMasterNode() string { - cmd := exec.Command("sh", "-c", "kubectl get nodes --show-labels --no-headers -o wide | grep nodetype=master | awk '{print $6}'") - out, err := cmd.CombinedOutput() - if err != nil { - log.Fatal(err) - } - return strings.Trim(string(out), " \n") -} - -func DetermineLoaderNode() string { - cmd := exec.Command("sh", "-c", "kubectl get nodes --show-labels --no-headers -o wide | grep nodetype=monitoring | awk '{print $6}'") - out, err := cmd.CombinedOutput() - if err != nil { - log.Fatal(err) - } - return strings.Trim(string(out), " \n") -} - -func DetermineOtherNodes(podNamePrefix string) string { - // Get the pod alias - cmdPodName := exec.Command("sh", "-c", fmt.Sprintf("kubectl get pods -n knative-serving --no-headers | grep %s- | awk '{print $1}'", podNamePrefix)) - out, err := cmdPodName.CombinedOutput() - - if err != nil { - log.Fatal("Error getting", podNamePrefix, "pod name:", err) - } - - // Get the private ip using the pod alias - podName := strings.Trim(string(out), "\n") - cmdNodeIP := exec.Command("sh", "-c", fmt.Sprintf("kubectl get pod %s -n knative-serving -o=jsonpath='{.status.hostIP}'", podName)) - out, err = cmdNodeIP.CombinedOutput() - - if err != nil { - log.Fatal("Error getting", cmdNodeIP, "node IP:", err) - } - - nodeIp := strings.Split(string(out), "\n")[0] - return strings.Trim(nodeIp, " ") -} - func RunScript(command string) { if command == "" { return diff --git a/pkg/common/validators.go b/pkg/common/validators.go index b9800fbb..0ae8138a 100644 --- a/pkg/common/validators.go +++ b/pkg/common/validators.go @@ -75,28 +75,6 @@ func CheckMultiLoaderConfig(multiLoaderConfig MultiLoaderConfiguration) { log.Info("All experiments configs are valid") } -// check platform specific multi-loader configuration -func CheckMultiLoaderPlatformSpecificConfig(multiLoaderConfig MultiLoaderConfiguration, nodeGroup NodeGroup, platform string) { - log.Info("Checking platform specific multi-loader configuration") - // For knative platform, there is an additional check for nodes - if platform == "Knative" || platform == "Knative-RPS" { - // Check if metrics are valid - for _, metric := range multiLoaderConfig.Metrics { - CheckCollectableMetrics(metric) - } - // Check nodes - CheckNode(nodeGroup.MasterNode) - CheckNode(nodeGroup.AutoScalerNode) - CheckNode(nodeGroup.ActivatorNode) - CheckNode(nodeGroup.LoaderNode) - for _, node := range nodeGroup.WorkerNodes { - CheckNode(node) - } - log.Info("Nodes are reachable") - } - log.Info("All platform specific configs are valid") -} - func CheckCollectableMetrics(metrics string) { if !slices.Contains(ValidCollectableMetrics, metrics) { log.Fatal("Invalid metrics ", metrics) diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index 8a1b5581..ae0ef969 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -56,14 +56,6 @@ func NewMultiLoaderRunner(configPath string, verbosity string, iatGeneration boo Platform: platform, } - // For knative platform, help to determine and validate nodes in cluster - if platform == "Knative" || platform == "Knative-RPS" { - nodeGroup := determineNodes(multiLoaderConfig) - // add to runner - runner.NodeGroup = nodeGroup - common.CheckMultiLoaderPlatformSpecificConfig(multiLoaderConfig, nodeGroup, platform) - } - return &runner, nil } @@ -81,32 +73,6 @@ func determinePlatform(multiLoaderConfig common.MultiLoaderConfiguration) string return loaderConfig.Platform } -func determineNodes(multiLoaderConfig common.MultiLoaderConfiguration) common.NodeGroup { - var nodeGroup common.NodeGroup - nodeGroup.MasterNode = multiLoaderConfig.MasterNode - nodeGroup.AutoScalerNode = multiLoaderConfig.AutoScalerNode - nodeGroup.ActivatorNode = multiLoaderConfig.ActivatorNode - nodeGroup.LoaderNode = multiLoaderConfig.LoaderNode - nodeGroup.WorkerNodes = multiLoaderConfig.WorkerNodes - - if len(nodeGroup.WorkerNodes) == 0 { - nodeGroup.WorkerNodes = common.DetermineWorkerNodes() - } - if nodeGroup.MasterNode == "" { - nodeGroup.MasterNode = common.DetermineMasterNode() - } - if nodeGroup.LoaderNode == "" { - nodeGroup.LoaderNode = common.DetermineLoaderNode() - } - if nodeGroup.AutoScalerNode == "" { - nodeGroup.AutoScalerNode = common.DetermineOtherNodes("autoscaler") - } - if nodeGroup.ActivatorNode == "" { - nodeGroup.ActivatorNode = common.DetermineOtherNodes("activator") - } - return nodeGroup -} - func (d *MultiLoaderRunner) RunDryRun() { log.Info("Running dry run") d.DryRun = true From e4b26d1067c6e4ba09c1363559fb069ba7475053 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 18:42:34 +0800 Subject: [PATCH 25/37] add multi loader tests Signed-off-by: Lenson --- .../runner/multi_loader_runner_test.go | 237 ++++++++++++++++++ .../test_configs/test_base_loader_config.json | 28 +++ .../test_multi_loader_config.json | 17 ++ 3 files changed, 282 insertions(+) create mode 100644 tools/multi_loader/runner/multi_loader_runner_test.go create mode 100644 tools/multi_loader/runner/test_configs/test_base_loader_config.json create mode 100644 tools/multi_loader/runner/test_configs/test_multi_loader_config.json diff --git a/tools/multi_loader/runner/multi_loader_runner_test.go b/tools/multi_loader/runner/multi_loader_runner_test.go new file mode 100644 index 00000000..c6e2f07f --- /dev/null +++ b/tools/multi_loader/runner/multi_loader_runner_test.go @@ -0,0 +1,237 @@ +package runner + +import ( + "os" + "path/filepath" + "strings" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/vhive-serverless/loader/pkg/common" +) + +var ( + multiLoaderTestConfigPath string + configPath string +) + +func init() { + wd, _ := os.Getwd() + multiLoaderTestConfigPath = filepath.Join(wd, "./test_configs/test_multi_loader_config.json") + configPath = filepath.Join(wd, "./test_configs/base_loader_config.json") + log.Info("Test config path: ", multiLoaderTestConfigPath) + log.Info("Test config path: ", configPath) +} + +func TestUnpackExperiment(t *testing.T) { + // helper func to validate unpacked experiments + validateUnpackedExperiment := func(t *testing.T, experimentConfig []common.LoaderStudy, studyConfig common.LoaderStudy, expectedNames []string, expectedOutputPrefixes []string) { + if len(experimentConfig) != len(expectedNames) { + t.Errorf("Expected %d sub-experiments, got %d", len(expectedNames), len(experimentConfig)) + return + } + + for i, subExp := range experimentConfig { + // check name + if subExp.Name != expectedNames[i] { + t.Errorf("Expected subexperiment name '%s', got '%s'", expectedNames[i], subExp.Name) + } + + // validate selected configs + if subExp.Config["ExperimentDuration"] != studyConfig.Config["ExperimentDuration"] { + t.Errorf("Expected ExperimentDuration %v, got %v", studyConfig.Config["ExperimentDuration"], subExp.Config["ExperimentDuration"]) + } + if subExp.OutputDir != studyConfig.OutputDir { + t.Errorf("Expected OutputDir '%s', got '%s'", studyConfig.OutputDir, subExp.OutputDir) + } + + // check OutputPathPrefix if needed + if len(expectedOutputPrefixes) > 0 { + if outputPathPrefix, ok := subExp.Config["OutputPathPrefix"].(string); !(ok && strings.HasSuffix(outputPathPrefix, expectedOutputPrefixes[i])) { + t.Errorf("Expected OutputPathPrefix '%s', got '%s'", expectedOutputPrefixes[i], subExp.Config["OutputPathPrefix"]) + } + } + } + } + + // create multiloader + multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false) + if err != nil { + t.Fatalf("Failed to create multi-loader driver: %v", err) + } + + + t.Run("Unpack using TracesDir (Success)", func(t *testing.T) { + // Set TracesDir to test directory + multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "./test_multi_trace" + + for _, experiment := range multiLoader.MultiLoaderConfig.Studies { + subExperiments := multiLoader.unpackStudy(experiment) + expectedNames := []string{experiment.Name, experiment.Name, experiment.Name} + expectedOutputPrefixes := []string{"example_1_test", "example_2_test", "example_3.1_test"} + validateUnpackedExperiment(t, subExperiments, experiment, expectedNames, expectedOutputPrefixes) + } + }) + + t.Run("Unpack using TracesDir (Failure: Incorrect Dir)", func(t *testing.T) { + expectFatal(t, func() { + multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "./test_multi_trace_incorrect" + for _, experiment := range multiLoader.MultiLoaderConfig.Studies { + _ = multiLoader.unpackStudy(experiment) + } + }) + }) + + t.Run("Unpack using TraceFormat and TraceValues", func(t *testing.T) { + multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "" + + for _, experiment := range multiLoader.MultiLoaderConfig.Studies { + subExperiments := multiLoader.unpackStudy(experiment) + expectedNames := make([]string, len(experiment.TraceValues)) + for i, traceValue := range experiment.TraceValues { + expectedNames[i] = experiment.Name + "_" + traceValue.(string) + } + validateUnpackedExperiment(t, subExperiments, experiment, expectedNames, nil) + } + }) + + t.Run("Unpack using tracePath", func(t *testing.T) { + multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "" + multiLoader.MultiLoaderConfig.Studies[0].TracesFormat = "" + multiLoader.MultiLoaderConfig.Studies[0].TraceValues = nil + + for _, experiment := range multiLoader.MultiLoaderConfig.Studies { + subExperiments := multiLoader.unpackStudy(experiment) + expectedNames := []string{experiment.Name} + validateUnpackedExperiment(t, subExperiments, experiment, expectedNames, nil) + } + }) +} + +func TestPrepareExperiment(t *testing.T) { + // Create a new multi-loader driver with the test config path + multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false) + if err != nil { + t.Fatalf("Failed to create multi-loader driver: %v", err) + } + + subExperiment := common.LoaderStudy{ + Name: "example_1", + Config: map[string]interface{}{ + "ExperimentDuration": 10, + "TracePath": "./test_multi_trace/example_1_test", + "OutputPathPrefix": "./test_output/example_1_test", + }, + } + + if err := os.MkdirAll(filepath.Dir(EXPERIMENT_TEMP_CONFIG_PATH), 0755); err != nil { + t.Fatalf("Failed to create temp config directory: %v", err) + } + multiLoader.prepareExperiment(subExperiment) + + // Check that the output directory and config file were created + outputDir := "./test_output" + tempConfigPath := EXPERIMENT_TEMP_CONFIG_PATH + + // Verify the output directory exists + if _, err := os.Stat(outputDir); os.IsNotExist(err) { + t.Errorf("Expected output directory '%s' to be created, but it was not", outputDir) + } + + // Verify the temporary config file exists + if _, err := os.Stat(tempConfigPath); os.IsNotExist(err) { + t.Errorf("Expected temp config file '%s' to be created, but it was not", tempConfigPath) + } + + // Clean up created files and directories + os.RemoveAll("./tools") + os.RemoveAll(outputDir) +} + +// Test mergeConfigurations method +func TestMergeConfig(t *testing.T){ + // Create a new multi-loader driver with the test config path + multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false) + if err != nil { + t.Fatalf("Failed to create multi-loader driver: %v", err) + } + experiment := common.LoaderStudy{ + Name: "example_1", + Config: map[string]interface{}{ + "ExperimentDuration": 10, + "TracePath": "./test_multi_trace/example_1_test", + "OutputPathPrefix": "./test_output/example_1_test", + }, + } + outputConfig := multiLoader.mergeConfigurations("./test_configs/test_base_loader_config.json", experiment) + // Check if the configurations are merged + if outputConfig.TracePath != "./test_multi_trace/example_1_test" { + t.Errorf("Expected TracePath to be './test_multi_trace/example_1_test', got %v", experiment.Config["TracePath"]) + } + if outputConfig.OutputPathPrefix != "./test_output/example_1_test" { + t.Errorf("Expected OutputPathPrefix to be './test_output/example_1_test', got %v", experiment.Config["OutputPathPrefix"]) + } + if outputConfig.ExperimentDuration != 10 { + t.Errorf("Expected ExperimentDuration to be 10, got %v", experiment.Config["ExperimentDuration"]) + } +} + +func TestMultiConfigValidator(t *testing.T){ + // Create a new multi-loader driver with the test config path + multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false) + if err != nil { + t.Fatalf("Failed to create multi-loader driver: %v", err) + } + t.Run("CheckMultiLoaderConfig (Success)", func(t *testing.T) { + // Check if all paths are valid + common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) + }) + + t.Run("CheckMultiLoaderConfig (Failure: No Study)", func(t *testing.T) { + expectFatal(t, func() { + temp := multiLoader.MultiLoaderConfig.Studies + multiLoader.MultiLoaderConfig.Studies = nil + common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) + multiLoader.MultiLoaderConfig.Studies = temp + }) + }) + + t.Run("CheckMultiLoaderConfig (Failure: Missing TracesDir, TracesFormat, TraceValues)", func(t *testing.T) { + expectFatal(t, func() { + multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "" + multiLoader.MultiLoaderConfig.Studies[0].TracesFormat = "" + multiLoader.MultiLoaderConfig.Studies[0].TraceValues = nil + common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) + }) + }) + + t.Run("CheckMultiLoaderConfig (Failure: Invalid TracesFormat)", func(t *testing.T) { + expectFatal(t, func() { + multiLoader.MultiLoaderConfig.Studies[0].TracesFormat = "invalid_format" + common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) + }) + }) + + t.Run("CheckMultiLoaderConfig (Failure: Missing TracesValues)", func(t *testing.T) { + expectFatal(t, func() { + multiLoader.MultiLoaderConfig.Studies[0].TraceValues = nil + multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "" + multiLoader.MultiLoaderConfig.Studies[0].TracesFormat = "example_{}_test" + common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) + }) + }) +} + +func expectFatal(t *testing.T,funcToTest func()) { + fatal := false + originalExitFunc := log.StandardLogger().ExitFunc + + // replace logrus exit function + log.StandardLogger().ExitFunc = func(int) { fatal = true } + + funcToTest() + // restore original state + log.StandardLogger().ExitFunc = originalExitFunc + assert.True(t, fatal, "Expected log.Fatal to be called") +} \ No newline at end of file diff --git a/tools/multi_loader/runner/test_configs/test_base_loader_config.json b/tools/multi_loader/runner/test_configs/test_base_loader_config.json new file mode 100644 index 00000000..592bbb6f --- /dev/null +++ b/tools/multi_loader/runner/test_configs/test_base_loader_config.json @@ -0,0 +1,28 @@ +{ + "Seed": 42, + + "Platform": "Test", + "InvokeProtocol" : "grpc", + "YAMLSelector": "container", + "EndpointPort": 80, + + "BusyLoopOnSandboxStartup": false, + + "TracePath": "data/traces/example", + "Granularity": "minute", + "OutputPathPrefix": "data/out/experiment", + "IATDistribution": "exponential", + "CPULimit": "1vCPU", + "ExperimentDuration": 5, + "WarmupDuration": 0, + + "IsPartiallyPanic": false, + "EnableZipkinTracing": false, + "EnableMetricsScrapping": false, + "MetricScrapingPeriodSeconds": 15, + "AutoscalingMetric": "concurrency", + + "GRPCConnectionTimeoutSeconds": 15, + "GRPCFunctionTimeoutSeconds": 900, + "DAGMode": false +} diff --git a/tools/multi_loader/runner/test_configs/test_multi_loader_config.json b/tools/multi_loader/runner/test_configs/test_multi_loader_config.json new file mode 100644 index 00000000..7c07c14e --- /dev/null +++ b/tools/multi_loader/runner/test_configs/test_multi_loader_config.json @@ -0,0 +1,17 @@ +{ + "Studies": [ + { + "Name": "test-experiment", + "Config": { + "ExperimentDuration": 1 + }, + "IatGeneration": false, + "Generated": false, + "Verbosity": "info", + "TracesFormat": "data/traces/{}", + "TraceValues": ["example", "example2"], + "OutputDir": "data/out/multi-test" + } + ], + "BaseConfigPath": "./test_configs/test_base_loader_config.json" +} \ No newline at end of file From 37b1d832942d07886899f9665b31f638aad42ca5 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sat, 16 Nov 2024 18:49:35 +0800 Subject: [PATCH 26/37] add multi loader documentation Signed-off-by: Lenson --- docs/multi_loader.md | 107 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 docs/multi_loader.md diff --git a/docs/multi_loader.md b/docs/multi_loader.md new file mode 100644 index 00000000..02af5984 --- /dev/null +++ b/docs/multi_loader.md @@ -0,0 +1,107 @@ +# Multi-Loader + +A wrapper around loader to run multiple experiments at once with additional features like validation, dry-run, log collection + +## Prerequisites +As a wrapper around loader, multi-loader requires the initial cluster setup to be completed. See [vHive Loader to create a cluster](https://github.com/vhive-serverless/invitro/blob/main/docs/loader.md#create-a-cluster) + +## Configuration +### Multi-Loader Configuration +| Parameter name | Data type | Possible values | Default value | Description | +|---------------------|--------------------|-----------------|---------------|------------------------------------------------------------| +| Studies | []LoaderStudy | N/A | N/A | A list of loader studies with their respective configurations. See [LoaderStudy](#loaderstudy) | +| BaseConfigPath | string | "tools/multi_loader/base_loader_config.json" | N/A | Path to the base configuration file | +| PreScript | string | any bash command | "" | (Optional) A global script that runs once before all experiments | +| PostScript | string | any bash command | "" | (Optional) A global script that runs once after all experiments | + +### LoaderStudy +| Parameter name | Data type | Possible values | Default value | Description | +|-----------------------|------------------------|-------------------------------|---------------|--------------------------------------------------------------------| +| Config | map[string]interface{} | Any field in [LoaderConfiguration](https://github.com/vhive-serverless/invitro/blob/main/docs/configuration.md#loader-configuration-file-format) | N/A | The configuration for each loader experiment which overrides configurations in baseLoaderConfig | +| Name | string | N/A | N/A | The name of the loader experiment | +| TracesDir | string | N/A | N/A | Directory containing the traces for the experiment | +| TracesFormat | string | "data/traces/example_{}" | N/A | Format of the trace files **The format string "{}" is required** | +| TraceValues | []interface{} | ["any", 0, 1.1] | N/A | Values of the trace files Replaces the "{}" in TraceFormat | +| OutputDir | string | any | data/out/{Name} | (Optional) Output directory for experiment results | +| Verbosity | string | "info", "debug", "trace" | "info" | (Optional) Verbosity level for logging the experiment | +| IatGeneration | bool | true, false | false | (Optional) Whether to Generate iats only and skip invocations | +| Generated | bool | true, false | false | (Optional) if iats were already generated | +| PreScript | string | any bash Command | "" | (Optional) Local script that runs this specific experiment | +| PostScript | string | any bash Command | "" | (Optional) Local script that runs this specific experiment | + +> **_Important_**: Only one of the following is required: +> 1. `TracesDir`, or +> 2. `TracesFormat` and `TraceValues`, or +> 3. `TracePath` within the `LoaderExperiment`'s `Config` field +> +> If more than one is defined, the order of precedence is as follows: +> 1. `TracesDir`, +> 2. `TracesFormat` and `TraceValues`, +> 3. `TracePath` + +> **_Note_**: +> The `Config` field follows the same structure as the [LoaderConfiguration](https://github.com/vhive-serverless/invitro/blob/main/docs/configuration.md#loader-configuration-file-format). +> Any field defined in `Config` will override the corresponding value from the configuration in `BaseConfigPath`, but only for that specific experiment. +> For example, if `BaseConfigPath` has `ExperimentDuration` set to 5 minutes, and you define `ExperimentDuration` as 10 minutes in `Config`, that particular experiment will run for 10 minutes instead. + +## Command Flags + +The multi-loader accepts the almost the same command-line flags as loader. + +> **_Note_**: These flags will subsequently be used during the execution of loader.go for **every experiment**. If you would like to define these flag for specific experiments only, define it in [LoaderStudy](#loaderstudy) + +Available flags: + +- **`--multiLoaderConfig`** *(default: `tools/multi_loader/multi_loader_config.json`)*: + Specifies the path to the multi-loader configuration file. This file contains settings and parameters that define how the multi-loader operates [see above](#multi-loader-configuration) + +- **`--verbosity`** *(default: `info`)*: + Sets the logging verbosity level. You can choose from the following options: + - `info`: Standard information messages. + - `debug`: Detailed debugging messages. + - `trace`: Extremely verbose logging, including detailed execution traces. + +- **`--iatGeneration`** *(default: `false`)*: + If set to `true`, the multi-loader will generate inter-arrival times (IATs) only and skip the invocation of actual workloads. This is useful for scenarios where you want to analyze or generate IATs without executing the associated load. + +- **`--generated`** *(default: `false`)*: + Indicates whether IATs have already been generated. If set to `true`, the multi-loader will use the existing IATs instead of generating new ones. + + +## Multi-loader Overall Flow + +1. **Initialization** + - Flags for configuration file path, verbosity, IAT generation, and execution mode are parsed + - Logger is initialized based on verbosity level + +3. **Experiment Execution Flow** + - The multi-loader runner is instantiated with the provided configuration path. + - A dry run is executed to validate the setup for all studies: + - If any dry run fails, the execution terminates. + - If all dry runs succeed, proceed to actual runs: + - Global pre-scripts are executed. + - Each experiment undergoes the following steps: + 1. **Pre-Execution Setup** + - Experiment-specific pre-scripts are executed. + - Necessary directories and folders are created. + - Each sub-experiment is unpacked and prepared + 2. **Experiment Invocation** + - The loader is executed with generated configurations and related flags + 3. **Post-Execution Steps** + - Experiment-specific post-scripts are executed + - Cleanup tasks are performed + +4. **Completion** + - Global post-scripts are executed. + - Run Make Clean + +### How the Dry Run Works + +The dry run mode executes the loader with the `--dryRun` flag set to true after the unpacking of experiments defined in the multi-loader configurations. + +In this mode, the loader performs the following actions: + +- **Configuration Validation**: It verifies the experiment configurations without deploying any functions or generating invocations. +- **Error Handling**: If a fatal error occurs at any point, the experiment will halt immediately. + +The purpose is to ensure that your configurations are correct and to identify any potential issues before actual execution. \ No newline at end of file From 1128742c3084fef61fb9c64af3868dc7ccf9d626 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sun, 17 Nov 2024 14:07:35 +0800 Subject: [PATCH 27/37] fix unpack from trace dir naming Signed-off-by: Lenson --- tools/multi_loader/runner/multi_loader_runner.go | 2 ++ tools/multi_loader/runner/multi_loader_runner_test.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index ae0ef969..cd5036b1 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -158,6 +158,8 @@ func (d *MultiLoaderRunner) unpackFromTraceDir(study common.LoaderStudy) []commo for _, file := range files { newExperiment := d.createNewStudy(study, file.Name()) + newExperiment.Config["TracePath"] = path.Join(study.TracesDir, file.Name()) + newExperiment.Name += "_" + file.Name() experiments = append(experiments, newExperiment) } return experiments diff --git a/tools/multi_loader/runner/multi_loader_runner_test.go b/tools/multi_loader/runner/multi_loader_runner_test.go index c6e2f07f..ce6e648b 100644 --- a/tools/multi_loader/runner/multi_loader_runner_test.go +++ b/tools/multi_loader/runner/multi_loader_runner_test.go @@ -68,7 +68,7 @@ func TestUnpackExperiment(t *testing.T) { for _, experiment := range multiLoader.MultiLoaderConfig.Studies { subExperiments := multiLoader.unpackStudy(experiment) - expectedNames := []string{experiment.Name, experiment.Name, experiment.Name} + expectedNames := []string{"test-experiment_example_1_test", "test-experiment_example_2_test", "test-experiment_example_3.1_test"} expectedOutputPrefixes := []string{"example_1_test", "example_2_test", "example_3.1_test"} validateUnpackedExperiment(t, subExperiments, experiment, expectedNames, expectedOutputPrefixes) } From 97809c0d02c1b4f81cc7873ddb71a3a7d71bee96 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sun, 22 Dec 2024 13:52:35 +0800 Subject: [PATCH 28/37] fix formatting Signed-off-by: Lenson --- tools/multi_loader/base_loader_config.json | 9 +- tools/multi_loader/multi_loader.go | 8 +- tools/multi_loader/multi_loader_config.json | 4 +- .../runner/multi_loader_runner.go | 104 +++++++++--------- .../runner/multi_loader_runner_test.go | 25 ++--- .../test_configs/test_base_loader_config.json | 9 +- .../test_multi_loader_config.json | 5 +- 7 files changed, 79 insertions(+), 85 deletions(-) diff --git a/tools/multi_loader/base_loader_config.json b/tools/multi_loader/base_loader_config.json index beae7786..209e3043 100644 --- a/tools/multi_loader/base_loader_config.json +++ b/tools/multi_loader/base_loader_config.json @@ -1,13 +1,10 @@ { "Seed": 42, - "Platform": "Knative", - "InvokeProtocol" : "grpc", + "InvokeProtocol": "grpc", "YAMLSelector": "container", "EndpointPort": 80, - "BusyLoopOnSandboxStartup": false, - "TracePath": "data/traces/example", "Granularity": "minute", "OutputPathPrefix": "data/out/experiment", @@ -15,14 +12,12 @@ "CPULimit": "1vCPU", "ExperimentDuration": 5, "WarmupDuration": 0, - "IsPartiallyPanic": false, "EnableZipkinTracing": false, "EnableMetricsScrapping": false, "MetricScrapingPeriodSeconds": 15, "AutoscalingMetric": "concurrency", - "GRPCConnectionTimeoutSeconds": 15, "GRPCFunctionTimeoutSeconds": 900, "DAGMode": false -} +} \ No newline at end of file diff --git a/tools/multi_loader/multi_loader.go b/tools/multi_loader/multi_loader.go index a801ce68..c9ba571e 100644 --- a/tools/multi_loader/multi_loader.go +++ b/tools/multi_loader/multi_loader.go @@ -10,10 +10,10 @@ import ( ) var ( - multiLoaderConfigPath = flag.String("multiLoaderConfig", "tools/multi_loader/multi_loader_config.json", "Path to multi loader configuration file") - verbosity = flag.String("verbosity", "info", "Logging verbosity - choose from [info, debug, trace]") - iatGeneration = flag.Bool("iatGeneration", false, "Generate iats only and skip invocations") - generated = flag.Bool("generated", false, "If iats were already generated") + multiLoaderConfigPath = flag.String("multiLoaderConfig", "tools/multi_loader/multi_loader_config.json", "Path to multi loader configuration file") + verbosity = flag.String("verbosity", "info", "Logging verbosity - choose from [info, debug, trace]") + iatGeneration = flag.Bool("iatGeneration", false, "Generate iats only and skip invocations") + generated = flag.Bool("generated", false, "If iats were already generated") ) func init() { diff --git a/tools/multi_loader/multi_loader_config.json b/tools/multi_loader/multi_loader_config.json index bbe65974..55e4491b 100644 --- a/tools/multi_loader/multi_loader_config.json +++ b/tools/multi_loader/multi_loader_config.json @@ -10,7 +10,9 @@ "Verbosity": "info", "TracesDir": "", "TracesFormat": "data/traces/multi-exp-test/example_{}_test", - "TraceValues": [1], + "TraceValues": [ + 1 + ], "OutputDir": "data/out/multi-test", "PreScript": "", "PostScript": "" diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index cd5036b1..700f9533 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -19,43 +19,43 @@ import ( ) const ( - LOADER_PATH = "cmd/loader.go" - TIME_FORMAT = "Jan_02_1504" + LOADER_PATH = "cmd/loader.go" + TIME_FORMAT = "Jan_02_1504" EXPERIMENT_TEMP_CONFIG_PATH = "tools/multi_loader/current_running_config.json" - NUM_OF_RETRIES = 2 + NUM_OF_RETRIES = 2 ) type MultiLoaderRunner struct { - MultiLoaderConfig common.MultiLoaderConfiguration - NodeGroup common.NodeGroup - DryRunSuccess bool - Verbosity string - IatGeneration bool - Generated bool - DryRun bool - Platform string + MultiLoaderConfig common.MultiLoaderConfiguration + NodeGroup common.NodeGroup + DryRunSuccess bool + Verbosity string + IatGeneration bool + Generated bool + DryRun bool + Platform string } // init multi loader runner func NewMultiLoaderRunner(configPath string, verbosity string, iatGeneration bool, generated bool) (*MultiLoaderRunner, error) { - multiLoaderConfig := config.ReadMultiLoaderConfigurationFile(configPath) + multiLoaderConfig := config.ReadMultiLoaderConfigurationFile(configPath) // validate configuration common.CheckMultiLoaderConfig(multiLoaderConfig) - + // determine platform platform := determinePlatform(multiLoaderConfig) - runner := MultiLoaderRunner{ - MultiLoaderConfig: multiLoaderConfig, - DryRunSuccess: true, - Verbosity: verbosity, - IatGeneration: iatGeneration, - Generated: generated, - DryRun: false, - Platform: platform, - } - + runner := MultiLoaderRunner{ + MultiLoaderConfig: multiLoaderConfig, + DryRunSuccess: true, + Verbosity: verbosity, + IatGeneration: iatGeneration, + Generated: generated, + DryRun: false, + Platform: platform, + } + return &runner, nil } @@ -74,32 +74,32 @@ func determinePlatform(multiLoaderConfig common.MultiLoaderConfiguration) string } func (d *MultiLoaderRunner) RunDryRun() { - log.Info("Running dry run") - d.DryRun = true - d.run() + log.Info("Running dry run") + d.DryRun = true + d.run() } func (d *MultiLoaderRunner) RunActual() { - log.Info("Running actual experiments") - d.DryRun = false - d.run() + log.Info("Running actual experiments") + d.DryRun = false + d.run() } -func (d *MultiLoaderRunner) run(){ +func (d *MultiLoaderRunner) run() { // Run global prescript common.RunScript(d.MultiLoaderConfig.PreScript) // Iterate over studies and run them for _, study := range d.MultiLoaderConfig.Studies { log.Info("Setting up experiment: ", study.Name) // Run pre script - common.RunScript(study.PreScript) + common.RunScript(study.PreScript) // Unpack study to a list of studies with different loader configs sparseExperiments := d.unpackStudy(study) // Iterate over sparse experiments, prepare and run for _, experiment := range sparseExperiments { - if d.DryRun{ + if d.DryRun { log.Info("Dry Running: ", experiment.Name) } // Prepare experiment: merge with base config, create output dir and write merged config to temp file @@ -118,7 +118,7 @@ func (d *MultiLoaderRunner) run(){ } // Run post script common.RunScript(study.PostScript) - if len(sparseExperiments) > 1 && !d.DryRun{ + if len(sparseExperiments) > 1 && !d.DryRun { log.Info("All experiments for ", study.Name, " completed") } } @@ -126,11 +126,11 @@ func (d *MultiLoaderRunner) run(){ common.RunScript(d.MultiLoaderConfig.PostScript) } -/** +/** * As a study can have multiple experiments, this function will unpack the study -* but first by duplicating the study to multiple studies with different values +* but first by duplicating the study to multiple studies with different values * in the config field. Those values will override the base loader config later -*/ + */ func (d *MultiLoaderRunner) unpackStudy(experiment common.LoaderStudy) []common.LoaderStudy { log.Info("Unpacking experiment ", experiment.Name) var experiments []common.LoaderStudy @@ -138,7 +138,7 @@ func (d *MultiLoaderRunner) unpackStudy(experiment common.LoaderStudy) []common. // if user specified a trace directory if experiment.TracesDir != "" { experiments = d.unpackFromTraceDir(experiment) - // user define trace format and values instead of directory + // user define trace format and values instead of directory } else if experiment.TracesFormat != "" && len(experiment.TraceValues) > 0 { experiments = d.unpackFromTraceValues(experiment) } else { @@ -223,17 +223,17 @@ func (d *MultiLoaderRunner) addCommandFlags(study common.LoaderStudy) { } if !study.Generated { study.Generated = d.Generated - } + } } /** * Prepare experiment by merging with base config, creating output directory and writing experiment config to temp file -*/ + */ func (d *MultiLoaderRunner) prepareExperiment(experiment common.LoaderStudy) { log.Info("Preparing ", experiment.Name) // Merge base configs with experiment configs experimentConfig := d.mergeConfigurations(d.MultiLoaderConfig.BaseConfigPath, experiment) - + // Create output directory outputDir := path.Dir(experimentConfig.OutputPathPrefix) @@ -246,7 +246,7 @@ func (d *MultiLoaderRunner) prepareExperiment(experiment common.LoaderStudy) { /** * Merge base configs with partial loader configs -*/ + */ func (d *MultiLoaderRunner) mergeConfigurations(baseConfigPath string, experiment common.LoaderStudy) config.LoaderConfiguration { // Read base configuration baseConfigByteValue, err := os.ReadFile(baseConfigPath) @@ -254,7 +254,7 @@ func (d *MultiLoaderRunner) mergeConfigurations(baseConfigPath string, experimen log.Fatal(err) } log.Debug("Experiment configuration ", experiment.Config) - + var mergedConfig config.LoaderConfiguration // Unmarshal base configuration if err = json.Unmarshal(baseConfigByteValue, &mergedConfig); err != nil { @@ -262,7 +262,7 @@ func (d *MultiLoaderRunner) mergeConfigurations(baseConfigPath string, experimen } log.Debug("Base configuration ", mergedConfig) - + // merge experiment config onto base config experimentConfigBytes, _ := json.Marshal(experiment.Config) if err = json.Unmarshal(experimentConfigBytes, &mergedConfig); err != nil { @@ -303,7 +303,7 @@ func (d *MultiLoaderRunner) runExperiment(experiment common.LoaderStudy) error { log.Info("Retrying experiment ", experiment.Name) logFile.WriteString("==================================RETRYING==================================\n") experiment.Verbosity = "debug" - } else{ + } else { // Experiment failed set dry run flag to false d.DryRunSuccess = false log.Error("Check log file for more information: ", logFilePath) @@ -311,7 +311,7 @@ func (d *MultiLoaderRunner) runExperiment(experiment common.LoaderStudy) error { return err } continue - }else{ + } else { break } } @@ -329,14 +329,14 @@ func (d *MultiLoaderRunner) executeLoaderCommand(experiment common.LoaderStudy, stdout, _ := cmd.StdoutPipe() stderr, _ := cmd.StderrPipe() - + if err := cmd.Start(); err != nil { return err } - + go d.logLoaderStdOutput(stdout, logFile) go d.logLoaderStdError(stderr, logFile) - + return cmd.Wait() } @@ -346,14 +346,14 @@ func (d *MultiLoaderRunner) logLoaderStdOutput(stdPipe io.ReadCloser, logFile *o m := scanner.Text() // write to log file logFile.WriteString(m + "\n") - + // Log key information if m == "" { continue } logType := common.ParseLogType(m) message := common.ParseLogMessage(m) - + switch logType { case "debug": log.Debug(message) @@ -361,7 +361,7 @@ func (d *MultiLoaderRunner) logLoaderStdOutput(stdPipe io.ReadCloser, logFile *o log.Trace(message) default: if strings.Contains(message, "Number of successful invocations:") || strings.Contains(message, "Number of failed invocations:") { - log.Info(strings.ReplaceAll(strings.ReplaceAll(message, "\\t", " ",), "\\n", "")) + log.Info(strings.ReplaceAll(strings.ReplaceAll(message, "\\t", " "), "\\n", "")) } } } @@ -373,7 +373,7 @@ func (d *MultiLoaderRunner) logLoaderStdError(stdPipe io.ReadCloser, logFile *os m := scanner.Text() // write to log file logFile.WriteString(m + "\n") - + if m == "" { continue } diff --git a/tools/multi_loader/runner/multi_loader_runner_test.go b/tools/multi_loader/runner/multi_loader_runner_test.go index ce6e648b..f4ffa903 100644 --- a/tools/multi_loader/runner/multi_loader_runner_test.go +++ b/tools/multi_loader/runner/multi_loader_runner_test.go @@ -13,7 +13,7 @@ import ( var ( multiLoaderTestConfigPath string - configPath string + configPath string ) func init() { @@ -61,7 +61,6 @@ func TestUnpackExperiment(t *testing.T) { t.Fatalf("Failed to create multi-loader driver: %v", err) } - t.Run("Unpack using TracesDir (Success)", func(t *testing.T) { // Set TracesDir to test directory multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "./test_multi_trace" @@ -73,7 +72,7 @@ func TestUnpackExperiment(t *testing.T) { validateUnpackedExperiment(t, subExperiments, experiment, expectedNames, expectedOutputPrefixes) } }) - + t.Run("Unpack using TracesDir (Failure: Incorrect Dir)", func(t *testing.T) { expectFatal(t, func() { multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "./test_multi_trace_incorrect" @@ -120,8 +119,8 @@ func TestPrepareExperiment(t *testing.T) { Name: "example_1", Config: map[string]interface{}{ "ExperimentDuration": 10, - "TracePath": "./test_multi_trace/example_1_test", - "OutputPathPrefix": "./test_output/example_1_test", + "TracePath": "./test_multi_trace/example_1_test", + "OutputPathPrefix": "./test_output/example_1_test", }, } @@ -130,7 +129,7 @@ func TestPrepareExperiment(t *testing.T) { } multiLoader.prepareExperiment(subExperiment) - // Check that the output directory and config file were created + // Check that the output directory and config file were created outputDir := "./test_output" tempConfigPath := EXPERIMENT_TEMP_CONFIG_PATH @@ -139,7 +138,7 @@ func TestPrepareExperiment(t *testing.T) { t.Errorf("Expected output directory '%s' to be created, but it was not", outputDir) } - // Verify the temporary config file exists + // Verify the temporary config file exists if _, err := os.Stat(tempConfigPath); os.IsNotExist(err) { t.Errorf("Expected temp config file '%s' to be created, but it was not", tempConfigPath) } @@ -150,7 +149,7 @@ func TestPrepareExperiment(t *testing.T) { } // Test mergeConfigurations method -func TestMergeConfig(t *testing.T){ +func TestMergeConfig(t *testing.T) { // Create a new multi-loader driver with the test config path multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false) if err != nil { @@ -160,8 +159,8 @@ func TestMergeConfig(t *testing.T){ Name: "example_1", Config: map[string]interface{}{ "ExperimentDuration": 10, - "TracePath": "./test_multi_trace/example_1_test", - "OutputPathPrefix": "./test_output/example_1_test", + "TracePath": "./test_multi_trace/example_1_test", + "OutputPathPrefix": "./test_output/example_1_test", }, } outputConfig := multiLoader.mergeConfigurations("./test_configs/test_base_loader_config.json", experiment) @@ -177,7 +176,7 @@ func TestMergeConfig(t *testing.T){ } } -func TestMultiConfigValidator(t *testing.T){ +func TestMultiConfigValidator(t *testing.T) { // Create a new multi-loader driver with the test config path multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false) if err != nil { @@ -223,7 +222,7 @@ func TestMultiConfigValidator(t *testing.T){ }) } -func expectFatal(t *testing.T,funcToTest func()) { +func expectFatal(t *testing.T, funcToTest func()) { fatal := false originalExitFunc := log.StandardLogger().ExitFunc @@ -234,4 +233,4 @@ func expectFatal(t *testing.T,funcToTest func()) { // restore original state log.StandardLogger().ExitFunc = originalExitFunc assert.True(t, fatal, "Expected log.Fatal to be called") -} \ No newline at end of file +} diff --git a/tools/multi_loader/runner/test_configs/test_base_loader_config.json b/tools/multi_loader/runner/test_configs/test_base_loader_config.json index 592bbb6f..b20768b1 100644 --- a/tools/multi_loader/runner/test_configs/test_base_loader_config.json +++ b/tools/multi_loader/runner/test_configs/test_base_loader_config.json @@ -1,13 +1,10 @@ { "Seed": 42, - "Platform": "Test", - "InvokeProtocol" : "grpc", + "InvokeProtocol": "grpc", "YAMLSelector": "container", "EndpointPort": 80, - "BusyLoopOnSandboxStartup": false, - "TracePath": "data/traces/example", "Granularity": "minute", "OutputPathPrefix": "data/out/experiment", @@ -15,14 +12,12 @@ "CPULimit": "1vCPU", "ExperimentDuration": 5, "WarmupDuration": 0, - "IsPartiallyPanic": false, "EnableZipkinTracing": false, "EnableMetricsScrapping": false, "MetricScrapingPeriodSeconds": 15, "AutoscalingMetric": "concurrency", - "GRPCConnectionTimeoutSeconds": 15, "GRPCFunctionTimeoutSeconds": 900, "DAGMode": false -} +} \ No newline at end of file diff --git a/tools/multi_loader/runner/test_configs/test_multi_loader_config.json b/tools/multi_loader/runner/test_configs/test_multi_loader_config.json index 7c07c14e..9138a040 100644 --- a/tools/multi_loader/runner/test_configs/test_multi_loader_config.json +++ b/tools/multi_loader/runner/test_configs/test_multi_loader_config.json @@ -9,7 +9,10 @@ "Generated": false, "Verbosity": "info", "TracesFormat": "data/traces/{}", - "TraceValues": ["example", "example2"], + "TraceValues": [ + "example", + "example2" + ], "OutputDir": "data/out/multi-test" } ], From b60581ef5b23e75f93ec493a09a774b5fed64563 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sun, 22 Dec 2024 14:16:43 +0800 Subject: [PATCH 29/37] rename createNewStudy function name Signed-off-by: Lenson --- tools/multi_loader/runner/multi_loader_runner.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index 700f9533..0db66c5f 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -157,7 +157,7 @@ func (d *MultiLoaderRunner) unpackFromTraceDir(study common.LoaderStudy) []commo } for _, file := range files { - newExperiment := d.createNewStudy(study, file.Name()) + newExperiment := d.duplicateStudy(study, file.Name()) newExperiment.Config["TracePath"] = path.Join(study.TracesDir, file.Name()) newExperiment.Name += "_" + file.Name() experiments = append(experiments, newExperiment) @@ -170,7 +170,7 @@ func (d *MultiLoaderRunner) unpackFromTraceValues(study common.LoaderStudy) []co for _, traceValue := range study.TraceValues { tracePath := strings.Replace(study.TracesFormat, common.TraceFormatString, fmt.Sprintf("%v", traceValue), -1) fileName := path.Base(tracePath) - newExperiment := d.createNewStudy(study, fileName) + newExperiment := d.duplicateStudy(study, fileName) newExperiment.Config["TracePath"] = tracePath newExperiment.Name += "_" + fileName experiments = append(experiments, newExperiment) @@ -187,12 +187,15 @@ func (d *MultiLoaderRunner) unpackSingleExperiment(study common.LoaderStudy) []c pathDir = study.OutputDir } study.OutputDir = pathDir - newExperiment := d.createNewStudy(study, study.Name) + newExperiment := d.duplicateStudy(study, study.Name) experiments = append(experiments, newExperiment) return experiments } -func (d *MultiLoaderRunner) createNewStudy(study common.LoaderStudy, fileName string) common.LoaderStudy { +/** +* Creates a deepcopy of a given study and updates relevant fields to utilise the provided new filename + */ +func (d *MultiLoaderRunner) duplicateStudy(study common.LoaderStudy, newFileName string) common.LoaderStudy { newStudy, err := common.DeepCopy(study) if err != nil { log.Fatal(err) @@ -206,8 +209,8 @@ func (d *MultiLoaderRunner) createNewStudy(study common.LoaderStudy, fileName st study.OutputDir, study.Name, dryRunAdditionalPath, - time.Now().Format(TIME_FORMAT)+"_"+fileName, - fileName, + time.Now().Format(TIME_FORMAT)+"_"+newFileName, + newFileName, ) d.addCommandFlags(newStudy) return newStudy From e44c33f9fbea52afeed003b646a050cd2be34fb5 Mon Sep 17 00:00:00 2001 From: Lenson Date: Sun, 22 Dec 2024 14:56:18 +0800 Subject: [PATCH 30/37] refactor common files to multiloader folder Signed-off-by: Lenson refactor multiloader functions Signed-off-by: Lenson --- pkg/common/config_types.go | 27 --------- pkg/common/constants.go | 15 ----- pkg/common/node_types.go | 9 --- pkg/common/validators.go | 43 -------------- pkg/config/parser.go | 16 ------ tools/multi_loader/common/constants.go | 15 +++++ tools/multi_loader/common/utils.go | 40 +++++++++++++ tools/multi_loader/common/validators.go | 52 +++++++++++++++++ .../runner/multi_loader_runner.go | 56 ++++++++----------- .../runner/multi_loader_runner_test.go | 19 ++++--- tools/multi_loader/types/config_types.go | 27 +++++++++ tools/multi_loader/types/node_types.go | 9 +++ 12 files changed, 175 insertions(+), 153 deletions(-) delete mode 100644 pkg/common/config_types.go delete mode 100644 pkg/common/node_types.go create mode 100644 tools/multi_loader/common/constants.go create mode 100644 tools/multi_loader/common/utils.go create mode 100644 tools/multi_loader/common/validators.go create mode 100644 tools/multi_loader/types/config_types.go create mode 100644 tools/multi_loader/types/node_types.go diff --git a/pkg/common/config_types.go b/pkg/common/config_types.go deleted file mode 100644 index 4cbe0f3e..00000000 --- a/pkg/common/config_types.go +++ /dev/null @@ -1,27 +0,0 @@ -package common - -type MultiLoaderConfiguration struct { - Studies []LoaderStudy `json:"Studies"` - BaseConfigPath string `json:"BaseConfigPath"` - // Optional - PreScript string `json:"PreScript"` - PostScript string `json:"PostScript"` -} - -type LoaderStudy struct { - Name string `json:"Name"` - Config map[string]interface{} `json:"Config"` - // A combination of format and values or just dir should be specified - TracesDir string `json:"TracesDir"` - - TracesFormat string `json:"TracesFormat"` - TraceValues []interface{} `json:"TraceValues"` - - // Optional - OutputDir string `json:"OutputDir"` - Verbosity string `json:"Verbosity"` - IatGeneration bool `json:"IatGeneration"` - Generated bool `json:"Generated"` - PreScript string `json:"PreScript"` - PostScript string `json:"PostScript"` -} \ No newline at end of file diff --git a/pkg/common/constants.go b/pkg/common/constants.go index d8f6e280..bf455791 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -98,21 +98,6 @@ const ( AwsTraceFuncRepositoryName = "invitro_trace_function_aws" ) -// Multi-loader Constants -const ( - TraceFormatString = "{}" -) - -// Multi-loader possible collectable metrics -const ( - Activator string = "activator" - AutoScaler string = "autoscaler" - TOP string = "top" - Prometheus string = "prometheus" -) - -var ValidCollectableMetrics = []string{Activator, AutoScaler, TOP, Prometheus} - // CPULimits const ( CPULimit1vCPU string = "1vCPU" diff --git a/pkg/common/node_types.go b/pkg/common/node_types.go deleted file mode 100644 index 05890592..00000000 --- a/pkg/common/node_types.go +++ /dev/null @@ -1,9 +0,0 @@ -package common - -type NodeGroup struct { - MasterNode string - AutoScalerNode string - ActivatorNode string - LoaderNode string - WorkerNodes []string -} \ No newline at end of file diff --git a/pkg/common/validators.go b/pkg/common/validators.go index 0ae8138a..7341b546 100644 --- a/pkg/common/validators.go +++ b/pkg/common/validators.go @@ -5,9 +5,7 @@ import ( "net" "os" "os/exec" - "path" "slices" - "strings" log "github.com/sirupsen/logrus" ) @@ -40,47 +38,6 @@ func IsValidIP(ip string) bool { return parsedIP != nil } -// Check general multi-loader configuration that applies to all platforms -func CheckMultiLoaderConfig(multiLoaderConfig MultiLoaderConfiguration) { - log.Info("Checking multi-loader configuration") - // Check if all paths are valid - CheckPath(multiLoaderConfig.BaseConfigPath) - // Check each study - if len(multiLoaderConfig.Studies) == 0 { - log.Fatal("No study found in configuration file") - } - for _, study := range multiLoaderConfig.Studies { - // Check trace directory - // if configs does not have TracePath or OutputPathPreix, either TracesDir or (TracesFormat and TraceValues) should be defined along with OutputDir - if study.TracesDir == "" && (study.TracesFormat == "" || len(study.TraceValues) == 0) { - if _, ok := study.Config["TracePath"]; !ok { - log.Fatal("Missing one of TracesDir, TracesFormat & TraceValues, Config.TracePath in multi_loader_config ", study.Name) - } - } - if study.TracesFormat != "" { - // check if trace format contains TRACE_FORMAT_STRING - if !strings.Contains(study.TracesFormat, TraceFormatString) { - log.Fatal("Invalid TracesFormat in multi_loader_config ", study.Name, ". Missing ", TraceFormatString, " in format") - } - } - if study.OutputDir == "" { - if _, ok := study.Config["OutputPathPrefix"]; !ok { - log.Warn("Missing one of OutputDir or Config.OutputPathPrefix in multi_loader_config ", study.Name) - // set default output directory - study.OutputDir = path.Join("data", "out", study.Name) - log.Warn("Setting default output directory to ", study.OutputDir) - } - } - } - log.Info("All experiments configs are valid") -} - -func CheckCollectableMetrics(metrics string) { - if !slices.Contains(ValidCollectableMetrics, metrics) { - log.Fatal("Invalid metrics ", metrics) - } -} - func CheckCPULimit(cpuLimit string) { if !slices.Contains(ValidCPULimits, cpuLimit) { log.Fatal("Invalid CPU Limit ", cpuLimit) diff --git a/pkg/config/parser.go b/pkg/config/parser.go index fb95d1c7..060849be 100644 --- a/pkg/config/parser.go +++ b/pkg/config/parser.go @@ -29,7 +29,6 @@ import ( "os" log "github.com/sirupsen/logrus" - "github.com/vhive-serverless/loader/pkg/common" ) type FailureConfiguration struct { @@ -118,18 +117,3 @@ func ReadFailureConfiguration(path string) *FailureConfiguration { return &config } - -func ReadMultiLoaderConfigurationFile(path string) common.MultiLoaderConfiguration { - byteValue, err := os.ReadFile(path) - if err != nil { - log.Fatal(err) - } - - var config common.MultiLoaderConfiguration - err = json.Unmarshal(byteValue, &config) - if err != nil { - log.Fatal(err) - } - - return config -} diff --git a/tools/multi_loader/common/constants.go b/tools/multi_loader/common/constants.go new file mode 100644 index 00000000..2645c501 --- /dev/null +++ b/tools/multi_loader/common/constants.go @@ -0,0 +1,15 @@ +package common + +const ( + TraceFormatString = "{}" +) + +// Multi-loader possible collectable metrics +const ( + Activator string = "activator" + AutoScaler string = "autoscaler" + TOP string = "top" + Prometheus string = "prometheus" +) + +var ValidCollectableMetrics = []string{Activator, AutoScaler, TOP, Prometheus} diff --git a/tools/multi_loader/common/utils.go b/tools/multi_loader/common/utils.go new file mode 100644 index 00000000..a97ea49a --- /dev/null +++ b/tools/multi_loader/common/utils.go @@ -0,0 +1,40 @@ +package common + +import ( + "encoding/json" + "os" + + log "github.com/sirupsen/logrus" + + "github.com/vhive-serverless/loader/pkg/config" + "github.com/vhive-serverless/loader/tools/multi_loader/types" +) + +func ReadMultiLoaderConfigurationFile(path string) types.MultiLoaderConfiguration { + byteValue, err := os.ReadFile(path) + if err != nil { + log.Fatal(err) + } + + var config types.MultiLoaderConfiguration + err = json.Unmarshal(byteValue, &config) + if err != nil { + log.Fatal(err) + } + + return config +} + +func DeterminePlatformFromConfig(multiLoaderConfig types.MultiLoaderConfiguration) string { + // Determine platform + baseConfigByteValue, err := os.ReadFile(multiLoaderConfig.BaseConfigPath) + if err != nil { + log.Fatal(err) + } + var loaderConfig config.LoaderConfiguration + // Unmarshal base configuration + if err = json.Unmarshal(baseConfigByteValue, &loaderConfig); err != nil { + log.Fatal(err) + } + return loaderConfig.Platform +} diff --git a/tools/multi_loader/common/validators.go b/tools/multi_loader/common/validators.go new file mode 100644 index 00000000..1b818b91 --- /dev/null +++ b/tools/multi_loader/common/validators.go @@ -0,0 +1,52 @@ +package common + +import ( + "path" + "slices" + "strings" + + log "github.com/sirupsen/logrus" + "github.com/vhive-serverless/loader/pkg/common" + "github.com/vhive-serverless/loader/tools/multi_loader/types" +) + +// Check general multi-loader configuration that applies to all platforms +func CheckMultiLoaderConfig(multiLoaderConfig types.MultiLoaderConfiguration) { + log.Info("Checking multi-loader configuration") + // Check if all paths are valid + common.CheckPath(multiLoaderConfig.BaseConfigPath) + // Check each study + if len(multiLoaderConfig.Studies) == 0 { + log.Fatal("No study found in configuration file") + } + for _, study := range multiLoaderConfig.Studies { + // Check trace directory + // if configs does not have TracePath or OutputPathPreix, either TracesDir or (TracesFormat and TraceValues) should be defined along with OutputDir + if study.TracesDir == "" && (study.TracesFormat == "" || len(study.TraceValues) == 0) { + if _, ok := study.Config["TracePath"]; !ok { + log.Fatal("Missing one of TracesDir, TracesFormat & TraceValues, Config.TracePath in multi_loader_config ", study.Name) + } + } + if study.TracesFormat != "" { + // check if trace format contains TRACE_FORMAT_STRING + if !strings.Contains(study.TracesFormat, TraceFormatString) { + log.Fatal("Invalid TracesFormat in multi_loader_config ", study.Name, ". Missing ", TraceFormatString, " in format") + } + } + if study.OutputDir == "" { + if _, ok := study.Config["OutputPathPrefix"]; !ok { + log.Warn("Missing one of OutputDir or Config.OutputPathPrefix in multi_loader_config ", study.Name) + // set default output directory + study.OutputDir = path.Join("data", "out", study.Name) + log.Warn("Setting default output directory to ", study.OutputDir) + } + } + } + log.Info("All experiments configs are valid") +} + +func CheckCollectableMetrics(metrics string) { + if !slices.Contains(ValidCollectableMetrics, metrics) { + log.Fatal("Invalid metrics ", metrics) + } +} diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index 0db66c5f..a183100d 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -14,6 +14,8 @@ import ( "github.com/vhive-serverless/loader/pkg/common" "github.com/vhive-serverless/loader/pkg/config" + ml_common "github.com/vhive-serverless/loader/tools/multi_loader/common" + "github.com/vhive-serverless/loader/tools/multi_loader/types" log "github.com/sirupsen/logrus" ) @@ -26,8 +28,8 @@ const ( ) type MultiLoaderRunner struct { - MultiLoaderConfig common.MultiLoaderConfiguration - NodeGroup common.NodeGroup + MultiLoaderConfig types.MultiLoaderConfiguration + NodeGroup types.NodeGroup DryRunSuccess bool Verbosity string IatGeneration bool @@ -38,13 +40,13 @@ type MultiLoaderRunner struct { // init multi loader runner func NewMultiLoaderRunner(configPath string, verbosity string, iatGeneration bool, generated bool) (*MultiLoaderRunner, error) { - multiLoaderConfig := config.ReadMultiLoaderConfigurationFile(configPath) + multiLoaderConfig := ml_common.ReadMultiLoaderConfigurationFile(configPath) // validate configuration - common.CheckMultiLoaderConfig(multiLoaderConfig) + ml_common.CheckMultiLoaderConfig(multiLoaderConfig) // determine platform - platform := determinePlatform(multiLoaderConfig) + platform := ml_common.DeterminePlatformFromConfig(multiLoaderConfig) runner := MultiLoaderRunner{ MultiLoaderConfig: multiLoaderConfig, @@ -59,20 +61,6 @@ func NewMultiLoaderRunner(configPath string, verbosity string, iatGeneration boo return &runner, nil } -func determinePlatform(multiLoaderConfig common.MultiLoaderConfiguration) string { - // Determine platform - baseConfigByteValue, err := os.ReadFile(multiLoaderConfig.BaseConfigPath) - if err != nil { - log.Fatal(err) - } - var loaderConfig config.LoaderConfiguration - // Unmarshal base configuration - if err = json.Unmarshal(baseConfigByteValue, &loaderConfig); err != nil { - log.Fatal(err) - } - return loaderConfig.Platform -} - func (d *MultiLoaderRunner) RunDryRun() { log.Info("Running dry run") d.DryRun = true @@ -131,9 +119,9 @@ func (d *MultiLoaderRunner) run() { * but first by duplicating the study to multiple studies with different values * in the config field. Those values will override the base loader config later */ -func (d *MultiLoaderRunner) unpackStudy(experiment common.LoaderStudy) []common.LoaderStudy { +func (d *MultiLoaderRunner) unpackStudy(experiment types.LoaderStudy) []types.LoaderStudy { log.Info("Unpacking experiment ", experiment.Name) - var experiments []common.LoaderStudy + var experiments []types.LoaderStudy // if user specified a trace directory if experiment.TracesDir != "" { @@ -149,8 +137,8 @@ func (d *MultiLoaderRunner) unpackStudy(experiment common.LoaderStudy) []common. return experiments } -func (d *MultiLoaderRunner) unpackFromTraceDir(study common.LoaderStudy) []common.LoaderStudy { - var experiments []common.LoaderStudy +func (d *MultiLoaderRunner) unpackFromTraceDir(study types.LoaderStudy) []types.LoaderStudy { + var experiments []types.LoaderStudy files, err := os.ReadDir(study.TracesDir) if err != nil { log.Fatal(err) @@ -165,10 +153,10 @@ func (d *MultiLoaderRunner) unpackFromTraceDir(study common.LoaderStudy) []commo return experiments } -func (d *MultiLoaderRunner) unpackFromTraceValues(study common.LoaderStudy) []common.LoaderStudy { - var experiments []common.LoaderStudy +func (d *MultiLoaderRunner) unpackFromTraceValues(study types.LoaderStudy) []types.LoaderStudy { + var experiments []types.LoaderStudy for _, traceValue := range study.TraceValues { - tracePath := strings.Replace(study.TracesFormat, common.TraceFormatString, fmt.Sprintf("%v", traceValue), -1) + tracePath := strings.Replace(study.TracesFormat, ml_common.TraceFormatString, fmt.Sprintf("%v", traceValue), -1) fileName := path.Base(tracePath) newExperiment := d.duplicateStudy(study, fileName) newExperiment.Config["TracePath"] = tracePath @@ -178,8 +166,8 @@ func (d *MultiLoaderRunner) unpackFromTraceValues(study common.LoaderStudy) []co return experiments } -func (d *MultiLoaderRunner) unpackSingleExperiment(study common.LoaderStudy) []common.LoaderStudy { - var experiments []common.LoaderStudy +func (d *MultiLoaderRunner) unpackSingleExperiment(study types.LoaderStudy) []types.LoaderStudy { + var experiments []types.LoaderStudy pathDir := "" if study.Config["OutputPathPrefix"] != nil { pathDir = path.Dir(study.Config["OutputPathPrefix"].(string)) @@ -195,7 +183,7 @@ func (d *MultiLoaderRunner) unpackSingleExperiment(study common.LoaderStudy) []c /** * Creates a deepcopy of a given study and updates relevant fields to utilise the provided new filename */ -func (d *MultiLoaderRunner) duplicateStudy(study common.LoaderStudy, newFileName string) common.LoaderStudy { +func (d *MultiLoaderRunner) duplicateStudy(study types.LoaderStudy, newFileName string) types.LoaderStudy { newStudy, err := common.DeepCopy(study) if err != nil { log.Fatal(err) @@ -216,7 +204,7 @@ func (d *MultiLoaderRunner) duplicateStudy(study common.LoaderStudy, newFileName return newStudy } -func (d *MultiLoaderRunner) addCommandFlags(study common.LoaderStudy) { +func (d *MultiLoaderRunner) addCommandFlags(study types.LoaderStudy) { // Add flags to experiment config if study.Verbosity == "" { study.Verbosity = d.Verbosity @@ -232,7 +220,7 @@ func (d *MultiLoaderRunner) addCommandFlags(study common.LoaderStudy) { /** * Prepare experiment by merging with base config, creating output directory and writing experiment config to temp file */ -func (d *MultiLoaderRunner) prepareExperiment(experiment common.LoaderStudy) { +func (d *MultiLoaderRunner) prepareExperiment(experiment types.LoaderStudy) { log.Info("Preparing ", experiment.Name) // Merge base configs with experiment configs experimentConfig := d.mergeConfigurations(d.MultiLoaderConfig.BaseConfigPath, experiment) @@ -250,7 +238,7 @@ func (d *MultiLoaderRunner) prepareExperiment(experiment common.LoaderStudy) { /** * Merge base configs with partial loader configs */ -func (d *MultiLoaderRunner) mergeConfigurations(baseConfigPath string, experiment common.LoaderStudy) config.LoaderConfiguration { +func (d *MultiLoaderRunner) mergeConfigurations(baseConfigPath string, experiment types.LoaderStudy) config.LoaderConfiguration { // Read base configuration baseConfigByteValue, err := os.ReadFile(baseConfigPath) if err != nil { @@ -284,7 +272,7 @@ func (d *MultiLoaderRunner) writeExperimentConfigToTempFile(experimentConfig con } } -func (d *MultiLoaderRunner) runExperiment(experiment common.LoaderStudy) error { +func (d *MultiLoaderRunner) runExperiment(experiment types.LoaderStudy) error { log.Info("Running ", experiment.Name) log.Debug("Experiment configuration ", experiment.Config) @@ -322,7 +310,7 @@ func (d *MultiLoaderRunner) runExperiment(experiment common.LoaderStudy) error { return nil } -func (d *MultiLoaderRunner) executeLoaderCommand(experiment common.LoaderStudy, logFile *os.File) error { +func (d *MultiLoaderRunner) executeLoaderCommand(experiment types.LoaderStudy, logFile *os.File) error { cmd := exec.Command("go", "run", LOADER_PATH, "--config="+EXPERIMENT_TEMP_CONFIG_PATH, "--verbosity="+experiment.Verbosity, diff --git a/tools/multi_loader/runner/multi_loader_runner_test.go b/tools/multi_loader/runner/multi_loader_runner_test.go index f4ffa903..a914eafe 100644 --- a/tools/multi_loader/runner/multi_loader_runner_test.go +++ b/tools/multi_loader/runner/multi_loader_runner_test.go @@ -8,7 +8,8 @@ import ( log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - "github.com/vhive-serverless/loader/pkg/common" + ml_common "github.com/vhive-serverless/loader/tools/multi_loader/common" + "github.com/vhive-serverless/loader/tools/multi_loader/types" ) var ( @@ -26,7 +27,7 @@ func init() { func TestUnpackExperiment(t *testing.T) { // helper func to validate unpacked experiments - validateUnpackedExperiment := func(t *testing.T, experimentConfig []common.LoaderStudy, studyConfig common.LoaderStudy, expectedNames []string, expectedOutputPrefixes []string) { + validateUnpackedExperiment := func(t *testing.T, experimentConfig []types.LoaderStudy, studyConfig types.LoaderStudy, expectedNames []string, expectedOutputPrefixes []string) { if len(experimentConfig) != len(expectedNames) { t.Errorf("Expected %d sub-experiments, got %d", len(expectedNames), len(experimentConfig)) return @@ -115,7 +116,7 @@ func TestPrepareExperiment(t *testing.T) { t.Fatalf("Failed to create multi-loader driver: %v", err) } - subExperiment := common.LoaderStudy{ + subExperiment := types.LoaderStudy{ Name: "example_1", Config: map[string]interface{}{ "ExperimentDuration": 10, @@ -155,7 +156,7 @@ func TestMergeConfig(t *testing.T) { if err != nil { t.Fatalf("Failed to create multi-loader driver: %v", err) } - experiment := common.LoaderStudy{ + experiment := types.LoaderStudy{ Name: "example_1", Config: map[string]interface{}{ "ExperimentDuration": 10, @@ -184,14 +185,14 @@ func TestMultiConfigValidator(t *testing.T) { } t.Run("CheckMultiLoaderConfig (Success)", func(t *testing.T) { // Check if all paths are valid - common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) + ml_common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) }) t.Run("CheckMultiLoaderConfig (Failure: No Study)", func(t *testing.T) { expectFatal(t, func() { temp := multiLoader.MultiLoaderConfig.Studies multiLoader.MultiLoaderConfig.Studies = nil - common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) + ml_common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) multiLoader.MultiLoaderConfig.Studies = temp }) }) @@ -201,14 +202,14 @@ func TestMultiConfigValidator(t *testing.T) { multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "" multiLoader.MultiLoaderConfig.Studies[0].TracesFormat = "" multiLoader.MultiLoaderConfig.Studies[0].TraceValues = nil - common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) + ml_common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) }) }) t.Run("CheckMultiLoaderConfig (Failure: Invalid TracesFormat)", func(t *testing.T) { expectFatal(t, func() { multiLoader.MultiLoaderConfig.Studies[0].TracesFormat = "invalid_format" - common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) + ml_common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) }) }) @@ -217,7 +218,7 @@ func TestMultiConfigValidator(t *testing.T) { multiLoader.MultiLoaderConfig.Studies[0].TraceValues = nil multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "" multiLoader.MultiLoaderConfig.Studies[0].TracesFormat = "example_{}_test" - common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) + ml_common.CheckMultiLoaderConfig(multiLoader.MultiLoaderConfig) }) }) } diff --git a/tools/multi_loader/types/config_types.go b/tools/multi_loader/types/config_types.go new file mode 100644 index 00000000..7c9ce36a --- /dev/null +++ b/tools/multi_loader/types/config_types.go @@ -0,0 +1,27 @@ +package types + +type MultiLoaderConfiguration struct { + Studies []LoaderStudy `json:"Studies"` + BaseConfigPath string `json:"BaseConfigPath"` + // Optional + PreScript string `json:"PreScript"` + PostScript string `json:"PostScript"` +} + +type LoaderStudy struct { + Name string `json:"Name"` + Config map[string]interface{} `json:"Config"` + // A combination of format and values or just dir should be specified + TracesDir string `json:"TracesDir"` + + TracesFormat string `json:"TracesFormat"` + TraceValues []interface{} `json:"TraceValues"` + + // Optional + OutputDir string `json:"OutputDir"` + Verbosity string `json:"Verbosity"` + IatGeneration bool `json:"IatGeneration"` + Generated bool `json:"Generated"` + PreScript string `json:"PreScript"` + PostScript string `json:"PostScript"` +} diff --git a/tools/multi_loader/types/node_types.go b/tools/multi_loader/types/node_types.go new file mode 100644 index 00000000..96b11bfb --- /dev/null +++ b/tools/multi_loader/types/node_types.go @@ -0,0 +1,9 @@ +package types + +type NodeGroup struct { + MasterNode string + AutoScalerNode string + ActivatorNode string + LoaderNode string + WorkerNodes []string +} From 0876e40f929d2ee028d1038ea09df45e1cd9e1a5 Mon Sep 17 00:00:00 2001 From: Lenson Date: Tue, 31 Dec 2024 10:20:07 +0800 Subject: [PATCH 31/37] rename multiloader driver to runner Signed-off-by: Lenson --- tools/multi_loader/multi_loader.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/multi_loader/multi_loader.go b/tools/multi_loader/multi_loader.go index c9ba571e..decf0a7c 100644 --- a/tools/multi_loader/multi_loader.go +++ b/tools/multi_loader/multi_loader.go @@ -40,22 +40,21 @@ func initLogger() { func main() { log.Info("Starting multiloader") - // Create multi loader driver - multiLoaderDriver, err := runner.NewMultiLoaderRunner(*multiLoaderConfigPath, *verbosity, *iatGeneration, *generated) + // Create multi loader runner + multiLoaderRunner, err := runner.NewMultiLoaderRunner(*multiLoaderConfigPath, *verbosity, *iatGeneration, *generated) if err != nil { log.Fatalf("Failed to create multi loader driver: %v", err) } // Dry run - multiLoaderDriver.RunDryRun() + multiLoaderRunner.RunDryRun() // Check if dry run was successful - if !multiLoaderDriver.DryRunSuccess { + if !multiLoaderRunner.DryRunSuccess { log.Fatal("Dry run failed. Exiting...") } // Actual run - log.Info("Running experiments") - multiLoaderDriver.RunActual() + multiLoaderRunner.RunActual() // Finish log.Info("All experiments completed") From 038f2c314f1245f5e700ddcc30933f46d27e3d0e Mon Sep 17 00:00:00 2001 From: Lenson Date: Tue, 31 Dec 2024 10:32:30 +0800 Subject: [PATCH 32/37] update logs Signed-off-by: Lenson update log verbosity Signed-off-by: Lenson update logs Signed-off-by: Lenson update logs Signed-off-by: Lenson --- pkg/common/utilities.go | 4 +- tools/multi_loader/common/validators.go | 4 +- tools/multi_loader/multi_loader.go | 2 +- .../runner/multi_loader_runner.go | 50 +++++++++++-------- 4 files changed, 35 insertions(+), 25 deletions(-) diff --git a/pkg/common/utilities.go b/pkg/common/utilities.go index a841ece5..27e6b039 100644 --- a/pkg/common/utilities.go +++ b/pkg/common/utilities.go @@ -166,12 +166,12 @@ func RunScript(command string) { if command == "" { return } - logger.Info("Running command ", command) + logger.Debug("Running command ", command) cmd, err := exec.Command("/bin/sh", command).Output() if err != nil { log.Fatal(err) } - logger.Info(string(cmd)) + logger.Debug(string(cmd)) } func ParseLogType(logString string) string { diff --git a/tools/multi_loader/common/validators.go b/tools/multi_loader/common/validators.go index 1b818b91..26fff0e8 100644 --- a/tools/multi_loader/common/validators.go +++ b/tools/multi_loader/common/validators.go @@ -12,7 +12,7 @@ import ( // Check general multi-loader configuration that applies to all platforms func CheckMultiLoaderConfig(multiLoaderConfig types.MultiLoaderConfiguration) { - log.Info("Checking multi-loader configuration") + log.Debug("Checking multi-loader configuration") // Check if all paths are valid common.CheckPath(multiLoaderConfig.BaseConfigPath) // Check each study @@ -42,7 +42,7 @@ func CheckMultiLoaderConfig(multiLoaderConfig types.MultiLoaderConfiguration) { } } } - log.Info("All experiments configs are valid") + log.Debug("All experiments configs are valid") } func CheckCollectableMetrics(metrics string) { diff --git a/tools/multi_loader/multi_loader.go b/tools/multi_loader/multi_loader.go index decf0a7c..c9e5930b 100644 --- a/tools/multi_loader/multi_loader.go +++ b/tools/multi_loader/multi_loader.go @@ -57,5 +57,5 @@ func main() { multiLoaderRunner.RunActual() // Finish - log.Info("All experiments completed") + log.Info("All studies completed") } diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index a183100d..264b01ca 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -62,7 +62,7 @@ func NewMultiLoaderRunner(configPath string, verbosity string, iatGeneration boo } func (d *MultiLoaderRunner) RunDryRun() { - log.Info("Running dry run") + log.Info("Running dry runs") d.DryRun = true d.run() } @@ -78,7 +78,7 @@ func (d *MultiLoaderRunner) run() { common.RunScript(d.MultiLoaderConfig.PreScript) // Iterate over studies and run them for _, study := range d.MultiLoaderConfig.Studies { - log.Info("Setting up experiment: ", study.Name) + log.Debug("Setting up study: ", study.Name) // Run pre script common.RunScript(study.PreScript) @@ -87,9 +87,7 @@ func (d *MultiLoaderRunner) run() { // Iterate over sparse experiments, prepare and run for _, experiment := range sparseExperiments { - if d.DryRun { - log.Info("Dry Running: ", experiment.Name) - } + // Prepare experiment: merge with base config, create output dir and write merged config to temp file d.prepareExperiment(experiment) @@ -100,7 +98,7 @@ func (d *MultiLoaderRunner) run() { // Check if should continue this study if err != nil { - log.Info("Experiment failed: ", experiment.Name, ". Skipping remaining experiments in study...") + log.Error("Experiment failed: ", experiment.Name, ". Skipping remaining experiments in study...") break } } @@ -119,19 +117,19 @@ func (d *MultiLoaderRunner) run() { * but first by duplicating the study to multiple studies with different values * in the config field. Those values will override the base loader config later */ -func (d *MultiLoaderRunner) unpackStudy(experiment types.LoaderStudy) []types.LoaderStudy { - log.Info("Unpacking experiment ", experiment.Name) +func (d *MultiLoaderRunner) unpackStudy(study types.LoaderStudy) []types.LoaderStudy { + log.Debug("Unpacking study ", study.Name) var experiments []types.LoaderStudy // if user specified a trace directory - if experiment.TracesDir != "" { - experiments = d.unpackFromTraceDir(experiment) + if study.TracesDir != "" { + experiments = d.unpackFromTraceDir(study) // user define trace format and values instead of directory - } else if experiment.TracesFormat != "" && len(experiment.TraceValues) > 0 { - experiments = d.unpackFromTraceValues(experiment) + } else if study.TracesFormat != "" && len(study.TraceValues) > 0 { + experiments = d.unpackFromTraceValues(study) } else { // Theres only one experiment in the study - experiments = d.unpackSingleExperiment(experiment) + experiments = d.unpackSingleExperiment(study) } return experiments @@ -205,7 +203,7 @@ func (d *MultiLoaderRunner) duplicateStudy(study types.LoaderStudy, newFileName } func (d *MultiLoaderRunner) addCommandFlags(study types.LoaderStudy) { - // Add flags to experiment config + // Add flags to study config if study.Verbosity == "" { study.Verbosity = d.Verbosity } @@ -221,7 +219,7 @@ func (d *MultiLoaderRunner) addCommandFlags(study types.LoaderStudy) { * Prepare experiment by merging with base config, creating output directory and writing experiment config to temp file */ func (d *MultiLoaderRunner) prepareExperiment(experiment types.LoaderStudy) { - log.Info("Preparing ", experiment.Name) + log.Debug("Preparing ", experiment.Name) // Merge base configs with experiment configs experimentConfig := d.mergeConfigurations(d.MultiLoaderConfig.BaseConfigPath, experiment) @@ -273,7 +271,11 @@ func (d *MultiLoaderRunner) writeExperimentConfigToTempFile(experimentConfig con } func (d *MultiLoaderRunner) runExperiment(experiment types.LoaderStudy) error { - log.Info("Running ", experiment.Name) + if d.DryRun { + log.Info("Dry running ", experiment.Name) + } else { + log.Info("Running ", experiment.Name) + } log.Debug("Experiment configuration ", experiment.Config) // Create the log file @@ -306,7 +308,7 @@ func (d *MultiLoaderRunner) runExperiment(experiment types.LoaderStudy) error { break } } - log.Info("Completed ", experiment.Name) + log.Debug("Completed ", experiment.Name) return nil } @@ -368,17 +370,25 @@ func (d *MultiLoaderRunner) logLoaderStdError(stdPipe io.ReadCloser, logFile *os if m == "" { continue } + + // if its go downloading logs, output to debug + if strings.Contains(m, "go: downloading") { + log.Debug(m) + continue + } + log.Error(m) } } func (d *MultiLoaderRunner) performCleanup() { - log.Info("Runnning Cleanup") + log.Debug("Runnning Cleanup") // Run make clean if err := exec.Command("make", "clean").Run(); err != nil { - log.Error(err) + log.Error("Error occured while running cleanup", err) } - log.Info("Cleanup completed") // Remove temp file os.Remove(EXPERIMENT_TEMP_CONFIG_PATH) + + log.Debug("Cleanup completed") } From 9648622c12a40c8b0bb4a89356b4bf717be3ffb6 Mon Sep 17 00:00:00 2001 From: Lenson Date: Tue, 31 Dec 2024 11:21:05 +0800 Subject: [PATCH 33/37] add loader experiment Signed-off-by: Lenson --- .../runner/multi_loader_runner.go | 82 +++++++++++-------- tools/multi_loader/types/config_types.go | 11 +++ 2 files changed, 59 insertions(+), 34 deletions(-) diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index 264b01ca..86145904 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -117,9 +117,9 @@ func (d *MultiLoaderRunner) run() { * but first by duplicating the study to multiple studies with different values * in the config field. Those values will override the base loader config later */ -func (d *MultiLoaderRunner) unpackStudy(study types.LoaderStudy) []types.LoaderStudy { +func (d *MultiLoaderRunner) unpackStudy(study types.LoaderStudy) []types.LoaderExperiment { log.Debug("Unpacking study ", study.Name) - var experiments []types.LoaderStudy + var experiments []types.LoaderExperiment // if user specified a trace directory if study.TracesDir != "" { @@ -135,37 +135,34 @@ func (d *MultiLoaderRunner) unpackStudy(study types.LoaderStudy) []types.LoaderS return experiments } -func (d *MultiLoaderRunner) unpackFromTraceDir(study types.LoaderStudy) []types.LoaderStudy { - var experiments []types.LoaderStudy +func (d *MultiLoaderRunner) unpackFromTraceDir(study types.LoaderStudy) []types.LoaderExperiment { + var experiments []types.LoaderExperiment files, err := os.ReadDir(study.TracesDir) if err != nil { log.Fatal(err) } for _, file := range files { - newExperiment := d.duplicateStudy(study, file.Name()) - newExperiment.Config["TracePath"] = path.Join(study.TracesDir, file.Name()) - newExperiment.Name += "_" + file.Name() + tracePath := path.Join(study.TracesDir, file.Name()) + newExperiment := d.createExperimentFromStudy(study, file.Name(), tracePath) experiments = append(experiments, newExperiment) } return experiments } -func (d *MultiLoaderRunner) unpackFromTraceValues(study types.LoaderStudy) []types.LoaderStudy { - var experiments []types.LoaderStudy +func (d *MultiLoaderRunner) unpackFromTraceValues(study types.LoaderStudy) []types.LoaderExperiment { + var experiments []types.LoaderExperiment for _, traceValue := range study.TraceValues { tracePath := strings.Replace(study.TracesFormat, ml_common.TraceFormatString, fmt.Sprintf("%v", traceValue), -1) fileName := path.Base(tracePath) - newExperiment := d.duplicateStudy(study, fileName) - newExperiment.Config["TracePath"] = tracePath - newExperiment.Name += "_" + fileName + newExperiment := d.createExperimentFromStudy(study, fileName, tracePath) experiments = append(experiments, newExperiment) } return experiments } -func (d *MultiLoaderRunner) unpackSingleExperiment(study types.LoaderStudy) []types.LoaderStudy { - var experiments []types.LoaderStudy +func (d *MultiLoaderRunner) unpackSingleExperiment(study types.LoaderStudy) []types.LoaderExperiment { + var experiments []types.LoaderExperiment pathDir := "" if study.Config["OutputPathPrefix"] != nil { pathDir = path.Dir(study.Config["OutputPathPrefix"].(string)) @@ -173,52 +170,69 @@ func (d *MultiLoaderRunner) unpackSingleExperiment(study types.LoaderStudy) []ty pathDir = study.OutputDir } study.OutputDir = pathDir - newExperiment := d.duplicateStudy(study, study.Name) + newExperiment := d.createExperimentFromStudy(study, study.Name, "") experiments = append(experiments, newExperiment) return experiments } /** -* Creates a deepcopy of a given study and updates relevant fields to utilise the provided new filename +* Creates a LoaderExperiment form a given study and updates relevant expereiment fields */ -func (d *MultiLoaderRunner) duplicateStudy(study types.LoaderStudy, newFileName string) types.LoaderStudy { - newStudy, err := common.DeepCopy(study) +func (d *MultiLoaderRunner) createExperimentFromStudy(study types.LoaderStudy, experimentName string, tracePath string) types.LoaderExperiment { + experiment := types.LoaderExperiment{ + Name: study.Name + "_" + experimentName, + OutputDir: study.OutputDir, + PreScript: study.PreScript, + PostScript: study.PostScript, + } + + // Deep copy of study config for new experiment + studyConfig, err := common.DeepCopy(study.Config) if err != nil { log.Fatal(err) } + // Update config to duplicated study configs + experiment.Config = studyConfig + // Update OutputPathPrefix dryRunAdditionalPath := "" if d.DryRun { dryRunAdditionalPath = "dry_run" } - newStudy.Config["OutputPathPrefix"] = path.Join( + studyConfig["OutputPathPrefix"] = path.Join( study.OutputDir, study.Name, dryRunAdditionalPath, - time.Now().Format(TIME_FORMAT)+"_"+newFileName, - newFileName, + time.Now().Format(TIME_FORMAT)+"_"+experimentName, + experimentName, ) - d.addCommandFlags(newStudy) - return newStudy + // Add loader command flags + d.addCommandFlagsToExperiment(experiment) + + // Update trace path + if tracePath != "" { + studyConfig["TracePath"] = tracePath + } + return experiment } -func (d *MultiLoaderRunner) addCommandFlags(study types.LoaderStudy) { +func (d *MultiLoaderRunner) addCommandFlagsToExperiment(experiment types.LoaderExperiment) { // Add flags to study config - if study.Verbosity == "" { - study.Verbosity = d.Verbosity + if experiment.Verbosity == "" { + experiment.Verbosity = d.Verbosity } - if !study.IatGeneration { - study.IatGeneration = d.IatGeneration + if !experiment.IatGeneration { + experiment.IatGeneration = d.IatGeneration } - if !study.Generated { - study.Generated = d.Generated + if !experiment.Generated { + experiment.Generated = d.Generated } } /** * Prepare experiment by merging with base config, creating output directory and writing experiment config to temp file */ -func (d *MultiLoaderRunner) prepareExperiment(experiment types.LoaderStudy) { +func (d *MultiLoaderRunner) prepareExperiment(experiment types.LoaderExperiment) { log.Debug("Preparing ", experiment.Name) // Merge base configs with experiment configs experimentConfig := d.mergeConfigurations(d.MultiLoaderConfig.BaseConfigPath, experiment) @@ -236,7 +250,7 @@ func (d *MultiLoaderRunner) prepareExperiment(experiment types.LoaderStudy) { /** * Merge base configs with partial loader configs */ -func (d *MultiLoaderRunner) mergeConfigurations(baseConfigPath string, experiment types.LoaderStudy) config.LoaderConfiguration { +func (d *MultiLoaderRunner) mergeConfigurations(baseConfigPath string, experiment types.LoaderExperiment) config.LoaderConfiguration { // Read base configuration baseConfigByteValue, err := os.ReadFile(baseConfigPath) if err != nil { @@ -270,7 +284,7 @@ func (d *MultiLoaderRunner) writeExperimentConfigToTempFile(experimentConfig con } } -func (d *MultiLoaderRunner) runExperiment(experiment types.LoaderStudy) error { +func (d *MultiLoaderRunner) runExperiment(experiment types.LoaderExperiment) error { if d.DryRun { log.Info("Dry running ", experiment.Name) } else { @@ -312,7 +326,7 @@ func (d *MultiLoaderRunner) runExperiment(experiment types.LoaderStudy) error { return nil } -func (d *MultiLoaderRunner) executeLoaderCommand(experiment types.LoaderStudy, logFile *os.File) error { +func (d *MultiLoaderRunner) executeLoaderCommand(experiment types.LoaderExperiment, logFile *os.File) error { cmd := exec.Command("go", "run", LOADER_PATH, "--config="+EXPERIMENT_TEMP_CONFIG_PATH, "--verbosity="+experiment.Verbosity, diff --git a/tools/multi_loader/types/config_types.go b/tools/multi_loader/types/config_types.go index 7c9ce36a..21d53996 100644 --- a/tools/multi_loader/types/config_types.go +++ b/tools/multi_loader/types/config_types.go @@ -25,3 +25,14 @@ type LoaderStudy struct { PreScript string `json:"PreScript"` PostScript string `json:"PostScript"` } + +type LoaderExperiment struct { + Name string `json:"Name"` + Config map[string]interface{} `json:"Config"` + OutputDir string `json:"OutputDir"` + Verbosity string `json:"Verbosity"` + IatGeneration bool `json:"IatGeneration"` + Generated bool `json:"Generated"` + PreScript string `json:"PreScript"` + PostScript string `json:"PostScript"` +} From aa3b2b8d96aefdbb8d2dd13ebe8245edfaa6bcd1 Mon Sep 17 00:00:00 2001 From: Lenson Date: Mon, 30 Dec 2024 22:30:38 +0800 Subject: [PATCH 34/37] add multi-loader tests Signed-off-by: Lenson update test Signed-off-by: Lenson refactor multi-loader tests Signed-off-by: Lenson --- .github/actions/ci_knative_setup/action.yml | 45 +++++++++++ .github/workflows/e2e.yaml | 79 ------------------- .github/workflows/e2e_loader.yaml | 57 +++++++++++++ .github/workflows/e2e_multi_loader.yaml | 69 ++++++++++++++++ .github/workflows/tools-tests.yaml | 2 +- tools/.gitignore | 5 +- tools/multi_loader/common/utils.go | 12 +++ tools/multi_loader/multi_loader.go | 2 +- tools/multi_loader/multi_loader_config.json | 8 +- .../multi_loader_config_test.json | 1 + .../runner/multi_loader_runner_test.go | 35 ++++---- .../test_configs/test_base_loader_config.json | 23 ------ .../test_multi_loader_config.json | 20 ----- .../test_data/example_1_test/dirigent.csv | 5 ++ .../test_data/example_1_test/durations.csv | 5 ++ .../test_data/example_1_test/invocations.csv | 5 ++ .../test_data/example_1_test/memory.csv | 5 ++ .../test_data/example_2_test/dirigent.csv | 5 ++ .../test_data/example_2_test/durations.csv | 5 ++ .../test_data/example_2_test/invocations.csv | 5 ++ .../test_data/example_2_test/memory.csv | 5 ++ .../test_data/example_3.1_test/dirigent.csv | 5 ++ .../test_data/example_3.1_test/durations.csv | 5 ++ .../example_3.1_test/invocations.csv | 5 ++ .../test_data/example_3.1_test/memory.csv | 5 ++ 25 files changed, 276 insertions(+), 142 deletions(-) create mode 100644 .github/actions/ci_knative_setup/action.yml delete mode 100644 .github/workflows/e2e.yaml create mode 100644 .github/workflows/e2e_loader.yaml create mode 100644 .github/workflows/e2e_multi_loader.yaml create mode 100644 tools/multi_loader/multi_loader_config_test.json delete mode 100644 tools/multi_loader/runner/test_configs/test_base_loader_config.json delete mode 100644 tools/multi_loader/runner/test_configs/test_multi_loader_config.json create mode 100644 tools/multi_loader/test_data/example_1_test/dirigent.csv create mode 100644 tools/multi_loader/test_data/example_1_test/durations.csv create mode 100644 tools/multi_loader/test_data/example_1_test/invocations.csv create mode 100644 tools/multi_loader/test_data/example_1_test/memory.csv create mode 100644 tools/multi_loader/test_data/example_2_test/dirigent.csv create mode 100644 tools/multi_loader/test_data/example_2_test/durations.csv create mode 100644 tools/multi_loader/test_data/example_2_test/invocations.csv create mode 100644 tools/multi_loader/test_data/example_2_test/memory.csv create mode 100644 tools/multi_loader/test_data/example_3.1_test/dirigent.csv create mode 100644 tools/multi_loader/test_data/example_3.1_test/durations.csv create mode 100644 tools/multi_loader/test_data/example_3.1_test/invocations.csv create mode 100644 tools/multi_loader/test_data/example_3.1_test/memory.csv diff --git a/.github/actions/ci_knative_setup/action.yml b/.github/actions/ci_knative_setup/action.yml new file mode 100644 index 00000000..41491a6f --- /dev/null +++ b/.github/actions/ci_knative_setup/action.yml @@ -0,0 +1,45 @@ +name: CI Knative Setup +description: Sets up the CI environment with Kind, Serving, and Kourier for Knative platform type +inputs: + go-version: + description: Version of Go to set up + required: true + default: '1.22' +runs: + using: "composite" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + lfs: "true" + + - name: Checkout LFS objects + shell: bash + run: git lfs checkout + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: Create k8s Kind Cluster + shell: bash + run: bash ./scripts/konk-ci/01-kind.sh + + - name: Install Serving + shell: bash + run: bash ./scripts/konk-ci/02-serving.sh + + - name: Install Kourier + shell: bash + run: bash ./scripts/konk-ci/02-kourier.sh + + - name: Setup domain and autoscaler + shell: bash + run: | + INGRESS_HOST="127.0.0.1" + KNATIVE_DOMAIN=$INGRESS_HOST.sslip.io + kubectl patch configmap -n knative-serving config-domain -p "{\"data\": {\"$KNATIVE_DOMAIN\": \"\"}}" + kubectl patch configmap -n knative-serving config-autoscaler -p "{\"data\": {\"allow-zero-initial-scale\": \"true\"}}" + kubectl patch configmap -n knative-serving config-features -p "{\"data\": {\"kubernetes.podspec-affinity\": \"enabled\"}}" + kubectl label node knative-control-plane loader-nodetype=worker diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml deleted file mode 100644 index 94169c6e..00000000 --- a/.github/workflows/e2e.yaml +++ /dev/null @@ -1,79 +0,0 @@ -name: End-to-End Tests - -on: - schedule: - - cron: "0 9 * * 1" - workflow_dispatch: - push: - branches: [main] - pull_request: - branches: [main] - -env: - GOOS: linux - GO111MODULE: on - -jobs: - test-knative: - name: Test Knative Deployment - env: - KIND_VERSION: v0.22.0 - K8S_VERSION: v1.29 - YAML_DIR: workloads/container - - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - service: - [ - trace_func_go, - ] - steps: - - uses: actions/checkout@v4 - with: - lfs: "true" - - name: Checkout LFS objects - run: git lfs checkout - - - uses: actions/setup-go@v5 - with: - go-version: 1.22 - - - name: Create k8s Kind Cluster - run: bash ./scripts/konk-ci/01-kind.sh - - - name: Install Serving - run: bash ./scripts/konk-ci/02-serving.sh - - - name: Install Kourier - run: bash ./scripts/konk-ci/02-kourier.sh - - - name: Setup domain and autoscaler - run: | - INGRESS_HOST="127.0.0.1" - KNATIVE_DOMAIN=$INGRESS_HOST.sslip.io - kubectl patch configmap -n knative-serving config-domain -p "{\"data\": {\"$KNATIVE_DOMAIN\": \"\"}}" - kubectl patch configmap -n knative-serving config-autoscaler -p "{\"data\": {\"allow-zero-initial-scale\": \"true\"}}" - kubectl patch configmap -n knative-serving config-features -p "{\"data\": {\"kubernetes.podspec-affinity\": \"enabled\"}}" - kubectl label node knative-control-plane loader-nodetype=worker - - - name: Build and run loader - run: go run cmd/loader.go --config pkg/config/test_config.json - - - name: Check the output - run: test -f "data/out/experiment_duration_5.csv" && test $(grep true data/out/experiment_duration_5.csv | wc -l) -eq 0 # test the output file for errors (true means failure to invoke) - - - name: Print logs - if: ${{ always() }} - run: | - set -x - container_list=$(kubectl get pods -n default -o jsonpath="{.items[*].spec.containers[*].name}") - for container_name in $container_list - do - kubectl logs -n default -c $container_name -l serving.knative.dev/service=${{ matrix.service }} - done - - name: Down - if: ${{ always() }} - run: | - kn service delete --all \ No newline at end of file diff --git a/.github/workflows/e2e_loader.yaml b/.github/workflows/e2e_loader.yaml new file mode 100644 index 00000000..4df805b2 --- /dev/null +++ b/.github/workflows/e2e_loader.yaml @@ -0,0 +1,57 @@ +name: End-to-End Loader Test + +on: + schedule: + - cron: "0 9 * * 1" + workflow_dispatch: + push: + branches: [main] + pull_request: + branches: [main] + +env: + GOOS: linux + GO111MODULE: on + +jobs: + test-knative: + name: Test Knative Deployment + env: + KIND_VERSION: v0.22.0 + K8S_VERSION: v1.29 + YAML_DIR: workloads/container + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + service: + [ + trace_func_go, + ] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Kubernetes Kind Cluster + uses: ./.github/actions/ci_knative_setup + + - name: Build and run loader + run: go run cmd/loader.go --config pkg/config/test_config.json + + - name: Check the output + run: test -f "data/out/experiment_duration_5.csv" && test $(grep true data/out/experiment_duration_5.csv | wc -l) -eq 0 # test the output file for errors (true means failure to invoke) + + - name: Print logs + if: ${{ always() }} + run: | + set -x + container_list=$(kubectl get pods -n default -o jsonpath="{.items[*].spec.containers[*].name}") + for container_name in $container_list + do + kubectl logs -n default -c $container_name -l serving.knative.dev/service=${{ matrix.service }} + done + + - name: Down + if: ${{ always() }} + run: | + kn service delete --all \ No newline at end of file diff --git a/.github/workflows/e2e_multi_loader.yaml b/.github/workflows/e2e_multi_loader.yaml new file mode 100644 index 00000000..6d8e5016 --- /dev/null +++ b/.github/workflows/e2e_multi_loader.yaml @@ -0,0 +1,69 @@ +name: End-to-End Multi-Loader Tests + +on: + schedule: + - cron: "0 9 * * 1" + workflow_dispatch: + push: + branches: [main] + pull_request: + branches: [main] + +env: + GOOS: linux + GO111MODULE: on + +jobs: + test-multi-loader: + name: Test Multi-Loader with Knative Deployment + env: + KIND_VERSION: v0.22.0 + K8S_VERSION: v1.29 + YAML_DIR: workloads/container + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + service: + [ + trace_func_go, + ] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Kubernetes Kind Cluster + uses: ./.github/actions/ci_knative_setup + + - name: Build and run multi-loader + run: go run tools/multi_loader/multi_loader.go --multiLoaderConfigPath tools/multi_loader/multi_loader_config.json + + - name: Check multi_loader output + run: | + # check if there are 4 output folders: *_example_1_test, *_example_2_test, *_example_3_test & dry_run + folder_count=$(find data/out/multi-test/test-experiment -mindepth 1 -maxdepth 1 -type d | wc -l) + if [ "$folder_count" -ne 4 ]; then + echo "Output folder count is $folder_count, expected 4" + exit 1 + else + echo "Output correct number of folders" + fi + + # Check for errors in each output CSV file + for file in $(find . -name "*_example_1_test/example_1_test_duration_1.csv" -o -name "*_example_2_test/example_2_test_duration_1.csv" -o -name "*_example_3_test/example_3_test_duration_1.csv"); do + if [ ! -f "$file" ]; then + echo "File $file not found!" + exit 1 + fi + + if [ $(grep true "$file" | wc -l) -ne 0 ]; then + echo "Error found in $file" + exit 1 + fi + done + echo "No errors found in output files" + + - name: Down + if: ${{ always() }} + run: | + kn service delete --all diff --git a/.github/workflows/tools-tests.yaml b/.github/workflows/tools-tests.yaml index e7d496f3..cae40703 100644 --- a/.github/workflows/tools-tests.yaml +++ b/.github/workflows/tools-tests.yaml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - module: [ tools/plotter, ] + module: [ tools/plotter, tools/multi_loader/runner] steps: - name: Set up Golang uses: actions/setup-go@v5 diff --git a/tools/.gitignore b/tools/.gitignore index edcd023c..05184c99 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -9,4 +9,7 @@ profile_default/ ipython_config.py .env -.venv \ No newline at end of file +.venv + +multi_loader/multi_loader_config_test.json +!multi_loader/test_data/*/*.csv \ No newline at end of file diff --git a/tools/multi_loader/common/utils.go b/tools/multi_loader/common/utils.go index a97ea49a..63ad72c0 100644 --- a/tools/multi_loader/common/utils.go +++ b/tools/multi_loader/common/utils.go @@ -25,6 +25,18 @@ func ReadMultiLoaderConfigurationFile(path string) types.MultiLoaderConfiguratio return config } +func WriteMultiLoaderConfigurationFile(config types.MultiLoaderConfiguration, path string) { + configByteValue, err := json.Marshal(config) + if err != nil { + log.Fatal(err) + } + + err = os.WriteFile(path, configByteValue, 0644) + if err != nil { + log.Fatal(err) + } +} + func DeterminePlatformFromConfig(multiLoaderConfig types.MultiLoaderConfiguration) string { // Determine platform baseConfigByteValue, err := os.ReadFile(multiLoaderConfig.BaseConfigPath) diff --git a/tools/multi_loader/multi_loader.go b/tools/multi_loader/multi_loader.go index c9e5930b..77d1d3d1 100644 --- a/tools/multi_loader/multi_loader.go +++ b/tools/multi_loader/multi_loader.go @@ -10,7 +10,7 @@ import ( ) var ( - multiLoaderConfigPath = flag.String("multiLoaderConfig", "tools/multi_loader/multi_loader_config.json", "Path to multi loader configuration file") + multiLoaderConfigPath = flag.String("multiLoaderConfigPath", "tools/multi_loader/multi_loader_config.json", "Path to multi loader configuration file") verbosity = flag.String("verbosity", "info", "Logging verbosity - choose from [info, debug, trace]") iatGeneration = flag.Bool("iatGeneration", false, "Generate iats only and skip invocations") generated = flag.Bool("generated", false, "If iats were already generated") diff --git a/tools/multi_loader/multi_loader_config.json b/tools/multi_loader/multi_loader_config.json index 55e4491b..6210740f 100644 --- a/tools/multi_loader/multi_loader_config.json +++ b/tools/multi_loader/multi_loader_config.json @@ -1,7 +1,7 @@ { "Studies": [ { - "Name": "experiment1", + "Name": "test-experiment", "Config": { "ExperimentDuration": 1 }, @@ -9,9 +9,11 @@ "Generated": false, "Verbosity": "info", "TracesDir": "", - "TracesFormat": "data/traces/multi-exp-test/example_{}_test", + "TracesFormat": "tools/multi_loader/test_data/example_{}_test", "TraceValues": [ - 1 + "1", + "2", + 3.1 ], "OutputDir": "data/out/multi-test", "PreScript": "", diff --git a/tools/multi_loader/multi_loader_config_test.json b/tools/multi_loader/multi_loader_config_test.json new file mode 100644 index 00000000..58fad90e --- /dev/null +++ b/tools/multi_loader/multi_loader_config_test.json @@ -0,0 +1 @@ +{"Studies":[{"Name":"test-experiment","Config":{"ExperimentDuration":1},"TracesDir":"","TracesFormat":"tools/multi_loader/test_data/example_{}_test","TraceValues":["1","2",3.1],"OutputDir":"data/out/multi-test","Verbosity":"info","IatGeneration":false,"Generated":false,"PreScript":"","PostScript":""}],"BaseConfigPath":"../base_loader_config.json","PreScript":"","PostScript":""} \ No newline at end of file diff --git a/tools/multi_loader/runner/multi_loader_runner_test.go b/tools/multi_loader/runner/multi_loader_runner_test.go index a914eafe..a3c0d142 100644 --- a/tools/multi_loader/runner/multi_loader_runner_test.go +++ b/tools/multi_loader/runner/multi_loader_runner_test.go @@ -1,6 +1,7 @@ package runner import ( + "fmt" "os" "path/filepath" "strings" @@ -19,15 +20,21 @@ var ( func init() { wd, _ := os.Getwd() - multiLoaderTestConfigPath = filepath.Join(wd, "./test_configs/test_multi_loader_config.json") - configPath = filepath.Join(wd, "./test_configs/base_loader_config.json") + multiLoaderTestConfigPath = filepath.Join(wd, "../multi_loader_config.json") + configPath = filepath.Join(wd, "../base_loader_config.json") log.Info("Test config path: ", multiLoaderTestConfigPath) log.Info("Test config path: ", configPath) + + // Override the BaseConfigPath field in multi_loader_config.json + mlConfig := ml_common.ReadMultiLoaderConfigurationFile(multiLoaderTestConfigPath) + mlConfig.BaseConfigPath = "../base_loader_config.json" + multiLoaderTestConfigPath = "../multi_loader_config_test.json" + ml_common.WriteMultiLoaderConfigurationFile(mlConfig, multiLoaderTestConfigPath) } func TestUnpackExperiment(t *testing.T) { // helper func to validate unpacked experiments - validateUnpackedExperiment := func(t *testing.T, experimentConfig []types.LoaderStudy, studyConfig types.LoaderStudy, expectedNames []string, expectedOutputPrefixes []string) { + validateUnpackedExperiment := func(t *testing.T, experimentConfig []types.LoaderExperiment, studyConfig types.LoaderStudy, expectedNames []string, expectedOutputPrefixes []string) { if len(experimentConfig) != len(expectedNames) { t.Errorf("Expected %d sub-experiments, got %d", len(expectedNames), len(experimentConfig)) return @@ -64,7 +71,7 @@ func TestUnpackExperiment(t *testing.T) { t.Run("Unpack using TracesDir (Success)", func(t *testing.T) { // Set TracesDir to test directory - multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "./test_multi_trace" + multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "../test_data" for _, experiment := range multiLoader.MultiLoaderConfig.Studies { subExperiments := multiLoader.unpackStudy(experiment) @@ -76,7 +83,7 @@ func TestUnpackExperiment(t *testing.T) { t.Run("Unpack using TracesDir (Failure: Incorrect Dir)", func(t *testing.T) { expectFatal(t, func() { - multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "./test_multi_trace_incorrect" + multiLoader.MultiLoaderConfig.Studies[0].TracesDir = "../test_data_incorrect" for _, experiment := range multiLoader.MultiLoaderConfig.Studies { _ = multiLoader.unpackStudy(experiment) } @@ -90,7 +97,7 @@ func TestUnpackExperiment(t *testing.T) { subExperiments := multiLoader.unpackStudy(experiment) expectedNames := make([]string, len(experiment.TraceValues)) for i, traceValue := range experiment.TraceValues { - expectedNames[i] = experiment.Name + "_" + traceValue.(string) + expectedNames[i] = experiment.Name + "_example_" + fmt.Sprintf("%v", traceValue) + "_test" } validateUnpackedExperiment(t, subExperiments, experiment, expectedNames, nil) } @@ -103,7 +110,7 @@ func TestUnpackExperiment(t *testing.T) { for _, experiment := range multiLoader.MultiLoaderConfig.Studies { subExperiments := multiLoader.unpackStudy(experiment) - expectedNames := []string{experiment.Name} + expectedNames := []string{experiment.Name + "_" + experiment.Name} validateUnpackedExperiment(t, subExperiments, experiment, expectedNames, nil) } }) @@ -116,11 +123,11 @@ func TestPrepareExperiment(t *testing.T) { t.Fatalf("Failed to create multi-loader driver: %v", err) } - subExperiment := types.LoaderStudy{ + subExperiment := types.LoaderExperiment{ Name: "example_1", Config: map[string]interface{}{ "ExperimentDuration": 10, - "TracePath": "./test_multi_trace/example_1_test", + "TracePath": "../test_data/example_1_test", "OutputPathPrefix": "./test_output/example_1_test", }, } @@ -156,18 +163,18 @@ func TestMergeConfig(t *testing.T) { if err != nil { t.Fatalf("Failed to create multi-loader driver: %v", err) } - experiment := types.LoaderStudy{ + experiment := types.LoaderExperiment{ Name: "example_1", Config: map[string]interface{}{ "ExperimentDuration": 10, - "TracePath": "./test_multi_trace/example_1_test", + "TracePath": "../test_data/example_1_test", "OutputPathPrefix": "./test_output/example_1_test", }, } - outputConfig := multiLoader.mergeConfigurations("./test_configs/test_base_loader_config.json", experiment) + outputConfig := multiLoader.mergeConfigurations(configPath, experiment) // Check if the configurations are merged - if outputConfig.TracePath != "./test_multi_trace/example_1_test" { - t.Errorf("Expected TracePath to be './test_multi_trace/example_1_test', got %v", experiment.Config["TracePath"]) + if outputConfig.TracePath != "../test_data/example_1_test" { + t.Errorf("Expected TracePath to be '../test_data/example_1_test', got %v", experiment.Config["TracePath"]) } if outputConfig.OutputPathPrefix != "./test_output/example_1_test" { t.Errorf("Expected OutputPathPrefix to be './test_output/example_1_test', got %v", experiment.Config["OutputPathPrefix"]) diff --git a/tools/multi_loader/runner/test_configs/test_base_loader_config.json b/tools/multi_loader/runner/test_configs/test_base_loader_config.json deleted file mode 100644 index b20768b1..00000000 --- a/tools/multi_loader/runner/test_configs/test_base_loader_config.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "Seed": 42, - "Platform": "Test", - "InvokeProtocol": "grpc", - "YAMLSelector": "container", - "EndpointPort": 80, - "BusyLoopOnSandboxStartup": false, - "TracePath": "data/traces/example", - "Granularity": "minute", - "OutputPathPrefix": "data/out/experiment", - "IATDistribution": "exponential", - "CPULimit": "1vCPU", - "ExperimentDuration": 5, - "WarmupDuration": 0, - "IsPartiallyPanic": false, - "EnableZipkinTracing": false, - "EnableMetricsScrapping": false, - "MetricScrapingPeriodSeconds": 15, - "AutoscalingMetric": "concurrency", - "GRPCConnectionTimeoutSeconds": 15, - "GRPCFunctionTimeoutSeconds": 900, - "DAGMode": false -} \ No newline at end of file diff --git a/tools/multi_loader/runner/test_configs/test_multi_loader_config.json b/tools/multi_loader/runner/test_configs/test_multi_loader_config.json deleted file mode 100644 index 9138a040..00000000 --- a/tools/multi_loader/runner/test_configs/test_multi_loader_config.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "Studies": [ - { - "Name": "test-experiment", - "Config": { - "ExperimentDuration": 1 - }, - "IatGeneration": false, - "Generated": false, - "Verbosity": "info", - "TracesFormat": "data/traces/{}", - "TraceValues": [ - "example", - "example2" - ], - "OutputDir": "data/out/multi-test" - } - ], - "BaseConfigPath": "./test_configs/test_base_loader_config.json" -} \ No newline at end of file diff --git a/tools/multi_loader/test_data/example_1_test/dirigent.csv b/tools/multi_loader/test_data/example_1_test/dirigent.csv new file mode 100644 index 00000000..b94558b1 --- /dev/null +++ b/tools/multi_loader/test_data/example_1_test/dirigent.csv @@ -0,0 +1,5 @@ +HashFunction,Image,Port,Protocol,ScalingUpperBound,ScalingLowerBound,IterationMultiplier +c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf,empty,80,tcp,5,0,80 +a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600,empty,80,tcp,5,0,80 +7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7,empty,80,tcp,5,0,80 +ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091,empty,80,tcp,5,0,80 \ No newline at end of file diff --git a/tools/multi_loader/test_data/example_1_test/durations.csv b/tools/multi_loader/test_data/example_1_test/durations.csv new file mode 100644 index 00000000..417e2afc --- /dev/null +++ b/tools/multi_loader/test_data/example_1_test/durations.csv @@ -0,0 +1,5 @@ +HashOwner,HashApp,HashFunction,Average,Count,Minimum,Maximum,percentile_Average_0,percentile_Average_1,percentile_Average_25,percentile_Average_50,percentile_Average_75,percentile_Average_99,percentile_Average_100 +c455703077a17a9b8d0fc655d939fcc6d24d819fa9a1066b74f710c35a43cbc8,68baea05aa0c3619b6feb78c80a07e27e4e68f921d714b8125f916c3b3370bf2,c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf,100.0,57523.0,99.0,101.0,99.0,99.0,99.0,100.0,101.0,101.0,101.0 +d7028b01f2422ea9d4f695cae4085800eb34540674081a46bb4e4388e7995f7a,c8b8b9160f181010f509ee0af3266cbcb5a5f4c7700ba8d4396f1147e1ad3bbd,a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600,500.0,27697.0,499.0,501.0,500.0,500.0,500.0,500.0,501.0,501.0,501.0 +0728f480194febeddbc0d2a69a2cb38344e495f264d52c62102e7bd99505dbf2,2cc1c836a3b32265b4a36b6e6f56b2d7c0588a926df5d03a76b7b4388cbf2258,7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7,250.0,8897.0,249.0,251.0,249.0,249.0,249.0,250.0,251.0,251.0,251.0 +01d1a13d091b0cfc764a125ead054ad6a720cb29b0b1fc2c187e9be3311f09fe,90ba6ab2b1b9f5b5925d1e9ee6242d0f840ebcfb3221567bc2b5f24cb864b016,ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091,125.0,2880.0,124.0,126.0,124.0,124.0,124.0,125.0,126.0,126.0,126.0 \ No newline at end of file diff --git a/tools/multi_loader/test_data/example_1_test/invocations.csv b/tools/multi_loader/test_data/example_1_test/invocations.csv new file mode 100644 index 00000000..4adc8e16 --- /dev/null +++ b/tools/multi_loader/test_data/example_1_test/invocations.csv @@ -0,0 +1,5 @@ +HashOwner,HashApp,HashFunction,Trigger,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000,1001,1002,1003,1004,1005,1006,1007,1008,1009,1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,1117,1118,1119,1120,1121,1122,1123,1124,1125,1126,1127,1128,1129,1130,1131,1132,1133,1134,1135,1136,1137,1138,1139,1140,1141,1142,1143,1144,1145,1146,1147,1148,1149,1150,1151,1152,1153,1154,1155,1156,1157,1158,1159,1160,1161,1162,1163,1164,1165,1166,1167,1168,1169,1170,1171,1172,1173,1174,1175,1176,1177,1178,1179,1180,1181,1182,1183,1184,1185,1186,1187,1188,1189,1190,1191,1192,1193,1194,1195,1196,1197,1198,1199,1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211,1212,1213,1214,1215,1216,1217,1218,1219,1220,1221,1222,1223,1224,1225,1226,1227,1228,1229,1230,1231,1232,1233,1234,1235,1236,1237,1238,1239,1240,1241,1242,1243,1244,1245,1246,1247,1248,1249,1250,1251,1252,1253,1254,1255,1256,1257,1258,1259,1260,1261,1262,1263,1264,1265,1266,1267,1268,1269,1270,1271,1272,1273,1274,1275,1276,1277,1278,1279,1280,1281,1282,1283,1284,1285,1286,1287,1288,1289,1290,1291,1292,1293,1294,1295,1296,1297,1298,1299,1300,1301,1302,1303,1304,1305,1306,1307,1308,1309,1310,1311,1312,1313,1314,1315,1316,1317,1318,1319,1320,1321,1322,1323,1324,1325,1326,1327,1328,1329,1330,1331,1332,1333,1334,1335,1336,1337,1338,1339,1340,1341,1342,1343,1344,1345,1346,1347,1348,1349,1350,1351,1352,1353,1354,1355,1356,1357,1358,1359,1360,1361,1362,1363,1364,1365,1366,1367,1368,1369,1370,1371,1372,1373,1374,1375,1376,1377,1378,1379,1380,1381,1382,1383,1384,1385,1386,1387,1388,1389,1390,1391,1392,1393,1394,1395,1396,1397,1398,1399,1400,1401,1402,1403,1404,1405,1406,1407,1408,1409,1410,1411,1412,1413,1414,1415,1416,1417,1418,1419,1420,1421,1422,1423,1424,1425,1426,1427,1428,1429,1430,1431,1432,1433,1434,1435,1436,1437,1438,1439,1440 +c455703077a17a9b8d0fc655d939fcc6d24d819fa9a1066b74f710c35a43cbc8,68baea05aa0c3619b6feb78c80a07e27e4e68f921d714b8125f916c3b3370bf2,c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf,queue,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +d7028b01f2422ea9d4f695cae4085800eb34540674081a46bb4e4388e7995f7a,c8b8b9160f181010f509ee0af3266cbcb5a5f4c7700ba8d4396f1147e1ad3bbd,a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600,queue,0,0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +0728f480194febeddbc0d2a69a2cb38344e495f264d52c62102e7bd99505dbf2,2cc1c836a3b32265b4a36b6e6f56b2d7c0588a926df5d03a76b7b4388cbf2258,7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7,queue,0,0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +01d1a13d091b0cfc764a125ead054ad6a720cb29b0b1fc2c187e9be3311f09fe,90ba6ab2b1b9f5b5925d1e9ee6242d0f840ebcfb3221567bc2b5f24cb864b016,ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091,timer,0,0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 \ No newline at end of file diff --git a/tools/multi_loader/test_data/example_1_test/memory.csv b/tools/multi_loader/test_data/example_1_test/memory.csv new file mode 100644 index 00000000..46d8c30f --- /dev/null +++ b/tools/multi_loader/test_data/example_1_test/memory.csv @@ -0,0 +1,5 @@ +HashOwner,HashApp,HashFunction,SampleCount,AverageAllocatedMb,AverageAllocatedMb_pct1,AverageAllocatedMb_pct5,AverageAllocatedMb_pct25,AverageAllocatedMb_pct50,AverageAllocatedMb_pct75,AverageAllocatedMb_pct95,AverageAllocatedMb_pct99,AverageAllocatedMb_pct100 +c455703077a17a9b8d0fc655d939fcc6d24d819fa9a1066b74f710c35a43cbc8,68baea05aa0c3619b6feb78c80a07e27e4e68f921d714b8125f916c3b3370bf2,c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf,19342.0,120.0,100.0,102.0,114.0,123.0,127.0,136.0,143.0,152.0 +d7028b01f2422ea9d4f695cae4085800eb34540674081a46bb4e4388e7995f7a,c8b8b9160f181010f509ee0af3266cbcb5a5f4c7700ba8d4396f1147e1ad3bbd,a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600,22084.0,176.0,146.0,159.0,165.0,173.0,181.0,211.0,228.0,235.0 +0728f480194febeddbc0d2a69a2cb38344e495f264d52c62102e7bd99505dbf2,2cc1c836a3b32265b4a36b6e6f56b2d7c0588a926df5d03a76b7b4388cbf2258,7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7,14841.0,170.0,85.0,96.0,171.0,186.0,191.0,205.0,317.0,382.0 +01d1a13d091b0cfc764a125ead054ad6a720cb29b0b1fc2c187e9be3311f09fe,90ba6ab2b1b9f5b5925d1e9ee6242d0f840ebcfb3221567bc2b5f24cb864b016,ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091,17360.0,112.0,112.0,112.0,112.0,113.0,113.0,113.0,113.0,113.0 \ No newline at end of file diff --git a/tools/multi_loader/test_data/example_2_test/dirigent.csv b/tools/multi_loader/test_data/example_2_test/dirigent.csv new file mode 100644 index 00000000..b94558b1 --- /dev/null +++ b/tools/multi_loader/test_data/example_2_test/dirigent.csv @@ -0,0 +1,5 @@ +HashFunction,Image,Port,Protocol,ScalingUpperBound,ScalingLowerBound,IterationMultiplier +c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf,empty,80,tcp,5,0,80 +a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600,empty,80,tcp,5,0,80 +7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7,empty,80,tcp,5,0,80 +ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091,empty,80,tcp,5,0,80 \ No newline at end of file diff --git a/tools/multi_loader/test_data/example_2_test/durations.csv b/tools/multi_loader/test_data/example_2_test/durations.csv new file mode 100644 index 00000000..417e2afc --- /dev/null +++ b/tools/multi_loader/test_data/example_2_test/durations.csv @@ -0,0 +1,5 @@ +HashOwner,HashApp,HashFunction,Average,Count,Minimum,Maximum,percentile_Average_0,percentile_Average_1,percentile_Average_25,percentile_Average_50,percentile_Average_75,percentile_Average_99,percentile_Average_100 +c455703077a17a9b8d0fc655d939fcc6d24d819fa9a1066b74f710c35a43cbc8,68baea05aa0c3619b6feb78c80a07e27e4e68f921d714b8125f916c3b3370bf2,c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf,100.0,57523.0,99.0,101.0,99.0,99.0,99.0,100.0,101.0,101.0,101.0 +d7028b01f2422ea9d4f695cae4085800eb34540674081a46bb4e4388e7995f7a,c8b8b9160f181010f509ee0af3266cbcb5a5f4c7700ba8d4396f1147e1ad3bbd,a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600,500.0,27697.0,499.0,501.0,500.0,500.0,500.0,500.0,501.0,501.0,501.0 +0728f480194febeddbc0d2a69a2cb38344e495f264d52c62102e7bd99505dbf2,2cc1c836a3b32265b4a36b6e6f56b2d7c0588a926df5d03a76b7b4388cbf2258,7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7,250.0,8897.0,249.0,251.0,249.0,249.0,249.0,250.0,251.0,251.0,251.0 +01d1a13d091b0cfc764a125ead054ad6a720cb29b0b1fc2c187e9be3311f09fe,90ba6ab2b1b9f5b5925d1e9ee6242d0f840ebcfb3221567bc2b5f24cb864b016,ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091,125.0,2880.0,124.0,126.0,124.0,124.0,124.0,125.0,126.0,126.0,126.0 \ No newline at end of file diff --git a/tools/multi_loader/test_data/example_2_test/invocations.csv b/tools/multi_loader/test_data/example_2_test/invocations.csv new file mode 100644 index 00000000..4adc8e16 --- /dev/null +++ b/tools/multi_loader/test_data/example_2_test/invocations.csv @@ -0,0 +1,5 @@ +HashOwner,HashApp,HashFunction,Trigger,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000,1001,1002,1003,1004,1005,1006,1007,1008,1009,1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,1117,1118,1119,1120,1121,1122,1123,1124,1125,1126,1127,1128,1129,1130,1131,1132,1133,1134,1135,1136,1137,1138,1139,1140,1141,1142,1143,1144,1145,1146,1147,1148,1149,1150,1151,1152,1153,1154,1155,1156,1157,1158,1159,1160,1161,1162,1163,1164,1165,1166,1167,1168,1169,1170,1171,1172,1173,1174,1175,1176,1177,1178,1179,1180,1181,1182,1183,1184,1185,1186,1187,1188,1189,1190,1191,1192,1193,1194,1195,1196,1197,1198,1199,1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211,1212,1213,1214,1215,1216,1217,1218,1219,1220,1221,1222,1223,1224,1225,1226,1227,1228,1229,1230,1231,1232,1233,1234,1235,1236,1237,1238,1239,1240,1241,1242,1243,1244,1245,1246,1247,1248,1249,1250,1251,1252,1253,1254,1255,1256,1257,1258,1259,1260,1261,1262,1263,1264,1265,1266,1267,1268,1269,1270,1271,1272,1273,1274,1275,1276,1277,1278,1279,1280,1281,1282,1283,1284,1285,1286,1287,1288,1289,1290,1291,1292,1293,1294,1295,1296,1297,1298,1299,1300,1301,1302,1303,1304,1305,1306,1307,1308,1309,1310,1311,1312,1313,1314,1315,1316,1317,1318,1319,1320,1321,1322,1323,1324,1325,1326,1327,1328,1329,1330,1331,1332,1333,1334,1335,1336,1337,1338,1339,1340,1341,1342,1343,1344,1345,1346,1347,1348,1349,1350,1351,1352,1353,1354,1355,1356,1357,1358,1359,1360,1361,1362,1363,1364,1365,1366,1367,1368,1369,1370,1371,1372,1373,1374,1375,1376,1377,1378,1379,1380,1381,1382,1383,1384,1385,1386,1387,1388,1389,1390,1391,1392,1393,1394,1395,1396,1397,1398,1399,1400,1401,1402,1403,1404,1405,1406,1407,1408,1409,1410,1411,1412,1413,1414,1415,1416,1417,1418,1419,1420,1421,1422,1423,1424,1425,1426,1427,1428,1429,1430,1431,1432,1433,1434,1435,1436,1437,1438,1439,1440 +c455703077a17a9b8d0fc655d939fcc6d24d819fa9a1066b74f710c35a43cbc8,68baea05aa0c3619b6feb78c80a07e27e4e68f921d714b8125f916c3b3370bf2,c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf,queue,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +d7028b01f2422ea9d4f695cae4085800eb34540674081a46bb4e4388e7995f7a,c8b8b9160f181010f509ee0af3266cbcb5a5f4c7700ba8d4396f1147e1ad3bbd,a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600,queue,0,0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +0728f480194febeddbc0d2a69a2cb38344e495f264d52c62102e7bd99505dbf2,2cc1c836a3b32265b4a36b6e6f56b2d7c0588a926df5d03a76b7b4388cbf2258,7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7,queue,0,0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +01d1a13d091b0cfc764a125ead054ad6a720cb29b0b1fc2c187e9be3311f09fe,90ba6ab2b1b9f5b5925d1e9ee6242d0f840ebcfb3221567bc2b5f24cb864b016,ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091,timer,0,0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 \ No newline at end of file diff --git a/tools/multi_loader/test_data/example_2_test/memory.csv b/tools/multi_loader/test_data/example_2_test/memory.csv new file mode 100644 index 00000000..46d8c30f --- /dev/null +++ b/tools/multi_loader/test_data/example_2_test/memory.csv @@ -0,0 +1,5 @@ +HashOwner,HashApp,HashFunction,SampleCount,AverageAllocatedMb,AverageAllocatedMb_pct1,AverageAllocatedMb_pct5,AverageAllocatedMb_pct25,AverageAllocatedMb_pct50,AverageAllocatedMb_pct75,AverageAllocatedMb_pct95,AverageAllocatedMb_pct99,AverageAllocatedMb_pct100 +c455703077a17a9b8d0fc655d939fcc6d24d819fa9a1066b74f710c35a43cbc8,68baea05aa0c3619b6feb78c80a07e27e4e68f921d714b8125f916c3b3370bf2,c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf,19342.0,120.0,100.0,102.0,114.0,123.0,127.0,136.0,143.0,152.0 +d7028b01f2422ea9d4f695cae4085800eb34540674081a46bb4e4388e7995f7a,c8b8b9160f181010f509ee0af3266cbcb5a5f4c7700ba8d4396f1147e1ad3bbd,a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600,22084.0,176.0,146.0,159.0,165.0,173.0,181.0,211.0,228.0,235.0 +0728f480194febeddbc0d2a69a2cb38344e495f264d52c62102e7bd99505dbf2,2cc1c836a3b32265b4a36b6e6f56b2d7c0588a926df5d03a76b7b4388cbf2258,7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7,14841.0,170.0,85.0,96.0,171.0,186.0,191.0,205.0,317.0,382.0 +01d1a13d091b0cfc764a125ead054ad6a720cb29b0b1fc2c187e9be3311f09fe,90ba6ab2b1b9f5b5925d1e9ee6242d0f840ebcfb3221567bc2b5f24cb864b016,ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091,17360.0,112.0,112.0,112.0,112.0,113.0,113.0,113.0,113.0,113.0 \ No newline at end of file diff --git a/tools/multi_loader/test_data/example_3.1_test/dirigent.csv b/tools/multi_loader/test_data/example_3.1_test/dirigent.csv new file mode 100644 index 00000000..b94558b1 --- /dev/null +++ b/tools/multi_loader/test_data/example_3.1_test/dirigent.csv @@ -0,0 +1,5 @@ +HashFunction,Image,Port,Protocol,ScalingUpperBound,ScalingLowerBound,IterationMultiplier +c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf,empty,80,tcp,5,0,80 +a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600,empty,80,tcp,5,0,80 +7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7,empty,80,tcp,5,0,80 +ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091,empty,80,tcp,5,0,80 \ No newline at end of file diff --git a/tools/multi_loader/test_data/example_3.1_test/durations.csv b/tools/multi_loader/test_data/example_3.1_test/durations.csv new file mode 100644 index 00000000..417e2afc --- /dev/null +++ b/tools/multi_loader/test_data/example_3.1_test/durations.csv @@ -0,0 +1,5 @@ +HashOwner,HashApp,HashFunction,Average,Count,Minimum,Maximum,percentile_Average_0,percentile_Average_1,percentile_Average_25,percentile_Average_50,percentile_Average_75,percentile_Average_99,percentile_Average_100 +c455703077a17a9b8d0fc655d939fcc6d24d819fa9a1066b74f710c35a43cbc8,68baea05aa0c3619b6feb78c80a07e27e4e68f921d714b8125f916c3b3370bf2,c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf,100.0,57523.0,99.0,101.0,99.0,99.0,99.0,100.0,101.0,101.0,101.0 +d7028b01f2422ea9d4f695cae4085800eb34540674081a46bb4e4388e7995f7a,c8b8b9160f181010f509ee0af3266cbcb5a5f4c7700ba8d4396f1147e1ad3bbd,a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600,500.0,27697.0,499.0,501.0,500.0,500.0,500.0,500.0,501.0,501.0,501.0 +0728f480194febeddbc0d2a69a2cb38344e495f264d52c62102e7bd99505dbf2,2cc1c836a3b32265b4a36b6e6f56b2d7c0588a926df5d03a76b7b4388cbf2258,7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7,250.0,8897.0,249.0,251.0,249.0,249.0,249.0,250.0,251.0,251.0,251.0 +01d1a13d091b0cfc764a125ead054ad6a720cb29b0b1fc2c187e9be3311f09fe,90ba6ab2b1b9f5b5925d1e9ee6242d0f840ebcfb3221567bc2b5f24cb864b016,ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091,125.0,2880.0,124.0,126.0,124.0,124.0,124.0,125.0,126.0,126.0,126.0 \ No newline at end of file diff --git a/tools/multi_loader/test_data/example_3.1_test/invocations.csv b/tools/multi_loader/test_data/example_3.1_test/invocations.csv new file mode 100644 index 00000000..4adc8e16 --- /dev/null +++ b/tools/multi_loader/test_data/example_3.1_test/invocations.csv @@ -0,0 +1,5 @@ +HashOwner,HashApp,HashFunction,Trigger,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000,1001,1002,1003,1004,1005,1006,1007,1008,1009,1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,1117,1118,1119,1120,1121,1122,1123,1124,1125,1126,1127,1128,1129,1130,1131,1132,1133,1134,1135,1136,1137,1138,1139,1140,1141,1142,1143,1144,1145,1146,1147,1148,1149,1150,1151,1152,1153,1154,1155,1156,1157,1158,1159,1160,1161,1162,1163,1164,1165,1166,1167,1168,1169,1170,1171,1172,1173,1174,1175,1176,1177,1178,1179,1180,1181,1182,1183,1184,1185,1186,1187,1188,1189,1190,1191,1192,1193,1194,1195,1196,1197,1198,1199,1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211,1212,1213,1214,1215,1216,1217,1218,1219,1220,1221,1222,1223,1224,1225,1226,1227,1228,1229,1230,1231,1232,1233,1234,1235,1236,1237,1238,1239,1240,1241,1242,1243,1244,1245,1246,1247,1248,1249,1250,1251,1252,1253,1254,1255,1256,1257,1258,1259,1260,1261,1262,1263,1264,1265,1266,1267,1268,1269,1270,1271,1272,1273,1274,1275,1276,1277,1278,1279,1280,1281,1282,1283,1284,1285,1286,1287,1288,1289,1290,1291,1292,1293,1294,1295,1296,1297,1298,1299,1300,1301,1302,1303,1304,1305,1306,1307,1308,1309,1310,1311,1312,1313,1314,1315,1316,1317,1318,1319,1320,1321,1322,1323,1324,1325,1326,1327,1328,1329,1330,1331,1332,1333,1334,1335,1336,1337,1338,1339,1340,1341,1342,1343,1344,1345,1346,1347,1348,1349,1350,1351,1352,1353,1354,1355,1356,1357,1358,1359,1360,1361,1362,1363,1364,1365,1366,1367,1368,1369,1370,1371,1372,1373,1374,1375,1376,1377,1378,1379,1380,1381,1382,1383,1384,1385,1386,1387,1388,1389,1390,1391,1392,1393,1394,1395,1396,1397,1398,1399,1400,1401,1402,1403,1404,1405,1406,1407,1408,1409,1410,1411,1412,1413,1414,1415,1416,1417,1418,1419,1420,1421,1422,1423,1424,1425,1426,1427,1428,1429,1430,1431,1432,1433,1434,1435,1436,1437,1438,1439,1440 +c455703077a17a9b8d0fc655d939fcc6d24d819fa9a1066b74f710c35a43cbc8,68baea05aa0c3619b6feb78c80a07e27e4e68f921d714b8125f916c3b3370bf2,c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf,queue,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +d7028b01f2422ea9d4f695cae4085800eb34540674081a46bb4e4388e7995f7a,c8b8b9160f181010f509ee0af3266cbcb5a5f4c7700ba8d4396f1147e1ad3bbd,a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600,queue,0,0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +0728f480194febeddbc0d2a69a2cb38344e495f264d52c62102e7bd99505dbf2,2cc1c836a3b32265b4a36b6e6f56b2d7c0588a926df5d03a76b7b4388cbf2258,7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7,queue,0,0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +01d1a13d091b0cfc764a125ead054ad6a720cb29b0b1fc2c187e9be3311f09fe,90ba6ab2b1b9f5b5925d1e9ee6242d0f840ebcfb3221567bc2b5f24cb864b016,ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091,timer,0,0,0,0,0,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 \ No newline at end of file diff --git a/tools/multi_loader/test_data/example_3.1_test/memory.csv b/tools/multi_loader/test_data/example_3.1_test/memory.csv new file mode 100644 index 00000000..46d8c30f --- /dev/null +++ b/tools/multi_loader/test_data/example_3.1_test/memory.csv @@ -0,0 +1,5 @@ +HashOwner,HashApp,HashFunction,SampleCount,AverageAllocatedMb,AverageAllocatedMb_pct1,AverageAllocatedMb_pct5,AverageAllocatedMb_pct25,AverageAllocatedMb_pct50,AverageAllocatedMb_pct75,AverageAllocatedMb_pct95,AverageAllocatedMb_pct99,AverageAllocatedMb_pct100 +c455703077a17a9b8d0fc655d939fcc6d24d819fa9a1066b74f710c35a43cbc8,68baea05aa0c3619b6feb78c80a07e27e4e68f921d714b8125f916c3b3370bf2,c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf,19342.0,120.0,100.0,102.0,114.0,123.0,127.0,136.0,143.0,152.0 +d7028b01f2422ea9d4f695cae4085800eb34540674081a46bb4e4388e7995f7a,c8b8b9160f181010f509ee0af3266cbcb5a5f4c7700ba8d4396f1147e1ad3bbd,a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600,22084.0,176.0,146.0,159.0,165.0,173.0,181.0,211.0,228.0,235.0 +0728f480194febeddbc0d2a69a2cb38344e495f264d52c62102e7bd99505dbf2,2cc1c836a3b32265b4a36b6e6f56b2d7c0588a926df5d03a76b7b4388cbf2258,7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7,14841.0,170.0,85.0,96.0,171.0,186.0,191.0,205.0,317.0,382.0 +01d1a13d091b0cfc764a125ead054ad6a720cb29b0b1fc2c187e9be3311f09fe,90ba6ab2b1b9f5b5925d1e9ee6242d0f840ebcfb3221567bc2b5f24cb864b016,ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091,17360.0,112.0,112.0,112.0,112.0,113.0,113.0,113.0,113.0,113.0 \ No newline at end of file From 589a01de6e1209b52f17ae2a00ad9215883a7155 Mon Sep 17 00:00:00 2001 From: Lenson Date: Tue, 31 Dec 2024 11:49:55 +0800 Subject: [PATCH 35/37] add failfast flag Signed-off-by: Lenson update failfast flag description Signed-off-by: Lenson --- docs/multi_loader.md | 4 ++++ tools/multi_loader/multi_loader.go | 3 ++- tools/multi_loader/multi_loader_config_test.json | 1 - tools/multi_loader/runner/multi_loader_runner.go | 7 +++++-- tools/multi_loader/runner/multi_loader_runner_test.go | 8 ++++---- 5 files changed, 15 insertions(+), 8 deletions(-) delete mode 100644 tools/multi_loader/multi_loader_config_test.json diff --git a/docs/multi_loader.md b/docs/multi_loader.md index 02af5984..8f864c8b 100644 --- a/docs/multi_loader.md +++ b/docs/multi_loader.md @@ -67,6 +67,10 @@ Available flags: - **`--generated`** *(default: `false`)*: Indicates whether IATs have already been generated. If set to `true`, the multi-loader will use the existing IATs instead of generating new ones. +- **`--failFast`** *(default: `false`)*: + Determines whether the multi-loader should skip the study immediately after a failure. By default, the loader retries a failed experiment once with debug verbosity and skips the study only if the second attempt also fails. Setting this flag to `true` prevents the retry and skips the study after the first failure. + + ## Multi-loader Overall Flow diff --git a/tools/multi_loader/multi_loader.go b/tools/multi_loader/multi_loader.go index 77d1d3d1..4c6ae10d 100644 --- a/tools/multi_loader/multi_loader.go +++ b/tools/multi_loader/multi_loader.go @@ -14,6 +14,7 @@ var ( verbosity = flag.String("verbosity", "info", "Logging verbosity - choose from [info, debug, trace]") iatGeneration = flag.Bool("iatGeneration", false, "Generate iats only and skip invocations") generated = flag.Bool("generated", false, "If iats were already generated") + failFast = flag.Bool("failFast", false, "Determines whether a study should immediately skip to the next study upon failure") ) func init() { @@ -41,7 +42,7 @@ func initLogger() { func main() { log.Info("Starting multiloader") // Create multi loader runner - multiLoaderRunner, err := runner.NewMultiLoaderRunner(*multiLoaderConfigPath, *verbosity, *iatGeneration, *generated) + multiLoaderRunner, err := runner.NewMultiLoaderRunner(*multiLoaderConfigPath, *verbosity, *iatGeneration, *generated, *failFast) if err != nil { log.Fatalf("Failed to create multi loader driver: %v", err) } diff --git a/tools/multi_loader/multi_loader_config_test.json b/tools/multi_loader/multi_loader_config_test.json deleted file mode 100644 index 58fad90e..00000000 --- a/tools/multi_loader/multi_loader_config_test.json +++ /dev/null @@ -1 +0,0 @@ -{"Studies":[{"Name":"test-experiment","Config":{"ExperimentDuration":1},"TracesDir":"","TracesFormat":"tools/multi_loader/test_data/example_{}_test","TraceValues":["1","2",3.1],"OutputDir":"data/out/multi-test","Verbosity":"info","IatGeneration":false,"Generated":false,"PreScript":"","PostScript":""}],"BaseConfigPath":"../base_loader_config.json","PreScript":"","PostScript":""} \ No newline at end of file diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index 86145904..b80ae407 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -36,10 +36,11 @@ type MultiLoaderRunner struct { Generated bool DryRun bool Platform string + FailFast bool } // init multi loader runner -func NewMultiLoaderRunner(configPath string, verbosity string, iatGeneration bool, generated bool) (*MultiLoaderRunner, error) { +func NewMultiLoaderRunner(configPath string, verbosity string, iatGeneration bool, generated bool, failFast bool) (*MultiLoaderRunner, error) { multiLoaderConfig := ml_common.ReadMultiLoaderConfigurationFile(configPath) // validate configuration @@ -56,6 +57,7 @@ func NewMultiLoaderRunner(configPath string, verbosity string, iatGeneration boo Generated: generated, DryRun: false, Platform: platform, + FailFast: failFast, } return &runner, nil @@ -306,7 +308,8 @@ func (d *MultiLoaderRunner) runExperiment(experiment types.LoaderExperiment) err log.Error(err) log.Error("Experiment failed: ", experiment.Name) logFile.WriteString("Experiment failed: " + experiment.Name + ". Error: " + err.Error() + "\n") - if i == 0 && !d.DryRun { + // Retry if not dry run and not fail fast + if i == 0 && !d.DryRun && !d.FailFast { log.Info("Retrying experiment ", experiment.Name) logFile.WriteString("==================================RETRYING==================================\n") experiment.Verbosity = "debug" diff --git a/tools/multi_loader/runner/multi_loader_runner_test.go b/tools/multi_loader/runner/multi_loader_runner_test.go index a3c0d142..202633c6 100644 --- a/tools/multi_loader/runner/multi_loader_runner_test.go +++ b/tools/multi_loader/runner/multi_loader_runner_test.go @@ -64,7 +64,7 @@ func TestUnpackExperiment(t *testing.T) { } // create multiloader - multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false) + multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false, false) if err != nil { t.Fatalf("Failed to create multi-loader driver: %v", err) } @@ -118,7 +118,7 @@ func TestUnpackExperiment(t *testing.T) { func TestPrepareExperiment(t *testing.T) { // Create a new multi-loader driver with the test config path - multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false) + multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false, false) if err != nil { t.Fatalf("Failed to create multi-loader driver: %v", err) } @@ -159,7 +159,7 @@ func TestPrepareExperiment(t *testing.T) { // Test mergeConfigurations method func TestMergeConfig(t *testing.T) { // Create a new multi-loader driver with the test config path - multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false) + multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false, false) if err != nil { t.Fatalf("Failed to create multi-loader driver: %v", err) } @@ -186,7 +186,7 @@ func TestMergeConfig(t *testing.T) { func TestMultiConfigValidator(t *testing.T) { // Create a new multi-loader driver with the test config path - multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false) + multiLoader, err := NewMultiLoaderRunner(multiLoaderTestConfigPath, "info", false, false, false) if err != nil { t.Fatalf("Failed to create multi-loader driver: %v", err) } From d0b6e0faeb55af737bf9425ceff8ab3c85c64149 Mon Sep 17 00:00:00 2001 From: Lenson Date: Tue, 31 Dec 2024 17:22:25 +0800 Subject: [PATCH 36/37] update wordlist with multiloader specific words Signed-off-by: Lenson --- .github/configs/wordlist.txt | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/configs/wordlist.txt b/.github/configs/wordlist.txt index 3c403ef0..3d8b4ba6 100644 --- a/.github/configs/wordlist.txt +++ b/.github/configs/wordlist.txt @@ -781,4 +781,19 @@ async AsyncMode AsyncResponseURL AsyncWaitToCollectMin -Knative's \ No newline at end of file +Knative's +BaseConfigPath +LoaderStudy +LoaderExperiment +PostScript +PreScript +IatGeneration +LoaderConfiguration +PostScript +PreScript +TraceFormat +TraceValues +TracesDir +TracesFormat +baseLoaderConfig +iats \ No newline at end of file From b2c12951e50ed8c75d45111788c01232de9b7dbc Mon Sep 17 00:00:00 2001 From: Lenson Date: Wed, 1 Jan 2025 14:40:27 +0800 Subject: [PATCH 37/37] fix wrong indexing Signed-off-by: Lenson add progress in logging Signed-off-by: Lenson --- tools/multi_loader/runner/multi_loader_runner.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/multi_loader/runner/multi_loader_runner.go b/tools/multi_loader/runner/multi_loader_runner.go index b80ae407..0fc97a89 100644 --- a/tools/multi_loader/runner/multi_loader_runner.go +++ b/tools/multi_loader/runner/multi_loader_runner.go @@ -79,7 +79,7 @@ func (d *MultiLoaderRunner) run() { // Run global prescript common.RunScript(d.MultiLoaderConfig.PreScript) // Iterate over studies and run them - for _, study := range d.MultiLoaderConfig.Studies { + for si, study := range d.MultiLoaderConfig.Studies { log.Debug("Setting up study: ", study.Name) // Run pre script common.RunScript(study.PreScript) @@ -88,7 +88,12 @@ func (d *MultiLoaderRunner) run() { sparseExperiments := d.unpackStudy(study) // Iterate over sparse experiments, prepare and run - for _, experiment := range sparseExperiments { + for ei, experiment := range sparseExperiments { + if d.DryRun { + log.Info(fmt.Sprintf("[Study %d/%d][Experiment %d/%d] Dry running %s", si+1, len(d.MultiLoaderConfig.Studies), ei+1, len(sparseExperiments), experiment.Name)) + } else { + log.Info(fmt.Sprintf("[Study %d/%d][Experiment %d/%d] Running %s", si+1, len(d.MultiLoaderConfig.Studies), ei+1, len(sparseExperiments), experiment.Name)) + } // Prepare experiment: merge with base config, create output dir and write merged config to temp file d.prepareExperiment(experiment) @@ -287,11 +292,6 @@ func (d *MultiLoaderRunner) writeExperimentConfigToTempFile(experimentConfig con } func (d *MultiLoaderRunner) runExperiment(experiment types.LoaderExperiment) error { - if d.DryRun { - log.Info("Dry running ", experiment.Name) - } else { - log.Info("Running ", experiment.Name) - } log.Debug("Experiment configuration ", experiment.Config) // Create the log file