Porting Dirigent config to JSON

Signed-off-by: Lazar Cvetković <[email protected]>
vhive-serverless · Oct 29, 2024 · b6aad78 · b6aad78
1 parent 3e27862
commit b6aad78
Show file tree

Hide file tree

Showing 8 changed files with 146 additions and 31 deletions.
diff --git a/data/traces/example/dirigent.json b/data/traces/example/dirigent.json
@@ -0,0 +1,50 @@
+[
+  {
+    "HashFunction": "c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf",
+    "Image": "docker.io/cvetkovic/dirigent_grpc_function:latest",
+    "Port": 80,
+    "Protocol": "tcp",
+    "ScalingUpperBound": 1,
+    "ScalingLowerBound": 0,
+    "IterationMultiplier": 80,
+    "IOPercentage": 0,
+    "EnvVars": [],
+    "ProgramArgs": []
+  },
+  {
+    "HashFunction": "a2faad786b3c813b12ce57d349d5e62f6d0f22ceecfa86cd72a962853383b600",
+    "Image": "docker.io/cvetkovic/dirigent_grpc_function:latest",
+    "Port": 80,
+    "Protocol": "tcp",
+    "ScalingUpperBound": 1,
+    "ScalingLowerBound": 0,
+    "IterationMultiplier": 80,
+    "IOPercentage": 0,
+    "EnvVars": [],
+    "ProgramArgs": []
+  },
+  {
+    "HashFunction": "7dc5aeabc131669912e8c793c8925cc9928321f45f13a4af031592b4611630d7",
+    "Image": "docker.io/cvetkovic/dirigent_grpc_function:latest",
+    "Port": 80,
+    "Protocol": "tcp",
+    "ScalingUpperBound": 1,
+    "ScalingLowerBound": 0,
+    "IterationMultiplier": 80,
+    "IOPercentage": 0,
+    "EnvVars": [],
+    "ProgramArgs": []
+  },
+  {
+    "HashFunction": "ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091",
+    "Image": "docker.io/cvetkovic/dirigent_grpc_function:latest",
+    "Port": 80,
+    "Protocol": "tcp",
+    "ScalingUpperBound": 1,
+    "ScalingLowerBound": 0,
+    "IterationMultiplier": 80,
+    "IOPercentage": 0,
+    "EnvVars": [],
+    "ProgramArgs": []
+  }
+]
diff --git a/go.mod b/go.mod
@@ -1,6 +1,7 @@
 module github.com/vhive-serverless/loader
 
-go 1.22
+go 1.22.0
+
 toolchain go1.22.5
 
 require (
@@ -16,6 +17,7 @@ require (
 
 require (
 	github.com/aws/aws-lambda-go v1.47.0
+	github.com/containerd/log v0.1.0
 	github.com/stretchr/testify v1.9.0
 	github.com/vhive-serverless/vSwarm/utils/tracing/go v0.0.0-20240827121957-11be651eb39a
 	go.mongodb.org/mongo-driver v1.17.1

diff --git a/go.sum b/go.sum
@@ -11,6 +11,8 @@ github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1s
 github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A=
 github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY=
 github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

diff --git a/pkg/common/trace_types.go b/pkg/common/trace_types.go
@@ -71,16 +71,16 @@ type FunctionMemoryStats struct {
 }
 
 type DirigentMetadata struct {
-	HashFunction        string `csv:"HashFunction"`
-	Image               string `csv:"Image"`
-	Port                int    `csv:"Port"`
-	Protocol            string `csv:"Protocol"`
-	ScalingUpperBound   int    `csv:"ScalingUpperBound"`
-	ScalingLowerBound   int    `csv:"ScalingLowerBound"`
-	IterationMultiplier int    `csv:"IterationMultiplier"`
-	IOPercentage        int    `csv:"IOPercentage"`
-	EnvVars             string `csv:"EnvVars"`
-	ProgramArgs         string `csv:"ProgramArgs"`
+	HashFunction        string   `json:"HashFunction"`
+	Image               string   `json:"Image"`
+	Port                int      `json:"Port"`
+	Protocol            string   `json:"Protocol"`
+	ScalingUpperBound   int      `json:"ScalingUpperBound"`
+	ScalingLowerBound   int      `json:"ScalingLowerBound"`
+	IterationMultiplier int      `json:"IterationMultiplier"`
+	IOPercentage        int      `json:"IOPercentage"`
+	EnvVars             []string `json:"EnvVars"`
+	ProgramArgs         []string `json:"ProgramArgs"`
 }
 
 type Function struct {

diff --git a/pkg/driver/deployment/dirigent.go b/pkg/driver/deployment/dirigent.go
@@ -53,18 +53,6 @@ var registrationClient = &http.Client{
 	},
 }
 
-const CSVColumnDelimiter = " "
-
-func prepareEnvVars(data string) []string {
-	// FORMAT: arg1=value1 arg2=value2
-	return strings.Split(data, CSVColumnDelimiter)
-}
-
-func prepareProgramArgs(data string) []string {
-	// FORMAT: arg1 val1 arg2 val2
-	return strings.Split(data, CSVColumnDelimiter)
-}
-
 func deployDirigent(function *common.Function, controlPlaneAddress string, busyLoopOnColdStart bool) {
 	metadata := function.DirigentMetadata
 
@@ -80,8 +68,8 @@ func deployDirigent(function *common.Function, controlPlaneAddress string, busyL
 		"scaling_lower_bound": {strconv.Itoa(metadata.ScalingLowerBound)},
 		"requested_cpu":       {strconv.Itoa(function.CPURequestsMilli)},
 		"requested_memory":    {strconv.Itoa(function.MemoryRequestsMiB)},
-		"env_vars":            prepareEnvVars(metadata.EnvVars),
-		"program_args":        prepareProgramArgs(metadata.ProgramArgs),
+		"env_vars":            metadata.EnvVars,     // FORMAT: arg1=value1 arg2=value2 ...
+		"program_args":        metadata.ProgramArgs, // FORMAT: arg1 arg2 ...
 	}
 
 	if busyLoopOnColdStart {

diff --git a/pkg/trace/parser.go b/pkg/trace/parser.go
@@ -26,6 +26,7 @@ package trace
 
 import (
 	"encoding/csv"
+	"encoding/json"
 	"fmt"
 	"github.com/gocarina/gocsv"
 	"github.com/vhive-serverless/loader/pkg/common"
@@ -126,7 +127,7 @@ func (p *AzureTraceParser) Parse(platform string) []*common.Function {
 	invocationPath := p.DirectoryPath + "/invocations.csv"
 	runtimePath := p.DirectoryPath + "/durations.csv"
 	memoryPath := p.DirectoryPath + "/memory.csv"
-	dirigentPath := p.DirectoryPath + "/dirigent.csv"
+	dirigentPath := p.DirectoryPath + "/dirigent.json"
 
 	invocationTrace := parseInvocationTrace(invocationPath, p.duration)
 	runtimeTrace := parseRuntimeTrace(runtimePath)
@@ -258,21 +259,20 @@ func parseMemoryTrace(traceFile string) *[]common.FunctionMemoryStats {
 }
 
 func parseDirigentMetadata(traceFile string, platform string) *[]common.DirigentMetadata {
-	if platform != "Dirigent" {
+	if strings.ToLower(platform) != "dirigent" {
 		return nil
 	}
 
 	log.Infof("Parsing Dirigent metadata: %s", traceFile)
 
-	f, err := os.Open(traceFile)
+	data, err := os.ReadFile(traceFile)
 	if err != nil {
-		log.Error("Failed to open trace memory specification file.")
+		log.Error("Failed to read Dirigent trace file.")
 		return nil
 	}
-	defer f.Close()
 
 	var metadata []common.DirigentMetadata
-	err = gocsv.UnmarshalFile(f, &metadata)
+	err = json.Unmarshal(data, &metadata)
 	if err != nil {
 		log.Fatal("Failed to parse trace runtime specification.")
 	}

diff --git a/pkg/trace/parser_test.go b/pkg/trace/parser_test.go
@@ -133,3 +133,39 @@ func TestParserWrapper(t *testing.T) {
 		t.Error("Unexpected results.")
 	}
 }
+
+func TestDirigentParser(t *testing.T) {
+	data := *parseDirigentMetadata("test_data/dirigent.json", "Dirigent")
+
+	if len(data) != 2 {
+		t.Error("Unexpected results.")
+	}
+
+	if !(data[0].HashFunction == "c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf" &&
+		data[0].Image == "docker.io/vhiveease/relay:latest" &&
+		data[0].Port == 50000 &&
+		data[0].Protocol == "tcp" &&
+		data[0].ScalingUpperBound == 1 &&
+		data[0].ScalingLowerBound == 1 &&
+		data[0].IterationMultiplier == 80 &&
+		data[0].IOPercentage == 0 &&
+		len(data[0].EnvVars) == 1 &&
+		len(data[0].ProgramArgs) == 8) {
+
+		t.Error("Unexpected results.")
+	}
+
+	if !(data[1].HashFunction == "ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091" &&
+		data[1].Image == "docker.io/cvetkovic/dirigent_grpc_function:latest" &&
+		data[1].Port == 80 &&
+		data[1].Protocol == "tcp" &&
+		data[1].ScalingUpperBound == 1 &&
+		data[1].ScalingLowerBound == 0 &&
+		data[1].IterationMultiplier == 80 &&
+		data[1].IOPercentage == 0 &&
+		len(data[1].EnvVars) == 0 &&
+		len(data[1].ProgramArgs) == 0) {
+
+		t.Error("Unexpected results.")
+	}
+}
diff --git a/pkg/trace/test_data/dirigent.json b/pkg/trace/test_data/dirigent.json
@@ -0,0 +1,37 @@
+[
+  {
+    "HashFunction": "c13acdc7567b225971cef2416a3a2b03c8a4d8d154df48afe75834e2f5c59ddf",
+    "Image": "docker.io/vhiveease/relay:latest",
+    "Port": 50000,
+    "Protocol": "tcp",
+    "ScalingUpperBound": 1,
+    "ScalingLowerBound": 1,
+    "IterationMultiplier": 80,
+    "IOPercentage": 0,
+    "EnvVars": [
+      "ENABLE_TRACING=true"
+    ],
+    "ProgramArgs": [
+      "--addr=0.0.0.0:50000",
+      "--function-endpoint-url=0.0.0.0",
+      "--function-endpoint-port=50051",
+      "--function-name=aes-go",
+      "--value=10",
+      "--generator=random",
+      "--lowerBound=1000",
+      "--upperBound=1001"
+    ]
+  },
+  {
+    "HashFunction": "ae8a1640fa932024f59b38a0b001808b5c64612bd60c6f3eb80ba9461ba2d091",
+    "Image": "docker.io/cvetkovic/dirigent_grpc_function:latest",
+    "Port": 80,
+    "Protocol": "tcp",
+    "ScalingUpperBound": 1,
+    "ScalingLowerBound": 0,
+    "IterationMultiplier": 80,
+    "IOPercentage": 0,
+    "EnvVars": [],
+    "ProgramArgs": []
+  }
+]