Adding logic for creating and managing DAG invocations

Signed-off-by: Kway Yi Shen <[email protected]>
vhive-serverless · Nov 26, 2024 · 8cb5f88 · 8cb5f88
2 parents eb570ac + 4fd007a
commit 8cb5f88
Show file tree

Hide file tree

Showing 34 changed files with 448 additions and 98 deletions.
diff --git a/.github/configs/wordlist.txt b/.github/configs/wordlist.txt
@@ -777,4 +777,7 @@ FailNode
 FailureEnabled
 Prepull
 PrepullMode
-async
+async
+AsyncMode
+AsyncResponseURL
+AsyncWaitToCollectMin
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -4,47 +4,47 @@ updates:
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
     ignore:
       - dependency-name: "*"
         update-types: [ "version-update:semver-patch" ]
 
   - package-ecosystem: "gomod"
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
     ignore:
       - dependency-name: "*"
         update-types: [ "version-update:semver-patch" ]
 
   - package-ecosystem: "gomod"
     directory: "/tools/invoker"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
     ignore:
       - dependency-name: "*"
         update-types: [ "version-update:semver-patch" ]
 
   - package-ecosystem: "gomod"
     directory: "/tools/benchmark/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
     ignore:
       - dependency-name: "*"
         update-types: [ "version-update:semver-patch" ]
 
   - package-ecosystem: "gomod"
     directory: "/tools/generateTimeline/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
 
   - package-ecosystem: "gomod"
     directory: "/tools/helloworld"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
     ignore:
       - dependency-name: "*"
         update-types: [ "version-update:semver-patch" ]
@@ -54,23 +54,23 @@ updates:
     # Look for `.github/workflows` in the `root` directory
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
     ignore:
       - dependency-name: "*"
         update-types: [ "version-update:semver-patch" ]
 
   - package-ecosystem: "pip"
     directory: "/tools/trace_synthesizer/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
     ignore:
       - dependency-name: "*"
         update-types: [ "version-update:semver-patch" ]
 
   - package-ecosystem: "pip"
     directory: "/sampler/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
     ignore:
       - dependency-name: "*"
         update-types: [ "version-update:semver-patch" ]
diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml
@@ -18,7 +18,7 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - name: Checkout Sources
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - id: set-matrix
         run: ./utils/golangci-lint/resolve-modules.sh
 
@@ -39,7 +39,7 @@ jobs:
         run: sudo apt update && sudo apt install libsnmp-dev
 
       - name: Checkout code into go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Lint with golangci-lint
         uses: golangci/[email protected]

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -30,7 +30,7 @@ jobs:
             trace_func_go,
           ]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           lfs: "true"
       - name: Checkout LFS objects

diff --git a/.github/workflows/e2e_aws.yaml b/.github/workflows/e2e_aws.yaml
@@ -2,8 +2,6 @@ name: End-to-End AWS Tests
 
 on:
  workflow_dispatch:
- push:
-   branches: [main]
 
 env:
   GOOS: linux
@@ -31,7 +29,7 @@ jobs:
           fi
 
       - name: Checkout GitHub Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: "true"
 

diff --git a/.github/workflows/enable-auto-merge.yml b/.github/workflows/enable-auto-merge.yml
diff --git a/.github/workflows/integration_tests.yaml b/.github/workflows/integration_tests.yaml
@@ -24,7 +24,7 @@ jobs:
 
     steps:
     - name: Check out code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         lfs: 'true'
 

diff --git a/.github/workflows/linters.yaml b/.github/workflows/linters.yaml
@@ -10,7 +10,7 @@ jobs:
     name: Spellcheck
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: rojopolis/[email protected]
       name: Spellcheck
       with:
@@ -19,7 +19,7 @@ jobs:
     name: Commitlint
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: actions/setup-node@v4
@@ -39,7 +39,7 @@ jobs:
     name: LinkCheck
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: gaurav-nelson/github-action-markdown-link-check@v1
         with:
           use-quiet-mode: "yes"

diff --git a/.github/workflows/preprocessing_tests.yaml b/.github/workflows/preprocessing_tests.yaml
@@ -26,13 +26,13 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: 'true'
 
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.10'
 
       - uses: actions/cache@v4
         with:

diff --git a/.github/workflows/tools-tests.yaml b/.github/workflows/tools-tests.yaml
@@ -30,7 +30,7 @@ jobs:
         go-version: 1.22
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Run tests
       working-directory: ${{ matrix.module }}

diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml
@@ -32,7 +32,7 @@ jobs:
               trace,
           ]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           lfs: 'true'
 
@@ -68,4 +68,4 @@ jobs:
         run: pip install -r ./requirements.txt
 
       - name: Run loader tests
-        run: go test -v -cover -race ./pkg/${{ matrix.module }}
+        run: go test -v -cover -race ./pkg/${{ matrix.module }}
diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,7 @@ data/azure
 !data/traces/example/
 pkg/generator/*.png
 pkg/generator/*.txt
+!pkg/generator/requirements.txt
 pkg/driver/*.csv
 pkg/workload/openwhisk/*.zip
 *.log

diff --git a/cmd/config_dirigent_dandelion_rps.json b/cmd/config_dirigent_dandelion_rps.json
@@ -8,6 +8,10 @@
   "DirigentControlPlaneIP": "10.0.1.253:9091",
   "BusyLoopOnSandboxStartup": false,
 
+  "AsyncMode": false,
+  "AsyncResponseURL": "10.0.1.3:8082",
+  "AsyncWaitToCollectMin": 1,
+
   "RpsTarget": 1,
   "RpsColdStartRatioPercentage": 0,
   "RpsCooldownSeconds": 10,

diff --git a/cmd/config_dirigent_dandelion_trace.json b/cmd/config_dirigent_dandelion_trace.json
@@ -8,6 +8,10 @@
   "DirigentControlPlaneIP": "10.0.1.253:9091",
   "BusyLoopOnSandboxStartup": false,
 
+  "AsyncMode": false,
+  "AsyncResponseURL": "10.0.1.3:8082",
+  "AsyncWaitToCollectMin": 1,
+
   "TracePath": "data/traces/example",
   "Granularity": "minute",
   "OutputPathPrefix": "data/out/experiment",

diff --git a/cmd/config_dirigent_rps.json b/cmd/config_dirigent_rps.json
@@ -8,6 +8,10 @@
   "DirigentControlPlaneIP": "10.0.1.253:9092",
   "BusyLoopOnSandboxStartup": false,
 
+  "AsyncMode": false,
+  "AsyncResponseURL": "10.0.1.3:8082",
+  "AsyncWaitToCollectMin": 1,
+
   "RpsTarget": 1,
   "RpsColdStartRatioPercentage": 0,
   "RpsCooldownSeconds": 10,

diff --git a/cmd/config_dirigent_trace.json b/cmd/config_dirigent_trace.json
@@ -8,6 +8,10 @@
   "DirigentControlPlaneIP": "10.0.1.253:9092",
   "BusyLoopOnSandboxStartup": false,
 
+  "AsyncMode": false,
+  "AsyncResponseURL": "10.0.1.253:8082",
+  "AsyncWaitToCollectMin": 1,
+
   "TracePath": "data/traces/example",
   "Granularity": "minute",
   "OutputPathPrefix": "data/out/experiment",

diff --git a/config/requirements.txt b/config/requirements.txt
@@ -1,3 +1,3 @@
 pandas==2.0.3
 psutil==6.1.0
-statsmodels==0.14.0
+statsmodels==0.14.4
diff --git a/config/requirements_dev.txt b/config/requirements_dev.txt
@@ -11,10 +11,10 @@ idna==3.7
 language-selector==0.1
 lxml==5.3.0
 netifaces==0.11.0
-numpy==2.1.3
+numpy==1.26.4
 packaging==24.2
 pandas==2.0.3
-patsy==0.5.2
+patsy==1.0.1
 pssh==2.3.1
 PyGObject==3.48.1
 pymacaroons==0.13.0
@@ -31,7 +31,7 @@ scipy==1.13.0
 six==1.16.0
 soupsieve==2.5
 ssh-import-id==5.11
-statsmodels==0.14.0
+statsmodels==0.14.4
 ubuntu-advantage-tools==20.3
 ufw==0.36
 unattended-upgrades==0.1

diff --git a/docs/configuration.md b/docs/configuration.md
@@ -9,6 +9,9 @@
 | EndpointPort                 | int       | > 0                                                                 | 80                  | Port to be appended to the service URL                                               |
 | DirigentControlPlaneIP       | string    | N/A                                                                 | N/A                 | IP address of the Dirigent control plane (for function deployment)                   |
 | BusyLoopOnSandboxStartup     | bool      | true/false                                                          | false               | Enable artificial delay on sandbox startup                                           |
+| AsyncMode [^6]               | bool      | true/false                                                          | false               | Enable asynchronous invocations in Dirigent                                          |
+| AsyncResponseURL [^6]        | string    | N/A                                                                 | N/A                 | URL from which to collect invocation responses                                       |
+| AsyncWaitToCollectMin [^6]   | int       | >= 0                                                                | 0                   | Time after experiment ends after which to collect invocation results                 |  
 | RpsTarget                    | int       | >= 0                                                                | 0                   | Number of requests per second to issue                                               | 
 | RpsColdStartRatioPercentage  | int       | >= 0 && <= 100                                                      | 0                   | Percentage of cold starts out of specified RPS                                       | 
 | RpsCooldownSeconds           | int       | > 0                                                                 | 0                   | The time it takes for the autoscaler to downscale function (higher for higher RPS)   |
@@ -30,8 +33,8 @@
 | MetricScrapingPeriodSeconds  | int       | > 0                                                                 | 15                  | Period of Prometheus metrics scrapping                                               |
 | GRPCConnectionTimeoutSeconds | int       | > 0                                                                 | 60                  | Timeout for establishing a gRPC connection                                           |
 | GRPCFunctionTimeoutSeconds   | int       | > 0                                                                 | 90                  | Maximum time given to function to execute[^5]                                        |
-| DAGMode                      | bool      | true/false                                                          | false               | Generates DAG workflows iteratively with functions in TracePath [^6]. Frequency and IAT of the DAG follows their respective entry function, while Duration and Memory of each function will follow their respective values in TracePath.                                                                                                              |                            
-| EnableDAGDataset             | bool      | true/false                                                          | true                |  Generate width and depth from data/traces/example/dag_structure.csv[^7]                                                                                                      |
+| DAGMode                      | bool      | true/false                                                          | false               | Generates DAG workflows iteratively with functions in TracePath [^7]. Frequency and IAT of the DAG follows their respective entry function, while Duration and Memory of each function will follow their respective values in TracePath.                                                                                                              |                            
+| EnableDAGDataset             | bool      | true/false                                                          | true                |  Generate width and depth from data/traces/example/dag_structure.csv[^8]                                                                                                      |
 | Width                        | int       | > 0                                                                 | 2                   | Default width of DAG                                                                 |
 | Depth                        | int       | > 0                                                                 | 2                   | Default depth of DAG                                                                 |
 
@@ -53,9 +56,11 @@ this [table](https://cloud.google.com/functions/pricing#compute_time) for Google
 [^5]: Function can execute for at most 15 minutes as in AWS
 Lambda; https://aws.amazon.com/about-aws/whats-new/2018/10/aws-lambda-supports-functions-that-can-run-up-to-15-minutes/
 
-[^6]: The generated DAGs consist of unique functions. The shape of each DAG is determined either ```Width,Depth``` or calculated based on ```EnableDAGDAtaset```.
+[^6]: Dirigent specific
 
-[^7]: A [data sample](https://github.com/icanforce/Orion-OSDI22/blob/main/Public_Dataset/dag_structure.xlsx) of DAG structures has been created based on past Microsoft Azure traces. Width and Depth are determined based on probabilities of this sample.
+[^7]: The generated DAGs consist of unique functions. The shape of each DAG is determined either ```Width,Depth``` or calculated based on ```EnableDAGDAtaset```.
+
+[^8]: A [data sample](https://github.com/icanforce/Orion-OSDI22/blob/main/Public_Dataset/dag_structure.xlsx) of DAG structures has been created based on past Microsoft Azure traces. Width and Depth are determined based on probabilities of this sample.
 
 ---
 

diff --git a/go.mod b/go.mod
@@ -1,6 +1,7 @@
 module github.com/vhive-serverless/loader
 
-go 1.22.0
+go 1.22.7
+
 toolchain go1.22.9
 
 require (