initial work

Signed-off-by: Heinz N. Gies <[email protected]> Signed-off-by: Matthias Wahl <[email protected]> Move startup and stop logic into ClusterNode and RunningClusterNode It looked cleaner from my angle and now we can properly stop the machinery and all its tasks. Unify cluster start and join to one command: cluster start --join ... Signed-off-by: Matthias Wahl <[email protected]> Update to openraft higher-cote branch rebased on openraft main Signed-off-by: Matthias Wahl <[email protected]> Do not connect in TremorNetwork::new_client this call is not guarded by a timeout, so if we cannot connect we are going to hang in the TCP connect call. Signed-off-by: Matthias Wahl <[email protected]> improve docs for tremor_common::file::is_empty Signed-off-by: Matthias Wahl <[email protected]> Redesign flow task and state machine. deploying a flow does not automatically start it. That needs to be done explicitly in a second step. Signed-off-by: Matthias Wahl <[email protected]> Signed-off-by: Heinz N. Gies <[email protected]> Add remove_node operation to the API client Signed-off-by: Matthias Wahl <[email protected]> k8s autoscaling experiment Signed-off-by: Heinz N. Gies <[email protected]> Fail the client if it returns a non-200, so it doesn't try to deserialize an error struct as a regular response Signed-off-by: Matthias Wahl <[email protected]> Refactor our custom raft statemachines into seperate sub-statemachines Signed-off-by: Matthias Wahl <[email protected]> Make sure the network impl reconnects upon errors Signed-off-by: Matthias Wahl <[email protected]> Add some more cluster tests Signed-off-by: Matthias Wahl <[email protected]> Add app id and flow id to common ids and logs Signed-off-by: Matthias Wahl <[email protected]> Update HTTP server library and API Signed-off-by: Heinz N. Gies <[email protected]> Change config to defaults and improve errors Signed-off-by: Heinz N. Gies <[email protected]> Allow connectors access to raft api Signed-off-by: Heinz N. Gies <[email protected]> Remove old API Signed-off-by: Heinz N. Gies <[email protected]> Add clustered kv and API improvements Signed-off-by: Heinz N. Gies <[email protected]> Add cluster::node_id tremorscript function Signed-off-by: Heinz N. Gies <[email protected]> Improve networking and cleanup * Add two examples * Update tremor new to create clustered apps * Fix memory corruption when apps are uninstalled * Try optimizing storage and n2n coms Signed-off-by: Heinz N. Gies <[email protected]> Remove unused deps Signed-off-by: Heinz N. Gies <[email protected]> Remove unused LINE_REGEXP lazy_static in tremor-pipeline to fix build Signed-off-by: Darach Ennis <[email protected]> Fix non consistant reads Signed-off-by: Heinz N. Gies <[email protected]>
tremor-rs · Oct 30, 2023 · 443b008 · 443b008
1 parent 2155413
commit 443b008
Show file tree

Hide file tree

Showing 241 changed files with 11,535 additions and 4,480 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -3,6 +3,7 @@ name: test-on-arm64
 
 platform:
   arch: arm64
+  os: linux
 
 steps:
   - name: test

diff --git a/.github/checks/copyright.sh b/.github/checks/copyright.sh
@@ -4,7 +4,7 @@ count=0
 
 for file in $(find . -name '*.rs' | grep -v '/target')
 do
-    if ! grep 'Copyright \(....-\)\?202[12], The Tremor Team' "$file" > /dev/null
+    if ! grep 'Copyright \(....-\)\?202[123], The Tremor Team' "$file" > /dev/null
     then
         echo "##[error] Copyright missing in $file"
         count=$((count + 1))

diff --git a/.github/checks/safety.sh b/.github/checks/safety.sh
@@ -25,7 +25,7 @@ EOF
 
 
 
-files=$(find . -name '*.rs' | grep -v -f .checkignore | grep -v 'test.rs$')
+files=$(find . -name '*.rs' | grep -v -f .checkignore | grep -v 'test.rs$' | grep -v '/test/')
 
 while getopts hamuiprebldxcft opt; do
     case $opt in

diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml
@@ -31,8 +31,6 @@ jobs:
       - uses: actions/checkout@v3
       - name: Check for unused dependencies (tremor-runtime)
         run: ./.github/checks/deps.sh .
-      - name: Check for unused dependencies (tremor-api)
-        run: ./.github/checks/deps.sh tremor-api
       - name: Check for unused dependencies (tremor-influx)
         run: ./.github/checks/deps.sh tremor-influx
       - name: Check for unused dependencies (tremor-pipeline)

diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
@@ -30,7 +30,7 @@ jobs:
 
     env:
       TREMOR_PATH: "${{ github.workspace }}/tremor-script/lib:${{ github.workspace }}/tremor-cli/tests/lib"
-      RUSTFLAGS: -D warnings -C target-feature=${{ matrix.target_feature }} --cfg tokio_unstable
+      RUSTFLAGS: -D warnings -C target-feature=${{ matrix.target_feature }}
 
     runs-on: ${{ matrix.os }}
     steps:

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -9,7 +9,11 @@
       "request": "launch",
       "name": "Debug tremor-cli",
       "cargo": {
-        "args": ["build", "-p", "tremor-cli"],
+        "args": [
+          "build",
+          "-p",
+          "tremor-cli"
+        ],
         "filter": {
           "name": "tremor",
           "kind": "bin"
@@ -48,54 +52,54 @@
     {
       "type": "lldb",
       "request": "launch",
-      "name": "Debug value::test::obj_eq unit test",
+      "name": "Debug raft::test::cluster_test unit test",
       "cargo": {
         "args": [
           "test",
           "--no-run",
-          "--package",
-          "tremor-value",
-          "value::test::obj_eq"
-        ]
+          "--package=tremor-runtime",
+        ],
+        "filter": {
+          "name": "tremor-runtime",
+          "kind": "lib"
+        }
       },
-      "args": [],
+      "args": [
+        "raft::test::cluster_test"
+      ],
       "cwd": "${workspaceFolder}"
     },
-
     {
       "type": "lldb",
       "request": "launch",
       "name": "Debug script_error unit test",
       "cargo": {
-        "args": ["test", "--no-run", "pp_embed_unrecognized_token2"],
+        "args": [
+          "test",
+          "--no-run",
+          "pp_embed_unrecognized_token2"
+        ],
         "filter": {
           "name": "script_error",
           "kind": "test"
         }
       },
-      "args": ["${selectedText}"],
-      "cwd": "${workspaceFolder}"
-    },
-    {
-      "type": "lldb",
-      "request": "launch",
-      "name": "Debug unit tests in library 'tremor-api'",
-      "cargo": {
-        "args": ["test", "--no-run", "--lib", "--package=tremor-api"],
-        "filter": {
-          "name": "tremor-api",
-          "kind": "lib"
-        }
-      },
-      "args": [],
+      "args": [
+        "${selectedText}"
+      ],
       "cwd": "${workspaceFolder}"
     },
     {
       "type": "lldb",
       "request": "launch",
       "name": "Debug unit tests in library 'tremor-pipeline'",
       "cargo": {
-        "args": ["test", "--no-run", "--lib", "--package=tremor-pipeline"],
+        "args": [
+          "test",
+          "--no-run",
+          "--lib",
+          "--package=tremor-pipeline"
+        ],
         "filter": {
           "name": "tremor-pipeline",
           "kind": "lib"
@@ -109,21 +113,32 @@
       "request": "launch",
       "name": "Debug integration test 'query'",
       "cargo": {
-        "args": ["test", "--no-run", "--test=query"],
+        "args": [
+          "test",
+          "--no-run",
+          "--test=query"
+        ],
         "filter": {
           "name": "query",
           "kind": "test"
         }
       },
-      "args": ["window_mixed_1"],
+      "args": [
+        "window_mixed_1"
+      ],
       "cwd": "${workspaceFolder}"
     },
     {
       "type": "lldb",
       "request": "launch",
       "name": "Debug unit tests in library 'tremor-influx'",
       "cargo": {
-        "args": ["test", "--no-run", "--lib", "--package=tremor-influx"],
+        "args": [
+          "test",
+          "--no-run",
+          "--lib",
+          "--package=tremor-influx"
+        ],
         "filter": {
           "name": "tremor-influx",
           "kind": "lib"
@@ -156,7 +171,12 @@
       "request": "launch",
       "name": "Debug unit tests in library 'tremor_script'",
       "cargo": {
-        "args": ["test", "--no-run", "--lib", "--package=tremor-script"],
+        "args": [
+          "test",
+          "--no-run",
+          "--lib",
+          "--package=tremor-script"
+        ],
         "filter": {
           "name": "tremor_script",
           "kind": "lib"
@@ -170,7 +190,11 @@
       "request": "launch",
       "name": "Debug executable 'tremor-script'",
       "cargo": {
-        "args": ["build", "--bin=tremor-script", "--package=tremor-script"],
+        "args": [
+          "build",
+          "--bin=tremor-script",
+          "--package=tremor-script"
+        ],
         "filter": {
           "name": "tremor-script",
           "kind": "bin"
@@ -203,7 +227,12 @@
       "request": "launch",
       "name": "Debug unit tests in library 'tremor-runtime'",
       "cargo": {
-        "args": ["test", "--no-run", "--lib", "--package=tremor-runtime"],
+        "args": [
+          "test",
+          "--no-run",
+          "--lib",
+          "--package=tremor-runtime"
+        ],
         "filter": {
           "name": "tremor-runtime",
           "kind": "lib"
@@ -350,7 +379,11 @@
       "request": "launch",
       "name": "Debug executable 'tremor-query'",
       "cargo": {
-        "args": ["build", "--bin=tremor-query", "--package=tremor-query"],
+        "args": [
+          "build",
+          "--bin=tremor-query",
+          "--package=tremor-query"
+        ],
         "filter": {
           "name": "tremor-query",
           "kind": "bin"
@@ -383,7 +416,11 @@
       "request": "launch",
       "name": "Debug executable 'tremor'",
       "cargo": {
-        "args": ["build", "--bin=tremor", "--package=tremor-cli"],
+        "args": [
+          "build",
+          "--bin=tremor",
+          "--package=tremor-cli"
+        ],
         "filter": {
           "name": "tremor",
           "kind": "bin"
@@ -397,7 +434,11 @@
       "request": "launch",
       "name": "Debug real-workflow-throughput-json",
       "cargo": {
-        "args": ["build", "--bin=tremor", "--package=tremor-cli"],
+        "args": [
+          "build",
+          "--bin=tremor",
+          "--package=tremor-cli"
+        ],
         "filter": {
           "name": "tremor",
           "kind": "bin"
@@ -416,7 +457,12 @@
       "request": "launch",
       "name": "Release real-workflow-throughput-json",
       "cargo": {
-        "args": ["build", "--bin=tremor", "--package=tremor-cli", "--release"],
+        "args": [
+          "build",
+          "--bin=tremor",
+          "--package=tremor-cli",
+          "--release"
+        ],
         "filter": {
           "name": "tremor",
           "kind": "bin"
@@ -435,7 +481,12 @@
       "request": "launch",
       "name": "Debug unit tests in executable 'tremor'",
       "cargo": {
-        "args": ["test", "--no-run", "--bin=tremor", "--package=tremor-cli"],
+        "args": [
+          "test",
+          "--no-run",
+          "--bin=tremor",
+          "--package=tremor-cli"
+        ],
         "filter": {
           "name": "tremor",
           "kind": "bin"
@@ -449,7 +500,11 @@
       "request": "launch",
       "name": "Debug executable 'tremor-tool'",
       "cargo": {
-        "args": ["build", "--bin=tremor-tool", "--package=tremor-tool"],
+        "args": [
+          "build",
+          "--bin=tremor-tool",
+          "--package=tremor-tool"
+        ],
         "filter": {
           "name": "tremor-tool",
           "kind": "bin"
@@ -478,4 +533,4 @@
       "cwd": "${workspaceFolder}"
     }
   ]
-}
+}