enable lr

CorfuDB · Feb 6, 2024 · 61e3918 · 61e3918
1 parent 3c3a267
commit 61e3918
Show file tree

Hide file tree

Showing 19 changed files with 459 additions and 98 deletions.
diff --git a/cloud/corfu/.swp b/cloud/corfu/.swp
diff --git a/cloud/corfu/cluster_deploy.sh b/cloud/corfu/cluster_deploy.sh
@@ -0,0 +1,146 @@
+#!/bin/zsh
+
+cluster_setup() {
+  k3d cluster delete corfu
+  rm -rf /tmp/k3dvol
+
+  k3d cluster create corfu \
+    --volume /tmp/k3dvol:/tmp/k3dvol \
+    -p "8082:30080@agent:0" \
+    --agents 4
+}
+
+image_imports() {
+  k3d image import corfudb/corfu-server:0.3.2-SNAPSHOT -c corfu
+
+  k3d image import corfudb/corfu-server:0.4.0-SNAPSHOT -c corfu
+  k3d image import corfudb2/corfu-server:0.4.0-SNAPSHOT -c corfu
+
+  k3d image import corfudb/corfu-client-example:latest -c corfu
+}
+
+helm_setup() {
+  helm repo add jetstack https://charts.jetstack.io
+  helm repo update
+  helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version v1.8.0 --set installCRDs=true
+}
+
+init_v1_cluster() {
+  helm install corfu corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb/corfu-server --set image.tag=0.3.2-SNAPSHOT
+  helm install corfu2 corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb/corfu-server --set image.tag=0.3.2-SNAPSHOT --set lr.name="log-replication2" --set nameOverride="corfu2" --set serviceAccount.name="corfu2" --set nameOverride="corfu2" --set fullnameOverride="corfu2" --set cluster.type="sink"
+  sleep 30
+}
+
+init_v2_cluster() {
+  helm install corfu corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb/corfu-server --set image.tag=0.4.0-SNAPSHOT --set version.new=true
+  helm install corfu2 corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb2/corfu-server --set image.tag=0.4.0-SNAPSHOT --set lr.name="log-replication2" --set nameOverride="corfu2" --set serviceAccount.name="corfu2" --set nameOverride="corfu2" --set fullnameOverride="corfu2" --set cluster.type="sink" --set version.new=true
+  sleep 30
+}
+
+cluster_verify() {
+  local lr_version=$1
+
+  # Wait for Corfu to be ready
+  while ! kubectl logs corfu-0 -c corfu | grep -q "DATA"; do
+    echo "Corfu is not ready yet..."
+    sleep 15
+  done
+  echo "Corfu is Ready!!!!"
+
+  # Get the leader of the log replication
+  lr_leader=""
+  while true; do
+    if kubectl logs log-replication-0 | grep -q "acquired"; then
+      lr_leader="log-replication-0"
+      break
+    fi
+    if kubectl logs log-replication-1 | grep -q "acquired"; then
+      lr_leader="log-replication-1"
+      break
+    fi
+    if kubectl logs log-replication-2 | grep -q "acquired"; then
+      lr_leader="log-replication-2"
+      break
+    fi
+  done
+
+  echo "LR Leader is: $lr_leader"
+
+  lr_ready_str=""
+  if [ $lr_version = "V2" ]; then
+    lr_ready_str="Received leadership response from node"
+  else
+    lr_ready_str="Negotiation complete"
+  fi
+
+  # Wait for the log replication leader to be ready
+  while ! kubectl logs $lr_leader | grep -q $lr_ready_str; do
+    echo "LR is not ready yet..."
+    sleep 10
+  done
+
+  echo "Ready to Replicate!!!!"
+}
+
+cluster_upgrade() {
+  helm upgrade corfu corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb/corfu-server --set image.tag=0.4.0-SNAPSHOT --set version.new=true
+  helm upgrade corfu2 corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb2/corfu-server --set image.tag=0.4.0-SNAPSHOT --set lr.name="log-replication2" --set nameOverride="corfu2" --set serviceAccount.name="corfu2" --set nameOverride="corfu2" --set fullnameOverride="corfu2" --set cluster.type="sink" --set version.new=true
+
+  while kubectl describe pods --all-namespaces | grep -q "0.3.2-SNAPSHOT"; do
+    echo "Waiting for pods to be re-imaged..."
+    sleep 10
+  done
+
+  echo "Cluster upgrade complete!!!"
+}
+
+
+cluster_test() {
+  echo "Writing Data To Source..."
+  helm install corfu-client corfu-client-example-helm --set tls.enabled=false --set jobs.job=1
+
+  while ! kubectl get pods -o wide | grep corfu-client | grep -q Completed; do
+    echo "Waiting for test to finish..."
+    sleep 5
+  done
+
+
+  helm uninstall corfu-client
+  while kubectl get pods -o wide | grep -q corfu-client; do
+    echo "Removing test agent..."
+    sleep 5
+  done
+
+  echo "Test Complete!!!"
+}
+
+cluster_test_validate() {
+    echo "Starting test validation!!!"
+    helm install corfu-client corfu-client-example-helm --set tls.enabled=false --set jobs.job=2
+
+    while ! kubectl get pods -o wide | grep corfu-client | grep -q Completed; do
+      echo "Waiting for validation to complete..."
+      sleep 5
+    done
+
+    helm uninstall corfu-client
+    while kubectl get pods -o wide | grep -q corfu-client; do
+      echo "Removing test agent..."
+      sleep 5
+    done
+
+    echo "Validation Complete!!!"
+}
+
+cluster_setup
+image_imports
+helm_setup
+
+init_v1_cluster
+cluster_verify V1
+cluster_test
+
+cluster_upgrade
+
+ cluster_verify V2
+ cluster_test_validate
diff --git a/cloud/corfu/corfu-client-example-helm/templates/job.yaml b/cloud/corfu/corfu-client-example-helm/templates/job.yaml
@@ -9,12 +9,12 @@ spec:
         containers:
           - name: corfu-client
             image: corfudb/corfu-client-example:latest
-            imagePullPolicy: Always
+            imagePullPolicy: Never
             command:
               - "sh"
               - "-c"
               - |
-                java -cp *.jar org.corfudb.cloud.runtime.example.Main {{ .Values.corfuEndpoint }}
+                java -cp *.jar org.corfudb.cloud.runtime.example.Main {{ .Values.corfuEndpoint }} {{ .Values.jobs.job }}
                 {{- if .Values.tls.enabled }} \
                 /certs/keystore.jks /password/password /certs/truststore.jks /password/password
                 {{- end }}

diff --git a/cloud/corfu/corfu-client-example-helm/values.yaml b/cloud/corfu/corfu-client-example-helm/values.yaml
@@ -2,9 +2,12 @@ image:
   registry: "docker.io"
   repository: "corfudb/corfu-client-example"
   tag: "latest"
-  pullPolicy: Always
+  pullPolicy: Never
 corfuEndpoint: "corfu-0.corfu-headless.default.svc.cluster.local"
 tls:
   enabled: true
   certificateName: corfu-certificate-tls
   passwordName: corfu-password
+jobs:
+  # 1: test, 2: validate, 3: test & validate
+  job: 3
diff --git a/cloud/corfu/corfu-client-example/build.gradle.kts b/cloud/corfu/corfu-client-example/build.gradle.kts
@@ -8,15 +8,15 @@ repositories {
     mavenLocal()
     mavenCentral()
 
-    maven {
-        name = "GitHubPackages"
-        url = uri("https://maven.pkg.github.com/corfudb/corfudb")
-        // For accessing GitHub Secrets in CorfuDB repo
-        credentials {
-            username = System.getenv("PKG_USERNAME")
-            password = System.getenv("PUBLISH_TOKEN")
-        }
-    }
+//    maven {
+//        name = "GitHubPackages"
+//        url = uri("https://maven.pkg.github.com/corfudb/corfudb")
+//        // For accessing GitHub Secrets in CorfuDB repo
+//        credentials {
+//            username = System.getenv("PKG_USERNAME")
+//            password = System.getenv("PUBLISH_TOKEN")
+//        }
+//    }
 }
 
 val corfuVersion = "0.3.2-SNAPSHOT"
@@ -29,6 +29,8 @@ dependencies {
     implementation("org.corfudb:runtime:${corfuVersion}") {
         exclude(group = "io.netty", module = "netty-tcnative")
     }
+    implementation("org.corfudb:infrastructure:${corfuVersion}")
+    implementation("com.github.luben:zstd-jni:1.4.8-1")
 
     testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}")
 }