diff --git a/cloud/corfu/.swp b/cloud/corfu/.swp new file mode 100644 index 00000000..cfd55278 Binary files /dev/null and b/cloud/corfu/.swp differ diff --git a/cloud/corfu/cluster_deploy.sh b/cloud/corfu/cluster_deploy.sh new file mode 100755 index 00000000..13286e63 --- /dev/null +++ b/cloud/corfu/cluster_deploy.sh @@ -0,0 +1,143 @@ +#!/bin/zsh + +cluster_setup() { + k3d cluster delete corfu + rm -rf /tmp/k3dvol + + k3d cluster create corfu \ + --volume /tmp/k3dvol:/tmp/k3dvol \ + -p "8082:30080@agent:0" \ + --agents 4 +} + +image_imports() { + k3d image import corfudb/corfu-server:0.3.2-SNAPSHOT -c corfu + k3d image import corfudb/corfu-server:0.4.0-SNAPSHOT -c corfu + k3d image import corfudb/corfu-client-example:latest -c corfu +} + +helm_setup() { + helm repo add jetstack https://charts.jetstack.io + helm repo update + helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version v1.8.0 --set installCRDs=true +} + +init_v1_cluster() { + helm install corfu corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb/corfu-server --set image.tag=0.3.2-SNAPSHOT + helm install corfu2 corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb/corfu-server --set image.tag=0.3.2-SNAPSHOT --set lr.name="log-replication2" --set nameOverride="corfu2" --set serviceAccount.name="corfu2" --set nameOverride="corfu2" --set fullnameOverride="corfu2" --set cluster.type="sink" + sleep 30 +} + +init_v2_cluster() { + helm install corfu corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb/corfu-server --set image.tag=0.4.0-SNAPSHOT --set version.new=true + helm install corfu2 corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb/corfu-server --set image.tag=0.4.0-SNAPSHOT --set lr.name="log-replication2" --set nameOverride="corfu2" --set serviceAccount.name="corfu2" --set nameOverride="corfu2" --set fullnameOverride="corfu2" --set cluster.type="sink" --set version.new=true + sleep 30 +} + +cluster_verify() { + local lr_version=$1 + + # Wait for Corfu to be ready + while ! kubectl logs corfu-0 -c corfu | grep -q "DATA"; do + echo "Corfu is not ready yet..." + sleep 15 + done + echo "Corfu is Ready!!!!" + + # Get the leader of the log replication + lr_leader="" + while true; do + if kubectl logs log-replication-0 | grep -q "acquired"; then + lr_leader="log-replication-0" + break + fi + if kubectl logs log-replication-1 | grep -q "acquired"; then + lr_leader="log-replication-1" + break + fi + if kubectl logs log-replication-2 | grep -q "acquired"; then + lr_leader="log-replication-2" + break + fi + done + + echo "LR Leader is: $lr_leader" + + lr_ready_str="" + if [ $lr_version = "V2" ]; then + lr_ready_str="Received leadership response from node" + else + lr_ready_str="Negotiation complete" + fi + + # Wait for the log replication leader to be ready + while ! kubectl logs $lr_leader | grep -q $lr_ready_str; do + echo "LR is not ready yet..." + sleep 10 + done + + echo "Ready to Replicate!!!!" +} + +cluster_upgrade() { + helm upgrade corfu corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb/corfu-server --set image.tag=0.4.0-SNAPSHOT --set version.new=true + helm upgrade corfu2 corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb/corfu-server --set image.tag=0.4.0-SNAPSHOT --set lr.name="log-replication2" --set nameOverride="corfu2" --set serviceAccount.name="corfu2" --set nameOverride="corfu2" --set fullnameOverride="corfu2" --set cluster.type="sink" --set version.new=true + + while kubectl describe pods --all-namespaces | grep -q "0.3.2-SNAPSHOT"; do + echo "Waiting for pods to be re-imaged..." + sleep 10 + done + + echo "Cluster upgrade complete!!!" +} + + +cluster_test() { + echo "Writing Data To Source..." + helm install corfu-client corfu-client-example-helm --set tls.enabled=false --set jobs.job=1 + + while ! kubectl get pods -o wide | grep corfu-client | grep -q Completed; do + echo "Waiting for test to finish..." + sleep 5 + done + + + helm uninstall corfu-client + while kubectl get pods -o wide | grep -q corfu-client; do + echo "Removing test agent..." + sleep 5 + done + + echo "Test Complete!!!" +} + +cluster_test_validate() { + echo "Starting test validation!!!" + helm install corfu-client corfu-client-example-helm --set tls.enabled=false --set jobs.job=2 + + while ! kubectl get pods -o wide | grep corfu-client | grep -q Completed; do + echo "Waiting for validation to complete..." + sleep 5 + done + + helm uninstall corfu-client + while kubectl get pods -o wide | grep -q corfu-client; do + echo "Removing test agent..." + sleep 5 + done + + echo "Validation Complete!!!" +} + +cluster_setup +image_imports +helm_setup + +init_v1_cluster +cluster_verify V1 +cluster_test + +cluster_upgrade + +cluster_verify V2 +cluster_test_validate \ No newline at end of file diff --git a/cloud/corfu/corfu-client-example-helm/templates/job.yaml b/cloud/corfu/corfu-client-example-helm/templates/job.yaml index a5715db7..75b1ef03 100644 --- a/cloud/corfu/corfu-client-example-helm/templates/job.yaml +++ b/cloud/corfu/corfu-client-example-helm/templates/job.yaml @@ -9,12 +9,12 @@ spec: containers: - name: corfu-client image: corfudb/corfu-client-example:latest - imagePullPolicy: Always + imagePullPolicy: Never command: - "sh" - "-c" - | - java -cp *.jar org.corfudb.cloud.runtime.example.Main {{ .Values.corfuEndpoint }} + java -cp *.jar org.corfudb.cloud.runtime.example.Main {{ .Values.corfuEndpoint }} {{ .Values.jobs.job }} {{- if .Values.tls.enabled }} \ /certs/keystore.jks /password/password /certs/truststore.jks /password/password {{- end }} diff --git a/cloud/corfu/corfu-client-example-helm/values.yaml b/cloud/corfu/corfu-client-example-helm/values.yaml index c08fdd0c..b119857c 100644 --- a/cloud/corfu/corfu-client-example-helm/values.yaml +++ b/cloud/corfu/corfu-client-example-helm/values.yaml @@ -2,9 +2,12 @@ image: registry: "docker.io" repository: "corfudb/corfu-client-example" tag: "latest" - pullPolicy: Always + pullPolicy: Never corfuEndpoint: "corfu-0.corfu-headless.default.svc.cluster.local" tls: enabled: true certificateName: corfu-certificate-tls passwordName: corfu-password +jobs: + # 1: test, 2: validate, 3: test & validate + job: 3 diff --git a/cloud/corfu/corfu-client-example/build.gradle.kts b/cloud/corfu/corfu-client-example/build.gradle.kts index ab609368..5325b827 100644 --- a/cloud/corfu/corfu-client-example/build.gradle.kts +++ b/cloud/corfu/corfu-client-example/build.gradle.kts @@ -8,15 +8,15 @@ repositories { mavenLocal() mavenCentral() - maven { - name = "GitHubPackages" - url = uri("https://maven.pkg.github.com/corfudb/corfudb") - // For accessing GitHub Secrets in CorfuDB repo - credentials { - username = System.getenv("PKG_USERNAME") - password = System.getenv("PUBLISH_TOKEN") - } - } +// maven { +// name = "GitHubPackages" +// url = uri("https://maven.pkg.github.com/corfudb/corfudb") +// // For accessing GitHub Secrets in CorfuDB repo +// credentials { +// username = System.getenv("PKG_USERNAME") +// password = System.getenv("PUBLISH_TOKEN") +// } +// } } val corfuVersion = "0.3.2-SNAPSHOT" @@ -29,6 +29,8 @@ dependencies { implementation("org.corfudb:runtime:${corfuVersion}") { exclude(group = "io.netty", module = "netty-tcnative") } + implementation("org.corfudb:infrastructure:${corfuVersion}") + implementation("com.github.luben:zstd-jni:1.4.8-1") testImplementation("org.junit.jupiter:junit-jupiter-engine:${junitVersion}") } diff --git a/cloud/corfu/corfu-client-example/src/main/java/org.corfudb.cloud.runtime.example/Main.java b/cloud/corfu/corfu-client-example/src/main/java/org.corfudb.cloud.runtime.example/Main.java index da23444f..90a29c5d 100644 --- a/cloud/corfu/corfu-client-example/src/main/java/org.corfudb.cloud.runtime.example/Main.java +++ b/cloud/corfu/corfu-client-example/src/main/java/org.corfudb.cloud.runtime.example/Main.java @@ -1,14 +1,28 @@ package org.corfudb.cloud.runtime.example; import com.google.common.reflect.TypeToken; +import java.util.Map; +import org.corfudb.infrastructure.logreplication.proto.Sample; +import org.corfudb.infrastructure.logreplication.proto.LogReplicationMetadata.LogReplicationMetadataKey; +import org.corfudb.infrastructure.logreplication.proto.LogReplicationMetadata.LogReplicationMetadataVal; +import org.corfudb.runtime.CorfuOptions; import org.corfudb.runtime.CorfuRuntime; -import org.corfudb.runtime.collections.PersistentCorfuTable; +import org.corfudb.runtime.collections.CorfuTable; +import org.corfudb.runtime.collections.CorfuStore; +import org.corfudb.runtime.collections.CorfuRecord; +import org.corfudb.runtime.collections.Table; +import org.corfudb.runtime.collections.TableOptions; +import org.corfudb.runtime.collections.TxnContext; +import org.corfudb.runtime.CorfuStoreMetadata.TableName; +import org.corfudb.runtime.CorfuStoreMetadata.TableDescriptors; +import org.corfudb.runtime.CorfuStoreMetadata.TableMetadata; +import org.corfudb.runtime.view.ObjectsView; import org.corfudb.util.NodeLocator; +import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.time.Duration; import java.util.Collections; -import java.util.Map; - +import java.util.concurrent.TimeUnit; /** * This tutorial demonstrates a simple Corfu application. @@ -32,7 +46,7 @@ private static CorfuRuntime getRuntimeAndConnect(String host, boolean tlsEnabled CorfuRuntime.CorfuRuntimeParameters.CorfuRuntimeParametersBuilder builder = CorfuRuntime.CorfuRuntimeParameters .builder() - .connectionTimeout(Duration.ofSeconds(2)) + .connectionTimeout(Duration.ofSeconds(20)) .layoutServers(Collections.singletonList(loc)); if (tlsEnabled) { builder.tlsEnabled(tlsEnabled) @@ -47,7 +61,7 @@ private static CorfuRuntime getRuntimeAndConnect(String host, boolean tlsEnabled } // Sample code - public static void main(String[] args) { + public static void main(String[] args) throws Exception { System.out.println("Start application. Got args: " + Arrays.toString(args)); // Parse the options given, using docopt. /* @@ -62,6 +76,7 @@ public static void main(String[] args) { * which is a Java object that contains all of the Corfu utilities exposed to applications. */ String ip = "localhost"; + int job = 3; boolean tlsEnabled = false; String keyStore = ""; String keyStorePassword = ""; @@ -72,63 +87,144 @@ public static void main(String[] args) { ip = args[0]; } if (args.length >= 2) { - keyStore = args[1]; + job = Integer.parseInt(args[1]); } if (args.length >= 3) { - keyStorePassword = args[2]; + keyStore = args[2]; } if (args.length >= 4) { - trustStore = args[3]; + keyStorePassword = args[3]; } if (args.length >= 5) { - trustStorePassword = args[4]; + trustStore = args[4]; + } + if (args.length >= 6) { + trustStorePassword = args[5]; } - if (args.length == 5) { + if (args.length == 6) { tlsEnabled = true; } + if (job >= 1) { + if (job != 2) { + test(ip, tlsEnabled, keyStore, keyStorePassword, trustStore, trustStorePassword); + } + validate(ip, tlsEnabled, keyStore, keyStorePassword, trustStore, trustStorePassword); + } + } + public static void test(String ip, boolean tlsEnabled, String keyStore, String keyStorePassword, String trustStore, String trustStorePassword) throws Exception { + CorfuRuntime runtimeSource = getRuntimeAndConnect(ip, tlsEnabled, keyStore, keyStorePassword, trustStore, trustStorePassword); + CorfuRuntime runtimeSink = getRuntimeAndConnect("corfu2-0.corfu2-headless.default.svc.cluster.local", + tlsEnabled, keyStore, keyStorePassword, trustStore, trustStorePassword); - CorfuRuntime runtime = getRuntimeAndConnect(ip, tlsEnabled, keyStore, keyStorePassword, trustStore, trustStorePassword); + CorfuStore corfuStoreSource = new CorfuStore(runtimeSource); + CorfuStore corfuStoreSink = new CorfuStore(runtimeSink); - /** - * Obviously, this application is not doing much yet, - * but you can already invoke getRuntimeAndConnect to test if you can connect to a deployed Corfu service. - * - * Above, you will need to point it to a host and port which is running the service. - * See {@link https://github.com/CorfuDB/CorfuDB} for instructions on how to deploy Corfu. - */ + String NAMESPACE = "LR-Test"; + String streamA = "MyTestTable"; - /** - * Next, we will illustrate how to declare a Java object backed by a Corfu Stream. - * A Corfu Stream is a log dedicated specifically to the history of updates of one object. - * We will instantiate a stream by giving it a name "A", - * and then instantiate an object by specifying its class - */ - Map map = runtime.getObjectsView() - .build() - .setStreamName("A") // stream name - .setTypeToken(new TypeToken>() {}) - .open(); // instantiate the object! + Table mapA = corfuStoreSource.openTable( + NAMESPACE, + streamA, + Sample.StringKey.class, + Sample.IntValue.class, + Sample.Metadata.class, + TableOptions.builder().schemaOptions( + CorfuOptions.SchemaOptions.newBuilder() + .setIsFederated(true) + .build()) + .build() + ); - /** - * The magic has already happened! map is an in-memory view of a shared map, backed by the Corfu log. - * The application can perform put and get on this map from different application instances, - * crash and restart applications, and so on. - * The map will persist and be consistent across all applications. - * - * For example, try the following code repeatedly in a sequence, in between run/exit, - * from multiple instances, and see the different interleaving of values that result. - */ - Integer previous = map.get("a"); - if (previous == null) { - System.out.println("This is the first time we were run!"); - map.put("a", 1); + Table mapASink = corfuStoreSink.openTable( + NAMESPACE, + streamA, + Sample.StringKey.class, + Sample.IntValue.class, + Sample.Metadata.class, + TableOptions.builder().schemaOptions( + CorfuOptions.SchemaOptions.newBuilder() + .setIsFederated(true) + .build()) + .build() + ); + + int totalEntries = 200; + int startIndex = 0; + + int maxIndex = totalEntries + startIndex; + for (int i = startIndex; i < maxIndex; i++) { + try (TxnContext txn = corfuStoreSource.txn(NAMESPACE)) { + txn.putRecord(mapA, Sample.StringKey.newBuilder().setKey(String.valueOf(i)).build(), + Sample.IntValue.newBuilder().setValue(i).build(), null); + txn.commit(); + } + } + + try (TxnContext txn = corfuStoreSource.txn(NAMESPACE)) { + int tableSize = txn.getTable(streamA).count(); + System.out.println("Size of source table after adding entries is: " + tableSize); + txn.commit(); } - else { - map.put("a", ++previous); - System.out.println("This is the " + previous + " time we were run!"); + } + + public static void validate(String ip, boolean tlsEnabled, String keyStore, String keyStorePassword, String trustStore, String trustStorePassword) throws Exception { + CorfuRuntime runtimeSource = getRuntimeAndConnect(ip, tlsEnabled, keyStore, keyStorePassword, trustStore, trustStorePassword); + CorfuRuntime runtimeSink = getRuntimeAndConnect("corfu2-0.corfu2-headless.default.svc.cluster.local", + tlsEnabled, keyStore, keyStorePassword, trustStore, trustStorePassword); + + CorfuStore corfuStoreSource = new CorfuStore(runtimeSource); + CorfuStore corfuStoreSink = new CorfuStore(runtimeSink); + + String NAMESPACE = "LR-Test"; + String streamA = "MyTestTable"; + + Table mapA = corfuStoreSource.openTable( + NAMESPACE, + streamA, + Sample.StringKey.class, + Sample.IntValue.class, + Sample.Metadata.class, + TableOptions.builder().schemaOptions( + CorfuOptions.SchemaOptions.newBuilder() + .setIsFederated(true) + .build()) + .build() + ); + + Table mapASink = corfuStoreSink.openTable( + NAMESPACE, + streamA, + Sample.StringKey.class, + Sample.IntValue.class, + Sample.Metadata.class, + TableOptions.builder().schemaOptions( + CorfuOptions.SchemaOptions.newBuilder() + .setIsFederated(true) + .build()) + .build() + ); + + while (true) { + try (TxnContext txn = corfuStoreSource.txn(NAMESPACE)) { + int tableSize = txn.getTable(streamA).count(); + System.out.println("Size of source table is: " + tableSize); + txn.commit(); + } + + try (TxnContext txn = corfuStoreSink.txn(NAMESPACE)) { + int tableSize = txn.getTable(streamA).count(); + + System.out.println("Size of sink table is: " + tableSize); + txn.commit(); + + if (tableSize == 200) { + break; + } + } + TimeUnit.SECONDS.sleep(5); } } } \ No newline at end of file diff --git a/cloud/corfu/corfu/config/corfu_plugin_config.properties b/cloud/corfu/corfu/config/corfu_plugin_config.properties index 81e90c01..9065d87f 100644 --- a/cloud/corfu/corfu/config/corfu_plugin_config.properties +++ b/cloud/corfu/corfu/config/corfu_plugin_config.properties @@ -1,15 +1,27 @@ -# Transport Plugin Configuration transport_adapter_JAR_path=/app/corfu.jar +GRPC_transport_adapter_server_class_name=org.corfudb.infrastructure.logreplication.transport.sample.GRPCLogReplicationServerChannelAdapter +GRPC_transport_adapter_client_class_name=org.corfudb.infrastructure.logreplication.transport.sample.GRPCLogReplicationClientChannelAdapter +NETTY_transport_adapter_server_class_name=org.corfudb.infrastructure.logreplication.transport.sample.NettyLogReplicationServerChannelAdapter +NETTY_transport_adapter_client_class_name=org.corfudb.infrastructure.logreplication.transport.sample.NettyLogReplicationClientChannelAdapter + +# Transport plugin selector +transport_plugin_selector_JAR_path=/app/corfu.jar +transport_plugin_selector_class_name=org.corfudb.infrastructure.logreplication.infrastructure.plugins.DefaultTransportPluginSelector + +# Transport Plugin Configuration transport_adapter_server_class_name=org.corfudb.infrastructure.logreplication.transport.sample.GRPCLogReplicationServerChannelAdapter transport_adapter_client_class_name=org.corfudb.infrastructure.logreplication.transport.sample.GRPCLogReplicationClientChannelAdapter + # Stream Fetcher Plugin Configuration stream_fetcher_plugin_JAR_path=/app/corfu.jar -stream_fetcher_plugin_class_name=org.corfudb.infrastructure.logreplication.infrastructure.plugins.DefaultLogReplicationConfigAdapter +stream_fetcher_plugin_class_name=org.corfudb.infrastructure.logreplication.infrastructure.plugins.DefaultLogReplicationCloudConfigAdapter + # Topology Manager Plugin Configuration topology_manager_adapter_JAR_path=/app/corfu.jar -topology_manager_adapter_class_name=org.corfudb.infrastructure.logreplication.infrastructure.plugins.DefaultClusterManager +topology_manager_adapter_class_name=org.corfudb.infrastructure.logreplication.infrastructure.plugins.DefaultCloudClusterManager + # Snapshot Sync Configuration (Plugin) snapshot_sync_plugin_JAR_path=/app/corfu.jar snapshot_sync_plugin_class_name=org.corfudb.infrastructure.logreplication.infrastructure.plugins.DefaultSnapshotSyncPlugin -saas_endpoint=corfu:9000 -local_node_id_path=/usr/share/corfu/conf/serial_number \ No newline at end of file + +saas_endpoint=corfu-0.corfu-headless.default.svc.cluster.local:9000 \ No newline at end of file diff --git a/cloud/corfu/corfu/files/init_layout.py b/cloud/corfu/corfu/files/init_layout.py index 27c703bd..b8a99e51 100644 --- a/cloud/corfu/corfu/files/init_layout.py +++ b/cloud/corfu/corfu/files/init_layout.py @@ -20,7 +20,7 @@ def generate_layout(args): layout_template["layoutServers"] = fqdn_list layout_template["sequencers"] = fqdn_list layout_template["segments"][0]["stripes"][0]["logServers"] = fqdn_list - layout_template["clusterId"] = "123e4567-e89b-12d3-a456-556642440000" + layout_template["clusterId"] = "456e4567-e89b-12d3-a456-556642440001" if args.type == "source" else "456e4567-e89b-12d3-a456-556642440002" # print layout print("Generated layout:") @@ -39,6 +39,7 @@ def main(): parser.add_argument('--replica', '-r', type=int, required=True, help='The replica of Corfu cluster.') parser.add_argument('--statefulset', type=str, default='corfu', help='Corfu statefulset name.') parser.add_argument('--headless', type=str, default='corfu-headless', required=True, help='Corfu headless service name.') + parser.add_argument('--type', type=str, default='source', required=True, help='Source or sink.') args = parser.parse_args() generate_layout(args) diff --git a/cloud/corfu/corfu/serial_number/serial_number b/cloud/corfu/corfu/serial_number/serial_number new file mode 100644 index 00000000..1c02b1a7 --- /dev/null +++ b/cloud/corfu/corfu/serial_number/serial_number @@ -0,0 +1 @@ +serial=B1310142-311D-B7AE-D6A7-F66DEE607871 \ No newline at end of file diff --git a/cloud/corfu/corfu/templates/Deployment.yaml b/cloud/corfu/corfu/templates/Deployment.yaml new file mode 100644 index 00000000..650f0702 --- /dev/null +++ b/cloud/corfu/corfu/templates/Deployment.yaml @@ -0,0 +1,53 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "corfu.service.lr" . }} + labels: + {{- include "corfu.labels" . | indent 4 }} +spec: + replicas: {{ include "corfu.replicas" . }} + updateStrategy: + type: RollingUpdate + serviceName: {{ include "corfu.service.lr" . }} + selector: + matchLabels: + {{- include "corfu.selectors.lr" . | nindent 6 }} + template: + metadata: + labels: + {{- include "corfu.selectors.lr" . | nindent 8 }} + spec: + containers: + - name: {{ include "corfu.service.lr" . }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.lr.port }} + protocol: TCP + env: + - name: CONFIG_FILE_PATH + value: "/usr/share/corfu/conf/corfu_replication_config.properties" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + VolumeMounts: + - name: log-dir + mountPath: /var/log/corfu-log-replication + - name: config-dir + mountPath: /config/corfu-log-replication + - name: lr + mountPath: /common/configs/ + command: + - "sh" + - "-c" + - "java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/corfu/corfu_oom.hprof -XX:+HeapDumpOnOutOfMemoryError -Djdk.nio.maxCachedBufferSize=1048576 -Dio.netty.recycler.maxCapacityPerThread=0 -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -Djava.io.tmpdir=/image/corfu-server/temp -cp /app/corfu.jar:/opt/vmware/log-replication/log-replication_deploy.jar -Djava.io.tmpdir=/tmp org.corfudb.infrastructure.CorfuServer --plugin=/usr/share/corfu/conf/corfu_plugin_config.properties -d DEBUG 9010 -m" \ No newline at end of file diff --git a/cloud/corfu/corfu/templates/_helpers.tpl b/cloud/corfu/corfu/templates/_helpers.tpl index ef4f144d..999b445d 100644 --- a/cloud/corfu/corfu/templates/_helpers.tpl +++ b/cloud/corfu/corfu/templates/_helpers.tpl @@ -1,5 +1,5 @@ {{- define "corfu.name" -}} -corfu +{{ .Values.nameOverride }} {{- end }} {{- define "corfu.fullname" -}} @@ -30,6 +30,14 @@ type: {{ .Values.type | default "config" | quote }} app.kubernetes.io/name: {{ include "corfu.fullname" . }} {{- end }} +{{- define "corfu.service.lr" -}} +{{- .Values.lr.name }} +{{- end }} + +{{- define "corfu.selectors.lr" -}} +app.kubernetes.io/name: {{ include "corfu.service.lr" . }} +{{- end }} + {{/* If replicas tag is defined in its own helm chart values.yaml it will always override the global value. If not, we will use the global value. diff --git a/cloud/corfu/corfu/templates/certificate.yaml b/cloud/corfu/corfu/templates/certificate.yaml index 1ecb8133..20ff3337 100644 --- a/cloud/corfu/corfu/templates/certificate.yaml +++ b/cloud/corfu/corfu/templates/certificate.yaml @@ -1,5 +1,5 @@ {{- if .Values.tls.certificate.enabled }} -{{- $dns := printf "*.corfu-headless.%s.svc.cluster.local" .Release.Namespace }} +{{- $dns := printf "*.%s-headless.%s.svc.cluster.local" .Values.nameOverride .Release.Namespace }} apiVersion: cert-manager.io/v1 kind: Certificate metadata: @@ -25,6 +25,7 @@ spec: - client auth dnsNames: - corfu + - corfu2 - "{{ $dns }}" issuerRef: name: {{ .Values.tls.certificate.issuer.name }} diff --git a/cloud/corfu/corfu/templates/configmap-lr.yaml b/cloud/corfu/corfu/templates/configmap-lr.yaml new file mode 100644 index 00000000..a6254cd1 --- /dev/null +++ b/cloud/corfu/corfu/templates/configmap-lr.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "corfu.fullname" . }}-corfu-configs-lr + labels: + {{- include "corfu.labels" . | indent 4 }} +data: + {{- with .Files.Glob "serial_number/*" }} + {{- .AsConfig | nindent 2 }} + {{- end }} \ No newline at end of file diff --git a/cloud/corfu/corfu/templates/service-lr.yaml b/cloud/corfu/corfu/templates/service-lr.yaml new file mode 100644 index 00000000..e292939b --- /dev/null +++ b/cloud/corfu/corfu/templates/service-lr.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "corfu.service.lr" . }} + labels: + {{- include "corfu.labels" . | indent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.lr.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "corfu.selectors.lr" . | nindent 4 }} \ No newline at end of file diff --git a/cloud/corfu/corfu/templates/service.yaml b/cloud/corfu/corfu/templates/service.yaml index e542a21a..e51036ec 100644 --- a/cloud/corfu/corfu/templates/service.yaml +++ b/cloud/corfu/corfu/templates/service.yaml @@ -33,7 +33,7 @@ spec: apiVersion: v1 kind: Service metadata: - name: {{ include "corfu.fullname" . }}-headless + name: {{ include "corfu.name" . }}-headless labels: {{- include "corfu.labels" . | indent 4 }} spec: diff --git a/cloud/corfu/corfu/templates/serviceaccount.yaml b/cloud/corfu/corfu/templates/serviceaccount.yaml index bc0d7b6b..5fa800f3 100644 --- a/cloud/corfu/corfu/templates/serviceaccount.yaml +++ b/cloud/corfu/corfu/templates/serviceaccount.yaml @@ -26,7 +26,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ include "corfu.name" . }}-role-binding + name: {{ include "corfu.serviceAccountName" . }}-role-binding labels: {{- include "corfu.labels" . | indent 4 }} subjects: diff --git a/cloud/corfu/corfu/templates/statefulset.yaml b/cloud/corfu/corfu/templates/statefulset.yaml index 7c2296eb..1dde49b2 100644 --- a/cloud/corfu/corfu/templates/statefulset.yaml +++ b/cloud/corfu/corfu/templates/statefulset.yaml @@ -5,8 +5,10 @@ metadata: labels: {{- include "corfu.labels" . | indent 4 }} spec: - serviceName: {{ include "corfu.fullname" . }}-headless + serviceName: {{ include "corfu.name" . }}-headless replicas: {{ include "corfu.replicas" . }} + updateStrategy: + type: RollingUpdate podManagementPolicy: Parallel selector: matchLabels: @@ -29,7 +31,7 @@ spec: {{- toYaml .Values.podSecurityContext | nindent 8 }} initContainers: - name: create-layout - image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} command: - "/bin/bash" @@ -37,7 +39,7 @@ spec: - | set -e python3 /etc/corfu/configs/init_layout.py --template /etc/corfu/configs/layout_template.json --layout /usr/share/configs/layout.json \ - --replica {{ include "corfu.replicas" . }} --statefulset {{ include "corfu.name" . }} --headless {{ include "corfu.name" . }}-headless --port {{ .Values.service.port }} + --replica {{ include "corfu.replicas" . }} --statefulset {{ include "corfu.fullname" . }} --headless {{ include "corfu.name" . }}-headless --port {{ .Values.service.port }} --type {{ .Values.cluster.type }} env: - name: POD_NAMESPACE valueFrom: @@ -46,11 +48,11 @@ spec: volumeMounts: - name: workdir mountPath: /usr/share/configs - - name: corfu-layout-configs + - name: {{ include "corfu.fullname" . }}-layout-configs mountPath: /etc/corfu/configs containers: - name: bootstrap-corfu - image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} volumeMounts: - name: workdir @@ -74,7 +76,8 @@ spec: echo "Current node is bootstrapped. Nothing to do." sleep inf fi - + + {{- if .Values.version.new }} {{- if .Values.tls.enabled }} certs_truststore_file=/certs/truststore.jks if test -f "$certs_truststore_file"; @@ -90,7 +93,7 @@ spec: --enable-tls --keystore=/certs/keystore.jks --truststore=/certs/truststore.jks \ --keystore-password-file=/password/password --truststore-password-file=/password/password \ {{- end }} - --retries 10 --timeout 5000 $POD_NAME.corfu-headless.$POD_NAMESPACE.svc.cluster.local:9000 \ + --retries 10 --timeout 5000 $POD_NAME.{{ include "corfu.name" . }}-headless.$POD_NAMESPACE.svc.cluster.local:9000 \ &> /tmp/ping.log if grep -q "All nodes are reachable!" "/tmp/ping.log"; then @@ -99,13 +102,14 @@ spec: cat /tmp/ping.log exit 1 fi + {{- end }} echo "bootstrap local node with layout:" cat /configs/layout.json echo /usr/share/corfu/bin/corfu_bootstrap_cluster --layout=/configs/layout.json \ - -n "$POD_NAME.corfu-headless.$POD_NAMESPACE.svc.cluster.local:9000" \ + -n "$POD_NAME.{{ include "corfu.name" . }}-headless.$POD_NAMESPACE.svc.cluster.local:9000" \ --connection-timeout 2000 \ {{- if .Values.tls.enabled }} --enable-tls --keystore=/certs/keystore.jks --truststore=/certs/truststore.jks \ @@ -130,8 +134,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: {{ include "corfu.fullname" . }} - image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" + - name: corfu + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} readinessProbe: exec: @@ -156,7 +160,7 @@ spec: - "sh" - "-c" - | - output=$(curl $POD_NAME.corfu-headless.$POD_NAMESPACE.svc.cluster.local:8080/health | jq -r .liveness.status) + output=$(curl $POD_NAME.{{ include "corfu.name" . }}-headless.$POD_NAMESPACE.svc.cluster.local:8080/health | jq -r .liveness.status) if [ "$output" == "DOWN"]; then exit 1 fi @@ -174,7 +178,7 @@ spec: -Dlogback.configurationFile=/usr/share/corfu/conf/logback.prod.xml \ -Djava.io.tmpdir=/tmp \ org.corfudb.infrastructure.CorfuServer \ - -a $POD_NAME.corfu-headless.$POD_NAMESPACE.svc.cluster.local \ + -a $POD_NAME.{{ include "corfu.name" . }}-headless.$POD_NAMESPACE.svc.cluster.local \ {{- if .Values.persistence.enabled }} -l /config \ {{- else }} @@ -188,10 +192,12 @@ spec: -k {{ .Values.extraServerArgs.sequencerCacheSize }} \ -d {{ .Values.extraServerArgs.logLevel }} \ --log-size-quota-percentage {{ .Values.extraServerArgs.logSizeQuotaPercentage }} \ + {{- if .Values.version.new }} --compactor-script {{ .Values.extraServerArgs.compactorScript }} \ --compactor-config {{ .Values.extraServerArgs.compactorConfig }} \ --compaction-trigger-freq-ms {{ .Values.extraServerArgs.compactorTriggerFreqMs }} \ --health-port={{ .Values.service.healthPort }} \ + {{- end }} {{- if .Values.extraServerArgs.metricsEnabled }} --metrics \ {{- end }} diff --git a/cloud/corfu/corfu/values.yaml b/cloud/corfu/corfu/values.yaml index 402ca348..10ff6db3 100644 --- a/cloud/corfu/corfu/values.yaml +++ b/cloud/corfu/corfu/values.yaml @@ -1,4 +1,4 @@ -nameOverride: "" +nameOverride: "corfu" fullnameOverride: "corfu" branch: master commitSha: "" @@ -7,7 +7,7 @@ image: registry: "docker.io" repository: "corfudb/corfu-server" tag: "cloud" - pullPolicy: Always + pullPolicy: IfNotPresent imagePullSecretsEnabled: false imagePullSecrets: - name: "secret" @@ -15,6 +15,11 @@ persistence: enabled: true storageSize: 2Gi storageClass: "local-path" +cluster: + type: "source" +lr: + port: 9010 + name: "log-replication" resources: {} resourcesLR: {} service: @@ -42,6 +47,7 @@ tls: passwordName: corfu-password password: "MTIzNDU2" jvmArgsFilePath: "java_opts/default" +jvmArgsFilePathLR: "java_opts/lr" extraServerArgs: cacheHeapRatio: 0.2 sequencerCacheSize: "5000000" @@ -51,4 +57,5 @@ extraServerArgs: compactorConfig: "/usr/share/corfu/conf/corfu-compactor-config.yml" compactorTriggerFreqMs: 900000 metricsEnabled: true - +version: + new: false \ No newline at end of file diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index cd699eeb..0f946754 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -3,24 +3,24 @@ repositories { mavenLocal() mavenCentral() - maven { - name = "GitHubPackages" - url = uri("https://maven.pkg.github.com/corfudb/corfudb") - // For accessing GitHub Secrets in CorfuDB repo - credentials { - username = System.getenv("PKG_USERNAME") - password = System.getenv("PUBLISH_TOKEN") - } - } - - maven { - name = "corfudbCloudPackages" - url = uri("https://maven.pkg.github.com/corfudb/corfudb-cloud") - credentials { - username = System.getenv("PKG_USERNAME") - password = System.getenv("PUBLISH_TOKEN") - } - } +// maven { +// name = "GitHubPackages" +// url = uri("https://maven.pkg.github.com/corfudb/corfudb") +// // For accessing GitHub Secrets in CorfuDB repo +// credentials { +// username = System.getenv("PKG_USERNAME") +// password = System.getenv("PUBLISH_TOKEN") +// } +// } +// +// maven { +// name = "corfudbCloudPackages" +// url = uri("https://maven.pkg.github.com/corfudb/corfudb-cloud") +// credentials { +// username = System.getenv("PKG_USERNAME") +// password = System.getenv("PUBLISH_TOKEN") +// } +// } } configurations.all {