enable lr (test)

cloud/corfu/corfu/config/corfu_plugin_config.properties

@@ -1,15 +1,28 @@
-# Transport Plugin Configuration
 transport_adapter_JAR_path=/app/corfu.jar
+GRPC_transport_adapter_server_class_name=org.corfudb.infrastructure.logreplication.transport.sample.GRPCLogReplicationServerChannelAdapter
+GRPC_transport_adapter_client_class_name=org.corfudb.infrastructure.logreplication.transport.sample.GRPCLogReplicationClientChannelAdapter
+NETTY_transport_adapter_server_class_name=org.corfudb.infrastructure.logreplication.transport.sample.NettyLogReplicationServerChannelAdapter
+NETTY_transport_adapter_client_class_name=org.corfudb.infrastructure.logreplication.transport.sample.NettyLogReplicationClientChannelAdapter
+
+# Transport plugin selector
+transport_plugin_selector_JAR_path=/app/corfu.jar
+transport_plugin_selector_class_name=org.corfudb.infrastructure.logreplication.infrastructure.plugins.DefaultTransportPluginSelector
+
+# Transport Plugin Configuration
 transport_adapter_server_class_name=org.corfudb.infrastructure.logreplication.transport.sample.GRPCLogReplicationServerChannelAdapter
 transport_adapter_client_class_name=org.corfudb.infrastructure.logreplication.transport.sample.GRPCLogReplicationClientChannelAdapter
+
 # Stream Fetcher Plugin Configuration
 stream_fetcher_plugin_JAR_path=/app/corfu.jar
 stream_fetcher_plugin_class_name=org.corfudb.infrastructure.logreplication.infrastructure.plugins.DefaultLogReplicationConfigAdapter
+
 # Topology Manager Plugin Configuration
 topology_manager_adapter_JAR_path=/app/corfu.jar
 topology_manager_adapter_class_name=org.corfudb.infrastructure.logreplication.infrastructure.plugins.DefaultClusterManager
+
 # Snapshot Sync Configuration (Plugin)
 snapshot_sync_plugin_JAR_path=/app/corfu.jar
 snapshot_sync_plugin_class_name=org.corfudb.infrastructure.logreplication.infrastructure.plugins.DefaultSnapshotSyncPlugin
-saas_endpoint=corfu:9000
+
+saas_endpoint=corfu-0.corfu-headless.default.svc.cluster.local:9000
 local_node_id_path=/usr/share/corfu/conf/serial_number

cloud/corfu/corfu/files/init_layout.py

@@ -20,7 +20,7 @@ def generate_layout(args):
   layout_template["layoutServers"] = fqdn_list
   layout_template["sequencers"] = fqdn_list
   layout_template["segments"][0]["stripes"][0]["logServers"] = fqdn_list
-  layout_template["clusterId"] = "123e4567-e89b-12d3-a456-556642440000"
+  layout_template["clusterId"] = "456e4567-e89b-12d3-a456-556642440001" if args.type == "source" else "456e4567-e89b-12d3-a456-556642440002"
 
   # print layout
   print("Generated layout:")
@@ -39,6 +39,7 @@ def main():
   parser.add_argument('--replica', '-r', type=int, required=True, help='The replica of Corfu cluster.')
   parser.add_argument('--statefulset', type=str, default='corfu', help='Corfu statefulset name.')
   parser.add_argument('--headless', type=str, default='corfu-headless', required=True, help='Corfu headless service name.')
+  parser.add_argument('--type', type=str, default='source', required=True, help='Source or sink.')
   args = parser.parse_args()
 
   generate_layout(args)

cloud/corfu/corfu/serial_number/serial_number

@@ -0,0 +1 @@
+serial=B1310142-311D-B7AE-D6A7-F66DEE607871

cloud/corfu/corfu/templates/Deployment.yaml

@@ -0,0 +1,50 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "corfu.service.lr" . }}
+  labels:
+    {{- include "corfu.labels" . | indent 4 }}
+spec:
+  replicas: {{ include "corfu.replicas" . }}
+  selector:
+    matchLabels:
+      {{- include "corfu.selectors.lr" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+        {{- include "corfu.selectors.lr" . | nindent 8 }}
+    spec:
+      containers:
+      - name: {{ include "corfu.service.lr" . }}
+        image:  "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        ports:
+        - name: http
+          containerPort: {{ .Values.lr.port }}
+          protocol: TCP
+        env:
+          - name: CONFIG_FILE_PATH
+            value: "/usr/share/corfu/conf/corfu_replication_config.properties"
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: POD_UID
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.uid
+        VolumeMounts:
+          - name: log-dir
+            mountPath: /var/log/corfu-log-replication
+          - name: config-dir
+            mountPath: /config/corfu-log-replication
+          - name: lr
+            mountPath: /common/configs/
+        command:
+          - "sh"
+          - "-c"
+          - "java -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/corfu/corfu_oom.hprof -XX:+HeapDumpOnOutOfMemoryError -Djdk.nio.maxCachedBufferSize=1048576 -Dio.netty.recycler.maxCapacityPerThread=0 -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -Djava.io.tmpdir=/image/corfu-server/temp -cp /app/corfu.jar:/opt/vmware/log-replication/log-replication_deploy.jar -Djava.io.tmpdir=/tmp org.corfudb.infrastructure.CorfuServer --plugin=/usr/share/corfu/conf/corfu_plugin_config.properties -d DEBUG 9010 -m"

cloud/corfu/corfu/templates/_helpers.tpl

@@ -1,5 +1,5 @@
 {{- define "corfu.name" -}}
-corfu
+{{ .Values.nameOverride }}
 {{- end }}
 
 {{- define "corfu.fullname" -}}
@@ -30,6 +30,14 @@ type: {{ .Values.type | default "config" | quote }}
 app.kubernetes.io/name: {{ include "corfu.fullname" . }}
 {{- end }}
 
+{{- define "corfu.service.lr" -}}
+{{- .Values.lr.name }}
+{{- end }}
+
+{{- define "corfu.selectors.lr" -}}
+app.kubernetes.io/name: {{ include "corfu.service.lr" . }}
+{{- end }}
+
 {{/*
 If replicas tag is defined in its own helm chart values.yaml
 it will always override the global value. If not, we will use the global value.

cloud/corfu/corfu/templates/certificate.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.tls.certificate.enabled }}
-{{- $dns := printf "*.corfu-headless.%s.svc.cluster.local" .Release.Namespace }}
+{{- $dns := printf "*.%s-headless.%s.svc.cluster.local" .Values.nameOverride .Release.Namespace }}
 apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
@@ -25,6 +25,7 @@ spec:
     - client auth
   dnsNames:
     - corfu
+    - corfu2
     - "{{ $dns }}"
   issuerRef:
     name: {{ .Values.tls.certificate.issuer.name }}

cloud/corfu/corfu/templates/configmap-lr.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "corfu.fullname" . }}-corfu-configs-lr
+  labels:
+    {{- include "corfu.labels" . | indent 4 }}
+data:
+  {{- with .Files.Glob "serial_number/*" }}
+    {{- .AsConfig | nindent 2 }}
+    {{- end }}

cloud/corfu/corfu/templates/service-lr.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "corfu.service.lr" . }}
+  labels:
+    {{- include "corfu.labels" . | indent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.lr.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "corfu.selectors.lr" . | nindent 4 }}

cloud/corfu/corfu/templates/service.yaml

@@ -33,7 +33,7 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "corfu.fullname" . }}-headless
+  name:   {{ include "corfu.name" . }}-headless
   labels:
     {{- include "corfu.labels" . | indent 4 }}
 spec:

cloud/corfu/corfu/templates/serviceaccount.yaml

@@ -26,7 +26,7 @@ rules:
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
-  name: {{ include "corfu.name" . }}-role-binding
+  name: {{ include "corfu.serviceAccountName" . }}-role-binding
   labels:
     {{- include "corfu.labels" . | indent 4 }}
 subjects:

cloud/corfu/corfu/templates/statefulset.yaml

@@ -5,7 +5,7 @@ metadata:
   labels:
     {{- include "corfu.labels" . | indent 4 }}
 spec:
-  serviceName: {{ include "corfu.fullname" . }}-headless
+  serviceName: {{ include "corfu.name" . }}-headless
   replicas: {{ include "corfu.replicas" . }}
   podManagementPolicy: Parallel
   selector:
@@ -29,15 +29,15 @@ spec:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
       initContainers:
         - name: create-layout
-          image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           command:
             - "/bin/bash"
             - "-c"
             - |
               set -e
               python3 /etc/corfu/configs/init_layout.py --template /etc/corfu/configs/layout_template.json --layout /usr/share/configs/layout.json \
-              --replica {{ include "corfu.replicas" . }} --statefulset {{ include "corfu.name" . }} --headless {{ include "corfu.name" . }}-headless --port {{ .Values.service.port }}
+              --replica {{ include "corfu.replicas" . }} --statefulset {{ include "corfu.fullname" . }} --headless {{ include "corfu.name" . }}-headless --port {{ .Values.service.port }} --type {{ .Values.cluster.type }}
           env:
             - name: POD_NAMESPACE
               valueFrom:
@@ -46,11 +46,11 @@ spec:
           volumeMounts:
             - name: workdir
               mountPath: /usr/share/configs
-            - name: corfu-layout-configs
+            - name: {{ include "corfu.fullname" . }}-layout-configs
               mountPath: /etc/corfu/configs
       containers:
         - name: bootstrap-corfu
-          image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           volumeMounts:
             - name: workdir
@@ -90,7 +90,7 @@ spec:
               --enable-tls --keystore=/certs/keystore.jks  --truststore=/certs/truststore.jks \
               --keystore-password-file=/password/password --truststore-password-file=/password/password \
               {{- end }}
-              --retries 10 --timeout 5000 $POD_NAME.corfu-headless.$POD_NAMESPACE.svc.cluster.local:9000 \
+              --retries 10 --timeout 5000 $POD_NAME.{{ include "corfu.name" . }}-headless.$POD_NAMESPACE.svc.cluster.local:9000 \
               &> /tmp/ping.log
 
               if grep -q "All nodes are reachable!" "/tmp/ping.log"; then
@@ -105,7 +105,7 @@ spec:
               echo
 
               /usr/share/corfu/bin/corfu_bootstrap_cluster --layout=/configs/layout.json \
-              -n "$POD_NAME.corfu-headless.$POD_NAMESPACE.svc.cluster.local:9000" \
+              -n "$POD_NAME.{{ include "corfu.name" . }}-headless.$POD_NAMESPACE.svc.cluster.local:9000" \
               --connection-timeout 2000 \
               {{- if .Values.tls.enabled }}
               --enable-tls --keystore=/certs/keystore.jks  --truststore=/certs/truststore.jks \
@@ -130,8 +130,8 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
-        - name: {{ include "corfu.fullname" . }}
-          image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        - name: corfu
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           readinessProbe:
             exec:
@@ -156,7 +156,7 @@ spec:
                 - "sh"
                 - "-c"
                 - |
-                  output=$(curl $POD_NAME.corfu-headless.$POD_NAMESPACE.svc.cluster.local:8080/health | jq -r .liveness.status)
+                  output=$(curl $POD_NAME.{{ include "corfu.name" . }}-headless.$POD_NAMESPACE.svc.cluster.local:8080/health | jq -r .liveness.status)
                   if [ "$output" == "DOWN"]; then
                     exit 1
                   fi
@@ -174,7 +174,7 @@ spec:
               -Dlogback.configurationFile=/usr/share/corfu/conf/logback.prod.xml \
               -Djava.io.tmpdir=/tmp \
               org.corfudb.infrastructure.CorfuServer \
-              -a $POD_NAME.corfu-headless.$POD_NAMESPACE.svc.cluster.local \
+              -a $POD_NAME.{{ include "corfu.name" . }}-headless.$POD_NAMESPACE.svc.cluster.local \
               {{- if .Values.persistence.enabled }}
               -l /config \
               {{- else }}

cloud/corfu/corfu/values.yaml

@@ -1,4 +1,4 @@
-nameOverride: ""
+nameOverride: "corfu"
 fullnameOverride: "corfu"
 branch: master
 commitSha: ""
@@ -7,14 +7,19 @@ image:
   registry: "docker.io"
   repository: "corfudb/corfu-server"
   tag: "cloud"
-  pullPolicy: Always
+  pullPolicy: IfNotPresent
 imagePullSecretsEnabled: false
 imagePullSecrets:
   - name: "secret"
 persistence:
   enabled: true
   storageSize: 2Gi
   storageClass: "local-path"
+cluster:
+  type: "source"
+lr:
+  port: 9010
+  name: "log-replication"
 resources: {}
 resourcesLR: {}
 service:
@@ -42,6 +47,7 @@ tls:
   passwordName: corfu-password
   password: "MTIzNDU2"
 jvmArgsFilePath: "java_opts/default"
+jvmArgsFilePathLR: "java_opts/lr"
 extraServerArgs:
   cacheHeapRatio: 0.2
   sequencerCacheSize: "5000000"

cloud/corfu/test.sh

@@ -0,0 +1,24 @@
+#!/bin/zsh
+
+k3d cluster delete corfu
+rm -rf /tmp/k3dvol
+
+k3d cluster create corfu \
+  --volume /tmp/k3dvol:/tmp/k3dvol \
+  -p "8082:30080@agent:0" \
+  --agents 2
+
+k3d image import corfudb/corfu-server:cloud -c corfu
+k3d image import corfudb2/corfu-server:cloud -c corfu
+
+k3d image import corfudb/corfu-server:0.4.0-SNAPSHOT -c corfu
+k3d image import corfudb2/corfu-server:0.4.0-SNAPSHOT -c corfu
+
+helm repo add jetstack https://charts.jetstack.io
+helm repo update
+
+helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version v1.8.0 --set installCRDs=true
+helm install corfu corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb/corfu-server --set image.tag=0.4.0-SNAPSHOT
+helm install corfu2 corfu --set tls.enabled=false --set tls.certificate.enabled=false --set global.replicas=3 --set image.repository=corfudb2/corfu-server --set image.tag=0.4.0-SNAPSHOT --set lr.name="log-replication2" --set nameOverride="corfu2" --set serviceAccount.name="corfu2" --set nameOverride="corfu2" --set fullnameOverride="corfu2" --set cluster.type="sink"
+
+sleep 10