added etcd to experiments folder

.hgignore

@@ -20,3 +20,4 @@ examples/ricartagrawala/gonadoresults/*
 examples/chainkv/results/*
 examples/sum/failed
 userstudy/*
+*log_archive/*

dinvRT/assert.go

@@ -365,6 +365,7 @@ func InitDistributedAssert(addr string, assertPeers []string, processName string
 		goVecLogger.LogLocalEvent("Cannot parse assert peers dying!!")
 		os.Exit(1)
 	}
+
 	//remove self from peer list
 	tmpNeighbours := make([]string, 0)
 	for _, n := range peers {

examples/lib.sh

@@ -39,7 +39,7 @@ function runLogMerger {
         echo "merge logs with extra args '$1'"
     fi
 
-    dinv -v -l $1 ./*Encoded.txt ./*Log.txt
+    dinv -l $1 $2 $3 $4 ./*Encoded.txt ./*Log.txt
     # dinv -v -l $1 -name="fruits" -shiviz ./*Encoded.txt ./*Log.txt
     echo "logmerger took $(($(date +'%s') - $t1))s to run"
 }
@@ -48,7 +48,9 @@ function runDaikon  {
     t1=$(date +'%s')
     echo "run daikon"
     # redirect output both to output.txt and stdout
-    java daikon.Daikon ./*.dtrace | tee output.txt
+    for file in ./*.dtrace; do
+        java daikon.Daikon $file >>  output.txt
+    done
     echo "daikon took $(($(date +'%s') - $t1))s to run"
 }
 

experiements/etcd/arraytest.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+assertOP[0]="NONE"
+assertOP[1]="STRONGLEADER"
+assertOP[2]="LOGMATCHING"
+assertOP[3]="LEADERAGREEMENT"
+
+leaderOP[0]="true"
+leaderOP[1]="false"
+
+sampleOP[0]="1"
+sampleOP[1]="10"
+sampleOP[2]="100"
+
+clientOP[0]="4"
+clientOP[1]="40"
+clientOP[2]="400"
+
+for assert in ${assertOP[@]}
+do
+    for leader in ${leaderOP[@]}
+    do
+        for sample in ${sampleOP[@]}
+        do
+            for client in ${clientOP[@]}
+            do
+                echo "$assert $leader $sample $client"
+
+            done
+        done
+    done
+done
+
+

experiements/etcd/azure/-p

@@ -0,0 +1,2 @@
+
+

experiements/etcd/azure/-r

@@ -0,0 +1,2 @@
+
+

experiements/etcd/azure/blast.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#./blast.sh words.txt serverIpPort command id
+echo "starting azure client"
+lout=latency$4.txt
+echo "" > $lout
+i=0
+for word in $(<$1)
+do
+    a=$(($(date +%s%N)/1000000))
+    ETCDCTL_API=3 $3 --endpoints=$2:2379 put $i "$word"
+    b=$(($(date +%s%N)/1000000))
+    lat=0
+    let lat=b-a
+    echo $lat >> $lout
+
+#    ETCDCTL_API=3 ../bin/etcdctl --endpoints=localhost:2379 get $i
+    i=$((i+1))
+done
+
+
+#i=0
+#for word in $(<$1)
+#do
+#    ETCDCTL_API=3 $3 --endpoints=$2:2379 get $i
+#    i=$((i+1))
+#done

experiements/etcd/azure/client.sh

@@ -0,0 +1,20 @@
+#client makes requets to a raft webserver and keeps track of the latency of each request
+
+#client.sh intput.txt [server ip:port]
+LatOut=latency.txt
+echo "" > $LatOut
+for word in $(<$1)
+do
+    echo $word
+    echo $1 $2 $3 $4
+    a=$(($(date +%s%N)/1000000))
+    ETCDCTL_API=3 $3 --endpoints=$2:2379 put $i "$word"
+    b=$(($(date +%s%N)/1000000))
+    latency=0
+    let latatency=b-a
+    echo $latency
+    #echo "$latency" >> $LatOut
+    #echo "making request"
+    #ETCDCTL_API=3 $3 --endpoints=$2 put $i "$word"
+
+done

experiements/etcd/azure/cluster.sh

@@ -0,0 +1,245 @@
+#!/bin/bash
+
+#Author Stewart Grant
+#Feb 2016
+#FSE deadline push
+
+#This script mannages running etcd clusters on azure. The clusters are
+#run in vm's which are ssh'd into. The names of all of the VMs are
+#abbrivated below #Etcd is launched along with a client that makes puts
+#and gets. After a predetermined ammount of time the cluster is killed
+#and the logs are retrieved via scp.
+
+#arguments
+#1 function
+#2 clients
+#3 name
+#4 assertType
+#5 leader
+#6 sample
+#7 bug
+
+
+
+
+#VM's their public and private IP's
+#stewart-test-1
+ST1G=52.228.27.112
+ST1P=10.0.1.4
+#stewart-test-2
+ST2G=52.228.32.101
+ST2P=10.0.1.5
+#stewart
+S1G=13.64.239.61
+S1P=10.0.0.4
+#stewart2
+S2G=13.64.247.122
+S2P=10.0.0.5
+#stewart3
+S3G=13.64.242.139
+S3P=10.0.0.6
+#stewartbig
+SBG=13.64.149.118
+SBP=10.0.0.7
+
+
+#map a subset of the VM's to the current cluster
+GLOBALS1=$S1G
+LOCALS1=$S1P
+GLOBALS2=$S2G
+LOCALS2=$S2P
+GLOBALS3=$S3G
+LOCALS3=$S3P
+
+#GLOBAL information true of all VMs
+HOMEA=/home/stewart
+DINV=$HOMEA/go/src/bitbucket.org/bestchai/dinv
+ETCD=$HOMEA/go/src/github.com/coreos/etcd
+ETCDCMD=$HOMEA/go/src/github.com/coreos/etcd/bin/etcd
+ETCDCTL=$HOMEA/go/src/github.com/coreos/etcd/bin/etcdctl
+AZURENODE=/dinv/azure/node.sh
+#different clients
+CLIENT=/dinv/azure/blast.sh
+#CLIENT=/dinv/azure/client.sh
+CLIENTMGR=/dinv/azure/measure.sh
+
+#LOCAL
+DINVDIR=/home/stewartgrant/go/src/bitbucket.org/bestchai/dinv
+
+#MEASUREMENT INFO
+MEASURE=true
+
+
+#TEXT=$ETCD/dinv/kahn.in
+#TEXT=/usr/share/dict/words
+TEXT=$ETCD/dinv/in.in
+
+
+USAGE="USAGE\n-k kill all nodes in the cluster\n-p pull from etcd-dinv repo\n-l logmerger\n-d Daikon\n-c clean"
+
+function onall {
+    ssh stewart@$GLOBALS1  -x $1  &
+    ssh stewart@$GLOBALS2  -x $1  &
+    ssh stewart@$GLOBALS3  -x $1  &
+    ssh stewart@$SBG -x $1  &
+    #ssh stewart@$ST1G -x $1 &
+    #ssh stewart@$ST2G -x $1 &
+    #ssh stewart@$SG -x $1 &
+    #ssh stewart@$S2G -x $1 &
+    #ssh stewart@$S3G -x $1 &
+}
+
+
+#kill all the nodes
+if [ "$1" == "-k" ];then
+    echo kill
+    onall "killall etcd; killall blast.sh"
+    exit
+fi
+
+#have all the nodes pull new code
+if [ "$1" == "-p" ];then
+    echo clean
+    $DINVDIR/examples/lib.sh clean
+    echo push
+    cd ../../
+    git add --all
+    git commit -m "updating raft for peers"
+    git push
+    cd dinv/azure
+    echo pull
+    onall "cd $ETCD && git pull && ./build ; cd $DINV && hg pull"
+    exit
+fi
+
+#run logmerger
+if [ "$1" == "-l" ];then
+    sudo -E go install ../../../../../bitbucket.org/bestchai/dinv
+    $DINVDIR/examples/lib.sh runLogMerger "-plan=SCM -shiviz"
+    exit
+fi
+
+#run daikon
+if [ "$1" == "-d" ];then
+    $DINVDIR/examples/lib.sh runDaikon
+    exit
+fi
+
+#clean
+if [ "$1" == "-c" ];then
+    echo clean
+    onall "cd; rm *.txt"
+    $DINVDIR/examples/lib.sh clean
+    exit
+fi
+
+
+if [ "$1" == "-r" ];then
+    echo run
+
+    onall "cd; pwd; rm bug*"
+
+    #Example execute ssh 
+    #ssh [email protected] -x "mkdir test"
+
+    #Example execute scp
+    #scp [email protected]:/home/stewart/azureinstall.sh astest
+
+    #LOCAL CLUSTER
+    CLUSTER="infra0=http://$LOCALS1:2380,infra1=http://$LOCALS2:2380,infra2=http://$LOCALS3:2380"
+    ASSERT="$LOCALS1:12000,$LOCALS2:12000,$LOCALS3:12000"
+    echo "ssh stewart@$GLOBALS1 -x $ETCD$AZURENODE 0 $GLOBALS1 $LOCALS1 $CLUSTER $ASSERT $4 $5 $6 $7"
+    ssh stewart@$GLOBALS1 -x "$ETCD$AZURENODE 0 $GLOBALS1 $LOCALS1 $CLUSTER $ASSERT $4 $5 $6 $7" &
+    ssh stewart@$GLOBALS2 -x "$ETCD$AZURENODE 1 $GLOBALS2 $LOCALS2 $CLUSTER $ASSERT $4 $5 $6 $7" &
+    ssh stewart@$GLOBALS3 -x "$ETCD$AZURENODE 2 $GLOBALS3 $LOCALS3 $CLUSTER $ASSERT $4 $5 $6 $7" &
+
+    #GLOBAL CLUSTER
+    #CLUSTER="infra0=http://$GLOBALS1:2380,infra1=http://$GLOBALS2:2380,infra2=http://$GLOBALS3:2380"
+    #ASSERT="$GLOBALS1:12000,$GLOBALS2:12000,$GLOBALS3:12000"
+    #ssh stewart@$GLOBALS1 -x "$ETCD$AZURENODE 0 $GLOBALS1 $GLOBALS1 $CLUSTER $ASSERT" &
+    #ssh stewart@$GLOBALS2 -x "$ETCD$AZURENODE 1 $GLOBALS2 $GLOBALS2 $CLUSTER $ASSERT" &
+    #ssh stewart@$GLOBALS3 -x "$ETCD$AZURENODE 2 $GLOBALS3 $GLOBALS3 $CLUSTER $ASSERT" &
+    sleep 5
+
+    if [ "$MEASURE" = false ] ; then
+        #run the client on on the same node it's sending to
+        ssh stewart@$GLOBALS1 -x "echo $ETCD/dinv/azure/client.sh $TEXT $LOCALS1 && $ETCD/dinv/azure/client.sh $TEXT $LOCALS1 $ETCDCTL" &
+        #kill allthe hosts
+        sleep 10
+        echo kill
+        onall "killall etcd"
+        onall "killall blast"
+    fi
+
+    #run the client on a node seperate from the one receving
+    #ssh stewart@$GLOBALS1 -x "$ETCD$CLIENT $TEXT $LOCALSS2" &
+
+    if [ "$MEASURE" = true ] ; then
+
+        EXP=$3
+        RUNTIME=10
+        CLIENTS=$2
+        #run the client locally
+        #./measure.sh /usr/share/dict/words $GLOBALS1:2379 $RUNTIME
+        echo "STARTING CLIENT"
+
+        #ssh stewart@$GLOBALS1 -x "echo $ETCD$CLIENT $TEXT $LOCALS1 && $ETCD$CLIENT $TEXT $LOCALS1 $ETCDCTL"
+        ##BLOCK HERE
+        ssh stewart@$SBG -x "echo $ETCD$CLIENTMGR $TEXT $LOCALS1 && $ETCD$CLIENTMGR $TEXT $LOCALS1 $RUNTIME $CLIENTS $ETCDCTL $ETCD$CLIENT"
+        #kill allthe hosts
+        echo kill
+        onall "killall etcd"
+        onall "killall blast"
+    fi
+
+    #wait for the test to run
+
+    scp stewart@$GLOBALS1:/home/stewart/*.txt ./
+    scp stewart@$GLOBALS2:/home/stewart/*.txt ./
+    scp stewart@$GLOBALS3:/home/stewart/*.txt ./
+    scp stewart@$SBG:/home/stewart/*.txt ./
+    ssh stewart@$SBG -x "rm lat*"
+    echo DONE!
+
+    if [ "$MEASURE" = true ] ; then
+        #get the latency from the requests
+        cat latency* > agg.txt
+        rm latency*
+        R -q -e "x <- read.csv('agg.txt', header = F); summary(x); sd(x[ , 1])" > stats.txt
+
+        #get the bug catching times
+        #get the earliest bug starting time
+        output=bs.txt
+        echo "" > $output
+        for file in bugstart*; do
+           cat $file >>$output
+           rm $file
+           echo "" >> $output
+        done
+        START=`sort $output | head -2`
+        rm $output
+        #get the earliest bug catching time
+        output=bc.txt
+        echo "" > $output
+        for file in bugcatch*; do
+           cat $file >> $output
+           rm $file
+           echo "" >> $output
+        done
+        CATCH=`sort $output | head -2`
+        rm $output
+        echo $CATCH - $START
+        BUGTIME=`echo $CATCH - $START | bc`
+
+        MEDIAN=`grep Median stats.txt |cut -d: -f2`
+        MEAN=`grep Mean stats.txt |cut -d: -f2`
+        SD=`grep "\[1\]" stats.txt |cut -d' ' -f2`
+        #./client.sh /usr/share/dict/words $GLOBALS1:2379
+        TP=`grep -E '[0-9]' agg.txt | wc -l | cut -f1`
+        let "RPS=$TP/$RUNTIME"
+        echo "$EXP,$CLIENTS,$RPS,$MEDIAN,$MEAN,$SD,$BUGTIME" >> measurements.txt
+    fi
+    exit
+fi
+
+echo -e $USAGE

experiements/etcd/azure/last_run

@@ -0,0 +1 @@
+./log_archive/02-22_15-45-34/