perftest

#!/bin/bash

shopt -s extglob

if [ "`uname -s`" = "Darwin" ] ; then
    export nwif=en0
    libsuffix=.0.dylib
else
    export nwif=eth0
    libsuffix=.so.0
fi
bandwidth=""
remotedir="$PWD"
provision=false
asynclist="sync async"
modelist="listener polling waitset"
modelistdefault=true
rate=""
sizelist="0 20 50 100 200 500 1000 2000 5000 10000 20000 50000 100000 200000 500000 1000000"
timeout=30
loopback=true
resultdir="throughput-result"
resultdirdefault=true
force=false
netstats=false
watermarks=""
remotes=""
security="none"

operation=throughput

usage () {
    cat >&2 <<EOF
usage: $0 [OPTIONS] user@remote [user@remote...]

OPTIONS
  -i IF        use network interface IF (default: $nwif) local
  -I IF        use network interface IF remotely (default: same as local)
  -b 100|1000|B  network bandwidth (100Mbps/1000Gbps or B bits/sec) for
               calculating load % given load in bytes/second (default:
               report Mb/s)
  -d DIR       use DIR on remote (default: PWD)
  -p           provision required binaries in DIR (default: $provision)
               first ssh's in to try mkdir -p DIR, then follows up with scp
  -t DUR       run for DUR seconds per size (default $timeout)
  -a ASYNCLIST run for delivery async settings ASYNCLIST (default:
               "$asynclist")
  -m MODELIST  run with subscriber mode settings MODELIST (default: 
               "$modelist")
  -s SIZELIST  run for sizes in SIZELIST (default: "$sizelist")
               if SIZELIST is empty, it uses ddsperf's OU topic instead
  -R RATE      set rate to RATE (e.g., -R 100Hz) rather than run as fast
               as possible
  -S MODE      whether and how to configure DDS Security:
                 none              no security (default)
                 sign              synonym for rtps-sign
                 encrypt           synonym for payload-encrypt
                 rtps-sign         RTPS message signing
                 rtps-encrypt      RTPS message encryption
                 metadata-sign     RTPS submessage signing
                 metadata-encrypt  RTPS submessage encryption
                 payload-encrypt   payload encryption only
                 *.xml             governance XML file to use
               the various \"encrypt\" modes use AES-256
               uses secperf directory for storing generated security config
  -l LOOPBACK  enable/disable multicast loopback (true/false, default: $loopback)
  -o DIR       store results in dir (default: $resultdir)
  -O OPER      operation: \"throughput\" or \"latency\" (default: $operation)
  -f           "force": first do "rm -rf" of output dir, then create it
  -W           set high water mark to 100kB
  -X           run ping to first remote; on macOS: log interface stats

Local host runs "ddsperf" in subscriber mode, first remote runs it publisher
mode, further remotes also run subcribers.  It assumes these are available in
DIR/bin.  It also assumes that ssh user@remote works without requiring a
password.
EOF
    exit 1
}

dokill () {
    [ -n "$netstats_pids" ] && kill $netstats_pids
    for r in $remotes ; do
        ssh $r "kill -9 \`cat $remotedir/$operation-test-$remote_oper-*.pid\` ; rm $remotedir/$operation-test-$remote_oper-*.pid"
    done
}

dokill_and_exit () {
    dokill
    exit 1
}

while getopts "fi:I:b:d:pa:m:s:S:t:o:O:l:R:WX" opt ; do
    case $opt in
        f) force=true ;;
        i) nwif="$OPTARG" ;;
        I) rnwif="$OPTARG" ;;
        b) case "$OPTARG" in
               100) bandwidth=:1e8 ;;
               1000) bandwidth=:1e9 ;;
               *) bandwidth="${OPTARG:+:}$OPTARG" ;;
           esac ;;
        d) remotedir="$OPTARG" ;;
        p) provision=true ;;
        a) asynclist="$OPTARG" ;;
        m) modelist="$OPTARG" ; modelistdefault=false ;;
        s) sizelist="$OPTARG" ;;
        S) security="$OPTARG" ;;
        l) loopback="$OPTARG" ;;
        t) timeout="$OPTARG" ;;
        o) resultdir="$OPTARG" ; resultdirdefault=false ;;
        O) operation="$OPTARG" ;;
        R) rate="$OPTARG" ;;
        W) watermarks="<Watermarks><WhcHigh>100kB</WhcHigh></Watermarks>" ;;
        X) netstats=true ;;
        h) usage ;;
    esac
done
shift $((OPTIND-1))
if [ $# -lt 1 ] ; then usage ; fi
remotes="$@"
[ -n "$security" ] || security="none"

case "$operation" in
    throughput)
        local_oper="pub"
        remote_oper="sub"
        allowedmodes="@(listener|polling|waitset)"
        ;;
    latency)
        local_oper="ping"
        remote_oper="pong"
        allowedmodes="@(listener|waitset)"
        if $modelistdefault ; then modelist="listener waitset" ; fi
        if $resultdirdefault ; then resultdir="latency-result" ; fi
        ;;
    *) echo "unsupported test operation: $operation" >&2 ; exit 1 ;;
esac
for m in $modelist ; do
    case $m in
        $allowedmodes) ;;
        *) echo "unsupported mode \"$m\" for operation \"$operation\"" >&2 ; exit 1 ;;
    esac
done
for a in $asynclist ; do
    case $a in
        sync|async) ;;
        *) echo "unsupported sync/async \"$a\" for operation \"$operation\"" >&2 ; exit 1 ;;
    esac
done
case "$security" in
    none)
        gensec=false
        gensec_governance=false
        ;;
    sign|encrypt|rtps-sign|rtps-encrypt|metadata-sign|metadata-encrypt|payload-encrypt)
        gensec=true
        gensec_governance=true
        ;;
    *.xml)
        cp "$security" $secdir/governance.xml || exit 1
        gensec=true
        gensec_governance=false
        ;;
    *) echo "unsupport security mode \"$security\"" >&2 ; exit 1 ;;
esac

secfiles="secperf/sloth_priv_key.pem secperf/sloth_cert.pem secperf/id_ca_cert.pem secperf/perm_ca_cert.pem secperf/governance.p7s secperf/permissions.p7s"
if $gensec ; then
    mkdir -p secperf || { echo "can't create secperf directory" >&2 ; exit 1 ; }
    if [ "`uname -s`" = "Darwin" -a "`which openssl`" = /usr/bin/openssl ] ; then
        if which brew >/dev/null 2>&1 ; then
            openssl="`brew --prefix openssl`/bin/openssl"
        fi
    else
        openssl=openssl
    fi
    [ -n "$openssl" ] || { echo "this needs a functioning openssl executable" 2>&1 ; exit 1 ; }

    if $gensec_governance ; then
        case $security in
            rtps-sign|sign)
                rtps_protection=SIGN
                metadata_protection=NONE
                data_protection=NONE
                ;;
            rtps-encrypt)
                rtps_protection=ENCRYPT
                metadata_protection=NONE
                data_protection=NONE
                ;;
            metadata-sign)
                rtps_protection=NONE
                metadata_protection=SIGN
                data_protection=NONE
                ;;
            metadata-encrypt)
                rtps_protection=NONE
                metadata_protection=ENCRYPT
                data_protection=NONE
                ;;
            payload-encrypt|encrypt)
                rtps_protection=NONE
                metadata_protection=NONE
                data_protection=ENCRYPT
                ;;
            *) exit 1 ;;
        esac
        cat >secperf/governance.xml <<EOF
<?xml version="1.0" encoding=\"utf-8\"?>
<dds xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="https://www.omg.org/spec/DDS-SECURITY/20170901/omg_shared_ca_governance.xsd">
  <domain_access_rules>
    <domain_rule>
      <domains> <id_range> <min>0</min> <max>230</max> </id_range> </domains>
      <allow_unauthenticated_participants>false</allow_unauthenticated_participants>
      <enable_join_access_control>true</enable_join_access_control>
      <discovery_protection_kind>NONE</discovery_protection_kind>
      <liveliness_protection_kind>NONE</liveliness_protection_kind>
      <rtps_protection_kind>$rtps_protection</rtps_protection_kind>
      <topic_access_rules>
        <topic_rule>
          <topic_expression>*</topic_expression>
          <enable_discovery_protection>true</enable_discovery_protection>
          <enable_liveliness_protection>true</enable_liveliness_protection>
          <enable_read_access_control>true</enable_read_access_control>
          <enable_write_access_control>true</enable_write_access_control>
          <metadata_protection_kind>$metadata_protection</metadata_protection_kind>
          <data_protection_kind>$data_protection</data_protection_kind>
        </topic_rule>
      </topic_access_rules>
    </domain_rule>
  </domain_access_rules>
</dds>
EOF
        cat >secperf/permissions.xml <<EOF
<?xml version="1.0" encoding="utf-8" ?>
<dds xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
     xsi:noNamespaceSchemaLocation="https://www.omg.org/spec/DDS-SECURITY/20170901/omg_shared_ca_permissions.xsd">
  <permissions>
    <grant name="default_permissions">
      <subject_name>emailAddress=alice@cycloneddssecurity.adlinktech.com,CN=Alice Example,O=Example Organization,OU=Organizational Unit Name,L=Locality Name,ST=OV,C=NL</subject_name>
      <validity>
        <!-- Format is CCYY-MM-DDThh:mm:ss[Z|(+|-)hh:mm] in GMT -->
        <not_before>2020-01-01T01:00:00</not_before>
        <not_after>2120-01-01T01:00:00</not_after>
      </validity>
      <allow_rule>
        <domains> <id_range> <min>0</min> <max>230</max> </id_range> </domains>
        <publish>
          <topics> <topic>*</topic> </topics>
          <partitions> <partition>*</partition> </partitions>
        </publish>
        <subscribe>
          <topics> <topic>*</topic> </topics>
          <partitions> <partition>*</partition> </partitions>
        </subscribe>
      </allow_rule>
      <default>DENY</default>
    </grant>
  </permissions>
</dds>
EOF

        # generate any missing keys/certificates
        [ -r secperf/id_ca_priv_key.pem ] || $openssl genrsa -out secperf/id_ca_priv_key.pem 2048
        [ -r secperf/id_ca_cert.pem ] || $openssl req -x509 -key secperf/id_ca_priv_key.pem -out secperf/id_ca_cert.pem -days 3650 -subj "/C=NL/ST=OV/L=Locality Name/OU=Example OU/O=Example ID CA Organization/CN=Example ID CA/emailAddress=authority@cycloneddssecurity.adlinktech.com"
        [ -r secperf/perm_ca_priv_key.pem ] || $openssl genrsa -out secperf/perm_ca_priv_key.pem 2048
        [ -r secperf/perm_ca_cert.pem ] || $openssl req -x509 -key secperf/perm_ca_priv_key.pem -out secperf/perm_ca_cert.pem -days 3650 -subj "/C=NL/ST=OV/L=Locality Name/OU=Example OU/O=Example CA Organization/CN=Example Permissions CA/emailAddress=authority@cycloneddssecurity.adlinktech.com"
        [ -r secperf/sloth_priv_key.pem ] || $openssl genrsa -out secperf/sloth_priv_key.pem 2048
        [ -r secperf/sloth.csr ] || $openssl req -new -key secperf/sloth_priv_key.pem -out secperf/sloth.csr -subj "/C=NL/ST=OV/L=Locality Name/OU=Organizational Unit Name/O=Example Organization/CN=Alice Example/emailAddress=alice@cycloneddssecurity.adlinktech.com"
        [ -r secperf/sloth_cert.pem ] || $openssl x509 -req -CA secperf/id_ca_cert.pem -CAkey secperf/id_ca_priv_key.pem -CAcreateserial -days 3650 -in secperf/sloth.csr -out secperf/sloth_cert.pem

        # always sign permissions, governance
        $openssl smime -sign -in secperf/permissions.xml -text -out secperf/permissions.p7s -signer secperf/perm_ca_cert.pem -inkey secperf/perm_ca_priv_key.pem
        $openssl smime -sign -in secperf/governance.xml -text -out secperf/governance.p7s -signer secperf/perm_ca_cert.pem -inkey secperf/perm_ca_priv_key.pem
    fi
fi

[ -z "$rnwif" ] && rnwif=$nwif
cfg=cdds-simple.xml
cat >$cfg <<EOF
<CycloneDDS>
  <Domain id="17">
    <General>
      <Interfaces><NetworkInterface name="\${nwif}" /></Interfaces>
      <EnableMulticastLoopback>$loopback</EnableMulticastLoopback>
      <MaxMessageSize>65500B</MaxMessageSize>
    </General>
EOF
if [ "$security" != "none" ] ; then
cat >>$cfg <<EOF
    <Security>
      <Authentication>
        <Library initFunction="init_authentication" finalizeFunction="finalize_authentication" path="dds_security_auth"/>
        <IdentityCA>file:\${secdir}/id_ca_cert.pem</IdentityCA>
        <IdentityCertificate>file:\${secdir}/sloth_cert.pem</IdentityCertificate>
        <PrivateKey>file:\${secdir}/sloth_priv_key.pem</PrivateKey>
      </Authentication>
      <Cryptographic>
        <Library initFunction="init_crypto" finalizeFunction="finalize_crypto" path="dds_security_crypto"/>
      </Cryptographic>
      <AccessControl>
        <Library initFunction="init_access_control" finalizeFunction="finalize_access_control" path="dds_security_ac"/>
        <PermissionsCA>file:\${secdir}/perm_ca_cert.pem</PermissionsCA>
        <Governance>file:\${secdir}/governance.p7s</Governance>
        <Permissions>file:\${secdir}/permissions.p7s</Permissions>
      </AccessControl>
    </Security>
EOF
fi
cat >>$cfg <<EOF
    <Internal>
      <SynchronousDeliveryPriorityThreshold>\${async:-0}</SynchronousDeliveryPriorityThreshold>
      <LeaseDuration>2s</LeaseDuration>
      <MinimumSocketReceiveBufferSize>6MB</MinimumSocketReceiveBufferSize>
      $watermarks
    </Internal>
    <Tracing>
      <Verbosity>fine</Verbosity>
      <Category>\${trace}</Category>
      <OutputFile>\${logdir}/cdds.log</OutputFile>
    </Tracing>
  </Domain>
</CycloneDDS>
EOF

localbindir=""
locallibdir=""
for x in "" /Release /RelWithDebInfo /Debug ; do
    if [ -x bin$x/ddsperf -a -f lib$x/libddsc$libsuffix ] ; then
        localbindir=bin$x
        locallibdir=lib$x
        break
    fi
done
if [ -z "$localbindir" -o -z "$locallibdir" ] ; then
    echo "ddsperf or libddsc$libsuffix not found on the local machine" >&2
    exit 1
fi

if [ ! -d $resultdir ] ; then
    mkdir $resultdir
elif $force ; then
    rm -rf $resultdir
    mkdir $resultdir
elif [ `ls $resultdir | wc -l` -gt 0 ] ; then
    echo "output directory $resultdir is non-empty" >&2
    exit 1
fi
[ -d $resultdir ] || { echo "output directory $resultdir doesn't exist" >&2 ; exit 1 ; }

if $provision ; then
    echo "provisioning ..."
    for r in $remotes ; do
        ssh $r mkdir -p $remotedir $remotedir/bin $remotedir/lib
        scp $locallibdir/libddsc$libsuffix $r:$remotedir/lib
        scp $localbindir/ddsperf $r:$remotedir/bin
    done
fi
if [ "$security" != "none" ] ; then
    ssh $r mkdir -p $remotedir $remotedir/secperf
    scp $secfiles $r:$remotedir/secperf
fi

topic=KS
[ -z "$sizelist" ] && topic=OU

export CYCLONEDDS_URI=file://$PWD/$cfg
for r in $remotes ; do
    scp $cfg $r:$remotedir || { echo "failed to copy $cfg to $remote:$PWD" >&2 ; exit 1 ; }
done

for async_mode in $asynclist ; do
    case "$async_mode" in
        sync) async=0 ;;
        async) async=1 ;;
        *) echo "$async_mode: invalid setting for ASYNC" >&2 ; continue ;;
    esac
    export async
    for sub_mode in $modelist ; do
        echo "======== ASYNC $async MODE $sub_mode ========="
        
        subpids=""
        netstat_pids=""
        trap dokill_and_exit SIGINT
        cat > run-remote.tmp <<EOF
export CYCLONEDDS_URI=file://$remotedir/$cfg
export async=$async
export nwif=$rnwif
export logdir=.
export secdir=./secperf
#export trace=trace,-content
cd $remotedir
remotebindir=""
for x in "" /Release /RelWithDebInfo Debug ; do
    if [ -x bin\$x/ddsperf ] ; then
        remotebindir=bin\$x
        break
    fi
done
if [ -z "\$remotebindir" ] ; then
    echo "ddsperf not found on remote machine" >&2
    exit 1
fi
#/usr/sbin/tcpdump -c 20000 -s 0 -w /tmp/x.pcap -i eth0 -Z erik 'udp[8:4]=0x52545053' & tcpdumppid=\$!
#/usr/sbin/tcpdump -c 20000 -s 0 -w /tmp/x.pcap -i eth0 -Z erik 'udp' & tcpdumppid=\$!
\$remotebindir/ddsperf -1l -X -d $rnwif$bandwidth -c -T $topic $remote_oper $sub_mode > $remote_oper.log & pid=\$!
echo \$pid > $operation-test-$remote_oper-\$pid.pid
wait \$pid
[ -n "\$tcpdumppid" ] && kill -INT \$tcpdumppid
EOF
        for r in $remotes ; do
            scp run-remote.tmp $r:$remotedir
            ssh $r ". $remotedir/run-remote.tmp" & subpids="$subpids $!"
        done

        outdir=$resultdir/$async_mode-$sub_mode
        mkdir $outdir
        rm -f $outdir/$local_oper.log
        export logdir=$outdir
        export secdir=./secperf
        if $netstats ; then
            ping `echo $1 | sed -e 's/.*@//'` > $outdir/icmpping.log & netstats_pids=$!
            if [ "`uname -s`" = "Darwin" ] ; then
                rm -f $outdir/netstat.log
                bash -c "while true ; do /System/Library/PrivateFrameworks/Apple80211.framework/Versions/Current/Resources/airport -I >> $outdir/netstat.log ; sleep 1; done" & netstats_pids="$netstats_pids $!"
            fi
        fi
        for size in ${sizelist:-0} ; do
            echo "size $size"
            #export trace=trace,-content
            $localbindir/ddsperf -Q minmatch:$# -Q initwait:3 \
                        -X -c -d $nwif$bandwidth \
                        -D $timeout -T $topic \
                        $local_oper size $size $rate | \
                tee -a $outdir/$local_oper.log
        done

        dokill
        wait
        for r in $remotes ; do
            scp $r:$remotedir/$remote_oper.log $outdir/$remote_oper-$r.log
        done

        if [ $operation = "throughput" ] ; then
            # write a summary, one line per second
            # col 0: raw network receive bandwidth (Mb/s)
            # col 1: appl receive bandwidth, 1s trailing average (Mb/s)
            # col 2: appl receive bandwidth, 10s trailing average (Mb/s)
            # (this assumes the network interface name is eth, en, or lo, optionally followed by a 0)
            perl -ne 'if(/size \d+ total.* (\d+\.\d+) Mb\/s.* (\d+\.\d+) Mb\/s/){$r=$1;$r10=$2;}if(/(?:eth|en|lo)0?: xmit.*? recv (\d+\.\d+)/){$rnet=$1;}if(/discarded\s+(\d+)/){printf "%f %f %f %u\n", $rnet, $r, $r10, $1;}' $outdir/sub-$1.log > $resultdir/summary-$async_mode-$sub_mode.txt
            perl -ne 'if(/(?:eth|en|lo)0?: xmit (\d+\.\d+) Mb\/s/){printf "%f\n", $1}' $outdir/pub.log > $resultdir/net-xmit-bytes.txt
            perl -ne 'print "$1 $2 $3 $4 $5\n" if /^(\d+)\s+(\d+)(\s+\d+)$/ || /^\[\d+\]\s+(\d+\.\d+)\s+discarded\s+\d+\s+rexmit\s+(\d+)\s+[A-Za-z_ ]+(\d+)[A-Za-z_ ]+(\d+)[A-Za-z_ ]+(\d+)$/' $outdir/pub.log > $resultdir/rexmit-bytes.txt
            if $netstats ; then
                perl -ne 'print "$1\n" if /time=([0-9.]+)/' $outdir/icmpping.log > $resultdir/lat.log
                if [ "`uname -s`" = "Darwin" ] ; then
                    perl -ne 'if(/CtlRSSI:\s*(-?\d+)/){$rssi=$1;}if(/CtlNoise:\s*(-?\d+)/){$noise=$1;}if(/maxRate:\s*(\d+)/){$max=$1;}if(/lastTxRate:\s*(\d+)/){$lasttx=$1;print "$max $lasttx $rssi $noise\n"}' $outdir/netstat.log > $resultdir/net.log
                fi
            fi
        fi
    done
done