iperf-server-start

#!/bin/bash
exec >iperf-server-stderrout.txt 
exec 2>&1

. /usr/bin/iperf-base || (echo "/usr/bin/iperf-base not found"; exit 1)

dump_runtime
validate_label
validate_sw_prereqs jq ss iperf3 getopt ip

# defaults
control_port=""
data_port=""
ifname=""
cpu_pin=""
ipv="4"

longopts="ipv:,ifname:,cpu-pin:,bitrate:,passthru:"
opts=$(getopt -q -o "" --longoptions "$longopts" -n "getopt.sh" -- "$@");
if [ $? -ne 0 ]; then
    exit_error  "Unrecognized option specified"
fi
eval set -- "$opts";
echo "opts= $opts"
while true; do
    case "$1" in
	    --cpu-pin)
	        shift;
	        cpu_pin=$1
	        echo "cpu_pin=${cpu_pin}"
	        shift;
	        ;;
	    --ipv)
	        shift;
	        ipv=$1
	        echo "ipv=${ipv}"
	        shift;
	        ;;
        --ifname)
            shift;
            ifname="$1"
            echo "ifname=$ifname"
            shift
            ;;
        --protocol|--length|--remotehost|--time|--bitrate)
            shift;
            shift;
            ;;
        --passthru) # debug mode iperf options that user just wants us to pass-thru
            shift;
            pt_opts=$1
            echo "pt_opts=$pt_opts"
            shift
            ;;
        --)
            shift;
            break
            ;;
        *)
            exit_error "Invalid option: $1"
    esac
done

case "${cpu_pin}" in
    ""|"numa"|"cpu-numa")
	;;
    *)
	exit_error "unsupported cpu-pin value '${cpu_pin}'"
	;;
esac

uname -a

# A benchmark server can provide both an IP and ports to the client.
# The server must package these in a JSON, which will be sent to
# the endpoint.  The endpoint may make adjustments to the IP and
# ports depending on the network and security model.

# There are always 2 ports for these iperf tests, and ports should always
# start at 30000 to avoid port number reassignment
let "control_port = 2 * $id"
let "control_port = $control_port + 30000"
let "data_port = $control_port + 1"

# If --remotehost is not used by the user, they can pass --ifname, a netdev name
# like "eth0", and the server (below) will try to resolve this to the actual IP address.
# This allows the user to not know the IP address in advance.
ip=""
ip -j a >ip.json
if [ ! -z "$ifname" ]; then
    if echo $ifname | grep -q '%id:'; then
        # We have the role ID variable embedded, so we need to extract the value
        before_var=`echo $ifname | sed -e 's/\(.*\)%id:.*%/\1/'`
        after_var=`echo $ifname | sed -e 's/.*%id:.*%\(.*\)/\1/'`
        var=`echo $ifname | sed -e 's/%id:\(.*\)%/\1/' | jq -r '."'$id'"'`
        ifname="${before_var}${var}${after_var}"
    fi
    if [ "${ifname}" == "default-route" ]; then
       echo "Looking for default route interface..."
       ip -j route > route.json
       ifname=$(jq --raw-output 'to_entries | .[] | if (.value.dst == "default") then .value.dev else null end' route.json | grep -v null)
       if [ -z "${ifname}" ]; then
           exit_error "could not find default route interface"
       else
           echo "Found default route on interface ${ifname}"
       fi
    else
       echo "--ifname=${ifname} was found"
    fi
    echo "Searching for IP for interface ${ifname}..."

    echo "--ifname=$ifname was found, searching for IP"
    # Find the element in the 'ip a' array that matches the ifname we are looking for
    ifname_idx=`jq 'to_entries | .[] | if (.value.ifname == "'$ifname'") then .key else null end' ip.json | grep -v null`
    if [ ! -z "$ifname_idx" ]; then
        if [ "$ipv" == "4" ]; then
            echo "$ifname matches index $ifname_idx, looking for ipv4 (inet) in addr_info:"
            jq '.['$ifname_idx'].addr_info' ip.json
            # Find the first ipv4 address
            addr_idx=`jq '.['$ifname_idx'].addr_info | to_entries | .[] | if (.value.family == "inet") then .key else null end' ip.json | grep -v null | head -1`
            if [ ! -z $addr_idx ]; then
                echo "Found inet in index $addr_idx:"
                jq -r '.['$ifname_idx'].addr_info['$addr_idx']' ip.json
                ip=`jq -r '.['$ifname_idx'].addr_info['$addr_idx'].local' ip.json`
            else
                addr_info=`jq '.['$ifname_idx'].addr_info' ip.json`
                exit_error "An IPV4 address could not be found for $ifname in addr_info: $addr_info"
            fi
        elif [ "$ipv" == "6" ]; then
            echo "$ifname matches index $ifname_idx, looking for ipv6 (inet6) in addr_info:"
            jq '.['$ifname_idx'].addr_info' ip.json
            # Find the first ipv4 address
            addr_idx=`jq '.['$ifname_idx'].addr_info | to_entries | .[] | if (.value.family == "inet6") then .key else null end' ip.json | grep -v null | head -1`
            if [ ! -z $addr_idx ]; then
                echo "Found inet in index $addr_idx:"
                jq -r '.['$ifname_idx'].addr_info['$addr_idx']' ip.json
                ip=`jq -r '.['$ifname_idx'].addr_info['$addr_idx'].local' ip.json`
            else
                addr_info=`jq '.['$ifname_idx'].addr_info' ip.json`
                exit_error "An IPV6 address could not be found for $ifname in addr_info: $addr_info"
            fi
        else
            exit_error "The value for --ipv, $ipv, is not supported.  Use 4 for ipv4 or 6 for ipv6"
        fi
    else
        interfaces=`jq -r '. | to_entries | .[] | .value.ifname' ip.json`
        exit_error "could not find netdev $ifname, interfaces are: $interfaces"
    fi
    if [ -z "$ip" ]; then
        exit_error "IP was not found from $ifname"
    else
        echo "IP was found: $ip"
    fi
else
    echo "Not going to look for an IP since --ifname was not used."
fi

# Queue ip/port info, to be sent to endpoint
echo '{"recipient":{"type":"all","id":"all"},"user-object":{"svc":{"ip":"'$ip'","ports":['$control_port,$data_port']}}}' >msgs/tx/svc

echo "Existing listening ports:"
ss -tlnp
echo

# Kill any existing process using this port
echo "Attempt to kill port $control_port via 'ss' if it exists:"
ss -tlnpK '( sport = '$control_port' )'
echo

pid=`ss -Otlnp '( sport = '$control_port' )' | awk '{print $6}' | grep -v Peer | awk -Fpid= '{print $2}' | awk -F, '{print $1}' | sort | uniq`
if [ ! -z "$pid" ]; then
    echo "Trying to force kill PID $pid with 'kill -9'"
    kill -9 $pid
fi

echo "Existing listening ports (after kill):"
ss -tlnp
echo
#TODO: exit with error if ports still used

ifname_numa_node=
ifname_numa_node_cpus=
if [ -e /sys/class/net/${ifname}/device/numa_node ]; then
    ifname_numa_node=$(cat /sys/class/net/${ifname}/device/numa_node)
    if [ "$ifname_numa_node" == "-1" ]; then
        # VM case, use numa 0
        ifname_numa_node=0
    fi
else
    # When topo=internode topo, the device is eth0. Default it to numa node0.
    # In other word, if there is no numa associated with ifname, just use numa 0
    ifname_numa_node=0
fi

ifname_numa_node_cpus=$(cat /sys/devices/system/node/node${ifname_numa_node}/cpulist)
# convert to comma separated list if there are ranges i.e 0-5 items
ifname_numa_node_cpus=$(echo $ifname_numa_node_cpus | perl -pe 's/(\d+)-(\d+)/join(",",$1..$2)/eg')
echo "ifname=${ifname}"
echo "ifname_numa_node=${ifname_numa_node}"
echo "ifname_numa_node_cpus=${ifname_numa_node_cpus}"

cpu_pin_cmd=""
case "${cpu_pin}" in
    "numa")
        cpu_pin_cmd="taskset --cpu-list ${ifname_numa_node_cpus}"
        ;;
    "cpu-numa")
        assign-pin-cpu server ${ifname_numa_node_cpus}
        ;;
esac

passthru_opts=$(echo "$pt_opts" | sed 's/,/ /g')
cmd="${cpu_pin_cmd} iperf3 -s --format k -p $control_port $passthru_opts"
if [ "$ipv" == "6" ]; then
    cmd+=" -6"
fi
echo "going to run: $cmd"
export MALLOC_CHECK_=2
$cmd 2>&1 >iperf-server-result.txt &
pid=$!
echo $pid >iperf-server.pid
# Wait a little bit to see if the server errored out
sleep 10
if grep -q -i error iperf-server-result.txt; then
    exit_error "Server failed to start: `grep -i error iperf-server-results.txt`"
fi
echo ps:
ps aux | grep iperf3
echo
echo ss -tlnp:
ss -tlnp
exit 0