trafficgen-server-start

#!/bin/bash
exec >trafficgen-server-start-stderrout.txt
exec 2>&1

. /usr/bin/trafficgen-base || (echo "/usr/bin/trafficgen-base not found"; exit 1)

dump_runtime
validate_label

# Defaults
sample_dir=`/bin/pwd`
switch_type="testpmd"
num_devices=
devices=()
res_devs=()
res_devs_node=()
dev_count_node=()
node_cpus=()
dev_macs=()
total_devs=0
testpmd_forward_mode="mac"
testpmd_queues="1"
testpmd_queues_per_pmd="1"
testpmd_descriptors="2048"
testpmd_smt_mode="grouped"
testpmd_smt="on"
testpmd_mtu="1518"
testpmd_enable_scatter="off"
testpmd_enable_rx_cksum="off"
testpmd_enable_rss_udp="off"
testpmd_devopt=""
testpmd_mbuf_size=""
testpmd_burst=""
testpmd_dst_macs=""

# Options processing
re='^(--[^=]+)=([^=]+)'
while [ $# -gt 0 ]; do
    if [[ "$1" =~ $re ]]; then
        arg="${BASH_REMATCH[1]}"
        val="${BASH_REMATCH[2]}"
        shift
    else
        arg="$1"
        shift
        val="$1"
        shift
    fi
    case "$arg" in
        --switch-type)
            switch_type="$val" # Can be "testpmd" or "null"
            ;;
        # The following two are needed to determine DPDK device IDs (0,1,2...)
        --server-devices)
            devices=(`echo "$val" | sed -e 's/,/ /g'`)
            num_devices=${#devices[@]}
            if [ $((num_devices%2)) -ne 0 ]; then
                exit_error "Number of devices ($num_devices) must be a multiple of 2, exiting" 1 "$sample_dir"
            fi
            ;;
        --testpmd-forward-mode)
            testpmd_forward_mode="$val"
            ;;
        --testpmd-queues)
            testpmd_queues="$val"
            ;;
        --testpmd-queues-per-pmd)
            testpmd_queues_per_pmd="$val"
            ;;
        --testpmd-descriptors)
            testpmd_descriptors="$val"
            ;;
        --testpmd-smt-mode)
            testpmd_smt_mode="$val"
            ;;
        --testpmd-smt)
            testpmd_smt="$val"
            ;;
        --testpmd-mtu)
            testpmd_mtu="$val"
            ;;
        --testpmd-devopt)
            testpmd_devopt="$val"
            ;;
        --testpmd-enable-scatter)
            testpmd_enable_scatter="$val"
            ;;
        --testpmd-enable-rx-cksum)
            testpmd_enable_rx_cksum="$val"
            ;;
        --testpmd-enable-rss-udp)
            testpmd_enable_rss_udp="$val"
            ;;
        --testpmd-mbuf-size)
            testpmd_mbuf_size="$val"
            ;;
        --testpmd-burst)
            testpmd_burst="$val"
            ;;
        --testpmd-dst-macs)
            testpmd_dst_macs="$val"
            ;;
    esac
done

if [ "$switch_type" == "testpmd" ]; then
    if [ -z "$devices" ]; then
        exit_error "Value for --server-devices was not found, exiting" 1 "$sample_dir"
    fi

    # trafficgen-server may receive a message about MACs from the traffic-generator
    echo "These files exist in ./msgs/rx:"
    /bin/ls -l msgs/rx
    file="msgs/rx/infra-start-end:1"
    if [ -e "$file" ]; then
        echo "Found $file:"
        cat $file
        testpmd_dst_macs=`jq -r '.macs[]' $file | tr "\n" "," | sed -e 's/,$//'`
    fi

    # Build testpmd cmdline opts for device selection
    echo "Resolving devices for testpmd based on devices [$devices]"
    testpmd_devices=""

    count=0
    for this_device in ${devices[@]}; do
        resolve_device this_res_dev $this_device
        res_devs[$count]=$this_res_dev
        this_node=$(cat "/sys/bus/pci/devices/${res_devs[$count]}/numa_node")
        if [ "$this_node" == "-1" ]; then
            echo "Reassigning NUMA node to 0 (from -1) for device $this_device"
            this_node=0
        fi
        res_devs_node[$count]=$this_node
        # Track total number of devices per node so CPUs can be allocated later
        if [ ! -n ${num_devs_node[$this_node]} ]; then
            num_devs_node[$this_node]=1
        else
            ((num_devs_node[this_node]++))
        fi
        testpmd_devices+=" --allow ${res_devs[$count]}"
        if [ -n "${testpmd_devopt}" ]; then
            testpmd_devices+=",${testpmd_devopt}"
        fi
        echo "Device $count: ${res_devs[$count]}, NUMA node: ${res_devs_node[$count]}"
        ((count++))
    done

    if [ -z "${testpmd_devices}" ]; then
        exit_error "Testpmd devices could not be found, exiting" 1 "$sample_dir"
    fi

    if [ -z "${HK_CPUS}" ] ; then
        exit_error "There are no housekeeping CPUs to use for testpmd" 1 "$sample_dir"
    else
        echo "HK_CPUS=${HK_CPUS}"
    fi

    # figure out the CPUs to use
    HK_CPUS_SEPARATED=$(echo "${HK_CPUS}" | sed -e "s/,/ /g")
    HK_CPUS_ARRAY=(${HK_CPUS_SEPARATED})
    if [ ${#HK_CPUS_ARRAY[@]} -lt 1 ]; then
        exit_error "You must have at least 1 HK_CPUS" 1 "$sample_dir"
    fi

    if [ -z "${WORKLOAD_CPUS}" ]; then
        exit_error "There are no dedicated/isolated CPUs to use for testpmd" 1 "$sample_dir"
    else
        echo "WORKLOAD_CPUS=${WORKLOAD_CPUS}"
    fi
    WORKLOAD_CPUS_SEPARATED=$(echo "${WORKLOAD_CPUS}" | sed -e "s/,/ /g")

    # Build a sequence of '--cpu' options for get-cpus-ordered.py that includes all of the workload cpus
    cpu_str=""
    for cpu in $WORKLOAD_CPUS_SEPARATED; do
        cpu_str+=" --cpu $cpu"
    done

    # Find the number of cpus needed per NUMA node, based on number of devices per node (and queues/pmd)
    #
    # First CPU is housekeeping for testpmd main thread
    testpmd_cpus="0@${HK_CPUS_ARRAY[0]},"
    cpu_idx=0
    for this_node in ${!num_devs_node[@]}; do
        echo "finding cpus for node [$this_node]"
        num_devs=${num_devs_node[$this_node]}
        ((total_devs+=num_devs))
        echo "num_devs: [$num_devs]"
        num_cpus=`echo "scale=0; $num_devs*$testpmd_queues/$testpmd_queues_per_pmd" | bc`
        # In case truncated to 0
        if [ $num_cpus -eq 0 ]; then
            num_cpus=1
        fi
        echo "num_cpus: [$num_cpus]"
        echo "$TOOLBOX_HOME/bin/get-cpus-ordered.py --smt $testpmd_smt --smt-enumeration $testpmd_smt_mode --numa-node $this_node $cpu_str"
        this_node_cpus=$($TOOLBOX_HOME/bin/get-cpus-ordered.py\
                                      --smt $testpmd_smt\
                                      --smt-enumeration $testpmd_smt_mode\
                                      --numa-node $this_node $cpu_str)
        echo "output:"
        echo "$this_node_cpus"
        this_node_cpus_trimmed=`echo -e "$this_node_cpus" | grep "final cpus:" | awk '{ print $3 }' | cut -d, -f-$num_cpus | sed -e "s/,/ /g"`
        echo "this_node_cpus_trimmed: [ $this_node_cpus_trimmed ]"
        for this_cpu in `echo $this_node_cpus_trimmed | sed -e 's/,/ /g'`; do
            ((cpu_idx++))
            testpmd_cpus+="$cpu_idx@$this_cpu,"
        done
    done
    testpmd_cpus=$(echo "$testpmd_cpus" | sed -e "s/,$//")


    # Detect what the testpmd numa memory configuration should be
    if pushd /sys/devices/system/node > /dev/null; then
        testpmd_mem=""

        for node in $(ls -1d node*); do
            NODE_NUM=$(echo ${node} | sed -e "s/node//")
            if echo " "${!num_devs_node[@]}" " | grep -q -P "\s$NODE_NUM\s"; then
                testpmd_mem+="1024,"
            else
                testpmd_mem+="0,"
            fi
        done

        testpmd_mem=$(echo "${testpmd_mem}" | sed -e "s/,$//")

        popd > /dev/null
    fi

    testpmd_output="trafficgen-testpmd-stderrout.txt"
    testpmd_bin=$( command -v dpdk-testpmd || command -v testpmd )
    testpmd_opts=" --lcores ${testpmd_cpus}"
    testpmd_opts+=" --file-prefix ${cs_label}"
    testpmd_opts+=" --socket-mem ${testpmd_mem}"
    testpmd_opts+=" --huge-dir /dev/hugepages ${testpmd_devices}"
    testpmd_opts+=" -v"
    testpmd_opts+=" --"
    testpmd_opts+=" --nb-ports $total_devs --nb-cores ${cpu_idx} --auto-start --stats-period=5"
    testpmd_opts+=" --rxq ${testpmd_queues} --txq ${testpmd_queues}"
    testpmd_opts+=" --rxd ${testpmd_descriptors} --txd ${testpmd_descriptors}"
    testpmd_opts+=" --max-pkt-len=${testpmd_mtu}"
    testpmd_opts+=" --record-core-cycles"
    testpmd_opts+=" --record-burst-stats"
    if [ "$testpmd_enable_scatter" == "on" ]; then
        testpmd_opts+=" --enable-scatter"
    elif [ "$testpmd_enable_scatter" == "off" ]; then
        echo "--enable-scatter will not be used"
    else
        exit_error "value for --testpmd-enable-scatter ($testpmd_enable_scatter) is not valid.  Use either 'on' or 'off'"
    fi

    if [ "$testpmd_enable_rx_cksum" == "on" ]; then
        testpmd_opts+=" --enable-rx-cksum"
    elif [ "$testpmd_enable_rx_cksum" == "off" ]; then
        echo "--enable-rx-cksum will not be used"
    else
        exit_error "value for --testpmd-enable-rx-cksum ($testpmd_enable_rx_cksum) is not valid.  Use either 'on' or 'off'"
    fi

    if [ "$testpmd_enable_rss_udp" == "on" ]; then
        testpmd_opts+=" --rss-udp"
    elif [ "$testpmd_enable_rss_udp" == "off" ]; then
        echo "--enable-rss-udp will not be used"
    else
        exit_error "value for --testpmd-enable-rss-udp ($testpmd_enable_rss_udp) is not valid.  Use either 'on' or 'off'"
    fi

    if [ ! -z "$testpmd_burst" ]; then
        testpmd_opts+=" --burst=$testpmd_burst"
    fi

    # Bump up the default mbuf if the MTU is large,
    # but allow a user-defined mbuf size to not be changed
    if [ "${testpmd_mtu}" -gt 2048 -a -z "$testpmd_mbuf_size" ]; then
        testpmd_mbuf_size=16384
        testpmd_opts+=" --mbuf-size=${testpmd_mbuf_size}"
    fi

    if [ ! -z "$testpmd_mbufs" ]; then
        testpmd_opts+=" --total-num-mbufs=${testpmd_mbufs}"
    fi

    if [ "$testpmd_forward_mode" == "mac" ]; then
        mac_file="msgs/rx/infra-start-end:1"
        if [ -z "$testpmd_dst_macs" ]; then
            exit_error  "[ERROR] Using forware-mode = mac, but did not get MAC addresses from TREX or --testpmd-dst-macs" 1 "$sample_dir"
        fi
        idx=0
        for this_mac in `echo $testpmd_dst_macs | sed -e 's/,/ /g'`; do
            testpmd_opts+=" --eth-peer $idx,$this_mac"
            ((idx++))
        done
        testpmd_opts+=" --forward-mode mac"
        if [ $idx -ne $total_devs ]; then
            exit_error "The number of destination macs did not match the number of devices for testpmd.  In order to use forward-mode=mac, the number of TRex devices and number of testpmd devices must match exactly." 1 $sample_dir
        fi
    else
        testpmd_opts+=" "
    fi


    echo "Going to run: $testpmd_bin $testpmd_opts"
    ( ${testpmd_bin} ${testpmd_opts} 2>&1 & echo $! > trafficgen-server.pid ) | ${TOOLBOX_HOME}/bin/timestamper.py > ${testpmd_output} &
    sleep 5 # TODO: need a better wait
    if [ ! -e $testpmd_output ]; then
        exit_error "Could not find testpmd output file: [$testpmd_output]" 1 $sample_dir
    fi

    # Find the MACs for all devices, to be sent to TRex
    if [ "$testpmd_forward_mode" == "mac" ]; then
        sleep 10 # TODO: need a better wait
        if [ ! -e $testpmd_output ]; then
            exit_error "Could not find testpmd output file: [$testpmd_output]" 1 $sample_dir
        fi
        echo '{"recipient":{"type":"all","id":"all"},"user-object":{"macs":[' >>msgs/tx/svc
        dev_idx=0
        while [ $dev_idx -lt $total_devs ]; do
            this_mac=`egrep "Port $dev_idx: [A-Fa-f0-9]{2}:[A-Fa-f0-9]{2}:[A-Fa-f0-9]{2}:[A-Fa-f0-9]{2}:[A-Fa-f0-9]{2}:[A-Fa-f0-9]{2}" $testpmd_output | awk -F"$dev_idx: " '{print $2}'`
            echo "MAC$dev_idx: $this_mac"
            ## MAC info, to be sent to endpoint and then forwarded to the client
            echo '"'$this_mac'"' >>msgs/tx/svc
            if [ $dev_idx -lt $((total_devs-1)) ]; then
                echo ', ' >>msgs/tx/svc
            fi
            dev_mac[$dev_idx]=$this_mac
            let dev_idx=$dev_idx+1
        done
        echo "]}}" >>msgs/tx/svc
    fi
elif [ "$switch_type" == "null" ]; then
    echo "Using switch_type null, no setup required"
else
    exit_error "Invalid switch type used: $switch_type"
fi