test/scripts/core_pinning.sh

#!/bin/bash
#
# core_pinning_auto.sh -- script to test vpp (debug-build) core-pinning
#                      -- in bare-metal, containers (docker, lxc)
#
DOCKER_CONTAINER_NAME="vpp_core_pinning"
VPP_SOCK_PATH=/run/vpp
CONTAINER_CPU_RANGE="4-7"
TEST_SUCCESS=0
TEST_FAIL=0
if [ ! $WS_ROOT ]
then
	if [ ! -d "../../../vpp" ]; then
		echo "VPP workspace path invalid"
		echo "Please execute script from vpp/test/scripts folder.."
		exit 1
	fi
	WS_ROOT="$(dirname $(readlink -e "../../../vpp"))/$(basename "../../../vpp")"
fi
# Get available CPU count on host machine
host_cpulist=$(cat /sys/devices/system/cpu/online)
startcpu="${host_cpulist%-*}"
endcpu="${host_cpulist#*\-}"
cpucount="$(($endcpu - $startcpu + 1))"
if [ $cpucount -lt 8 ]
then
	echo "Current host machine has $cpucount CPUs"
    echo "A minimum of 8 CPUs is required to run testcases, exiting.."
    exit 1
fi
# Check that container 'vpp_core_pinning' does not already exist
count=$(docker ps -a | grep -c "$DOCKER_CONTAINER_NAME")
if [ $count -ne 0 ]
then
	echo "Error: docker container $DOCKER_CONTAINER_NAME already exists"
	echo "Remove it using 'docker stop/docker rm', then re-run test"
	exit 1
fi
# Check that there is no vpp instance currently running on the machine
count=$(pgrep vpp | wc -l)
if [ $count -ne 0 ]
then
	echo "Error: a vpp instance is currently running on this machine"
	echo "Please stop the running instance, then re-run test"
	exit 1
fi
mkdir -p $VPP_SOCK_PATH

# Function to parse main core
parse_maincore () {
    main_core_args=$1
    main_core_parsed=$main_core_args
    if [ $main_core_args = "auto" ];
    then
        main_core_parsed="0"
        if [ -n "$SKIP_CORE" ]
        then
            main_core_parsed=$(($main_core_parsed + $SKIP_CORE))
        fi
        if [ -n "$CONTAINER_RESTRAIN_CPUSET" ]
        then
            main_core_parsed=(${container_cpus[ $main_core_parsed ]})
        fi
    fi
    echo $main_core_parsed
}

# Function to parse n workers range to an array
# e.g. "4" is parsed to ('0','1','2','3')
parse_workers_n () {
    workers_n_args=$1
    workers_n_parsed=()
    main_core_increment="0"
    skip_core_increment="0"
    if [ -n "$SKIP_CORE" ]
        then
            skip_core_increment=$(($SKIP_CORE))
        fi

    for ((i=0;i<$workers_n_args;i++)); do

    if [ -n "$CONTAINER_RESTRAIN_CPUSET" ]
        then
        if [ $(( ${container_cpus[ $(($i + $skip_core_increment)) ]})) -eq $(("$parsed_main_core")) ]
        then
            main_core_increment=$(($main_core_increment + 1))
        fi
            workers_n_parsed+=" ${container_cpus[ $(($i + $main_core_increment + $skip_core_increment)) ]}"
        else
        if [ $(( $skip_core_increment + $i)) -eq $(("$parsed_main_core")) ]
        then
            main_core_increment=$(($main_core_increment + 1))
        fi
            workers_n_parsed+=" $(($i + $main_core_increment + $skip_core_increment))"
        fi
    done
    echo $workers_n_parsed
}

# Function to parse corelist range to an array
# e.g. "0,3-5,7" is parsed to ('0','3','4','5','7')
parse_corelist () {
    corelist_args=$1
    corelist_args=$(echo $corelist_args | grep -Po '[0-9]+-[0-9]+|[0-9]+')
    corelist_parsed=()
    for corelist_elt in ${corelist_args[@]};do
        if [ $(echo $corelist_elt | grep -Po '[0-9]+-[0-9]+') ]
        then
            startcpu="${corelist_elt%-*}"
            endcpu="${corelist_elt#*\-}"
            cpucount="$(($endcpu - $startcpu))"
            for ((i=0;i<=$cpucount;i++)); do
              corelist_parsed+=" $(($i+$startcpu))"
            done
        elif [ $(echo $corelist_elt | grep -Po '[0-9]+') ]
        then
            corelist_parsed+=" ${corelist_elt}"
        fi
    done
    echo $corelist_parsed
}
# Test VPP core pinning configuration
test_pinning_conf () {
    VPP_CPU_EXTRA_OPTIONS=""
	if [ -n "$CORELIST_WORKERS" ];
	then
		VPP_CPU_EXTRA_OPTIONS=" corelist-workers ${CORELIST_WORKERS}"
	fi
    if [ -n "$WORKERS_AUTO" ];
	then
		VPP_CPU_EXTRA_OPTIONS=" workers ${WORKERS_AUTO}"
	fi
    if [ -n "$SKIP_CORE" ];
	then
		VPP_CPU_EXTRA_OPTIONS="${VPP_CPU_EXTRA_OPTIONS} skip-cores ${SKIP_CORE}"
	fi
	echo "TEST - conf 'cpu {main-core ${MAIN_CORE} ${VPP_CPU_EXTRA_OPTIONS}}'"
	if [ -z "$CONTAINER_RESTRAIN_CPUSET" ];
	then
		VPP_CONTAINER_CPUSET=""
		echo "(Running vpp in container with full host cpuset $host_cpulist)"
	else
		VPP_CONTAINER_CPUSET="--cpuset-cpus $CONTAINER_CPU_RANGE"
		echo "(Running vpp in container with limited cpuset $CONTAINER_CPU_RANGE)"
	fi
	(docker run -d ${VPP_CONTAINER_CPUSET} --name="$DOCKER_CONTAINER_NAME" \
	-e LD_LIBRARY_PATH="/vpp/build-root/build-vpp_debug-native/vpp/lib/x86_64-linux-gnu/" -v $VPP_SOCK_PATH:$VPP_SOCK_PATH \
	-v $WS_ROOT:/vpp  ubuntu:22.04 sh -c "/vpp/build-root/build-vpp_debug-native/vpp/bin/vpp unix {interactive \
	nodaemon cli-listen $VPP_SOCK_PATH/cli.sock} cpu {main-core ${MAIN_CORE} ${VPP_CPU_EXTRA_OPTIONS} } plugins \
	{ plugin dpdk_plugin.so {disable } }" > /dev/null )
	sleep 3 # wait for VPP to initialize socket
	# Change access permissions on vpp cli socket
	# docker exec -it "$DOCKER_CONTAINER_NAME" /bin/bash -c "chmod 777  $VPP_SOCK_PATH/cli.sock"  > /dev/null
	# check if vppctl can connect to vpp container instance
	$WS_ROOT/build-root/build-vpp_debug-native/vpp/bin/vppctl -s $VPP_SOCK_PATH/cli.sock show threads  1> /dev/null
	# get CPUs vpp instance in container is running on
	taskset_vpp_cpus=($( taskset --all-tasks -pc $(pgrep vpp) | grep -e ".$" -o))
	rc=$?
	# parse list of user requested CPUs for vpp
	requested_cpus=()
    parsed_main_core=$(parse_maincore ${MAIN_CORE})
	requested_cpus+=($parsed_main_core)
    if [ -n "$CORELIST_WORKERS" ];
    then
	    requested_cpus+=($(parse_corelist ${CORELIST_WORKERS}))
    fi
    if [ -n "$WORKERS_AUTO" ];
    then
	    requested_cpus+=($(parse_workers_n ${WORKERS_AUTO}))
    fi

	# parse list of expected CPUs used by vpp
	expected_cpu_mapping=()
    expected_cpu_mapping=("${requested_cpus[@]}")
	echo "CPUs requested by user:      [${requested_cpus[@]}]"
	echo "--------------------"
	echo "Expected CPU Mapping:        [${expected_cpu_mapping[@]}]"
	echo "VPP pinning (taskset):       [${taskset_vpp_cpus[@]}]"
	#check if expected CPU mapping matches CPUs vpp instance in container is running on
	failure_cond=""
	for index in ${!taskset_vpp_cpus[@]}; do
		if [ ${taskset_vpp_cpus[$index]} -ne  ${expected_cpu_mapping[ $index ]} ]
		then
			failure_cond="t"
		fi
	done
	if [ $rc -eq 0 ] && [ -z "$failure_cond" ]
	then
		echo "Test Successful"
		TEST_SUCCESS=$(($TEST_SUCCESS+1))
	else
		echo "Test Failed"
		TEST_FAIL=$(($TEST_FAIL+1))
	fi
	echo "=============================================="
	echo " "
	# Stop & destroy container instance
	docker stop $DOCKER_CONTAINER_NAME  &> /dev/null
	docker rm -f $DOCKER_CONTAINER_NAME &> /dev/null
}
test_invalid_conf () {
	if [ -n "$CORELIST_WORKERS" ];
	then
		VPP_CPU_EXTRA_OPTIONS=" corelist-workers ${CORELIST_WORKERS}"
	fi
    if [ -n "$WORKERS_AUTO" ];
	then
		VPP_CPU_EXTRA_OPTIONS=" workers ${WORKERS_AUTO}"
	fi
    if [ -n "$SKIP_CORE" ];
	then
		VPP_CPU_EXTRA_OPTIONS="${VPP_CPU_EXTRA_OPTIONS} skip-cores ${SKIP_CORE}"
	fi
	echo "TEST - conf 'cpu {main-core ${MAIN_CORE} ${VPP_CPU_EXTRA_OPTIONS}}'"
	if [ -z "$CONTAINER_RESTRAIN_CPUSET" ];
	then
		VPP_CONTAINER_CPUSET=""
		echo "(Running vpp in container with full host cpuset $host_cpulist)"
	else
		VPP_CONTAINER_CPUSET="--cpuset-cpus $CONTAINER_CPU_RANGE"
		echo "(Running vpp in container with limited cpuset $CONTAINER_CPU_RANGE)"
	fi
	(docker run -d --cpuset-cpus $CONTAINER_CPU_RANGE --name="$DOCKER_CONTAINER_NAME" \
	-e LD_LIBRARY_PATH="/vpp/build-root/build-vpp_debug-native/vpp/lib/x86_64-linux-gnu/" -v $VPP_SOCK_PATH:$VPP_SOCK_PATH \
	-v $WS_ROOT:/vpp  ubuntu:22.04 sh -c "/vpp/build-root/build-vpp_debug-native/vpp/bin/vpp unix {interactive \
	nodaemon cli-listen $VPP_SOCK_PATH/cli.sock} cpu {main-core ${MAIN_CORE} ${VPP_CPU_EXTRA_OPTIONS}} plugins \
	{ plugin dpdk_plugin.so {disable } }"  > /dev/null)
	sleep 3 # wait for vpp to initialize socket
	# check if vpp launched with invalid configuration
	taskset --all-tasks -pc $(pgrep vpp) &> /dev/null
	rc=$?
	if [ $rc -eq 1 ]
	then
        echo " "
		echo "OK... VPP did not launch with invalid configuration"
		TEST_SUCCESS=$(($TEST_SUCCESS+1))
	else
        echo " "
		echo "Failure... VPP launched with wrong configuration"
		TEST_FAIL=$(($TEST_FAIL+1))
	fi
	echo "=============================================="
	echo " "
	# Stop & destroy container instance
	docker stop $DOCKER_CONTAINER_NAME  &> /dev/null
	docker rm -f $DOCKER_CONTAINER_NAME &> /dev/null
}
run_tests () {
	container_cpus=($(parse_corelist ${CONTAINER_CPU_RANGE}))
	echo "TESTING VALID CORE PINNING CONFIGURATIONS"
	echo " "
    WORKERS_AUTO=""
    SKIP_CORE=""
	CONTAINER_RESTRAIN_CPUSET=""
	CORELIST_WORKERS="1-3"
	MAIN_CORE="0"
	test_pinning_conf
    WORKERS_AUTO=""
    SKIP_CORE=""
	CONTAINER_RESTRAIN_CPUSET=""
	CORELIST_WORKERS="0,2-3"
	MAIN_CORE="1"
	test_pinning_conf
    WORKERS_AUTO=""
    SKIP_CORE=""
	CONTAINER_RESTRAIN_CPUSET=""
	CORELIST_WORKERS="0-2"
	MAIN_CORE="3"
	test_pinning_conf
    WORKERS_AUTO="2"
    SKIP_CORE=""
	CONTAINER_RESTRAIN_CPUSET=""
    CORELIST_WORKERS=""
	MAIN_CORE="auto"
	test_pinning_conf
    WORKERS_AUTO="3"
    SKIP_CORE=""
	CONTAINER_RESTRAIN_CPUSET="t"
    CORELIST_WORKERS=""
	MAIN_CORE="auto"
	test_pinning_conf
    WORKERS_AUTO="2"
    SKIP_CORE="1"
	CONTAINER_RESTRAIN_CPUSET="t"
    CORELIST_WORKERS=""
	MAIN_CORE="auto"
	test_pinning_conf
    WORKERS_AUTO="2"
    SKIP_CORE=""
	CONTAINER_RESTRAIN_CPUSET="t"
    CORELIST_WORKERS=""
	MAIN_CORE="5"
	test_pinning_conf
	echo "TESTING NON-VALID CORE PINNING CONFIGURATIONS"
	echo " "
    WORKERS_AUTO=""
    SKIP_CORE=""
	CONTAINER_RESTRAIN_CPUSET="t"
	CORELIST_WORKERS="1-3"
	MAIN_CORE="0"
	test_invalid_conf
	WORKERS_AUTO="3"
    SKIP_CORE="1"
	CONTAINER_RESTRAIN_CPUSET="t"
	CORELIST_WORKERS=""
	MAIN_CORE="auto"
	test_invalid_conf
	WORKERS_AUTO="5"
    SKIP_CORE=""
	CONTAINER_RESTRAIN_CPUSET="t"
	CORELIST_WORKERS=""
	MAIN_CORE="auto"
	test_invalid_conf
	WORKERS_AUTO=""
    SKIP_CORE="4"
	CONTAINER_RESTRAIN_CPUSET="t"
	CORELIST_WORKERS=""
	MAIN_CORE="auto"
	test_invalid_conf
	echo " "
	echo "========================"
	echo "RESULTS:"
	echo "SUCCESS: $TEST_SUCCESS"
	echo "FAILURE: $TEST_FAIL"
	echo "========================"
	echo " "
}
run_tests