tests: support multiple filter expressions
Support multiple comma-delimited filter expressions,
e.g. to run both bfd and ip4 tests, it's now possible to do:
make test TEST=bfd,ip4
Same goes for wildcards, e.g.:
make test TEST=bfd,..test_longest_prefix_match,..test_icmp_error
Type: improvement
Change-Id: I0cceaa443cb612dca955f301c7407959f9a71a6e
Signed-off-by: Klement Sekera <klement.sekera@gmail.com>
diff --git a/test/Makefile b/test/Makefile
index e5e9975..4418740 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -404,52 +404,165 @@
@echo " test-shell-debug - enter shell with test environment (debug build)"
@echo " test-refresh-deps - refresh the Python dependencies for the tests"
@echo ""
- @echo "Arguments controlling test runs:"
+ @echo "Environment variables controlling test runs:"
@echo ""
- @echo " V=[0|1|2] - set test verbosity level"
- @echo " 0=ERROR, 1=INFO, 2=DEBUG"
- @echo " TEST_JOBS=[<n>|auto] - use at most <n> parallel python processes for test execution, if auto, set to number of available cpus (default: 1)"
- @echo " MAX_VPP_CPUS=[<n>|auto]- use at most <n> cpus for running vpp main and worker threads, if auto, set to number of available cpus (default: auto)"
- @echo " CACHE_OUTPUT=[0|n|no] - disable cache VPP stdout/stderr and log as one block after test finishes (default: yes)"
- @echo " FAILFAST=[1|y|yes] - fail fast if 1, otherwise complete all tests"
- @echo " TIMEOUT=<timeout> - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)"
- @echo " RETRIES=<n> - retry failed tests <n> times"
- @echo " DEBUG=<type> - set VPP debugging kind"
- @echo " DEBUG=core - detect coredump and load it in gdb on crash"
- @echo " DEBUG=gdb - allow easy debugging by printing VPP PID"
- @echo " and waiting for user input before running"
- @echo " and tearing down a testcase"
- @echo " DEBUG=gdbserver - run gdb inside a gdb server, otherwise"
- @echo " same as above"
- @echo " DEBUG=attach - attach test case to already running vpp in gdb (see test-start-vpp-in-gdb)"
- @echo " STEP=[1|y|yes] - enable stepping through a testcase (for testcase debugging)"
- @echo " SANITY=[0|n|no] - disable sanity import of vpp-api/sanity vpp run before running tests"
- @echo " EXTENDED_TESTS=[1|y|yes] - run extended tests"
- @echo " TEST=<filter> - filter the set of tests:"
- @echo " by file-name - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py"
- @echo " by file-suffix - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py"
- @echo " by wildcard - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'"
- @echo " e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name"
- @echo " TEST='bfd.*.*' is equivalent to above example of filter by file-suffix"
- @echo " TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class"
- @echo " TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase"
- @echo " TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes"
- @echo " VARIANT=<variant> - specify which march node variant to unit test"
- @echo " e.g. VARIANT=skx test the skx march variants"
- @echo " e.g. VARIANT=icl test the icl march variants"
- @echo " COREDUMP_SIZE=<size> - pass <size> as unix { coredump-size <size> } argument to vpp"
- @echo " e.g. COREDUMP_SIZE=4g"
- @echo " COREDUMP_SIZE=unlimited"
- @echo " COREDUMP_COMPRESS=[1|y|yes] - compress core files if not debugging them"
- @echo " EXTERN_TESTS=<path> - path to out-of-tree test_<name>.py files containing test cases"
- @echo " EXTERN_PLUGINS=<path> - path to out-of-tree plugins to be loaded by vpp under test"
- @echo " EXTERN_COV_DIR=<path> - path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report"
- @echo " PROFILE=[1|y|yes] - enable profiling of test framework via cProfile module"
- @echo " PROFILE_SORT_BY=opt - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime)"
- @echo " PROFILE_OUTPUT=file - output profiling info to file - use absolute path (default: stdout)"
- @echo " TEST_DEBUG=[1|y|yes] - enable debugging of the test framework itself (expert)"
- @echo " API_FUZZ=[1|y|yes] - enable VPP api fuzz testing"
- @echo " RND_SEED=<seed> - Seed RND with given seed"
+ @echo " V=[0|1|2]"
+ @echo " set test verbosity level: 0=ERROR, 1=INFO, 2=DEBUG"
+ @echo ""
+ @echo " TEST_JOBS=[<n>|auto]"
+ @echo " use at most <n> parallel python processes for test"
+ @echo " execution, if auto, set to number of available cpus"
+ @echo " (default: 1)"
+ @echo ""
+ @echo " MAX_VPP_CPUS=[<n>|auto]"
+ @echo " use at most <n> cpus for running vpp"
+ @echo " 'auto' sets to number of available cpus"
+ @echo " (default: auto)"
+ @echo ""
+ @echo " CACHE_OUTPUT=[0|n|no]"
+ @echo " disable caching VPP stdout/stderr and logging it"
+ @echo " as one block after test finishes"
+ @echo " (default: yes)"
+ @echo ""
+ @echo " FAILFAST=[1|y|yes]"
+ @echo " if enabled, stop running tests on first failure"
+ @echo " otherwise finish running whole suite"
+ @echo " (default: no)"
+ @echo ""
+ @echo " TIMEOUT=<timeout>"
+ @echo " fail test suite if any single test takes longer"
+ @echo " than <timeout> (in seconds) to finish"
+ @echo " (default: 600)"
+ @echo ""
+ @echo " RETRIES=<n>"
+ @echo " retry failed tests <n> times"
+ @echo " (default: 0)"
+ @echo ""
+ @echo " DEBUG=<type>"
+ @echo " configure VPP debugging:"
+ @echo " DEBUG=core"
+ @echo " detect coredump and load it in gdb on crash"
+ @echo ""
+ @echo " DEBUG=gdb"
+ @echo " print VPP PID and wait for user input before"
+ @echo " running and tearing down a testcase, allowing"
+ @echo " easy gdb attach"
+ @echo ""
+ @echo " DEBUG=gdbserver"
+ @echo " same as above, but run gdb inside a gdb server"
+ @echo ""
+ @echo " DEBUG=attach"
+ @echo " attach to existing vpp in running in gdb"
+ @echo " (see test-start-vpp-in-gdb)"
+ @echo " (default: none)"
+ @echo ""
+ @echo " STEP=[1|y|yes]"
+ @echo " enable stepping through a testcase"
+ @echo " (default: no)"
+ @echo ""
+ @echo " SANITY=[0|n|no]"
+ @echo " disable sanity import of vpp-api/vpp sanity"
+ @echo " run before running tests"
+ @echo " (default: yes)"
+ @echo ""
+ @echo " EXTENDED_TESTS=[1|y|yes]"
+ @echo " run extended tests"
+ @echo " (default: no)"
+ @echo ""
+ @echo " TEST=<filter>,[<filter>],..."
+ @echo " only run tests matching one or more comma-delimited"
+ @echo " filter expressions"
+ @echo ""
+ @echo " simple filter:"
+ @echo " file name or file suffix select all tests from a file"
+ @echo " examples:"
+ @echo " TEST=test_bfd"
+ @echo " TEST=bfd"
+ @echo " equivalent expressions selecting all"
+ @echo " tests defined in test_bfd.py"
+ @echo ""
+ @echo " wildcard filter:"
+ @echo " advanced filtering based on test file, test class"
+ @echo " and test function"
+ @echo " each filter expression is in the form of"
+ @echo " <file>.<class>.<test function>"
+ @echo " each of the tokens can be left empty or replaced"
+ @echo " with '*' to select all objects available"
+ @echo " examples:"
+ @echo " TEST=test_bfd.*.*"
+ @echo " TEST=test_bfd.."
+ @echo " TEST=bfd.*.*"
+ @echo " TEST=bfd.."
+ @echo " select all tests defined in test_bfd.py"
+ @echo " TEST=bfd.BFDAPITestCase.*"
+ @echo " TEST=bfd.BFDAPITestCase."
+ @echo " select all tests from test_bfd.py"
+ @echo " which are part of BFDAPITestCase class"
+ @echo " TEST=bfd.BFDAPITestCase.test_add_bfd"
+ @echo " select a single test named test_add_bfd"
+ @echo " from test_bfd.py/BFDAPITestCase"
+ @echo " TEST=..test_add_bfd"
+ @echo " TEST=*.*.test_add_bfd"
+ @echo " select all test functions named test_add_bfd"
+ @echo " from all files/classes"
+ @echo " TEST=bfd,ip4,..test_icmp_error"
+ @echo " select all test functions in test_bfd.py,"
+ @echo " test_ip4.py and all test functions named"
+ @echo " 'test_icmp_error' in all files"
+ @echo " (default: '')"
+ @echo ""
+ @echo " VARIANT=<variant>"
+ @echo " specify which march node variant to unit test"
+ @echo " e.g. VARIANT=skx test the skx march variants"
+ @echo " e.g. VARIANT=icl test the icl march variants"
+ @echo " (default: '')"
+ @echo ""
+ @echo " COREDUMP_SIZE=<size>"
+ @echo " pass <size> as unix { coredump-size <size> } argument"
+ @echo " to vpp, e.g. COREDUMP_SIZE=4g or COREDUMP_SIZE=unlimited"
+ @echo " (default: '')"
+ @echo ""
+ @echo " COREDUMP_COMPRESS=[1|y|yes]"
+ @echo " if no debug option is set, compress any core files"
+ @echo " (default: no)"
+ @echo ""
+ @echo " EXTERN_TESTS=<path>"
+ @echo " include out-of-tree test_*.py files under <path>"
+ @echo " (default: '')"
+ @echo ""
+ @echo " EXTERN_PLUGINS=<path>"
+ @echo " load out-of-tree vpp plugins in <path>"
+ @echo " (default: '')"
+ @echo ""
+ @echo " EXTERN_COV_DIR=<path>"
+ @echo " path to out-of-tree prefix, where source, object"
+ @echo " and .gcda files can be found for coverage report"
+ @echo " (default: '')"
+ @echo ""
+ @echo " PROFILE=[1|y|yes]"
+ @echo " enable profiling of test framework via cProfile module"
+ @echo " (default: no)"
+ @echo ""
+ @echo " PROFILE_SORT_BY=opt"
+ @echo " sort profiling report by opt - see cProfile documentation"
+ @echo " for possible values"
+ @echo " (default: cumtime)"
+ @echo ""
+ @echo " PROFILE_OUTPUT=file"
+ @echo " output profiling info to file - use absolute path"
+ @echo " (default: stdout)"
+ @echo ""
+ @echo " TEST_DEBUG=[1|y|yes]"
+ @echo " enable debugging of the test framework itself (expert)"
+ @echo " (default: no)"
+ @echo ""
+ @echo " API_FUZZ=[1|y|yes]"
+ @echo " enable VPP api fuzz testing"
+ @echo " (default: no)"
+ @echo ""
+ @echo " RND_SEED=<seed>"
+ @echo " random seed used by test framework"
+ @echo " (default: time.time())"
@echo ""
@echo "Starting VPP in GDB for use with DEBUG=attach:"
@echo ""
diff --git a/test/config.py b/test/config.py
index d2f14c8..578cc40 100644
--- a/test/config.py
+++ b/test/config.py
@@ -122,7 +122,8 @@
)
filter_help_string = """\
-expression consists of 3 string selectors separated by '.' separators:
+expression consists of one or more filters separated by commas (',')
+filter consists of 3 string selectors separated by dots ('.')
<file>.<class>.<function>
@@ -142,6 +143,8 @@
test_add_bfd from test_bfd.py/BFDAPITestCase
4. '.*.test_add_bfd' selects all test functions named test_add_bfd
from all files/classes
+5. 'bfd,ip4,..test_icmp_error' selects all test functions in test_bfd.py,
+ test_ip4.py and all test functions named 'test_icmp_error' in all files
"""
parser.add_argument(
"--filter", action="store", metavar="FILTER_EXPRESSION", help=filter_help_string
diff --git a/test/run_tests.py b/test/run_tests.py
index 85344ca..3bbf1bc 100644
--- a/test/run_tests.py
+++ b/test/run_tests.py
@@ -631,7 +631,7 @@
if "." in f:
parts = f.split(".")
if len(parts) > 3:
- raise Exception("Unrecognized %s option: %s" % (test_option, f))
+ raise Exception(f"Invalid test filter: {test_filter}")
if len(parts) > 2:
if parts[2] not in ("*", ""):
filter_func_name = parts[2]
@@ -677,21 +677,40 @@
class FilterByTestOption:
- def __init__(self, filter_file_name, filter_class_name, filter_func_name):
- self.filter_file_name = filter_file_name
- self.filter_class_name = filter_class_name
- self.filter_func_name = filter_func_name
+ def __init__(self, filters):
+ self.filters = filters
def __call__(self, file_name, class_name, func_name):
- if self.filter_file_name:
- fn_match = fnmatch.fnmatch(file_name, self.filter_file_name)
- if not fn_match:
+ def test_one(
+ filter_file_name,
+ filter_class_name,
+ filter_func_name,
+ file_name,
+ class_name,
+ func_name,
+ ):
+ if filter_file_name:
+ fn_match = fnmatch.fnmatch(file_name, filter_file_name)
+ if not fn_match:
+ return False
+ if filter_class_name and class_name != filter_class_name:
return False
- if self.filter_class_name and class_name != self.filter_class_name:
- return False
- if self.filter_func_name and func_name != self.filter_func_name:
- return False
- return True
+ if filter_func_name and func_name != filter_func_name:
+ return False
+ return True
+
+ for filter_file_name, filter_class_name, filter_func_name in self.filters:
+ if test_one(
+ filter_file_name,
+ filter_class_name,
+ filter_func_name,
+ file_name,
+ class_name,
+ func_name,
+ ):
+ return True
+
+ return False
class FilterByClassList:
@@ -938,14 +957,17 @@
descriptions = True
print("Running tests using custom test runner.")
- filter_file, filter_class, filter_func = parse_test_filter(config.filter)
+ filters = [(parse_test_filter(f)) for f in config.filter.split(",")]
print(
- "Selected filters: file=%s, class=%s, function=%s"
- % (filter_file, filter_class, filter_func)
+ "Selected filters: ",
+ "|".join(
+ f"file={filter_file}, class={filter_class}, function={filter_func}"
+ for filter_file, filter_class, filter_func in filters
+ ),
)
- filter_cb = FilterByTestOption(filter_file, filter_class, filter_func)
+ filter_cb = FilterByTestOption(filters)
cb = SplitToSuitesCallback(filter_cb)
for d in config.test_src_dir: