Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 1 | .PHONY: verify-python-path |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 2 | |
Klement Sekera | f413bef | 2017-08-15 07:09:02 +0200 | [diff] [blame] | 3 | VPP_TEST_FAILED_DIR=/tmp/vpp-failed-unittests/ |
| 4 | |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 5 | verify-python-path: |
| 6 | ifndef VPP_PYTHON_PREFIX |
| 7 | $(error VPP_PYTHON_PREFIX is not set) |
| 8 | endif |
| 9 | |
Klement Sekera | b93d142 | 2017-03-02 08:16:20 +0100 | [diff] [blame] | 10 | .PHONY: verify-no-running-vpp |
| 11 | |
| 12 | ifdef VPP_ZOMBIE_NOCHECK |
| 13 | VPP_PIDS= |
| 14 | else |
| 15 | VPP_PIDS=$(shell pgrep -d, -x vpp_main) |
| 16 | endif |
| 17 | |
Klement Sekera | db4e84c | 2017-08-11 10:06:15 +0200 | [diff] [blame] | 18 | ifeq ($(DEBUG),gdb) |
| 19 | FORCE_FOREGROUND=1 |
| 20 | else ifeq ($(DEBUG),gdbserver) |
| 21 | FORCE_FOREGROUND=1 |
Klement Sekera | b90be67 | 2017-10-10 06:44:05 +0200 | [diff] [blame] | 22 | else ifeq ($(DEBUG),core) |
| 23 | FORCE_FOREGROUND=1 |
Klement Sekera | db4e84c | 2017-08-11 10:06:15 +0200 | [diff] [blame] | 24 | else |
| 25 | FORCE_FOREGROUND=0 |
| 26 | endif |
| 27 | |
Klement Sekera | b93d142 | 2017-03-02 08:16:20 +0100 | [diff] [blame] | 28 | verify-no-running-vpp: |
| 29 | @if [ "$(VPP_PIDS)" != "" ]; then \ |
| 30 | echo; \ |
| 31 | echo "*** Existing vpp processes detected (PID(s): $(VPP_PIDS)). Running tests under these conditions is not supported. ***"; \ |
| 32 | echo; \ |
| 33 | ps -fp $(VPP_PIDS);\ |
| 34 | echo; \ |
| 35 | false; \ |
| 36 | fi |
| 37 | |
Klement Sekera | 993e0ed | 2017-03-16 09:14:59 +0100 | [diff] [blame] | 38 | UNITTEST_EXTRA_OPTS= |
| 39 | UNITTEST_FAILFAST_OPTS= |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 40 | |
| 41 | ifeq ($(FAILFAST),1) |
Klement Sekera | 993e0ed | 2017-03-16 09:14:59 +0100 | [diff] [blame] | 42 | UNITTEST_EXTRA_OPTS=-f |
| 43 | endif |
| 44 | |
| 45 | ifneq ($(EXTERN_TESTS),) |
| 46 | UNITTEST_EXTRA_OPTS=$(UNITTEST_FAILFAST_OPTS) -d $(EXTERN_TESTS) |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 47 | endif |
| 48 | |
Klement Sekera | ebbaf55 | 2018-02-17 13:41:33 +0100 | [diff] [blame] | 49 | ifeq ($(TEST_DEBUG),1) |
| 50 | VPP_PYTHON_PREFIX:=$(VPP_PYTHON_PREFIX)/debug |
| 51 | PYTHON_EXTRA_DEPENDS=objgraph pympler |
| 52 | else |
| 53 | PYTHON_EXTRA_DEPENDS= |
| 54 | endif |
| 55 | |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 56 | PYTHON_VENV_PATH=$(VPP_PYTHON_PREFIX)/virtualenv |
Neale Ranns | 5b4a020 | 2018-05-01 05:55:59 -0700 | [diff] [blame] | 57 | PYTHON_DEPENDS=$(PYTHON_EXTRA_DEPENDS) faulthandler six scapy==2.4.0 pexpect cryptography subprocess32 cffi git+https://github.com/vpp-dev/py-lispnetworking |
vagrant | 937bf30 | 2017-04-07 01:48:47 +0000 | [diff] [blame] | 58 | SCAPY_SOURCE=$(shell find $(PYTHON_VENV_PATH) -name site-packages) |
| 59 | BUILD_COV_DIR=$(BR)/test-cov |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 60 | |
vagrant | 937bf30 | 2017-04-07 01:48:47 +0000 | [diff] [blame] | 61 | GET_PIP_SCRIPT=$(VPP_PYTHON_PREFIX)/get-pip.py |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 62 | PIP_INSTALL_DONE=$(VPP_PYTHON_PREFIX)/pip-install.done |
| 63 | PIP_PATCH_DONE=$(VPP_PYTHON_PREFIX)/pip-patch.done |
| 64 | PAPI_INSTALL_DONE=$(VPP_PYTHON_PREFIX)/papi-install.done |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 65 | |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 66 | PAPI_INSTALL_FLAGS=$(PIP_INSTALL_DONE) $(PIP_PATCH_DONE) $(PAPI_INSTALL_DONE) |
| 67 | |
vagrant | 937bf30 | 2017-04-07 01:48:47 +0000 | [diff] [blame] | 68 | ifeq ($(PYTHON),) |
| 69 | PYTHON_INTERP=python2.7 |
| 70 | else |
| 71 | PYTHON_INTERP=$(PYTHON) |
| 72 | endif |
| 73 | |
| 74 | $(GET_PIP_SCRIPT): |
| 75 | @mkdir -p $(VPP_PYTHON_PREFIX) |
| 76 | @bash -c "cd $(VPP_PYTHON_PREFIX) && curl -O https://bootstrap.pypa.io/get-pip.py" |
| 77 | |
| 78 | $(PIP_INSTALL_DONE): $(GET_PIP_SCRIPT) |
| 79 | @virtualenv $(PYTHON_VENV_PATH) -p $(PYTHON_INTERP) |
| 80 | @bash -c "source $(PYTHON_VENV_PATH)/bin/activate && python $(GET_PIP_SCRIPT)" |
Klement Sekera | 778c276 | 2016-11-08 02:00:28 +0100 | [diff] [blame] | 81 | @bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install $(PYTHON_DEPENDS)" |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 82 | @touch $@ |
| 83 | |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 84 | $(PIP_PATCH_DONE): $(PIP_INSTALL_DONE) |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 85 | @echo --- patching --- |
Dave Wallace | 6f692d6 | 2017-01-26 21:07:10 -0500 | [diff] [blame] | 86 | @sleep 1 # Ensure python recompiles patched *.py files -> *.pyc |
Neale Ranns | 2bc9402 | 2018-02-25 12:27:18 -0800 | [diff] [blame] | 87 | for f in $(CURDIR)/patches/scapy-2.4/*.patch ; do \ |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 88 | echo Applying patch: $$(basename $$f) ; \ |
| 89 | patch -p1 -d $(SCAPY_SOURCE) < $$f ; \ |
| 90 | done |
| 91 | @touch $@ |
| 92 | |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 93 | $(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE) |
Damjan Marion | 37d77e9 | 2016-12-28 15:14:46 +0100 | [diff] [blame] | 94 | @bash -c "source $(PYTHON_VENV_PATH)/bin/activate && cd $(WS_ROOT)/src/vpp-api/python && python setup.py install" |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 95 | @touch $@ |
| 96 | |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 97 | define retest-func |
Klement Sekera | 75e7d13 | 2017-09-20 08:26:30 +0200 | [diff] [blame] | 98 | @env VPP_TEST_FAILED_DIR=$(VPP_TEST_FAILED_DIR) scripts/setsid_wrapper.sh $(FORCE_FOREGROUND) $(PYTHON_VENV_PATH)/bin/activate python run_tests.py -d $(TEST_DIR) $(UNITTEST_EXTRA_OPTS) || env VPP_TEST_FAILED_DIR=$(VPP_TEST_FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 99 | endef |
| 100 | |
Klement Sekera | c160ae0 | 2017-03-09 08:20:13 +0100 | [diff] [blame] | 101 | .PHONY: sanity |
| 102 | |
Klement Sekera | 41a9e51 | 2017-10-17 07:59:31 +0200 | [diff] [blame] | 103 | ifeq ($(SANITY),no) |
| 104 | SANITY_IMPORT_VPP_PAPI_CMD=true |
| 105 | SANITY_RUN_VPP_CMD=true |
| 106 | else |
| 107 | SANITY_IMPORT_VPP_PAPI_CMD=source $(PYTHON_VENV_PATH)/bin/activate && python sanity_import_vpp_papi.py |
| 108 | SANITY_RUN_VPP_CMD=source $(PYTHON_VENV_PATH)/bin/activate && python sanity_run_vpp.py |
| 109 | endif |
| 110 | |
Klement Sekera | c160ae0 | 2017-03-09 08:20:13 +0100 | [diff] [blame] | 111 | sanity: verify-no-running-vpp |
Klement Sekera | da38715 | 2018-01-24 13:41:50 +0100 | [diff] [blame] | 112 | @sys_req/dev_shm_size.sh |
Klement Sekera | 41a9e51 | 2017-10-17 07:59:31 +0200 | [diff] [blame] | 113 | @bash -c "$(SANITY_IMPORT_VPP_PAPI_CMD) ||\ |
Klement Sekera | c160ae0 | 2017-03-09 08:20:13 +0100 | [diff] [blame] | 114 | (echo \"*******************************************************************\" &&\ |
| 115 | echo \"* Sanity check failed, cannot import vpp_papi\" &&\ |
| 116 | echo \"* to debug: \" &&\ |
| 117 | echo \"* 1. enter test shell: make test-shell\" &&\ |
| 118 | echo \"* 2. execute debugger: gdb python -ex 'run sanity_import_vpp_papi.py'\" &&\ |
| 119 | echo \"*******************************************************************\" &&\ |
| 120 | false)" |
Klement Sekera | 41a9e51 | 2017-10-17 07:59:31 +0200 | [diff] [blame] | 121 | @bash -c "$(SANITY_RUN_VPP_CMD) ||\ |
Klement Sekera | 3747c75 | 2017-04-10 06:30:17 +0200 | [diff] [blame] | 122 | (echo \"*******************************************************************\" &&\ |
| 123 | echo \"* Sanity check failed, cannot run vpp\" &&\ |
| 124 | echo \"*******************************************************************\" &&\ |
| 125 | false)" |
Klement Sekera | c160ae0 | 2017-03-09 08:20:13 +0100 | [diff] [blame] | 126 | |
Klement Sekera | 8f2a4ea | 2017-05-04 06:15:18 +0200 | [diff] [blame] | 127 | .PHONY: ext |
| 128 | ext: |
| 129 | make -C ext |
| 130 | |
Ed Kern | 56ccc23 | 2018-04-02 16:42:48 -0600 | [diff] [blame] | 131 | test-dep: verify-python-path $(PAPI_INSTALL_DONE) |
| 132 | |
Klement Sekera | 8f2a4ea | 2017-05-04 06:15:18 +0200 | [diff] [blame] | 133 | test: verify-python-path $(PAPI_INSTALL_DONE) ext sanity reset |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 134 | $(call retest-func) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 135 | |
Klement Sekera | c160ae0 | 2017-03-09 08:20:13 +0100 | [diff] [blame] | 136 | retest: verify-python-path sanity reset |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 137 | $(call retest-func) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 138 | |
Klement Sekera | a86e289 | 2017-03-09 08:01:52 +0100 | [diff] [blame] | 139 | shell: verify-python-path $(PAPI_INSTALL_DONE) |
| 140 | @echo "source $(PYTHON_VENV_PATH)/bin/activate;\ |
| 141 | echo '***';\ |
| 142 | echo VPP_TEST_BUILD_DIR=$(VPP_TEST_BUILD_DIR);\ |
| 143 | echo VPP_TEST_BIN=$(VPP_TEST_BIN);\ |
| 144 | echo VPP_TEST_PLUGIN_PATH=$(VPP_TEST_PLUGIN_PATH);\ |
| 145 | echo VPP_TEST_INSTALL_PATH=$(VPP_TEST_INSTALL_PATH);\ |
Klement Sekera | 47e275b | 2017-03-21 08:21:25 +0100 | [diff] [blame] | 146 | echo EXTERN_TESTS=$(EXTERN_TESTS);\ |
| 147 | echo EXTERN_PLUGINS=$(EXTERN_PLUGINS);\ |
Klement Sekera | cd6cb98 | 2017-05-11 06:55:32 +0200 | [diff] [blame] | 148 | echo EXTERN_COV_DIR=$(EXTERN_COV_DIR);\ |
Klement Sekera | a86e289 | 2017-03-09 08:01:52 +0100 | [diff] [blame] | 149 | echo LD_LIBRARY_PATH=$(LD_LIBRARY_PATH);\ |
| 150 | echo '***';\ |
| 151 | exec </dev/tty" | bash -i |
| 152 | |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 153 | .PHONY: wipe doc |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 154 | |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 155 | reset: |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 156 | @rm -f /dev/shm/vpp-unittest-* |
| 157 | @rm -rf /tmp/vpp-unittest-* |
Klement Sekera | f413bef | 2017-08-15 07:09:02 +0200 | [diff] [blame] | 158 | @rm -rf $(VPP_TEST_FAILED_DIR) |
| 159 | @mkdir $(VPP_TEST_FAILED_DIR) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 160 | |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 161 | wipe: reset |
Klement Sekera | a61d125 | 2017-09-27 06:48:44 +0200 | [diff] [blame] | 162 | @make -C ext clean |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 163 | @rm -rf $(PYTHON_VENV_PATH) |
| 164 | @rm -f $(PAPI_INSTALL_FLAGS) |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 165 | |
Klement Sekera | 6320514 | 2017-02-03 08:18:53 +0100 | [diff] [blame] | 166 | doc: verify-python-path $(PIP_PATCH_DONE) |
vagrant | 937bf30 | 2017-04-07 01:48:47 +0000 | [diff] [blame] | 167 | @virtualenv $(PYTHON_VENV_PATH) -p $(PYTHON_INTERP) |
Klement Sekera | 6320514 | 2017-02-03 08:18:53 +0100 | [diff] [blame] | 168 | @bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install sphinx sphinx-rtd-theme" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 169 | @bash -c "source $(PYTHON_VENV_PATH)/bin/activate && make -C doc WS_ROOT=$(WS_ROOT) BR=$(BR) NO_VPP_PAPI=1 html" |
| 170 | |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 171 | .PHONY: wipe-doc |
| 172 | |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 173 | wipe-doc: |
| 174 | @make -C doc wipe BR=$(BR) |
| 175 | |
Klement Sekera | cae7134 | 2018-02-22 14:47:00 +0100 | [diff] [blame] | 176 | cov: wipe-cov reset ext verify-python-path $(PAPI_INSTALL_DONE) |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 177 | @lcov --zerocounters --directory $(VPP_TEST_BUILD_DIR) |
Klement Sekera | cd6cb98 | 2017-05-11 06:55:32 +0200 | [diff] [blame] | 178 | @test -z "$(EXTERN_COV_DIR)" || lcov --zerocounters --directory $(EXTERN_COV_DIR) |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 179 | $(call retest-func) |
| 180 | @mkdir $(BUILD_COV_DIR) |
| 181 | @lcov --capture --directory $(VPP_TEST_BUILD_DIR) --output-file $(BUILD_COV_DIR)/coverage.info |
Klement Sekera | cd6cb98 | 2017-05-11 06:55:32 +0200 | [diff] [blame] | 182 | @test -z "$(EXTERN_COV_DIR)" || lcov --capture --directory $(EXTERN_COV_DIR) --output-file $(BUILD_COV_DIR)/extern-coverage.info |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 183 | @genhtml $(BUILD_COV_DIR)/coverage.info --output-directory $(BUILD_COV_DIR)/html |
Klement Sekera | cd6cb98 | 2017-05-11 06:55:32 +0200 | [diff] [blame] | 184 | @test -z "$(EXTERN_COV_DIR)" || genhtml $(BUILD_COV_DIR)/extern-coverage.info --output-directory $(BUILD_COV_DIR)/extern-html |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 185 | @echo |
| 186 | @echo "Build finished. Code coverage report is in $(BUILD_COV_DIR)/html/index.html" |
Klement Sekera | cd6cb98 | 2017-05-11 06:55:32 +0200 | [diff] [blame] | 187 | @test -z "$(EXTERN_COV_DIR)" || echo "Code coverage report for out-of-tree objects is in $(BUILD_COV_DIR)/extern-html/index.html" |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 188 | |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 189 | .PHONY: wipe-cov |
| 190 | |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 191 | wipe-cov: wipe |
| 192 | @rm -rf $(BUILD_COV_DIR) |
| 193 | |
Klement Sekera | 72715ee | 2017-01-17 10:37:05 +0100 | [diff] [blame] | 194 | .PHONY: checkstyle |
| 195 | checkstyle: verify-python-path |
vagrant | 937bf30 | 2017-04-07 01:48:47 +0000 | [diff] [blame] | 196 | @virtualenv $(PYTHON_VENV_PATH) -p $(PYTHON_INTERP) |
Marco Varlese | 767fcb6 | 2018-02-25 11:28:52 +0100 | [diff] [blame] | 197 | @bash -c "source $(PYTHON_VENV_PATH)/bin/activate && pip install pycodestyle" |
Klement Sekera | 72715ee | 2017-01-17 10:37:05 +0100 | [diff] [blame] | 198 | @bash -c "source $(PYTHON_VENV_PATH)/bin/activate &&\ |
Chris Luke | d042194 | 2018-04-10 15:19:54 -0400 | [diff] [blame] | 199 | pycodestyle --show-source --ignore=W504,E126,E241,E226,E305,E704,E741,E722 -v $(WS_ROOT)/test/*.py ||\ |
Klement Sekera | 72715ee | 2017-01-17 10:37:05 +0100 | [diff] [blame] | 200 | (echo \"*******************************************************************\" &&\ |
| 201 | echo \"* Test framework PEP8 compliance check FAILED \" &&\ |
| 202 | echo \"*******************************************************************\" &&\ |
| 203 | false)" |
| 204 | @echo "*******************************************************************" |
| 205 | @echo "* Test framework PEP8 compliance check passed" |
| 206 | @echo "*******************************************************************" |
| 207 | |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 208 | help: |
| 209 | @echo "Running tests:" |
| 210 | @echo "" |
Klement Sekera | 8713493 | 2017-03-07 11:39:27 +0100 | [diff] [blame] | 211 | @echo " test - build and run (basic) functional tests" |
| 212 | @echo " test-debug - build and run (basic) functional tests (debug build)" |
| 213 | @echo " test-all - build and run (all) functional tests" |
| 214 | @echo " test-all-debug - build and run (all) functional tests (debug build)" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 215 | @echo " retest - run functional tests" |
| 216 | @echo " retest-debug - run functional tests (debug build)" |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 217 | @echo " test-wipe - wipe (temporary) files generated by unit tests" |
Klement Sekera | a86e289 | 2017-03-09 08:01:52 +0100 | [diff] [blame] | 218 | @echo " test-shell - enter shell with test environment" |
| 219 | @echo " test-shell-debug - enter shell with test environment (debug build)" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 220 | @echo "" |
| 221 | @echo "Arguments controlling test runs:" |
| 222 | @echo " V=[0|1|2] - set test verbosity level" |
Klement Sekera | a3d933c | 2017-11-06 09:46:00 +0100 | [diff] [blame] | 223 | @echo " CACHE_OUTPUT=[0|1] - cache VPP stdout/stderr and log as one block after test finishes (default: 1)" |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 224 | @echo " FAILFAST=[0|1] - fail fast if 1, complete all tests if 0" |
Neale Ranns | f051072 | 2018-01-31 11:35:41 -0800 | [diff] [blame] | 225 | @echo " TIMEOUT=<timeout> - fail test suite if any single test takes longer than <timeout> (in seconds) to finish" |
Klement Sekera | df2b980 | 2017-10-05 10:26:03 +0200 | [diff] [blame] | 226 | @echo " RETRIES=<n> - retry failed tests <n> times" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 227 | @echo " DEBUG=<type> - set VPP debugging kind" |
| 228 | @echo " DEBUG=core - detect coredump and load it in gdb on crash" |
| 229 | @echo " DEBUG=gdb - allow easy debugging by printing VPP PID " |
| 230 | @echo " and waiting for user input before running " |
| 231 | @echo " and tearing down a testcase" |
| 232 | @echo " DEBUG=gdbserver - run gdb inside a gdb server, otherwise " |
| 233 | @echo " same as above" |
| 234 | @echo " STEP=[yes|no] - ease debugging by stepping through a testcase " |
Klement Sekera | 41a9e51 | 2017-10-17 07:59:31 +0200 | [diff] [blame] | 235 | @echo " SANITY=[yes|no] - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes)" |
Dave Wallace | 05e4f9b | 2018-03-02 13:33:45 -0500 | [diff] [blame] | 236 | @echo " EXTENDED_TESTS=[1|y] - run extended tests" |
Klement Sekera | 104543f | 2017-02-03 07:29:43 +0100 | [diff] [blame] | 237 | @echo " TEST=<filter> - filter the set of tests:" |
| 238 | @echo " by file-name - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py" |
| 239 | @echo " by file-suffix - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py" |
| 240 | @echo " by wildcard - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'" |
| 241 | @echo " e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name" |
| 242 | @echo " TEST='bfd.*.*' is equivalent to above example of filter by file-suffix" |
| 243 | @echo " TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class" |
| 244 | @echo " TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase" |
| 245 | @echo " TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 246 | @echo "" |
Klement Sekera | b93d142 | 2017-03-02 08:16:20 +0100 | [diff] [blame] | 247 | @echo " VPP_ZOMBIE_NOCHECK=1 - skip checking for vpp (zombie) processes (CAUTION)" |
Klement Sekera | 80a7f0a | 2017-03-02 11:27:11 +0100 | [diff] [blame] | 248 | @echo " COREDUMP_SIZE=<size> - pass <size> as unix { coredump-size <size> } argument to vpp" |
| 249 | @echo " e.g. COREDUMP_SIZE=4g" |
| 250 | @echo " COREDUMP_SIZE=unlimited" |
Klement Sekera | 47e275b | 2017-03-21 08:21:25 +0100 | [diff] [blame] | 251 | @echo " EXTERN_TESTS=<path> - path to out-of-tree test_<name>.py files containing test cases" |
| 252 | @echo " EXTERN_PLUGINS=<path>- path to out-of-tree plugins to be loaded by vpp under test" |
Klement Sekera | cd6cb98 | 2017-05-11 06:55:32 +0200 | [diff] [blame] | 253 | @echo " EXTERN_COV_DIR=<path>- path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report" |
Klement Sekera | b93d142 | 2017-03-02 08:16:20 +0100 | [diff] [blame] | 254 | @echo "" |
Klement Sekera | ebbaf55 | 2018-02-17 13:41:33 +0100 | [diff] [blame] | 255 | @echo " TEST_DEBUG=1 - turn on debugging of the test framework itself (expert)" |
| 256 | @echo "" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 257 | @echo "Creating test documentation" |
| 258 | @echo " test-doc - generate documentation for test framework" |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 259 | @echo " test-wipe-doc - wipe documentation for test framework" |
| 260 | @echo "" |
| 261 | @echo "Creating test code coverage report" |
| 262 | @echo " test-cov - generate code coverage report for test framework" |
| 263 | @echo " test-wipe-cov - wipe code coverage report for test framework" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 264 | @echo "" |
Klement Sekera | 72715ee | 2017-01-17 10:37:05 +0100 | [diff] [blame] | 265 | @echo "Verifying code-style" |
| 266 | @echo " test-checkstyle - check PEP8 compliance" |
| 267 | @echo "" |