Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 1 | .PHONY: verify-test-dir |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 2 | |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 3 | FAILED_DIR=/tmp/vpp-failed-unittests/ |
Klement Sekera | f413bef | 2017-08-15 07:09:02 +0200 | [diff] [blame] | 4 | |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 5 | verify-test-dir: |
| 6 | ifndef TEST_DIR |
| 7 | $(error TEST_DIR is not set) |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 8 | endif |
| 9 | |
Klement Sekera | b93d142 | 2017-03-02 08:16:20 +0100 | [diff] [blame] | 10 | .PHONY: verify-no-running-vpp |
| 11 | |
| 12 | ifdef VPP_ZOMBIE_NOCHECK |
| 13 | VPP_PIDS= |
| 14 | else |
| 15 | VPP_PIDS=$(shell pgrep -d, -x vpp_main) |
| 16 | endif |
| 17 | |
Klement Sekera | db4e84c | 2017-08-11 10:06:15 +0200 | [diff] [blame] | 18 | ifeq ($(DEBUG),gdb) |
| 19 | FORCE_FOREGROUND=1 |
| 20 | else ifeq ($(DEBUG),gdbserver) |
| 21 | FORCE_FOREGROUND=1 |
Klement Sekera | b90be67 | 2017-10-10 06:44:05 +0200 | [diff] [blame] | 22 | else ifeq ($(DEBUG),core) |
| 23 | FORCE_FOREGROUND=1 |
juraj.linkes | 184870a | 2018-07-16 14:22:01 +0200 | [diff] [blame] | 24 | else ifeq ($(STEP),yes) |
| 25 | FORCE_FOREGROUND=1 |
| 26 | else ifeq ($(STEP),y) |
| 27 | FORCE_FOREGROUND=1 |
| 28 | else ifeq ($(STEP),1) |
| 29 | FORCE_FOREGROUND=1 |
Klement Sekera | db4e84c | 2017-08-11 10:06:15 +0200 | [diff] [blame] | 30 | else |
| 31 | FORCE_FOREGROUND=0 |
| 32 | endif |
| 33 | |
Klement Sekera | bbfa5fd | 2018-06-27 13:54:32 +0200 | [diff] [blame] | 34 | ifdef PROFILE_OUTPUT |
| 35 | PROFILE_OUTPUT_OPTS=-o $(PROFILE_OUTPUT) |
| 36 | endif |
| 37 | |
| 38 | ifndef PROFILE_SORT_BY |
| 39 | PROFILE_SORT_BY=cumtime |
| 40 | endif |
| 41 | |
| 42 | ifeq ($(PROFILE),1) |
| 43 | PYTHON_PROFILE_OPTS=-m cProfile $(PROFILE_OUTPUT_OPTS) -s $(PROFILE_SORT_BY) |
| 44 | FORCE_FOREGROUND=1 |
| 45 | endif |
| 46 | |
Klement Sekera | b93d142 | 2017-03-02 08:16:20 +0100 | [diff] [blame] | 47 | verify-no-running-vpp: |
| 48 | @if [ "$(VPP_PIDS)" != "" ]; then \ |
| 49 | echo; \ |
| 50 | echo "*** Existing vpp processes detected (PID(s): $(VPP_PIDS)). Running tests under these conditions is not supported. ***"; \ |
| 51 | echo; \ |
| 52 | ps -fp $(VPP_PIDS);\ |
| 53 | echo; \ |
| 54 | false; \ |
| 55 | fi |
| 56 | |
Klement Sekera | 993e0ed | 2017-03-16 09:14:59 +0100 | [diff] [blame] | 57 | UNITTEST_EXTRA_OPTS= |
| 58 | UNITTEST_FAILFAST_OPTS= |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 59 | |
| 60 | ifeq ($(FAILFAST),1) |
Klement Sekera | 993e0ed | 2017-03-16 09:14:59 +0100 | [diff] [blame] | 61 | UNITTEST_EXTRA_OPTS=-f |
| 62 | endif |
| 63 | |
| 64 | ifneq ($(EXTERN_TESTS),) |
| 65 | UNITTEST_EXTRA_OPTS=$(UNITTEST_FAILFAST_OPTS) -d $(EXTERN_TESTS) |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 66 | endif |
| 67 | |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 68 | VENV_PATH=$(TEST_DIR)/run/venv |
| 69 | |
Klement Sekera | ebbaf55 | 2018-02-17 13:41:33 +0100 | [diff] [blame] | 70 | ifeq ($(TEST_DEBUG),1) |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 71 | TEST_RUN_DIR:=$(VENV_PATH)/run-debug |
Klement Sekera | ebbaf55 | 2018-02-17 13:41:33 +0100 | [diff] [blame] | 72 | PYTHON_EXTRA_DEPENDS=objgraph pympler |
| 73 | else |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 74 | TEST_RUN_DIR:=$(VENV_PATH)/run |
Klement Sekera | ebbaf55 | 2018-02-17 13:41:33 +0100 | [diff] [blame] | 75 | PYTHON_EXTRA_DEPENDS= |
| 76 | endif |
| 77 | |
Paul Vinciguerra | ab9e20b | 2018-11-23 03:43:52 -0800 | [diff] [blame] | 78 | PYTHON_DEPENDS=$(PYTHON_EXTRA_DEPENDS) -r requirements.txt |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 79 | SCAPY_SOURCE=$(shell find $(VENV_PATH) -name site-packages) |
| 80 | BUILD_COV_DIR=$(TEST_DIR)/coverage |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 81 | |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 82 | GET_PIP_SCRIPT=$(TEST_RUN_DIR)/get-pip.py |
| 83 | PIP_INSTALL_DONE=$(TEST_RUN_DIR)/pip-install.done |
| 84 | PIP_PATCH_DONE=$(TEST_RUN_DIR)/pip-patch.done |
| 85 | PAPI_INSTALL_DONE=$(TEST_RUN_DIR)/papi-install.done |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 86 | |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 87 | PAPI_INSTALL_FLAGS=$(PIP_INSTALL_DONE) $(PIP_PATCH_DONE) $(PAPI_INSTALL_DONE) |
| 88 | |
vagrant | 937bf30 | 2017-04-07 01:48:47 +0000 | [diff] [blame] | 89 | ifeq ($(PYTHON),) |
| 90 | PYTHON_INTERP=python2.7 |
| 91 | else |
| 92 | PYTHON_INTERP=$(PYTHON) |
| 93 | endif |
| 94 | |
| 95 | $(GET_PIP_SCRIPT): |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 96 | @mkdir -p $(TEST_RUN_DIR) |
| 97 | @bash -c "cd $(TEST_RUN_DIR) && curl -O https://bootstrap.pypa.io/get-pip.py" |
vagrant | 937bf30 | 2017-04-07 01:48:47 +0000 | [diff] [blame] | 98 | |
| 99 | $(PIP_INSTALL_DONE): $(GET_PIP_SCRIPT) |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 100 | @virtualenv $(VENV_PATH) -p $(PYTHON_INTERP) |
| 101 | @bash -c "source $(VENV_PATH)/bin/activate && python $(GET_PIP_SCRIPT)" |
| 102 | @bash -c "source $(VENV_PATH)/bin/activate && pip install $(PYTHON_DEPENDS)" |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 103 | @touch $@ |
| 104 | |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 105 | $(PIP_PATCH_DONE): $(PIP_INSTALL_DONE) |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 106 | @echo --- patching --- |
Dave Wallace | 6f692d6 | 2017-01-26 21:07:10 -0500 | [diff] [blame] | 107 | @sleep 1 # Ensure python recompiles patched *.py files -> *.pyc |
Neale Ranns | 2bc9402 | 2018-02-25 12:27:18 -0800 | [diff] [blame] | 108 | for f in $(CURDIR)/patches/scapy-2.4/*.patch ; do \ |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 109 | echo Applying patch: $$(basename $$f) ; \ |
| 110 | patch -p1 -d $(SCAPY_SOURCE) < $$f ; \ |
| 111 | done |
| 112 | @touch $@ |
| 113 | |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 114 | $(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE) |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 115 | @bash -c "source $(VENV_PATH)/bin/activate && cd $(WS_ROOT)/src/vpp-api/python && python setup.py install" |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 116 | @touch $@ |
| 117 | |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 118 | define retest-func |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 119 | @env FORCE_FOREGROUND=$(FORCE_FOREGROUND) FAILED_DIR=$(FAILED_DIR) VENV_PATH=$(VENV_PATH) scripts/setsid_wrapper.sh $(FORCE_FOREGROUND) $(VENV_PATH)/bin/activate python $(PYTHON_PROFILE_OPTS) run_tests.py -d $(TEST_DIR) $(UNITTEST_EXTRA_OPTS) || env FAILED_DIR=$(FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 120 | endef |
| 121 | |
Klement Sekera | c160ae0 | 2017-03-09 08:20:13 +0100 | [diff] [blame] | 122 | .PHONY: sanity |
| 123 | |
Klement Sekera | 41a9e51 | 2017-10-17 07:59:31 +0200 | [diff] [blame] | 124 | ifeq ($(SANITY),no) |
| 125 | SANITY_IMPORT_VPP_PAPI_CMD=true |
| 126 | SANITY_RUN_VPP_CMD=true |
| 127 | else |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 128 | SANITY_IMPORT_VPP_PAPI_CMD=source $(VENV_PATH)/bin/activate && python sanity_import_vpp_papi.py |
| 129 | SANITY_RUN_VPP_CMD=source $(VENV_PATH)/bin/activate && python sanity_run_vpp.py |
Klement Sekera | 41a9e51 | 2017-10-17 07:59:31 +0200 | [diff] [blame] | 130 | endif |
| 131 | |
juraj.linkes | 184870a | 2018-07-16 14:22:01 +0200 | [diff] [blame] | 132 | ifndef TEST_JOBS |
| 133 | PARALLEL_ILLEGAL=0 |
| 134 | else ifeq ($(FORCE_FOREGROUND),0) |
| 135 | PARALLEL_ILLEGAL=0 |
| 136 | else ifeq ($(TEST_JOBS),auto) |
| 137 | PARALLEL_ILLEGAL=0 |
| 138 | else ifeq ($(TEST_JOBS),1) |
| 139 | PARALLEL_ILLEGAL=0 |
| 140 | else |
| 141 | PARALLEL_ILLEGAL=1 |
| 142 | endif |
| 143 | |
Klement Sekera | c160ae0 | 2017-03-09 08:20:13 +0100 | [diff] [blame] | 144 | sanity: verify-no-running-vpp |
Klement Sekera | da38715 | 2018-01-24 13:41:50 +0100 | [diff] [blame] | 145 | @sys_req/dev_shm_size.sh |
juraj.linkes | 184870a | 2018-07-16 14:22:01 +0200 | [diff] [blame] | 146 | @bash -c "test $(PARALLEL_ILLEGAL) -eq 0 ||\ |
| 147 | (echo \"*******************************************************************\" &&\ |
| 148 | echo \"* Sanity check failed, TEST_JOBS is not 1 or 'auto' and DEBUG, STEP or PROFILE is set\" &&\ |
| 149 | echo \"*******************************************************************\" &&\ |
| 150 | false)" |
Klement Sekera | 41a9e51 | 2017-10-17 07:59:31 +0200 | [diff] [blame] | 151 | @bash -c "$(SANITY_IMPORT_VPP_PAPI_CMD) ||\ |
Klement Sekera | c160ae0 | 2017-03-09 08:20:13 +0100 | [diff] [blame] | 152 | (echo \"*******************************************************************\" &&\ |
| 153 | echo \"* Sanity check failed, cannot import vpp_papi\" &&\ |
| 154 | echo \"* to debug: \" &&\ |
| 155 | echo \"* 1. enter test shell: make test-shell\" &&\ |
| 156 | echo \"* 2. execute debugger: gdb python -ex 'run sanity_import_vpp_papi.py'\" &&\ |
| 157 | echo \"*******************************************************************\" &&\ |
| 158 | false)" |
Klement Sekera | 41a9e51 | 2017-10-17 07:59:31 +0200 | [diff] [blame] | 159 | @bash -c "$(SANITY_RUN_VPP_CMD) ||\ |
Klement Sekera | 3747c75 | 2017-04-10 06:30:17 +0200 | [diff] [blame] | 160 | (echo \"*******************************************************************\" &&\ |
| 161 | echo \"* Sanity check failed, cannot run vpp\" &&\ |
| 162 | echo \"*******************************************************************\" &&\ |
| 163 | false)" |
Klement Sekera | c160ae0 | 2017-03-09 08:20:13 +0100 | [diff] [blame] | 164 | |
Klement Sekera | 8f2a4ea | 2017-05-04 06:15:18 +0200 | [diff] [blame] | 165 | .PHONY: ext |
| 166 | ext: |
Klement Sekera | e61ad8d | 2018-11-15 14:27:04 +0100 | [diff] [blame] | 167 | @if [ "$(EXTENDED_TESTS)" = "yes" ]; then\ |
| 168 | make -C ext;\ |
| 169 | fi |
Klement Sekera | 8f2a4ea | 2017-05-04 06:15:18 +0200 | [diff] [blame] | 170 | |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 171 | test-dep: verify-test-dir $(PAPI_INSTALL_DONE) |
Ed Kern | 56ccc23 | 2018-04-02 16:42:48 -0600 | [diff] [blame] | 172 | |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 173 | test: verify-test-dir $(PAPI_INSTALL_DONE) ext sanity reset |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 174 | $(call retest-func) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 175 | |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 176 | retest: verify-test-dir sanity reset |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 177 | $(call retest-func) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 178 | |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 179 | shell: verify-test-dir $(PAPI_INSTALL_DONE) |
| 180 | @echo "source $(VENV_PATH)/bin/activate;\ |
Klement Sekera | a86e289 | 2017-03-09 08:01:52 +0100 | [diff] [blame] | 181 | echo '***';\ |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 182 | echo _BUILD_DIR=$(VPP_BUILD_DIR);\ |
| 183 | echo _BIN=$(VPP_BIN);\ |
| 184 | echo _PLUGIN_PATH=$(VPP_PLUGIN_PATH);\ |
| 185 | echo _INSTALL_PATH=$(VPP_INSTALL_PATH);\ |
Klement Sekera | 47e275b | 2017-03-21 08:21:25 +0100 | [diff] [blame] | 186 | echo EXTERN_TESTS=$(EXTERN_TESTS);\ |
| 187 | echo EXTERN_PLUGINS=$(EXTERN_PLUGINS);\ |
Klement Sekera | cd6cb98 | 2017-05-11 06:55:32 +0200 | [diff] [blame] | 188 | echo EXTERN_COV_DIR=$(EXTERN_COV_DIR);\ |
Klement Sekera | a86e289 | 2017-03-09 08:01:52 +0100 | [diff] [blame] | 189 | echo LD_LIBRARY_PATH=$(LD_LIBRARY_PATH);\ |
| 190 | echo '***';\ |
| 191 | exec </dev/tty" | bash -i |
| 192 | |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 193 | .PHONY: wipe doc |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 194 | |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 195 | reset: |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 196 | @rm -f /dev/shm/vpp-unittest-* |
| 197 | @rm -rf /tmp/vpp-unittest-* |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 198 | @rm -rf $(FAILED_DIR) |
| 199 | @mkdir $(FAILED_DIR) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 200 | |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 201 | wipe: reset |
Klement Sekera | a61d125 | 2017-09-27 06:48:44 +0200 | [diff] [blame] | 202 | @make -C ext clean |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 203 | @rm -rf $(VENV_PATH) |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 204 | @rm -f $(PAPI_INSTALL_FLAGS) |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 205 | |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 206 | doc: verify-test-dir $(PIP_PATCH_DONE) |
| 207 | @virtualenv $(VENV_PATH) -p $(PYTHON_INTERP) |
| 208 | @bash -c "source $(VENV_PATH)/bin/activate && pip install sphinx sphinx-rtd-theme" |
| 209 | @bash -c "source $(VENV_PATH)/bin/activate && make -C doc WS_ROOT=$(WS_ROOT) BR=$(BR) NO_VPP_PAPI=1 html" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 210 | |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 211 | .PHONY: wipe-doc |
| 212 | |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 213 | wipe-doc: |
| 214 | @make -C doc wipe BR=$(BR) |
| 215 | |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 216 | cov: wipe-cov reset ext verify-test-dir $(PAPI_INSTALL_DONE) |
| 217 | @lcov --zerocounters --directory $(_BUILD_DIR) |
Klement Sekera | cd6cb98 | 2017-05-11 06:55:32 +0200 | [diff] [blame] | 218 | @test -z "$(EXTERN_COV_DIR)" || lcov --zerocounters --directory $(EXTERN_COV_DIR) |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 219 | $(call retest-func) |
| 220 | @mkdir $(BUILD_COV_DIR) |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 221 | @lcov --capture --directory $(_BUILD_DIR) --output-file $(BUILD_COV_DIR)/coverage.info |
Klement Sekera | cd6cb98 | 2017-05-11 06:55:32 +0200 | [diff] [blame] | 222 | @test -z "$(EXTERN_COV_DIR)" || lcov --capture --directory $(EXTERN_COV_DIR) --output-file $(BUILD_COV_DIR)/extern-coverage.info |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 223 | @genhtml $(BUILD_COV_DIR)/coverage.info --output-directory $(BUILD_COV_DIR)/html |
Klement Sekera | cd6cb98 | 2017-05-11 06:55:32 +0200 | [diff] [blame] | 224 | @test -z "$(EXTERN_COV_DIR)" || genhtml $(BUILD_COV_DIR)/extern-coverage.info --output-directory $(BUILD_COV_DIR)/extern-html |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 225 | @echo |
| 226 | @echo "Build finished. Code coverage report is in $(BUILD_COV_DIR)/html/index.html" |
Klement Sekera | cd6cb98 | 2017-05-11 06:55:32 +0200 | [diff] [blame] | 227 | @test -z "$(EXTERN_COV_DIR)" || echo "Code coverage report for out-of-tree objects is in $(BUILD_COV_DIR)/extern-html/index.html" |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 228 | |
Klement Sekera | 9f9d07b | 2016-12-12 09:20:50 +0100 | [diff] [blame] | 229 | .PHONY: wipe-cov |
| 230 | |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 231 | wipe-cov: wipe |
| 232 | @rm -rf $(BUILD_COV_DIR) |
| 233 | |
Klement Sekera | 72715ee | 2017-01-17 10:37:05 +0100 | [diff] [blame] | 234 | .PHONY: checkstyle |
Klement Sekera | b8c72a4 | 2018-11-08 11:21:39 +0100 | [diff] [blame] | 235 | checkstyle: verify-test-dir |
| 236 | @virtualenv $(VENV_PATH) -p $(PYTHON_INTERP) |
| 237 | @bash -c "source $(VENV_PATH)/bin/activate && pip install pycodestyle" |
| 238 | @bash -c "source $(VENV_PATH)/bin/activate &&\ |
Chris Luke | d042194 | 2018-04-10 15:19:54 -0400 | [diff] [blame] | 239 | pycodestyle --show-source --ignore=W504,E126,E241,E226,E305,E704,E741,E722 -v $(WS_ROOT)/test/*.py ||\ |
Klement Sekera | 72715ee | 2017-01-17 10:37:05 +0100 | [diff] [blame] | 240 | (echo \"*******************************************************************\" &&\ |
| 241 | echo \"* Test framework PEP8 compliance check FAILED \" &&\ |
| 242 | echo \"*******************************************************************\" &&\ |
| 243 | false)" |
| 244 | @echo "*******************************************************************" |
| 245 | @echo "* Test framework PEP8 compliance check passed" |
| 246 | @echo "*******************************************************************" |
| 247 | |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 248 | help: |
| 249 | @echo "Running tests:" |
| 250 | @echo "" |
Klement Sekera | 8713493 | 2017-03-07 11:39:27 +0100 | [diff] [blame] | 251 | @echo " test - build and run (basic) functional tests" |
| 252 | @echo " test-debug - build and run (basic) functional tests (debug build)" |
| 253 | @echo " test-all - build and run (all) functional tests" |
| 254 | @echo " test-all-debug - build and run (all) functional tests (debug build)" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 255 | @echo " retest - run functional tests" |
| 256 | @echo " retest-debug - run functional tests (debug build)" |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 257 | @echo " test-wipe - wipe (temporary) files generated by unit tests" |
Klement Sekera | a86e289 | 2017-03-09 08:01:52 +0100 | [diff] [blame] | 258 | @echo " test-shell - enter shell with test environment" |
| 259 | @echo " test-shell-debug - enter shell with test environment (debug build)" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 260 | @echo "" |
| 261 | @echo "Arguments controlling test runs:" |
| 262 | @echo " V=[0|1|2] - set test verbosity level" |
juraj.linkes | 184870a | 2018-07-16 14:22:01 +0200 | [diff] [blame] | 263 | @echo " TEST_JOBS=[<n>|auto] - use <n> parallel processes for test execution or automatic discovery of maximum acceptable processes (default: 1)" |
Klement Sekera | a3d933c | 2017-11-06 09:46:00 +0100 | [diff] [blame] | 264 | @echo " CACHE_OUTPUT=[0|1] - cache VPP stdout/stderr and log as one block after test finishes (default: 1)" |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 265 | @echo " FAILFAST=[0|1] - fail fast if 1, complete all tests if 0" |
juraj.linkes | 184870a | 2018-07-16 14:22:01 +0200 | [diff] [blame] | 266 | @echo " TIMEOUT=<timeout> - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)" |
Klement Sekera | df2b980 | 2017-10-05 10:26:03 +0200 | [diff] [blame] | 267 | @echo " RETRIES=<n> - retry failed tests <n> times" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 268 | @echo " DEBUG=<type> - set VPP debugging kind" |
| 269 | @echo " DEBUG=core - detect coredump and load it in gdb on crash" |
juraj.linkes | 184870a | 2018-07-16 14:22:01 +0200 | [diff] [blame] | 270 | @echo " DEBUG=gdb - allow easy debugging by printing VPP PID" |
| 271 | @echo " and waiting for user input before running" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 272 | @echo " and tearing down a testcase" |
juraj.linkes | 184870a | 2018-07-16 14:22:01 +0200 | [diff] [blame] | 273 | @echo " DEBUG=gdbserver - run gdb inside a gdb server, otherwise" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 274 | @echo " same as above" |
juraj.linkes | 184870a | 2018-07-16 14:22:01 +0200 | [diff] [blame] | 275 | @echo " STEP=[yes|no] - ease debugging by stepping through a testcase" |
Klement Sekera | 41a9e51 | 2017-10-17 07:59:31 +0200 | [diff] [blame] | 276 | @echo " SANITY=[yes|no] - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes)" |
Dave Wallace | 05e4f9b | 2018-03-02 13:33:45 -0500 | [diff] [blame] | 277 | @echo " EXTENDED_TESTS=[1|y] - run extended tests" |
Klement Sekera | 104543f | 2017-02-03 07:29:43 +0100 | [diff] [blame] | 278 | @echo " TEST=<filter> - filter the set of tests:" |
| 279 | @echo " by file-name - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py" |
| 280 | @echo " by file-suffix - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py" |
| 281 | @echo " by wildcard - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'" |
| 282 | @echo " e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name" |
| 283 | @echo " TEST='bfd.*.*' is equivalent to above example of filter by file-suffix" |
| 284 | @echo " TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class" |
| 285 | @echo " TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase" |
| 286 | @echo " TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 287 | @echo "" |
Klement Sekera | b93d142 | 2017-03-02 08:16:20 +0100 | [diff] [blame] | 288 | @echo " VPP_ZOMBIE_NOCHECK=1 - skip checking for vpp (zombie) processes (CAUTION)" |
Klement Sekera | 80a7f0a | 2017-03-02 11:27:11 +0100 | [diff] [blame] | 289 | @echo " COREDUMP_SIZE=<size> - pass <size> as unix { coredump-size <size> } argument to vpp" |
| 290 | @echo " e.g. COREDUMP_SIZE=4g" |
| 291 | @echo " COREDUMP_SIZE=unlimited" |
Klement Sekera | 47e275b | 2017-03-21 08:21:25 +0100 | [diff] [blame] | 292 | @echo " EXTERN_TESTS=<path> - path to out-of-tree test_<name>.py files containing test cases" |
| 293 | @echo " EXTERN_PLUGINS=<path>- path to out-of-tree plugins to be loaded by vpp under test" |
Klement Sekera | cd6cb98 | 2017-05-11 06:55:32 +0200 | [diff] [blame] | 294 | @echo " EXTERN_COV_DIR=<path>- path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report" |
Klement Sekera | b93d142 | 2017-03-02 08:16:20 +0100 | [diff] [blame] | 295 | @echo "" |
Klement Sekera | bbfa5fd | 2018-06-27 13:54:32 +0200 | [diff] [blame] | 296 | @echo " PROFILE=1 - enable profiling of test framework via cProfile module" |
| 297 | @echo " PROFILE_SORT_BY=opt - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime)" |
| 298 | @echo " PROFILE_OUTPUT=file - output profiling info to file - use absolute path (default: stdout)" |
| 299 | @echo "" |
Klement Sekera | ebbaf55 | 2018-02-17 13:41:33 +0100 | [diff] [blame] | 300 | @echo " TEST_DEBUG=1 - turn on debugging of the test framework itself (expert)" |
| 301 | @echo "" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 302 | @echo "Creating test documentation" |
| 303 | @echo " test-doc - generate documentation for test framework" |
Juraj Sloboda | 0279b29 | 2016-11-16 19:50:24 +0100 | [diff] [blame] | 304 | @echo " test-wipe-doc - wipe documentation for test framework" |
| 305 | @echo "" |
| 306 | @echo "Creating test code coverage report" |
| 307 | @echo " test-cov - generate code coverage report for test framework" |
| 308 | @echo " test-wipe-cov - wipe code coverage report for test framework" |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 309 | @echo "" |
Klement Sekera | 72715ee | 2017-01-17 10:37:05 +0100 | [diff] [blame] | 310 | @echo "Verifying code-style" |
| 311 | @echo " test-checkstyle - check PEP8 compliance" |
| 312 | @echo "" |