blob: d4d74bb62b4765d0c8d8333be8fcbdee60e8dcc9 [file] [log] [blame]
BenoƮt Ganne9fb6d402019-04-15 15:28:21 +02001ASAN_OPTIONS?=verify_asan_link_order=0:detect_leaks=0:abort_on_error=1:unmap_shadow_on_exit=1:disable_coredump=0
2export ASAN_OPTIONS
3
Dave Wallacef0797d12019-11-15 17:29:15 -05004.PHONY: verify-env
5verify-env:
6ifndef WS_ROOT
7 $(error WS_ROOT is not set)
8endif
9ifndef BR
10 $(error BR is not set)
11endif
Klement Sekerab8c72a42018-11-08 11:21:39 +010012ifndef TEST_DIR
13 $(error TEST_DIR is not set)
Klement Sekera277b89c2016-10-28 13:20:27 +020014endif
15
Dave Wallacef0797d12019-11-15 17:29:15 -050016export TEST_BR = $(BR)/build-test
17export TEST_DOC_BR = $(TEST_BR)/doc
18export BUILD_TEST_SRC = $(TEST_BR)/src
19FAILED_DIR=/tmp/vpp-failed-unittests/
20PLUGIN_TEST_DIRS=$(shell find $(PLUGIN_SRC_DIR) -type d -name test -exec echo -n " -d {}" \;)
21CORE_TEST_DIRS=$(shell find $(WS_ROOT)/src -not \( -path $(INTERN_PLUGIN_SRC_DIR) -prune \) -type d -name test -exec echo -n " -d {}" \;)
22VPP_TEST_DIRS=$(shell ls -d $(TEST_DIR)$(PLUGIN_TEST_DIRS)$(CORE_TEST_DIRS) $(EXTERN_TESTS))
Dave Wallacefd243742021-03-16 22:12:07 -040023VPP_TEST_SRC=$(shell for dir in $(VPP_TEST_DIRS) ; do ls $$dir/*.py 2>/dev/null; done)
Dave Wallacef0797d12019-11-15 17:29:15 -050024
Klement Sekerab93d1422017-03-02 08:16:20 +010025.PHONY: verify-no-running-vpp
26
27ifdef VPP_ZOMBIE_NOCHECK
28VPP_PIDS=
29else
30VPP_PIDS=$(shell pgrep -d, -x vpp_main)
31endif
32
Klement Sekeradb4e84c2017-08-11 10:06:15 +020033ifeq ($(DEBUG),gdb)
34FORCE_FOREGROUND=1
35else ifeq ($(DEBUG),gdbserver)
36FORCE_FOREGROUND=1
Dave Wallace24564332019-10-21 02:53:14 +000037else ifeq ($(DEBUG),gdb-all)
38FORCE_FOREGROUND=1
39else ifeq ($(DEBUG),gdbserver-all)
40FORCE_FOREGROUND=1
Klement Sekerab90be672017-10-10 06:44:05 +020041else ifeq ($(DEBUG),core)
42FORCE_FOREGROUND=1
juraj.linkes184870a2018-07-16 14:22:01 +020043else ifeq ($(STEP),yes)
44FORCE_FOREGROUND=1
45else ifeq ($(STEP),y)
46FORCE_FOREGROUND=1
47else ifeq ($(STEP),1)
48FORCE_FOREGROUND=1
Klement Sekeradb4e84c2017-08-11 10:06:15 +020049else
50FORCE_FOREGROUND=0
51endif
52
Klement Sekerabbfa5fd2018-06-27 13:54:32 +020053ifdef PROFILE_OUTPUT
54PROFILE_OUTPUT_OPTS=-o $(PROFILE_OUTPUT)
55endif
56
57ifndef PROFILE_SORT_BY
58PROFILE_SORT_BY=cumtime
59endif
60
61ifeq ($(PROFILE),1)
62PYTHON_PROFILE_OPTS=-m cProfile $(PROFILE_OUTPUT_OPTS) -s $(PROFILE_SORT_BY)
63FORCE_FOREGROUND=1
64endif
65
Klement Sekerab93d1422017-03-02 08:16:20 +010066verify-no-running-vpp:
67 @if [ "$(VPP_PIDS)" != "" ]; then \
68 echo; \
69 echo "*** Existing vpp processes detected (PID(s): $(VPP_PIDS)). Running tests under these conditions is not supported. ***"; \
70 echo; \
71 ps -fp $(VPP_PIDS);\
72 echo; \
73 false; \
74 fi
75
Klement Sekera993e0ed2017-03-16 09:14:59 +010076UNITTEST_EXTRA_OPTS=
77UNITTEST_FAILFAST_OPTS=
Klement Sekeraacb9b8e2017-02-14 02:55:31 +010078
79ifeq ($(FAILFAST),1)
Klement Sekera993e0ed2017-03-16 09:14:59 +010080UNITTEST_EXTRA_OPTS=-f
81endif
82
83ifneq ($(EXTERN_TESTS),)
84UNITTEST_EXTRA_OPTS=$(UNITTEST_FAILFAST_OPTS) -d $(EXTERN_TESTS)
Klement Sekeraacb9b8e2017-02-14 02:55:31 +010085endif
86
Dave Wallacef0797d12019-11-15 17:29:15 -050087VENV_PATH=$(TEST_BR)/venv
Klement Sekerab8c72a42018-11-08 11:21:39 +010088
Klement Sekeraebbaf552018-02-17 13:41:33 +010089ifeq ($(TEST_DEBUG),1)
Dave Wallacef0797d12019-11-15 17:29:15 -050090VENV_RUN_DIR:=$(VENV_PATH)/run-debug
Klement Sekeraebbaf552018-02-17 13:41:33 +010091else
Dave Wallacef0797d12019-11-15 17:29:15 -050092VENV_RUN_DIR:=$(VENV_PATH)/run
Klement Sekeraebbaf552018-02-17 13:41:33 +010093endif
94
vagrant937bf302017-04-07 01:48:47 +000095ifeq ($(PYTHON),)
Ole Troan6e6ad642020-02-04 13:28:13 +010096PYTHON_INTERP=python3
vagrant937bf302017-04-07 01:48:47 +000097else
98PYTHON_INTERP=$(PYTHON)
99endif
100
Dave Wallacefd243742021-03-16 22:12:07 -0400101empty:=
102space:= $(empty) $(empty)
103export PYTHONPATH=$(subst $(space),:,$(VPP_TEST_DIRS))
104export PYTHONPYCACHEPREFIX=$(TEST_BR)/pycache
105
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200106PYTHON_VERSION=$(shell $(PYTHON_INTERP) -c 'import sys; print(sys.version_info.major)')
Aloys Augustinef29fa42020-05-25 18:07:24 +0200107PIP_VERSION=20.1.1
108# Keep in sync with requirements.txt
109PIP_TOOLS_VERSION=5.1.2
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200110PYTHON_DEPENDS=requirements-$(PYTHON_VERSION).txt
Ole Troan6e6ad642020-02-04 13:28:13 +0100111SCAPY_SOURCE=$(shell find $(VENV_PATH)/lib/python* -name site-packages)
Dave Wallacef0797d12019-11-15 17:29:15 -0500112BUILD_COV_DIR=$(TEST_BR)/coverage
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200113
Aloys Augustinef29fa42020-05-25 18:07:24 +0200114PIP_TOOLS_INSTALL_DONE=$(VENV_RUN_DIR)/pip-tools-install-$(PYTHON_VERSION)-$(PIP_TOOLS_VERSION).done
115PIP_INSTALL_DONE=$(VENV_RUN_DIR)/pip-install-$(PYTHON_VERSION)-$(PIP_VERSION).done
Dave Wallacef0797d12019-11-15 17:29:15 -0500116PIP_PATCH_DONE=$(VENV_RUN_DIR)/pip-patch-$(PYTHON_VERSION).done
117PAPI_INSTALL_DONE=$(VENV_RUN_DIR)/papi-install-$(PYTHON_VERSION).done
118PAPI_PYTHON_SRC_DIR=$(WS_ROOT)/src/vpp-api/python
119PAPI_WIPE_DIST=$(WS_ROOT)/src/vpp-api/vapi/__pycache__ \
120 $(PAPI_PYTHON_SRC_DIR)/build \
121 $(PAPI_PYTHON_SRC_DIR)/vpp_papi.egg-info \
122 $(PAPI_PYTHON_SRC_DIR)/vpp_papi/__pycache__
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200123
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200124$(PIP_TOOLS_INSTALL_DONE):
125 @rm -rf $(VENV_PATH)
Dave Wallacef0797d12019-11-15 17:29:15 -0500126 @mkdir -p $(VENV_RUN_DIR)
Klement Sekera7554aef2021-03-30 19:46:06 +0200127 @$(PYTHON_INTERP) -m venv $(VENV_PATH)
Aloys Augustin8adb92b2019-07-23 15:19:28 +0200128 # pip version pinning
129 @bash -c "source $(VENV_PATH)/bin/activate && \
130 $(PYTHON_INTERP) -m pip install pip===$(PIP_VERSION)"
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200131 @bash -c "source $(VENV_PATH)/bin/activate && \
132 $(PYTHON_INTERP) -m pip install pip-tools===$(PIP_TOOLS_VERSION)"
133 @touch $@
134
Aloys Augustin8eca60d2020-06-24 11:33:20 +0200135$(PYTHON_DEPENDS): requirements.txt
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200136 @bash -c "source $(VENV_PATH)/bin/activate && \
137 CUSTOM_COMPILE_COMMAND='make test-refresh-deps (or update requirements.txt)' \
138 $(PYTHON_INTERP) -m piptools compile -q --generate-hashes requirements.txt --output-file $@"
139
Aloys Augustin8eca60d2020-06-24 11:33:20 +0200140$(PIP_INSTALL_DONE): $(PIP_TOOLS_INSTALL_DONE) $(PYTHON_DEPENDS)
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200141 @bash -c "source $(VENV_PATH)/bin/activate && \
142 $(PYTHON_INTERP) -m piptools sync $(PYTHON_DEPENDS)"
Neale Ranns177bbdc2016-11-15 09:46:51 +0000143 @touch $@
144
Klement Sekera9f9d07b2016-12-12 09:20:50 +0100145$(PIP_PATCH_DONE): $(PIP_INSTALL_DONE)
Neale Ranns177bbdc2016-11-15 09:46:51 +0000146 @echo --- patching ---
Dave Wallace6f692d62017-01-26 21:07:10 -0500147 @sleep 1 # Ensure python recompiles patched *.py files -> *.pyc
snaramre5d4b8912019-12-13 23:39:35 +0000148 for f in $(CURDIR)/patches/scapy-2.4.3/*.patch ; do \
Neale Ranns177bbdc2016-11-15 09:46:51 +0000149 echo Applying patch: $$(basename $$f) ; \
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200150 patch --forward -p1 -d $(SCAPY_SOURCE) < $$f ; \
151 retCode=$$?; \
Aloys Augustin8eca60d2020-06-24 11:33:20 +0200152 [ $$retCode -gt 1 ] && exit $$retCode; \
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200153 done; \
154 touch $@
Neale Ranns177bbdc2016-11-15 09:46:51 +0000155
Klement Sekera9f9d07b2016-12-12 09:20:50 +0100156$(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE)
Dave Wallacef0797d12019-11-15 17:29:15 -0500157 @bash -c "source $(VENV_PATH)/bin/activate && $(PYTHON_INTERP) -m pip install -e $(PAPI_PYTHON_SRC_DIR)"
Neale Ranns177bbdc2016-11-15 09:46:51 +0000158 @touch $@
159
Dave Wallacef0797d12019-11-15 17:29:15 -0500160.PHONY: refresh-deps
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200161refresh-deps: clean-deps $(PYTHON_DEPENDS)
162
Dave Wallacef0797d12019-11-15 17:29:15 -0500163.PHONY: clean-deps
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200164clean-deps:
165 @rm -f $(PYTHON_DEPENDS)
166
Dave Wallacef0797d12019-11-15 17:29:15 -0500167INTERN_PLUGIN_SRC_DIR=$(WS_ROOT)/src/plugins
Dave Wallacea43c93f2019-08-22 00:32:29 +0000168ifneq ($(EXTERN_PLUGIN_SRC_DIR),)
169PLUGIN_SRC_DIR=$(EXTERN_PLUGIN_SRC_DIR)
170else
Dave Wallacef0797d12019-11-15 17:29:15 -0500171PLUGIN_SRC_DIR=$(INTERN_PLUGIN_SRC_DIR)
Dave Wallacea43c93f2019-08-22 00:32:29 +0000172endif
Dave Wallacea43c93f2019-08-22 00:32:29 +0000173
Juraj Sloboda0279b292016-11-16 19:50:24 +0100174define retest-func
Dave Wallacef0797d12019-11-15 17:29:15 -0500175@env FORCE_FOREGROUND=$(FORCE_FOREGROUND) FAILED_DIR=$(FAILED_DIR) VENV_PATH=$(VENV_PATH) scripts/setsid_wrapper.sh $(FORCE_FOREGROUND) $(VENV_PATH)/bin/activate $(PYTHON_INTERP) $(PYTHON_PROFILE_OPTS) $(BUILD_TEST_SRC)/run_tests.py -d $(BUILD_TEST_SRC) $(UNITTEST_EXTRA_OPTS) || env FAILED_DIR=$(FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh
Juraj Sloboda0279b292016-11-16 19:50:24 +0100176endef
177
Klement Sekerac160ae02017-03-09 08:20:13 +0100178.PHONY: sanity
179
Klement Sekera41a9e512017-10-17 07:59:31 +0200180ifeq ($(SANITY),no)
181SANITY_IMPORT_VPP_PAPI_CMD=true
182SANITY_RUN_VPP_CMD=true
183else
Dave Wallacef0797d12019-11-15 17:29:15 -0500184SANITY_IMPORT_VPP_PAPI_CMD=source $(VENV_PATH)/bin/activate && $(PYTHON_INTERP) $(BUILD_TEST_SRC)/sanity_import_vpp_papi.py
Dave Wallacefd243742021-03-16 22:12:07 -0400185SANITY_RUN_VPP_CMD=source $(VENV_PATH)/bin/activate && $(PYTHON_INTERP) $(BUILD_TEST_SRC)/sanity_run_vpp.py
Klement Sekera41a9e512017-10-17 07:59:31 +0200186endif
187
juraj.linkes184870a2018-07-16 14:22:01 +0200188ifndef TEST_JOBS
189PARALLEL_ILLEGAL=0
190else ifeq ($(FORCE_FOREGROUND),0)
191PARALLEL_ILLEGAL=0
192else ifeq ($(TEST_JOBS),auto)
193PARALLEL_ILLEGAL=0
194else ifeq ($(TEST_JOBS),1)
195PARALLEL_ILLEGAL=0
196else
197PARALLEL_ILLEGAL=1
198endif
199
Dave Wallacef0797d12019-11-15 17:29:15 -0500200sanity: test-dep verify-no-running-vpp
juraj.linkes184870a2018-07-16 14:22:01 +0200201 @bash -c "test $(PARALLEL_ILLEGAL) -eq 0 ||\
202 (echo \"*******************************************************************\" &&\
203 echo \"* Sanity check failed, TEST_JOBS is not 1 or 'auto' and DEBUG, STEP or PROFILE is set\" &&\
204 echo \"*******************************************************************\" &&\
205 false)"
Klement Sekera41a9e512017-10-17 07:59:31 +0200206 @bash -c "$(SANITY_IMPORT_VPP_PAPI_CMD) ||\
Klement Sekerac160ae02017-03-09 08:20:13 +0100207 (echo \"*******************************************************************\" &&\
208 echo \"* Sanity check failed, cannot import vpp_papi\" &&\
209 echo \"* to debug: \" &&\
210 echo \"* 1. enter test shell: make test-shell\" &&\
211 echo \"* 2. execute debugger: gdb python -ex 'run sanity_import_vpp_papi.py'\" &&\
212 echo \"*******************************************************************\" &&\
213 false)"
Klement Sekera41a9e512017-10-17 07:59:31 +0200214 @bash -c "$(SANITY_RUN_VPP_CMD) ||\
Klement Sekera3747c752017-04-10 06:30:17 +0200215 (echo \"*******************************************************************\" &&\
216 echo \"* Sanity check failed, cannot run vpp\" &&\
217 echo \"*******************************************************************\" &&\
218 false)"
Klement Sekerac160ae02017-03-09 08:20:13 +0100219
Dave Wallacef0797d12019-11-15 17:29:15 -0500220.PHONY: ext-test-apps
221ext-test-apps:
222 make -C ext test-apps
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200223
Dave Wallacef0797d12019-11-15 17:29:15 -0500224$(BUILD_TEST_SRC): verify-env
Dave Wallacefd243742021-03-16 22:12:07 -0400225 @rm -rf $@
Dave Wallacef0797d12019-11-15 17:29:15 -0500226 @mkdir -p $@
Neale Ranns02da9262021-01-08 08:43:14 +0000227 @for file in $(VPP_TEST_SRC); do if [ ! -L $$file ] && [ ! -e $(BUILD_TEST_SRC)/$$(basename $$file) ] ; then ln -s $$file $(BUILD_TEST_SRC) ; fi ; done
Ed Kern56ccc232018-04-02 16:42:48 -0600228
Dave Wallacef0797d12019-11-15 17:29:15 -0500229$(FAILED_DIR): reset
230 @mkdir -p $@
231
232.PHONY: test-dep
233test-dep: $(BUILD_TEST_SRC) $(PAPI_INSTALL_DONE) $(FAILED_DIR)
234
235.PHONY: test
236test: test-dep ext-test-apps sanity
Juraj Sloboda0279b292016-11-16 19:50:24 +0100237 $(call retest-func)
Klement Sekeraf62ae122016-10-11 11:47:09 +0200238
Dave Wallacef0797d12019-11-15 17:29:15 -0500239.PHONY: retest
240retest: verify-env sanity $(FAILED_DIR)
Juraj Sloboda0279b292016-11-16 19:50:24 +0100241 $(call retest-func)
Klement Sekeraf62ae122016-10-11 11:47:09 +0200242
Dave Wallacef0797d12019-11-15 17:29:15 -0500243.PHONY: shell
244shell: test-dep
Klement Sekerab8c72a42018-11-08 11:21:39 +0100245 @echo "source $(VENV_PATH)/bin/activate;\
Dave Wallacef0797d12019-11-15 17:29:15 -0500246 cd $(BUILD_TEST_SRC);\
Dave Wallacefd243742021-03-16 22:12:07 -0400247 export PYTHONPATH=$(PYTHONPATH);\
Paul Vinciguerrac701e572019-12-19 16:09:43 -0500248 export RND_SEED=$(RND_SEED);\
Klement Sekeraa86e2892017-03-09 08:01:52 +0100249 echo '***';\
Dave Wallacefd243742021-03-16 22:12:07 -0400250 echo PYTHONPATH=$(PYTHONPATH);\
Paul Vinciguerrac701e572019-12-19 16:09:43 -0500251 echo RND_SEED=$(RND_SEED);\
Klement Sekera593635d2019-02-07 19:56:50 +0100252 echo VPP_BUILD_DIR=$(VPP_BUILD_DIR);\
253 echo VPP_BIN=$(VPP_BIN);\
254 echo VPP_PLUGIN_PATH=$(VPP_PLUGIN_PATH);\
Dave Barach7d31ab22019-05-08 19:18:18 -0400255 echo VPP_TEST_PLUGIN_PATH=$(VPP_TEST_PLUGIN_PATH);\
Klement Sekera593635d2019-02-07 19:56:50 +0100256 echo VPP_INSTALL_PATH=$(VPP_INSTALL_PATH);\
Klement Sekera47e275b2017-03-21 08:21:25 +0100257 echo EXTERN_TESTS=$(EXTERN_TESTS);\
258 echo EXTERN_PLUGINS=$(EXTERN_PLUGINS);\
Klement Sekeracd6cb982017-05-11 06:55:32 +0200259 echo EXTERN_COV_DIR=$(EXTERN_COV_DIR);\
Klement Sekeraa86e2892017-03-09 08:01:52 +0100260 echo LD_LIBRARY_PATH=$(LD_LIBRARY_PATH);\
261 echo '***';\
262 exec </dev/tty" | bash -i
263
Dave Wallacef0797d12019-11-15 17:29:15 -0500264.PHONY: reset
Neale Ranns177bbdc2016-11-15 09:46:51 +0000265reset:
Klement Sekeraaa7a2a12021-03-18 15:22:57 +0100266 @rm -f /dev/shm/vpp-unittest-*
Klement Sekeraf62ae122016-10-11 11:47:09 +0200267 @rm -rf /tmp/vpp-unittest-*
Dave Wallacef0797d12019-11-15 17:29:15 -0500268 @rm -f /tmp/api_post_mortem.*
Klement Sekerab8c72a42018-11-08 11:21:39 +0100269 @rm -rf $(FAILED_DIR)
Klement Sekeraf62ae122016-10-11 11:47:09 +0200270
Dave Wallacef0797d12019-11-15 17:29:15 -0500271.PHONY: wipe
Neale Ranns177bbdc2016-11-15 09:46:51 +0000272wipe: reset
Klement Sekeraa61d1252017-09-27 06:48:44 +0200273 @make -C ext clean
Klement Sekerab8c72a42018-11-08 11:21:39 +0100274 @rm -rf $(VENV_PATH)
Neale Ranns02da9262021-01-08 08:43:14 +0000275 @rm -rf $(patsubst %,%/__pycache__, $(VPP_TEST_DIRS))
276 @rm -rf $(BUILD_TEST_SRC)
Neale Ranns177bbdc2016-11-15 09:46:51 +0000277
Aloys Augustin1f2e1462020-04-07 15:43:33 +0200278$(TEST_DOC_BR): $(PIP_INSTALL_DONE)
Dave Wallacef0797d12019-11-15 17:29:15 -0500279 @mkdir -p $@
Dave Wallacef0797d12019-11-15 17:29:15 -0500280 @bash -c "source $(VENV_PATH)/bin/activate && make -C doc html"
281
282.PHONY: doc
283doc: $(BUILD_TEST_SRC) $(PIP_PATCH_DONE) $(TEST_DOC_BR)
284 @echo
285 @echo "Test Documentation URL: $(TEST_DOC_BR)/html/index.html"
286 @echo "Run 'make test-wipe-doc test-doc' to rebuild the test docs"
287 @echo
Klement Sekera277b89c2016-10-28 13:20:27 +0200288
Klement Sekera9f9d07b2016-12-12 09:20:50 +0100289.PHONY: wipe-doc
Klement Sekera277b89c2016-10-28 13:20:27 +0200290wipe-doc:
Dave Wallacef0797d12019-11-15 17:29:15 -0500291 @rm -rf $(TEST_DOC_BR)
Klement Sekera277b89c2016-10-28 13:20:27 +0200292
Dave Wallacef0797d12019-11-15 17:29:15 -0500293$(BUILD_COV_DIR):
294 @mkdir -p $@
295
296.PHONY: cov
297cov: wipe-cov test-dep ext $(BUILD_COV_DIR)
Klement Sekera593635d2019-02-07 19:56:50 +0100298 @lcov --zerocounters --directory $(VPP_BUILD_DIR)
Klement Sekeracd6cb982017-05-11 06:55:32 +0200299 @test -z "$(EXTERN_COV_DIR)" || lcov --zerocounters --directory $(EXTERN_COV_DIR)
Juraj Sloboda0279b292016-11-16 19:50:24 +0100300 $(call retest-func)
Klement Sekera593635d2019-02-07 19:56:50 +0100301 @lcov --capture --directory $(VPP_BUILD_DIR) --output-file $(BUILD_COV_DIR)/coverage.info
Klement Sekeracd6cb982017-05-11 06:55:32 +0200302 @test -z "$(EXTERN_COV_DIR)" || lcov --capture --directory $(EXTERN_COV_DIR) --output-file $(BUILD_COV_DIR)/extern-coverage.info
Juraj Sloboda0279b292016-11-16 19:50:24 +0100303 @genhtml $(BUILD_COV_DIR)/coverage.info --output-directory $(BUILD_COV_DIR)/html
Klement Sekeracd6cb982017-05-11 06:55:32 +0200304 @test -z "$(EXTERN_COV_DIR)" || genhtml $(BUILD_COV_DIR)/extern-coverage.info --output-directory $(BUILD_COV_DIR)/extern-html
Juraj Sloboda0279b292016-11-16 19:50:24 +0100305 @echo
306 @echo "Build finished. Code coverage report is in $(BUILD_COV_DIR)/html/index.html"
Klement Sekeracd6cb982017-05-11 06:55:32 +0200307 @test -z "$(EXTERN_COV_DIR)" || echo "Code coverage report for out-of-tree objects is in $(BUILD_COV_DIR)/extern-html/index.html"
Juraj Sloboda0279b292016-11-16 19:50:24 +0100308
Klement Sekera9f9d07b2016-12-12 09:20:50 +0100309.PHONY: wipe-cov
Juraj Sloboda0279b292016-11-16 19:50:24 +0100310wipe-cov: wipe
311 @rm -rf $(BUILD_COV_DIR)
312
Dave Wallacef0797d12019-11-15 17:29:15 -0500313.PHONY: wipe-papi
314wipe-papi:
315 @rm -rf $(PAPI_INSTALL_DONE) $(PAPI_WIPE_DIST)
Paul Vinciguerra2648e442019-07-09 16:49:46 -0400316
Dave Wallacef0797d12019-11-15 17:29:15 -0500317.PHONY: wipe-all
318wipe-all: wipe wipe-papi wipe-doc wipe-cov
319 @rm -rf $(TEST_BR)
Paul Vinciguerra2648e442019-07-09 16:49:46 -0400320
Klement Sekerae0b63b22021-03-24 23:24:52 +0100321.PHONY: checkstyle-diff
322checkstyle-diff: $(BUILD_TEST_SRC) $(PIP_INSTALL_DONE)
323 @bash -c "source $(VENV_PATH)/bin/activate &&\
324 $(PYTHON_INTERP) -m pip install pycodestyle"
325 @bash -c "source $(VENV_PATH)/bin/activate &&\
326 cd $(WS_ROOT) && git diff --name-only --no-color --relative HEAD~1 ':!*.patch' | grep '.py$$' | xargs -n 1 -I XXX \
327 pycodestyle --show-source --ignore=W504,E126,E241,E226,E305,E704,E741,E722 -v XXX ||\
328 (echo \"*********************************************************************\" &&\
329 echo \"* Test framework PEP8 compliance check FAILED (checked changed files)\" &&\
330 echo \"*********************************************************************\" &&\
331 false)"
332 @echo "*********************************************************************"
333 @echo "* Test framework PEP8 compliance check passed (checked changed files)"
334 @echo "*********************************************************************"
335
Klement Sekera72715ee2017-01-17 10:37:05 +0100336.PHONY: checkstyle
Dave Wallacef0797d12019-11-15 17:29:15 -0500337checkstyle: $(BUILD_TEST_SRC) $(PIP_INSTALL_DONE)
Ole Troan4aacc012019-10-23 11:47:33 +0200338 @bash -c "source $(VENV_PATH)/bin/activate &&\
339 $(PYTHON_INTERP) -m pip install pycodestyle"
Klement Sekerab8c72a42018-11-08 11:21:39 +0100340 @bash -c "source $(VENV_PATH)/bin/activate &&\
Dave Wallacef0797d12019-11-15 17:29:15 -0500341 pycodestyle --show-source --ignore=W504,E126,E241,E226,E305,E704,E741,E722 -v $(BUILD_TEST_SRC)/*.py ||\
Klement Sekera72715ee2017-01-17 10:37:05 +0100342 (echo \"*******************************************************************\" &&\
Klement Sekerae0b63b22021-03-24 23:24:52 +0100343 echo \"* Test framework PEP8 compliance check FAILED (checked all files)\" &&\
Klement Sekera72715ee2017-01-17 10:37:05 +0100344 echo \"*******************************************************************\" &&\
345 false)"
346 @echo "*******************************************************************"
Klement Sekerae0b63b22021-03-24 23:24:52 +0100347 @echo "* Test framework PEP8 compliance check passed (checked all files)"
Klement Sekera72715ee2017-01-17 10:37:05 +0100348 @echo "*******************************************************************"
349
Dave Wallacef0797d12019-11-15 17:29:15 -0500350.PHONY: help
Klement Sekera277b89c2016-10-28 13:20:27 +0200351help:
352 @echo "Running tests:"
353 @echo ""
Dave Wallacef0797d12019-11-15 17:29:15 -0500354 @echo " test - build and run (basic) functional tests"
355 @echo " test-debug - build and run (basic) functional tests (debug build)"
356 @echo " test-all - build and run functional and extended tests"
357 @echo " test-all-debug - build and run functional and extended tests (debug build)"
358 @echo " retest - run functional tests"
359 @echo " retest-debug - run functional tests (debug build)"
360 @echo " retest-all - run functional and extended tests"
361 @echo " retest-all-debug - run functional and extended tests (debug build)"
362 @echo " test-cov - generate code coverage report for test framework"
363 @echo " test-gcov - build and run functional tests (gcov build)"
364
365 @echo " test-wipe - wipe (temporary) files generated by unit tests"
366 @echo " test-wipe-cov - wipe code coverage report for test framework"
367 @echo " test-wipe-doc - wipe documentation for test framework"
368 @echo " test-wipe-papi - rebuild vpp_papi sources"
369 @echo " test-wipe-all - wipe (temporary) files generated by unit tests, docs, and coverage"
370 @echo " test-shell - enter shell with test environment"
371 @echo " test-shell-debug - enter shell with test environment (debug build)"
372 @echo " test-checkstyle - check PEP8 compliance for test framework"
373 @echo " test-refresh-deps - refresh the Python dependencies for the tests"
Klement Sekera277b89c2016-10-28 13:20:27 +0200374 @echo ""
375 @echo "Arguments controlling test runs:"
Dave Wallacef0797d12019-11-15 17:29:15 -0500376 @echo " V=[0|1|2] - set test verbosity level"
377 @echo " 0=ERROR, 1=INFO, 2=DEBUG"
378 @echo " TEST_JOBS=[<n>|auto] - use <n> parallel processes for test execution or automatic discovery of maximum acceptable processes (default: 1)"
379 @echo " CACHE_OUTPUT=[0|1] - cache VPP stdout/stderr and log as one block after test finishes (default: 1)"
380 @echo " FAILFAST=[0|1] - fail fast if 1, complete all tests if 0"
381 @echo " TIMEOUT=<timeout> - fail test suite if any single test takes longer than <timeout> (in seconds) to finish (default: 600)"
382 @echo " RETRIES=<n> - retry failed tests <n> times"
383 @echo " DEBUG=<type> - set VPP debugging kind"
384 @echo " DEBUG=core - detect coredump and load it in gdb on crash"
385 @echo " DEBUG=gdb - allow easy debugging by printing VPP PID"
386 @echo " and waiting for user input before running"
387 @echo " and tearing down a testcase"
388 @echo " DEBUG=gdbserver - run gdb inside a gdb server, otherwise"
389 @echo " same as above"
390 @echo " STEP=[yes|no] - ease debugging by stepping through a testcase"
391 @echo " SANITY=[yes|no] - perform sanity import of vpp-api/sanity vpp run before running tests (default: yes)"
392 @echo " EXTENDED_TESTS=[1|y] - used by '[re]test-all' & '[re]test-all-debug' to run extended tests"
393 @echo " TEST=<filter> - filter the set of tests:"
394 @echo " by file-name - only run tests from specified file, e.g. TEST=test_bfd selects all tests from test_bfd.py"
395 @echo " by file-suffix - same as file-name, but 'test_' is omitted e.g. TEST=bfd selects all tests from test_bfd.py"
396 @echo " by wildcard - wildcard filter is <file>.<class>.<test function>, each can be replaced by '*'"
397 @echo " e.g. TEST='test_bfd.*.*' is equivalent to above example of filter by file-name"
398 @echo " TEST='bfd.*.*' is equivalent to above example of filter by file-suffix"
399 @echo " TEST='bfd.BFDAPITestCase.*' selects all tests from test_bfd.py which are part of BFDAPITestCase class"
400 @echo " TEST='bfd.BFDAPITestCase.test_add_bfd' selects a single test named test_add_bfd from test_bfd.py/BFDAPITestCase"
401 @echo " TEST='*.*.test_add_bfd' selects all test functions named test_add_bfd from all files/classes"
Klement Sekera277b89c2016-10-28 13:20:27 +0200402 @echo ""
Ray Kinsella4830e4f2020-03-10 14:35:32 +0000403 @echo " VARIANT=<variant> - specify which march node variant to unit test"
Ray Kinsellaf9c8fe42020-08-14 15:27:10 +0100404 @echo " e.g. VARIANT=skx test the skx march variants"
405 @echo " e.g. VARIANT=icl test the icl march variants"
Ray Kinsella4830e4f2020-03-10 14:35:32 +0000406 @echo ""
Dave Wallacef0797d12019-11-15 17:29:15 -0500407 @echo " VPP_ZOMBIE_NOCHECK=1 - skip checking for vpp (zombie) processes (CAUTION)"
408 @echo " COREDUMP_SIZE=<size> - pass <size> as unix { coredump-size <size> } argument to vpp"
409 @echo " e.g. COREDUMP_SIZE=4g"
410 @echo " COREDUMP_SIZE=unlimited"
411 @echo " COREDUMP_COMPRESS=1 - compress core files if not debugging them"
412 @echo " EXTERN_TESTS=<path> - path to out-of-tree test_<name>.py files containing test cases"
413 @echo " EXTERN_PLUGINS=<path> - path to out-of-tree plugins to be loaded by vpp under test"
414 @echo " EXTERN_COV_DIR=<path> - path to out-of-tree prefix, where source, object and .gcda files can be found for coverage report"
Klement Sekerab93d1422017-03-02 08:16:20 +0100415 @echo ""
Dave Wallacef0797d12019-11-15 17:29:15 -0500416 @echo " PROFILE=1 - enable profiling of test framework via cProfile module"
417 @echo " PROFILE_SORT_BY=opt - sort profiling report by opt - consult cProfile documentation for possible values (default: cumtime)"
418 @echo " PROFILE_OUTPUT=file - output profiling info to file - use absolute path (default: stdout)"
Klement Sekerabbfa5fd2018-06-27 13:54:32 +0200419 @echo ""
Dave Wallacef0797d12019-11-15 17:29:15 -0500420 @echo " TEST_DEBUG=1 - turn on debugging of the test framework itself (expert)"
Klement Sekeraebbaf552018-02-17 13:41:33 +0100421 @echo ""
Dave Wallacef0797d12019-11-15 17:29:15 -0500422 @echo " SKIP_AARCH64=1 - skip tests that are failing on the ARM platorm in FD.io CI"
juraj.linkes68ebc832018-11-29 09:37:08 +0100423 @echo ""
Dave Wallacef0797d12019-11-15 17:29:15 -0500424 @echo " RND_SEED=seed - Seed RND with given seed"
Klement Sekera45a95dd2019-11-05 11:18:25 +0000425 @echo ""
Klement Sekera277b89c2016-10-28 13:20:27 +0200426 @echo "Creating test documentation"
Dave Wallacef0797d12019-11-15 17:29:15 -0500427 @echo " test-doc - generate documentation for test framework"
428 @echo " test-wipe-doc - wipe documentation for test framework"
Juraj Sloboda0279b292016-11-16 19:50:24 +0100429 @echo ""
430 @echo "Creating test code coverage report"
Dave Wallacef0797d12019-11-15 17:29:15 -0500431 @echo " test-cov - generate code coverage report for test framework"
432 @echo " test-wipe-cov - wipe code coverage report for test framework"
Klement Sekera277b89c2016-10-28 13:20:27 +0200433 @echo ""
Klement Sekera72715ee2017-01-17 10:37:05 +0100434 @echo "Verifying code-style"
Dave Wallacef0797d12019-11-15 17:29:15 -0500435 @echo " test-checkstyle - check PEP8 compliance"
Klement Sekera72715ee2017-01-17 10:37:05 +0100436 @echo ""