blob: 4ff0c152a302dcdb73f3d94e3752305d631c2cbe [file] [log] [blame]
BenoƮt Ganne9fb6d402019-04-15 15:28:21 +02001ASAN_OPTIONS?=verify_asan_link_order=0:detect_leaks=0:abort_on_error=1:unmap_shadow_on_exit=1:disable_coredump=0
2export ASAN_OPTIONS
3
Dave Wallacef0797d12019-11-15 17:29:15 -05004.PHONY: verify-env
5verify-env:
6ifndef WS_ROOT
7 $(error WS_ROOT is not set)
8endif
9ifndef BR
10 $(error BR is not set)
11endif
Klement Sekerab8c72a42018-11-08 11:21:39 +010012ifndef TEST_DIR
13 $(error TEST_DIR is not set)
Klement Sekera277b89c2016-10-28 13:20:27 +020014endif
15
Dave Wallaceeddd8e32021-05-12 21:43:59 -040016export TEST_BR = $(TEST_DIR)
Dave Wallacef0797d12019-11-15 17:29:15 -050017FAILED_DIR=/tmp/vpp-failed-unittests/
Dave Wallaceeddd8e32021-05-12 21:43:59 -040018VPP_TEST_DIRS=$(shell ls -d $(TEST_DIR) $(EXTERN_TESTS))
Dave Wallacef0797d12019-11-15 17:29:15 -050019
Klement Sekerae2636852021-03-16 12:52:12 +010020FORCE_NO_WIPE=0
Klement Sekeradb4e84c2017-08-11 10:06:15 +020021ifeq ($(DEBUG),gdb)
22FORCE_FOREGROUND=1
23else ifeq ($(DEBUG),gdbserver)
24FORCE_FOREGROUND=1
Dave Wallace24564332019-10-21 02:53:14 +000025else ifeq ($(DEBUG),gdb-all)
26FORCE_FOREGROUND=1
27else ifeq ($(DEBUG),gdbserver-all)
28FORCE_FOREGROUND=1
Klement Sekerab90be672017-10-10 06:44:05 +020029else ifeq ($(DEBUG),core)
30FORCE_FOREGROUND=1
Klement Sekerae2636852021-03-16 12:52:12 +010031else ifeq ($(DEBUG),attach)
32FORCE_FOREGROUND=1
33FORCE_NO_WIPE=1
juraj.linkes184870a2018-07-16 14:22:01 +020034else ifeq ($(STEP),yes)
35FORCE_FOREGROUND=1
36else ifeq ($(STEP),y)
37FORCE_FOREGROUND=1
38else ifeq ($(STEP),1)
39FORCE_FOREGROUND=1
Klement Sekeradb4e84c2017-08-11 10:06:15 +020040else
41FORCE_FOREGROUND=0
42endif
43
Klement Sekerabbfa5fd2018-06-27 13:54:32 +020044ifdef PROFILE_OUTPUT
45PROFILE_OUTPUT_OPTS=-o $(PROFILE_OUTPUT)
46endif
47
48ifndef PROFILE_SORT_BY
49PROFILE_SORT_BY=cumtime
50endif
51
52ifeq ($(PROFILE),1)
Klement Sekerab23ffd72021-05-31 16:08:53 +020053PYTHON_OPTS="-m cProfile $(PROFILE_OUTPUT_OPTS) -s $(PROFILE_SORT_BY)"
Klement Sekerabbfa5fd2018-06-27 13:54:32 +020054FORCE_FOREGROUND=1
55endif
56
Saima Yunusc7f93b32022-08-10 03:25:31 -040057VENV_BR_DIR=$(BR)/test
58VENV_PATH=$(VENV_BR_DIR)/venv
Klement Sekerab8c72a42018-11-08 11:21:39 +010059
Klement Sekeraebbaf552018-02-17 13:41:33 +010060ifeq ($(TEST_DEBUG),1)
Dave Wallacef0797d12019-11-15 17:29:15 -050061VENV_RUN_DIR:=$(VENV_PATH)/run-debug
Klement Sekeraebbaf552018-02-17 13:41:33 +010062else
Dave Wallacef0797d12019-11-15 17:29:15 -050063VENV_RUN_DIR:=$(VENV_PATH)/run
Klement Sekeraebbaf552018-02-17 13:41:33 +010064endif
65
vagrant937bf302017-04-07 01:48:47 +000066ifeq ($(PYTHON),)
Ole Troan6e6ad642020-02-04 13:28:13 +010067PYTHON_INTERP=python3
vagrant937bf302017-04-07 01:48:47 +000068else
69PYTHON_INTERP=$(PYTHON)
70endif
71
Klement Sekerab23ffd72021-05-31 16:08:53 +020072ifeq ($(V),)
73V=0
74endif
75
Aloys Augustin28fa8cb2019-06-21 17:04:29 +020076PYTHON_VERSION=$(shell $(PYTHON_INTERP) -c 'import sys; print(sys.version_info.major)')
Dave Wallace7b8b4652023-08-15 19:05:26 -040077PIP_VERSION=23.2.1
Aloys Augustinef29fa42020-05-25 18:07:24 +020078# Keep in sync with requirements.txt
Dave Wallace7b8b4652023-08-15 19:05:26 -040079PIP_TOOLS_VERSION=7.3.0
80PIP_SETUPTOOLS_VERSION=68.1.0
Aloys Augustin28fa8cb2019-06-21 17:04:29 +020081PYTHON_DEPENDS=requirements-$(PYTHON_VERSION).txt
Ole Troan6e6ad642020-02-04 13:28:13 +010082SCAPY_SOURCE=$(shell find $(VENV_PATH)/lib/python* -name site-packages)
Dave Wallace1c95e122023-06-21 23:14:38 -040083BUILD_COV_DIR=$(BR)/test-coverage
Aloys Augustin28fa8cb2019-06-21 17:04:29 +020084
Aloys Augustinef29fa42020-05-25 18:07:24 +020085PIP_TOOLS_INSTALL_DONE=$(VENV_RUN_DIR)/pip-tools-install-$(PYTHON_VERSION)-$(PIP_TOOLS_VERSION).done
86PIP_INSTALL_DONE=$(VENV_RUN_DIR)/pip-install-$(PYTHON_VERSION)-$(PIP_VERSION).done
Dave Wallacef0797d12019-11-15 17:29:15 -050087PIP_PATCH_DONE=$(VENV_RUN_DIR)/pip-patch-$(PYTHON_VERSION).done
88PAPI_INSTALL_DONE=$(VENV_RUN_DIR)/papi-install-$(PYTHON_VERSION).done
89PAPI_PYTHON_SRC_DIR=$(WS_ROOT)/src/vpp-api/python
90PAPI_WIPE_DIST=$(WS_ROOT)/src/vpp-api/vapi/__pycache__ \
91 $(PAPI_PYTHON_SRC_DIR)/build \
92 $(PAPI_PYTHON_SRC_DIR)/vpp_papi.egg-info \
93 $(PAPI_PYTHON_SRC_DIR)/vpp_papi/__pycache__
Aloys Augustin28fa8cb2019-06-21 17:04:29 +020094
Aloys Augustin28fa8cb2019-06-21 17:04:29 +020095$(PIP_TOOLS_INSTALL_DONE):
96 @rm -rf $(VENV_PATH)
Dave Wallacef0797d12019-11-15 17:29:15 -050097 @mkdir -p $(VENV_RUN_DIR)
Klement Sekera7554aef2021-03-30 19:46:06 +020098 @$(PYTHON_INTERP) -m venv $(VENV_PATH)
Aloys Augustin8adb92b2019-07-23 15:19:28 +020099 # pip version pinning
100 @bash -c "source $(VENV_PATH)/bin/activate && \
Klement Sekeraa8a9a8a2021-04-10 12:37:04 +0200101 python3 -m pip install pip===$(PIP_VERSION)"
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200102 @bash -c "source $(VENV_PATH)/bin/activate && \
Klement Sekeraa8a9a8a2021-04-10 12:37:04 +0200103 python3 -m pip install pip-tools===$(PIP_TOOLS_VERSION)"
Dave Wallacefa5aabb2022-02-05 09:27:05 -0500104 @bash -c "source $(VENV_PATH)/bin/activate && \
105 python3 -m pip install setuptools===$(PIP_SETUPTOOLS_VERSION)"
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200106 @touch $@
107
Aloys Augustin8eca60d2020-06-24 11:33:20 +0200108$(PYTHON_DEPENDS): requirements.txt
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200109 @bash -c "source $(VENV_PATH)/bin/activate && \
110 CUSTOM_COMPILE_COMMAND='make test-refresh-deps (or update requirements.txt)' \
Klement Sekeraa8a9a8a2021-04-10 12:37:04 +0200111 python3 -m piptools compile -q --generate-hashes requirements.txt --output-file $@"
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200112
Aloys Augustin8eca60d2020-06-24 11:33:20 +0200113$(PIP_INSTALL_DONE): $(PIP_TOOLS_INSTALL_DONE) $(PYTHON_DEPENDS)
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200114 @bash -c "source $(VENV_PATH)/bin/activate && \
Klement Sekeraa8a9a8a2021-04-10 12:37:04 +0200115 python3 -m piptools sync $(PYTHON_DEPENDS)"
Neale Ranns177bbdc2016-11-15 09:46:51 +0000116 @touch $@
117
Klement Sekera9f9d07b2016-12-12 09:20:50 +0100118$(PIP_PATCH_DONE): $(PIP_INSTALL_DONE)
Neale Ranns177bbdc2016-11-15 09:46:51 +0000119 @echo --- patching ---
Dave Wallace6f692d62017-01-26 21:07:10 -0500120 @sleep 1 # Ensure python recompiles patched *.py files -> *.pyc
snaramre5d4b8912019-12-13 23:39:35 +0000121 for f in $(CURDIR)/patches/scapy-2.4.3/*.patch ; do \
Neale Ranns177bbdc2016-11-15 09:46:51 +0000122 echo Applying patch: $$(basename $$f) ; \
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200123 patch --forward -p1 -d $(SCAPY_SOURCE) < $$f ; \
124 retCode=$$?; \
Aloys Augustin8eca60d2020-06-24 11:33:20 +0200125 [ $$retCode -gt 1 ] && exit $$retCode; \
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200126 done; \
127 touch $@
Neale Ranns177bbdc2016-11-15 09:46:51 +0000128
Klement Sekera9f9d07b2016-12-12 09:20:50 +0100129$(PAPI_INSTALL_DONE): $(PIP_PATCH_DONE)
Klement Sekeraa8a9a8a2021-04-10 12:37:04 +0200130 @bash -c "source $(VENV_PATH)/bin/activate && python3 -m pip install -e $(PAPI_PYTHON_SRC_DIR)"
Neale Ranns177bbdc2016-11-15 09:46:51 +0000131 @touch $@
132
Dave Wallacef0797d12019-11-15 17:29:15 -0500133.PHONY: refresh-deps
Dave Wallace7b8b4652023-08-15 19:05:26 -0400134refresh-deps: clean-deps $(PIP_INSTALL_DONE) $(PYTHON_DEPENDS)
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200135
Dave Wallacef0797d12019-11-15 17:29:15 -0500136.PHONY: clean-deps
Aloys Augustin28fa8cb2019-06-21 17:04:29 +0200137clean-deps:
138 @rm -f $(PYTHON_DEPENDS)
139
Dave Wallacef0797d12019-11-15 17:29:15 -0500140INTERN_PLUGIN_SRC_DIR=$(WS_ROOT)/src/plugins
Dave Wallacea43c93f2019-08-22 00:32:29 +0000141ifneq ($(EXTERN_PLUGIN_SRC_DIR),)
142PLUGIN_SRC_DIR=$(EXTERN_PLUGIN_SRC_DIR)
143else
Dave Wallacef0797d12019-11-15 17:29:15 -0500144PLUGIN_SRC_DIR=$(INTERN_PLUGIN_SRC_DIR)
Dave Wallacea43c93f2019-08-22 00:32:29 +0000145endif
Dave Wallacea43c93f2019-08-22 00:32:29 +0000146
Klement Sekerac160ae02017-03-09 08:20:13 +0100147.PHONY: sanity
148
juraj.linkes184870a2018-07-16 14:22:01 +0200149ifndef TEST_JOBS
150PARALLEL_ILLEGAL=0
Klement Sekerab23ffd72021-05-31 16:08:53 +0200151TEST_JOBS=1
juraj.linkes184870a2018-07-16 14:22:01 +0200152else ifeq ($(FORCE_FOREGROUND),0)
153PARALLEL_ILLEGAL=0
Klement Sekerab23ffd72021-05-31 16:08:53 +0200154else ifneq ($(findstring $(TEST_JOBS),1 auto),)
juraj.linkes184870a2018-07-16 14:22:01 +0200155PARALLEL_ILLEGAL=0
156else
157PARALLEL_ILLEGAL=1
158endif
159
Klement Sekerab23ffd72021-05-31 16:08:53 +0200160ifneq ($(DEBUG),)
161SANITY=no
162endif
163
164ifneq ($(findstring $(SANITY),0 n no),)
165SANITY_IMPORT_VPP_PAPI_CMD=true
166ARG0=
167else
168SANITY_IMPORT_VPP_PAPI_CMD=source $(VENV_PATH)/bin/activate && $(PYTHON_INTERP) sanity_import_vpp_papi.py
169ARG0=--sanity
170endif
171
172ARG1=
173ifneq ($(findstring $(FAILFAST),1 y yes),)
174ARG1=--failfast
175endif
176
177ARG2=
178ifneq ($(findstring $(EXTENDED_TESTS),1 y yes),)
179ARG2=--extended
180endif
181
182ARG3=
183ifneq ($(EXTERN_TESTS),)
184ARG3=--test-src-dir $(EXTERN_TESTS)
185endif
186
187ARG4=
188ifneq ($(findstring $(FORCE_FOREGROUND),1 y yes),)
189ARG4=--force-foreground
190endif
191
192ARG5=
193ifneq ($(findstring $(COREDUMP_COMPRESS),1 y yes),)
194ARG5=--compress-core
195endif
196
197ARG6=
198ifneq ($(findstring $(STEP),1 y yes),)
199ARG6=--step
200endif
201
202ARG7=
Dave Wallace878a9f52023-08-08 15:25:21 -0400203ifneq ($(findstring $(TEST_GCOV),1 y yes),)
Klement Sekerab23ffd72021-05-31 16:08:53 +0200204ARG7=--gcov
205endif
206
207ARG8=
208ifneq ($(EXTERN_PLUGINS),)
209ARG8=--extern-plugin-dir=$(EXTERN_PLUGINS)
210endif
211
212ARG9=
213ifneq ($(DEBUG),)
214ARG9=--debug=$(DEBUG)
215endif
216
217ARG10=
218ifneq ($(COREDUMP_SIZE),)
219ARG10=--coredump-size=$(COREDUMP_SIZE)
220endif
221
222ARG11=
223ifneq ($(VARIANT),)
224ARG11=--variant=$(VARIANT)
225endif
226
227ARG12=--cache-vpp-output
228ifneq ($(findstring $(CACHE_OUTPUT),0 n no),)
229ARG12=
230endif
231
232ARG13=
233ifneq ($(MAX_VPP_CPUS),)
234ARG13=--max-vpp-cpus=$(MAX_VPP_CPUS)
235endif
236
237ARG14=
238ifneq ($(TIMEOUT),)
239ARG14=--timeout=$(TIMEOUT)
240endif
241
242ARG15=
243ifneq ($(findstring $(TEST_DEBUG),1 y yes),)
244ARG15=--debug-framework
245endif
246
247ARG16=
248ifneq ($(findstring $(API_FUZZ),1 y yes),)
249ARG16=--api-fuzz=on
250endif
251
Ole Troan37157da2022-12-01 11:22:06 +0100252ARG17=
253ifneq ($(EXTERN_APIDIR),)
254ARG17=--extern-apidir=$(EXTERN_APIDIR)
255endif
256
Andrew Yourtchenkobc378782023-09-26 16:01:21 +0200257EXC_PLUGINS_ARG=
258ifneq ($(VPP_EXCLUDED_PLUGINS),)
259# convert the comma-separated list into N invocations of the argument to exclude a plugin
260EXC_PLUGINS_ARG=$(shell echo "${VPP_EXCLUDED_PLUGINS}" | sed 's/\([^,]*\)/--excluded-plugin=\1/g; s/,/ /g')
261endif
262
263
264
Ole Troan37157da2022-12-01 11:22:06 +0100265EXTRA_ARGS=$(ARG0) $(ARG1) $(ARG2) $(ARG3) $(ARG4) $(ARG5) $(ARG6) $(ARG7) $(ARG8) $(ARG9) $(ARG10) $(ARG11) $(ARG12) $(ARG13) $(ARG14) $(ARG15) $(ARG16) $(ARG17)
Klement Sekerab23ffd72021-05-31 16:08:53 +0200266
Andrew Yourtchenkobc378782023-09-26 16:01:21 +0200267RUN_TESTS_ARGS=--failed-dir=$(FAILED_DIR) --verbose=$(V) --jobs=$(TEST_JOBS) --filter=$(TEST) --retries=$(RETRIES) --venv-dir=$(VENV_PATH) --vpp-ws-dir=$(WS_ROOT) --vpp-tag=$(TAG) --rnd-seed=$(RND_SEED) --vpp-worker-count="$(VPP_WORKER_COUNT)" --keep-pcaps $(PLUGIN_PATH_ARGS) $(EXC_PLUGINS_ARG) $(TEST_PLUGIN_PATH_ARGS) $(EXTRA_ARGS)
Klement Sekerab3fc6582022-03-10 11:47:45 +0100268RUN_SCRIPT_ARGS=--python-opts=$(PYTHON_OPTS)
Klement Sekerab23ffd72021-05-31 16:08:53 +0200269
270define retest-func
Klement Sekerab3fc6582022-03-10 11:47:45 +0100271@scripts/run.sh $(RUN_SCRIPT_ARGS) $(RUN_TESTS_ARGS) || env FAILED_DIR=$(FAILED_DIR) COMPRESS_FAILED_TEST_LOGS=$(COMPRESS_FAILED_TEST_LOGS) scripts/compress_failed.sh
Klement Sekerab23ffd72021-05-31 16:08:53 +0200272endef
273
Klement Sekerae2636852021-03-16 12:52:12 +0100274sanity: test-dep
juraj.linkes184870a2018-07-16 14:22:01 +0200275 @bash -c "test $(PARALLEL_ILLEGAL) -eq 0 ||\
276 (echo \"*******************************************************************\" &&\
277 echo \"* Sanity check failed, TEST_JOBS is not 1 or 'auto' and DEBUG, STEP or PROFILE is set\" &&\
278 echo \"*******************************************************************\" &&\
279 false)"
Klement Sekera41a9e512017-10-17 07:59:31 +0200280 @bash -c "$(SANITY_IMPORT_VPP_PAPI_CMD) ||\
Klement Sekerac160ae02017-03-09 08:20:13 +0100281 (echo \"*******************************************************************\" &&\
282 echo \"* Sanity check failed, cannot import vpp_papi\" &&\
283 echo \"* to debug: \" &&\
284 echo \"* 1. enter test shell: make test-shell\" &&\
285 echo \"* 2. execute debugger: gdb python -ex 'run sanity_import_vpp_papi.py'\" &&\
286 echo \"*******************************************************************\" &&\
287 false)"
288
Dave Wallacef0797d12019-11-15 17:29:15 -0500289$(FAILED_DIR): reset
290 @mkdir -p $@
291
292.PHONY: test-dep
Dave Wallaceeddd8e32021-05-12 21:43:59 -0400293test-dep: $(PAPI_INSTALL_DONE) $(FAILED_DIR)
Dave Wallacef0797d12019-11-15 17:29:15 -0500294
295.PHONY: test
Damjan Mariona2d6d352021-05-25 18:29:24 +0200296test: test-dep sanity
Juraj Sloboda0279b292016-11-16 19:50:24 +0100297 $(call retest-func)
Klement Sekeraf62ae122016-10-11 11:47:09 +0200298
Dave Wallacef0797d12019-11-15 17:29:15 -0500299.PHONY: retest
300retest: verify-env sanity $(FAILED_DIR)
Juraj Sloboda0279b292016-11-16 19:50:24 +0100301 $(call retest-func)
Klement Sekeraf62ae122016-10-11 11:47:09 +0200302
Dave Wallacef0797d12019-11-15 17:29:15 -0500303.PHONY: shell
304shell: test-dep
Klement Sekerab8c72a42018-11-08 11:21:39 +0100305 @echo "source $(VENV_PATH)/bin/activate;\
Paul Vinciguerrac701e572019-12-19 16:09:43 -0500306 export RND_SEED=$(RND_SEED);\
Klement Sekeraa86e2892017-03-09 08:01:52 +0100307 echo '***';\
Dave Wallacefd243742021-03-16 22:12:07 -0400308 echo PYTHONPATH=$(PYTHONPATH);\
Paul Vinciguerrac701e572019-12-19 16:09:43 -0500309 echo RND_SEED=$(RND_SEED);\
Klement Sekera593635d2019-02-07 19:56:50 +0100310 echo VPP_BUILD_DIR=$(VPP_BUILD_DIR);\
Klement Sekera593635d2019-02-07 19:56:50 +0100311 echo VPP_PLUGIN_PATH=$(VPP_PLUGIN_PATH);\
Dave Barach7d31ab22019-05-08 19:18:18 -0400312 echo VPP_TEST_PLUGIN_PATH=$(VPP_TEST_PLUGIN_PATH);\
Klement Sekera593635d2019-02-07 19:56:50 +0100313 echo VPP_INSTALL_PATH=$(VPP_INSTALL_PATH);\
Klement Sekera47e275b2017-03-21 08:21:25 +0100314 echo EXTERN_TESTS=$(EXTERN_TESTS);\
315 echo EXTERN_PLUGINS=$(EXTERN_PLUGINS);\
Klement Sekerab23ffd72021-05-31 16:08:53 +0200316 echo EXTERN_COV_DIR=$(EXTERN_COV_DIR);\
Klement Sekeraa86e2892017-03-09 08:01:52 +0100317 echo LD_LIBRARY_PATH=$(LD_LIBRARY_PATH);\
318 echo '***';\
319 exec </dev/tty" | bash -i
320
Dave Wallacef0797d12019-11-15 17:29:15 -0500321.PHONY: reset
Neale Ranns177bbdc2016-11-15 09:46:51 +0000322reset:
Klement Sekeraaa7a2a12021-03-18 15:22:57 +0100323 @rm -f /dev/shm/vpp-unittest-*
Klement Sekerae2636852021-03-16 12:52:12 +0100324 @if [ $(FORCE_NO_WIPE) -eq "0" ] ; then rm -rf /tmp/vpp-unittest-*; fi
Dave Wallacef0797d12019-11-15 17:29:15 -0500325 @rm -f /tmp/api_post_mortem.*
Klement Sekerab8c72a42018-11-08 11:21:39 +0100326 @rm -rf $(FAILED_DIR)
Naveen Joy7ea7ab52021-05-11 10:31:18 -0700327 @rm -rf /tmp/vpp-vm-tests
Klement Sekeraf62ae122016-10-11 11:47:09 +0200328
Dave Wallacef0797d12019-11-15 17:29:15 -0500329.PHONY: wipe
Neale Ranns177bbdc2016-11-15 09:46:51 +0000330wipe: reset
Saima Yunusc7f93b32022-08-10 03:25:31 -0400331 @rm -rf $(VENV_BR_DIR)
Neale Ranns02da9262021-01-08 08:43:14 +0000332 @rm -rf $(patsubst %,%/__pycache__, $(VPP_TEST_DIRS))
Neale Ranns177bbdc2016-11-15 09:46:51 +0000333
Dave Wallacef0797d12019-11-15 17:29:15 -0500334$(BUILD_COV_DIR):
335 @mkdir -p $@
336
Dave Wallace1c95e122023-06-21 23:14:38 -0400337.PHONY: cov-prep
338cov-prep: test-dep
Klement Sekera593635d2019-02-07 19:56:50 +0100339 @lcov --zerocounters --directory $(VPP_BUILD_DIR)
Klement Sekeracd6cb982017-05-11 06:55:32 +0200340 @test -z "$(EXTERN_COV_DIR)" || lcov --zerocounters --directory $(EXTERN_COV_DIR)
Dave Wallace1c95e122023-06-21 23:14:38 -0400341
342.PHONY: cov-post
343cov-post: wipe-cov $(BUILD_COV_DIR)
344 @lcov --capture \
345 --directory $(VPP_BUILD_DIR) \
346 --output-file $(BUILD_COV_DIR)/coverage.info
347 @test -z "$(EXTERN_COV_DIR)" || \
348 lcov --capture \
349 --directory $(EXTERN_COV_DIR) \
350 --output-file $(BUILD_COV_DIR)/extern-coverage.info
351 @lcov --remove $(BUILD_COV_DIR)/coverage.info \
352 "/usr/include/*" "*/build-root/*" "/opt/*" "/usr/lib/*" \
Dave Wallaceef68bb52023-08-09 12:58:22 -0400353 "*_test.*" "*vat*" "*rdma*" "*/vpp-api/client/*" "*/plugins/af_packet/*" \
354 "*/plugins/af_xdp/*" "*/plugins/avf/*" "*/plugins/dma_intel/*" \
355 "*/plugins/hs_apps/*" "*/plugins/vmxnet3/*" "*/vnet/devices/virtio/*" \
Dave Wallace1c95e122023-06-21 23:14:38 -0400356 -o $(BUILD_COV_DIR)/coverage-filtered.info
357 @genhtml $(BUILD_COV_DIR)/coverage-filtered.info \
358 --output-directory $(BUILD_COV_DIR)/html
359 @test -z "$(EXTERN_COV_DIR)" || \
360 genhtml $(BUILD_COV_DIR)/extern-coverage.info \
361 --output-directory $(BUILD_COV_DIR)/extern-html
Juraj Sloboda0279b292016-11-16 19:50:24 +0100362 @echo
363 @echo "Build finished. Code coverage report is in $(BUILD_COV_DIR)/html/index.html"
Klement Sekeracd6cb982017-05-11 06:55:32 +0200364 @test -z "$(EXTERN_COV_DIR)" || echo "Code coverage report for out-of-tree objects is in $(BUILD_COV_DIR)/extern-html/index.html"
Juraj Sloboda0279b292016-11-16 19:50:24 +0100365
Dave Wallace1c95e122023-06-21 23:14:38 -0400366.PHONY: cov
367cov:
368 make -C . cov-prep test cov-post
369
Klement Sekera9f9d07b2016-12-12 09:20:50 +0100370.PHONY: wipe-cov
Juraj Sloboda0279b292016-11-16 19:50:24 +0100371wipe-cov: wipe
372 @rm -rf $(BUILD_COV_DIR)
373
Dave Wallacef0797d12019-11-15 17:29:15 -0500374.PHONY: wipe-papi
375wipe-papi:
376 @rm -rf $(PAPI_INSTALL_DONE) $(PAPI_WIPE_DIST)
Paul Vinciguerra2648e442019-07-09 16:49:46 -0400377
Dave Wallacef0797d12019-11-15 17:29:15 -0500378.PHONY: wipe-all
Nathan Skrzypczak9ad39c02021-08-19 11:38:06 +0200379wipe-all: wipe wipe-papi wipe-cov
Dave Wallacef0797d12019-11-15 17:29:15 -0500380 @rm -rf $(TEST_BR)
Paul Vinciguerra2648e442019-07-09 16:49:46 -0400381
Klement Sekerae2636852021-03-16 12:52:12 +0100382.PHONY: start-gdb
383start-gdb: sanity
Klement Sekerab23ffd72021-05-31 16:08:53 +0200384 @bash -c "source $(VENV_PATH)/bin/activate && python3 -c 'from debug import start_vpp_in_gdb; start_vpp_in_gdb()' $(RUN_TESTS_ARGS)"
Klement Sekerae2636852021-03-16 12:52:12 +0100385
Klement Sekerad9b0c6f2022-04-26 19:02:15 +0200386.PHONY: checkstyle-python-all
387checkstyle-python-all: $(PIP_INSTALL_DONE)
Ole Troan4aacc012019-10-23 11:47:33 +0200388 @bash -c "source $(VENV_PATH)/bin/activate &&\
Klement Sekerae2159bd2022-05-11 16:54:09 +0200389 black -t py39 --check --diff $(WS_ROOT) ||\
Klement Sekerad9b0c6f2022-04-26 19:02:15 +0200390 (echo \"*************************************************************************\" &&\
391 echo \"* Test framework PEP8 compliance check FAILED (maybe: make fixstyle-python)\" &&\
392 echo \"*************************************************************************\" &&\
393 false)"
Klement Sekera72715ee2017-01-17 10:37:05 +0100394 @echo "*******************************************************************"
Klement Sekerad9b0c6f2022-04-26 19:02:15 +0200395 @echo "* Test framework PEP8 compliance check passed"
396 @echo "*******************************************************************"
397
398.PHONY: checkstyle
399checkstyle: checkstyle-python-all
400
401.PHONY: fixstyle-python-all
402fixstyle-python-all: $(PIP_INSTALL_DONE)
403 @bash -c "source $(VENV_PATH)/bin/activate &&\
404 black -t py39 $(WS_ROOT) ||\
405 (echo \"*************************************************************************\" &&\
406 echo \"* Test framework PEP8 compliance check FAILED (maybe: make fixstyle-python)\" &&\
407 echo \"*************************************************************************\" &&\
408 false)"
409 @echo "*******************************************************************"
410 @echo "* Test framework PEP8 compliance check passed"
Klement Sekera72715ee2017-01-17 10:37:05 +0100411 @echo "*******************************************************************"
412
Dave Wallacef0797d12019-11-15 17:29:15 -0500413.PHONY: help
Klement Sekera277b89c2016-10-28 13:20:27 +0200414help:
415 @echo "Running tests:"
416 @echo ""
Dave Wallacef0797d12019-11-15 17:29:15 -0500417 @echo " test - build and run (basic) functional tests"
418 @echo " test-debug - build and run (basic) functional tests (debug build)"
Dave Wallace1c95e122023-06-21 23:14:38 -0400419 @echo " test-cov - generate code coverage report for functional tests"
420 @echo " test-cov-prep - coverage phase #1 : prepare lcov"
421 @echo " test-cov-build - coverage phase #2 : build gcov image & run tests against it (use TEST=)"
422 @echo " test-cov-post - coverage phase #3 : generate lcov html report"
Dave Wallacef0797d12019-11-15 17:29:15 -0500423 @echo " test-all - build and run functional and extended tests"
424 @echo " test-all-debug - build and run functional and extended tests (debug build)"
Dave Wallace1c95e122023-06-21 23:14:38 -0400425 @echo " test-all-cov - generate code coverage report for functional and extended tests"
Dave Wallacef0797d12019-11-15 17:29:15 -0500426 @echo " retest - run functional tests"
427 @echo " retest-debug - run functional tests (debug build)"
428 @echo " retest-all - run functional and extended tests"
429 @echo " retest-all-debug - run functional and extended tests (debug build)"
Dave Wallacef0797d12019-11-15 17:29:15 -0500430 @echo " test-wipe - wipe (temporary) files generated by unit tests"
431 @echo " test-wipe-cov - wipe code coverage report for test framework"
Dave Wallacef0797d12019-11-15 17:29:15 -0500432 @echo " test-wipe-papi - rebuild vpp_papi sources"
Nathan Skrzypczak9ad39c02021-08-19 11:38:06 +0200433 @echo " test-wipe-all - wipe (temporary) files generated by unit tests, and coverage"
Dave Wallacef0797d12019-11-15 17:29:15 -0500434 @echo " test-shell - enter shell with test environment"
435 @echo " test-shell-debug - enter shell with test environment (debug build)"
Klement Sekerae2636852021-03-16 12:52:12 +0100436 @echo " test-refresh-deps - refresh the Python dependencies for the tests"
Klement Sekera277b89c2016-10-28 13:20:27 +0200437 @echo ""
Klement Sekera08c50e32023-04-14 17:44:04 +0200438 @echo "Environment variables controlling test runs:"
Klement Sekerab23ffd72021-05-31 16:08:53 +0200439 @echo ""
Klement Sekera08c50e32023-04-14 17:44:04 +0200440 @echo " V=[0|1|2]"
441 @echo " set test verbosity level: 0=ERROR, 1=INFO, 2=DEBUG"
442 @echo ""
443 @echo " TEST_JOBS=[<n>|auto]"
444 @echo " use at most <n> parallel python processes for test"
445 @echo " execution, if auto, set to number of available cpus"
446 @echo " (default: 1)"
447 @echo ""
448 @echo " MAX_VPP_CPUS=[<n>|auto]"
449 @echo " use at most <n> cpus for running vpp"
450 @echo " 'auto' sets to number of available cpus"
451 @echo " (default: auto)"
452 @echo ""
453 @echo " CACHE_OUTPUT=[0|n|no]"
454 @echo " disable caching VPP stdout/stderr and logging it"
455 @echo " as one block after test finishes"
456 @echo " (default: yes)"
457 @echo ""
458 @echo " FAILFAST=[1|y|yes]"
459 @echo " if enabled, stop running tests on first failure"
460 @echo " otherwise finish running whole suite"
461 @echo " (default: no)"
462 @echo ""
463 @echo " TIMEOUT=<timeout>"
464 @echo " fail test suite if any single test takes longer"
465 @echo " than <timeout> (in seconds) to finish"
466 @echo " (default: 600)"
467 @echo ""
468 @echo " RETRIES=<n>"
469 @echo " retry failed tests <n> times"
470 @echo " (default: 0)"
471 @echo ""
472 @echo " DEBUG=<type>"
473 @echo " configure VPP debugging:"
474 @echo " DEBUG=core"
475 @echo " detect coredump and load it in gdb on crash"
476 @echo ""
477 @echo " DEBUG=gdb"
478 @echo " print VPP PID and wait for user input before"
479 @echo " running and tearing down a testcase, allowing"
480 @echo " easy gdb attach"
481 @echo ""
482 @echo " DEBUG=gdbserver"
483 @echo " same as above, but run gdb inside a gdb server"
484 @echo ""
485 @echo " DEBUG=attach"
486 @echo " attach to existing vpp in running in gdb"
487 @echo " (see test-start-vpp-in-gdb)"
488 @echo " (default: none)"
489 @echo ""
490 @echo " STEP=[1|y|yes]"
491 @echo " enable stepping through a testcase"
492 @echo " (default: no)"
493 @echo ""
494 @echo " SANITY=[0|n|no]"
495 @echo " disable sanity import of vpp-api/vpp sanity"
496 @echo " run before running tests"
497 @echo " (default: yes)"
498 @echo ""
499 @echo " EXTENDED_TESTS=[1|y|yes]"
500 @echo " run extended tests"
501 @echo " (default: no)"
502 @echo ""
503 @echo " TEST=<filter>,[<filter>],..."
504 @echo " only run tests matching one or more comma-delimited"
505 @echo " filter expressions"
506 @echo ""
507 @echo " simple filter:"
508 @echo " file name or file suffix select all tests from a file"
509 @echo " examples:"
510 @echo " TEST=test_bfd"
511 @echo " TEST=bfd"
512 @echo " equivalent expressions selecting all"
513 @echo " tests defined in test_bfd.py"
514 @echo ""
515 @echo " wildcard filter:"
516 @echo " advanced filtering based on test file, test class"
517 @echo " and test function"
518 @echo " each filter expression is in the form of"
519 @echo " <file>.<class>.<test function>"
520 @echo " each of the tokens can be left empty or replaced"
521 @echo " with '*' to select all objects available"
522 @echo " examples:"
523 @echo " TEST=test_bfd.*.*"
524 @echo " TEST=test_bfd.."
525 @echo " TEST=bfd.*.*"
526 @echo " TEST=bfd.."
527 @echo " select all tests defined in test_bfd.py"
528 @echo " TEST=bfd.BFDAPITestCase.*"
529 @echo " TEST=bfd.BFDAPITestCase."
530 @echo " select all tests from test_bfd.py"
531 @echo " which are part of BFDAPITestCase class"
532 @echo " TEST=bfd.BFDAPITestCase.test_add_bfd"
533 @echo " select a single test named test_add_bfd"
534 @echo " from test_bfd.py/BFDAPITestCase"
535 @echo " TEST=..test_add_bfd"
536 @echo " TEST=*.*.test_add_bfd"
537 @echo " select all test functions named test_add_bfd"
538 @echo " from all files/classes"
539 @echo " TEST=bfd,ip4,..test_icmp_error"
540 @echo " select all test functions in test_bfd.py,"
541 @echo " test_ip4.py and all test functions named"
542 @echo " 'test_icmp_error' in all files"
543 @echo " (default: '')"
544 @echo ""
545 @echo " VARIANT=<variant>"
546 @echo " specify which march node variant to unit test"
547 @echo " e.g. VARIANT=skx test the skx march variants"
548 @echo " e.g. VARIANT=icl test the icl march variants"
549 @echo " (default: '')"
550 @echo ""
551 @echo " COREDUMP_SIZE=<size>"
552 @echo " pass <size> as unix { coredump-size <size> } argument"
553 @echo " to vpp, e.g. COREDUMP_SIZE=4g or COREDUMP_SIZE=unlimited"
554 @echo " (default: '')"
555 @echo ""
556 @echo " COREDUMP_COMPRESS=[1|y|yes]"
557 @echo " if no debug option is set, compress any core files"
558 @echo " (default: no)"
559 @echo ""
560 @echo " EXTERN_TESTS=<path>"
561 @echo " include out-of-tree test_*.py files under <path>"
562 @echo " (default: '')"
563 @echo ""
564 @echo " EXTERN_PLUGINS=<path>"
565 @echo " load out-of-tree vpp plugins in <path>"
566 @echo " (default: '')"
567 @echo ""
568 @echo " EXTERN_COV_DIR=<path>"
569 @echo " path to out-of-tree prefix, where source, object"
570 @echo " and .gcda files can be found for coverage report"
571 @echo " (default: '')"
572 @echo ""
573 @echo " PROFILE=[1|y|yes]"
574 @echo " enable profiling of test framework via cProfile module"
575 @echo " (default: no)"
576 @echo ""
577 @echo " PROFILE_SORT_BY=opt"
578 @echo " sort profiling report by opt - see cProfile documentation"
579 @echo " for possible values"
580 @echo " (default: cumtime)"
581 @echo ""
582 @echo " PROFILE_OUTPUT=file"
583 @echo " output profiling info to file - use absolute path"
584 @echo " (default: stdout)"
585 @echo ""
586 @echo " TEST_DEBUG=[1|y|yes]"
587 @echo " enable debugging of the test framework itself (expert)"
588 @echo " (default: no)"
589 @echo ""
Dave Wallace1c95e122023-06-21 23:14:38 -0400590 @echo " TEST_GCOV=[1|y|yes]"
591 @echo " enable tests specifically designed soley for code coverage"
592 @echo " (default: no)"
593 @echo ""
Klement Sekera08c50e32023-04-14 17:44:04 +0200594 @echo " API_FUZZ=[1|y|yes]"
595 @echo " enable VPP api fuzz testing"
596 @echo " (default: no)"
597 @echo ""
598 @echo " RND_SEED=<seed>"
599 @echo " random seed used by test framework"
600 @echo " (default: time.time())"
Klement Sekera45a95dd2019-11-05 11:18:25 +0000601 @echo ""
Klement Sekerae2636852021-03-16 12:52:12 +0100602 @echo "Starting VPP in GDB for use with DEBUG=attach:"
603 @echo ""
604 @echo " test-start-vpp-in-gdb - start VPP in gdb (release)"
605 @echo " test-start-vpp-debug-in-gdb - start VPP in gdb (debug)"
606 @echo ""