tests docs: fix lcov code coverage report generation
- Updated/rebased version of https://gerrit.fd.io/r/c/vpp/+/34199
Type: test
Change-Id: I43913ecfd11a4578bdb10c4be76253fe38d57976
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>
diff --git a/.gitignore b/.gitignore
index ec9ee7b..bcfee43 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,14 +16,8 @@
/build-root/*.tar.xz
/build-root/*.changes
/build-root/rpmbuild/
-/build-root/test-doc/
-/build-root/test-cov/
+/build-root/test-coverage/
/build-root/test/
-/test/run/
-/test/build/
-/test/coverage/
-/test/vapi_test/
-/test/doc/build/
/build-config.mk
/build/external/*.tar.gz
/build/external/*.tar.xz
diff --git a/Makefile b/Makefile
index 88d42df..2b781fb 100644
--- a/Makefile
+++ b/Makefile
@@ -65,7 +65,7 @@
DEB_DEPENDS = curl build-essential autoconf automake ccache
DEB_DEPENDS += debhelper dkms git libtool libapr1-dev dh-python
DEB_DEPENDS += libconfuse-dev git-review exuberant-ctags cscope pkg-config
-DEB_DEPENDS += lcov chrpath autoconf libnuma-dev
+DEB_DEPENDS += gcovr lcov chrpath autoconf libnuma-dev
DEB_DEPENDS += python3-all python3-setuptools check
DEB_DEPENDS += libffi-dev python3-ply
DEB_DEPENDS += cmake ninja-build uuid-dev python3-jsonschema python3-yaml
@@ -426,7 +426,7 @@
export TEST_DIR ?= $(WS_ROOT)/test
define test
- $(if $(filter-out $(2),retest),make -C $(BR) PLATFORM=vpp TAG=$(1) vpp-install,)
+ $(if $(filter-out $(2),retest),make -C $(BR) PLATFORM=vpp TAG=$(1) CC=$(CC) vpp-install,)
$(eval libs:=lib lib64)
make -C test \
VPP_BUILD_DIR=$(BR)/build-$(1)-native/vpp \
@@ -449,10 +449,28 @@
test-debug:
$(call test,vpp_debug,test)
-.PHONY: test-gcov
-test-gcov:
+.PHONY: test-cov
+test-cov:
+ $(eval CC=gcc)
+ $(eval TEST_GCOV=1)
+ $(call test,vpp_gcov,cov)
+
+.PHONY: test-cov-build
+test-cov-build:
+ $(eval CC=gcc)
+ $(eval TEST_GCOV=1)
$(call test,vpp_gcov,test)
+.PHONY: test-cov-prep
+test-cov-prep:
+ $(eval CC=gcc)
+ $(call test,vpp_gcov,cov-prep)
+
+.PHONY: test-cov-post
+test-cov-post:
+ $(eval CC=gcc)
+ $(call test,vpp_gcov,cov-post)
+
.PHONY: test-all
test-all:
$(eval EXTENDED_TESTS=1)
@@ -463,6 +481,13 @@
$(eval EXTENDED_TESTS=1)
$(call test,vpp_debug,test)
+.PHONY: test-all-cov
+test-all-cov:
+ $(eval CC=gcc)
+ $(eval TEST_GCOV=1)
+ $(eval EXTENDED_TESTS=1)
+ $(call test,vpp_gcov,test)
+
.PHONY: papi-wipe
papi-wipe: test-wipe-papi
$(call banner,"This command is deprecated. Please use 'test-wipe-papi'")
@@ -487,8 +512,10 @@
test-shell-debug:
$(call test,vpp_debug,shell)
-.PHONY: test-shell-gcov
-test-shell-gcov:
+.PHONY: test-shell-cov
+test-shell-cov:
+ $(eval CC=gcc)
+ $(eval TEST_GCOV=1)
$(call test,vpp_gcov,shell)
.PHONY: test-dep
@@ -505,13 +532,9 @@
@echo "make test-wipe-doc is DEPRECATED"
sleep 300
-.PHONY: test-cov
-test-cov:
- $(eval EXTENDED_TESTS=1)
- $(call test,vpp_gcov,cov)
-
.PHONY: test-wipe-cov
test-wipe-cov:
+ $(call make,$(PLATFORM)_gcov,$(addsuffix -wipe,$(TARGETS)))
@make -C test wipe-cov
.PHONY: test-wipe-all
diff --git a/docs/developer/build-run-debug/code_coverage.rst b/docs/developer/build-run-debug/code_coverage.rst
new file mode 100644
index 0000000..efe750c
--- /dev/null
+++ b/docs/developer/build-run-debug/code_coverage.rst
@@ -0,0 +1,45 @@
+.. _lcov_code_coverage:
+
+Code coverage with lcov
+=======================
+
+Prerequisites
+-------------
+
+Ensure required packages are installed:
+
+::
+
+ $ make install-deps
+
+The Linux gcov and lcov tools can be fussy about gcc / g++ compiler
+versions. As of this writing, Ubuntu 22.04 gcov / lcov works with
+the latest gcc version (``11.3.0``).
+
+Generate coverage for a test case
+---------------------------------
+
+As a first run, in order to generate the coverage report of
+a specific plugin or test, run for example
+
+::
+
+ $ make test-cov TEST=fib
+
+Then open the file ``.build-root/test-coverage/html/index.html`` in a Chrome browser.
+
+Improving test coverage
+-----------------------
+
+When doing modifications on the test cases, you can run
+
+::
+
+ # This will run the test & report the result in the coverage data
+ $ make test-cov-build TEST=fib
+
+ # This will generate the html report with the current coverage data
+ $ make test-cov-post
+
+ # To reset the coverage data use
+ $ make test-cov-prep
diff --git a/docs/developer/build-run-debug/index.rst b/docs/developer/build-run-debug/index.rst
index f8bfeab..c99615f 100644
--- a/docs/developer/build-run-debug/index.rst
+++ b/docs/developer/build-run-debug/index.rst
@@ -12,3 +12,4 @@
testing_vpp
gdb_examples
cross_compile_macos
+ code_coverage
diff --git a/docs/developer/extras/index.rst b/docs/developer/extras/index.rst
index a0f2d9b..2ee96be 100644
--- a/docs/developer/extras/index.rst
+++ b/docs/developer/extras/index.rst
@@ -7,7 +7,6 @@
.. toctree::
:maxdepth: 2
- lcov
snap
strongswan
vpp_config
diff --git a/docs/developer/extras/lcov.rst b/docs/developer/extras/lcov.rst
deleted file mode 120000
index 9f8de24..0000000
--- a/docs/developer/extras/lcov.rst
+++ /dev/null
@@ -1 +0,0 @@
-../../../extras/lcov/README.rst
\ No newline at end of file
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index 80db33a..d28c6df 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -397,7 +397,6 @@
gethostbyname
gettingsources
gettingstarted
-ggcov
gid
Gigabit
GigE
diff --git a/extras/lcov/README.rst b/extras/lcov/README.rst
deleted file mode 100644
index 3dab951..0000000
--- a/extras/lcov/README.rst
+++ /dev/null
@@ -1,48 +0,0 @@
-.. _lcov_code_coverage:
-
-Code coverage with lcov
-=======================
-
-Prerequisites
--------------
-
-The Linux gcov and lcov tools are fussy about gcc / g++ compiler
-versions. As of this writing, Ubuntu 18.04 gcov / lcov work with these
-toolchain versions:
-
-$ gcc –version gcc (Ubuntu 7.4.0-1ubuntu1~18.04.1) 7.4.0 $ g++ –version
-g++ (Ubuntu 8.3.0-6ubuntu1~18.04.1) 8.3.0
-
-Refer to
-https://askubuntu.com/questions/26498/how-to-choose-the-default-gcc-and-g-version
-for information on how to install multiple gcc / g++ versions, and
-switch between them.
-
-You’ll need to install the following additional packages:
-
-$ sudo apt-get install gcovr ggcov lcov
-
-Compile an instrumented vpp image
----------------------------------
-
-Two ways:
-
-$ cd $ make test-gcov $ ## interrupt compilation after building the
-image
-
-or $ cd /build-root $ make PLATFORM=vpp TAG=vpp_gcov vpp-install
-
-Initialize the lcov database
-----------------------------
-
-$ cd $ ./extras/lcov/lcov_prep $ make test-gcov or make TEST=my_test
-test-gcov $ # repeat or vary as desired to increase reported coverage $
-# Generate the report: $ ./extras/lcov/lcov_post
-
-You can run vpp manually, do anything you like. Results are cumulative
-until you re-run the “prep” script.
-
-Look at the results
--------------------
-
-Point a browser at file:////build-root/html/index.html
diff --git a/extras/lcov/lcov_post b/extras/lcov/lcov_post
deleted file mode 100755
index 1aaf986..0000000
--- a/extras/lcov/lcov_post
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-set -e
-
-cd build-root
-rm -rf html
-mkdir html
-lcov --no-checksum --directory . --capture --output-file out.info
-lcov --remove out.info \
- "/usr/include/*" "*/build-root/*" "/opt/*" "/usr/lib/*" \
- -o filtered.info
-genhtml filtered.info -o html
diff --git a/extras/lcov/lcov_prep b/extras/lcov/lcov_prep
deleted file mode 100755
index 1212d04..0000000
--- a/extras/lcov/lcov_prep
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-
-set -e
-
-cd build-root
-rm -rf html
-lcov --capture --initial --directory . --output-file lcov.out
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index fb4463e..b6d48f5 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -127,6 +127,7 @@
elseif (${CMAKE_BUILD_TYPE_LC} MATCHES "gcov")
add_compile_options(-O0 -fprofile-arcs -ftest-coverage)
add_compile_definitions(CLIB_DEBUG CLIB_GCOV)
+ link_libraries(gcov)
endif()
set(BUILD_TYPES release debug coverity gcov)
diff --git a/src/vlib/unix/main.c b/src/vlib/unix/main.c
index 74ad19e..f09a537 100644
--- a/src/vlib/unix/main.c
+++ b/src/vlib/unix/main.c
@@ -145,17 +145,6 @@
break;
}
-#ifdef CLIB_GCOV
- /*
- * Test framework sends SIGTERM, so we need to flush the
- * code coverage stats here.
- */
- {
- void __gcov_flush (void);
- __gcov_flush ();
- }
-#endif
-
/* Null terminate. */
vec_add1 (syslog_msg, 0);
diff --git a/test/Makefile b/test/Makefile
index 41d28f9..27e7501 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -80,7 +80,7 @@
PIP_SETUPTOOLS_VERSION=62.1.0
PYTHON_DEPENDS=requirements-$(PYTHON_VERSION).txt
SCAPY_SOURCE=$(shell find $(VENV_PATH)/lib/python* -name site-packages)
-BUILD_COV_DIR=$(TEST_BR)/coverage
+BUILD_COV_DIR=$(BR)/test-coverage
PIP_TOOLS_INSTALL_DONE=$(VENV_RUN_DIR)/pip-tools-install-$(PYTHON_VERSION)-$(PIP_TOOLS_VERSION).done
PIP_INSTALL_DONE=$(VENV_RUN_DIR)/pip-install-$(PYTHON_VERSION)-$(PIP_VERSION).done
@@ -326,19 +326,36 @@
$(BUILD_COV_DIR):
@mkdir -p $@
-.PHONY: cov
-cov: wipe-cov test-dep ext $(BUILD_COV_DIR)
+.PHONY: cov-prep
+cov-prep: test-dep
@lcov --zerocounters --directory $(VPP_BUILD_DIR)
@test -z "$(EXTERN_COV_DIR)" || lcov --zerocounters --directory $(EXTERN_COV_DIR)
- $(call retest-func)
- @lcov --capture --directory $(VPP_BUILD_DIR) --output-file $(BUILD_COV_DIR)/coverage.info
- @test -z "$(EXTERN_COV_DIR)" || lcov --capture --directory $(EXTERN_COV_DIR) --output-file $(BUILD_COV_DIR)/extern-coverage.info
- @genhtml $(BUILD_COV_DIR)/coverage.info --output-directory $(BUILD_COV_DIR)/html
- @test -z "$(EXTERN_COV_DIR)" || genhtml $(BUILD_COV_DIR)/extern-coverage.info --output-directory $(BUILD_COV_DIR)/extern-html
+
+.PHONY: cov-post
+cov-post: wipe-cov $(BUILD_COV_DIR)
+ @lcov --capture \
+ --directory $(VPP_BUILD_DIR) \
+ --output-file $(BUILD_COV_DIR)/coverage.info
+ @test -z "$(EXTERN_COV_DIR)" || \
+ lcov --capture \
+ --directory $(EXTERN_COV_DIR) \
+ --output-file $(BUILD_COV_DIR)/extern-coverage.info
+ @lcov --remove $(BUILD_COV_DIR)/coverage.info \
+ "/usr/include/*" "*/build-root/*" "/opt/*" "/usr/lib/*" \
+ -o $(BUILD_COV_DIR)/coverage-filtered.info
+ @genhtml $(BUILD_COV_DIR)/coverage-filtered.info \
+ --output-directory $(BUILD_COV_DIR)/html
+ @test -z "$(EXTERN_COV_DIR)" || \
+ genhtml $(BUILD_COV_DIR)/extern-coverage.info \
+ --output-directory $(BUILD_COV_DIR)/extern-html
@echo
@echo "Build finished. Code coverage report is in $(BUILD_COV_DIR)/html/index.html"
@test -z "$(EXTERN_COV_DIR)" || echo "Code coverage report for out-of-tree objects is in $(BUILD_COV_DIR)/extern-html/index.html"
+.PHONY: cov
+cov:
+ make -C . cov-prep test cov-post
+
.PHONY: wipe-cov
wipe-cov: wipe
@rm -rf $(BUILD_COV_DIR)
@@ -388,14 +405,17 @@
@echo ""
@echo " test - build and run (basic) functional tests"
@echo " test-debug - build and run (basic) functional tests (debug build)"
+ @echo " test-cov - generate code coverage report for functional tests"
+ @echo " test-cov-prep - coverage phase #1 : prepare lcov"
+ @echo " test-cov-build - coverage phase #2 : build gcov image & run tests against it (use TEST=)"
+ @echo " test-cov-post - coverage phase #3 : generate lcov html report"
@echo " test-all - build and run functional and extended tests"
@echo " test-all-debug - build and run functional and extended tests (debug build)"
+ @echo " test-all-cov - generate code coverage report for functional and extended tests"
@echo " retest - run functional tests"
@echo " retest-debug - run functional tests (debug build)"
@echo " retest-all - run functional and extended tests"
@echo " retest-all-debug - run functional and extended tests (debug build)"
- @echo " test-cov - generate code coverage report for test framework"
- @echo " test-gcov - build and run functional tests (gcov build)"
@echo " test-wipe - wipe (temporary) files generated by unit tests"
@echo " test-wipe-cov - wipe code coverage report for test framework"
@echo " test-wipe-papi - rebuild vpp_papi sources"
@@ -556,6 +576,10 @@
@echo " enable debugging of the test framework itself (expert)"
@echo " (default: no)"
@echo ""
+ @echo " TEST_GCOV=[1|y|yes]"
+ @echo " enable tests specifically designed soley for code coverage"
+ @echo " (default: no)"
+ @echo ""
@echo " API_FUZZ=[1|y|yes]"
@echo " enable VPP api fuzz testing"
@echo " (default: no)"
@@ -569,7 +593,3 @@
@echo " test-start-vpp-in-gdb - start VPP in gdb (release)"
@echo " test-start-vpp-debug-in-gdb - start VPP in gdb (debug)"
@echo ""
- @echo "Creating test code coverage report:"
- @echo ""
- @echo " test-cov - generate code coverage report for test framework"
- @echo " test-wipe-cov - wipe code coverage report for test framework"