blob: 718055ae7df7d457473abf200c8ddd382e1e9643 [file] [log] [blame]
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -04001#!/usr/bin/env ksh
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -05002# this has been hacked to work with bash; ksh is preferred
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -04003
4#==================================================================================
E. Scott Daniels5efb1e62019-05-02 17:09:35 +00005# Copyright (c) 2019 Nokia
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -04006# Copyright (c) 2018-2019 AT&T Intellectual Property.
7#
8# Licensed under the Apache License, Version 2.0 (the "License");
9# you may not use this file except in compliance with the License.
10# You may obtain a copy of the License at
11#
12# http://www.apache.org/licenses/LICENSE-2.0
13#
14# Unless required by applicable law or agreed to in writing, software
15# distributed under the License is distributed on an "AS IS" BASIS,
16# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17# See the License for the specific language governing permissions and
18# limitations under the License.
19#==================================================================================
20
21
22#
23# Mnemonic: unit_test.ksh
E. Scott Daniels5efb1e62019-05-02 17:09:35 +000024# Abstract: Execute unit test(s) in the directory and produce a more
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040025# meaningful summary than gcov gives by default (exclude
26# coverage on the unit test functions).
27#
E. Scott Daniels5efb1e62019-05-02 17:09:35 +000028# Test files must be named *_test.c, or must explicitly be
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040029# supplied on the command line. Functions in the test
30# files will not be reported on provided that they have
31# their prototype (all on the SAME line) as:
32# static type name() {
33#
E. Scott Daniels5efb1e62019-05-02 17:09:35 +000034# Functions with coverage less than 80% will be reported as
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040035# [LOW] in the output. A file is considered to pass if the
36# overall execution percentage for the file is >= 80% regardless
37# of the number of functions that reported low.
38#
39# Test programmes are built prior to execution. Plan-9 mk is
40# the preferred builder, but as it's not widly adopted (sigh)
E. Scott Daniels5efb1e62019-05-02 17:09:35 +000041# make is assumed and -M will shift to Plan-9. Use -C xxx to
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040042# invoke a customised builder.
43#
44# For a module which does not pass, we will attempt to boost
45# the coverage by discounting the unexecuted lines which are
E. Scott Daniels5efb1e62019-05-02 17:09:35 +000046# inside of if() statements that are checking return from
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040047# (m)alloc() calls or are checking for nil pointers as these
48# cases are likely impossible to drive. When discount testing
49# is done both the failure message from the original analysis
E. Scott Daniels5efb1e62019-05-02 17:09:35 +000050# and a pass/fail message from the discount test are listed,
51# but only the result of the discount test is taken into
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040052# consideration with regard to overall success.
53#
E. Scott Daniels8dd46412019-04-16 20:47:54 +000054# Overall Pass/Fail
55# By default the overall state is based only on the success
E. Scott Daniels5efb1e62019-05-02 17:09:35 +000056# or failure of the unit tests and NOT on the perceived
E. Scott Daniels8dd46412019-04-16 20:47:54 +000057# state of coverage. If the -s (strict) option is given, then
58# overall state will be failure if code coverage expectations
59# are not met.
60#
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040061# Date: 16 January 2018
62# Author: E. Scott Daniels
63# -------------------------------------------------------------------------
64
65function usage {
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -050066 echo "usage: $0 [-G|-M|-C custom-command-string] [-a] [-c cov-target] [-f] [-F] [-v] [-x] [files]"
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040067 echo " if -C is used to provide a custom build command then it must "
68 echo " contain a %s which will be replaced with the unit test file name."
69 echo ' e.g.: -C "mk -a %s"'
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -050070 echo " -a always run coverage (even on failed modules)"
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040071 echo " -c allows user to set the target coverage for a module to pass; default is 80"
72 echo " -f forces a discount check (normally done only if coverage < target)"
E. Scott Daniels8dd46412019-04-16 20:47:54 +000073 echo " -F show only failures at the function level"
74 echo " -s strict mode; code coverage must also pass to result in a good exit code"
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040075 echo " -v will write additional information to the tty and save the disccounted file if discount run or -f given"
E. Scott Daniels5efb1e62019-05-02 17:09:35 +000076 echo " -x generates the coverage XML files for Sonar (implies -f)"
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040077}
78
E. Scott Daniels5efb1e62019-05-02 17:09:35 +000079# read through the given file and add any functions that are static to the
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040080# ignored list. Only test and test tools files should be parsed.
81#
82function add_ignored_func {
83 if [[ ! -r $1 ]]
84 then
E. Scott Daniels17398002020-04-01 12:39:47 -040085 echo ">>>> can't find file to ignore: $1"
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -040086 return
87 fi
88
89 typeset f=""
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -050090 goop=$(
91 grep "^static.*(.*).*{" $1 | awk ' # get list of test functions to ignore
92 {
93 gsub( "[(].*", "" )
94 gsub( "[*]", "" )
95 if( $2 == "struct" ) { # static struct goober function
96 printf( "%s ", $4 )
97 } else {
98 printf( "%s ", $3 ) # static goober-type funct
99 }
100 }
101 ' )
102
103 iflist="$iflist $goop" # this goop hack because bash can't read from a loop
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400104}
105
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000106
107# Merge two coverage files to preserve the total lines covered by different
108# test programmes.
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400109#
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000110function merge_cov {
111 if [[ -z $1 || -z $2 ]]
112 then
113 return
114 fi
115
116 if [[ ! -e $1 || ! -e $2 ]]
117 then
118 return
119 fi
120
121 (
122 cat $1
123 echo "==merge=="
124 cat $2
125 ) | awk '
126 /^==merge==/ {
127 merge = 1
128 next
129 }
130
131 merge && /#####:/ {
132 line = $2+0
133 if( executed[line] ) {
134 $1 = sprintf( "%9d:", executed[line] )
135 }
136 }
137
138 merge {
139 print
140 next
141 }
142
143 {
144 line = $2+0
145 if( $1+0 > 0 ) {
146 executed[line] = $1+0
147 }
148 }
149 '
150}
151
152#
153# Parse the .gcov file and discount any unexecuted lines which are in if()
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400154# blocks that are testing the result of alloc/malloc calls, or testing for
155# nil pointers. The feeling is that these might not be possible to drive
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000156# and shoudn't contribute to coverage deficiencies.
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400157#
158# In verbose mode, the .gcov file is written to stdout and any unexecuted
159# line which is discounted is marked with ===== replacing the ##### marking
160# that gcov wrote.
161#
162# The return value is 0 for pass; non-zero for fail.
163function discount_an_checks {
164 typeset f="$1"
165
166 mct=$( get_mct ${1%.gcov} ) # see if a special coverage target is defined for this
167
168 if [[ ! -f $1 ]]
169 then
170 if [[ -f ${1##*/} ]]
171 then
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000172 f=${1##*/}
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400173 else
174 echo "cant find: $f"
175 return
176 fi
177 fi
178
179 awk -v module_cov_target=$mct \
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000180 -v cfail=${cfail:-WARN} \
181 -v show_all=$show_all \
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400182 -v full_name="${1}" \
183 -v module="${f%.*}" \
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000184 -v chatty=$chatty \
185 -v replace_flags=$replace_flags \
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400186 '
187 function spit_line( ) {
188 if( chatty ) {
189 printf( "%s\n", $0 )
190 }
191 }
192
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400193 /-:/ { # skip unexecutable lines
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400194 spit_line()
195 seq++ # allow blank lines in a sequence group
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000196 next
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400197 }
198
199 {
200 nexec++ # number of executable lines
201 }
202
203 /#####:/ {
204 unexec++;
205 if( $2+0 != seq+1 ) {
206 prev_malloc = 0
207 prev_if = 0
208 seq = 0
209 spit_line()
210 next
211 }
212
213 if( prev_if && prev_malloc ) {
214 if( prev_malloc ) {
215 #printf( "allow discount: %s\n", $0 )
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000216 if( replace_flags ) {
217 gsub( "#####", " 1", $0 )
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400218 }
219 discount++;
220 }
221 }
222
223 seq++;;
224 spit_line()
225 next;
226 }
227
228 /if[(].*alloc.*{/ { # if( (x = malloc( ... )) != NULL ) or if( (p = sym_alloc(...)) != NULL )
229 seq = $2+0
230 prev_malloc = 1
231 prev_if = 1
232 spit_line()
233 next
234 }
235
236 /if[(].* == NULL/ { # a nil check likely not easily forced if it wasnt driven
237 prev_malloc = 1
238 prev_if = 1
239 spit_line()
240 seq = $2+0
241 next
242 }
243
244 /if[(]/ {
245 if( seq+1 == $2+0 && prev_malloc ) { // malloc on previous line
246 prev_if = 1
247 } else {
248 prev_malloc = 0
249 prev_if = 0
250 }
251 spit_line()
252 next
253 }
254
255 /alloc[(]/ {
256 seq = $2+0
257 prev_malloc = 1
258 spit_line()
259 next
260 }
261
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000262 {
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400263 spit_line()
264 }
265
266 END {
267 net = unexec - discount
268 orig_cov = ((nexec-unexec)/nexec)*100 # original coverage
269 adj_cov = ((nexec-net)/nexec)*100 # coverage after discount
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000270 pass_fail = adj_cov < module_cov_target ? cfail : "PASS"
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400271 rc = adj_cov < module_cov_target ? 1 : 0
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000272 if( pass_fail == cfail || show_all ) {
273 if( chatty ) {
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000274 printf( "[%s] %s executable=%d unexecuted=%d discounted=%d net_unex=%d cov=%d%% ==> %d%% target=%d%%\n",
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000275 pass_fail, full_name ? full_name : module, nexec, unexec, discount, net, orig_cov, adj_cov, module_cov_target )
276 } else {
277 printf( "[%s] %d%% (%d%%) %s\n", pass_fail, adj_cov, orig_cov, full_name ? full_name : module )
278 }
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400279 }
280
281 exit( rc )
282 }
283 ' $f
284}
285
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000286# Given a file name ($1) see if it is in the ./.targets file. If it is
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400287# return the coverage listed, else return (echo) the default $module_cov_target
288#
289function get_mct {
290 typeset v=$module_cov_target
291
292 if [[ -f ./.targets ]]
293 then
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500294 grep "^$1 " ./.targets | head -1 | read stuff
295 tv="${stuff##* }" # assume junk tv; ditch junk
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400296 fi
297
298 echo ${tv:-$v}
299}
300
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000301# Remove unneeded coverage files, then generate the xml files that can be given
302# to sonar. gcov.xml is based on the "raw" coverage and dcov.xml is based on
303# the discounted coverage.
304#
305function mk_xml {
306 rm -fr *_test.c.gcov test_*.c.gcov *_test.c.dcov test_*.c.dcov # we don't report on the unit test code, so ditch
307 cat *.gcov | cov2xml.ksh >gcov.xml
308 cat *.dcov | cov2xml.ksh >dcov.xml
309}
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400310
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000311
312# -----------------------------------------------------------------------------------------------------------------
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400313
E. Scott Daniels17398002020-04-01 12:39:47 -0400314if [[ -z $BUILD_PATH ]]
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000315then
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000316
E. Scott Daniels17398002020-04-01 12:39:47 -0400317 # we assume that the project has been built in the ../[.]build directory
318 if [[ -d ../build ]]
319 then
320 export BUILD_PATH=../build
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000321 else
E. Scott Daniels17398002020-04-01 12:39:47 -0400322 if [[ -d ../.build ]]
323 then
324 export BUILD_PATH=../.build
325 else
326 echo "[WARN] cannot find build directory (tried ../build and ../.build); things might not work"
327 echo ""
328 fi
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000329 fi
330fi
331
E. Scott Daniels17398002020-04-01 12:39:47 -0400332export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BUILD_PATH/lib:$BUILD_PATH/lib64
333export C_INCLUDE_PATH=$C_INCLUDE_PATH:$BUILD_PATH/include
E. Scott Daniels69e742d2019-06-11 13:35:39 -0400334export LIBRARY_PATH=$LD_LIBRARY_PATH
335
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500336# The Makefile sets specific includes for things
337#export C_INCLUDE_PATH="../src/rmr/common/include:../src/rmr/si/include:$C_INCLUDE_PATH"
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400338
339module_cov_target=80
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500340builder="make -B %s" # default to plain ole make
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400341verbose=0
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500342show_all=1 # show all things -F sets to show failures only
343strict=0 # -s (strict) will set; when off, coverage state ignored in final pass/fail
344show_output=0 # show output from each test execution (-S)
E. Scott Danielsa41c6f52019-04-23 18:24:25 +0000345quiet=0
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000346gen_xml=0
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500347replace_flags=1 # replace ##### in gcov for discounted lines
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400348run_nano_tests=0 # can nolonger be turned on
349run_nng_tests=0 # -N will enable
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500350always_gcov=0 # -a sets to always run gcov even if failure
351save_gcov=1 # -o turns this off
352out_dir=${UT_COVERAGE_DIR:-/tmp/rmr_gcov} # -O changes output directory
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400353
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500354export RMR_WARNING=1 # turn on warnings
E. Scott Daniels6511ac72019-08-27 10:17:21 -0400355
E. Scott Daniels0d4def62020-01-28 16:50:27 -0500356ulimit -c unlimited
357
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400358while [[ $1 == "-"* ]]
359do
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000360 case $1 in
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400361 -C) builder="$2"; shift;; # custom build command
362 -G) builder="gmake %s";;
363 -M) builder="mk -a %s";; # use plan-9 mk (better, but sadly not widly used)
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400364 -N) run_nng_tests=1;;
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500365 -O) out_dir=$2; shift;;
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400366
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500367 -a) always_gcov=1;;
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400368 -c) module_cov_target=$2; shift;;
E. Scott Danielsd9de79a2019-10-31 09:20:33 -0400369 -e) capture_file=$2; >$capture_file; shift;; # capture errors from failed tests rather than spewing on tty
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000370 -f) force_discounting=1;
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000371 trigger_discount_str="WARN|FAIL|PASS" # check all outcomes for each module
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400372 ;;
373
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000374 -F) show_all=0;;
375
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500376 -n) noexec=1;;
377 -o) save_gcov=0;;
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000378 -s) strict=1;; # coverage counts toward pass/fail state
E. Scott Danielsd7109572019-04-18 14:01:16 +0000379 -S) show_output=1;; # test output shown even on success
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400380 -v) (( verbose++ ));;
E. Scott Danielsa41c6f52019-04-23 18:24:25 +0000381 -q) quiet=1;; # less chatty when spilling error log files
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000382 -x) gen_xml=1
383 force_discounting=1
384 trigger_discount_str="WARN|FAIL|PASS" # check all outcomes for each module
E. Scott Daniels7958ddf2019-05-14 20:07:36 +0000385 rm -fr *cov.xml
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000386 ;;
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400387
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500388
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400389 -h) usage; exit 0;;
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400390 --help) usage; exit 0;;
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400391 -\?) usage; exit 0;;
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400392
393 *) echo "unrecognised option: $1" >&2
394 usage >&2
395 exit 1
396 ;;
397 esac
398
399 shift
400done
401
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000402
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400403if (( strict )) # if in strict mode, coverage shortcomings are failures
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000404then
405 cfail="FAIL"
406else
407 cfail="WARN"
408fi
409if [[ -z $trigger_discount_str ]]
410then
411 trigger_discount_str="$cfail"
412fi
413
414
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400415if [[ -z $1 ]]
416then
417 flist=""
418 for tfile in *_test.c
419 do
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000420 if [[ $tfile != *"static_test.c" ]]
421 then
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400422 if (( ! run_nng_tests )) && [[ $tfile == *"nng"* ]] # drop any nng file unless -N given
423 then
424 continue
425 fi
426 if [[ $tfile == *"nano"* ]] # no longer support nano tests; drop regardless
E. Scott Danielsb0ba22e2019-08-06 13:26:12 -0400427 then
428 continue
429 fi
E. Scott Daniels17398002020-04-01 12:39:47 -0400430
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400431 echo "<INFO> add test: $tfile" >&2
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000432 flist="${flist}$tfile "
433 fi
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400434 done
435else
436 flist="$@"
437fi
438
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000439
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500440if (( noexec ))
441then
442 echo "no exec mode; would test these:"
443 for tf in $flist
444 do
445 echo " $tf"
446 done
447 exit 0
448fi
449
E. Scott Daniels7958ddf2019-05-14 20:07:36 +0000450rm -fr *.gcov # ditch the previous coverage files
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000451ut_errors=0 # unit test errors (not coverage errors)
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400452errors=0
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000453
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400454if ! touch /tmp/PPID$$.noise
455then
456 echo "<ERR> unable to write to /tmp???"
457fi
458
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400459for tfile in $flist
460do
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000461 for x in *.gcov
462 do
463 if [[ -e $x ]]
464 then
465 cp $x $x-
466 fi
467 done
468
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400469 echo "$tfile --------------------------------------"
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500470 ( # all noise is now captured into a tmp file to support quiet mode
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500471 bcmd=$( printf "$builder" "${tfile%.c}" )
472 if ! $bcmd >/tmp/PID$$.log 2>&1
E. Scott Danielsb0ba22e2019-08-06 13:26:12 -0400473 then
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500474 echo "[FAIL] cannot build $tfile"
475 cat /tmp/PID$$.log
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400476 # do NOT remove tmp files; bash seens to not gen a new PID for subshells
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500477 exit 1
E. Scott Danielsb0ba22e2019-08-06 13:26:12 -0400478 fi
479
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500480 iflist="main sig_clean_exit " # ignore external functions from our tools
481 add_ignored_func $tfile # ignore all static functions in our test driver
482 add_ignored_func test_support.c # ignore all static functions in our test tools
483 add_ignored_func test_nng_em.c # the nng/nano emulated things
484 add_ignored_func test_si95_em.c # the si emulated things
485 add_ignored_func test_common_em.c # the common emulation functions
486 for f in *_static_test.c # all static modules here
487 do
488 if(( ! run_nano_tests )) && [[ $f == *"nano"* ]]
E. Scott Danielsd9de79a2019-10-31 09:20:33 -0400489 then
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500490 continue
491 fi
492
493 add_ignored_func $f
494 done
495
496 if ! ./${tfile%.c} >/tmp/PID$$.log 2>&1
497 then
498 echo "[FAIL] unit test failed for: $tfile"
E. Scott Daniels17398002020-04-01 12:39:47 -0400499 if [[ -n $capture_file ]]
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500500 then
501 echo "all errors captured in $capture_file, listing only fail message on tty"
502 echo "$tfile --------------------------------------" >>$capture_file
503 cat /tmp/PID$$.log >>$capture_file
504 grep "^<FAIL>" /tmp/PID$$.log
505 echo ""
E. Scott Danielsd9de79a2019-10-31 09:20:33 -0400506 else
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500507 if (( quiet ))
508 then
E. Scott Daniels17398002020-04-01 12:39:47 -0400509 grep "^<" /tmp/PID$$.log|egrep -v "^<SIEM>|^<EM>" # in quiet mode just dump <...> messages which are assumed from the test programme not appl
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500510 else
511 cat /tmp/PID$$.log
512 fi
513 fi
514 (( ut_errors++ )) # cause failure even if not in strict mode
515 if (( ! always_gcov ))
516 then
E. Scott Daniels17398002020-04-01 12:39:47 -0400517 exit 1 # we are in a subshell, must exit bad
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500518 fi
519 else
520 if (( show_output ))
521 then
522 printf "\n============= test programme output =======================\n"
E. Scott Danielsd9de79a2019-10-31 09:20:33 -0400523 cat /tmp/PID$$.log
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500524 printf "===========================================================\n"
E. Scott Danielsd9de79a2019-10-31 09:20:33 -0400525 fi
E. Scott Danielsa41c6f52019-04-23 18:24:25 +0000526 fi
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400527
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500528 (
529 touch ./.targets
530 sed '/^#/ d; /^$/ d; s/^/TARGET: /' ./.targets
531 gcov -f ${tfile%.c} | sed "s/'//g"
532 ) | awk \
533 -v cfail=$cfail \
534 -v show_all=$show_all \
535 -v ignore_list="$iflist" \
536 -v module_cov_target=$module_cov_target \
537 -v chatty=$verbose \
538 '
539 BEGIN {
540 announce_target = 1;
541 nignore = split( ignore_list, ignore, " " )
542 for( i = 1; i <= nignore; i++ ) {
543 imap[ignore[i]] = 1
544 }
545
546 exit_code = 0 # assume good
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400547 }
548
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500549 /^TARGET:/ {
550 if( NF > 1 ) {
551 target[$2] = $NF
552 }
553 next;
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400554 }
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400555
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500556 /File.*_test/ || /File.*test_/ { # dont report on test files
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400557 skip = 1
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500558 file = 1
559 fname = $2
560 next
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400561 }
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400562
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500563 /File/ {
564 skip = 0
565 file = 1
566 fname = $2
567 next
568 }
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400569
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500570 /Function/ {
571 fname = $2
572 file = 0
573 if( imap[fname] ) {
574 fname = "skipped: " fname # should never see and make it smell if we do
575 skip = 1
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400576 } else {
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500577 skip = 0
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400578 }
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500579 next
580 }
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000581
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500582 skip { next }
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000583
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500584 /Lines executed/ {
585 split( $0, a, ":" )
586 pct = a[2]+0
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000587
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500588 if( file ) {
589 if( announce_target ) { # announce default once at start
590 announce_target = 0;
591 printf( "\n[INFO] default target coverage for modules is %d%%\n", module_cov_target )
592 }
593
594 if( target[fname] ) {
595 mct = target[fname]
596 announce_target = 1;
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000597 } else {
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500598 mct = module_cov_target
599 }
600
601 if( announce_target ) { # annoucne for module if different from default
602 printf( "[INFO] target coverage for %s is %d%%\n", fname, mct )
603 }
604
605 if( pct < mct ) {
606 printf( "[%s] %3d%% %s\n", cfail, pct, fname ) # CAUTION: write only 3 things here
607 exit_code = 1
608 } else {
609 printf( "[PASS] %3d%% %s\n", pct, fname )
610 }
611
612 announce_target = 0;
613 } else {
614 if( pct < 70 ) {
615 printf( "[LOW] %3d%% %s\n", pct, fname )
616 } else {
617 if( pct < 80 ) {
618 printf( "[MARG] %3d%% %s\n", pct, fname )
619 } else {
620 if( show_all ) {
621 printf( "[OK] %3d%% %s\n", pct, fname )
622 }
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000623 }
624 }
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400625 }
626 }
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000627
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500628 END {
629 printf( "\n" );
630 exit( exit_code )
631 }
632 ' >/tmp/PID$$.log # capture output to run discount on failures
633 rc=$?
634 cat /tmp/PID$$.log
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400635
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400636 if (( rc || force_discounting )) # didn't pass, or forcing, see if discounting helps
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000637 then
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500638 if (( ! verbose ))
639 then
640 echo "[INFO] checking to see if discounting improves coverage for failures listed above"
641 fi
642
643 # preferred, but breaks under bash
644 #egrep "$trigger_discount_str" /tmp/PID$$.log | while read state junk name
645 egrep "$trigger_discount_str" /tmp/PID$$.log | while read stuff
646 do
647 set stuff # this hack required because bash cant read into mult vars
648 state="$1"
649 name="$3"
650
651 if ! discount_an_checks $name.gcov >/tmp/PID$$.disc
652 then
653 (( errors++ ))
654 fi
655
656 tail -1 /tmp/PID$$.disc | grep '\['
657
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400658 if (( verbose > 1 )) # updated file was generated, keep here
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500659 then
660 echo "[INFO] discounted coverage info in: ${tfile##*/}.dcov"
661 fi
662
663 mv /tmp/PID$$.disc ${name##*/}.dcov
664 done
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000665 fi
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400666 )>/tmp/PID$$.noise 2>&1
E. Scott Daniels17398002020-04-01 12:39:47 -0400667 if (( $? != 0 ))
668 then
669 (( ut_errors++ ))
670 cat /tmp/PID$$.noise
671 continue
672 fi
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000673
674 for x in *.gcov # merge any previous coverage file with this one
675 do
676 if [[ -e $x && -e $x- ]]
677 then
678 merge_cov $x $x- >/tmp/PID$$.mc
679 cp /tmp/PID$$.mc $x
680 rm $x-
681 fi
682 done
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500683
684 if (( ! quiet ))
685 then
686 cat /tmp/PID$$.noise
687 fi
E. Scott Daniels17398002020-04-01 12:39:47 -0400688done
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000689
690echo ""
691echo "[INFO] final discount checks on merged gcov files"
692show_all=1
693for xx in *.gcov
694do
695 if [[ $xx != *"test"* ]]
696 then
697 of=${xx%.gcov}.dcov
E. Scott Danielsfc2112d2020-04-21 12:51:05 -0400698 discount_an_checks $xx >$of
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500699 if [[ -n $of ]]
700 then
701 tail -1 $of | grep '\['
702 fi
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000703 fi
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400704done
705
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500706if (( save_gcov ))
707then
708 echo ""
709 ok=1
710 if [[ ! -d $outdir ]]
711 then
712 if ! mkdir -p $out_dir
713 then
714 echo "[WARN] unable to save .gcov files in $out_dir"
715 ok=0
716 fi
717 fi
718
719 if (( ok ))
720 then
721 rm -fr $out_dir/*
722 echo "[INFO] gcov files saved in $out_dir for push to remote system(s)"
723 cp *.gcov $out_dir/
724 rm -f $out_dir/*_test.c.gcov $out_dir/test_*.c.gcov
725 rm -f ./*_test.c.gcov ./test_*.c.gcov
726 fi
727else
728 echo "[INFO] .gcov files were not saved for remote system"
729fi
730
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000731state=0 # final state
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400732rm -f /tmp/PID$$.*
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000733if (( strict )) # fail if some coverage failed too
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400734then
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000735 if (( errors + ut_errors ))
736 then
737 state=1
738 fi
739else # not strict; fail only if unit tests themselves failed
740 if (( ut_errors ))
741 then
742 state=1
743 fi
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400744fi
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000745
746echo""
747if (( state ))
748then
749 echo "[FAIL] overall unit testing fails: coverage errors=$errors unit test errors=$ut_errors"
750else
751 echo "[PASS] overall unit testing passes"
E. Scott Daniels5efb1e62019-05-02 17:09:35 +0000752 if (( gen_xml ))
753 then
754 mk_xml
755 fi
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000756fi
E. Scott Danielsfc5c77b2020-02-21 13:24:29 -0500757
E. Scott Daniels8dd46412019-04-16 20:47:54 +0000758exit $state
Ashwin Sridharanfd9cc7a2019-04-03 16:47:02 -0400759