blob: 2e3f9781e5d8960633721926280ba67ce1f30ca1 [file] [log] [blame]
Damjan Marionf56b77a2016-10-03 19:44:57 +02001#!/usr/bin/env python
Damjan Marionf56b77a2016-10-03 19:44:57 +02002
Klement Sekeraacb9b8e2017-02-14 02:55:31 +01003from __future__ import print_function
4import gc
5import sys
6import os
7import select
Damjan Marionf56b77a2016-10-03 19:44:57 +02008import unittest
Klement Sekeraf62ae122016-10-11 11:47:09 +02009import tempfile
Klement Sekera277b89c2016-10-28 13:20:27 +020010import time
Klement Sekeraf62ae122016-10-11 11:47:09 +020011import resource
Klement Sekera3658adc2017-06-07 08:19:47 +020012import faulthandler
Klement Sekera6a6f4f72017-11-09 09:16:39 +010013import random
Klement Sekerae4504c62016-12-08 10:16:41 +010014from collections import deque
Klement Sekeraacb9b8e2017-02-14 02:55:31 +010015from threading import Thread, Event
Klement Sekera909a6a12017-08-08 04:33:53 +020016from inspect import getdoc, isclass
Klement Sekerab91017a2017-02-09 06:04:36 +010017from traceback import format_exception
Klement Sekeraacb9b8e2017-02-14 02:55:31 +010018from logging import FileHandler, DEBUG, Formatter
19from scapy.packet import Raw
Klement Sekera277b89c2016-10-28 13:20:27 +020020from hook import StepHook, PollHook
Klement Sekeraf62ae122016-10-11 11:47:09 +020021from vpp_pg_interface import VppPGInterface
Klement Sekeradab231a2016-12-21 08:50:14 +010022from vpp_sub_interface import VppSubInterface
Matej Klotton0178d522016-11-04 11:11:44 +010023from vpp_lo_interface import VppLoInterface
Klement Sekeraf62ae122016-10-11 11:47:09 +020024from vpp_papi_provider import VppPapiProvider
Klement Sekera277b89c2016-10-28 13:20:27 +020025from log import *
Klement Sekera10db26f2017-01-11 08:16:53 +010026from vpp_object import VppObjectRegistry
Klement Sekeraacb9b8e2017-02-14 02:55:31 +010027if os.name == 'posix' and sys.version_info[0] < 3:
28 # using subprocess32 is recommended by python official documentation
29 # @ https://docs.python.org/2/library/subprocess.html
30 import subprocess32 as subprocess
31else:
32 import subprocess
Klement Sekeraf62ae122016-10-11 11:47:09 +020033
34"""
35 Test framework module.
36
37 The module provides a set of tools for constructing and running tests and
38 representing the results.
39"""
40
Klement Sekeraf62ae122016-10-11 11:47:09 +020041
Damjan Marionf56b77a2016-10-03 19:44:57 +020042class _PacketInfo(object):
Klement Sekeraf62ae122016-10-11 11:47:09 +020043 """Private class to create packet info object.
44
45 Help process information about the next packet.
46 Set variables to default values.
Klement Sekeraf62ae122016-10-11 11:47:09 +020047 """
Matej Klotton86d87c42016-11-11 11:38:55 +010048 #: Store the index of the packet.
Damjan Marionf56b77a2016-10-03 19:44:57 +020049 index = -1
Matej Klotton86d87c42016-11-11 11:38:55 +010050 #: Store the index of the source packet generator interface of the packet.
Damjan Marionf56b77a2016-10-03 19:44:57 +020051 src = -1
Matej Klotton86d87c42016-11-11 11:38:55 +010052 #: Store the index of the destination packet generator interface
53 #: of the packet.
Damjan Marionf56b77a2016-10-03 19:44:57 +020054 dst = -1
Pavel Kotucek59dda062017-03-02 15:22:47 +010055 #: Store expected ip version
56 ip = -1
57 #: Store expected upper protocol
58 proto = -1
Matej Klotton86d87c42016-11-11 11:38:55 +010059 #: Store the copy of the former packet.
Damjan Marionf56b77a2016-10-03 19:44:57 +020060 data = None
Damjan Marionf56b77a2016-10-03 19:44:57 +020061
Matej Klotton16a14cd2016-12-07 15:09:13 +010062 def __eq__(self, other):
63 index = self.index == other.index
64 src = self.src == other.src
65 dst = self.dst == other.dst
66 data = self.data == other.data
67 return index and src and dst and data
68
Klement Sekeraf62ae122016-10-11 11:47:09 +020069
Klement Sekeraacb9b8e2017-02-14 02:55:31 +010070def pump_output(testclass):
71 """ pump output from vpp stdout/stderr to proper queues """
Klement Sekera6a6f4f72017-11-09 09:16:39 +010072 stdout_fragment = ""
73 stderr_fragment = ""
Klement Sekeraacb9b8e2017-02-14 02:55:31 +010074 while not testclass.pump_thread_stop_flag.wait(0):
75 readable = select.select([testclass.vpp.stdout.fileno(),
76 testclass.vpp.stderr.fileno(),
77 testclass.pump_thread_wakeup_pipe[0]],
78 [], [])[0]
79 if testclass.vpp.stdout.fileno() in readable:
Klement Sekera6a6f4f72017-11-09 09:16:39 +010080 read = os.read(testclass.vpp.stdout.fileno(), 102400)
81 if len(read) > 0:
82 split = read.splitlines(True)
83 if len(stdout_fragment) > 0:
84 split[0] = "%s%s" % (stdout_fragment, split[0])
85 if len(split) > 0 and split[-1].endswith("\n"):
86 limit = None
87 else:
88 limit = -1
89 stdout_fragment = split[-1]
90 testclass.vpp_stdout_deque.extend(split[:limit])
91 if not testclass.cache_vpp_output:
92 for line in split[:limit]:
93 testclass.logger.debug(
94 "VPP STDOUT: %s" % line.rstrip("\n"))
Klement Sekeraacb9b8e2017-02-14 02:55:31 +010095 if testclass.vpp.stderr.fileno() in readable:
Klement Sekera6a6f4f72017-11-09 09:16:39 +010096 read = os.read(testclass.vpp.stderr.fileno(), 102400)
97 if len(read) > 0:
98 split = read.splitlines(True)
99 if len(stderr_fragment) > 0:
100 split[0] = "%s%s" % (stderr_fragment, split[0])
101 if len(split) > 0 and split[-1].endswith("\n"):
102 limit = None
103 else:
104 limit = -1
105 stderr_fragment = split[-1]
106 testclass.vpp_stderr_deque.extend(split[:limit])
107 if not testclass.cache_vpp_output:
108 for line in split[:limit]:
109 testclass.logger.debug(
110 "VPP STDERR: %s" % line.rstrip("\n"))
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100111 # ignoring the dummy pipe here intentionally - the flag will take care
112 # of properly terminating the loop
Klement Sekera277b89c2016-10-28 13:20:27 +0200113
114
Klement Sekera87134932017-03-07 11:39:27 +0100115def running_extended_tests():
116 try:
117 s = os.getenv("EXTENDED_TESTS")
118 return True if s.lower() in ("y", "yes", "1") else False
119 except:
120 return False
121 return False
122
123
Klement Sekerad3e671e2017-09-29 12:36:37 +0200124def running_on_centos():
125 try:
126 os_id = os.getenv("OS_ID")
127 return True if "centos" in os_id.lower() else False
128 except:
129 return False
130 return False
131
132
Klement Sekera909a6a12017-08-08 04:33:53 +0200133class KeepAliveReporter(object):
134 """
135 Singleton object which reports test start to parent process
136 """
137 _shared_state = {}
138
139 def __init__(self):
140 self.__dict__ = self._shared_state
141
142 @property
143 def pipe(self):
144 return self._pipe
145
146 @pipe.setter
147 def pipe(self, pipe):
148 if hasattr(self, '_pipe'):
149 raise Exception("Internal error - pipe should only be set once.")
150 self._pipe = pipe
151
152 def send_keep_alive(self, test):
153 """
154 Write current test tmpdir & desc to keep-alive pipe to signal liveness
155 """
Klement Sekera3f6ff192017-08-11 06:56:05 +0200156 if self.pipe is None:
157 # if not running forked..
158 return
159
Klement Sekera909a6a12017-08-08 04:33:53 +0200160 if isclass(test):
161 desc = test.__name__
162 else:
163 desc = test.shortDescription()
164 if not desc:
165 desc = str(test)
166
Dave Wallacee2efd122017-09-30 22:04:21 -0400167 self.pipe.send((desc, test.vpp_bin, test.tempdir, test.vpp.pid))
Klement Sekera909a6a12017-08-08 04:33:53 +0200168
169
Damjan Marionf56b77a2016-10-03 19:44:57 +0200170class VppTestCase(unittest.TestCase):
Matej Klotton86d87c42016-11-11 11:38:55 +0100171 """This subclass is a base class for VPP test cases that are implemented as
172 classes. It provides methods to create and run test case.
Klement Sekeraf62ae122016-10-11 11:47:09 +0200173 """
174
175 @property
176 def packet_infos(self):
177 """List of packet infos"""
178 return self._packet_infos
179
Klement Sekeradab231a2016-12-21 08:50:14 +0100180 @classmethod
181 def get_packet_count_for_if_idx(cls, dst_if_index):
182 """Get the number of packet info for specified destination if index"""
183 if dst_if_index in cls._packet_count_for_dst_if_idx:
184 return cls._packet_count_for_dst_if_idx[dst_if_index]
185 else:
186 return 0
Klement Sekeraf62ae122016-10-11 11:47:09 +0200187
188 @classmethod
189 def instance(cls):
190 """Return the instance of this testcase"""
191 return cls.test_instance
192
Damjan Marionf56b77a2016-10-03 19:44:57 +0200193 @classmethod
Klement Sekera277b89c2016-10-28 13:20:27 +0200194 def set_debug_flags(cls, d):
195 cls.debug_core = False
196 cls.debug_gdb = False
197 cls.debug_gdbserver = False
198 if d is None:
199 return
200 dl = d.lower()
201 if dl == "core":
Klement Sekera277b89c2016-10-28 13:20:27 +0200202 cls.debug_core = True
203 elif dl == "gdb":
204 cls.debug_gdb = True
205 elif dl == "gdbserver":
206 cls.debug_gdbserver = True
207 else:
208 raise Exception("Unrecognized DEBUG option: '%s'" % d)
209
210 @classmethod
Damjan Marionf56b77a2016-10-03 19:44:57 +0200211 def setUpConstants(cls):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200212 """ Set-up the test case class based on environment variables """
213 try:
Klement Sekera277b89c2016-10-28 13:20:27 +0200214 s = os.getenv("STEP")
215 cls.step = True if s.lower() in ("y", "yes", "1") else False
Klement Sekeraf62ae122016-10-11 11:47:09 +0200216 except:
Klement Sekera277b89c2016-10-28 13:20:27 +0200217 cls.step = False
218 try:
219 d = os.getenv("DEBUG")
220 except:
221 d = None
Klement Sekeraa3d933c2017-11-06 09:46:00 +0100222 try:
223 c = os.getenv("CACHE_OUTPUT", "1")
224 cls.cache_vpp_output = \
Klement Sekerae1783992017-11-07 03:19:16 +0100225 False if c.lower() in ("n", "no", "0") else True
Klement Sekeraa3d933c2017-11-06 09:46:00 +0100226 except:
227 cls.cache_vpp_output = True
Klement Sekera277b89c2016-10-28 13:20:27 +0200228 cls.set_debug_flags(d)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200229 cls.vpp_bin = os.getenv('VPP_TEST_BIN', "vpp")
Pierre Pfistercd8e3182016-10-07 16:30:03 +0100230 cls.plugin_path = os.getenv('VPP_TEST_PLUGIN_PATH')
Klement Sekera47e275b2017-03-21 08:21:25 +0100231 cls.extern_plugin_path = os.getenv('EXTERN_PLUGINS')
232 plugin_path = None
233 if cls.plugin_path is not None:
234 if cls.extern_plugin_path is not None:
235 plugin_path = "%s:%s" % (
236 cls.plugin_path, cls.extern_plugin_path)
Klement Sekera6abbc282017-03-24 05:47:15 +0100237 else:
238 plugin_path = cls.plugin_path
Klement Sekera47e275b2017-03-21 08:21:25 +0100239 elif cls.extern_plugin_path is not None:
240 plugin_path = cls.extern_plugin_path
Klement Sekera01bbbe92016-11-02 09:25:05 +0100241 debug_cli = ""
242 if cls.step or cls.debug_gdb or cls.debug_gdbserver:
243 debug_cli = "cli-listen localhost:5002"
Klement Sekera80a7f0a2017-03-02 11:27:11 +0100244 coredump_size = None
245 try:
246 size = os.getenv("COREDUMP_SIZE")
247 if size is not None:
248 coredump_size = "coredump-size %s" % size
249 except:
250 pass
251 if coredump_size is None:
Dave Wallacee2efd122017-09-30 22:04:21 -0400252 coredump_size = "coredump-size unlimited"
Klement Sekera80a7f0a2017-03-02 11:27:11 +0100253 cls.vpp_cmdline = [cls.vpp_bin, "unix",
Dave Wallacee2efd122017-09-30 22:04:21 -0400254 "{", "nodaemon", debug_cli, "full-coredump",
255 coredump_size, "}", "api-trace", "{", "on", "}",
Damjan Marion374e2c52017-03-09 20:38:15 +0100256 "api-segment", "{", "prefix", cls.shm_prefix, "}",
257 "plugins", "{", "plugin", "dpdk_plugin.so", "{",
258 "disable", "}", "}"]
Klement Sekera47e275b2017-03-21 08:21:25 +0100259 if plugin_path is not None:
260 cls.vpp_cmdline.extend(["plugin_path", plugin_path])
Klement Sekera277b89c2016-10-28 13:20:27 +0200261 cls.logger.info("vpp_cmdline: %s" % cls.vpp_cmdline)
262
263 @classmethod
264 def wait_for_enter(cls):
265 if cls.debug_gdbserver:
266 print(double_line_delim)
267 print("Spawned GDB server with PID: %d" % cls.vpp.pid)
268 elif cls.debug_gdb:
269 print(double_line_delim)
270 print("Spawned VPP with PID: %d" % cls.vpp.pid)
271 else:
272 cls.logger.debug("Spawned VPP with PID: %d" % cls.vpp.pid)
273 return
274 print(single_line_delim)
275 print("You can debug the VPP using e.g.:")
276 if cls.debug_gdbserver:
277 print("gdb " + cls.vpp_bin + " -ex 'target remote localhost:7777'")
278 print("Now is the time to attach a gdb by running the above "
279 "command, set up breakpoints etc. and then resume VPP from "
280 "within gdb by issuing the 'continue' command")
281 elif cls.debug_gdb:
282 print("gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid)
283 print("Now is the time to attach a gdb by running the above "
284 "command and set up breakpoints etc.")
285 print(single_line_delim)
286 raw_input("Press ENTER to continue running the testcase...")
287
288 @classmethod
289 def run_vpp(cls):
290 cmdline = cls.vpp_cmdline
291
292 if cls.debug_gdbserver:
Klement Sekera931be3a2016-11-03 05:36:01 +0100293 gdbserver = '/usr/bin/gdbserver'
294 if not os.path.isfile(gdbserver) or \
295 not os.access(gdbserver, os.X_OK):
296 raise Exception("gdbserver binary '%s' does not exist or is "
297 "not executable" % gdbserver)
298
299 cmdline = [gdbserver, 'localhost:7777'] + cls.vpp_cmdline
Klement Sekera277b89c2016-10-28 13:20:27 +0200300 cls.logger.info("Gdbserver cmdline is %s", " ".join(cmdline))
301
Klement Sekera931be3a2016-11-03 05:36:01 +0100302 try:
303 cls.vpp = subprocess.Popen(cmdline,
304 stdout=subprocess.PIPE,
305 stderr=subprocess.PIPE,
306 bufsize=1)
307 except Exception as e:
308 cls.logger.critical("Couldn't start vpp: %s" % e)
309 raise
310
Klement Sekera277b89c2016-10-28 13:20:27 +0200311 cls.wait_for_enter()
Pierre Pfistercd8e3182016-10-07 16:30:03 +0100312
Damjan Marionf56b77a2016-10-03 19:44:57 +0200313 @classmethod
314 def setUpClass(cls):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200315 """
316 Perform class setup before running the testcase
317 Remove shared memory files, start vpp and connect the vpp-api
318 """
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100319 gc.collect() # run garbage collection first
Klement Sekera6a6f4f72017-11-09 09:16:39 +0100320 random.seed()
Klement Sekera277b89c2016-10-28 13:20:27 +0200321 cls.logger = getLogger(cls.__name__)
Klement Sekeraf62ae122016-10-11 11:47:09 +0200322 cls.tempdir = tempfile.mkdtemp(
Klement Sekeraf413bef2017-08-15 07:09:02 +0200323 prefix='vpp-unittest-%s-' % cls.__name__)
Klement Sekera027dbd52017-04-11 06:01:53 +0200324 cls.file_handler = FileHandler("%s/log.txt" % cls.tempdir)
325 cls.file_handler.setFormatter(
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100326 Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s',
327 datefmt="%H:%M:%S"))
Klement Sekera027dbd52017-04-11 06:01:53 +0200328 cls.file_handler.setLevel(DEBUG)
329 cls.logger.addHandler(cls.file_handler)
Klement Sekeraf62ae122016-10-11 11:47:09 +0200330 cls.shm_prefix = cls.tempdir.split("/")[-1]
331 os.chdir(cls.tempdir)
Klement Sekera277b89c2016-10-28 13:20:27 +0200332 cls.logger.info("Temporary dir is %s, shm prefix is %s",
333 cls.tempdir, cls.shm_prefix)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200334 cls.setUpConstants()
Klement Sekeradab231a2016-12-21 08:50:14 +0100335 cls.reset_packet_infos()
Klement Sekera9225dee2016-12-12 08:36:58 +0100336 cls._captures = []
337 cls._zombie_captures = []
Klement Sekeraf62ae122016-10-11 11:47:09 +0200338 cls.verbose = 0
Klement Sekera085f5c02016-11-24 01:59:16 +0100339 cls.vpp_dead = False
Klement Sekera10db26f2017-01-11 08:16:53 +0100340 cls.registry = VppObjectRegistry()
Klement Sekera3747c752017-04-10 06:30:17 +0200341 cls.vpp_startup_failed = False
Klement Sekera909a6a12017-08-08 04:33:53 +0200342 cls.reporter = KeepAliveReporter()
Klement Sekeraf62ae122016-10-11 11:47:09 +0200343 # need to catch exceptions here because if we raise, then the cleanup
344 # doesn't get called and we might end with a zombie vpp
345 try:
Klement Sekera277b89c2016-10-28 13:20:27 +0200346 cls.run_vpp()
Dave Wallacee2efd122017-09-30 22:04:21 -0400347 cls.reporter.send_keep_alive(cls)
Klement Sekerae4504c62016-12-08 10:16:41 +0100348 cls.vpp_stdout_deque = deque()
Klement Sekerae4504c62016-12-08 10:16:41 +0100349 cls.vpp_stderr_deque = deque()
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100350 cls.pump_thread_stop_flag = Event()
351 cls.pump_thread_wakeup_pipe = os.pipe()
352 cls.pump_thread = Thread(target=pump_output, args=(cls,))
Klement Sekeraaeeac3b2017-02-14 07:11:52 +0100353 cls.pump_thread.daemon = True
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100354 cls.pump_thread.start()
Klement Sekera7bb873a2016-11-18 07:38:42 +0100355 cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls)
Klement Sekera085f5c02016-11-24 01:59:16 +0100356 if cls.step:
357 hook = StepHook(cls)
358 else:
359 hook = PollHook(cls)
360 cls.vapi.register_hook(hook)
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100361 cls.sleep(0.1, "after vpp startup, before initial poll")
Klement Sekera3747c752017-04-10 06:30:17 +0200362 try:
363 hook.poll_vpp()
364 except:
365 cls.vpp_startup_failed = True
366 cls.logger.critical(
367 "VPP died shortly after startup, check the"
368 " output to standard error for possible cause")
369 raise
Klement Sekera085f5c02016-11-24 01:59:16 +0100370 try:
371 cls.vapi.connect()
372 except:
373 if cls.debug_gdbserver:
374 print(colorize("You're running VPP inside gdbserver but "
375 "VPP-API connection failed, did you forget "
376 "to 'continue' VPP from within gdb?", RED))
377 raise
Klement Sekeraf62ae122016-10-11 11:47:09 +0200378 except:
Klement Sekera0529a742016-12-02 07:05:24 +0100379 t, v, tb = sys.exc_info()
Klement Sekera085f5c02016-11-24 01:59:16 +0100380 try:
381 cls.quit()
382 except:
383 pass
Klement Sekera0529a742016-12-02 07:05:24 +0100384 raise t, v, tb
Damjan Marionf56b77a2016-10-03 19:44:57 +0200385
Damjan Marionf56b77a2016-10-03 19:44:57 +0200386 @classmethod
387 def quit(cls):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200388 """
389 Disconnect vpp-api, kill vpp and cleanup shared memory files
390 """
Klement Sekera277b89c2016-10-28 13:20:27 +0200391 if (cls.debug_gdbserver or cls.debug_gdb) and hasattr(cls, 'vpp'):
392 cls.vpp.poll()
393 if cls.vpp.returncode is None:
394 print(double_line_delim)
395 print("VPP or GDB server is still running")
396 print(single_line_delim)
Klement Sekerada505f62017-01-04 12:58:53 +0100397 raw_input("When done debugging, press ENTER to kill the "
398 "process and finish running the testcase...")
Klement Sekera277b89c2016-10-28 13:20:27 +0200399
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100400 os.write(cls.pump_thread_wakeup_pipe[1], 'ding dong wake up')
401 cls.pump_thread_stop_flag.set()
402 if hasattr(cls, 'pump_thread'):
403 cls.logger.debug("Waiting for pump thread to stop")
404 cls.pump_thread.join()
405 if hasattr(cls, 'vpp_stderr_reader_thread'):
406 cls.logger.debug("Waiting for stdderr pump to stop")
407 cls.vpp_stderr_reader_thread.join()
408
Klement Sekeraf62ae122016-10-11 11:47:09 +0200409 if hasattr(cls, 'vpp'):
Klement Sekera0529a742016-12-02 07:05:24 +0100410 if hasattr(cls, 'vapi'):
411 cls.vapi.disconnect()
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100412 del cls.vapi
Klement Sekeraf62ae122016-10-11 11:47:09 +0200413 cls.vpp.poll()
414 if cls.vpp.returncode is None:
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100415 cls.logger.debug("Sending TERM to vpp")
Klement Sekeraf62ae122016-10-11 11:47:09 +0200416 cls.vpp.terminate()
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100417 cls.logger.debug("Waiting for vpp to die")
418 cls.vpp.communicate()
Klement Sekeraf62ae122016-10-11 11:47:09 +0200419 del cls.vpp
Damjan Marionf56b77a2016-10-03 19:44:57 +0200420
Klement Sekera3747c752017-04-10 06:30:17 +0200421 if cls.vpp_startup_failed:
422 stdout_log = cls.logger.info
423 stderr_log = cls.logger.critical
424 else:
425 stdout_log = cls.logger.info
426 stderr_log = cls.logger.info
427
Klement Sekerae4504c62016-12-08 10:16:41 +0100428 if hasattr(cls, 'vpp_stdout_deque'):
Klement Sekera3747c752017-04-10 06:30:17 +0200429 stdout_log(single_line_delim)
430 stdout_log('VPP output to stdout while running %s:', cls.__name__)
431 stdout_log(single_line_delim)
Klement Sekerae4504c62016-12-08 10:16:41 +0100432 vpp_output = "".join(cls.vpp_stdout_deque)
Klement Sekera027dbd52017-04-11 06:01:53 +0200433 with open(cls.tempdir + '/vpp_stdout.txt', 'w') as f:
434 f.write(vpp_output)
Klement Sekera3747c752017-04-10 06:30:17 +0200435 stdout_log('\n%s', vpp_output)
436 stdout_log(single_line_delim)
Klement Sekera277b89c2016-10-28 13:20:27 +0200437
Klement Sekerae4504c62016-12-08 10:16:41 +0100438 if hasattr(cls, 'vpp_stderr_deque'):
Klement Sekera3747c752017-04-10 06:30:17 +0200439 stderr_log(single_line_delim)
440 stderr_log('VPP output to stderr while running %s:', cls.__name__)
441 stderr_log(single_line_delim)
Klement Sekerae4504c62016-12-08 10:16:41 +0100442 vpp_output = "".join(cls.vpp_stderr_deque)
Klement Sekera027dbd52017-04-11 06:01:53 +0200443 with open(cls.tempdir + '/vpp_stderr.txt', 'w') as f:
444 f.write(vpp_output)
Klement Sekera3747c752017-04-10 06:30:17 +0200445 stderr_log('\n%s', vpp_output)
446 stderr_log(single_line_delim)
Klement Sekera277b89c2016-10-28 13:20:27 +0200447
Damjan Marionf56b77a2016-10-03 19:44:57 +0200448 @classmethod
449 def tearDownClass(cls):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200450 """ Perform final cleanup after running all tests in this test-case """
Damjan Marionf56b77a2016-10-03 19:44:57 +0200451 cls.quit()
Klement Sekera027dbd52017-04-11 06:01:53 +0200452 cls.file_handler.close()
Damjan Marionf56b77a2016-10-03 19:44:57 +0200453
Damjan Marionf56b77a2016-10-03 19:44:57 +0200454 def tearDown(self):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200455 """ Show various debug prints after each test """
Klement Sekerab91017a2017-02-09 06:04:36 +0100456 self.logger.debug("--- tearDown() for %s.%s(%s) called ---" %
457 (self.__class__.__name__, self._testMethodName,
458 self._testMethodDoc))
Klement Sekeraf62ae122016-10-11 11:47:09 +0200459 if not self.vpp_dead:
Jan49c0fca2016-10-26 15:44:27 +0200460 self.logger.debug(self.vapi.cli("show trace"))
Neale Ranns88fc83e2017-04-05 08:11:14 -0700461 self.logger.info(self.vapi.ppcli("show interface"))
Jan49c0fca2016-10-26 15:44:27 +0200462 self.logger.info(self.vapi.ppcli("show hardware"))
463 self.logger.info(self.vapi.ppcli("show error"))
464 self.logger.info(self.vapi.ppcli("show run"))
Klement Sekera10db26f2017-01-11 08:16:53 +0100465 self.registry.remove_vpp_config(self.logger)
Dave Wallace90c55722017-02-16 11:25:26 -0500466 # Save/Dump VPP api trace log
467 api_trace = "vpp_api_trace.%s.log" % self._testMethodName
468 tmp_api_trace = "/tmp/%s" % api_trace
469 vpp_api_trace_log = "%s/%s" % (self.tempdir, api_trace)
470 self.logger.info(self.vapi.ppcli("api trace save %s" % api_trace))
471 self.logger.info("Moving %s to %s\n" % (tmp_api_trace,
472 vpp_api_trace_log))
473 os.rename(tmp_api_trace, vpp_api_trace_log)
474 self.logger.info(self.vapi.ppcli("api trace dump %s" %
475 vpp_api_trace_log))
Klement Sekera1b686402017-03-02 11:29:19 +0100476 else:
477 self.registry.unregister_all(self.logger)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200478
Damjan Marionf56b77a2016-10-03 19:44:57 +0200479 def setUp(self):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200480 """ Clear trace before running each test"""
Klement Sekera909a6a12017-08-08 04:33:53 +0200481 self.reporter.send_keep_alive(self)
Klement Sekerab91017a2017-02-09 06:04:36 +0100482 self.logger.debug("--- setUp() for %s.%s(%s) called ---" %
483 (self.__class__.__name__, self._testMethodName,
484 self._testMethodDoc))
Klement Sekera0c1519b2016-12-08 05:03:32 +0100485 if self.vpp_dead:
486 raise Exception("VPP is dead when setting up the test")
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100487 self.sleep(.1, "during setUp")
Klement Sekerae4504c62016-12-08 10:16:41 +0100488 self.vpp_stdout_deque.append(
489 "--- test setUp() for %s.%s(%s) starts here ---\n" %
490 (self.__class__.__name__, self._testMethodName,
491 self._testMethodDoc))
492 self.vpp_stderr_deque.append(
493 "--- test setUp() for %s.%s(%s) starts here ---\n" %
494 (self.__class__.__name__, self._testMethodName,
495 self._testMethodDoc))
Klement Sekeraf62ae122016-10-11 11:47:09 +0200496 self.vapi.cli("clear trace")
497 # store the test instance inside the test class - so that objects
498 # holding the class can access instance methods (like assertEqual)
499 type(self).test_instance = self
Damjan Marionf56b77a2016-10-03 19:44:57 +0200500
Damjan Marionf56b77a2016-10-03 19:44:57 +0200501 @classmethod
Klement Sekeraf62ae122016-10-11 11:47:09 +0200502 def pg_enable_capture(cls, interfaces):
503 """
504 Enable capture on packet-generator interfaces
Damjan Marionf56b77a2016-10-03 19:44:57 +0200505
Klement Sekeraf62ae122016-10-11 11:47:09 +0200506 :param interfaces: iterable interface indexes
Damjan Marionf56b77a2016-10-03 19:44:57 +0200507
Klement Sekeraf62ae122016-10-11 11:47:09 +0200508 """
509 for i in interfaces:
510 i.enable_capture()
Damjan Marionf56b77a2016-10-03 19:44:57 +0200511
Damjan Marionf56b77a2016-10-03 19:44:57 +0200512 @classmethod
Klement Sekera9225dee2016-12-12 08:36:58 +0100513 def register_capture(cls, cap_name):
514 """ Register a capture in the testclass """
515 # add to the list of captures with current timestamp
516 cls._captures.append((time.time(), cap_name))
517 # filter out from zombies
518 cls._zombie_captures = [(stamp, name)
519 for (stamp, name) in cls._zombie_captures
520 if name != cap_name]
521
522 @classmethod
523 def pg_start(cls):
524 """ Remove any zombie captures and enable the packet generator """
525 # how long before capture is allowed to be deleted - otherwise vpp
526 # crashes - 100ms seems enough (this shouldn't be needed at all)
527 capture_ttl = 0.1
528 now = time.time()
529 for stamp, cap_name in cls._zombie_captures:
530 wait = stamp + capture_ttl - now
531 if wait > 0:
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100532 cls.sleep(wait, "before deleting capture %s" % cap_name)
Klement Sekera9225dee2016-12-12 08:36:58 +0100533 now = time.time()
534 cls.logger.debug("Removing zombie capture %s" % cap_name)
535 cls.vapi.cli('packet-generator delete %s' % cap_name)
536
Klement Sekeraf62ae122016-10-11 11:47:09 +0200537 cls.vapi.cli("trace add pg-input 50") # 50 is maximum
538 cls.vapi.cli('packet-generator enable')
Klement Sekera9225dee2016-12-12 08:36:58 +0100539 cls._zombie_captures = cls._captures
540 cls._captures = []
Damjan Marionf56b77a2016-10-03 19:44:57 +0200541
Damjan Marionf56b77a2016-10-03 19:44:57 +0200542 @classmethod
Klement Sekeraf62ae122016-10-11 11:47:09 +0200543 def create_pg_interfaces(cls, interfaces):
544 """
Matej Klotton8d8a1da2016-12-22 11:06:56 +0100545 Create packet-generator interfaces.
Damjan Marionf56b77a2016-10-03 19:44:57 +0200546
Matej Klotton8d8a1da2016-12-22 11:06:56 +0100547 :param interfaces: iterable indexes of the interfaces.
548 :returns: List of created interfaces.
Damjan Marionf56b77a2016-10-03 19:44:57 +0200549
Klement Sekeraf62ae122016-10-11 11:47:09 +0200550 """
551 result = []
552 for i in interfaces:
553 intf = VppPGInterface(cls, i)
554 setattr(cls, intf.name, intf)
555 result.append(intf)
556 cls.pg_interfaces = result
557 return result
558
Matej Klotton0178d522016-11-04 11:11:44 +0100559 @classmethod
560 def create_loopback_interfaces(cls, interfaces):
561 """
Matej Klotton8d8a1da2016-12-22 11:06:56 +0100562 Create loopback interfaces.
Matej Klotton0178d522016-11-04 11:11:44 +0100563
Matej Klotton8d8a1da2016-12-22 11:06:56 +0100564 :param interfaces: iterable indexes of the interfaces.
565 :returns: List of created interfaces.
Matej Klotton0178d522016-11-04 11:11:44 +0100566 """
567 result = []
568 for i in interfaces:
569 intf = VppLoInterface(cls, i)
570 setattr(cls, intf.name, intf)
571 result.append(intf)
572 cls.lo_interfaces = result
573 return result
574
Damjan Marionf56b77a2016-10-03 19:44:57 +0200575 @staticmethod
576 def extend_packet(packet, size):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200577 """
578 Extend packet to given size by padding with spaces
579 NOTE: Currently works only when Raw layer is present.
580
581 :param packet: packet
582 :param size: target size
583
584 """
Damjan Marionf56b77a2016-10-03 19:44:57 +0200585 packet_len = len(packet) + 4
586 extend = size - packet_len
587 if extend > 0:
588 packet[Raw].load += ' ' * extend
Damjan Marionf56b77a2016-10-03 19:44:57 +0200589
Klement Sekeradab231a2016-12-21 08:50:14 +0100590 @classmethod
591 def reset_packet_infos(cls):
592 """ Reset the list of packet info objects and packet counts to zero """
593 cls._packet_infos = {}
594 cls._packet_count_for_dst_if_idx = {}
Klement Sekeraf62ae122016-10-11 11:47:09 +0200595
Klement Sekeradab231a2016-12-21 08:50:14 +0100596 @classmethod
597 def create_packet_info(cls, src_if, dst_if):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200598 """
599 Create packet info object containing the source and destination indexes
600 and add it to the testcase's packet info list
601
Klement Sekeradab231a2016-12-21 08:50:14 +0100602 :param VppInterface src_if: source interface
603 :param VppInterface dst_if: destination interface
Klement Sekeraf62ae122016-10-11 11:47:09 +0200604
605 :returns: _PacketInfo object
606
607 """
Damjan Marionf56b77a2016-10-03 19:44:57 +0200608 info = _PacketInfo()
Klement Sekeradab231a2016-12-21 08:50:14 +0100609 info.index = len(cls._packet_infos)
610 info.src = src_if.sw_if_index
611 info.dst = dst_if.sw_if_index
612 if isinstance(dst_if, VppSubInterface):
613 dst_idx = dst_if.parent.sw_if_index
614 else:
615 dst_idx = dst_if.sw_if_index
616 if dst_idx in cls._packet_count_for_dst_if_idx:
617 cls._packet_count_for_dst_if_idx[dst_idx] += 1
618 else:
619 cls._packet_count_for_dst_if_idx[dst_idx] = 1
620 cls._packet_infos[info.index] = info
Damjan Marionf56b77a2016-10-03 19:44:57 +0200621 return info
Damjan Marionf56b77a2016-10-03 19:44:57 +0200622
Damjan Marionf56b77a2016-10-03 19:44:57 +0200623 @staticmethod
624 def info_to_payload(info):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200625 """
626 Convert _PacketInfo object to packet payload
627
628 :param info: _PacketInfo object
629
630 :returns: string containing serialized data from packet info
631 """
Pavel Kotucek59dda062017-03-02 15:22:47 +0100632 return "%d %d %d %d %d" % (info.index, info.src, info.dst,
633 info.ip, info.proto)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200634
Damjan Marionf56b77a2016-10-03 19:44:57 +0200635 @staticmethod
636 def payload_to_info(payload):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200637 """
638 Convert packet payload to _PacketInfo object
639
640 :param payload: packet payload
641
642 :returns: _PacketInfo object containing de-serialized data from payload
643
644 """
Damjan Marionf56b77a2016-10-03 19:44:57 +0200645 numbers = payload.split()
646 info = _PacketInfo()
647 info.index = int(numbers[0])
648 info.src = int(numbers[1])
649 info.dst = int(numbers[2])
Pavel Kotucek59dda062017-03-02 15:22:47 +0100650 info.ip = int(numbers[3])
651 info.proto = int(numbers[4])
Damjan Marionf56b77a2016-10-03 19:44:57 +0200652 return info
Damjan Marionf56b77a2016-10-03 19:44:57 +0200653
Damjan Marionf56b77a2016-10-03 19:44:57 +0200654 def get_next_packet_info(self, info):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200655 """
656 Iterate over the packet info list stored in the testcase
657 Start iteration with first element if info is None
658 Continue based on index in info if info is specified
659
660 :param info: info or None
661 :returns: next info in list or None if no more infos
662 """
Damjan Marionf56b77a2016-10-03 19:44:57 +0200663 if info is None:
664 next_index = 0
665 else:
666 next_index = info.index + 1
Klement Sekeradab231a2016-12-21 08:50:14 +0100667 if next_index == len(self._packet_infos):
Damjan Marionf56b77a2016-10-03 19:44:57 +0200668 return None
669 else:
Klement Sekeradab231a2016-12-21 08:50:14 +0100670 return self._packet_infos[next_index]
Damjan Marionf56b77a2016-10-03 19:44:57 +0200671
Klement Sekeraf62ae122016-10-11 11:47:09 +0200672 def get_next_packet_info_for_interface(self, src_index, info):
673 """
674 Search the packet info list for the next packet info with same source
675 interface index
676
677 :param src_index: source interface index to search for
678 :param info: packet info - where to start the search
679 :returns: packet info or None
680
681 """
Damjan Marionf56b77a2016-10-03 19:44:57 +0200682 while True:
683 info = self.get_next_packet_info(info)
684 if info is None:
685 return None
Klement Sekeraf62ae122016-10-11 11:47:09 +0200686 if info.src == src_index:
Damjan Marionf56b77a2016-10-03 19:44:57 +0200687 return info
Damjan Marionf56b77a2016-10-03 19:44:57 +0200688
Klement Sekeraf62ae122016-10-11 11:47:09 +0200689 def get_next_packet_info_for_interface2(self, src_index, dst_index, info):
690 """
691 Search the packet info list for the next packet info with same source
692 and destination interface indexes
693
694 :param src_index: source interface index to search for
695 :param dst_index: destination interface index to search for
696 :param info: packet info - where to start the search
697 :returns: packet info or None
698
699 """
Damjan Marionf56b77a2016-10-03 19:44:57 +0200700 while True:
Klement Sekeraf62ae122016-10-11 11:47:09 +0200701 info = self.get_next_packet_info_for_interface(src_index, info)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200702 if info is None:
703 return None
Klement Sekeraf62ae122016-10-11 11:47:09 +0200704 if info.dst == dst_index:
Damjan Marionf56b77a2016-10-03 19:44:57 +0200705 return info
Damjan Marionf56b77a2016-10-03 19:44:57 +0200706
Klement Sekera0e3c0de2016-09-29 14:43:44 +0200707 def assert_equal(self, real_value, expected_value, name_or_class=None):
708 if name_or_class is None:
Klement Sekera239790f2017-02-16 10:53:53 +0100709 self.assertEqual(real_value, expected_value)
Klement Sekera0e3c0de2016-09-29 14:43:44 +0200710 return
711 try:
712 msg = "Invalid %s: %d('%s') does not match expected value %d('%s')"
713 msg = msg % (getdoc(name_or_class).strip(),
714 real_value, str(name_or_class(real_value)),
715 expected_value, str(name_or_class(expected_value)))
716 except:
717 msg = "Invalid %s: %s does not match expected value %s" % (
718 name_or_class, real_value, expected_value)
719
720 self.assertEqual(real_value, expected_value, msg)
721
Klement Sekerab17dd962017-01-09 07:43:48 +0100722 def assert_in_range(self,
723 real_value,
724 expected_min,
725 expected_max,
726 name=None):
Klement Sekera0e3c0de2016-09-29 14:43:44 +0200727 if name is None:
728 msg = None
729 else:
730 msg = "Invalid %s: %s out of range <%s,%s>" % (
731 name, real_value, expected_min, expected_max)
732 self.assertTrue(expected_min <= real_value <= expected_max, msg)
733
Klement Sekeraacb9b8e2017-02-14 02:55:31 +0100734 @classmethod
735 def sleep(cls, timeout, remark=None):
736 if hasattr(cls, 'logger'):
Klement Sekera3cfa5582017-04-19 07:10:58 +0000737 cls.logger.debug("Starting sleep for %ss (%s)" % (timeout, remark))
738 before = time.time()
Klement Sekeraa57a9702017-02-02 06:58:07 +0100739 time.sleep(timeout)
Klement Sekera3cfa5582017-04-19 07:10:58 +0000740 after = time.time()
741 if after - before > 2 * timeout:
Klement Sekera60c12232017-07-18 10:33:06 +0200742 cls.logger.error("unexpected time.sleep() result - "
743 "slept for %ss instead of ~%ss!" % (
744 after - before, timeout))
Klement Sekera3cfa5582017-04-19 07:10:58 +0000745 if hasattr(cls, 'logger'):
746 cls.logger.debug(
747 "Finished sleep (%s) - slept %ss (wanted %ss)" % (
748 remark, after - before, timeout))
Klement Sekeraa57a9702017-02-02 06:58:07 +0100749
Neale Ranns52fae862018-01-08 04:41:42 -0800750 def send_and_assert_no_replies(self, intf, pkts, remark=""):
751 self.vapi.cli("clear trace")
752 intf.add_stream(pkts)
753 self.pg_enable_capture(self.pg_interfaces)
754 self.pg_start()
755 timeout = 1
756 for i in self.pg_interfaces:
757 i.get_capture(0, timeout=timeout)
758 i.assert_nothing_captured(remark=remark)
759 timeout = 0.1
760
761 def send_and_expect(self, input, pkts, output):
762 self.vapi.cli("clear trace")
763 input.add_stream(pkts)
764 self.pg_enable_capture(self.pg_interfaces)
765 self.pg_start()
766 rx = output.get_capture(len(pkts))
767 return rx
768
Damjan Marionf56b77a2016-10-03 19:44:57 +0200769
Klement Sekera87134932017-03-07 11:39:27 +0100770class TestCasePrinter(object):
771 _shared_state = {}
772
773 def __init__(self):
774 self.__dict__ = self._shared_state
775 if not hasattr(self, "_test_case_set"):
776 self._test_case_set = set()
777
778 def print_test_case_heading_if_first_time(self, case):
779 if case.__class__ not in self._test_case_set:
780 print(double_line_delim)
781 print(colorize(getdoc(case.__class__).splitlines()[0], YELLOW))
782 print(double_line_delim)
783 self._test_case_set.add(case.__class__)
784
785
Damjan Marionf56b77a2016-10-03 19:44:57 +0200786class VppTestResult(unittest.TestResult):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200787 """
788 @property result_string
789 String variable to store the test case result string.
790 @property errors
791 List variable containing 2-tuples of TestCase instances and strings
792 holding formatted tracebacks. Each tuple represents a test which
793 raised an unexpected exception.
794 @property failures
795 List variable containing 2-tuples of TestCase instances and strings
796 holding formatted tracebacks. Each tuple represents a test where
797 a failure was explicitly signalled using the TestCase.assert*()
798 methods.
799 """
800
Damjan Marionf56b77a2016-10-03 19:44:57 +0200801 def __init__(self, stream, descriptions, verbosity):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200802 """
Klement Sekerada505f62017-01-04 12:58:53 +0100803 :param stream File descriptor to store where to report test results.
804 Set to the standard error stream by default.
805 :param descriptions Boolean variable to store information if to use
806 test case descriptions.
Klement Sekeraf62ae122016-10-11 11:47:09 +0200807 :param verbosity Integer variable to store required verbosity level.
808 """
Damjan Marionf56b77a2016-10-03 19:44:57 +0200809 unittest.TestResult.__init__(self, stream, descriptions, verbosity)
810 self.stream = stream
811 self.descriptions = descriptions
812 self.verbosity = verbosity
813 self.result_string = None
Klement Sekera87134932017-03-07 11:39:27 +0100814 self.printer = TestCasePrinter()
Damjan Marionf56b77a2016-10-03 19:44:57 +0200815
Damjan Marionf56b77a2016-10-03 19:44:57 +0200816 def addSuccess(self, test):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200817 """
818 Record a test succeeded result
819
820 :param test:
821
822 """
Klement Sekerab91017a2017-02-09 06:04:36 +0100823 if hasattr(test, 'logger'):
824 test.logger.debug("--- addSuccess() %s.%s(%s) called"
825 % (test.__class__.__name__,
826 test._testMethodName,
827 test._testMethodDoc))
Damjan Marionf56b77a2016-10-03 19:44:57 +0200828 unittest.TestResult.addSuccess(self, test)
Klement Sekera277b89c2016-10-28 13:20:27 +0200829 self.result_string = colorize("OK", GREEN)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200830
Klement Sekeraf62ae122016-10-11 11:47:09 +0200831 def addSkip(self, test, reason):
832 """
833 Record a test skipped.
834
835 :param test:
836 :param reason:
837
838 """
Klement Sekerab91017a2017-02-09 06:04:36 +0100839 if hasattr(test, 'logger'):
840 test.logger.debug("--- addSkip() %s.%s(%s) called, reason is %s"
841 % (test.__class__.__name__,
842 test._testMethodName,
843 test._testMethodDoc,
844 reason))
Klement Sekeraf62ae122016-10-11 11:47:09 +0200845 unittest.TestResult.addSkip(self, test, reason)
Klement Sekera277b89c2016-10-28 13:20:27 +0200846 self.result_string = colorize("SKIP", YELLOW)
Klement Sekeraf62ae122016-10-11 11:47:09 +0200847
Klement Sekeraf413bef2017-08-15 07:09:02 +0200848 def symlink_failed(self, test):
849 logger = None
850 if hasattr(test, 'logger'):
851 logger = test.logger
852 if hasattr(test, 'tempdir'):
853 try:
854 failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
855 link_path = '%s/%s-FAILED' % (failed_dir,
856 test.tempdir.split("/")[-1])
857 if logger:
858 logger.debug("creating a link to the failed test")
859 logger.debug("os.symlink(%s, %s)" %
860 (test.tempdir, link_path))
861 os.symlink(test.tempdir, link_path)
862 except Exception as e:
863 if logger:
864 logger.error(e)
865
Klement Sekeradf2b9802017-10-05 10:26:03 +0200866 def send_failure_through_pipe(self, test):
867 if hasattr(self, 'test_framework_failed_pipe'):
868 pipe = self.test_framework_failed_pipe
869 if pipe:
870 pipe.send(test.__class__)
871
Damjan Marionf56b77a2016-10-03 19:44:57 +0200872 def addFailure(self, test, err):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200873 """
874 Record a test failed result
875
876 :param test:
877 :param err: error message
878
879 """
Klement Sekerab91017a2017-02-09 06:04:36 +0100880 if hasattr(test, 'logger'):
881 test.logger.debug("--- addFailure() %s.%s(%s) called, err is %s"
882 % (test.__class__.__name__,
883 test._testMethodName,
884 test._testMethodDoc, err))
885 test.logger.debug("formatted exception is:\n%s" %
886 "".join(format_exception(*err)))
Damjan Marionf56b77a2016-10-03 19:44:57 +0200887 unittest.TestResult.addFailure(self, test, err)
Klement Sekeraf62ae122016-10-11 11:47:09 +0200888 if hasattr(test, 'tempdir'):
Klement Sekera277b89c2016-10-28 13:20:27 +0200889 self.result_string = colorize("FAIL", RED) + \
Klement Sekeraf62ae122016-10-11 11:47:09 +0200890 ' [ temp dir used by test case: ' + test.tempdir + ' ]'
Klement Sekeraf413bef2017-08-15 07:09:02 +0200891 self.symlink_failed(test)
Klement Sekeraf62ae122016-10-11 11:47:09 +0200892 else:
Klement Sekera277b89c2016-10-28 13:20:27 +0200893 self.result_string = colorize("FAIL", RED) + ' [no temp dir]'
Damjan Marionf56b77a2016-10-03 19:44:57 +0200894
Klement Sekeradf2b9802017-10-05 10:26:03 +0200895 self.send_failure_through_pipe(test)
896
Damjan Marionf56b77a2016-10-03 19:44:57 +0200897 def addError(self, test, err):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200898 """
899 Record a test error result
Damjan Marionf56b77a2016-10-03 19:44:57 +0200900
Klement Sekeraf62ae122016-10-11 11:47:09 +0200901 :param test:
902 :param err: error message
903
904 """
Klement Sekerab91017a2017-02-09 06:04:36 +0100905 if hasattr(test, 'logger'):
906 test.logger.debug("--- addError() %s.%s(%s) called, err is %s"
907 % (test.__class__.__name__,
908 test._testMethodName,
909 test._testMethodDoc, err))
910 test.logger.debug("formatted exception is:\n%s" %
911 "".join(format_exception(*err)))
Klement Sekeraf62ae122016-10-11 11:47:09 +0200912 unittest.TestResult.addError(self, test, err)
913 if hasattr(test, 'tempdir'):
Klement Sekera277b89c2016-10-28 13:20:27 +0200914 self.result_string = colorize("ERROR", RED) + \
Klement Sekeraf62ae122016-10-11 11:47:09 +0200915 ' [ temp dir used by test case: ' + test.tempdir + ' ]'
Klement Sekeraf413bef2017-08-15 07:09:02 +0200916 self.symlink_failed(test)
Klement Sekeraf62ae122016-10-11 11:47:09 +0200917 else:
Klement Sekera277b89c2016-10-28 13:20:27 +0200918 self.result_string = colorize("ERROR", RED) + ' [no temp dir]'
Klement Sekeraf62ae122016-10-11 11:47:09 +0200919
Klement Sekeradf2b9802017-10-05 10:26:03 +0200920 self.send_failure_through_pipe(test)
921
Damjan Marionf56b77a2016-10-03 19:44:57 +0200922 def getDescription(self, test):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200923 """
924 Get test description
925
926 :param test:
927 :returns: test description
928
929 """
Damjan Marionf56b77a2016-10-03 19:44:57 +0200930 # TODO: if none print warning not raise exception
931 short_description = test.shortDescription()
932 if self.descriptions and short_description:
933 return short_description
934 else:
935 return str(test)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200936
Damjan Marionf56b77a2016-10-03 19:44:57 +0200937 def startTest(self, test):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200938 """
939 Start a test
940
941 :param test:
942
943 """
Klement Sekera87134932017-03-07 11:39:27 +0100944 self.printer.print_test_case_heading_if_first_time(test)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200945 unittest.TestResult.startTest(self, test)
946 if self.verbosity > 0:
Klement Sekeraf62ae122016-10-11 11:47:09 +0200947 self.stream.writeln(
948 "Starting " + self.getDescription(test) + " ...")
949 self.stream.writeln(single_line_delim)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200950
Damjan Marionf56b77a2016-10-03 19:44:57 +0200951 def stopTest(self, test):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200952 """
953 Stop a test
954
955 :param test:
956
957 """
Damjan Marionf56b77a2016-10-03 19:44:57 +0200958 unittest.TestResult.stopTest(self, test)
959 if self.verbosity > 0:
Klement Sekeraf62ae122016-10-11 11:47:09 +0200960 self.stream.writeln(single_line_delim)
Klement Sekera52e84f32017-01-13 07:25:25 +0100961 self.stream.writeln("%-73s%s" % (self.getDescription(test),
Klement Sekerada505f62017-01-04 12:58:53 +0100962 self.result_string))
Klement Sekeraf62ae122016-10-11 11:47:09 +0200963 self.stream.writeln(single_line_delim)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200964 else:
Klement Sekera52e84f32017-01-13 07:25:25 +0100965 self.stream.writeln("%-73s%s" % (self.getDescription(test),
Klement Sekerada505f62017-01-04 12:58:53 +0100966 self.result_string))
Damjan Marionf56b77a2016-10-03 19:44:57 +0200967
Damjan Marionf56b77a2016-10-03 19:44:57 +0200968 def printErrors(self):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200969 """
970 Print errors from running the test case
971 """
Damjan Marionf56b77a2016-10-03 19:44:57 +0200972 self.stream.writeln()
973 self.printErrorList('ERROR', self.errors)
974 self.printErrorList('FAIL', self.failures)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200975
Damjan Marionf56b77a2016-10-03 19:44:57 +0200976 def printErrorList(self, flavour, errors):
Klement Sekeraf62ae122016-10-11 11:47:09 +0200977 """
978 Print error list to the output stream together with error type
979 and test case description.
980
981 :param flavour: error type
982 :param errors: iterable errors
983
984 """
Damjan Marionf56b77a2016-10-03 19:44:57 +0200985 for test, err in errors:
Klement Sekeraf62ae122016-10-11 11:47:09 +0200986 self.stream.writeln(double_line_delim)
987 self.stream.writeln("%s: %s" %
988 (flavour, self.getDescription(test)))
989 self.stream.writeln(single_line_delim)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200990 self.stream.writeln("%s" % err)
Damjan Marionf56b77a2016-10-03 19:44:57 +0200991
992
Klement Sekeradf2b9802017-10-05 10:26:03 +0200993class Filter_by_test_option:
994 def __init__(self, filter_file_name, filter_class_name, filter_func_name):
995 self.filter_file_name = filter_file_name
996 self.filter_class_name = filter_class_name
997 self.filter_func_name = filter_func_name
998
999 def __call__(self, file_name, class_name, func_name):
1000 if self.filter_file_name and file_name != self.filter_file_name:
1001 return False
1002 if self.filter_class_name and class_name != self.filter_class_name:
1003 return False
1004 if self.filter_func_name and func_name != self.filter_func_name:
1005 return False
1006 return True
1007
1008
Damjan Marionf56b77a2016-10-03 19:44:57 +02001009class VppTestRunner(unittest.TextTestRunner):
Klement Sekeraf62ae122016-10-11 11:47:09 +02001010 """
Klement Sekera104543f2017-02-03 07:29:43 +01001011 A basic test runner implementation which prints results to standard error.
Klement Sekeraf62ae122016-10-11 11:47:09 +02001012 """
1013 @property
1014 def resultclass(self):
1015 """Class maintaining the results of the tests"""
1016 return VppTestResult
Damjan Marionf56b77a2016-10-03 19:44:57 +02001017
Klement Sekeradf2b9802017-10-05 10:26:03 +02001018 def __init__(self, keep_alive_pipe=None, failed_pipe=None,
1019 stream=sys.stderr, descriptions=True,
Klement Sekera3f6ff192017-08-11 06:56:05 +02001020 verbosity=1, failfast=False, buffer=False, resultclass=None):
Klement Sekera7a161da2017-01-17 13:42:48 +01001021 # ignore stream setting here, use hard-coded stdout to be in sync
1022 # with prints from VppTestCase methods ...
1023 super(VppTestRunner, self).__init__(sys.stdout, descriptions,
1024 verbosity, failfast, buffer,
1025 resultclass)
Klement Sekera909a6a12017-08-08 04:33:53 +02001026 reporter = KeepAliveReporter()
Klement Sekeradf2b9802017-10-05 10:26:03 +02001027 reporter.pipe = keep_alive_pipe
1028 # this is super-ugly, but very simple to implement and works as long
1029 # as we run only one test at the same time
1030 VppTestResult.test_framework_failed_pipe = failed_pipe
Klement Sekera7a161da2017-01-17 13:42:48 +01001031
Klement Sekera104543f2017-02-03 07:29:43 +01001032 test_option = "TEST"
1033
1034 def parse_test_option(self):
1035 try:
1036 f = os.getenv(self.test_option)
1037 except:
1038 f = None
1039 filter_file_name = None
1040 filter_class_name = None
1041 filter_func_name = None
1042 if f:
1043 if '.' in f:
1044 parts = f.split('.')
1045 if len(parts) > 3:
1046 raise Exception("Unrecognized %s option: %s" %
1047 (self.test_option, f))
1048 if len(parts) > 2:
1049 if parts[2] not in ('*', ''):
1050 filter_func_name = parts[2]
1051 if parts[1] not in ('*', ''):
1052 filter_class_name = parts[1]
1053 if parts[0] not in ('*', ''):
1054 if parts[0].startswith('test_'):
1055 filter_file_name = parts[0]
1056 else:
1057 filter_file_name = 'test_%s' % parts[0]
1058 else:
1059 if f.startswith('test_'):
1060 filter_file_name = f
1061 else:
1062 filter_file_name = 'test_%s' % f
1063 return filter_file_name, filter_class_name, filter_func_name
1064
Klement Sekeradf2b9802017-10-05 10:26:03 +02001065 @staticmethod
1066 def filter_tests(tests, filter_cb):
Klement Sekera104543f2017-02-03 07:29:43 +01001067 result = unittest.suite.TestSuite()
1068 for t in tests:
1069 if isinstance(t, unittest.suite.TestSuite):
1070 # this is a bunch of tests, recursively filter...
Klement Sekeradf2b9802017-10-05 10:26:03 +02001071 x = filter_tests(t, filter_cb)
Klement Sekera104543f2017-02-03 07:29:43 +01001072 if x.countTestCases() > 0:
1073 result.addTest(x)
1074 elif isinstance(t, unittest.TestCase):
1075 # this is a single test
1076 parts = t.id().split('.')
1077 # t.id() for common cases like this:
1078 # test_classifier.TestClassifier.test_acl_ip
1079 # apply filtering only if it is so
1080 if len(parts) == 3:
Klement Sekeradf2b9802017-10-05 10:26:03 +02001081 if not filter_cb(parts[0], parts[1], parts[2]):
Klement Sekera104543f2017-02-03 07:29:43 +01001082 continue
1083 result.addTest(t)
1084 else:
1085 # unexpected object, don't touch it
1086 result.addTest(t)
1087 return result
1088
Damjan Marionf56b77a2016-10-03 19:44:57 +02001089 def run(self, test):
Klement Sekeraf62ae122016-10-11 11:47:09 +02001090 """
1091 Run the tests
1092
1093 :param test:
1094
1095 """
Klement Sekera3658adc2017-06-07 08:19:47 +02001096 faulthandler.enable() # emit stack trace to stderr if killed by signal
Klement Sekeraf62ae122016-10-11 11:47:09 +02001097 print("Running tests using custom test runner") # debug message
Klement Sekera104543f2017-02-03 07:29:43 +01001098 filter_file, filter_class, filter_func = self.parse_test_option()
1099 print("Active filters: file=%s, class=%s, function=%s" % (
1100 filter_file, filter_class, filter_func))
Klement Sekeradf2b9802017-10-05 10:26:03 +02001101 filter_cb = Filter_by_test_option(
1102 filter_file, filter_class, filter_func)
1103 filtered = self.filter_tests(test, filter_cb)
Klement Sekera104543f2017-02-03 07:29:43 +01001104 print("%s out of %s tests match specified filters" % (
1105 filtered.countTestCases(), test.countTestCases()))
Klement Sekera3747c752017-04-10 06:30:17 +02001106 if not running_extended_tests():
1107 print("Not running extended tests (some tests will be skipped)")
Klement Sekera104543f2017-02-03 07:29:43 +01001108 return super(VppTestRunner, self).run(filtered)
Neale Ranns812ed392017-10-16 04:20:13 -07001109
1110
1111class Worker(Thread):
1112 def __init__(self, args, logger):
1113 self.logger = logger
1114 self.args = args
1115 self.result = None
1116 super(Worker, self).__init__()
1117
1118 def run(self):
1119 executable = self.args[0]
1120 self.logger.debug("Running executable w/args `%s'" % self.args)
1121 env = os.environ.copy()
1122 env["CK_LOG_FILE_NAME"] = "-"
1123 self.process = subprocess.Popen(
1124 self.args, shell=False, env=env, preexec_fn=os.setpgrp,
1125 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1126 out, err = self.process.communicate()
1127 self.logger.debug("Finished running `%s'" % executable)
1128 self.logger.info("Return code is `%s'" % self.process.returncode)
1129 self.logger.info(single_line_delim)
1130 self.logger.info("Executable `%s' wrote to stdout:" % executable)
1131 self.logger.info(single_line_delim)
1132 self.logger.info(out)
1133 self.logger.info(single_line_delim)
1134 self.logger.info("Executable `%s' wrote to stderr:" % executable)
1135 self.logger.info(single_line_delim)
1136 self.logger.error(err)
1137 self.logger.info(single_line_delim)
1138 self.result = self.process.returncode