Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 1 | #!/usr/bin/env python |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 2 | |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 3 | from __future__ import print_function |
| 4 | import gc |
| 5 | import sys |
| 6 | import os |
| 7 | import select |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 8 | import unittest |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 9 | import tempfile |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 10 | import time |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 11 | import resource |
Klement Sekera | e4504c6 | 2016-12-08 10:16:41 +0100 | [diff] [blame] | 12 | from collections import deque |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 13 | from threading import Thread, Event |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 14 | from inspect import getdoc |
Klement Sekera | b91017a | 2017-02-09 06:04:36 +0100 | [diff] [blame] | 15 | from traceback import format_exception |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 16 | from logging import FileHandler, DEBUG, Formatter |
| 17 | from scapy.packet import Raw |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 18 | from hook import StepHook, PollHook |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 19 | from vpp_pg_interface import VppPGInterface |
Klement Sekera | dab231a | 2016-12-21 08:50:14 +0100 | [diff] [blame] | 20 | from vpp_sub_interface import VppSubInterface |
Matej Klotton | 0178d52 | 2016-11-04 11:11:44 +0100 | [diff] [blame] | 21 | from vpp_lo_interface import VppLoInterface |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 22 | from vpp_papi_provider import VppPapiProvider |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 23 | from log import * |
Klement Sekera | 10db26f | 2017-01-11 08:16:53 +0100 | [diff] [blame] | 24 | from vpp_object import VppObjectRegistry |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 25 | if os.name == 'posix' and sys.version_info[0] < 3: |
| 26 | # using subprocess32 is recommended by python official documentation |
| 27 | # @ https://docs.python.org/2/library/subprocess.html |
| 28 | import subprocess32 as subprocess |
| 29 | else: |
| 30 | import subprocess |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 31 | |
| 32 | """ |
| 33 | Test framework module. |
| 34 | |
| 35 | The module provides a set of tools for constructing and running tests and |
| 36 | representing the results. |
| 37 | """ |
| 38 | |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 39 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 40 | class _PacketInfo(object): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 41 | """Private class to create packet info object. |
| 42 | |
| 43 | Help process information about the next packet. |
| 44 | Set variables to default values. |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 45 | """ |
Matej Klotton | 86d87c4 | 2016-11-11 11:38:55 +0100 | [diff] [blame] | 46 | #: Store the index of the packet. |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 47 | index = -1 |
Matej Klotton | 86d87c4 | 2016-11-11 11:38:55 +0100 | [diff] [blame] | 48 | #: Store the index of the source packet generator interface of the packet. |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 49 | src = -1 |
Matej Klotton | 86d87c4 | 2016-11-11 11:38:55 +0100 | [diff] [blame] | 50 | #: Store the index of the destination packet generator interface |
| 51 | #: of the packet. |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 52 | dst = -1 |
Matej Klotton | 86d87c4 | 2016-11-11 11:38:55 +0100 | [diff] [blame] | 53 | #: Store the copy of the former packet. |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 54 | data = None |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 55 | |
Matej Klotton | 16a14cd | 2016-12-07 15:09:13 +0100 | [diff] [blame] | 56 | def __eq__(self, other): |
| 57 | index = self.index == other.index |
| 58 | src = self.src == other.src |
| 59 | dst = self.dst == other.dst |
| 60 | data = self.data == other.data |
| 61 | return index and src and dst and data |
| 62 | |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 63 | |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 64 | def pump_output(testclass): |
| 65 | """ pump output from vpp stdout/stderr to proper queues """ |
| 66 | while not testclass.pump_thread_stop_flag.wait(0): |
| 67 | readable = select.select([testclass.vpp.stdout.fileno(), |
| 68 | testclass.vpp.stderr.fileno(), |
| 69 | testclass.pump_thread_wakeup_pipe[0]], |
| 70 | [], [])[0] |
| 71 | if testclass.vpp.stdout.fileno() in readable: |
| 72 | read = os.read(testclass.vpp.stdout.fileno(), 1024) |
| 73 | testclass.vpp_stdout_deque.append(read) |
| 74 | if testclass.vpp.stderr.fileno() in readable: |
| 75 | read = os.read(testclass.vpp.stderr.fileno(), 1024) |
| 76 | testclass.vpp_stderr_deque.append(read) |
| 77 | # ignoring the dummy pipe here intentionally - the flag will take care |
| 78 | # of properly terminating the loop |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 79 | |
| 80 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 81 | class VppTestCase(unittest.TestCase): |
Matej Klotton | 86d87c4 | 2016-11-11 11:38:55 +0100 | [diff] [blame] | 82 | """This subclass is a base class for VPP test cases that are implemented as |
| 83 | classes. It provides methods to create and run test case. |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 84 | """ |
| 85 | |
| 86 | @property |
| 87 | def packet_infos(self): |
| 88 | """List of packet infos""" |
| 89 | return self._packet_infos |
| 90 | |
Klement Sekera | dab231a | 2016-12-21 08:50:14 +0100 | [diff] [blame] | 91 | @classmethod |
| 92 | def get_packet_count_for_if_idx(cls, dst_if_index): |
| 93 | """Get the number of packet info for specified destination if index""" |
| 94 | if dst_if_index in cls._packet_count_for_dst_if_idx: |
| 95 | return cls._packet_count_for_dst_if_idx[dst_if_index] |
| 96 | else: |
| 97 | return 0 |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 98 | |
| 99 | @classmethod |
| 100 | def instance(cls): |
| 101 | """Return the instance of this testcase""" |
| 102 | return cls.test_instance |
| 103 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 104 | @classmethod |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 105 | def set_debug_flags(cls, d): |
| 106 | cls.debug_core = False |
| 107 | cls.debug_gdb = False |
| 108 | cls.debug_gdbserver = False |
| 109 | if d is None: |
| 110 | return |
| 111 | dl = d.lower() |
| 112 | if dl == "core": |
| 113 | if resource.getrlimit(resource.RLIMIT_CORE)[0] <= 0: |
| 114 | # give a heads up if this is actually useless |
Klement Sekera | b17dd96 | 2017-01-09 07:43:48 +0100 | [diff] [blame] | 115 | print(colorize("WARNING: core size limit is set 0, core files " |
| 116 | "will NOT be created", RED)) |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 117 | cls.debug_core = True |
| 118 | elif dl == "gdb": |
| 119 | cls.debug_gdb = True |
| 120 | elif dl == "gdbserver": |
| 121 | cls.debug_gdbserver = True |
| 122 | else: |
| 123 | raise Exception("Unrecognized DEBUG option: '%s'" % d) |
| 124 | |
| 125 | @classmethod |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 126 | def setUpConstants(cls): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 127 | """ Set-up the test case class based on environment variables """ |
| 128 | try: |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 129 | s = os.getenv("STEP") |
| 130 | cls.step = True if s.lower() in ("y", "yes", "1") else False |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 131 | except: |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 132 | cls.step = False |
| 133 | try: |
| 134 | d = os.getenv("DEBUG") |
| 135 | except: |
| 136 | d = None |
| 137 | cls.set_debug_flags(d) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 138 | cls.vpp_bin = os.getenv('VPP_TEST_BIN', "vpp") |
Pierre Pfister | cd8e318 | 2016-10-07 16:30:03 +0100 | [diff] [blame] | 139 | cls.plugin_path = os.getenv('VPP_TEST_PLUGIN_PATH') |
Klement Sekera | 01bbbe9 | 2016-11-02 09:25:05 +0100 | [diff] [blame] | 140 | debug_cli = "" |
| 141 | if cls.step or cls.debug_gdb or cls.debug_gdbserver: |
| 142 | debug_cli = "cli-listen localhost:5002" |
Klement Sekera | da505f6 | 2017-01-04 12:58:53 +0100 | [diff] [blame] | 143 | cls.vpp_cmdline = [cls.vpp_bin, |
| 144 | "unix", "{", "nodaemon", debug_cli, "}", |
Dave Wallace | 90c5572 | 2017-02-16 11:25:26 -0500 | [diff] [blame] | 145 | "api-trace", "{", "on", "}", |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 146 | "api-segment", "{", "prefix", cls.shm_prefix, "}"] |
Pierre Pfister | cd8e318 | 2016-10-07 16:30:03 +0100 | [diff] [blame] | 147 | if cls.plugin_path is not None: |
| 148 | cls.vpp_cmdline.extend(["plugin_path", cls.plugin_path]) |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 149 | cls.logger.info("vpp_cmdline: %s" % cls.vpp_cmdline) |
| 150 | |
| 151 | @classmethod |
| 152 | def wait_for_enter(cls): |
| 153 | if cls.debug_gdbserver: |
| 154 | print(double_line_delim) |
| 155 | print("Spawned GDB server with PID: %d" % cls.vpp.pid) |
| 156 | elif cls.debug_gdb: |
| 157 | print(double_line_delim) |
| 158 | print("Spawned VPP with PID: %d" % cls.vpp.pid) |
| 159 | else: |
| 160 | cls.logger.debug("Spawned VPP with PID: %d" % cls.vpp.pid) |
| 161 | return |
| 162 | print(single_line_delim) |
| 163 | print("You can debug the VPP using e.g.:") |
| 164 | if cls.debug_gdbserver: |
| 165 | print("gdb " + cls.vpp_bin + " -ex 'target remote localhost:7777'") |
| 166 | print("Now is the time to attach a gdb by running the above " |
| 167 | "command, set up breakpoints etc. and then resume VPP from " |
| 168 | "within gdb by issuing the 'continue' command") |
| 169 | elif cls.debug_gdb: |
| 170 | print("gdb " + cls.vpp_bin + " -ex 'attach %s'" % cls.vpp.pid) |
| 171 | print("Now is the time to attach a gdb by running the above " |
| 172 | "command and set up breakpoints etc.") |
| 173 | print(single_line_delim) |
| 174 | raw_input("Press ENTER to continue running the testcase...") |
| 175 | |
| 176 | @classmethod |
| 177 | def run_vpp(cls): |
| 178 | cmdline = cls.vpp_cmdline |
| 179 | |
| 180 | if cls.debug_gdbserver: |
Klement Sekera | 931be3a | 2016-11-03 05:36:01 +0100 | [diff] [blame] | 181 | gdbserver = '/usr/bin/gdbserver' |
| 182 | if not os.path.isfile(gdbserver) or \ |
| 183 | not os.access(gdbserver, os.X_OK): |
| 184 | raise Exception("gdbserver binary '%s' does not exist or is " |
| 185 | "not executable" % gdbserver) |
| 186 | |
| 187 | cmdline = [gdbserver, 'localhost:7777'] + cls.vpp_cmdline |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 188 | cls.logger.info("Gdbserver cmdline is %s", " ".join(cmdline)) |
| 189 | |
Klement Sekera | 931be3a | 2016-11-03 05:36:01 +0100 | [diff] [blame] | 190 | try: |
| 191 | cls.vpp = subprocess.Popen(cmdline, |
| 192 | stdout=subprocess.PIPE, |
| 193 | stderr=subprocess.PIPE, |
| 194 | bufsize=1) |
| 195 | except Exception as e: |
| 196 | cls.logger.critical("Couldn't start vpp: %s" % e) |
| 197 | raise |
| 198 | |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 199 | cls.wait_for_enter() |
Pierre Pfister | cd8e318 | 2016-10-07 16:30:03 +0100 | [diff] [blame] | 200 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 201 | @classmethod |
| 202 | def setUpClass(cls): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 203 | """ |
| 204 | Perform class setup before running the testcase |
| 205 | Remove shared memory files, start vpp and connect the vpp-api |
| 206 | """ |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 207 | gc.collect() # run garbage collection first |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 208 | cls.logger = getLogger(cls.__name__) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 209 | cls.tempdir = tempfile.mkdtemp( |
| 210 | prefix='vpp-unittest-' + cls.__name__ + '-') |
Klement Sekera | 6c7440c | 2016-12-23 09:16:39 +0100 | [diff] [blame] | 211 | file_handler = FileHandler("%s/log.txt" % cls.tempdir) |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 212 | file_handler.setFormatter( |
| 213 | Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s', |
| 214 | datefmt="%H:%M:%S")) |
Klement Sekera | 6c7440c | 2016-12-23 09:16:39 +0100 | [diff] [blame] | 215 | file_handler.setLevel(DEBUG) |
| 216 | cls.logger.addHandler(file_handler) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 217 | cls.shm_prefix = cls.tempdir.split("/")[-1] |
| 218 | os.chdir(cls.tempdir) |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 219 | cls.logger.info("Temporary dir is %s, shm prefix is %s", |
| 220 | cls.tempdir, cls.shm_prefix) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 221 | cls.setUpConstants() |
Klement Sekera | dab231a | 2016-12-21 08:50:14 +0100 | [diff] [blame] | 222 | cls.reset_packet_infos() |
Klement Sekera | 9225dee | 2016-12-12 08:36:58 +0100 | [diff] [blame] | 223 | cls._captures = [] |
| 224 | cls._zombie_captures = [] |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 225 | cls.verbose = 0 |
Klement Sekera | 085f5c0 | 2016-11-24 01:59:16 +0100 | [diff] [blame] | 226 | cls.vpp_dead = False |
Klement Sekera | 10db26f | 2017-01-11 08:16:53 +0100 | [diff] [blame] | 227 | cls.registry = VppObjectRegistry() |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 228 | print(double_line_delim) |
Matej Klotton | 86d87c4 | 2016-11-11 11:38:55 +0100 | [diff] [blame] | 229 | print(colorize(getdoc(cls).splitlines()[0], YELLOW)) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 230 | print(double_line_delim) |
| 231 | # need to catch exceptions here because if we raise, then the cleanup |
| 232 | # doesn't get called and we might end with a zombie vpp |
| 233 | try: |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 234 | cls.run_vpp() |
Klement Sekera | e4504c6 | 2016-12-08 10:16:41 +0100 | [diff] [blame] | 235 | cls.vpp_stdout_deque = deque() |
Klement Sekera | e4504c6 | 2016-12-08 10:16:41 +0100 | [diff] [blame] | 236 | cls.vpp_stderr_deque = deque() |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 237 | cls.pump_thread_stop_flag = Event() |
| 238 | cls.pump_thread_wakeup_pipe = os.pipe() |
| 239 | cls.pump_thread = Thread(target=pump_output, args=(cls,)) |
Klement Sekera | aeeac3b | 2017-02-14 07:11:52 +0100 | [diff] [blame] | 240 | cls.pump_thread.daemon = True |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 241 | cls.pump_thread.start() |
Klement Sekera | 7bb873a | 2016-11-18 07:38:42 +0100 | [diff] [blame] | 242 | cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls) |
Klement Sekera | 085f5c0 | 2016-11-24 01:59:16 +0100 | [diff] [blame] | 243 | if cls.step: |
| 244 | hook = StepHook(cls) |
| 245 | else: |
| 246 | hook = PollHook(cls) |
| 247 | cls.vapi.register_hook(hook) |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 248 | cls.sleep(0.1, "after vpp startup, before initial poll") |
Klement Sekera | 085f5c0 | 2016-11-24 01:59:16 +0100 | [diff] [blame] | 249 | hook.poll_vpp() |
| 250 | try: |
| 251 | cls.vapi.connect() |
| 252 | except: |
| 253 | if cls.debug_gdbserver: |
| 254 | print(colorize("You're running VPP inside gdbserver but " |
| 255 | "VPP-API connection failed, did you forget " |
| 256 | "to 'continue' VPP from within gdb?", RED)) |
| 257 | raise |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 258 | except: |
Klement Sekera | 0529a74 | 2016-12-02 07:05:24 +0100 | [diff] [blame] | 259 | t, v, tb = sys.exc_info() |
Klement Sekera | 085f5c0 | 2016-11-24 01:59:16 +0100 | [diff] [blame] | 260 | try: |
| 261 | cls.quit() |
| 262 | except: |
| 263 | pass |
Klement Sekera | 0529a74 | 2016-12-02 07:05:24 +0100 | [diff] [blame] | 264 | raise t, v, tb |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 265 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 266 | @classmethod |
| 267 | def quit(cls): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 268 | """ |
| 269 | Disconnect vpp-api, kill vpp and cleanup shared memory files |
| 270 | """ |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 271 | if (cls.debug_gdbserver or cls.debug_gdb) and hasattr(cls, 'vpp'): |
| 272 | cls.vpp.poll() |
| 273 | if cls.vpp.returncode is None: |
| 274 | print(double_line_delim) |
| 275 | print("VPP or GDB server is still running") |
| 276 | print(single_line_delim) |
Klement Sekera | da505f6 | 2017-01-04 12:58:53 +0100 | [diff] [blame] | 277 | raw_input("When done debugging, press ENTER to kill the " |
| 278 | "process and finish running the testcase...") |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 279 | |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 280 | os.write(cls.pump_thread_wakeup_pipe[1], 'ding dong wake up') |
| 281 | cls.pump_thread_stop_flag.set() |
| 282 | if hasattr(cls, 'pump_thread'): |
| 283 | cls.logger.debug("Waiting for pump thread to stop") |
| 284 | cls.pump_thread.join() |
| 285 | if hasattr(cls, 'vpp_stderr_reader_thread'): |
| 286 | cls.logger.debug("Waiting for stdderr pump to stop") |
| 287 | cls.vpp_stderr_reader_thread.join() |
| 288 | |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 289 | if hasattr(cls, 'vpp'): |
Klement Sekera | 0529a74 | 2016-12-02 07:05:24 +0100 | [diff] [blame] | 290 | if hasattr(cls, 'vapi'): |
| 291 | cls.vapi.disconnect() |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 292 | del cls.vapi |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 293 | cls.vpp.poll() |
| 294 | if cls.vpp.returncode is None: |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 295 | cls.logger.debug("Sending TERM to vpp") |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 296 | cls.vpp.terminate() |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 297 | cls.logger.debug("Waiting for vpp to die") |
| 298 | cls.vpp.communicate() |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 299 | del cls.vpp |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 300 | |
Klement Sekera | e4504c6 | 2016-12-08 10:16:41 +0100 | [diff] [blame] | 301 | if hasattr(cls, 'vpp_stdout_deque'): |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 302 | cls.logger.info(single_line_delim) |
| 303 | cls.logger.info('VPP output to stdout while running %s:', |
| 304 | cls.__name__) |
| 305 | cls.logger.info(single_line_delim) |
| 306 | f = open(cls.tempdir + '/vpp_stdout.txt', 'w') |
Klement Sekera | e4504c6 | 2016-12-08 10:16:41 +0100 | [diff] [blame] | 307 | vpp_output = "".join(cls.vpp_stdout_deque) |
| 308 | f.write(vpp_output) |
| 309 | cls.logger.info('\n%s', vpp_output) |
| 310 | cls.logger.info(single_line_delim) |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 311 | |
Klement Sekera | e4504c6 | 2016-12-08 10:16:41 +0100 | [diff] [blame] | 312 | if hasattr(cls, 'vpp_stderr_deque'): |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 313 | cls.logger.info(single_line_delim) |
| 314 | cls.logger.info('VPP output to stderr while running %s:', |
| 315 | cls.__name__) |
| 316 | cls.logger.info(single_line_delim) |
| 317 | f = open(cls.tempdir + '/vpp_stderr.txt', 'w') |
Klement Sekera | e4504c6 | 2016-12-08 10:16:41 +0100 | [diff] [blame] | 318 | vpp_output = "".join(cls.vpp_stderr_deque) |
| 319 | f.write(vpp_output) |
| 320 | cls.logger.info('\n%s', vpp_output) |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 321 | cls.logger.info(single_line_delim) |
| 322 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 323 | @classmethod |
| 324 | def tearDownClass(cls): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 325 | """ Perform final cleanup after running all tests in this test-case """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 326 | cls.quit() |
| 327 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 328 | def tearDown(self): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 329 | """ Show various debug prints after each test """ |
Klement Sekera | b91017a | 2017-02-09 06:04:36 +0100 | [diff] [blame] | 330 | self.logger.debug("--- tearDown() for %s.%s(%s) called ---" % |
| 331 | (self.__class__.__name__, self._testMethodName, |
| 332 | self._testMethodDoc)) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 333 | if not self.vpp_dead: |
Jan | 49c0fca | 2016-10-26 15:44:27 +0200 | [diff] [blame] | 334 | self.logger.debug(self.vapi.cli("show trace")) |
Neale Ranns | 177bbdc | 2016-11-15 09:46:51 +0000 | [diff] [blame] | 335 | self.logger.info(self.vapi.ppcli("show int")) |
Jan | 49c0fca | 2016-10-26 15:44:27 +0200 | [diff] [blame] | 336 | self.logger.info(self.vapi.ppcli("show hardware")) |
| 337 | self.logger.info(self.vapi.ppcli("show error")) |
| 338 | self.logger.info(self.vapi.ppcli("show run")) |
Klement Sekera | 10db26f | 2017-01-11 08:16:53 +0100 | [diff] [blame] | 339 | self.registry.remove_vpp_config(self.logger) |
Dave Wallace | 90c5572 | 2017-02-16 11:25:26 -0500 | [diff] [blame] | 340 | # Save/Dump VPP api trace log |
| 341 | api_trace = "vpp_api_trace.%s.log" % self._testMethodName |
| 342 | tmp_api_trace = "/tmp/%s" % api_trace |
| 343 | vpp_api_trace_log = "%s/%s" % (self.tempdir, api_trace) |
| 344 | self.logger.info(self.vapi.ppcli("api trace save %s" % api_trace)) |
| 345 | self.logger.info("Moving %s to %s\n" % (tmp_api_trace, |
| 346 | vpp_api_trace_log)) |
| 347 | os.rename(tmp_api_trace, vpp_api_trace_log) |
| 348 | self.logger.info(self.vapi.ppcli("api trace dump %s" % |
| 349 | vpp_api_trace_log)) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 350 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 351 | def setUp(self): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 352 | """ Clear trace before running each test""" |
Klement Sekera | b91017a | 2017-02-09 06:04:36 +0100 | [diff] [blame] | 353 | self.logger.debug("--- setUp() for %s.%s(%s) called ---" % |
| 354 | (self.__class__.__name__, self._testMethodName, |
| 355 | self._testMethodDoc)) |
Klement Sekera | 0c1519b | 2016-12-08 05:03:32 +0100 | [diff] [blame] | 356 | if self.vpp_dead: |
| 357 | raise Exception("VPP is dead when setting up the test") |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 358 | self.sleep(.1, "during setUp") |
Klement Sekera | e4504c6 | 2016-12-08 10:16:41 +0100 | [diff] [blame] | 359 | self.vpp_stdout_deque.append( |
| 360 | "--- test setUp() for %s.%s(%s) starts here ---\n" % |
| 361 | (self.__class__.__name__, self._testMethodName, |
| 362 | self._testMethodDoc)) |
| 363 | self.vpp_stderr_deque.append( |
| 364 | "--- test setUp() for %s.%s(%s) starts here ---\n" % |
| 365 | (self.__class__.__name__, self._testMethodName, |
| 366 | self._testMethodDoc)) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 367 | self.vapi.cli("clear trace") |
| 368 | # store the test instance inside the test class - so that objects |
| 369 | # holding the class can access instance methods (like assertEqual) |
| 370 | type(self).test_instance = self |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 371 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 372 | @classmethod |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 373 | def pg_enable_capture(cls, interfaces): |
| 374 | """ |
| 375 | Enable capture on packet-generator interfaces |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 376 | |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 377 | :param interfaces: iterable interface indexes |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 378 | |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 379 | """ |
| 380 | for i in interfaces: |
| 381 | i.enable_capture() |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 382 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 383 | @classmethod |
Klement Sekera | 9225dee | 2016-12-12 08:36:58 +0100 | [diff] [blame] | 384 | def register_capture(cls, cap_name): |
| 385 | """ Register a capture in the testclass """ |
| 386 | # add to the list of captures with current timestamp |
| 387 | cls._captures.append((time.time(), cap_name)) |
| 388 | # filter out from zombies |
| 389 | cls._zombie_captures = [(stamp, name) |
| 390 | for (stamp, name) in cls._zombie_captures |
| 391 | if name != cap_name] |
| 392 | |
| 393 | @classmethod |
| 394 | def pg_start(cls): |
| 395 | """ Remove any zombie captures and enable the packet generator """ |
| 396 | # how long before capture is allowed to be deleted - otherwise vpp |
| 397 | # crashes - 100ms seems enough (this shouldn't be needed at all) |
| 398 | capture_ttl = 0.1 |
| 399 | now = time.time() |
| 400 | for stamp, cap_name in cls._zombie_captures: |
| 401 | wait = stamp + capture_ttl - now |
| 402 | if wait > 0: |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 403 | cls.sleep(wait, "before deleting capture %s" % cap_name) |
Klement Sekera | 9225dee | 2016-12-12 08:36:58 +0100 | [diff] [blame] | 404 | now = time.time() |
| 405 | cls.logger.debug("Removing zombie capture %s" % cap_name) |
| 406 | cls.vapi.cli('packet-generator delete %s' % cap_name) |
| 407 | |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 408 | cls.vapi.cli("trace add pg-input 50") # 50 is maximum |
| 409 | cls.vapi.cli('packet-generator enable') |
Klement Sekera | 9225dee | 2016-12-12 08:36:58 +0100 | [diff] [blame] | 410 | cls._zombie_captures = cls._captures |
| 411 | cls._captures = [] |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 412 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 413 | @classmethod |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 414 | def create_pg_interfaces(cls, interfaces): |
| 415 | """ |
Matej Klotton | 8d8a1da | 2016-12-22 11:06:56 +0100 | [diff] [blame] | 416 | Create packet-generator interfaces. |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 417 | |
Matej Klotton | 8d8a1da | 2016-12-22 11:06:56 +0100 | [diff] [blame] | 418 | :param interfaces: iterable indexes of the interfaces. |
| 419 | :returns: List of created interfaces. |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 420 | |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 421 | """ |
| 422 | result = [] |
| 423 | for i in interfaces: |
| 424 | intf = VppPGInterface(cls, i) |
| 425 | setattr(cls, intf.name, intf) |
| 426 | result.append(intf) |
| 427 | cls.pg_interfaces = result |
| 428 | return result |
| 429 | |
Matej Klotton | 0178d52 | 2016-11-04 11:11:44 +0100 | [diff] [blame] | 430 | @classmethod |
| 431 | def create_loopback_interfaces(cls, interfaces): |
| 432 | """ |
Matej Klotton | 8d8a1da | 2016-12-22 11:06:56 +0100 | [diff] [blame] | 433 | Create loopback interfaces. |
Matej Klotton | 0178d52 | 2016-11-04 11:11:44 +0100 | [diff] [blame] | 434 | |
Matej Klotton | 8d8a1da | 2016-12-22 11:06:56 +0100 | [diff] [blame] | 435 | :param interfaces: iterable indexes of the interfaces. |
| 436 | :returns: List of created interfaces. |
Matej Klotton | 0178d52 | 2016-11-04 11:11:44 +0100 | [diff] [blame] | 437 | """ |
| 438 | result = [] |
| 439 | for i in interfaces: |
| 440 | intf = VppLoInterface(cls, i) |
| 441 | setattr(cls, intf.name, intf) |
| 442 | result.append(intf) |
| 443 | cls.lo_interfaces = result |
| 444 | return result |
| 445 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 446 | @staticmethod |
| 447 | def extend_packet(packet, size): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 448 | """ |
| 449 | Extend packet to given size by padding with spaces |
| 450 | NOTE: Currently works only when Raw layer is present. |
| 451 | |
| 452 | :param packet: packet |
| 453 | :param size: target size |
| 454 | |
| 455 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 456 | packet_len = len(packet) + 4 |
| 457 | extend = size - packet_len |
| 458 | if extend > 0: |
| 459 | packet[Raw].load += ' ' * extend |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 460 | |
Klement Sekera | dab231a | 2016-12-21 08:50:14 +0100 | [diff] [blame] | 461 | @classmethod |
| 462 | def reset_packet_infos(cls): |
| 463 | """ Reset the list of packet info objects and packet counts to zero """ |
| 464 | cls._packet_infos = {} |
| 465 | cls._packet_count_for_dst_if_idx = {} |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 466 | |
Klement Sekera | dab231a | 2016-12-21 08:50:14 +0100 | [diff] [blame] | 467 | @classmethod |
| 468 | def create_packet_info(cls, src_if, dst_if): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 469 | """ |
| 470 | Create packet info object containing the source and destination indexes |
| 471 | and add it to the testcase's packet info list |
| 472 | |
Klement Sekera | dab231a | 2016-12-21 08:50:14 +0100 | [diff] [blame] | 473 | :param VppInterface src_if: source interface |
| 474 | :param VppInterface dst_if: destination interface |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 475 | |
| 476 | :returns: _PacketInfo object |
| 477 | |
| 478 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 479 | info = _PacketInfo() |
Klement Sekera | dab231a | 2016-12-21 08:50:14 +0100 | [diff] [blame] | 480 | info.index = len(cls._packet_infos) |
| 481 | info.src = src_if.sw_if_index |
| 482 | info.dst = dst_if.sw_if_index |
| 483 | if isinstance(dst_if, VppSubInterface): |
| 484 | dst_idx = dst_if.parent.sw_if_index |
| 485 | else: |
| 486 | dst_idx = dst_if.sw_if_index |
| 487 | if dst_idx in cls._packet_count_for_dst_if_idx: |
| 488 | cls._packet_count_for_dst_if_idx[dst_idx] += 1 |
| 489 | else: |
| 490 | cls._packet_count_for_dst_if_idx[dst_idx] = 1 |
| 491 | cls._packet_infos[info.index] = info |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 492 | return info |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 493 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 494 | @staticmethod |
| 495 | def info_to_payload(info): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 496 | """ |
| 497 | Convert _PacketInfo object to packet payload |
| 498 | |
| 499 | :param info: _PacketInfo object |
| 500 | |
| 501 | :returns: string containing serialized data from packet info |
| 502 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 503 | return "%d %d %d" % (info.index, info.src, info.dst) |
| 504 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 505 | @staticmethod |
| 506 | def payload_to_info(payload): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 507 | """ |
| 508 | Convert packet payload to _PacketInfo object |
| 509 | |
| 510 | :param payload: packet payload |
| 511 | |
| 512 | :returns: _PacketInfo object containing de-serialized data from payload |
| 513 | |
| 514 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 515 | numbers = payload.split() |
| 516 | info = _PacketInfo() |
| 517 | info.index = int(numbers[0]) |
| 518 | info.src = int(numbers[1]) |
| 519 | info.dst = int(numbers[2]) |
| 520 | return info |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 521 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 522 | def get_next_packet_info(self, info): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 523 | """ |
| 524 | Iterate over the packet info list stored in the testcase |
| 525 | Start iteration with first element if info is None |
| 526 | Continue based on index in info if info is specified |
| 527 | |
| 528 | :param info: info or None |
| 529 | :returns: next info in list or None if no more infos |
| 530 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 531 | if info is None: |
| 532 | next_index = 0 |
| 533 | else: |
| 534 | next_index = info.index + 1 |
Klement Sekera | dab231a | 2016-12-21 08:50:14 +0100 | [diff] [blame] | 535 | if next_index == len(self._packet_infos): |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 536 | return None |
| 537 | else: |
Klement Sekera | dab231a | 2016-12-21 08:50:14 +0100 | [diff] [blame] | 538 | return self._packet_infos[next_index] |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 539 | |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 540 | def get_next_packet_info_for_interface(self, src_index, info): |
| 541 | """ |
| 542 | Search the packet info list for the next packet info with same source |
| 543 | interface index |
| 544 | |
| 545 | :param src_index: source interface index to search for |
| 546 | :param info: packet info - where to start the search |
| 547 | :returns: packet info or None |
| 548 | |
| 549 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 550 | while True: |
| 551 | info = self.get_next_packet_info(info) |
| 552 | if info is None: |
| 553 | return None |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 554 | if info.src == src_index: |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 555 | return info |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 556 | |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 557 | def get_next_packet_info_for_interface2(self, src_index, dst_index, info): |
| 558 | """ |
| 559 | Search the packet info list for the next packet info with same source |
| 560 | and destination interface indexes |
| 561 | |
| 562 | :param src_index: source interface index to search for |
| 563 | :param dst_index: destination interface index to search for |
| 564 | :param info: packet info - where to start the search |
| 565 | :returns: packet info or None |
| 566 | |
| 567 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 568 | while True: |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 569 | info = self.get_next_packet_info_for_interface(src_index, info) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 570 | if info is None: |
| 571 | return None |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 572 | if info.dst == dst_index: |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 573 | return info |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 574 | |
Klement Sekera | 0e3c0de | 2016-09-29 14:43:44 +0200 | [diff] [blame] | 575 | def assert_equal(self, real_value, expected_value, name_or_class=None): |
| 576 | if name_or_class is None: |
Klement Sekera | 239790f | 2017-02-16 10:53:53 +0100 | [diff] [blame] | 577 | self.assertEqual(real_value, expected_value) |
Klement Sekera | 0e3c0de | 2016-09-29 14:43:44 +0200 | [diff] [blame] | 578 | return |
| 579 | try: |
| 580 | msg = "Invalid %s: %d('%s') does not match expected value %d('%s')" |
| 581 | msg = msg % (getdoc(name_or_class).strip(), |
| 582 | real_value, str(name_or_class(real_value)), |
| 583 | expected_value, str(name_or_class(expected_value))) |
| 584 | except: |
| 585 | msg = "Invalid %s: %s does not match expected value %s" % ( |
| 586 | name_or_class, real_value, expected_value) |
| 587 | |
| 588 | self.assertEqual(real_value, expected_value, msg) |
| 589 | |
Klement Sekera | b17dd96 | 2017-01-09 07:43:48 +0100 | [diff] [blame] | 590 | def assert_in_range(self, |
| 591 | real_value, |
| 592 | expected_min, |
| 593 | expected_max, |
| 594 | name=None): |
Klement Sekera | 0e3c0de | 2016-09-29 14:43:44 +0200 | [diff] [blame] | 595 | if name is None: |
| 596 | msg = None |
| 597 | else: |
| 598 | msg = "Invalid %s: %s out of range <%s,%s>" % ( |
| 599 | name, real_value, expected_min, expected_max) |
| 600 | self.assertTrue(expected_min <= real_value <= expected_max, msg) |
| 601 | |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 602 | @classmethod |
| 603 | def sleep(cls, timeout, remark=None): |
| 604 | if hasattr(cls, 'logger'): |
| 605 | cls.logger.debug("Sleeping for %ss (%s)" % (timeout, remark)) |
Klement Sekera | a57a970 | 2017-02-02 06:58:07 +0100 | [diff] [blame] | 606 | time.sleep(timeout) |
| 607 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 608 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 609 | class VppTestResult(unittest.TestResult): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 610 | """ |
| 611 | @property result_string |
| 612 | String variable to store the test case result string. |
| 613 | @property errors |
| 614 | List variable containing 2-tuples of TestCase instances and strings |
| 615 | holding formatted tracebacks. Each tuple represents a test which |
| 616 | raised an unexpected exception. |
| 617 | @property failures |
| 618 | List variable containing 2-tuples of TestCase instances and strings |
| 619 | holding formatted tracebacks. Each tuple represents a test where |
| 620 | a failure was explicitly signalled using the TestCase.assert*() |
| 621 | methods. |
| 622 | """ |
| 623 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 624 | def __init__(self, stream, descriptions, verbosity): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 625 | """ |
Klement Sekera | da505f6 | 2017-01-04 12:58:53 +0100 | [diff] [blame] | 626 | :param stream File descriptor to store where to report test results. |
| 627 | Set to the standard error stream by default. |
| 628 | :param descriptions Boolean variable to store information if to use |
| 629 | test case descriptions. |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 630 | :param verbosity Integer variable to store required verbosity level. |
| 631 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 632 | unittest.TestResult.__init__(self, stream, descriptions, verbosity) |
| 633 | self.stream = stream |
| 634 | self.descriptions = descriptions |
| 635 | self.verbosity = verbosity |
| 636 | self.result_string = None |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 637 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 638 | def addSuccess(self, test): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 639 | """ |
| 640 | Record a test succeeded result |
| 641 | |
| 642 | :param test: |
| 643 | |
| 644 | """ |
Klement Sekera | b91017a | 2017-02-09 06:04:36 +0100 | [diff] [blame] | 645 | if hasattr(test, 'logger'): |
| 646 | test.logger.debug("--- addSuccess() %s.%s(%s) called" |
| 647 | % (test.__class__.__name__, |
| 648 | test._testMethodName, |
| 649 | test._testMethodDoc)) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 650 | unittest.TestResult.addSuccess(self, test) |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 651 | self.result_string = colorize("OK", GREEN) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 652 | |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 653 | def addSkip(self, test, reason): |
| 654 | """ |
| 655 | Record a test skipped. |
| 656 | |
| 657 | :param test: |
| 658 | :param reason: |
| 659 | |
| 660 | """ |
Klement Sekera | b91017a | 2017-02-09 06:04:36 +0100 | [diff] [blame] | 661 | if hasattr(test, 'logger'): |
| 662 | test.logger.debug("--- addSkip() %s.%s(%s) called, reason is %s" |
| 663 | % (test.__class__.__name__, |
| 664 | test._testMethodName, |
| 665 | test._testMethodDoc, |
| 666 | reason)) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 667 | unittest.TestResult.addSkip(self, test, reason) |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 668 | self.result_string = colorize("SKIP", YELLOW) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 669 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 670 | def addFailure(self, test, err): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 671 | """ |
| 672 | Record a test failed result |
| 673 | |
| 674 | :param test: |
| 675 | :param err: error message |
| 676 | |
| 677 | """ |
Klement Sekera | b91017a | 2017-02-09 06:04:36 +0100 | [diff] [blame] | 678 | if hasattr(test, 'logger'): |
| 679 | test.logger.debug("--- addFailure() %s.%s(%s) called, err is %s" |
| 680 | % (test.__class__.__name__, |
| 681 | test._testMethodName, |
| 682 | test._testMethodDoc, err)) |
| 683 | test.logger.debug("formatted exception is:\n%s" % |
| 684 | "".join(format_exception(*err))) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 685 | unittest.TestResult.addFailure(self, test, err) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 686 | if hasattr(test, 'tempdir'): |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 687 | self.result_string = colorize("FAIL", RED) + \ |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 688 | ' [ temp dir used by test case: ' + test.tempdir + ' ]' |
| 689 | else: |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 690 | self.result_string = colorize("FAIL", RED) + ' [no temp dir]' |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 691 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 692 | def addError(self, test, err): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 693 | """ |
| 694 | Record a test error result |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 695 | |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 696 | :param test: |
| 697 | :param err: error message |
| 698 | |
| 699 | """ |
Klement Sekera | b91017a | 2017-02-09 06:04:36 +0100 | [diff] [blame] | 700 | if hasattr(test, 'logger'): |
| 701 | test.logger.debug("--- addError() %s.%s(%s) called, err is %s" |
| 702 | % (test.__class__.__name__, |
| 703 | test._testMethodName, |
| 704 | test._testMethodDoc, err)) |
| 705 | test.logger.debug("formatted exception is:\n%s" % |
| 706 | "".join(format_exception(*err))) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 707 | unittest.TestResult.addError(self, test, err) |
| 708 | if hasattr(test, 'tempdir'): |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 709 | self.result_string = colorize("ERROR", RED) + \ |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 710 | ' [ temp dir used by test case: ' + test.tempdir + ' ]' |
| 711 | else: |
Klement Sekera | 277b89c | 2016-10-28 13:20:27 +0200 | [diff] [blame] | 712 | self.result_string = colorize("ERROR", RED) + ' [no temp dir]' |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 713 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 714 | def getDescription(self, test): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 715 | """ |
| 716 | Get test description |
| 717 | |
| 718 | :param test: |
| 719 | :returns: test description |
| 720 | |
| 721 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 722 | # TODO: if none print warning not raise exception |
| 723 | short_description = test.shortDescription() |
| 724 | if self.descriptions and short_description: |
| 725 | return short_description |
| 726 | else: |
| 727 | return str(test) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 728 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 729 | def startTest(self, test): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 730 | """ |
| 731 | Start a test |
| 732 | |
| 733 | :param test: |
| 734 | |
| 735 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 736 | unittest.TestResult.startTest(self, test) |
| 737 | if self.verbosity > 0: |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 738 | self.stream.writeln( |
| 739 | "Starting " + self.getDescription(test) + " ...") |
| 740 | self.stream.writeln(single_line_delim) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 741 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 742 | def stopTest(self, test): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 743 | """ |
| 744 | Stop a test |
| 745 | |
| 746 | :param test: |
| 747 | |
| 748 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 749 | unittest.TestResult.stopTest(self, test) |
| 750 | if self.verbosity > 0: |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 751 | self.stream.writeln(single_line_delim) |
Klement Sekera | 52e84f3 | 2017-01-13 07:25:25 +0100 | [diff] [blame] | 752 | self.stream.writeln("%-73s%s" % (self.getDescription(test), |
Klement Sekera | da505f6 | 2017-01-04 12:58:53 +0100 | [diff] [blame] | 753 | self.result_string)) |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 754 | self.stream.writeln(single_line_delim) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 755 | else: |
Klement Sekera | 52e84f3 | 2017-01-13 07:25:25 +0100 | [diff] [blame] | 756 | self.stream.writeln("%-73s%s" % (self.getDescription(test), |
Klement Sekera | da505f6 | 2017-01-04 12:58:53 +0100 | [diff] [blame] | 757 | self.result_string)) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 758 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 759 | def printErrors(self): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 760 | """ |
| 761 | Print errors from running the test case |
| 762 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 763 | self.stream.writeln() |
| 764 | self.printErrorList('ERROR', self.errors) |
| 765 | self.printErrorList('FAIL', self.failures) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 766 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 767 | def printErrorList(self, flavour, errors): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 768 | """ |
| 769 | Print error list to the output stream together with error type |
| 770 | and test case description. |
| 771 | |
| 772 | :param flavour: error type |
| 773 | :param errors: iterable errors |
| 774 | |
| 775 | """ |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 776 | for test, err in errors: |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 777 | self.stream.writeln(double_line_delim) |
| 778 | self.stream.writeln("%s: %s" % |
| 779 | (flavour, self.getDescription(test))) |
| 780 | self.stream.writeln(single_line_delim) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 781 | self.stream.writeln("%s" % err) |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 782 | |
| 783 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 784 | class VppTestRunner(unittest.TextTestRunner): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 785 | """ |
Klement Sekera | 104543f | 2017-02-03 07:29:43 +0100 | [diff] [blame] | 786 | A basic test runner implementation which prints results to standard error. |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 787 | """ |
| 788 | @property |
| 789 | def resultclass(self): |
| 790 | """Class maintaining the results of the tests""" |
| 791 | return VppTestResult |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 792 | |
Klement Sekera | 7a161da | 2017-01-17 13:42:48 +0100 | [diff] [blame] | 793 | def __init__(self, stream=sys.stderr, descriptions=True, verbosity=1, |
| 794 | failfast=False, buffer=False, resultclass=None): |
| 795 | # ignore stream setting here, use hard-coded stdout to be in sync |
| 796 | # with prints from VppTestCase methods ... |
| 797 | super(VppTestRunner, self).__init__(sys.stdout, descriptions, |
| 798 | verbosity, failfast, buffer, |
| 799 | resultclass) |
| 800 | |
Klement Sekera | 104543f | 2017-02-03 07:29:43 +0100 | [diff] [blame] | 801 | test_option = "TEST" |
| 802 | |
| 803 | def parse_test_option(self): |
| 804 | try: |
| 805 | f = os.getenv(self.test_option) |
| 806 | except: |
| 807 | f = None |
| 808 | filter_file_name = None |
| 809 | filter_class_name = None |
| 810 | filter_func_name = None |
| 811 | if f: |
| 812 | if '.' in f: |
| 813 | parts = f.split('.') |
| 814 | if len(parts) > 3: |
| 815 | raise Exception("Unrecognized %s option: %s" % |
| 816 | (self.test_option, f)) |
| 817 | if len(parts) > 2: |
| 818 | if parts[2] not in ('*', ''): |
| 819 | filter_func_name = parts[2] |
| 820 | if parts[1] not in ('*', ''): |
| 821 | filter_class_name = parts[1] |
| 822 | if parts[0] not in ('*', ''): |
| 823 | if parts[0].startswith('test_'): |
| 824 | filter_file_name = parts[0] |
| 825 | else: |
| 826 | filter_file_name = 'test_%s' % parts[0] |
| 827 | else: |
| 828 | if f.startswith('test_'): |
| 829 | filter_file_name = f |
| 830 | else: |
| 831 | filter_file_name = 'test_%s' % f |
| 832 | return filter_file_name, filter_class_name, filter_func_name |
| 833 | |
| 834 | def filter_tests(self, tests, filter_file, filter_class, filter_func): |
| 835 | result = unittest.suite.TestSuite() |
| 836 | for t in tests: |
| 837 | if isinstance(t, unittest.suite.TestSuite): |
| 838 | # this is a bunch of tests, recursively filter... |
| 839 | x = self.filter_tests(t, filter_file, filter_class, |
| 840 | filter_func) |
| 841 | if x.countTestCases() > 0: |
| 842 | result.addTest(x) |
| 843 | elif isinstance(t, unittest.TestCase): |
| 844 | # this is a single test |
| 845 | parts = t.id().split('.') |
| 846 | # t.id() for common cases like this: |
| 847 | # test_classifier.TestClassifier.test_acl_ip |
| 848 | # apply filtering only if it is so |
| 849 | if len(parts) == 3: |
| 850 | if filter_file and filter_file != parts[0]: |
| 851 | continue |
| 852 | if filter_class and filter_class != parts[1]: |
| 853 | continue |
| 854 | if filter_func and filter_func != parts[2]: |
| 855 | continue |
| 856 | result.addTest(t) |
| 857 | else: |
| 858 | # unexpected object, don't touch it |
| 859 | result.addTest(t) |
| 860 | return result |
| 861 | |
Damjan Marion | f56b77a | 2016-10-03 19:44:57 +0200 | [diff] [blame] | 862 | def run(self, test): |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 863 | """ |
| 864 | Run the tests |
| 865 | |
| 866 | :param test: |
| 867 | |
| 868 | """ |
Klement Sekera | acb9b8e | 2017-02-14 02:55:31 +0100 | [diff] [blame] | 869 | gc.disable() # disable garbage collection, we'll do that manually |
Klement Sekera | f62ae12 | 2016-10-11 11:47:09 +0200 | [diff] [blame] | 870 | print("Running tests using custom test runner") # debug message |
Klement Sekera | 104543f | 2017-02-03 07:29:43 +0100 | [diff] [blame] | 871 | filter_file, filter_class, filter_func = self.parse_test_option() |
| 872 | print("Active filters: file=%s, class=%s, function=%s" % ( |
| 873 | filter_file, filter_class, filter_func)) |
| 874 | filtered = self.filter_tests(test, filter_file, filter_class, |
| 875 | filter_func) |
| 876 | print("%s out of %s tests match specified filters" % ( |
| 877 | filtered.countTestCases(), test.countTestCases())) |
| 878 | return super(VppTestRunner, self).run(filtered) |