tests: disable failing tests on ubuntu-22.04 debian-11
Type: test
Change-Id: I7b2314a731c83b3dcd69c999edb8ebed53839724
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>
diff --git a/test/framework.py b/test/framework.py
index c85dec5..d130855 100644
--- a/test/framework.py
+++ b/test/framework.py
@@ -147,6 +147,8 @@
def pump_output(testclass):
"""pump output from vpp stdout/stderr to proper queues"""
+ if not hasattr(testclass, "vpp"):
+ return
stdout_fragment = ""
stderr_fragment = ""
while not testclass.pump_thread_stop_flag.is_set():
@@ -212,6 +214,17 @@
is_distro_ubuntu2204 = _is_distro_ubuntu2204()
+def _is_distro_debian11():
+ with open("/etc/os-release") as f:
+ for line in f.readlines():
+ if "bullseye" in line:
+ return True
+ return False
+
+
+is_distro_debian11 = _is_distro_debian11()
+
+
class KeepAliveReporter(object):
"""
Singleton object which reports test start to parent process
@@ -237,7 +250,7 @@
"""
Write current test tmpdir & desc to keep-alive pipe to signal liveness
"""
- if self.pipe is None:
+ if not hasattr(test, "vpp") or self.pipe is None:
# if not running forked..
return
@@ -259,6 +272,8 @@
FIXME_ASAN = 3
# marks suites broken on Ubuntu-22.04
FIXME_UBUNTU2204 = 4
+ # marks suites broken on Debian-11
+ FIXME_DEBIAN11 = 5
def create_tag_decorator(e):
@@ -276,6 +291,7 @@
tag_fixme_vpp_workers = create_tag_decorator(TestCaseTag.FIXME_VPP_WORKERS)
tag_fixme_asan = create_tag_decorator(TestCaseTag.FIXME_ASAN)
tag_fixme_ubuntu2204 = create_tag_decorator(TestCaseTag.FIXME_UBUNTU2204)
+tag_fixme_debian11 = create_tag_decorator(TestCaseTag.FIXME_DEBIAN11)
class DummyVpp:
@@ -358,6 +374,12 @@
cls = unittest.skip("Skipping @tag_fixme_ubuntu2204 tests")(cls)
@classmethod
+ def skip_fixme_debian11(cls):
+ """if distro is Debian-11 and @tag_fixme_debian11 mark for skip"""
+ if cls.has_tag(TestCaseTag.FIXME_DEBIAN11):
+ cls = unittest.skip("Skipping @tag_fixme_debian11 tests")(cls)
+
+ @classmethod
def instance(cls):
"""Return the instance of this testcase"""
return cls.test_instance
@@ -556,6 +578,10 @@
@classmethod
def run_vpp(cls):
+ if (
+ is_distro_ubuntu2204 == True and cls.has_tag(TestCaseTag.FIXME_UBUNTU2204)
+ ) or (is_distro_debian11 == True and cls.has_tag(TestCaseTag.FIXME_DEBIAN11)):
+ return
cls.logger.debug(f"Assigned cpus: {cls.cpus}")
cmdline = cls.vpp_cmdline
@@ -694,6 +720,8 @@
cls.attach_vpp()
else:
cls.run_vpp()
+ if not hasattr(cls, "vpp"):
+ return
cls.reporter.send_keep_alive(cls, "setUpClass")
VppTestResult.current_test_case_info = TestCaseInfo(
cls.logger, cls.tempdir, cls.vpp.pid, config.vpp
@@ -854,6 +882,8 @@
def tearDownClass(cls):
"""Perform final cleanup after running all tests in this test-case"""
cls.logger.debug("--- tearDownClass() for %s called ---" % cls.__name__)
+ if not hasattr(cls, "vpp"):
+ return
cls.reporter.send_keep_alive(cls, "tearDownClass")
cls.quit()
cls.file_handler.close()
@@ -871,6 +901,8 @@
"--- tearDown() for %s.%s(%s) called ---"
% (self.__class__.__name__, self._testMethodName, self._testMethodDoc)
)
+ if not hasattr(self, "vpp"):
+ return
try:
if not self.vpp_dead:
@@ -904,6 +936,8 @@
def setUp(self):
"""Clear trace before running each test"""
super(VppTestCase, self).setUp()
+ if not hasattr(self, "vpp"):
+ return
self.reporter.send_keep_alive(self)
if self.vpp_dead:
raise VppDiedError(
@@ -1008,6 +1042,9 @@
@classmethod
def create_pg_ip4_interfaces(cls, interfaces, gso=0, gso_size=0):
+ if not hasattr(cls, "vpp"):
+ cls.pg_interfaces = []
+ return cls.pg_interfaces
pgmode = VppEnum.vl_api_pg_interface_mode_t
return cls.create_pg_interfaces_internal(
interfaces, gso, gso_size, pgmode.PG_API_MODE_IP4
@@ -1015,6 +1052,9 @@
@classmethod
def create_pg_ip6_interfaces(cls, interfaces, gso=0, gso_size=0):
+ if not hasattr(cls, "vpp"):
+ cls.pg_interfaces = []
+ return cls.pg_interfaces
pgmode = VppEnum.vl_api_pg_interface_mode_t
return cls.create_pg_interfaces_internal(
interfaces, gso, gso_size, pgmode.PG_API_MODE_IP6
@@ -1022,6 +1062,9 @@
@classmethod
def create_pg_interfaces(cls, interfaces, gso=0, gso_size=0):
+ if not hasattr(cls, "vpp"):
+ cls.pg_interfaces = []
+ return cls.pg_interfaces
pgmode = VppEnum.vl_api_pg_interface_mode_t
return cls.create_pg_interfaces_internal(
interfaces, gso, gso_size, pgmode.PG_API_MODE_ETHERNET
@@ -1029,6 +1072,9 @@
@classmethod
def create_pg_ethernet_interfaces(cls, interfaces, gso=0, gso_size=0):
+ if not hasattr(cls, "vpp"):
+ cls.pg_interfaces = []
+ return cls.pg_interfaces
pgmode = VppEnum.vl_api_pg_interface_mode_t
return cls.create_pg_interfaces_internal(
interfaces, gso, gso_size, pgmode.PG_API_MODE_ETHERNET
@@ -1042,6 +1088,9 @@
:param count: number of interfaces created.
:returns: List of created interfaces.
"""
+ if not hasattr(cls, "vpp"):
+ cls.lo_interfaces = []
+ return cls.lo_interfaces
result = [VppLoInterface(cls) for i in range(count)]
for intf in result:
setattr(cls, intf.name, intf)
@@ -1056,6 +1105,9 @@
:param count: number of interfaces created.
:returns: List of created interfaces.
"""
+ if not hasattr(cls, "vpp"):
+ cls.bvi_interfaces = []
+ return cls.bvi_interfaces
result = [VppBviInterface(cls) for i in range(count)]
for intf in result:
setattr(cls, intf.name, intf)
@@ -1786,6 +1838,10 @@
test_title = colorize(f"FIXME on Ubuntu-22.04: {test_title}", RED)
test.skip_fixme_ubuntu2204()
+ if is_distro_debian11 == True and test.has_tag(TestCaseTag.FIXME_DEBIAN11):
+ test_title = colorize(f"FIXME on Debian-11: {test_title}", RED)
+ test.skip_fixme_debian11()
+
if hasattr(test, "vpp_worker_count"):
if test.vpp_worker_count == 0:
test_title += " [main thread only]"
diff --git a/test/hook.py b/test/hook.py
index 3429bda..58bbcaf 100644
--- a/test/hook.py
+++ b/test/hook.py
@@ -116,7 +116,7 @@
Poll the vpp status and throw an exception if it's not running
:raises VppDiedError: exception if VPP is not running anymore
"""
- if self.test.vpp_dead:
+ if not hasattr(self.test, "vpp") or self.test.vpp_dead:
# already dead, nothing to do
return
diff --git a/test/test_bfd.py b/test/test_bfd.py
index 7113370..67ddb4b 100644
--- a/test/test_bfd.py
+++ b/test/test_bfd.py
@@ -30,7 +30,8 @@
BFDState,
BFD_vpp_echo,
)
-from framework import tag_fixme_vpp_workers
+from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, tag_fixme_debian11
+from framework import is_distro_ubuntu2204, is_distro_debian11
from framework import VppTestCase, VppTestRunner
from framework import tag_run_solo
from util import ppp
@@ -818,6 +819,8 @@
@tag_run_solo
+@tag_fixme_ubuntu2204
+@tag_fixme_debian11
class BFD4TestCase(VppTestCase):
"""Bidirectional Forwarding Detection (BFD)"""
@@ -828,6 +831,10 @@
@classmethod
def setUpClass(cls):
+ if (is_distro_ubuntu2204 == True or is_distro_debian11 == True) and not hasattr(
+ cls, "vpp"
+ ):
+ return
super(BFD4TestCase, cls).setUpClass()
cls.vapi.cli("set log class bfd level debug")
try:
@@ -1722,6 +1729,7 @@
@tag_run_solo
@tag_fixme_vpp_workers
+@tag_fixme_ubuntu2204
class BFD6TestCase(VppTestCase):
"""Bidirectional Forwarding Detection (BFD) (IPv6)"""
@@ -1732,6 +1740,8 @@
@classmethod
def setUpClass(cls):
+ if is_distro_ubuntu2204 == True and not hasattr(cls, "vpp"):
+ return
super(BFD6TestCase, cls).setUpClass()
cls.vapi.cli("set log class bfd level debug")
try:
@@ -1755,6 +1765,8 @@
def setUp(self):
super(BFD6TestCase, self).setUp()
+ if is_distro_ubuntu2204 == True and not hasattr(self, "vpp"):
+ return
self.factory = AuthKeyFactory()
self.vapi.want_bfd_events()
self.pg0.enable_capture()
diff --git a/test/test_flowprobe.py b/test/test_flowprobe.py
index 141d745..e3ff224 100644
--- a/test/test_flowprobe.py
+++ b/test/test_flowprobe.py
@@ -13,7 +13,8 @@
from scapy.layers.inet6 import IPv6
from config import config
-from framework import tag_fixme_vpp_workers
+from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, tag_fixme_debian11
+from framework import is_distro_ubuntu2204, is_distro_debian11
from framework import VppTestCase, VppTestRunner
from framework import tag_run_solo
from vpp_object import VppObject
@@ -151,6 +152,10 @@
variables and configure VPP.
"""
super(MethodHolder, cls).setUpClass()
+ if (is_distro_ubuntu2204 == True or is_distro_debian11 == True) and not hasattr(
+ cls, "vpp"
+ ):
+ return
try:
# Create pg interfaces
cls.create_pg_interfaces(range(9))
@@ -337,6 +342,8 @@
@tag_run_solo
@tag_fixme_vpp_workers
+@tag_fixme_ubuntu2204
+@tag_fixme_debian11
class Flowprobe(MethodHolder):
"""Template verification, timer tests"""
diff --git a/test/test_ikev2.py b/test/test_ikev2.py
index ffddc79..f0fd205 100644
--- a/test/test_ikev2.py
+++ b/test/test_ikev2.py
@@ -19,7 +19,8 @@
from scapy.layers.inet6 import IPv6
from scapy.packet import raw, Raw
from scapy.utils import long_converter
-from framework import tag_fixme_vpp_workers
+from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, tag_fixme_debian11
+from framework import is_distro_ubuntu2204, is_distro_debian11
from framework import VppTestCase, VppTestRunner
from vpp_ikev2 import Profile, IDType, AuthMethod
from vpp_papi import VppEnum
@@ -2177,6 +2178,8 @@
WITH_KEX = True
+@tag_fixme_ubuntu2204
+@tag_fixme_debian11
class TestResponderVrf(TestResponderPsk, Ikev2Params):
"""test ikev2 responder - non-default table id"""
@@ -2186,6 +2189,10 @@
globals()["ikev2"] = _ikev2
super(IkePeer, cls).setUpClass()
+ if (is_distro_ubuntu2204 == True or is_distro_debian11 == True) and not hasattr(
+ cls, "vpp"
+ ):
+ return
cls.create_pg_interfaces(range(1))
cls.vapi.cli("ip table add 1")
cls.vapi.cli("set interface ip table pg0 1")
diff --git a/test/test_memif.py b/test/test_memif.py
index 9b15cd0..f0b6021 100644
--- a/test/test_memif.py
+++ b/test/test_memif.py
@@ -5,7 +5,7 @@
from scapy.layers.inet import IP, ICMP
from framework import VppTestCase, VppTestRunner
-from framework import tag_run_solo
+from framework import tag_run_solo, tag_fixme_debian11, is_distro_debian11
from remote_test import RemoteClass, RemoteVppTestCase
from vpp_memif import remove_all_memif_vpp_config, VppSocketFilename, VppMemif
from vpp_ip_route import VppIpRoute, VppRoutePath
@@ -13,6 +13,7 @@
@tag_run_solo
+@tag_fixme_debian11
class TestMemif(VppTestCase):
"""Memif Test Case"""
@@ -36,6 +37,9 @@
cls.remote_test.start_remote()
cls.remote_test.set_request_timeout(10)
super(TestMemif, cls).setUpClass()
+ if is_distro_debian11 == True and not hasattr(cls, "vpp"):
+ cls.remote_test.quit_remote()
+ return
cls.remote_test.setUpClass(cls.tempdir)
cls.create_pg_interfaces(range(1))
for pg in cls.pg_interfaces:
diff --git a/test/test_nat44_ed.py b/test/test_nat44_ed.py
index d90afd2..6b0eedc 100644
--- a/test/test_nat44_ed.py
+++ b/test/test_nat44_ed.py
@@ -5,6 +5,7 @@
from random import randint, choice
import scapy.compat
+from framework import tag_fixme_ubuntu2204, is_distro_ubuntu2204
from framework import VppTestCase, VppTestRunner
from scapy.data import IP_PROTOS
from scapy.layers.inet import IP, TCP, UDP, ICMP, GRE
@@ -159,6 +160,8 @@
@classmethod
def setUpClass(cls):
super().setUpClass()
+ if is_distro_ubuntu2204 == True and not hasattr(cls, "vpp"):
+ return
cls.create_pg_interfaces(range(12))
cls.interfaces = list(cls.pg_interfaces[:4])
@@ -2550,6 +2553,7 @@
out_if.unconfig()
+@tag_fixme_ubuntu2204
class TestNAT44EDMW(TestNAT44ED):
"""NAT44ED MW Test Case"""
diff --git a/test/test_nat44_ei.py b/test/test_nat44_ei.py
index 9eb127a..259c445 100644
--- a/test/test_nat44_ei.py
+++ b/test/test_nat44_ei.py
@@ -8,6 +8,7 @@
from io import BytesIO
import scapy.compat
+from framework import tag_fixme_debian11, is_distro_debian11
from framework import VppTestCase, VppTestRunner
from ipfix import IPFIX, Set, Template, Data, IPFIXDecoder
from scapy.all import (
@@ -910,6 +911,7 @@
return 1 + h % vpp_worker_count
+@tag_fixme_debian11
class TestNAT44EI(MethodHolder):
"""NAT44EI Test Cases"""
@@ -919,6 +921,8 @@
@classmethod
def setUpClass(cls):
super(TestNAT44EI, cls).setUpClass()
+ if is_distro_debian11 == True and not hasattr(cls, "vpp"):
+ return
cls.vapi.cli("set log class nat44-ei level debug")
cls.tcp_port_in = 6303
diff --git a/test/test_nat64.py b/test/test_nat64.py
index 214072a..05a2031 100644
--- a/test/test_nat64.py
+++ b/test/test_nat64.py
@@ -9,7 +9,7 @@
import scapy.compat
from config import config
-from framework import tag_fixme_vpp_workers
+from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, is_distro_ubuntu2204
from framework import VppTestCase, VppTestRunner
from ipfix import IPFIX, Set, Template, Data, IPFIXDecoder
from scapy.data import IP_PROTOS
@@ -34,6 +34,7 @@
@tag_fixme_vpp_workers
+@tag_fixme_ubuntu2204
class TestNAT64(VppTestCase):
"""NAT64 Test Cases"""
@@ -49,6 +50,8 @@
def setUpClass(cls):
super(TestNAT64, cls).setUpClass()
+ if is_distro_ubuntu2204 == True and not hasattr(cls, "vpp"):
+ return
cls.tcp_port_in = 6303
cls.tcp_port_out = 6303
cls.udp_port_in = 6304
diff --git a/test/test_neighbor.py b/test/test_neighbor.py
index 58d98a3..f663e73 100644
--- a/test/test_neighbor.py
+++ b/test/test_neighbor.py
@@ -4,7 +4,7 @@
import os
from socket import AF_INET, AF_INET6, inet_pton
-from framework import tag_fixme_vpp_workers
+from framework import tag_fixme_vpp_workers, tag_fixme_ubuntu2204, tag_fixme_debian11
from framework import VppTestCase, VppTestRunner
from vpp_neighbor import VppNeighbor, find_nbr
from vpp_ip_route import (
@@ -2196,6 +2196,7 @@
self.assertEqual(NUM_PKTS + 16, nd1.get_stats()["packets"])
+@tag_fixme_ubuntu2204
class NeighborAgeTestCase(VppTestCase):
"""ARP/ND Aging"""
diff --git a/test/test_wireguard.py b/test/test_wireguard.py
index 7055b7a..6da112a 100644
--- a/test/test_wireguard.py
+++ b/test/test_wireguard.py
@@ -41,6 +41,8 @@
from vpp_ip_route import VppIpRoute, VppRoutePath
from vpp_object import VppObject
from vpp_papi import VppEnum
+from framework import tag_fixme_ubuntu2204, tag_fixme_debian11
+from framework import is_distro_ubuntu2204, is_distro_debian11
from framework import VppTestCase
from re import compile
import unittest
@@ -493,6 +495,8 @@
self._test.assertEqual(rv.peer_index, self.index)
+@tag_fixme_ubuntu2204
+@tag_fixme_debian11
class TestWg(VppTestCase):
"""Wireguard Test Case"""
@@ -518,6 +522,10 @@
@classmethod
def setUpClass(cls):
super(TestWg, cls).setUpClass()
+ if (is_distro_ubuntu2204 == True or is_distro_debian11 == True) and not hasattr(
+ cls, "vpp"
+ ):
+ return
try:
cls.create_pg_interfaces(range(3))
for i in cls.pg_interfaces:
@@ -2227,6 +2235,8 @@
wg1.remove_vpp_config()
+@tag_fixme_ubuntu2204
+@tag_fixme_debian11
class WireguardHandoffTests(TestWg):
"""Wireguard Tests in multi worker setup"""