test/run_tests.py

#!/usr/bin/env python

import sys
import shutil
import os
import select
import unittest
import argparse
import time
from multiprocessing import Process, Pipe
from framework import VppTestRunner
from debug import spawn_gdb
from log import global_logger
from discover_tests import discover_tests
from subprocess import check_output, CalledProcessError
from util import check_core_path

# timeout which controls how long the child has to finish after seeing
# a core dump in test temporary directory. If this is exceeded, parent assumes
# that child process is stuck (e.g. waiting for shm mutex, which will never
# get unlocked) and kill the child
core_timeout = 3


def test_runner_wrapper(suite, keep_alive_pipe, result_pipe, failed_pipe):
    result = not VppTestRunner(
        keep_alive_pipe=keep_alive_pipe,
        failed_pipe=failed_pipe,
        verbosity=verbose,
        failfast=failfast).run(suite).wasSuccessful()
    result_pipe.send(result)
    result_pipe.close()
    keep_alive_pipe.close()
    failed_pipe.close()


class add_to_suite_callback:
    def __init__(self, suite):
        self.suite = suite

    def __call__(self, file_name, cls, method):
        suite.addTest(cls(method))


class Filter_by_class_list:
    def __init__(self, class_list):
        self.class_list = class_list

    def __call__(self, file_name, class_name, func_name):
        return class_name in self.class_list


def suite_from_failed(suite, failed):
    filter_cb = Filter_by_class_list(failed)
    suite = VppTestRunner.filter_tests(suite, filter_cb)
    if 0 == suite.countTestCases():
        raise Exception("Suite is empty after filtering out the failed tests!")
    return suite


def run_forked(suite):
    keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
    result_parent_end, result_child_end = Pipe(duplex=False)
    failed_parent_end, failed_child_end = Pipe(duplex=False)

    child = Process(target=test_runner_wrapper,
                    args=(suite, keep_alive_child_end, result_child_end,
                          failed_child_end))
    child.start()
    last_test_temp_dir = None
    last_test_vpp_binary = None
    last_test = None
    result = None
    failed = set()
    last_heard = time.time()
    core_detected_at = None
    debug_core = os.getenv("DEBUG", "").lower() == "core"
    while True:
        readable = select.select([keep_alive_parent_end.fileno(),
                                  result_parent_end.fileno(),
                                  failed_parent_end.fileno(),
                                  ],
                                 [], [], 1)[0]
        if result_parent_end.fileno() in readable:
            result = result_parent_end.recv()
            break
        if keep_alive_parent_end.fileno() in readable:
            while keep_alive_parent_end.poll():
                last_test, last_test_vpp_binary,\
                    last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
            last_heard = time.time()
        if failed_parent_end.fileno() in readable:
            while failed_parent_end.poll():
                failed_test = failed_parent_end.recv()
                failed.add(failed_test.__name__)
            last_heard = time.time()
        fail = False
        if last_heard + test_timeout < time.time() and \
                not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
            fail = True
            global_logger.critical("Timeout while waiting for child test "
                                   "runner process (last test running was "
                                   "`%s' in `%s')!" %
                                   (last_test, last_test_temp_dir))
        elif not child.is_alive():
            fail = True
            global_logger.critical("Child python process unexpectedly died "
                                   "(last test running was `%s' in `%s')!" %
                                   (last_test, last_test_temp_dir))
        elif last_test_temp_dir and last_test_vpp_binary:
            core_path = "%s/core" % last_test_temp_dir
            if os.path.isfile(core_path):
                if core_detected_at is None:
                    core_detected_at = time.time()
                elif core_detected_at + core_timeout < time.time():
                    if not os.path.isfile(
                            "%s/_core_handled" % last_test_temp_dir):
                        global_logger.critical(
                            "Child python process unresponsive and core-file "
                            "exists in test temporary directory!")
                        fail = True

        if fail:
            failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
            lttd = last_test_temp_dir.split("/")[-1]
            link_path = '%s%s-FAILED' % (failed_dir, lttd)
            global_logger.error("Creating a link to the failed " +
                                "test: %s -> %s" % (link_path, lttd))
            try:
                os.symlink(last_test_temp_dir, link_path)
            except Exception:
                pass
            api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
            if os.path.isfile(api_post_mortem_path):
                global_logger.error("Copying api_post_mortem.%d to %s" %
                                    (vpp_pid, last_test_temp_dir))
                shutil.copy2(api_post_mortem_path, last_test_temp_dir)
            if last_test_temp_dir and last_test_vpp_binary:
                core_path = "%s/core" % last_test_temp_dir
                if os.path.isfile(core_path):
                    global_logger.error("Core-file exists in test temporary "
                                        "directory: %s!" % core_path)
                    check_core_path(global_logger, core_path)
                    global_logger.debug("Running `file %s':" % core_path)
                    try:
                        info = check_output(["file", core_path])
                        global_logger.debug(info)
                    except CalledProcessError as e:
                        global_logger.error(
                            "Could not run `file' utility on core-file, "
                            "rc=%s" % e.returncode)
                        pass
                    if debug_core:
                        spawn_gdb(last_test_vpp_binary, core_path,
                                  global_logger)
            child.terminate()
            result = -1
            break
    keep_alive_parent_end.close()
    result_parent_end.close()
    failed_parent_end.close()
    return result, failed


if __name__ == '__main__':

    try:
        verbose = int(os.getenv("V", 0))
    except ValueError:
        verbose = 0

    default_test_timeout = 600  # 10 minutes
    try:
        test_timeout = int(os.getenv("TIMEOUT", default_test_timeout))
    except ValueError:
        test_timeout = default_test_timeout

    debug = os.getenv("DEBUG")

    s = os.getenv("STEP", "n")
    step = True if s.lower() in ("y", "yes", "1") else False

    parser = argparse.ArgumentParser(description="VPP unit tests")
    parser.add_argument("-f", "--failfast", action='count',
                        help="fast failure flag")
    parser.add_argument("-d", "--dir", action='append', type=str,
                        help="directory containing test files "
                             "(may be specified multiple times)")
    args = parser.parse_args()
    failfast = True if args.failfast == 1 else False

    suite = unittest.TestSuite()
    cb = add_to_suite_callback(suite)
    for d in args.dir:
        print("Adding tests from directory tree %s" % d)
        discover_tests(d, cb)

    try:
        retries = int(os.getenv("RETRIES", 0))
    except ValueError:
        retries = 0

    try:
        force_foreground = int(os.getenv("FORCE_FOREGROUND", 0))
    except ValueError:
        force_foreground = 0
    attempts = retries + 1
    if attempts > 1:
        print("Perform %s attempts to pass the suite..." % attempts)
    if (debug is not None and debug.lower() in ["gdb", "gdbserver"]) or step\
            or force_foreground:
        # don't fork if requiring interactive terminal..
        sys.exit(not VppTestRunner(
            verbosity=verbose, failfast=failfast).run(suite).wasSuccessful())
    else:
        while True:
            result, failed = run_forked(suite)
            attempts = attempts - 1
            print("%s test(s) failed, %s attempt(s) left" %
                  (len(failed), attempts))
            if len(failed) > 0 and attempts > 0:
                suite = suite_from_failed(suite, failed)
                continue
            sys.exit(result)