hs-test: generate core dump, fix docker logs in CI
Type: test
Change-Id: Ie1f66cdc061d3eccefc2ce58e977d88a33340038
Signed-off-by: Adrian Villin <avillin@cisco.com>
diff --git a/extras/hs-test/infra/container.go b/extras/hs-test/infra/container.go
index d8bddab..195f889 100644
--- a/extras/hs-test/infra/container.go
+++ b/extras/hs-test/infra/container.go
@@ -14,6 +14,7 @@
"text/template"
"time"
+ "github.com/cilium/cilium/pkg/sysctl"
containerTypes "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/image"
@@ -325,6 +326,15 @@
volumeSlice = append(volumeSlice, fmt.Sprintf("%s:%s", *VppSourceFileDir, *VppSourceFileDir))
}
+ core_pattern, err := sysctl.Read("kernel.core_pattern")
+ if err == nil {
+ index := strings.LastIndex(core_pattern, "/")
+ core_pattern = core_pattern[:index]
+ volumeSlice = append(volumeSlice, c.Suite.getLogDirPath()+":"+core_pattern)
+ } else {
+ c.Suite.Log(err)
+ }
+
if len(c.Volumes) > 0 {
for _, volume := range c.Volumes {
volumeSlice = append(volumeSlice, fmt.Sprintf("%s:%s", volume.HostDir, volume.ContainerDir))
@@ -435,21 +445,8 @@
return string(byteOutput)
}
-func (c *Container) getLogDirPath() string {
- testId := c.Suite.GetTestId()
- testName := c.Suite.GetCurrentTestName()
- logDirPath := logDir + testName + "/" + testId + "/"
-
- cmd := exec.Command("mkdir", "-p", logDirPath)
- if err := cmd.Run(); err != nil {
- Fail("mkdir error: " + fmt.Sprint(err))
- }
-
- return logDirPath
-}
-
func (c *Container) saveLogs() {
- testLogFilePath := c.getLogDirPath() + "container-" + c.Name + ".log"
+ testLogFilePath := c.Suite.getLogDirPath() + "container-" + c.Name + ".log"
logs, err := c.log(0)
if err != nil {
diff --git a/extras/hs-test/infra/hst_suite.go b/extras/hs-test/infra/hst_suite.go
index ff5b02e..271ef99 100644
--- a/extras/hs-test/infra/hst_suite.go
+++ b/extras/hs-test/infra/hst_suite.go
@@ -38,6 +38,7 @@
var IsDebugBuild = flag.Bool("debug_build", false, "some paths are different with debug build")
var UseCpu0 = flag.Bool("cpu0", false, "use cpu0")
var IsLeakCheck = flag.Bool("leak_check", false, "run leak-check tests")
+var ParallelTotal = flag.Lookup("ginkgo.parallel.total")
var NumaAwareCpuAlloc bool
var SuiteTimeout time.Duration
@@ -59,11 +60,39 @@
Docker *client.Client
}
+// used for colorful ReportEntry
+type StringerStruct struct {
+ Label string
+}
+
+// ColorableString for ReportEntry to use
+func (s StringerStruct) ColorableString() string {
+ return fmt.Sprintf("{{red}}%s{{/}}", s.Label)
+}
+
+// non-colorable String() is used by go's string formatting support but ignored by ReportEntry
+func (s StringerStruct) String() string {
+ return s.Label
+}
+
func getTestFilename() string {
_, filename, _, _ := runtime.Caller(2)
return filepath.Base(filename)
}
+func (s *HstSuite) getLogDirPath() string {
+ testId := s.GetTestId()
+ testName := s.GetCurrentTestName()
+ logDirPath := logDir + testName + "/" + testId + "/"
+
+ cmd := exec.Command("mkdir", "-p", logDirPath)
+ if err := cmd.Run(); err != nil {
+ Fail("mkdir error: " + fmt.Sprint(err))
+ }
+
+ return logDirPath
+}
+
func (s *HstSuite) newDockerClient() {
var err error
s.Docker, err = client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
@@ -117,6 +146,7 @@
if *IsPersistent {
return
}
+ s.WaitForCoreDump()
s.ResetContainers()
if s.Ip4AddrAllocator != nil {
@@ -183,7 +213,7 @@
out, err := container.log(20)
if err != nil {
s.Log("An error occured while obtaining '" + container.Name + "' container logs: " + fmt.Sprint(err))
- s.Log("The container might not be running - check logs in " + container.getLogDirPath())
+ s.Log("The container might not be running - check logs in " + s.getLogDirPath())
continue
}
s.Log("\nvvvvvvvvvvvvvvv " +
@@ -297,6 +327,71 @@
s.Skip("leak-check tests excluded")
}
}
+
+func (s *HstSuite) WaitForCoreDump() {
+ var filename string
+ var cmd *exec.Cmd
+ dir, err := os.Open(s.getLogDirPath())
+ if err != nil {
+ s.Log(err)
+ return
+ }
+ defer dir.Close()
+
+ files, err := dir.Readdirnames(0)
+ if err != nil {
+ s.Log(err)
+ return
+ }
+ for _, file := range files {
+ if strings.Contains(file, "core") {
+ filename = file
+ }
+ }
+ timeout := 60
+ waitTime := 5
+
+ if filename != "" {
+ filename := s.getLogDirPath() + filename
+ s.Log(fmt.Sprintf("WAITING FOR CORE DUMP (%s)", filename))
+ for i := waitTime; i <= timeout; i += waitTime {
+ fileInfo, err := os.Stat(filename)
+ if err != nil {
+ s.Log("Error while reading file info: " + fmt.Sprint(err))
+ return
+ }
+ currSize := fileInfo.Size()
+ s.Log(fmt.Sprintf("Waiting %ds/%ds...", i, timeout))
+ time.Sleep(time.Duration(waitTime) * time.Second)
+ fileInfo, _ = os.Stat(filename)
+
+ if currSize == fileInfo.Size() {
+ if *IsDebugBuild {
+ cmd = exec.Command("sudo", "gdb", "../../build-root/build-vpp_debug-native/vpp/bin/vpp",
+ "-c", filename, "-ex", "bt", "full", "-ex", "quit")
+ } else {
+ cmd = exec.Command("sudo", "gdb", "../../build-root/build-vpp-native/vpp/bin/vpp",
+ "-c", filename, "-ex", "bt", "full", "-ex", "quit")
+ }
+
+ s.Log(cmd)
+ output, _ := cmd.Output()
+ AddReportEntry("VPP Backtrace", StringerStruct{Label: string(output)})
+ os.WriteFile(s.getLogDirPath()+"backtrace.log", output, os.FileMode(0644))
+ if s.CpuAllocator.runningInCi {
+ err = os.Remove(filename)
+ if err == nil {
+ s.Log("removed " + filename)
+ } else {
+ s.Log(err)
+ }
+ }
+ return
+ }
+ }
+ }
+}
+
func (s *HstSuite) ResetContainers() {
for _, container := range s.StartedContainers {
container.stop()
diff --git a/extras/hs-test/infra/vppinstance.go b/extras/hs-test/infra/vppinstance.go
index dfb236b..a93921e 100644
--- a/extras/hs-test/infra/vppinstance.go
+++ b/extras/hs-test/infra/vppinstance.go
@@ -33,19 +33,15 @@
nodaemon
log %[1]s%[4]s
full-coredump
+ coredump-size unlimited
cli-listen %[1]s%[2]s
runtime-dir %[1]s/var/run
- gid vpp
}
api-trace {
on
}
-api-segment {
- gid vpp
-}
-
socksvr {
socket-name %[1]s%[3]s
}
@@ -479,7 +475,7 @@
}
func (vpp *VppInstance) saveLogs() {
- logTarget := vpp.Container.getLogDirPath() + "vppinstance-" + vpp.Container.Name + ".log"
+ logTarget := vpp.getSuite().getLogDirPath() + "vppinstance-" + vpp.Container.Name + ".log"
logSource := vpp.Container.GetHostWorkDir() + defaultLogFilePath
cmd := exec.Command("cp", logSource, logTarget)
vpp.getSuite().Log(cmd.String())