hs-test: pin CPUs to containers
Type: test
Change-Id: I412be2dec7ff352740e50e838e0ac466bf0a6674
Signed-off-by: Adrian Villin <avillin@cisco.com>
diff --git a/extras/hs-test/Makefile b/extras/hs-test/Makefile
index e247bf4..0db3958 100644
--- a/extras/hs-test/Makefile
+++ b/extras/hs-test/Makefile
@@ -68,7 +68,7 @@
@echo " UNCONFIGURE=[true|false] - unconfigure selected test"
@echo " DEBUG=[true|false] - attach VPP to GDB"
@echo " TEST=[test-name] - specific test to run"
- @echo " CPUS=[n-cpus] - number of cpus to run with vpp"
+ @echo " CPUS=[n-cpus] - number of cpus to allocate to VPP and containers"
@echo " VPPSRC=[path-to-vpp-src] - path to vpp source files (for gdb)"
@echo " PARALLEL=[n-cpus] - number of test processes to spawn to run in parallel"
@echo " REPEAT=[n] - repeat tests up to N times or until a failure occurs"
diff --git a/extras/hs-test/container.go b/extras/hs-test/container.go
index 0bdc3a2..83693f4 100644
--- a/extras/hs-test/container.go
+++ b/extras/hs-test/container.go
@@ -37,6 +37,7 @@
volumes map[string]Volume
envVars map[string]string
vppInstance *VppInstance
+ allocatedCpus []int
}
func newContainer(suite *HstSuite, yamlInput ContainerConfig) (*Container, error) {
@@ -160,6 +161,12 @@
return exechelper.Run(cmd)
}
+func (c *Container) allocateCpus() {
+ c.suite.containerCount += 1
+ c.allocatedCpus = c.suite.AllocateCpus()
+ c.suite.log("Allocated CPUs " + fmt.Sprint(c.allocatedCpus) + " to container " + c.name)
+}
+
func (c *Container) start() error {
cmd := "docker start " + c.name
c.suite.log(cmd)
@@ -175,6 +182,9 @@
if c.runDetached {
cmd += " -d"
}
+
+ c.allocateCpus()
+ cmd += fmt.Sprintf(" --cpuset-cpus=\"%d-%d\"", c.allocatedCpus[0], c.allocatedCpus[len(c.allocatedCpus)-1])
cmd += " " + c.getContainerArguments()
c.suite.log(cmd)
@@ -239,7 +249,6 @@
vpp := new(VppInstance)
vpp.container = c
vpp.cpus = cpus
- c.suite.vppContainerCount += 1
vpp.additionalConfig = append(vpp.additionalConfig, additionalConfigs...)
c.vppInstance = vpp
return vpp, nil
@@ -287,7 +296,7 @@
GinkgoHelper()
c.suite.log(containerExecCommand)
byteOutput, err := exechelper.CombinedOutput(containerExecCommand)
- c.suite.assertNil(err, err)
+ c.suite.assertNil(err, fmt.Sprint(err))
return string(byteOutput)
}
diff --git a/extras/hs-test/cpu.go b/extras/hs-test/cpu.go
index 69b4cab..49a7dfb 100644
--- a/extras/hs-test/cpu.go
+++ b/extras/hs-test/cpu.go
@@ -23,22 +23,26 @@
var cpuAllocator *CpuAllocatorT = nil
-func (c *CpuAllocatorT) Allocate(vppContainerCount int, nCpus int) (*CpuContext, error) {
+func (c *CpuAllocatorT) Allocate(containerCount int, nCpus int) (*CpuContext, error) {
var cpuCtx CpuContext
- maxCpu := GinkgoParallelProcess() * 2 * nCpus
- minCpu := (GinkgoParallelProcess() - 1) * 2 * nCpus
- if len(c.cpus) < maxCpu {
- vppContainerCount += 1
+
+ // splitting cpus into equal parts; this will over-allocate cores but it's good enough for now
+ maxContainerCount := 4
+ // skip CPU 0
+ minCpu := ((GinkgoParallelProcess() - 1) * maxContainerCount * nCpus) + 1
+ maxCpu := (GinkgoParallelProcess() * maxContainerCount * nCpus)
+
+ if len(c.cpus)-1 < maxCpu {
err := fmt.Errorf("could not allocate %d CPUs; available: %d; attempted to allocate cores %d-%d",
- nCpus*vppContainerCount, len(c.cpus), minCpu, minCpu+nCpus*vppContainerCount)
+ nCpus*containerCount, len(c.cpus)-1, minCpu, maxCpu)
return nil, err
}
- if vppContainerCount == 0 {
- cpuCtx.cpus = c.cpus[minCpu : maxCpu-nCpus]
- } else if vppContainerCount == 1 {
- cpuCtx.cpus = c.cpus[minCpu+nCpus : maxCpu]
+ if containerCount == 1 {
+ cpuCtx.cpus = c.cpus[minCpu : minCpu+nCpus]
+ } else if containerCount > 1 && containerCount <= maxContainerCount {
+ cpuCtx.cpus = c.cpus[minCpu+(nCpus*(containerCount-1)) : minCpu+(nCpus*containerCount)]
} else {
- return nil, fmt.Errorf("too many VPP containers; CPU allocation for >2 VPP containers is not implemented yet")
+ return nil, fmt.Errorf("too many containers; CPU allocation for >%d containers is not implemented", maxContainerCount)
}
cpuCtx.cpuAllocator = c
diff --git a/extras/hs-test/hst_suite.go b/extras/hs-test/hst_suite.go
index 9cb79c5..b8c0a42 100644
--- a/extras/hs-test/hst_suite.go
+++ b/extras/hs-test/hst_suite.go
@@ -30,19 +30,19 @@
var vppSourceFileDir = flag.String("vppsrc", "", "vpp source file directory")
type HstSuite struct {
- containers map[string]*Container
- vppContainerCount int
- volumes []string
- netConfigs []NetConfig
- netInterfaces map[string]*NetInterface
- ip4AddrAllocator *Ip4AddressAllocator
- testIds map[string]string
- cpuAllocator *CpuAllocatorT
- cpuContexts []*CpuContext
- cpuPerVpp int
- pid string
- logger *log.Logger
- logFile *os.File
+ containers map[string]*Container
+ containerCount int
+ volumes []string
+ netConfigs []NetConfig
+ netInterfaces map[string]*NetInterface
+ ip4AddrAllocator *Ip4AddressAllocator
+ testIds map[string]string
+ cpuAllocator *CpuAllocatorT
+ cpuContexts []*CpuContext
+ cpuPerVpp int
+ pid string
+ logger *log.Logger
+ logFile *os.File
}
func (s *HstSuite) SetupSuite() {
@@ -62,7 +62,7 @@
}
func (s *HstSuite) AllocateCpus() []int {
- cpuCtx, err := s.cpuAllocator.Allocate(s.vppContainerCount, s.cpuPerVpp)
+ cpuCtx, err := s.cpuAllocator.Allocate(s.containerCount, s.cpuPerVpp)
s.assertNil(err)
s.AddCpuContext(cpuCtx)
return cpuCtx.cpus
@@ -96,7 +96,7 @@
func (s *HstSuite) SetupTest() {
s.log("Test Setup")
- s.vppContainerCount = 0
+ s.containerCount = 0
s.skipIfUnconfiguring()
s.setupVolumes()
s.setupContainers()
diff --git a/extras/hs-test/suite_nginx_test.go b/extras/hs-test/suite_nginx_test.go
index c559496..ef4ca5c 100644
--- a/extras/hs-test/suite_nginx_test.go
+++ b/extras/hs-test/suite_nginx_test.go
@@ -48,10 +48,9 @@
append("enable").
append("use-app-socket-api").close()
- cpus := s.AllocateCpus()
// ... for proxy
vppProxyContainer := s.getContainerByName(vppProxyContainerName)
- proxyVpp, _ := vppProxyContainer.newVppInstance(cpus, sessionConfig)
+ proxyVpp, _ := vppProxyContainer.newVppInstance(vppProxyContainer.allocatedCpus, sessionConfig)
s.assertNil(proxyVpp.start())
clientInterface := s.getInterfaceByName(mirroringClientInterfaceName)
diff --git a/extras/hs-test/suite_no_topo_test.go b/extras/hs-test/suite_no_topo_test.go
index 625dca9..19a3e1f 100644
--- a/extras/hs-test/suite_no_topo_test.go
+++ b/extras/hs-test/suite_no_topo_test.go
@@ -45,9 +45,8 @@
append("enable").
append("use-app-socket-api").close()
- cpus := s.AllocateCpus()
container := s.getContainerByName(singleTopoContainerVpp)
- vpp, _ := container.newVppInstance(cpus, sessionConfig)
+ vpp, _ := container.newVppInstance(container.allocatedCpus, sessionConfig)
s.assertNil(vpp.start())
tapInterface := s.getInterfaceByName(tapInterfaceName)
diff --git a/extras/hs-test/suite_ns_test.go b/extras/hs-test/suite_ns_test.go
index 85b9091..377046c 100644
--- a/extras/hs-test/suite_ns_test.go
+++ b/extras/hs-test/suite_ns_test.go
@@ -48,9 +48,8 @@
append("evt_qs_memfd_seg").
append("event-queue-length 100000").close()
- cpus := s.AllocateCpus()
container := s.getContainerByName("vpp")
- vpp, _ := container.newVppInstance(cpus, sessionConfig)
+ vpp, _ := container.newVppInstance(container.allocatedCpus, sessionConfig)
s.assertNil(vpp.start())
idx, err := vpp.createAfPacket(s.getInterfaceByName(serverInterface))
diff --git a/extras/hs-test/suite_veth_test.go b/extras/hs-test/suite_veth_test.go
index d47bf8c..c3038ca 100644
--- a/extras/hs-test/suite_veth_test.go
+++ b/extras/hs-test/suite_veth_test.go
@@ -50,8 +50,7 @@
// ... For server
serverContainer := s.getContainerByName("server-vpp")
- cpus := s.AllocateCpus()
- serverVpp, err := serverContainer.newVppInstance(cpus, sessionConfig)
+ serverVpp, err := serverContainer.newVppInstance(serverContainer.allocatedCpus, sessionConfig)
s.assertNotNil(serverVpp, fmt.Sprint(err))
s.setupServerVpp()
@@ -59,8 +58,7 @@
// ... For client
clientContainer := s.getContainerByName("client-vpp")
- cpus = s.AllocateCpus()
- clientVpp, err := clientContainer.newVppInstance(cpus, sessionConfig)
+ clientVpp, err := clientContainer.newVppInstance(clientContainer.allocatedCpus, sessionConfig)
s.assertNotNil(clientVpp, fmt.Sprint(err))
s.setupClientVpp()