blob: 3ac5eee3aaa0a976d66a703d30a1df4f5d31ceb4 [file] [log] [blame]
Maros Ondrejicka11a03e92022-12-01 09:56:37 +01001package main
2
3import (
4 "fmt"
Maros Ondrejickadb823ed2022-12-14 16:30:04 +01005 "os"
Maros Ondrejickaffa3f602023-01-26 10:07:29 +01006 "os/exec"
Maros Ondrejickadb823ed2022-12-14 16:30:04 +01007 "strings"
Maros Ondrejicka85396a52023-02-28 12:49:43 +01008 "text/template"
Filip Tehlar4fa0ba62023-12-06 11:35:11 +01009 "time"
Maros Ondrejicka11a03e92022-12-01 09:56:37 +010010
11 "github.com/edwarnicke/exechelper"
Adrian Villincee15aa2024-03-14 11:42:55 -040012 . "github.com/onsi/ginkgo/v2"
Maros Ondrejicka11a03e92022-12-01 09:56:37 +010013)
14
Maros Ondrejickaa2d52622023-02-24 11:26:39 +010015const (
Filip Tehlara1bd50c2024-01-24 11:59:44 +010016 logDir string = "/tmp/hs-test/"
17 volumeDir string = "/volumes"
Maros Ondrejickaa2d52622023-02-24 11:26:39 +010018)
19
Maros Ondrejicka7550dd22023-02-07 20:40:27 +010020var (
21 workDir, _ = os.Getwd()
22)
23
Maros Ondrejicka0e79abb2022-12-06 19:46:24 +010024type Volume struct {
Maros Ondrejickaffa3f602023-01-26 10:07:29 +010025 hostDir string
26 containerDir string
27 isDefaultWorkDir bool
Maros Ondrejicka0e79abb2022-12-06 19:46:24 +010028}
29
30type Container struct {
Maros Ondrejickaffa3f602023-01-26 10:07:29 +010031 suite *HstSuite
Filip Tehlar3f951432023-01-13 21:33:43 +010032 isOptional bool
Filip Tehlarb41b0af2023-03-20 12:39:20 +010033 runDetached bool
Filip Tehlar3f951432023-01-13 21:33:43 +010034 name string
35 image string
Filip Tehlar3f951432023-01-13 21:33:43 +010036 extraRunningArgs string
37 volumes map[string]Volume
38 envVars map[string]string
Maros Ondrejickaffa3f602023-01-26 10:07:29 +010039 vppInstance *VppInstance
Maros Ondrejickadb823ed2022-12-14 16:30:04 +010040}
41
Adrian Villincee15aa2024-03-14 11:42:55 -040042func newContainer(suite *HstSuite, yamlInput ContainerConfig) (*Container, error) {
Maros Ondrejickadb823ed2022-12-14 16:30:04 +010043 containerName := yamlInput["name"].(string)
44 if len(containerName) == 0 {
45 err := fmt.Errorf("container name must not be blank")
46 return nil, err
47 }
48
49 var container = new(Container)
50 container.volumes = make(map[string]Volume)
51 container.envVars = make(map[string]string)
Adrian Villincee15aa2024-03-14 11:42:55 -040052 container.name = containerName
Filip Tehlara1bd50c2024-01-24 11:59:44 +010053 container.suite = suite
Maros Ondrejickadb823ed2022-12-14 16:30:04 +010054
55 if image, ok := yamlInput["image"]; ok {
56 container.image = image.(string)
57 } else {
58 container.image = "hs-test/vpp"
59 }
60
Filip Tehlar3f951432023-01-13 21:33:43 +010061 if args, ok := yamlInput["extra-args"]; ok {
62 container.extraRunningArgs = args.(string)
63 } else {
64 container.extraRunningArgs = ""
65 }
66
Maros Ondrejickadb823ed2022-12-14 16:30:04 +010067 if isOptional, ok := yamlInput["is-optional"]; ok {
68 container.isOptional = isOptional.(bool)
69 } else {
70 container.isOptional = false
71 }
72
Filip Tehlarb41b0af2023-03-20 12:39:20 +010073 if runDetached, ok := yamlInput["run-detached"]; ok {
74 container.runDetached = runDetached.(bool)
75 } else {
76 container.runDetached = true
77 }
78
Maros Ondrejickadb823ed2022-12-14 16:30:04 +010079 if _, ok := yamlInput["volumes"]; ok {
Adrian Villin637edda2024-05-06 06:55:34 -040080 workingVolumeDir := logDir + CurrentSpecReport().LeafNodeText + volumeDir
Filip Tehlara1bd50c2024-01-24 11:59:44 +010081 workDirReplacer := strings.NewReplacer("$HST_DIR", workDir)
82 volDirReplacer := strings.NewReplacer("$HST_VOLUME_DIR", workingVolumeDir)
Maros Ondrejickadb823ed2022-12-14 16:30:04 +010083 for _, volu := range yamlInput["volumes"].([]interface{}) {
84 volumeMap := volu.(ContainerConfig)
Filip Tehlara1bd50c2024-01-24 11:59:44 +010085 hostDir := workDirReplacer.Replace(volumeMap["host-dir"].(string))
86 hostDir = volDirReplacer.Replace(hostDir)
Maros Ondrejickadb823ed2022-12-14 16:30:04 +010087 containerDir := volumeMap["container-dir"].(string)
Maros Ondrejickaffa3f602023-01-26 10:07:29 +010088 isDefaultWorkDir := false
Maros Ondrejickadb823ed2022-12-14 16:30:04 +010089
Maros Ondrejickaffa3f602023-01-26 10:07:29 +010090 if isDefault, ok := volumeMap["is-default-work-dir"]; ok {
91 isDefaultWorkDir = isDefault.(bool)
Maros Ondrejickadb823ed2022-12-14 16:30:04 +010092 }
Maros Ondrejickaffa3f602023-01-26 10:07:29 +010093 container.addVolume(hostDir, containerDir, isDefaultWorkDir)
Maros Ondrejickadb823ed2022-12-14 16:30:04 +010094 }
95 }
96
97 if _, ok := yamlInput["vars"]; ok {
98 for _, envVar := range yamlInput["vars"].([]interface{}) {
Maros Ondrejickaffa3f602023-01-26 10:07:29 +010099 envVarMap := envVar.(ContainerConfig)
100 name := envVarMap["name"].(string)
101 value := envVarMap["value"].(string)
102 container.addEnvVar(name, value)
Maros Ondrejickadb823ed2022-12-14 16:30:04 +0100103 }
104 }
105 return container, nil
Maros Ondrejicka11a03e92022-12-01 09:56:37 +0100106}
107
Maros Ondrejickaffa3f602023-01-26 10:07:29 +0100108func (c *Container) getWorkDirVolume() (res Volume, exists bool) {
109 for _, v := range c.volumes {
110 if v.isDefaultWorkDir {
111 res = v
112 exists = true
113 return
114 }
115 }
116 return
117}
118
Maros Ondrejickae7625d02023-02-28 16:55:01 +0100119func (c *Container) getHostWorkDir() (res string) {
Maros Ondrejickaffa3f602023-01-26 10:07:29 +0100120 if v, ok := c.getWorkDirVolume(); ok {
121 res = v.hostDir
122 }
123 return
124}
125
Maros Ondrejickae7625d02023-02-28 16:55:01 +0100126func (c *Container) getContainerWorkDir() (res string) {
Maros Ondrejickaffa3f602023-01-26 10:07:29 +0100127 if v, ok := c.getWorkDirVolume(); ok {
128 res = v.containerDir
129 }
130 return
131}
132
Maros Ondrejickac2f76f42023-02-27 13:22:45 +0100133func (c *Container) getContainerArguments() string {
Filip Tehlarb41b0af2023-03-20 12:39:20 +0100134 args := "--ulimit nofile=90000:90000 --cap-add=all --privileged --network host --rm"
Maros Ondrejickac2f76f42023-02-27 13:22:45 +0100135 args += c.getVolumesAsCliOption()
136 args += c.getEnvVarsAsCliOption()
Filip Tehlar109f3ce2023-09-05 15:36:28 +0200137 if *vppSourceFileDir != "" {
138 args += fmt.Sprintf(" -v %s:%s", *vppSourceFileDir, *vppSourceFileDir)
139 }
Maros Ondrejickac2f76f42023-02-27 13:22:45 +0100140 args += " --name " + c.name + " " + c.image
Filip Tehlar9abba112023-03-07 10:13:19 +0100141 args += " " + c.extraRunningArgs
Maros Ondrejickac2f76f42023-02-27 13:22:45 +0100142 return args
143}
144
Filip Tehlar4fa0ba62023-12-06 11:35:11 +0100145func (c *Container) runWithRetry(cmd string) error {
146 nTries := 5
147 for i := 0; i < nTries; i++ {
148 err := exechelper.Run(cmd)
149 if err == nil {
150 return nil
151 }
152 time.Sleep(1 * time.Second)
153 }
154 return fmt.Errorf("failed to run container command")
155}
156
Filip Tehlar9abba112023-03-07 10:13:19 +0100157func (c *Container) create() error {
Maros Ondrejickac2f76f42023-02-27 13:22:45 +0100158 cmd := "docker create " + c.getContainerArguments()
Maros Ondrejickae7625d02023-02-28 16:55:01 +0100159 c.suite.log(cmd)
Filip Tehlar9abba112023-03-07 10:13:19 +0100160 return exechelper.Run(cmd)
Maros Ondrejickac2f76f42023-02-27 13:22:45 +0100161}
162
Filip Tehlar9abba112023-03-07 10:13:19 +0100163func (c *Container) start() error {
Maros Ondrejickac2f76f42023-02-27 13:22:45 +0100164 cmd := "docker start " + c.name
Maros Ondrejickae7625d02023-02-28 16:55:01 +0100165 c.suite.log(cmd)
Filip Tehlar4fa0ba62023-12-06 11:35:11 +0100166 return c.runWithRetry(cmd)
Maros Ondrejicka87531802022-12-19 20:35:27 +0100167}
168
Filip Tehlarb41b0af2023-03-20 12:39:20 +0100169func (c *Container) prepareCommand() (string, error) {
Maros Ondrejicka87531802022-12-19 20:35:27 +0100170 if c.name == "" {
Filip Tehlarb41b0af2023-03-20 12:39:20 +0100171 return "", fmt.Errorf("run container failed: name is blank")
Maros Ondrejicka87531802022-12-19 20:35:27 +0100172 }
173
Filip Tehlarb41b0af2023-03-20 12:39:20 +0100174 cmd := "docker run "
175 if c.runDetached {
176 cmd += " -d"
177 }
178 cmd += " " + c.getContainerArguments()
179
Maros Ondrejickae7625d02023-02-28 16:55:01 +0100180 c.suite.log(cmd)
Filip Tehlarb41b0af2023-03-20 12:39:20 +0100181 return cmd, nil
182}
183
184func (c *Container) combinedOutput() (string, error) {
185 cmd, err := c.prepareCommand()
Maros Ondrejicka11a03e92022-12-01 09:56:37 +0100186 if err != nil {
Filip Tehlarb41b0af2023-03-20 12:39:20 +0100187 return "", err
Maros Ondrejicka11a03e92022-12-01 09:56:37 +0100188 }
189
Filip Tehlarb41b0af2023-03-20 12:39:20 +0100190 byteOutput, err := exechelper.CombinedOutput(cmd)
191 return string(byteOutput), err
192}
193
194func (c *Container) run() error {
195 cmd, err := c.prepareCommand()
196 if err != nil {
197 return err
198 }
Filip Tehlar4fa0ba62023-12-06 11:35:11 +0100199 return c.runWithRetry(cmd)
Maros Ondrejicka11a03e92022-12-01 09:56:37 +0100200}
201
Maros Ondrejickaffa3f602023-01-26 10:07:29 +0100202func (c *Container) addVolume(hostDir string, containerDir string, isDefaultWorkDir bool) {
Maros Ondrejickadb823ed2022-12-14 16:30:04 +0100203 var volume Volume
204 volume.hostDir = hostDir
205 volume.containerDir = containerDir
Maros Ondrejickaffa3f602023-01-26 10:07:29 +0100206 volume.isDefaultWorkDir = isDefaultWorkDir
Maros Ondrejickadb823ed2022-12-14 16:30:04 +0100207 c.volumes[hostDir] = volume
Maros Ondrejicka0e79abb2022-12-06 19:46:24 +0100208}
209
Maros Ondrejickadb823ed2022-12-14 16:30:04 +0100210func (c *Container) getVolumesAsCliOption() string {
211 cliOption := ""
Maros Ondrejicka0e79abb2022-12-06 19:46:24 +0100212
213 if len(c.volumes) > 0 {
214 for _, volume := range c.volumes {
Maros Ondrejickadb823ed2022-12-14 16:30:04 +0100215 cliOption += fmt.Sprintf(" -v %s:%s", volume.hostDir, volume.containerDir)
Maros Ondrejicka0e79abb2022-12-06 19:46:24 +0100216 }
217 }
218
Maros Ondrejickadb823ed2022-12-14 16:30:04 +0100219 return cliOption
220}
221
Maros Ondrejickaffa3f602023-01-26 10:07:29 +0100222func (c *Container) addEnvVar(name string, value string) {
Maros Ondrejickadb823ed2022-12-14 16:30:04 +0100223 c.envVars[name] = value
224}
225
226func (c *Container) getEnvVarsAsCliOption() string {
227 cliOption := ""
228 if len(c.envVars) == 0 {
229 return cliOption
230 }
231
232 for name, value := range c.envVars {
233 cliOption += fmt.Sprintf(" -e %s=%s", name, value)
234 }
235 return cliOption
236}
237
Filip Tehlar608d0062023-04-28 10:29:47 +0200238func (c *Container) newVppInstance(cpus []int, additionalConfigs ...Stanza) (*VppInstance, error) {
Maros Ondrejickaffa3f602023-01-26 10:07:29 +0100239 vpp := new(VppInstance)
240 vpp.container = c
Filip Tehlar608d0062023-04-28 10:29:47 +0200241 vpp.cpus = cpus
242 vpp.additionalConfig = append(vpp.additionalConfig, additionalConfigs...)
Maros Ondrejickaffa3f602023-01-26 10:07:29 +0100243 c.vppInstance = vpp
Maros Ondrejickaffa3f602023-01-26 10:07:29 +0100244 return vpp, nil
245}
246
247func (c *Container) copy(sourceFileName string, targetFileName string) error {
248 cmd := exec.Command("docker", "cp", sourceFileName, c.name+":"+targetFileName)
249 return cmd.Run()
250}
251
252func (c *Container) createFile(destFileName string, content string) error {
Adrian Villincee15aa2024-03-14 11:42:55 -0400253 f, err := os.CreateTemp("/tmp", "hst-config"+c.suite.pid)
Maros Ondrejickaffa3f602023-01-26 10:07:29 +0100254 if err != nil {
255 return err
256 }
257 defer os.Remove(f.Name())
258
259 if _, err := f.Write([]byte(content)); err != nil {
260 return err
261 }
262 if err := f.Close(); err != nil {
263 return err
264 }
265 c.copy(f.Name(), destFileName)
266 return nil
267}
268
269/*
270 * Executes in detached mode so that the started application can continue to run
271 * without blocking execution of test
272 */
Maros Ondrejicka2908f8c2023-02-02 08:58:04 +0100273func (c *Container) execServer(command string, arguments ...any) {
274 serverCommand := fmt.Sprintf(command, arguments...)
275 containerExecCommand := "docker exec -d" + c.getEnvVarsAsCliOption() +
276 " " + c.name + " " + serverCommand
Adrian Villincee15aa2024-03-14 11:42:55 -0400277 GinkgoHelper()
Maros Ondrejickae7625d02023-02-28 16:55:01 +0100278 c.suite.log(containerExecCommand)
279 c.suite.assertNil(exechelper.Run(containerExecCommand))
Maros Ondrejickaffa3f602023-01-26 10:07:29 +0100280}
281
Maros Ondrejicka2908f8c2023-02-02 08:58:04 +0100282func (c *Container) exec(command string, arguments ...any) string {
283 cliCommand := fmt.Sprintf(command, arguments...)
284 containerExecCommand := "docker exec" + c.getEnvVarsAsCliOption() +
285 " " + c.name + " " + cliCommand
Adrian Villincee15aa2024-03-14 11:42:55 -0400286 GinkgoHelper()
Maros Ondrejickae7625d02023-02-28 16:55:01 +0100287 c.suite.log(containerExecCommand)
Maros Ondrejicka2908f8c2023-02-02 08:58:04 +0100288 byteOutput, err := exechelper.CombinedOutput(containerExecCommand)
adrianvillin7c675472024-02-12 02:44:53 -0500289 c.suite.assertNil(err, err)
Maros Ondrejicka2908f8c2023-02-02 08:58:04 +0100290 return string(byteOutput)
Maros Ondrejickadb823ed2022-12-14 16:30:04 +0100291}
292
Maros Ondrejickaa2d52622023-02-24 11:26:39 +0100293func (c *Container) getLogDirPath() string {
Maros Ondrejickae7625d02023-02-28 16:55:01 +0100294 testId := c.suite.getTestId()
Adrian Villincee15aa2024-03-14 11:42:55 -0400295 testName := CurrentSpecReport().LeafNodeText
Maros Ondrejickaa2d52622023-02-24 11:26:39 +0100296 logDirPath := logDir + testName + "/" + testId + "/"
297
298 cmd := exec.Command("mkdir", "-p", logDirPath)
299 if err := cmd.Run(); err != nil {
Adrian Villincee15aa2024-03-14 11:42:55 -0400300 Fail("mkdir error: " + fmt.Sprint(err))
Maros Ondrejickaa2d52622023-02-24 11:26:39 +0100301 }
302
303 return logDirPath
304}
305
306func (c *Container) saveLogs() {
307 cmd := exec.Command("docker", "inspect", "--format='{{.State.Status}}'", c.name)
308 if output, _ := cmd.CombinedOutput(); !strings.Contains(string(output), "running") {
309 return
310 }
311
312 testLogFilePath := c.getLogDirPath() + "container-" + c.name + ".log"
313
314 cmd = exec.Command("docker", "logs", "--details", "-t", c.name)
315 output, err := cmd.CombinedOutput()
316 if err != nil {
Adrian Villincee15aa2024-03-14 11:42:55 -0400317 Fail("fetching logs error: " + fmt.Sprint(err))
Maros Ondrejickaa2d52622023-02-24 11:26:39 +0100318 }
319
320 f, err := os.Create(testLogFilePath)
321 if err != nil {
Adrian Villincee15aa2024-03-14 11:42:55 -0400322 Fail("file create error: " + fmt.Sprint(err))
Maros Ondrejickaa2d52622023-02-24 11:26:39 +0100323 }
Maros Ondrejickae7625d02023-02-28 16:55:01 +0100324 fmt.Fprint(f, string(output))
Maros Ondrejickaa2d52622023-02-24 11:26:39 +0100325 f.Close()
326}
327
adrianvillin7c675472024-02-12 02:44:53 -0500328// Outputs logs from docker containers. Set 'maxLines' to 0 to output the full log.
329func (c *Container) log(maxLines int) (string, error) {
330 var cmd string
331 if maxLines == 0 {
332 cmd = "docker logs " + c.name
333 } else {
334 cmd = fmt.Sprintf("docker logs --tail %d %s", maxLines, c.name)
335 }
336
Maros Ondrejickae7625d02023-02-28 16:55:01 +0100337 c.suite.log(cmd)
Maros Ondrejickac2f76f42023-02-27 13:22:45 +0100338 o, err := exechelper.CombinedOutput(cmd)
adrianvillin7c675472024-02-12 02:44:53 -0500339 return string(o), err
Maros Ondrejickac2f76f42023-02-27 13:22:45 +0100340}
341
Maros Ondrejicka0e79abb2022-12-06 19:46:24 +0100342func (c *Container) stop() error {
Adrian Villin46d66002024-05-15 04:33:41 -0400343 if c.vppInstance != nil && c.vppInstance.apiStream != nil {
Maros Ondrejickaa2d52622023-02-24 11:26:39 +0100344 c.vppInstance.saveLogs()
Maros Ondrejickaffa3f602023-01-26 10:07:29 +0100345 c.vppInstance.disconnect()
346 }
347 c.vppInstance = nil
Maros Ondrejickaa2d52622023-02-24 11:26:39 +0100348 c.saveLogs()
Florin Corasf4fe0162023-01-17 13:02:51 -0800349 return exechelper.Run("docker stop " + c.name + " -t 0")
Maros Ondrejicka0e79abb2022-12-06 19:46:24 +0100350}
Maros Ondrejicka85396a52023-02-28 12:49:43 +0100351
352func (c *Container) createConfig(targetConfigName string, templateName string, values any) {
353 template := template.Must(template.ParseFiles(templateName))
354
Filip Tehlara1bd50c2024-01-24 11:59:44 +0100355 f, err := os.CreateTemp(logDir, "hst-config")
adrianvillin7c675472024-02-12 02:44:53 -0500356 c.suite.assertNil(err, err)
Maros Ondrejicka85396a52023-02-28 12:49:43 +0100357 defer os.Remove(f.Name())
358
359 err = template.Execute(f, values)
adrianvillin7c675472024-02-12 02:44:53 -0500360 c.suite.assertNil(err, err)
Maros Ondrejicka85396a52023-02-28 12:49:43 +0100361
362 err = f.Close()
adrianvillin7c675472024-02-12 02:44:53 -0500363 c.suite.assertNil(err, err)
Maros Ondrejicka85396a52023-02-28 12:49:43 +0100364
365 c.copy(f.Name(), targetConfigName)
366}
Maros Ondrejickaf4ddf162023-03-08 16:01:43 +0100367
368func init() {
369 cmd := exec.Command("mkdir", "-p", logDir)
370 if err := cmd.Run(); err != nil {
371 panic(err)
372 }
373}