John DeNisco | 68b0ee3 | 2017-09-27 16:35:23 -0400 | [diff] [blame] | 1 | # Copyright (c) 2016 Cisco and/or its affiliates. |
| 2 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 3 | # you may not use this file except in compliance with the License. |
| 4 | # You may obtain a copy of the License at: |
| 5 | # |
| 6 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 7 | # |
| 8 | # Unless required by applicable law or agreed to in writing, software |
| 9 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 11 | # See the License for the specific language governing permissions and |
| 12 | # limitations under the License. |
| 13 | |
| 14 | """QEMU utilities library.""" |
| 15 | |
| 16 | from time import time, sleep |
| 17 | import json |
| 18 | import logging |
| 19 | |
| 20 | from vpplib.VPPUtil import VPPUtil |
| 21 | from vpplib.constants import Constants |
| 22 | |
| 23 | |
| 24 | class NodeType(object): |
| 25 | """Defines node types used in topology dictionaries.""" |
| 26 | # Device Under Test (this node has VPP running on it) |
| 27 | DUT = 'DUT' |
| 28 | # Traffic Generator (this node has traffic generator on it) |
| 29 | TG = 'TG' |
| 30 | # Virtual Machine (this node running on DUT node) |
| 31 | VM = 'VM' |
| 32 | |
| 33 | |
| 34 | class QemuUtils(object): |
| 35 | """QEMU utilities.""" |
| 36 | |
John DeNisco | a7da67f | 2018-01-26 14:55:33 -0500 | [diff] [blame] | 37 | # noinspection PyDictCreation |
John DeNisco | 68b0ee3 | 2017-09-27 16:35:23 -0400 | [diff] [blame] | 38 | def __init__(self, qemu_id=1): |
| 39 | self._qemu_id = qemu_id |
| 40 | # Path to QEMU binary |
| 41 | self._qemu_bin = '/usr/bin/qemu-system-x86_64' |
| 42 | # QEMU Machine Protocol socket |
| 43 | self._qmp_sock = '/tmp/qmp{0}.sock'.format(self._qemu_id) |
| 44 | # QEMU Guest Agent socket |
| 45 | self._qga_sock = '/tmp/qga{0}.sock'.format(self._qemu_id) |
| 46 | # QEMU PID file |
| 47 | self._pid_file = '/tmp/qemu{0}.pid'.format(self._qemu_id) |
| 48 | self._qemu_opt = {} |
| 49 | # Default 1 CPU. |
| 50 | self._qemu_opt['smp'] = '-smp 1,sockets=1,cores=1,threads=1' |
| 51 | # Daemonize the QEMU process after initialization. Default one |
| 52 | # management interface. |
| 53 | self._qemu_opt['options'] = '-cpu host -daemonize -enable-kvm ' \ |
| 54 | '-machine pc,accel=kvm,usb=off,mem-merge=off ' \ |
| 55 | '-net nic,macaddr=52:54:00:00:{0:02x}:ff -balloon none'\ |
| 56 | .format(self._qemu_id) |
| 57 | self._qemu_opt['ssh_fwd_port'] = 10021 + qemu_id |
| 58 | # Default serial console port |
| 59 | self._qemu_opt['serial_port'] = 4555 + qemu_id |
| 60 | # Default 512MB virtual RAM |
| 61 | self._qemu_opt['mem_size'] = 512 |
| 62 | # Default huge page mount point, required for Vhost-user interfaces. |
| 63 | self._qemu_opt['huge_mnt'] = '/mnt/huge' |
| 64 | # Default do not allocate huge pages. |
| 65 | self._qemu_opt['huge_allocate'] = False |
| 66 | # Default image for CSIT virl setup |
| 67 | self._qemu_opt['disk_image'] = '/var/lib/vm/vhost-nested.img' |
| 68 | # VM node info dict |
| 69 | self._vm_info = { |
| 70 | 'type': NodeType.VM, |
| 71 | 'port': self._qemu_opt['ssh_fwd_port'], |
| 72 | 'username': 'cisco', |
| 73 | 'password': 'cisco', |
| 74 | 'interfaces': {}, |
| 75 | } |
| 76 | # Virtio queue count |
| 77 | self._qemu_opt['queues'] = 1 |
| 78 | self._vhost_id = 0 |
| 79 | self._ssh = None |
| 80 | self._node = None |
| 81 | self._socks = [self._qmp_sock, self._qga_sock] |
| 82 | |
| 83 | def qemu_set_bin(self, path): |
| 84 | """Set binary path for QEMU. |
| 85 | |
| 86 | :param path: Absolute path in filesystem. |
| 87 | :type path: str |
| 88 | """ |
| 89 | self._qemu_bin = path |
| 90 | |
| 91 | def qemu_set_smp(self, cpus, cores, threads, sockets): |
| 92 | """Set SMP option for QEMU. |
| 93 | |
| 94 | :param cpus: Number of CPUs. |
| 95 | :param cores: Number of CPU cores on one socket. |
| 96 | :param threads: Number of threads on one CPU core. |
| 97 | :param sockets: Number of discrete sockets in the system. |
| 98 | :type cpus: int |
| 99 | :type cores: int |
| 100 | :type threads: int |
| 101 | :type sockets: int |
| 102 | """ |
| 103 | self._qemu_opt['smp'] = '-smp {},cores={},threads={},sockets={}'.format( |
| 104 | cpus, cores, threads, sockets) |
| 105 | |
| 106 | def qemu_set_ssh_fwd_port(self, fwd_port): |
| 107 | """Set host port for guest SSH forwarding. |
| 108 | |
| 109 | :param fwd_port: Port number on host for guest SSH forwarding. |
| 110 | :type fwd_port: int |
| 111 | """ |
| 112 | self._qemu_opt['ssh_fwd_port'] = fwd_port |
| 113 | self._vm_info['port'] = fwd_port |
| 114 | |
| 115 | def qemu_set_serial_port(self, port): |
| 116 | """Set serial console port. |
| 117 | |
| 118 | :param port: Serial console port. |
| 119 | :type port: int |
| 120 | """ |
| 121 | self._qemu_opt['serial_port'] = port |
| 122 | |
| 123 | def qemu_set_mem_size(self, mem_size): |
| 124 | """Set virtual RAM size. |
| 125 | |
| 126 | :param mem_size: RAM size in Mega Bytes. |
| 127 | :type mem_size: int |
| 128 | """ |
| 129 | self._qemu_opt['mem_size'] = int(mem_size) |
| 130 | |
| 131 | def qemu_set_huge_mnt(self, huge_mnt): |
| 132 | """Set hugefile mount point. |
| 133 | |
| 134 | :param huge_mnt: System hugefile mount point. |
| 135 | :type huge_mnt: int |
| 136 | """ |
| 137 | self._qemu_opt['huge_mnt'] = huge_mnt |
| 138 | |
| 139 | def qemu_set_huge_allocate(self): |
| 140 | """Set flag to allocate more huge pages if needed.""" |
| 141 | self._qemu_opt['huge_allocate'] = True |
| 142 | |
| 143 | def qemu_set_disk_image(self, disk_image): |
| 144 | """Set disk image. |
| 145 | |
| 146 | :param disk_image: Path of the disk image. |
| 147 | :type disk_image: str |
| 148 | """ |
| 149 | self._qemu_opt['disk_image'] = disk_image |
| 150 | |
| 151 | def qemu_set_affinity(self, *host_cpus): |
| 152 | """Set qemu affinity by getting thread PIDs via QMP and taskset to list |
| 153 | of CPU cores. |
| 154 | |
| 155 | :param host_cpus: List of CPU cores. |
| 156 | :type host_cpus: list |
| 157 | """ |
| 158 | qemu_cpus = self._qemu_qmp_exec('query-cpus')['return'] |
| 159 | |
| 160 | if len(qemu_cpus) != len(host_cpus): |
| 161 | logging.debug('Host CPU count {0}, Qemu Thread count {1}'.format( |
| 162 | len(host_cpus), len(qemu_cpus))) |
| 163 | raise ValueError('Host CPU count must match Qemu Thread count') |
| 164 | |
| 165 | for qemu_cpu, host_cpu in zip(qemu_cpus, host_cpus): |
| 166 | cmd = 'taskset -pc {0} {1}'.format(host_cpu, qemu_cpu['thread_id']) |
| 167 | (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd) |
| 168 | if int(ret_code) != 0: |
| 169 | logging.debug('Set affinity failed {0}'.format(stderr)) |
| 170 | raise RuntimeError('Set affinity failed on {0}'.format( |
| 171 | self._node['host'])) |
| 172 | |
| 173 | def qemu_set_scheduler_policy(self): |
| 174 | """Set scheduler policy to SCHED_RR with priority 1 for all Qemu CPU |
| 175 | processes. |
| 176 | |
| 177 | :raises RuntimeError: Set scheduler policy failed. |
| 178 | """ |
| 179 | qemu_cpus = self._qemu_qmp_exec('query-cpus')['return'] |
| 180 | |
| 181 | for qemu_cpu in qemu_cpus: |
| 182 | cmd = 'chrt -r -p 1 {0}'.format(qemu_cpu['thread_id']) |
| 183 | (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd) |
| 184 | if int(ret_code) != 0: |
| 185 | logging.debug('Set SCHED_RR failed {0}'.format(stderr)) |
| 186 | raise RuntimeError('Set SCHED_RR failed on {0}'.format( |
| 187 | self._node['host'])) |
| 188 | |
| 189 | def qemu_set_node(self, node): |
| 190 | """Set node to run QEMU on. |
| 191 | |
| 192 | :param node: Node to run QEMU on. |
| 193 | :type node: dict |
| 194 | """ |
| 195 | self._node = node |
| 196 | self._vm_info['host'] = node['host'] |
| 197 | |
| 198 | def qemu_add_vhost_user_if(self, socket, server=True, mac=None): |
| 199 | """Add Vhost-user interface. |
| 200 | |
| 201 | :param socket: Path of the unix socket. |
| 202 | :param server: If True the socket shall be a listening socket. |
| 203 | :param mac: Vhost-user interface MAC address (optional, otherwise is |
| 204 | used auto-generated MAC 52:54:00:00:xx:yy). |
| 205 | :type socket: str |
| 206 | :type server: bool |
| 207 | :type mac: str |
| 208 | """ |
| 209 | self._vhost_id += 1 |
| 210 | # Create unix socket character device. |
| 211 | chardev = ' -chardev socket,id=char{0},path={1}'.format(self._vhost_id, |
| 212 | socket) |
| 213 | if server is True: |
| 214 | chardev += ',server' |
| 215 | self._qemu_opt['options'] += chardev |
| 216 | # Create Vhost-user network backend. |
| 217 | netdev = (' -netdev vhost-user,id=vhost{0},chardev=char{0},queues={1}' |
| 218 | .format(self._vhost_id, self._qemu_opt['queues'])) |
| 219 | self._qemu_opt['options'] += netdev |
| 220 | # If MAC is not specified use auto-generated MAC address based on |
| 221 | # template 52:54:00:00:<qemu_id>:<vhost_id>, e.g. vhost1 MAC of QEMU |
| 222 | # with ID 1 is 52:54:00:00:01:01 |
| 223 | if mac is None: |
| 224 | mac = '52:54:00:00:{0:02x}:{1:02x}'.\ |
| 225 | format(self._qemu_id, self._vhost_id) |
| 226 | extend_options = 'mq=on,csum=off,gso=off,guest_tso4=off,'\ |
| 227 | 'guest_tso6=off,guest_ecn=off,mrg_rxbuf=off' |
| 228 | # Create Virtio network device. |
| 229 | device = ' -device virtio-net-pci,netdev=vhost{0},mac={1},{2}'.format( |
| 230 | self._vhost_id, mac, extend_options) |
| 231 | self._qemu_opt['options'] += device |
| 232 | # Add interface MAC and socket to the node dict |
| 233 | if_data = {'mac_address': mac, 'socket': socket} |
| 234 | if_name = 'vhost{}'.format(self._vhost_id) |
| 235 | self._vm_info['interfaces'][if_name] = if_data |
| 236 | # Add socket to the socket list |
| 237 | self._socks.append(socket) |
| 238 | |
| 239 | def _qemu_qmp_exec(self, cmd): |
| 240 | """Execute QMP command. |
| 241 | |
| 242 | QMP is JSON based protocol which allows to control QEMU instance. |
| 243 | |
| 244 | :param cmd: QMP command to execute. |
| 245 | :type cmd: str |
| 246 | :return: Command output in python representation of JSON format. The |
| 247 | { "return": {} } response is QMP's success response. An error |
| 248 | response will contain the "error" keyword instead of "return". |
| 249 | """ |
| 250 | # To enter command mode, the qmp_capabilities command must be issued. |
| 251 | qmp_cmd = 'echo "{ \\"execute\\": \\"qmp_capabilities\\" }' \ |
| 252 | '{ \\"execute\\": \\"' + cmd + \ |
| 253 | '\\" }" | sudo -S socat - UNIX-CONNECT:' + self._qmp_sock |
| 254 | |
| 255 | (ret_code, stdout, stderr) = self._ssh.exec_command(qmp_cmd) |
| 256 | if int(ret_code) != 0: |
| 257 | logging.debug('QMP execute failed {0}'.format(stderr)) |
| 258 | raise RuntimeError('QMP execute "{0}"' |
| 259 | ' failed on {1}'.format(cmd, self._node['host'])) |
| 260 | logging.debug(stdout) |
| 261 | # Skip capabilities negotiation messages. |
| 262 | out_list = stdout.splitlines() |
| 263 | if len(out_list) < 3: |
| 264 | raise RuntimeError('Invalid QMP output on {0}'.format( |
| 265 | self._node['host'])) |
| 266 | return json.loads(out_list[2]) |
| 267 | |
| 268 | def _qemu_qga_flush(self): |
| 269 | """Flush the QGA parser state |
| 270 | """ |
| 271 | qga_cmd = '(printf "\xFF"; sleep 1) | sudo -S socat - UNIX-CONNECT:' + \ |
| 272 | self._qga_sock |
| 273 | # TODO: probably need something else |
| 274 | (ret_code, stdout, stderr) = self._ssh.exec_command(qga_cmd) |
| 275 | if int(ret_code) != 0: |
| 276 | logging.debug('QGA execute failed {0}'.format(stderr)) |
| 277 | raise RuntimeError('QGA execute "{0}" ' |
| 278 | 'failed on {1}'.format(qga_cmd, |
| 279 | self._node['host'])) |
| 280 | logging.debug(stdout) |
| 281 | if not stdout: |
| 282 | return {} |
| 283 | return json.loads(stdout.split('\n', 1)[0]) |
| 284 | |
| 285 | def _qemu_qga_exec(self, cmd): |
| 286 | """Execute QGA command. |
| 287 | |
| 288 | QGA provide access to a system-level agent via standard QMP commands. |
| 289 | |
| 290 | :param cmd: QGA command to execute. |
| 291 | :type cmd: str |
| 292 | """ |
| 293 | qga_cmd = '(echo "{ \\"execute\\": \\"' + \ |
| 294 | cmd + \ |
| 295 | '\\" }"; sleep 1) | sudo -S socat - UNIX-CONNECT:' + \ |
| 296 | self._qga_sock |
| 297 | (ret_code, stdout, stderr) = self._ssh.exec_command(qga_cmd) |
| 298 | if int(ret_code) != 0: |
| 299 | logging.debug('QGA execute failed {0}'.format(stderr)) |
| 300 | raise RuntimeError('QGA execute "{0}"' |
| 301 | ' failed on {1}'.format(cmd, self._node['host'])) |
| 302 | logging.debug(stdout) |
| 303 | if not stdout: |
| 304 | return {} |
| 305 | return json.loads(stdout.split('\n', 1)[0]) |
| 306 | |
| 307 | def _wait_until_vm_boot(self, timeout=60): |
| 308 | """Wait until QEMU VM is booted. |
| 309 | |
| 310 | Ping QEMU guest agent each 5s until VM booted or timeout. |
| 311 | |
| 312 | :param timeout: Waiting timeout in seconds (optional, default 60s). |
| 313 | :type timeout: int |
| 314 | """ |
| 315 | start = time() |
| 316 | while True: |
| 317 | if time() - start > timeout: |
| 318 | raise RuntimeError('timeout, VM {0} not booted on {1}'.format( |
| 319 | self._qemu_opt['disk_image'], self._node['host'])) |
| 320 | out = None |
| 321 | try: |
| 322 | self._qemu_qga_flush() |
| 323 | out = self._qemu_qga_exec('guest-ping') |
| 324 | except ValueError: |
| 325 | logging.debug('QGA guest-ping unexpected output {}'.format(out)) |
| 326 | # Empty output - VM not booted yet |
| 327 | if not out: |
| 328 | sleep(5) |
| 329 | # Non-error return - VM booted |
| 330 | elif out.get('return') is not None: |
| 331 | break |
| 332 | # Skip error and wait |
| 333 | elif out.get('error') is not None: |
| 334 | sleep(5) |
| 335 | else: |
| 336 | # If there is an unexpected output from QGA guest-info, try |
| 337 | # again until timeout. |
| 338 | logging.debug('QGA guest-ping unexpected output {}'.format(out)) |
| 339 | |
| 340 | logging.debug('VM {0} booted on {1}'.format(self._qemu_opt['disk_image'], |
| 341 | self._node['host'])) |
| 342 | |
| 343 | def _update_vm_interfaces(self): |
| 344 | """Update interface names in VM node dict.""" |
| 345 | # Send guest-network-get-interfaces command via QGA, output example: |
| 346 | # {"return": [{"name": "eth0", "hardware-address": "52:54:00:00:04:01"}, |
| 347 | # {"name": "eth1", "hardware-address": "52:54:00:00:04:02"}]} |
| 348 | out = self._qemu_qga_exec('guest-network-get-interfaces') |
| 349 | interfaces = out.get('return') |
| 350 | mac_name = {} |
| 351 | if not interfaces: |
| 352 | raise RuntimeError('Get VM {0} interface list failed on {1}'.format( |
| 353 | self._qemu_opt['disk_image'], self._node['host'])) |
| 354 | # Create MAC-name dict |
| 355 | for interface in interfaces: |
| 356 | if 'hardware-address' not in interface: |
| 357 | continue |
| 358 | mac_name[interface['hardware-address']] = interface['name'] |
| 359 | # Match interface by MAC and save interface name |
| 360 | for interface in self._vm_info['interfaces'].values(): |
| 361 | mac = interface.get('mac_address') |
| 362 | if_name = mac_name.get(mac) |
| 363 | if if_name is None: |
| 364 | logging.debug('Interface name for MAC {} not found'.format(mac)) |
| 365 | else: |
| 366 | interface['name'] = if_name |
| 367 | |
| 368 | def _huge_page_check(self, allocate=False): |
| 369 | """Huge page check.""" |
| 370 | huge_mnt = self._qemu_opt.get('huge_mnt') |
| 371 | mem_size = self._qemu_opt.get('mem_size') |
| 372 | |
| 373 | # Get huge pages information |
| 374 | huge_size = self._get_huge_page_size() |
| 375 | huge_free = self._get_huge_page_free(huge_size) |
| 376 | huge_total = self._get_huge_page_total(huge_size) |
| 377 | |
| 378 | # Check if memory reqested by qemu is available on host |
| 379 | if (mem_size * 1024) > (huge_free * huge_size): |
| 380 | # If we want to allocate hugepage dynamically |
| 381 | if allocate: |
| 382 | mem_needed = abs((huge_free * huge_size) - (mem_size * 1024)) |
| 383 | huge_to_allocate = ((mem_needed / huge_size) * 2) + huge_total |
| 384 | max_map_count = huge_to_allocate*4 |
| 385 | # Increase maximum number of memory map areas a process may have |
| 386 | cmd = 'echo "{0}" | sudo tee /proc/sys/vm/max_map_count'.format( |
| 387 | max_map_count) |
| 388 | (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd) |
| 389 | # Increase hugepage count |
| 390 | cmd = 'echo "{0}" | sudo tee /proc/sys/vm/nr_hugepages'.format( |
| 391 | huge_to_allocate) |
| 392 | (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd) |
| 393 | if int(ret_code) != 0: |
| 394 | logging.debug('Mount huge pages failed {0}'.format(stderr)) |
| 395 | raise RuntimeError('Mount huge pages failed on {0}'.format( |
| 396 | self._node['host'])) |
| 397 | # If we do not want to allocate dynamicaly end with error |
| 398 | else: |
| 399 | raise RuntimeError( |
| 400 | 'Not enough free huge pages: {0}, ' |
| 401 | '{1} MB'.format(huge_free, huge_free * huge_size) |
| 402 | ) |
| 403 | # Check if huge pages mount point exist |
| 404 | has_huge_mnt = False |
| 405 | (_, output, _) = self._ssh.exec_command('cat /proc/mounts') |
| 406 | for line in output.splitlines(): |
| 407 | # Try to find something like: |
| 408 | # none /mnt/huge hugetlbfs rw,relatime,pagesize=2048k 0 0 |
| 409 | mount = line.split() |
| 410 | if mount[2] == 'hugetlbfs' and mount[1] == huge_mnt: |
| 411 | has_huge_mnt = True |
| 412 | break |
| 413 | # If huge page mount point not exist create one |
| 414 | if not has_huge_mnt: |
| 415 | cmd = 'mkdir -p {0}'.format(huge_mnt) |
| 416 | (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd) |
| 417 | if int(ret_code) != 0: |
| 418 | logging.debug('Create mount dir failed: {0}'.format(stderr)) |
| 419 | raise RuntimeError('Create mount dir failed on {0}'.format( |
| 420 | self._node['host'])) |
| 421 | cmd = 'mount -t hugetlbfs -o pagesize=2048k none {0}'.format( |
| 422 | huge_mnt) |
| 423 | (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd) |
| 424 | if int(ret_code) != 0: |
| 425 | logging.debug('Mount huge pages failed {0}'.format(stderr)) |
| 426 | raise RuntimeError('Mount huge pages failed on {0}'.format( |
| 427 | self._node['host'])) |
| 428 | |
| 429 | def _get_huge_page_size(self): |
| 430 | """Get default size of huge pages in system. |
| 431 | |
| 432 | :returns: Default size of free huge pages in system. |
| 433 | :rtype: int |
| 434 | :raises: RuntimeError if reading failed for three times. |
| 435 | """ |
| 436 | # TODO: remove to dedicated library |
| 437 | cmd_huge_size = "grep Hugepagesize /proc/meminfo | awk '{ print $2 }'" |
| 438 | for _ in range(3): |
| 439 | (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_size) |
| 440 | if ret == 0: |
| 441 | try: |
| 442 | huge_size = int(out) |
| 443 | except ValueError: |
| 444 | logging.debug('Reading huge page size information failed') |
| 445 | else: |
| 446 | break |
| 447 | else: |
| 448 | raise RuntimeError('Getting huge page size information failed.') |
| 449 | return huge_size |
| 450 | |
| 451 | def _get_huge_page_free(self, huge_size): |
| 452 | """Get total number of huge pages in system. |
| 453 | |
| 454 | :param huge_size: Size of hugepages. |
| 455 | :type huge_size: int |
| 456 | :returns: Number of free huge pages in system. |
| 457 | :rtype: int |
| 458 | :raises: RuntimeError if reading failed for three times. |
| 459 | """ |
| 460 | # TODO: add numa aware option |
| 461 | # TODO: remove to dedicated library |
| 462 | cmd_huge_free = 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/'\ |
| 463 | 'free_hugepages'.format(huge_size) |
| 464 | for _ in range(3): |
| 465 | (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_free) |
| 466 | if ret == 0: |
| 467 | try: |
| 468 | huge_free = int(out) |
| 469 | except ValueError: |
| 470 | logging.debug('Reading free huge pages information failed') |
| 471 | else: |
| 472 | break |
| 473 | else: |
| 474 | raise RuntimeError('Getting free huge pages information failed.') |
| 475 | return huge_free |
| 476 | |
| 477 | def _get_huge_page_total(self, huge_size): |
| 478 | """Get total number of huge pages in system. |
| 479 | |
| 480 | :param huge_size: Size of hugepages. |
| 481 | :type huge_size: int |
| 482 | :returns: Total number of huge pages in system. |
| 483 | :rtype: int |
| 484 | :raises: RuntimeError if reading failed for three times. |
| 485 | """ |
| 486 | # TODO: add numa aware option |
| 487 | # TODO: remove to dedicated library |
| 488 | cmd_huge_total = 'cat /sys/kernel/mm/hugepages/hugepages-{0}kB/'\ |
| 489 | 'nr_hugepages'.format(huge_size) |
| 490 | for _ in range(3): |
| 491 | (ret, out, _) = self._ssh.exec_command_sudo(cmd_huge_total) |
| 492 | if ret == 0: |
| 493 | try: |
| 494 | huge_total = int(out) |
| 495 | except ValueError: |
| 496 | logging.debug('Reading total huge pages information failed') |
| 497 | else: |
| 498 | break |
| 499 | else: |
| 500 | raise RuntimeError('Getting total huge pages information failed.') |
| 501 | return huge_total |
| 502 | |
| 503 | def qemu_start(self): |
| 504 | """Start QEMU and wait until VM boot. |
| 505 | |
| 506 | :return: VM node info. |
| 507 | :rtype: dict |
| 508 | .. note:: First set at least node to run QEMU on. |
| 509 | .. warning:: Starts only one VM on the node. |
| 510 | """ |
| 511 | # SSH forwarding |
| 512 | ssh_fwd = '-net user,hostfwd=tcp::{0}-:22'.format( |
| 513 | self._qemu_opt.get('ssh_fwd_port')) |
| 514 | # Memory and huge pages |
| 515 | mem = '-object memory-backend-file,id=mem,size={0}M,mem-path={1},' \ |
| 516 | 'share=on -m {0} -numa node,memdev=mem'.format( |
| 517 | self._qemu_opt.get('mem_size'), self._qemu_opt.get('huge_mnt')) |
| 518 | |
| 519 | # By default check only if hugepages are available. |
| 520 | # If 'huge_allocate' is set to true try to allocate as well. |
| 521 | self._huge_page_check(allocate=self._qemu_opt.get('huge_allocate')) |
| 522 | |
| 523 | # Disk option |
| 524 | drive = '-drive file={0},format=raw,cache=none,if=virtio'.format( |
| 525 | self._qemu_opt.get('disk_image')) |
| 526 | # Setup QMP via unix socket |
| 527 | qmp = '-qmp unix:{0},server,nowait'.format(self._qmp_sock) |
| 528 | # Setup serial console |
| 529 | serial = '-chardev socket,host=127.0.0.1,port={0},id=gnc0,server,' \ |
| 530 | 'nowait -device isa-serial,chardev=gnc0'.format( |
| 531 | self._qemu_opt.get('serial_port')) |
| 532 | # Setup QGA via chardev (unix socket) and isa-serial channel |
| 533 | qga = '-chardev socket,path={0},server,nowait,id=qga0 ' \ |
| 534 | '-device isa-serial,chardev=qga0'.format(self._qga_sock) |
| 535 | # Graphic setup |
| 536 | graphic = '-monitor none -display none -vga none' |
| 537 | # PID file |
| 538 | pid = '-pidfile {}'.format(self._pid_file) |
| 539 | |
| 540 | # Run QEMU |
| 541 | cmd = '{0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}'.format( |
| 542 | self._qemu_bin, self._qemu_opt.get('smp'), mem, ssh_fwd, |
| 543 | self._qemu_opt.get('options'), |
| 544 | drive, qmp, serial, qga, graphic, pid) |
| 545 | (ret_code, _, stderr) = self._ssh.exec_command_sudo(cmd, timeout=300) |
| 546 | if int(ret_code) != 0: |
| 547 | logging.debug('QEMU start failed {0}'.format(stderr)) |
| 548 | raise RuntimeError('QEMU start failed on {0}'.format( |
| 549 | self._node['host'])) |
| 550 | logging.debug('QEMU running') |
| 551 | # Wait until VM boot |
| 552 | try: |
| 553 | self._wait_until_vm_boot() |
| 554 | except RuntimeError: |
| 555 | self.qemu_kill_all() |
| 556 | self.qemu_clear_socks() |
| 557 | raise |
| 558 | # Update interface names in VM node dict |
| 559 | self._update_vm_interfaces() |
| 560 | # Return VM node dict |
| 561 | return self._vm_info |
| 562 | |
| 563 | def qemu_quit(self): |
| 564 | """Quit the QEMU emulator.""" |
| 565 | out = self._qemu_qmp_exec('quit') |
| 566 | err = out.get('error') |
| 567 | if err is not None: |
| 568 | raise RuntimeError('QEMU quit failed on {0}, error: {1}'.format( |
| 569 | self._node['host'], json.dumps(err))) |
| 570 | |
| 571 | def qemu_system_powerdown(self): |
| 572 | """Power down the system (if supported).""" |
| 573 | out = self._qemu_qmp_exec('system_powerdown') |
| 574 | err = out.get('error') |
| 575 | if err is not None: |
| 576 | raise RuntimeError( |
| 577 | 'QEMU system powerdown failed on {0}, ' |
| 578 | 'error: {1}'.format(self._node['host'], json.dumps(err)) |
| 579 | ) |
| 580 | |
| 581 | def qemu_system_reset(self): |
| 582 | """Reset the system.""" |
| 583 | out = self._qemu_qmp_exec('system_reset') |
| 584 | err = out.get('error') |
| 585 | if err is not None: |
| 586 | raise RuntimeError( |
| 587 | 'QEMU system reset failed on {0}, ' |
| 588 | 'error: {1}'.format(self._node['host'], json.dumps(err))) |
| 589 | |
| 590 | def qemu_kill(self): |
| 591 | """Kill qemu process.""" |
| 592 | # Note: in QEMU start phase there are 3 QEMU processes because we |
| 593 | # daemonize QEMU |
| 594 | self._ssh.exec_command_sudo('chmod +r {}'.format(self._pid_file)) |
| 595 | self._ssh.exec_command_sudo('kill -SIGKILL $(cat {})' |
| 596 | .format(self._pid_file)) |
| 597 | # Delete PID file |
| 598 | cmd = 'rm -f {}'.format(self._pid_file) |
| 599 | self._ssh.exec_command_sudo(cmd) |
| 600 | |
| 601 | def qemu_kill_all(self, node=None): |
| 602 | """Kill all qemu processes on DUT node if specified. |
| 603 | |
| 604 | :param node: Node to kill all QEMU processes on. |
| 605 | :type node: dict |
| 606 | """ |
| 607 | if node: |
| 608 | self.qemu_set_node(node) |
| 609 | self._ssh.exec_command_sudo('pkill -SIGKILL qemu') |
| 610 | |
| 611 | def qemu_clear_socks(self): |
| 612 | """Remove all sockets created by QEMU.""" |
| 613 | # If serial console port still open kill process |
| 614 | cmd = 'fuser -k {}/tcp'.format(self._qemu_opt.get('serial_port')) |
| 615 | self._ssh.exec_command_sudo(cmd) |
| 616 | # Delete all created sockets |
| 617 | for sock in self._socks: |
| 618 | cmd = 'rm -f {}'.format(sock) |
| 619 | self._ssh.exec_command_sudo(cmd) |
| 620 | |
| 621 | def qemu_system_status(self): |
| 622 | """Return current VM status. |
| 623 | |
| 624 | VM should be in following status: |
| 625 | |
| 626 | - debug: QEMU running on a debugger |
| 627 | - finish-migrate: paused to finish the migration process |
| 628 | - inmigrate: waiting for an incoming migration |
| 629 | - internal-error: internal error has occurred |
| 630 | - io-error: the last IOP has failed |
| 631 | - paused: paused |
| 632 | - postmigrate: paused following a successful migrate |
| 633 | - prelaunch: QEMU was started with -S and guest has not started |
| 634 | - restore-vm: paused to restore VM state |
| 635 | - running: actively running |
| 636 | - save-vm: paused to save the VM state |
| 637 | - shutdown: shut down (and -no-shutdown is in use) |
| 638 | - suspended: suspended (ACPI S3) |
| 639 | - watchdog: watchdog action has been triggered |
| 640 | - guest-panicked: panicked as a result of guest OS panic |
| 641 | |
| 642 | :return: VM status. |
| 643 | :rtype: str |
| 644 | """ |
| 645 | out = self._qemu_qmp_exec('query-status') |
| 646 | ret = out.get('return') |
| 647 | if ret is not None: |
| 648 | return ret.get('status') |
| 649 | else: |
| 650 | err = out.get('error') |
| 651 | raise RuntimeError( |
| 652 | 'QEMU query-status failed on {0}, ' |
| 653 | 'error: {1}'.format(self._node['host'], json.dumps(err))) |
| 654 | |
| 655 | @staticmethod |
| 656 | def build_qemu(node, force_install=False, apply_patch=False): |
| 657 | """Build QEMU from sources. |
| 658 | |
| 659 | :param node: Node to build QEMU on. |
| 660 | :param force_install: If True, then remove previous build. |
| 661 | :param apply_patch: If True, then apply patches from qemu_patches dir. |
| 662 | :type node: dict |
| 663 | :type force_install: bool |
| 664 | :type apply_patch: bool |
| 665 | :raises: RuntimeError if building QEMU failed. |
| 666 | """ |
| 667 | |
| 668 | directory = ' --directory={0}'.format(Constants.QEMU_INSTALL_DIR) |
| 669 | version = ' --version={0}'.format(Constants.QEMU_INSTALL_VERSION) |
| 670 | force = ' --force' if force_install else '' |
| 671 | patch = ' --patch' if apply_patch else '' |
| 672 | |
| 673 | (ret_code, stdout, stderr) = VPPUtil. \ |
| 674 | exec_command( |
| 675 | "sudo -E sh -c '{0}/{1}/qemu_build.sh{2}{3}{4}{5}'". |
| 676 | format(Constants.REMOTE_FW_DIR, Constants.RESOURCES_LIB_SH, |
| 677 | version, directory, force, patch), 1000) |
| 678 | |
| 679 | if int(ret_code) != 0: |
| 680 | logging.debug('QEMU build failed {0}'.format(stdout + stderr)) |
| 681 | raise RuntimeError('QEMU build failed on {0}'.format(node['host'])) |