aria/multivim-plugin/system_tests/openstack_handler.py

########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
#    * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    * See the License for the specific language governing permissions and
#    * limitations under the License.

import random
import logging
import os
import time
import copy
from contextlib import contextmanager

from cinderclient import client as cinderclient
from keystoneauth1 import loading, session
import novaclient.client as nvclient
import neutronclient.v2_0.client as neclient
from retrying import retry

from cosmo_tester.framework.handlers import (
    BaseHandler,
    BaseCloudifyInputsConfigReader)
from cosmo_tester.framework.util import get_actual_keypath

logging.getLogger('neutronclient.client').setLevel(logging.INFO)
logging.getLogger('novaclient.client').setLevel(logging.INFO)


VOLUME_TERMINATION_TIMEOUT_SECS = 300


class OpenstackCleanupContext(BaseHandler.CleanupContext):

    def __init__(self, context_name, env):
        super(OpenstackCleanupContext, self).__init__(context_name, env)
        self.before_run = self.env.handler.openstack_infra_state()

    def cleanup(self):
        """
        Cleans resources created by the test.
        Resource that existed before the test will not be removed
        """
        super(OpenstackCleanupContext, self).cleanup()
        resources_to_teardown = self.get_resources_to_teardown(
            self.env, resources_to_keep=self.before_run)
        if self.skip_cleanup:
            self.logger.warn('[{0}] SKIPPING cleanup of resources: {1}'
                             .format(self.context_name, resources_to_teardown))
        else:
            self._clean(self.env, resources_to_teardown)

    @classmethod
    def clean_all(cls, env):
        """
        Cleans *all* resources, including resources that were not
        created by the test
        """
        super(OpenstackCleanupContext, cls).clean_all(env)
        resources_to_teardown = cls.get_resources_to_teardown(env)
        cls._clean(env, resources_to_teardown)

    @classmethod
    def _clean(cls, env, resources_to_teardown):
        cls.logger.info('Openstack handler will try to remove these resources:'
                        ' {0}'.format(resources_to_teardown))
        failed_to_remove = env.handler.remove_openstack_resources(
            resources_to_teardown)
        if failed_to_remove:
            trimmed_failed_to_remove = {key: value for key, value in
                                        failed_to_remove.iteritems()
                                        if value}
            if len(trimmed_failed_to_remove) > 0:
                msg = 'Openstack handler failed to remove some resources:' \
                      ' {0}'.format(trimmed_failed_to_remove)
                cls.logger.error(msg)
                raise RuntimeError(msg)

    @classmethod
    def get_resources_to_teardown(cls, env, resources_to_keep=None):
        all_existing_resources = env.handler.openstack_infra_state()
        if resources_to_keep:
            return env.handler.openstack_infra_state_delta(
                before=resources_to_keep, after=all_existing_resources)
        else:
            return all_existing_resources

    def update_server_id(self, server_name):

        # retrieve the id of the new server
        nova, _, _ = self.env.handler.openstack_clients()
        servers = nova.servers.list(
            search_opts={'name': server_name})
        if len(servers) > 1:
            raise RuntimeError(
                'Expected 1 server with name {0}, but found {1}'
                .format(server_name, len(servers)))

        new_server_id = servers[0].id

        # retrieve the id of the old server
        old_server_id = None
        servers = self.before_run['servers']
        for server_id, name in servers.iteritems():
            if server_name == name:
                old_server_id = server_id
                break
        if old_server_id is None:
            raise RuntimeError(
                'Could not find a server with name {0} '
                'in the internal cleanup context state'
                .format(server_name))

        # replace the id in the internal state
        servers[new_server_id] = servers.pop(old_server_id)


class CloudifyOpenstackInputsConfigReader(BaseCloudifyInputsConfigReader):

    def __init__(self, cloudify_config, manager_blueprint_path, **kwargs):
        super(CloudifyOpenstackInputsConfigReader, self).__init__(
            cloudify_config, manager_blueprint_path=manager_blueprint_path,
            **kwargs)

    @property
    def region(self):
        return self.config['region']

    @property
    def management_server_name(self):
        return self.config['manager_server_name']

    @property
    def agent_key_path(self):
        return self.config['agent_private_key_path']

    @property
    def management_user_name(self):
        return self.config['ssh_user']

    @property
    def management_key_path(self):
        return self.config['ssh_key_filename']

    @property
    def agent_keypair_name(self):
        return self.config['agent_public_key_name']

    @property
    def management_keypair_name(self):
        return self.config['manager_public_key_name']

    @property
    def use_existing_agent_keypair(self):
        return self.config['use_existing_agent_keypair']

    @property
    def use_existing_manager_keypair(self):
        return self.config['use_existing_manager_keypair']

    @property
    def external_network_name(self):
        return self.config['external_network_name']

    @property
    def keystone_username(self):
        return self.config['keystone_username']

    @property
    def keystone_password(self):
        return self.config['keystone_password']

    @property
    def keystone_tenant_name(self):
        return self.config['keystone_tenant_name']

    @property
    def keystone_url(self):
        return self.config['keystone_url']

    @property
    def neutron_url(self):
        return self.config.get('neutron_url', None)

    @property
    def management_network_name(self):
        return self.config['management_network_name']

    @property
    def management_subnet_name(self):
        return self.config['management_subnet_name']

    @property
    def management_router_name(self):
        return self.config['management_router']

    @property
    def agents_security_group(self):
        return self.config['agents_security_group_name']

    @property
    def management_security_group(self):
        return self.config['manager_security_group_name']


class OpenstackHandler(BaseHandler):

    CleanupContext = OpenstackCleanupContext
    CloudifyConfigReader = CloudifyOpenstackInputsConfigReader

    def before_bootstrap(self):
        super(OpenstackHandler, self).before_bootstrap()
        with self.update_cloudify_config() as patch:
            suffix = '-%06x' % random.randrange(16 ** 6)
            server_name_prop_path = 'manager_server_name'
            patch.append_value(server_name_prop_path, suffix)

    def after_bootstrap(self, provider_context):
        super(OpenstackHandler, self).after_bootstrap(provider_context)
        resources = provider_context['resources']
        agent_keypair = resources['agents_keypair']
        management_keypair = resources['management_keypair']
        self.remove_agent_keypair = agent_keypair['external_resource'] is False
        self.remove_management_keypair = \
            management_keypair['external_resource'] is False

    def after_teardown(self):
        super(OpenstackHandler, self).after_teardown()
        if self.remove_agent_keypair:
            agent_key_path = get_actual_keypath(self.env,
                                                self.env.agent_key_path,
                                                raise_on_missing=False)
            if agent_key_path:
                os.remove(agent_key_path)
        if self.remove_management_keypair:
            management_key_path = get_actual_keypath(
                self.env,
                self.env.management_key_path,
                raise_on_missing=False)
            if management_key_path:
                os.remove(management_key_path)

    def openstack_clients(self):
        creds = self._client_creds()
        params = {
            'region_name': creds.pop('region_name'),
        }

        loader = loading.get_plugin_loader("password")
        auth = loader.load_from_options(**creds)
        sess = session.Session(auth=auth, verify=True)

        params['session'] = sess

        nova = nvclient.Client('2', **params)
        neutron = neclient.Client(**params)
        cinder = cinderclient.Client('2', **params)

        return (nova, neutron, cinder)

    @retry(stop_max_attempt_number=5, wait_fixed=20000)
    def openstack_infra_state(self):
        """
        @retry decorator is used because this error sometimes occur:
        ConnectionFailed: Connection to neutron failed: Maximum
        attempts reached
        """
        nova, neutron, cinder = self.openstack_clients()
        try:
            prefix = self.env.resources_prefix
        except (AttributeError, KeyError):
            prefix = ''
        return {
            'networks': dict(self._networks(neutron, prefix)),
            'subnets': dict(self._subnets(neutron, prefix)),
            'routers': dict(self._routers(neutron, prefix)),
            'security_groups': dict(self._security_groups(neutron, prefix)),
            'servers': dict(self._servers(nova, prefix)),
            'key_pairs': dict(self._key_pairs(nova, prefix)),
            'floatingips': dict(self._floatingips(neutron, prefix)),
            'ports': dict(self._ports(neutron, prefix)),
            'volumes': dict(self._volumes(cinder, prefix))
        }

    def openstack_infra_state_delta(self, before, after):
        after = copy.deepcopy(after)
        return {
            prop: self._remove_keys(after[prop], before[prop].keys())
            for prop in before
        }

    def _find_keypairs_to_delete(self, nodes, node_instances):
        """Filter the nodes only returning the names of keypair nodes

        Examine node_instances and nodes, return the external_name of
        those node_instances, which correspond to a node that has a
        type == KeyPair

        To filter by deployment_id, simply make sure that the nodes and
        node_instances this method receives, are pre-filtered
        (ie. filter the nodes while fetching them from the manager)
        """
        keypairs = set()  # a set of (deployment_id, node_id) tuples

        for node in nodes:
            if node.get('type') != 'cloudify.openstack.nodes.KeyPair':
                continue
            # deployment_id isnt always present in local_env runs
            key = (node.get('deployment_id'), node['id'])
            keypairs.add(key)

        for node_instance in node_instances:
            key = (node_instance.get('deployment_id'),
                   node_instance['node_id'])
            if key not in keypairs:
                continue

            runtime_properties = node_instance['runtime_properties']
            if not runtime_properties:
                continue
            name = runtime_properties.get('external_name')
            if name:
                yield name

    def _delete_keypairs_by_name(self, keypair_names):
        nova, neutron, cinder = self.openstack_clients()
        existing_keypairs = nova.keypairs.list()

        for name in keypair_names:
            for keypair in existing_keypairs:
                if keypair.name == name:
                    nova.keypairs.delete(keypair)

    def remove_keypairs_from_local_env(self, local_env):
        """Query the local_env for nodes which are keypairs, remove them

        Similar to querying the manager, we can look up nodes in the local_env
        which is used for tests.
        """
        nodes = local_env.storage.get_nodes()
        node_instances = local_env.storage.get_node_instances()
        names = self._find_keypairs_to_delete(nodes, node_instances)
        self._delete_keypairs_by_name(names)

    def remove_keypairs_from_manager(self, deployment_id=None,
                                     rest_client=None):
        """Query the manager for nodes by deployment_id, delete keypairs

        Fetch nodes and node_instances from the manager by deployment_id
        (or all if not given), find which ones represent openstack keypairs,
        remove them.
        """
        if rest_client is None:
            rest_client = self.env.rest_client

        nodes = rest_client.nodes.list(deployment_id=deployment_id)
        node_instances = rest_client.node_instances.list(
            deployment_id=deployment_id)
        keypairs = self._find_keypairs_to_delete(nodes, node_instances)
        self._delete_keypairs_by_name(keypairs)

    def remove_keypair(self, name):
        """Delete an openstack keypair by name. If it doesnt exist, do nothing.
        """
        self._delete_keypairs_by_name([name])

    def remove_openstack_resources(self, resources_to_remove):
        # basically sort of a workaround, but if we get the order wrong
        # the first time, there is a chance things would better next time
        # 3'rd time can't really hurt, can it?
        # 3 is a charm
        for _ in range(3):
            resources_to_remove = self._remove_openstack_resources_impl(
                resources_to_remove)
            if all([len(g) == 0 for g in resources_to_remove.values()]):
                break
            # give openstack some time to update its data structures
            time.sleep(3)
        return resources_to_remove

    def _remove_openstack_resources_impl(self, resources_to_remove):
        nova, neutron, cinder = self.openstack_clients()

        servers = nova.servers.list()
        ports = neutron.list_ports()['ports']
        routers = neutron.list_routers()['routers']
        subnets = neutron.list_subnets()['subnets']
        networks = neutron.list_networks()['networks']
        # keypairs = nova.keypairs.list()
        floatingips = neutron.list_floatingips()['floatingips']
        security_groups = neutron.list_security_groups()['security_groups']
        volumes = cinder.volumes.list()

        failed = {
            'servers': {},
            'routers': {},
            'ports': {},
            'subnets': {},
            'networks': {},
            'key_pairs': {},
            'floatingips': {},
            'security_groups': {},
            'volumes': {}
        }

        volumes_to_remove = []
        for volume in volumes:
            if volume.id in resources_to_remove['volumes']:
                volumes_to_remove.append(volume)

        left_volumes = self._delete_volumes(nova, cinder, volumes_to_remove)
        for volume_id, ex in left_volumes.iteritems():
            failed['volumes'][volume_id] = ex

        for server in servers:
            if server.id in resources_to_remove['servers']:
                with self._handled_exception(server.id, failed, 'servers'):
                    nova.servers.delete(server)

        for router in routers:
            if router['id'] in resources_to_remove['routers']:
                with self._handled_exception(router['id'], failed, 'routers'):
                    for p in neutron.list_ports(
                            device_id=router['id'])['ports']:
                        neutron.remove_interface_router(router['id'], {
                            'port_id': p['id']
                        })
                    neutron.delete_router(router['id'])

        for port in ports:
            if port['id'] in resources_to_remove['ports']:
                with self._handled_exception(port['id'], failed, 'ports'):
                    neutron.delete_port(port['id'])

        for subnet in subnets:
            if subnet['id'] in resources_to_remove['subnets']:
                with self._handled_exception(subnet['id'], failed, 'subnets'):
                    neutron.delete_subnet(subnet['id'])

        for network in networks:
            if network['name'] == self.env.external_network_name:
                continue
            if network['id'] in resources_to_remove['networks']:
                with self._handled_exception(network['id'], failed,
                                             'networks'):
                    neutron.delete_network(network['id'])

        # TODO: implement key-pair creation and cleanup per tenant
        #
        # IMPORTANT: Do not remove key-pairs, they might be used
        # by another tenant (of the same user)
        #
        # for key_pair in keypairs:
        #     if key_pair.name == self.env.agent_keypair_name and \
        #             self.env.use_existing_agent_keypair:
        #             # this is a pre-existing agent key-pair, do not remove
        #             continue
        #     elif key_pair.name == self.env.management_keypair_name and \
        #             self.env.use_existing_manager_keypair:
        #             # this is a pre-existing manager key-pair, do not remove
        #             continue
        #     elif key_pair.id in resources_to_remove['key_pairs']:
        #         with self._handled_exception(key_pair.id, failed,
        #           'key_pairs'):
        #             nova.keypairs.delete(key_pair)

        for floatingip in floatingips:
            if floatingip['id'] in resources_to_remove['floatingips']:
                with self._handled_exception(floatingip['id'], failed,
                                             'floatingips'):
                    neutron.delete_floatingip(floatingip['id'])

        for security_group in security_groups:
            if security_group['name'] == 'default':
                continue
            if security_group['id'] in resources_to_remove['security_groups']:
                with self._handled_exception(security_group['id'],
                                             failed, 'security_groups'):
                    neutron.delete_security_group(security_group['id'])

        return failed

    def _delete_volumes(self, nova, cinder, existing_volumes):
        unremovables = {}
        end_time = time.time() + VOLUME_TERMINATION_TIMEOUT_SECS

        for volume in existing_volumes:
            # detach the volume
            if volume.status in ['available', 'error', 'in-use']:
                try:
                    self.logger.info('Detaching volume {0} ({1}), currently in'
                                     ' status {2} ...'.
                                     format(volume.name, volume.id,
                                            volume.status))
                    for attachment in volume.attachments:
                        nova.volumes.delete_server_volume(
                            server_id=attachment['server_id'],
                            attachment_id=attachment['id'])
                except Exception as e:
                    self.logger.warning('Attempt to detach volume {0} ({1})'
                                        ' yielded exception: "{2}"'.
                                        format(volume.name, volume.id,
                                               e))
                    unremovables[volume.id] = e
                    existing_volumes.remove(volume)

        time.sleep(3)
        for volume in existing_volumes:
            # delete the volume
            if volume.status in ['available', 'error', 'in-use']:
                try:
                    self.logger.info('Deleting volume {0} ({1}), currently in'
                                     ' status {2} ...'.
                                     format(volume.name, volume.id,
                                            volume.status))
                    cinder.volumes.delete(volume)
                except Exception as e:
                    self.logger.warning('Attempt to delete volume {0} ({1})'
                                        ' yielded exception: "{2}"'.
                                        format(volume.name, volume.id,
                                               e))
                    unremovables[volume.id] = e
                    existing_volumes.remove(volume)

        # wait for all volumes deletion until completed or timeout is reached
        while existing_volumes and time.time() < end_time:
            time.sleep(3)
            for volume in existing_volumes:
                volume_id = volume.id
                volume_name = volume.name
                try:
                    vol = cinder.volumes.get(volume_id)
                    if vol.status == 'deleting':
                        self.logger.debug('volume {0} ({1}) is being '
                                          'deleted...'.format(volume_name,
                                                              volume_id))
                    else:
                        self.logger.warning('volume {0} ({1}) is in '
                                            'unexpected status: {2}'.
                                            format(volume_name, volume_id,
                                                   vol.status))
                except Exception as e:
                    # the volume wasn't found, it was deleted
                    if hasattr(e, 'code') and e.code == 404:
                        self.logger.info('deleted volume {0} ({1})'.
                                         format(volume_name, volume_id))
                        existing_volumes.remove(volume)
                    else:
                        self.logger.warning('failed to remove volume {0} '
                                            '({1}), exception: {2}'.
                                            format(volume_name,
                                                   volume_id, e))
                        unremovables[volume_id] = e
                        existing_volumes.remove(volume)

        if existing_volumes:
            for volume in existing_volumes:
                # try to get the volume's status
                try:
                    vol = cinder.volumes.get(volume.id)
                    vol_status = vol.status
                except:
                    # failed to get volume... status is unknown
                    vol_status = 'unknown'

                unremovables[volume.id] = 'timed out while removing volume '\
                                          '{0} ({1}), current volume status '\
                                          'is {2}'.format(volume.name,
                                                          volume.id,
                                                          vol_status)

        if unremovables:
            self.logger.warning('failed to remove volumes: {0}'.format(
                unremovables))

        return unremovables

    def _client_creds(self):
        return {
            'username': self.env.keystone_username,
            'password': self.env.keystone_password,
            'auth_url': self.env.keystone_url,
            'project_name': self.env.keystone_tenant_name,
            'region_name': self.env.region
        }

    def _networks(self, neutron, prefix):
        return [(n['id'], n['name'])
                for n in neutron.list_networks()['networks']
                if self._check_prefix(n['name'], prefix)]

    def _subnets(self, neutron, prefix):
        return [(n['id'], n['name'])
                for n in neutron.list_subnets()['subnets']
                if self._check_prefix(n['name'], prefix)]

    def _routers(self, neutron, prefix):
        return [(n['id'], n['name'])
                for n in neutron.list_routers()['routers']
                if self._check_prefix(n['name'], prefix)]

    def _security_groups(self, neutron, prefix):
        return [(n['id'], n['name'])
                for n in neutron.list_security_groups()['security_groups']
                if self._check_prefix(n['name'], prefix)]

    def _servers(self, nova, prefix):
        return [(s.id, s.human_id)
                for s in nova.servers.list()
                if self._check_prefix(s.human_id, prefix)]

    def _key_pairs(self, nova, prefix):
        return [(kp.id, kp.name)
                for kp in nova.keypairs.list()
                if self._check_prefix(kp.name, prefix)]

    def _floatingips(self, neutron, prefix):
        return [(ip['id'], ip['floating_ip_address'])
                for ip in neutron.list_floatingips()['floatingips']]

    def _ports(self, neutron, prefix):
        return [(p['id'], p['name'])
                for p in neutron.list_ports()['ports']
                if self._check_prefix(p['name'], prefix)]

    def _volumes(self, cinder, prefix):
        return [(v.id, v.name) for v in cinder.volumes.list()
                if self._check_prefix(v.name, prefix)]

    def _check_prefix(self, name, prefix):
        # some openstack resources (eg. volumes) can have no display_name,
        # in which case it's None
        return name is None or name.startswith(prefix)

    def _remove_keys(self, dct, keys):
        for key in keys:
            if key in dct:
                del dct[key]
        return dct

    @contextmanager
    def _handled_exception(self, resource_id, failed, resource_group):
        try:
            yield
        except BaseException, ex:
            failed[resource_group][resource_id] = ex


handler = OpenstackHandler