From 617c92b0b56cac51b3110443c1ff29b951e812d7 Mon Sep 17 00:00:00 2001 From: Mikyung Kang Date: Thu, 13 Dec 2012 02:56:56 +0900 Subject: Baremetal VIF and Volume sub-drivers. This patch implements VIF and Volume subdrivers for baremetal driver. It was separated from review 11354. blueprint general-bare-metal-provisioning-framework. Change-Id: Id187d04b6bd7b838159d46d61be72256414d0e72 Co-authored-by: Mikyung Kang Co-authored-by: David Kang Co-authored-by: Ken Igarashi Co-authored-by: Arata Notsu --- nova/virt/baremetal/db/api.py | 8 + nova/virt/baremetal/db/sqlalchemy/api.py | 40 +++++ nova/virt/baremetal/driver.py | 64 ++++++- nova/virt/baremetal/vif_driver.py | 75 +++++++++ nova/virt/baremetal/volume_driver.py | 277 +++++++++++++++++++++++++++++++ 5 files changed, 463 insertions(+), 1 deletion(-) create mode 100644 nova/virt/baremetal/vif_driver.py create mode 100644 nova/virt/baremetal/volume_driver.py (limited to 'nova/virt') diff --git a/nova/virt/baremetal/db/api.py b/nova/virt/baremetal/db/api.py index 15d50dd66..0b8cf781c 100644 --- a/nova/virt/baremetal/db/api.py +++ b/nova/virt/baremetal/db/api.py @@ -148,6 +148,14 @@ def bm_interface_create(context, bm_node_id, address, datapath_id, port_no): datapath_id, port_no) +def bm_interface_set_vif_uuid(context, if_id, vif_uuid): + return IMPL.bm_interface_set_vif_uuid(context, if_id, vif_uuid) + + +def bm_interface_get_by_vif_uuid(context, vif_uuid): + return IMPL.bm_interface_get_by_vif_uuid(context, vif_uuid) + + def bm_interface_get_all_by_bm_node_id(context, bm_node_id): return IMPL.bm_interface_get_all_by_bm_node_id(context, bm_node_id) diff --git a/nova/virt/baremetal/db/sqlalchemy/api.py b/nova/virt/baremetal/db/sqlalchemy/api.py index bb85d677f..48606ac44 100644 --- a/nova/virt/baremetal/db/sqlalchemy/api.py +++ b/nova/virt/baremetal/db/sqlalchemy/api.py @@ -310,6 +310,46 @@ def bm_interface_create(context, bm_node_id, address, datapath_id, port_no): return ref.id +@require_admin_context +def bm_interface_set_vif_uuid(context, if_id, vif_uuid): + session = get_session() + with session.begin(): + bm_interface = model_query(context, models.BareMetalInterface, + read_deleted="no", session=session).\ + filter_by(id=if_id).\ + with_lockmode('update').\ + first() + if not bm_interface: + raise exception.NovaException(_("Baremetal interface %s " + "not found") % if_id) + + bm_interface.vif_uuid = vif_uuid + try: + session.add(bm_interface) + session.flush() + except exception.DBError, e: + # TODO(deva): clean up when db layer raises DuplicateKeyError + if str(e).find('IntegrityError') != -1: + raise exception.NovaException(_("Baremetal interface %s " + "already in use") % vif_uuid) + else: + raise e + + +@require_admin_context +def bm_interface_get_by_vif_uuid(context, vif_uuid): + result = model_query(context, models.BareMetalInterface, + read_deleted="no").\ + filter_by(vif_uuid=vif_uuid).\ + first() + + if not result: + raise exception.NovaException(_("Baremetal virtual interface %s " + "not found") % vif_uuid) + + return result + + @require_admin_context def bm_interface_get_all_by_bm_node_id(context, bm_node_id): result = model_query(context, models.BareMetalInterface, diff --git a/nova/virt/baremetal/driver.py b/nova/virt/baremetal/driver.py index 043fba421..e840c4e75 100644 --- a/nova/virt/baremetal/driver.py +++ b/nova/virt/baremetal/driver.py @@ -40,6 +40,12 @@ opts = [ cfg.StrOpt('baremetal_injected_network_template', default='$pybasedir/nova/virt/baremetal/interfaces.template', help='Template file for injected network'), + cfg.StrOpt('baremetal_vif_driver', + default='nova.virt.baremetal.vif_driver.BareMetalVIFDriver', + help='Baremetal VIF driver.'), + cfg.StrOpt('baremetal_volume_driver', + default='nova.virt.baremetal.volume_driver.LibvirtVolumeDriver', + help='Baremetal volume driver.'), cfg.ListOpt('instance_type_extra_specs', default=[], help='a list of additional capabilities corresponding to ' @@ -110,8 +116,12 @@ class BareMetalDriver(driver.ComputeDriver): self.baremetal_nodes = importutils.import_object( CONF.baremetal_driver) + self._vif_driver = importutils.import_object( + CONF.baremetal_vif_driver) self._firewall_driver = firewall.load_driver( default=DEFAULT_FIREWALL_DRIVER) + self._volume_driver = importutils.import_object( + CONF.baremetal_volume_driver, virtapi) self._image_cache_manager = imagecache.ImageCacheManager() extra_specs = {} @@ -177,6 +187,8 @@ class BareMetalDriver(driver.ComputeDriver): var = self.baremetal_nodes.define_vars(instance, network_info, block_device_info) + self._plug_vifs(instance, network_info, context=context) + self._firewall_driver.setup_basic_filtering(instance, network_info) self._firewall_driver.prepare_instance_filter(instance, network_info) @@ -195,8 +207,15 @@ class BareMetalDriver(driver.ComputeDriver): self.baremetal_nodes.activate_node(var, context, node, instance) self._firewall_driver.apply_instance_filter(instance, network_info) - pm.start_console() + block_device_mapping = driver.block_device_info_get_mapping( + block_device_info) + for vol in block_device_mapping: + connection_info = vol['connection_info'] + mountpoint = vol['mount_device'] + self.attach_volume(connection_info, instance['name'], + mountpoint) + pm.start_console() except Exception, e: # TODO(deva): add tooling that can revert a failed spawn _update_baremetal_state(context, node, instance, @@ -235,6 +254,15 @@ class BareMetalDriver(driver.ComputeDriver): ## power off the node state = pm.deactivate_node() + ## cleanup volumes + # NOTE(vish): we disconnect from volumes regardless + block_device_mapping = driver.block_device_info_get_mapping( + block_device_info) + for vol in block_device_mapping: + connection_info = vol['connection_info'] + mountpoint = vol['mount_device'] + self.detach_volume(connection_info, instance['name'], mountpoint) + self.baremetal_nodes.deactivate_bootloader(var, ctx, node, instance) self.baremetal_nodes.destroy_images(var, ctx, node, instance) @@ -243,6 +271,8 @@ class BareMetalDriver(driver.ComputeDriver): self._firewall_driver.unfilter_instance(instance, network_info=network_info) + self._unplug_vifs(instance, network_info) + _update_baremetal_state(ctx, node, None, state) def power_off(self, instance): @@ -257,6 +287,18 @@ class BareMetalDriver(driver.ComputeDriver): pm = get_power_manager(node) pm.activate_node() + def get_volume_connector(self, instance): + return self._volume_driver.get_volume_connector(instance) + + def attach_volume(self, connection_info, instance_name, mountpoint): + return self._volume_driver.attach_volume(connection_info, + instance_name, mountpoint) + + @exception.wrap_exception() + def detach_volume(self, connection_info, instance_name, mountpoint): + return self._volume_driver.detach_volume(connection_info, + instance_name, mountpoint) + def get_info(self, instance): # NOTE(deva): compute/manager.py expects to get NotFound exception # so we convert from InstanceNotFound @@ -355,6 +397,26 @@ class BareMetalDriver(driver.ComputeDriver): caps.append(data) return caps + def plug_vifs(self, instance, network_info): + """Plugin VIFs into networks.""" + self._plug_vifs(instance, network_info) + + def _plug_vifs(self, instance, network_info, context=None): + if not context: + context = nova_context.get_admin_context() + node = _get_baremetal_node_by_instance_uuid(instance['uuid']) + if node: + pifs = bmdb.bm_interface_get_all_by_bm_node_id(context, node['id']) + for pif in pifs: + if pif['vif_uuid']: + bmdb.bm_interface_set_vif_uuid(context, pif['id'], None) + for (network, mapping) in network_info: + self._vif_driver.plug(instance, (network, mapping)) + + def _unplug_vifs(self, instance, network_info): + for (network, mapping) in network_info: + self._vif_driver.unplug(instance, (network, mapping)) + def manage_image_cache(self, context, all_instances): """Manage the local cache of images.""" self._image_cache_manager.verify_base_images(context, all_instances) diff --git a/nova/virt/baremetal/vif_driver.py b/nova/virt/baremetal/vif_driver.py new file mode 100644 index 000000000..2dc03410b --- /dev/null +++ b/nova/virt/baremetal/vif_driver.py @@ -0,0 +1,75 @@ +# Copyright (c) 2012 NTT DOCOMO, INC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import context +from nova import exception +from nova.openstack.common import cfg +from nova.openstack.common import log as logging +from nova.virt.baremetal import db as bmdb +from nova.virt.vif import VIFDriver + +CONF = cfg.CONF + +LOG = logging.getLogger(__name__) + + +class BareMetalVIFDriver(VIFDriver): + + def _after_plug(self, instance, network, mapping, pif): + pass + + def _after_unplug(self, instance, network, mapping, pif): + pass + + def plug(self, instance, vif): + LOG.debug(_("plug: instance_uuid=%(uuid)s vif=%(vif)s") + % {'uuid': instance['uuid'], 'vif': vif}) + network, mapping = vif + vif_uuid = mapping['vif_uuid'] + ctx = context.get_admin_context() + node = bmdb.bm_node_get_by_instance_uuid(ctx, instance['uuid']) + + # TODO(deva): optimize this database query + # this is just searching for a free physical interface + pifs = bmdb.bm_interface_get_all_by_bm_node_id(ctx, node['id']) + for pif in pifs: + if not pif['vif_uuid']: + bmdb.bm_interface_set_vif_uuid(ctx, pif['id'], vif_uuid) + LOG.debug(_("pif:%(id)s is plugged (vif_uuid=%(vif_uuid)s)") + % {'id': pif['id'], 'vif_uuid': vif_uuid}) + self._after_plug(instance, network, mapping, pif) + return + + # NOTE(deva): should this really be raising an exception + # when there are no physical interfaces left? + raise exception.NovaException(_( + "Baremetal node: %(id)s has no available physical interface" + " for virtual interface %(vif_uuid)s") + % {'id': node['id'], 'vif_uuid': vif_uuid}) + + def unplug(self, instance, vif): + LOG.debug(_("unplug: instance_uuid=%(uuid)s vif=%(vif)s"), + {'uuid': instance['uuid'], 'vif': vif}) + network, mapping = vif + vif_uuid = mapping['vif_uuid'] + ctx = context.get_admin_context() + try: + pif = bmdb.bm_interface_get_by_vif_uuid(ctx, vif_uuid) + bmdb.bm_interface_set_vif_uuid(ctx, pif['id'], None) + LOG.debug(_("pif:%(id)s is unplugged (vif_uuid=%(vif_uuid)s)") + % {'id': pif['id'], 'vif_uuid': vif_uuid}) + self._after_unplug(instance, network, mapping, pif) + except exception.NovaException: + LOG.warn(_("no pif for vif_uuid=%s") % vif_uuid) diff --git a/nova/virt/baremetal/volume_driver.py b/nova/virt/baremetal/volume_driver.py new file mode 100644 index 000000000..bf8e47a63 --- /dev/null +++ b/nova/virt/baremetal/volume_driver.py @@ -0,0 +1,277 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# coding=utf-8 + +# Copyright (c) 2012 NTT DOCOMO, INC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re + +from nova import context as nova_context +from nova import exception +from nova.openstack.common import cfg +from nova.openstack.common import importutils +from nova.openstack.common import log as logging +from nova import utils +from nova.virt.baremetal import db as bmdb +from nova.virt import driver +from nova.virt.libvirt import utils as libvirt_utils + +opts = [ + cfg.BoolOpt('baremetal_use_unsafe_iscsi', + default=False, + help='Do not set this out of dev/test environments. ' + 'If a node does not have an fixed PXE IP address, ' + 'volumes are exported with globally opened ACL'), + cfg.StrOpt('baremetal_iscsi_iqn_prefix', + default='iqn.2010-10.org.openstack.baremetal', + help='iSCSI IQN prefix used in baremetal volume connections.'), + ] + +CONF = cfg.CONF +CONF.register_opts(opts) + +CONF.import_opt('libvirt_volume_drivers', 'nova.virt.libvirt.driver') + +LOG = logging.getLogger(__name__) + + +def _get_baremetal_node_by_instance_name(virtapi, instance_name): + context = nova_context.get_admin_context() + # TODO(deva): optimize this DB query. + # I don't think it should be _get_all + for node in bmdb.bm_node_get_all(context, service_host=CONF.host): + if not node['instance_uuid']: + continue + try: + inst = virtapi.instance_get_by_uuid(context, node['instance_uuid']) + if inst['name'] == instance_name: + return node + except exception.InstanceNotFound: + continue + + # raise exception if we found no matching instance + raise exception.InstanceNotFound(instance_name) + + +def _create_iscsi_export_tgtadm(path, tid, iqn): + utils.execute('tgtadm', '--lld', 'iscsi', + '--mode', 'target', + '--op', 'new', + '--tid', tid, + '--targetname', iqn, + run_as_root=True) + utils.execute('tgtadm', '--lld', 'iscsi', + '--mode', 'logicalunit', + '--op', 'new', + '--tid', tid, + '--lun', '1', + '--backing-store', path, + run_as_root=True) + + +def _allow_iscsi_tgtadm(tid, address): + utils.execute('tgtadm', '--lld', 'iscsi', + '--mode', 'target', + '--op', 'bind', + '--tid', tid, + '--initiator-address', address, + run_as_root=True) + + +def _delete_iscsi_export_tgtadm(tid): + try: + utils.execute('tgtadm', '--lld', 'iscsi', + '--mode', 'logicalunit', + '--op', 'delete', + '--tid', tid, + '--lun', '1', + run_as_root=True) + except exception.ProcessExecutionError: + pass + try: + utils.execute('tgtadm', '--lld', 'iscsi', + '--mode', 'target', + '--op', 'delete', + '--tid', tid, + run_as_root=True) + except exception.ProcessExecutionError: + pass + # Check if the tid is deleted, that is, check the tid no longer exists. + # If the tid dose not exist, tgtadm returns with exit_code 22. + # utils.execute() can check the exit_code if check_exit_code parameter is + # passed. But, regardless of whether check_exit_code contains 0 or not, + # if the exit_code is 0, the function dose not report errors. So we have to + # catch a ProcessExecutionError and test its exit_code is 22. + try: + utils.execute('tgtadm', '--lld', 'iscsi', + '--mode', 'target', + '--op', 'show', + '--tid', tid, + run_as_root=True) + except exception.ProcessExecutionError as e: + if e.exit_code == 22: + # OK, the tid is deleted + return + raise + raise exception.NovaException(_( + 'baremetal driver was unable to delete tid %s') % tid) + + +def _show_tgtadm(): + out, _ = utils.execute('tgtadm', '--lld', 'iscsi', + '--mode', 'target', + '--op', 'show', + run_as_root=True) + return out + + +def _list_backingstore_path(): + out = _show_tgtadm() + l = [] + for line in out.split('\n'): + m = re.search(r'Backing store path: (.*)$', line) + if m: + if '/' in m.group(1): + l.append(m.group(1)) + return l + + +def _get_next_tid(): + out = _show_tgtadm() + last_tid = 0 + for line in out.split('\n'): + m = re.search(r'^Target (\d+):', line) + if m: + tid = int(m.group(1)) + if last_tid < tid: + last_tid = tid + return last_tid + 1 + + +def _find_tid(iqn): + out = _show_tgtadm() + pattern = r'^Target (\d+): *' + re.escape(iqn) + for line in out.split('\n'): + m = re.search(pattern, line) + if m: + return int(m.group(1)) + return None + + +def _get_iqn(instance_name, mountpoint): + mp = mountpoint.replace('/', '-').strip('-') + iqn = '%s:%s-%s' % (CONF.baremetal_iscsi_iqn_prefix, + instance_name, + mp) + return iqn + + +class VolumeDriver(object): + + def __init__(self, virtapi): + super(VolumeDriver, self).__init__() + self.virtapi = virtapi + self._initiator = None + + def get_volume_connector(self, instance): + if not self._initiator: + self._initiator = libvirt_utils.get_iscsi_initiator() + if not self._initiator: + LOG.warn(_('Could not determine iscsi initiator name ' + 'for instance %s') % instance) + return { + 'ip': CONF.my_ip, + 'initiator': self._initiator, + 'host': CONF.host, + } + + def attach_volume(self, connection_info, instance_name, mountpoint): + raise NotImplementedError() + + def detach_volume(self, connection_info, instance_name, mountpoint): + raise NotImplementedError() + + +class LibvirtVolumeDriver(VolumeDriver): + """The VolumeDriver deligates to nova.virt.libvirt.volume.""" + + def __init__(self, virtapi): + super(LibvirtVolumeDriver, self).__init__(virtapi) + self.volume_drivers = {} + for driver_str in CONF.libvirt_volume_drivers: + driver_type, _sep, driver = driver_str.partition('=') + driver_class = importutils.import_class(driver) + self.volume_drivers[driver_type] = driver_class(self) + + def _volume_driver_method(self, method_name, connection_info, + *args, **kwargs): + driver_type = connection_info.get('driver_volume_type') + if not driver_type in self.volume_drivers: + raise exception.VolumeDriverNotFound(driver_type=driver_type) + driver = self.volume_drivers[driver_type] + method = getattr(driver, method_name) + return method(connection_info, *args, **kwargs) + + def attach_volume(self, connection_info, instance_name, mountpoint): + node = _get_baremetal_node_by_instance_name(self.virtapi, + instance_name) + ctx = nova_context.get_admin_context() + pxe_ip = bmdb.bm_pxe_ip_get_by_bm_node_id(ctx, node['id']) + if not pxe_ip: + if not CONF.baremetal_use_unsafe_iscsi: + raise exception.NovaException(_( + 'No fixed PXE IP is associated to %s') % instance_name) + + mount_device = mountpoint.rpartition("/")[2] + self._volume_driver_method('connect_volume', + connection_info, + mount_device) + device_path = connection_info['data']['device_path'] + iqn = _get_iqn(instance_name, mountpoint) + tid = _get_next_tid() + _create_iscsi_export_tgtadm(device_path, tid, iqn) + + if pxe_ip: + _allow_iscsi_tgtadm(tid, pxe_ip['address']) + else: + # NOTE(NTTdocomo): Since nova-compute does not know the + # instance's initiator ip, it allows any initiators + # to connect to the volume. This means other bare-metal + # instances that are not attached the volume can connect + # to the volume. Do not set CONF.baremetal_use_unsafe_iscsi + # out of dev/test environments. + # TODO(NTTdocomo): support CHAP + _allow_iscsi_tgtadm(tid, 'ALL') + + @exception.wrap_exception() + def detach_volume(self, connection_info, instance_name, mountpoint): + mount_device = mountpoint.rpartition("/")[2] + try: + iqn = _get_iqn(instance_name, mountpoint) + tid = _find_tid(iqn) + if tid is not None: + _delete_iscsi_export_tgtadm(tid) + else: + LOG.warn(_('detach volume could not find tid for %s') % iqn) + finally: + self._volume_driver_method('disconnect_volume', + connection_info, + mount_device) + + def get_all_block_devices(self): + """ + Return all block devices in use on this node. + """ + return _list_backingstore_path() -- cgit