Adding live migration server actions

Change-Id: I5e1f5dddaf45d1c6eae8666647425bff748b639e

12 files changed, 323 insertions, 59 deletions

diff --git a/nova/api/openstack/compute/contrib/admin_actions.py b/nova/api/openstack/compute/contrib/admin_actions.py
index cbd54117d..f68126fb2 100644
--- a/nova/api/openstack/compute/contrib/admin_actions.py
+++ b/nova/api/openstack/compute/contrib/admin_actions.py
@@ -272,6 +272,43 @@ class AdminActionsController(wsgi.Controller):
         resp.headers['Location'] = image_ref
         return resp
 
+    @wsgi.action('os-migrateLive')
+    @exception.novaclient_converter
+    @scheduler_api.redirect_handler
+    def _migrate_live(self, req, id, body):
+        """Permit admins to (live) migrate a server to a new host"""
+        context = req.environ["nova.context"]
+        # Expected to use AuthMiddleware.
+        # Otherwise, non-admin user can use live migration
+        if not context.is_admin:
+            msg = _("Live migration is admin only functionality")
+            raise exc.HTTPForbidden(explanation=msg)
+
+        try:
+            block_migration = body["os-migrateLive"]["block_migration"]
+            disk_over_commit = body["os-migrateLive"]["disk_over_commit"]
+            host = body["os-migrateLive"]["host"]
+        except (TypeError, KeyError):
+            msg = _("host and block_migration must be specified.")
+            raise exc.HTTPBadRequest(explanation=msg)
+
+        try:
+            instance = self.compute_api.get(context, id)
+            result = scheduler_api.live_migration(context,
+                                                  block_migration,
+                                                  disk_over_commit,
+                                                  instance["id"],
+                                                  host,
+                                                  topic=FLAGS.compute_topic)
+        except Exception, e:
+            msg = _("Live migration of instance %(id)s to host %(host)s"
+                    " failed") % locals()
+            LOG.exception(msg)
+            # Return messages from scheduler
+            raise exc.HTTPBadRequest(explanation=msg)
+
+        return webob.Response(status_int=202)
+
 
 class Admin_actions(extensions.ExtensionDescriptor):
     """Enable admin-only server actions
diff --git a/nova/api/openstack/compute/contrib/hosts.py b/nova/api/openstack/compute/contrib/hosts.py
index b522e6a98..53f1a064a 100644
--- a/nova/api/openstack/compute/contrib/hosts.py
+++ b/nova/api/openstack/compute/contrib/hosts.py
@@ -23,6 +23,7 @@ from nova.api.openstack import wsgi
 from nova.api.openstack import xmlutil
 from nova.api.openstack import extensions
 from nova import compute
+from nova import db
 from nova import exception
 from nova import flags
 from nova import log as logging
@@ -82,6 +83,15 @@ class HostDeserializer(wsgi.XMLDeserializer):
         return dict(body=updates)
 
 
+class HostShowTemplate(xmlutil.TemplateBuilder):
+    def construct(self):
+        root = xmlutil.TemplateElement('host', selector='host')
+        root.set('resource')
+        root.set('usage')
+
+        return xmlutil.MasterTemplate(root, 1)
+
+
 def _list_hosts(req, service=None):
     """Returns a summary list of hosts, optionally filtering
     by service type.
@@ -176,6 +186,72 @@ class HostController(object):
     def reboot(self, req, id):
         return self._host_power_action(req, host=id, action="reboot")
 
+    @wsgi.serializers(xml=HostShowTemplate)
+    def show(self, req, id):
+        """Shows the physical/usage resource given by hosts.
+
+        :param context: security context
+        :param host: hostname
+        :returns:
+            {'host': {'resource':D1, 'usage':{proj_id1:D2,..}}}
+
+            'resource' shows "available" and "in-use" vcpus, memory and disk.
+            'usage' shows "in-use" vcpus, memory and disk per project.
+
+            D1: {'vcpus': 16, 'memory_mb': 2048, 'local_gb': 2048,
+                 'vcpus_used': 12, 'memory_mb_used': 10240,
+                 'local_gb_used': 64}
+            D2: {'vcpus': 1, 'memory_mb': 2048, 'local_gb': 20}
+        """
+        host = id
+        context = req.environ['nova.context']
+        # Expected to use AuthMiddleware.
+        # Otherwise, non-admin user can use describe-resource
+        if not context.is_admin:
+            msg = _("Describe-resource is admin only functionality")
+            raise webob.exc.HTTPForbidden(explanation=msg)
+
+        # Getting compute node info and related instances info
+        try:
+            compute_ref = db.service_get_all_compute_by_host(context, host)
+            compute_ref = compute_ref[0]
+        except exception.ComputeHostNotFound:
+            raise webob.exc.HTTPNotFound(explanation=_("Host not found"))
+        instance_refs = db.instance_get_all_by_host(context,
+                                                    compute_ref['host'])
+
+        # Getting total available/used resource
+        compute_ref = compute_ref['compute_node'][0]
+        resource = {'vcpus': compute_ref['vcpus'],
+                    'memory_mb': compute_ref['memory_mb'],
+                    'local_gb': compute_ref['local_gb'],
+                    'vcpus_used': compute_ref['vcpus_used'],
+                    'memory_mb_used': compute_ref['memory_mb_used'],
+                    'local_gb_used': compute_ref['local_gb_used']}
+        usage = dict()
+        if not instance_refs:
+            return {'host':
+                    {'resource': resource, 'usage': usage}}
+
+        # Getting usage resource per project
+        project_ids = [i['project_id'] for i in instance_refs]
+        project_ids = list(set(project_ids))
+        for project_id in project_ids:
+            vcpus = [i['vcpus'] for i in instance_refs
+                     if i['project_id'] == project_id]
+
+            mem = [i['memory_mb']  for i in instance_refs
+                   if i['project_id'] == project_id]
+
+            disk = [i['root_gb'] + i['ephemeral_gb'] for i in instance_refs
+                    if i['project_id'] == project_id]
+
+            usage[project_id] = {'vcpus': reduce(lambda x, y: x + y, vcpus),
+                                 'memory_mb': reduce(lambda x, y: x + y, mem),
+                                 'local_gb': reduce(lambda x, y: x + y, disk)}
+
+        return {'host': {'resource': resource, 'usage': usage}}
+
 
 class Hosts(extensions.ExtensionDescriptor):
     """Admin-only host administration"""
diff --git a/nova/compute/manager.py b/nova/compute/manager.py
index b66f9ebe5..6f635a34f 100644
--- a/nova/compute/manager.py
+++ b/nova/compute/manager.py
@@ -1696,16 +1696,6 @@ class ComputeManager(manager.SchedulerDependentManager):
         tmp_file = os.path.join(FLAGS.instances_path, filename)
         os.remove(tmp_file)
 
-    @exception.wrap_exception(notifier=notifier, publisher_id=publisher_id())
-    def update_available_resource(self, context):
-        """See comments update_resource_info.
-
-        :param context: security context
-        :returns: See driver.update_available_resource()
-
-        """
-        self.driver.update_available_resource(context, self.host)
-
     def get_instance_disk_info(self, context, instance_name):
         """Getting infomation of instance's current disk.
 
@@ -2117,6 +2107,16 @@ class ComputeManager(manager.SchedulerDependentManager):
                          locals())
                 self._delete_instance(context, instance)
 
+    @manager.periodic_task
+    def update_available_resource(self, context):
+        """See driver.update_available_resource()
+
+        :param context: security context
+        :returns: See driver.update_available_resource()
+
+        """
+        self.driver.update_available_resource(context, self.host)
+
     def add_instance_fault_from_exc(self, context, instance_uuid, fault):
         """Adds the specified fault to the database."""
         if hasattr(fault, "code"):
diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py
index b05148651..59f93b952 100644
--- a/nova/scheduler/api.py
+++ b/nova/scheduler/api.py
@@ -419,3 +419,15 @@ def redirect_handler(f):
                 raise e.results
             return e.results
     return new_f
+
+
+def live_migration(context, block_migration, disk_over_commit,
+                   instance_id, dest, topic):
+    """Migrate a server to a new host"""
+    params = {"instance_id": instance_id,
+              "dest": dest,
+              "topic": topic,
+              "block_migration": block_migration,
+              "disk_over_commit": disk_over_commit}
+    return _call_scheduler("live_migration", context=context,
+                           params=params)
diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py
index 28975e004..06f613a45 100644
--- a/nova/scheduler/driver.py
+++ b/nova/scheduler/driver.py
@@ -446,11 +446,6 @@ class Scheduler(object):
         # if disk_over_commit is True,
         #  otherwise virtual disk size < available disk size.
 
-        # Refresh compute_nodes table
-        topic = db.queue_get_for(context, FLAGS.compute_topic, dest)
-        rpc.call(context, topic,
-                 {"method": "update_available_resource"})
-
         # Getting total available disk of host
         available_gb = self._get_compute_info(context,
                                               dest, 'disk_available_least')
diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py
index 99f51fab2..4a701496c 100644
--- a/nova/scheduler/manager.py
+++ b/nova/scheduler/manager.py
@@ -143,10 +143,6 @@ class SchedulerManager(manager.Manager):
                 'local_gb_used': 64}
 
         """
-        # Update latest compute_node table
-        topic = db.queue_get_for(context, FLAGS.compute_topic, host)
-        rpc.call(context, topic, {"method": "update_available_resource"})
-
         # Getting compute node info and related instances info
         compute_ref = db.service_get_all_compute_by_host(context, host)
         compute_ref = compute_ref[0]
diff --git a/nova/tests/api/openstack/compute/contrib/test_admin_actions.py b/nova/tests/api/openstack/compute/contrib/test_admin_actions.py
index f572b12d9..fca5f309f 100644
--- a/nova/tests/api/openstack/compute/contrib/test_admin_actions.py
+++ b/nova/tests/api/openstack/compute/contrib/test_admin_actions.py
@@ -21,10 +21,12 @@ from nova.api.openstack  import compute as compute_api
 from nova.api.openstack.compute import extensions
 from nova.api.openstack import wsgi
 from nova import compute
+from nova import context
 from nova import exception
 from nova import flags
 from nova import test
 from nova import utils
+from nova.scheduler import api as scheduler_api
 from nova.tests.api.openstack import fakes
 
 
@@ -59,6 +61,12 @@ def fake_compute_api_get(self, context, instance_id):
     return {'id': 1, 'uuid': instance_id}
 
 
+def fake_scheduler_api_live_migration(context, block_migration,
+                                      disk_over_commit, instance_id,
+                                      dest, topic):
+    return None
+
+
 class AdminActionsTest(test.TestCase):
 
     _actions = ('pause', 'unpause', 'suspend', 'resume', 'migrate',
@@ -81,6 +89,9 @@ class AdminActionsTest(test.TestCase):
         self.UUID = utils.gen_uuid()
         for _method in self._methods:
             self.stubs.Set(compute.API, _method, fake_compute_api)
+        self.stubs.Set(scheduler_api,
+                       'live_migration',
+                       fake_scheduler_api_live_migration)
 
     def test_admin_api_actions(self):
         self.maxDiff = None
@@ -112,6 +123,51 @@ class AdminActionsTest(test.TestCase):
             self.assertIn("invalid state for '%(_action)s'" % locals(),
                     res.body)
 
+    def test_migrate_live_enabled(self):
+        ctxt = context.get_admin_context()
+        ctxt.user_id = 'fake'
+        ctxt.project_id = 'fake'
+        ctxt.is_admin = True
+        app = fakes.wsgi_app(fake_auth_context=ctxt)
+        req = webob.Request.blank('/v2/fake/servers/%s/action' % self.UUID)
+        req.method = 'POST'
+        req.body = json.dumps({'os-migrateLive': {'host': 'hostname',
+                                               'block_migration': False,
+                                               'disk_over_commit': False}})
+        req.content_type = 'application/json'
+        res = req.get_response(app)
+        self.assertEqual(res.status_int, 202)
+
+    def test_migrate_live_forbidden(self):
+        ctxt = context.get_admin_context()
+        ctxt.user_id = 'fake'
+        ctxt.project_id = 'fake'
+        ctxt.is_admin = False
+        app = fakes.wsgi_app(fake_auth_context=ctxt)
+        req = webob.Request.blank('/v2/fake/servers/%s/action' % self.UUID)
+        req.method = 'POST'
+        req.body = json.dumps({'os-migrateLive': {'host': 'hostname',
+                                               'block_migration': False,
+                                               'disk_over_commit': False}})
+        req.content_type = 'application/json'
+        res = req.get_response(app)
+        self.assertEqual(res.status_int, 403)
+
+    def test_migrate_live_missing_dict_param(self):
+        ctxt = context.get_admin_context()
+        ctxt.user_id = 'fake'
+        ctxt.project_id = 'fake'
+        ctxt.is_admin = True
+        app = fakes.wsgi_app(fake_auth_context=ctxt)
+        req = webob.Request.blank('/v2/fake/servers/%s/action' % self.UUID)
+        req.method = 'POST'
+        req.body = json.dumps({'os-migrateLive': {'dummy': 'hostname',
+                                               'block_migration': False,
+                                               'disk_over_commit': False}})
+        req.content_type = 'application/json'
+        res = req.get_response(app)
+        self.assertEqual(res.status_int, 400)
+
 
 class CreateBackupTests(test.TestCase):
 
diff --git a/nova/tests/api/openstack/compute/contrib/test_hosts.py b/nova/tests/api/openstack/compute/contrib/test_hosts.py
index f0547eff7..925dc0d53 100644
--- a/nova/tests/api/openstack/compute/contrib/test_hosts.py
+++ b/nova/tests/api/openstack/compute/contrib/test_hosts.py
@@ -17,11 +17,14 @@ from lxml import etree
 import webob.exc
 
 from nova import context
+from nova import db
 from nova import exception
 from nova import flags
 from nova import log as logging
 from nova import test
 from nova.api.openstack.compute.contrib import hosts as os_hosts
+from nova.compute import power_state
+from nova.compute import vm_states
 from nova.scheduler import api as scheduler_api
 
 
@@ -52,6 +55,35 @@ def stub_host_power_action(context, host, action):
     return action
 
 
+def _create_instance(**kwargs):
+    """Create a test instance"""
+    ctxt = context.get_admin_context()
+    return db.instance_create(ctxt, _create_instance_dict(**kwargs))
+
+
+def _create_instance_dict(**kwargs):
+    """Create a dictionary for a test instance"""
+    inst = {}
+    inst['image_ref'] = 'cedef40a-ed67-4d10-800e-17455edce175'
+    inst['reservation_id'] = 'r-fakeres'
+    inst['user_id'] = kwargs.get('user_id', 'admin')
+    inst['project_id'] = kwargs.get('project_id', 'fake')
+    inst['instance_type_id'] = '1'
+    if 'host' in kwargs:
+        inst['host'] = kwargs.get('host')
+    inst['vcpus'] = kwargs.get('vcpus', 1)
+    inst['memory_mb'] = kwargs.get('memory_mb', 20)
+    inst['root_gb'] = kwargs.get('root_gb', 30)
+    inst['ephemeral_gb'] = kwargs.get('ephemeral_gb', 30)
+    inst['vm_state'] = kwargs.get('vm_state', vm_states.ACTIVE)
+    inst['power_state'] = kwargs.get('power_state', power_state.RUNNING)
+    inst['task_state'] = kwargs.get('task_state', None)
+    inst['availability_zone'] = kwargs.get('availability_zone', None)
+    inst['ami_launch_index'] = 0
+    inst['launched_on'] = kwargs.get('launched_on', 'dummy')
+    return inst
+
+
 class FakeRequest(object):
     environ = {"nova.context": context.get_admin_context()}
 
@@ -136,6 +168,102 @@ class HostTestCase(test.TestCase):
         self.assertRaises(exception.HostNotFound, self.controller.update,
                 self.req, "bogus_host_name", body={"status": "disable"})
 
+    def test_show_forbidden(self):
+        self.req.environ["nova.context"].is_admin = False
+        dest = 'dummydest'
+        self.assertRaises(webob.exc.HTTPForbidden,
+                          self.controller.show,
+                          self.req, dest)
+        self.req.environ["nova.context"].is_admin = True
+
+    def test_show_host_not_exist(self):
+        """A host given as an argument does not exists."""
+        self.req.environ["nova.context"].is_admin = True
+        dest = 'dummydest'
+        self.assertRaises(webob.exc.HTTPNotFound,
+                          self.controller.show,
+                          self.req, dest)
+
+    def _dic_is_equal(self, dic1, dic2, keys=None):
+        """Compares 2 dictionary contents(Helper method)"""
+        if not keys:
+            keys = ['vcpus', 'memory_mb', 'local_gb',
+                    'vcpus_used', 'memory_mb_used', 'local_gb_used']
+
+        for key in keys:
+            if not (dic1[key] == dic2[key]):
+                return False
+        return True
+
+    def _create_compute_service(self):
+        """Create compute-manager(ComputeNode and Service record)."""
+        ctxt = context.get_admin_context()
+        dic = {'host': 'dummy', 'binary': 'nova-compute', 'topic': 'compute',
+               'report_count': 0, 'availability_zone': 'dummyzone'}
+        s_ref = db.service_create(ctxt, dic)
+
+        dic = {'service_id': s_ref['id'],
+               'vcpus': 16, 'memory_mb': 32, 'local_gb': 100,
+               'vcpus_used': 16, 'memory_mb_used': 32, 'local_gb_used': 10,
+               'hypervisor_type': 'qemu', 'hypervisor_version': 12003,
+               'cpu_info': ''}
+        db.compute_node_create(ctxt, dic)
+
+        return db.service_get(ctxt, s_ref['id'])
+
+    def test_show_no_project(self):
+        """No instance are running on the given host."""
+        ctxt = context.get_admin_context()
+        s_ref = self._create_compute_service()
+
+        result = self.controller.show(self.req, s_ref['host'])
+
+        # result checking
+        c1 = ('resource' in result['host'] and
+              'usage' in result['host'])
+        compute_node = s_ref['compute_node'][0]
+        c2 = self._dic_is_equal(result['host']['resource'],
+                                compute_node)
+        c3 = result['host']['usage'] == {}
+        self.assertTrue(c1 and c2 and c3)
+        db.service_destroy(ctxt, s_ref['id'])
+
+    def test_show_works_correctly(self):
+        """show() works correctly as expected."""
+        ctxt = context.get_admin_context()
+        s_ref = self._create_compute_service()
+        i_ref1 = _create_instance(project_id='p-01', host=s_ref['host'])
+        i_ref2 = _create_instance(project_id='p-02', vcpus=3,
+                                       host=s_ref['host'])
+
+        result = self.controller.show(self.req, s_ref['host'])
+
+        c1 = ('resource' in result['host'] and
+              'usage' in result['host'])
+        compute_node = s_ref['compute_node'][0]
+        c2 = self._dic_is_equal(result['host']['resource'],
+                                compute_node)
+        c3 = result['host']['usage'].keys() == ['p-01', 'p-02']
+        keys = ['vcpus', 'memory_mb']
+        c4 = self._dic_is_equal(
+                 result['host']['usage']['p-01'], i_ref1, keys)
+        disk = i_ref2['root_gb'] + i_ref2['ephemeral_gb']
+        if result['host']['usage']['p-01']['local_gb'] == disk:
+            c6 = True
+        else:
+            c6 = False
+        c5 = self._dic_is_equal(
+                 result['host']['usage']['p-02'], i_ref2, keys)
+        if result['host']['usage']['p-02']['local_gb'] == disk:
+            c7 = True
+        else:
+            c7 = False
+        self.assertTrue(c1 and c2 and c3 and c4 and c5 and c6 and c7)
+
+        db.service_destroy(ctxt, s_ref['id'])
+        db.instance_destroy(ctxt, i_ref1['id'])
+        db.instance_destroy(ctxt, i_ref2['id'])
+
 
 class HostSerializerTest(test.TestCase):
     def setUp(self):
diff --git a/nova/tests/scheduler/test_scheduler.py b/nova/tests/scheduler/test_scheduler.py
index 26e1aaaba..7f8645db8 100644
--- a/nova/tests/scheduler/test_scheduler.py
+++ b/nova/tests/scheduler/test_scheduler.py
@@ -446,10 +446,6 @@ class SchedulerTestCase(test.TestCase):
         db.instance_get_all_by_host(self.context, dest).AndReturn(
                 [dict(memory_mb=256), dict(memory_mb=512)])
         # assert_compute_node_has_enough_disk()
-        db.queue_get_for(self.context, FLAGS.compute_topic,
-                dest).AndReturn('dest_queue1')
-        rpc.call(self.context, 'dest_queue1',
-                {'method': 'update_available_resource'})
         self.driver._get_compute_info(self.context, dest,
                 'disk_available_least').AndReturn(1025)
         db.queue_get_for(self.context, FLAGS.compute_topic,
@@ -698,10 +694,6 @@ class SchedulerTestCase(test.TestCase):
                 instance, dest)
 
         # Not enough disk
-        db.queue_get_for(self.context, FLAGS.compute_topic,
-                dest).AndReturn('dest_queue')
-        rpc.call(self.context, 'dest_queue',
-                {'method': 'update_available_resource'})
         self.driver._get_compute_info(self.context, dest,
                 'disk_available_least').AndReturn(1023)
         db.queue_get_for(self.context, FLAGS.compute_topic,
diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py
index f416e4a51..81b7cd48a 100644
--- a/nova/tests/test_compute.py
+++ b/nova/tests/test_compute.py
@@ -151,6 +151,7 @@ class BaseTestCase(test.TestCase):
         type_id = instance_types.get_instance_type_by_name(type_name)['id']
         inst['instance_type_id'] = type_id
         inst['ami_launch_index'] = 0
+        inst['memory_mb'] = 0
         inst['root_gb'] = 0
         inst['ephemeral_gb'] = 0
         inst.update(params)
diff --git a/nova/virt/fake.py b/nova/virt/fake.py
index 8c6d5481b..780d644eb 100644
--- a/nova/virt/fake.py
+++ b/nova/virt/fake.py
@@ -244,36 +244,7 @@ class FakeConnection(driver.ComputeDriver):
         pass
 
     def update_available_resource(self, ctxt, host):
-        """Updates compute manager resource info on ComputeNode table.
-
-        Since we don't have a real hypervisor, pretend we have lots of
-        disk and ram.
-        """
-
-        try:
-            service_ref = db.service_get_all_compute_by_host(ctxt, host)[0]
-        except exception.NotFound:
-            raise exception.ComputeServiceUnavailable(host=host)
-
-        # Updating host information
-        dic = {'vcpus': 1,
-               'memory_mb': 4096,
-               'local_gb': 1028,
-               'vcpus_used': 0,
-               'memory_mb_used': 0,
-               'local_gb_used': 0,
-               'hypervisor_type': 'fake',
-               'hypervisor_version': '1.0',
-               'service_id': service_ref['id'],
-               'cpu_info': '?'}
-
-        compute_node_ref = service_ref['compute_node']
-        if not compute_node_ref:
-            LOG.info(_('Compute_service record created for %s ') % host)
-            db.compute_node_create(ctxt, dic)
-        else:
-            LOG.info(_('Compute_service record updated for %s ') % host)
-            db.compute_node_update(ctxt, compute_node_ref[0]['id'], dic)
+        pass
 
     def compare_cpu(self, xml):
         """This method is supported only by libvirt."""
diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py
index 8ebaa276f..1c8774548 100644
--- a/nova/virt/libvirt/connection.py
+++ b/nova/virt/libvirt/connection.py
@@ -1540,8 +1540,8 @@ class LibvirtConnection(driver.ComputeDriver):
     def update_available_resource(self, ctxt, host):
         """Updates compute manager resource info on ComputeNode table.
 
-        This method is called when nova-coompute launches, and
-        whenever admin executes "nova-manage service update_resource".
+        This method is called as an periodic tasks and is used only
+        in live migration currently.
 
         :param ctxt: security context
         :param host: hostname that compute manager is currently running