First implementation of bp/live-migration-resource-calc

Fix based on revewer's comment

upgraded the migration version
nova/db/sqlalchemy/migrate_repo/versions/069_block_migration.py

rebase on master

Change-Id: Ia762f8dec761c3d595bc6fcd39f127f6d92306d2

6 files changed, 75 insertions, 86 deletions

diff --git a/nova/tests/fake_libvirt_utils.py b/nova/tests/fake_libvirt_utils.py
index 085a61bc9..b05f11178 100644
--- a/nova/tests/fake_libvirt_utils.py
+++ b/nova/tests/fake_libvirt_utils.py
@@ -33,7 +33,7 @@ def get_disk_size(path):
     return disk_sizes.get(path, 1024 * 1024 * 20)
 
 
-def get_backing_file(path):
+def get_disk_backing_file(path):
     return disk_backing_files.get(path, None)
 
 
diff --git a/nova/tests/scheduler/test_scheduler.py b/nova/tests/scheduler/test_scheduler.py
index 48f678d56..550964844 100644
--- a/nova/tests/scheduler/test_scheduler.py
+++ b/nova/tests/scheduler/test_scheduler.py
@@ -36,6 +36,7 @@ from nova import rpc
 from nova import utils
 from nova.scheduler import api
 from nova.scheduler import driver
+from nova.scheduler import zone_manager
 from nova.scheduler import manager
 from nova.scheduler.simple import SimpleScheduler
 from nova.compute import power_state
@@ -999,9 +1000,9 @@ class SimpleDriverTestCase(test.TestCase):
         self.mox.StubOutWithMock(driver_i, '_live_migration_common_check')
         driver_i._live_migration_src_check(nocare, nocare)
         driver_i._live_migration_dest_check(nocare, nocare,
-                                            i_ref['host'], False)
+                                            i_ref['host'], False, False)
         driver_i._live_migration_common_check(nocare, nocare,
-                                              i_ref['host'], False)
+                                            i_ref['host'], False, False)
         self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True)
         kwargs = {'instance_id': instance_id, 'dest': i_ref['host'],
                   'block_migration': False}
@@ -1013,7 +1014,8 @@ class SimpleDriverTestCase(test.TestCase):
         self.scheduler.live_migration(self.context, FLAGS.compute_topic,
                                       instance_id=instance_id,
                                       dest=i_ref['host'],
-                                      block_migration=False)
+                                      block_migration=False,
+                                      disk_over_commit=False)
 
         i_ref = db.instance_get(self.context, instance_id)
         self.assertTrue(i_ref['vm_state'] == vm_states.MIGRATING)
@@ -1095,7 +1097,22 @@ class SimpleDriverTestCase(test.TestCase):
 
         self.assertRaises(exception.ComputeServiceUnavailable,
                           self.scheduler.driver._live_migration_dest_check,
-                          self.context, i_ref, i_ref['host'], False)
+                          self.context, i_ref, i_ref['host'], False, False)
+
+        db.instance_destroy(self.context, instance_id)
+        db.service_destroy(self.context, s_ref['id'])
+
+    def test_live_migration_dest_check_not_alive(self):
+        """Confirms exception raises in case dest host does not exist."""
+        instance_id = _create_instance()['id']
+        i_ref = db.instance_get(self.context, instance_id)
+        t = utils.utcnow() - datetime.timedelta(10)
+        s_ref = self._create_compute_service(created_at=t, updated_at=t,
+                                             host=i_ref['host'])
+
+        self.assertRaises(exception.ComputeServiceUnavailable,
+                          self.scheduler.driver._live_migration_dest_check,
+                          self.context, i_ref, i_ref['host'], False, False)
 
         db.instance_destroy(self.context, instance_id)
         db.service_destroy(self.context, s_ref['id'])
@@ -1108,7 +1125,7 @@ class SimpleDriverTestCase(test.TestCase):
 
         self.assertRaises(exception.UnableToMigrateToSelf,
                           self.scheduler.driver._live_migration_dest_check,
-                          self.context, i_ref, i_ref['host'], False)
+                          self.context, i_ref, i_ref['host'], False, False)
 
         db.instance_destroy(self.context, instance_id)
         db.service_destroy(self.context, s_ref['id'])
@@ -1123,7 +1140,7 @@ class SimpleDriverTestCase(test.TestCase):
 
         self.assertRaises(exception.MigrationError,
                           self.scheduler.driver._live_migration_dest_check,
-                          self.context, i_ref, 'somewhere', False)
+                          self.context, i_ref, 'somewhere', False, False)
 
         db.instance_destroy(self.context, instance_id)
         db.instance_destroy(self.context, instance_id2)
@@ -1139,7 +1156,7 @@ class SimpleDriverTestCase(test.TestCase):
 
         self.assertRaises(exception.MigrationError,
                           self.scheduler.driver._live_migration_dest_check,
-                          self.context, i_ref, 'somewhere', True)
+                          self.context, i_ref, 'somewhere', True, False)
 
         db.instance_destroy(self.context, instance_id)
         db.instance_destroy(self.context, instance_id2)
@@ -1155,7 +1172,7 @@ class SimpleDriverTestCase(test.TestCase):
         ret = self.scheduler.driver._live_migration_dest_check(self.context,
                                                              i_ref,
                                                              'somewhere',
-                                                             False)
+                                                             False, False)
         self.assertTrue(ret is None)
         db.instance_destroy(self.context, instance_id)
         db.service_destroy(self.context, s_ref['id'])
@@ -1191,7 +1208,7 @@ class SimpleDriverTestCase(test.TestCase):
         #self.assertRaises(exception.SourceHostUnavailable,
         self.assertRaises(exception.FileNotFound,
                           self.scheduler.driver._live_migration_common_check,
-                          self.context, i_ref, dest, False)
+                          self.context, i_ref, dest, False, False)
 
         db.instance_destroy(self.context, instance_id)
         db.service_destroy(self.context, s_ref['id'])
@@ -1215,7 +1232,7 @@ class SimpleDriverTestCase(test.TestCase):
         self.mox.ReplayAll()
         self.assertRaises(exception.InvalidHypervisorType,
                           self.scheduler.driver._live_migration_common_check,
-                          self.context, i_ref, dest, False)
+                          self.context, i_ref, dest, False, False)
 
         db.instance_destroy(self.context, instance_id)
         db.service_destroy(self.context, s_ref['id'])
@@ -1241,7 +1258,7 @@ class SimpleDriverTestCase(test.TestCase):
         self.mox.ReplayAll()
         self.assertRaises(exception.DestinationHypervisorTooOld,
                           self.scheduler.driver._live_migration_common_check,
-                          self.context, i_ref, dest, False)
+                          self.context, i_ref, dest, False, False)
 
         db.instance_destroy(self.context, instance_id)
         db.service_destroy(self.context, s_ref['id'])
@@ -1275,6 +1292,7 @@ class SimpleDriverTestCase(test.TestCase):
             driver._live_migration_common_check(self.context,
                                                                i_ref,
                                                                dest,
+                                                               False,
                                                                False)
         except rpc.RemoteError, e:
             c = (e.exc_type == exception.InvalidCPUInfo)
@@ -1284,6 +1302,20 @@ class SimpleDriverTestCase(test.TestCase):
         db.service_destroy(self.context, s_ref['id'])
         db.service_destroy(self.context, s_ref2['id'])
 
+    def test_exception_puts_instance_in_error_state(self):
+        """Test that an exception from the scheduler puts an instance
+        in the ERROR state."""
+
+        scheduler = manager.SchedulerManager()
+        ctxt = context.get_admin_context()
+        inst = _create_instance()
+        self.assertRaises(Exception, scheduler._schedule,
+                          'failing_method', ctxt, 'scheduler',
+                          instance_id=inst['uuid'])
+
+        # Refresh the instance
+        inst = db.instance_get(ctxt, inst['id'])
+
 
 class MultiDriverTestCase(SimpleDriverTestCase):
     """Test case for multi driver."""
diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py
index 56906b81f..bf36e7431 100644
--- a/nova/tests/test_compute.py
+++ b/nova/tests/test_compute.py
@@ -1194,6 +1194,10 @@ class ComputeTestCase(BaseTestCase):
         self.mox.StubOutWithMock(rpc, 'call')
         rpc.call(c, FLAGS.volume_topic, {"method": "check_for_export",
                                          "args": {'instance_id': instance_id}})
+
+        self.mox.StubOutWithMock(self.compute.driver, 'get_instance_disk_info')
+        self.compute.driver.get_instance_disk_info(inst_ref.name)
+
         rpc.call(c, topic, {"method": "pre_live_migration",
                             "args": {'instance_id': instance_id,
                                      'block_migration': True,
diff --git a/nova/tests/test_instance_types.py b/nova/tests/test_instance_types.py
index 6c7b9d76e..aa277206f 100644
--- a/nova/tests/test_instance_types.py
+++ b/nova/tests/test_instance_types.py
@@ -144,7 +144,7 @@ class InstanceTypeTestCase(test.TestCase):
         instance_types.create(name, 256, 1, 120, 'flavor1')
         self.assertRaises(exception.ApiError,
                           instance_types.create,
-                          name, 256, 1, 120, 'flavor2')
+                          name, "256", 1, 120, 'flavor2')
 
     def test_duplicate_flavorids_fail(self):
         """Ensures that flavorid duplicates raise ApiError"""
diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py
index 15b7f7259..782e01563 100644
--- a/nova/tests/test_libvirt.py
+++ b/nova/tests/test_libvirt.py
@@ -757,67 +757,6 @@ class LibvirtConnTestCase(test.TestCase):
             self.assertEquals(conn.uri, testuri)
         db.instance_destroy(user_context, instance_ref['id'])
 
-    def test_update_available_resource_works_correctly(self):
-        """Confirm compute_node table is updated successfully."""
-        self.flags(instances_path='.')
-
-        # Prepare mocks
-        def getVersion():
-            return 12003
-
-        def getType():
-            return 'qemu'
-
-        def listDomainsID():
-            return []
-
-        service_ref = self.create_service(host='dummy')
-        self.create_fake_libvirt_mock(getVersion=getVersion,
-                                      getType=getType,
-                                      listDomainsID=listDomainsID)
-        self.mox.StubOutWithMock(connection.LibvirtConnection,
-                                 'get_cpu_info')
-        connection.LibvirtConnection.get_cpu_info().AndReturn('cpuinfo')
-
-        # Start test
-        self.mox.ReplayAll()
-        conn = connection.LibvirtConnection(False)
-        conn.update_available_resource(self.context, 'dummy')
-        service_ref = db.service_get(self.context, service_ref['id'])
-        compute_node = service_ref['compute_node'][0]
-
-        if sys.platform.upper() == 'LINUX2':
-            self.assertTrue(compute_node['vcpus'] >= 0)
-            self.assertTrue(compute_node['memory_mb'] > 0)
-            self.assertTrue(compute_node['local_gb'] > 0)
-            self.assertTrue(compute_node['vcpus_used'] == 0)
-            self.assertTrue(compute_node['memory_mb_used'] > 0)
-            self.assertTrue(compute_node['local_gb_used'] > 0)
-            self.assertTrue(len(compute_node['hypervisor_type']) > 0)
-            self.assertTrue(compute_node['hypervisor_version'] > 0)
-        else:
-            self.assertTrue(compute_node['vcpus'] >= 0)
-            self.assertTrue(compute_node['memory_mb'] == 0)
-            self.assertTrue(compute_node['local_gb'] > 0)
-            self.assertTrue(compute_node['vcpus_used'] == 0)
-            self.assertTrue(compute_node['memory_mb_used'] == 0)
-            self.assertTrue(compute_node['local_gb_used'] > 0)
-            self.assertTrue(len(compute_node['hypervisor_type']) > 0)
-            self.assertTrue(compute_node['hypervisor_version'] > 0)
-
-        db.service_destroy(self.context, service_ref['id'])
-
-    def test_update_resource_info_no_compute_record_found(self):
-        """Raise exception if no recorde found on services table."""
-        self.flags(instances_path='.')
-        self.create_fake_libvirt_mock()
-
-        self.mox.ReplayAll()
-        conn = connection.LibvirtConnection(False)
-        self.assertRaises(exception.ComputeServiceUnavailable,
-                          conn.update_available_resource,
-                          self.context, 'dummy')
-
     @test.skip_if(missing_libvirt(), "Test requires libvirt")
     def test_ensure_filtering_rules_for_instance_timeout(self):
         """ensure_filtering_fules_for_instance() finishes with timeout."""
@@ -950,7 +889,7 @@ class LibvirtConnTestCase(test.TestCase):
 
         # Test data
         instance_ref = db.instance_create(self.context, self.test_instance)
-        dummyjson = ('[{"path": "%s/disk", "local_gb": "10G",'
+        dummyjson = ('[{"path": "%s/disk", "disk_size": "10737418240",'
                      ' "type": "raw", "backing_file": ""}]')
 
         # Preparing mocks
@@ -984,6 +923,13 @@ class LibvirtConnTestCase(test.TestCase):
                     "<target dev='vdb' bus='virtio'/></disk>"
                     "</devices></domain>")
 
+        ret = ("image: /test/disk\n"
+               "file format: raw\n"
+               "virtual size: 20G (21474836480 bytes)\n"
+               "disk size: 3.1G\n"
+               "cluster_size: 2097152\n"
+               "backing file: /test/dummy (actual path: /backing/file)\n")
+
         # Preparing mocks
         vdmock = self.mox.CreateMock(libvirt.virDomain)
         self.mox.StubOutWithMock(vdmock, "XMLDesc")
@@ -998,18 +944,27 @@ class LibvirtConnTestCase(test.TestCase):
         fake_libvirt_utils.disk_sizes['/test/disk'] = 10 * GB
         fake_libvirt_utils.disk_sizes['/test/disk.local'] = 20 * GB
         fake_libvirt_utils.disk_backing_files['/test/disk.local'] = 'file'
+
+        self.mox.StubOutWithMock(os.path, "getsize")
+        os.path.getsize('/test/disk').AndReturn((10737418240))
+
+        self.mox.StubOutWithMock(utils, "execute")
+        utils.execute('qemu-img', 'info', '/test/disk.local').\
+            AndReturn((ret, ''))
+
+        os.path.getsize('/test/disk.local').AndReturn((21474836480))
+
         self.mox.ReplayAll()
         conn = connection.LibvirtConnection(False)
-        info = conn.get_instance_disk_info(self.context, instance_ref)
+        info = conn.get_instance_disk_info(instance_ref.name)
         info = utils.loads(info)
-
         self.assertEquals(info[0]['type'], 'raw')
-        self.assertEquals(info[1]['type'], 'qcow2')
         self.assertEquals(info[0]['path'], '/test/disk')
-        self.assertEquals(info[1]['path'], '/test/disk.local')
-        self.assertEquals(info[0]['local_gb'], '10G')
-        self.assertEquals(info[1]['local_gb'], '20G')
+        self.assertEquals(info[0]['disk_size'], 10737418240)
         self.assertEquals(info[0]['backing_file'], "")
+        self.assertEquals(info[1]['type'], 'qcow2')
+        self.assertEquals(info[1]['path'], '/test/disk.local')
+        self.assertEquals(info[1]['virt_disk_size'], 21474836480)
         self.assertEquals(info[1]['backing_file'], "file")
 
         db.instance_destroy(self.context, instance_ref['id'])
@@ -1188,6 +1143,9 @@ class HostStateTestCase(test.TestCase):
         def get_hypervisor_version(self):
             return 13091
 
+        def get_disk_available_least(self):
+            return 13091
+
     def test_update_status(self):
         self.mox.StubOutWithMock(connection, 'get_connection')
         connection.get_connection(True).AndReturn(self.FakeConnection())
diff --git a/nova/tests/test_virt_drivers.py b/nova/tests/test_virt_drivers.py
index 8c5cda6f0..9c17b3b0a 100644
--- a/nova/tests/test_virt_drivers.py
+++ b/nova/tests/test_virt_drivers.py
@@ -324,11 +324,6 @@ class _VirtDriverTestCase(test.TestCase):
         self.connection.refresh_provider_fw_rules()
 
     @catch_notimplementederror
-    def test_update_available_resource(self):
-        self.compute = self.start_service('compute', host='dummy')
-        self.connection.update_available_resource(self.ctxt, 'dummy')
-
-    @catch_notimplementederror
     def test_compare_cpu(self):
         cpu_info = '''{ "topology": {
                                "sockets": 1,