From 24aacd2c91c73245d444a2460ded1c5c94382f5f Mon Sep 17 00:00:00 2001
From: Joshua Harlow <harlowja@yahoo-inc.com>
Date: Fri, 29 Mar 2013 17:10:08 -0700
Subject: Limit the checks for block device becoming available.

Instead of previously looping potentially forever for a block
device mapping to become created from the volume api, basically
blocking a greenthread until this happens we now will have a number
of attempts (defaulting to 10) that we will wait for the volume api
to create said block device mapping (or error out).

Fixes bug 1162064

Change-Id: I6ff0b42aad48df735d09f91a0a9fe98e6abcf2eb
---
 nova/compute/manager.py            | 33 +++++++++++++++++++++++++--------
 nova/exception.py                  |  6 ++++++
 nova/tests/compute/test_compute.py | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 66 insertions(+), 8 deletions(-)

diff --git a/nova/compute/manager.py b/nova/compute/manager.py
index 6cb0bbb94..9bc0af1fd 100755
--- a/nova/compute/manager.py
+++ b/nova/compute/manager.py
@@ -673,6 +673,30 @@ class ComputeManager(manager.SchedulerDependentManager):
             network_info = network_info.legacy()
         return network_info
 
+    def _await_block_device_map_created(self, context, vol_id, max_tries=30,
+                                        wait_between=1):
+        # TODO(yamahata): creating volume simultaneously
+        #                 reduces creation time?
+        # TODO(yamahata): eliminate dumb polling
+        # TODO(harlowja): make the max_tries configurable or dynamic?
+        attempts = 0
+        start = time.time()
+        while attempts < max_tries:
+            volume = self.volume_api.get(context, vol_id)
+            volume_status = volume['status']
+            if volume_status != 'creating':
+                if volume_status != 'available':
+                    LOG.warn(_("Volume id: %s finished being created but was"
+                               " not set as 'available'"), vol_id)
+                # NOTE(harlowja): return how many attempts were tried
+                return attempts + 1
+            greenthread.sleep(wait_between)
+            attempts += 1
+        # NOTE(harlowja): Should only happen if we ran out of attempts
+        raise exception.VolumeNotCreated(volume_id=vol_id,
+                                         seconds=int(time.time() - start),
+                                         attempts=attempts)
+
     def _setup_block_device_mapping(self, context, instance, bdms):
         """setup volumes for block device mapping."""
         block_device_mapping = []
@@ -705,14 +729,7 @@ class ComputeManager(manager.SchedulerDependentManager):
                                                         bdm['snapshot_id'])
                 vol = self.volume_api.create(context, bdm['volume_size'],
                                              '', '', snapshot)
-                # TODO(yamahata): creating volume simultaneously
-                #                 reduces creation time?
-                # TODO(yamahata): eliminate dumb polling
-                while True:
-                    volume = self.volume_api.get(context, vol['id'])
-                    if volume['status'] != 'creating':
-                        break
-                    greenthread.sleep(1)
+                self._await_block_device_map_created(context, vol['id'])
                 self.conductor_api.block_device_mapping_update(
                     context, bdm['id'], {'volume_id': vol['id']})
                 bdm['volume_id'] = vol['id']
diff --git a/nova/exception.py b/nova/exception.py
index 1f32b0671..ceedfb827 100644
--- a/nova/exception.py
+++ b/nova/exception.py
@@ -244,6 +244,12 @@ class VolumeUnattached(Invalid):
     message = _("Volume %(volume_id)s is not attached to anything")
 
 
+class VolumeNotCreated(NovaException):
+    message = _("Volume %(volume_id)s did not finish being created"
+                " even after we waited %(seconds)s seconds or %(attempts)s"
+                " attempts.")
+
+
 class InvalidKeypair(Invalid):
     message = _("Keypair data is invalid")
 
diff --git a/nova/tests/compute/test_compute.py b/nova/tests/compute/test_compute.py
index 3a1526522..fe17421ed 100644
--- a/nova/tests/compute/test_compute.py
+++ b/nova/tests/compute/test_compute.py
@@ -248,6 +248,7 @@ class ComputeVolumeTestCase(BaseTestCase):
     def setUp(self):
         super(ComputeVolumeTestCase, self).setUp()
         self.volume_id = 'fake'
+        self.fetched_attempts = 0
         self.instance = {
             'id': 'fake',
             'uuid': 'fake',
@@ -283,6 +284,40 @@ class ComputeVolumeTestCase(BaseTestCase):
                                    '/dev/vdb', self.instance)
         self.assertEqual(self.cinfo.get('serial'), self.volume_id)
 
+    def test_await_block_device_created_to_slow(self):
+
+        def never_get(context, vol_id):
+            return {
+                'status': 'creating',
+                'id': 'blah',
+            }
+
+        self.stubs.Set(self.compute.volume_api, 'get', never_get)
+        self.assertRaises(exception.VolumeNotCreated,
+                          self.compute._await_block_device_map_created,
+                          self.context, '1', max_tries=2, wait_between=0.1)
+
+    def test_await_block_device_created_slow(self):
+        c = self.compute
+
+        def slow_get(context, vol_id):
+            while self.fetched_attempts < 2:
+                self.fetched_attempts += 1
+                return {
+                    'status': 'creating',
+                    'id': 'blah',
+                }
+            return {
+                'status': 'available',
+                'id': 'blah',
+            }
+
+        self.stubs.Set(c.volume_api, 'get', slow_get)
+        attempts = c._await_block_device_map_created(self.context, '1',
+                                                     max_tries=4,
+                                                     wait_between=0.1)
+        self.assertEqual(attempts, 3)
+
     def test_boot_volume_serial(self):
         block_device_mapping = [{
             'id': 1,
-- 
cgit