Fix wait_for_deleted function in SmokeTests.

Fixes the wait_for_not_running function in smoketests/base.py so
that it short circuits when an instance ID is deleted rather than
waits 60 seconds (the timeout). Previously this function had
two problems:

 1) It used Boto's instance.update() instead of instance.update(validate=True).
 Since update() returns returns quietly if no data about the instance was
 returned (aka the instance has been deleted) it effectively made this
 function almost always wait for the timeout (60 seconds). This makes the
 tests take about a minute longer than they need to unless #2 below
 occurs...

 2) It also introduced a race condition in that... if hit just right
 the instance.state would get set to 'terminated'. Which would cause
 the loop to short circuit. This might seem correct at first glance but
 in fact causes intermittent failures in some of the netadmin tests
 because security groups can't be deleted if a running (or exiting)
 instance is still assigned to them. In fact wait_for_not_running
 was added to guard against just this in the first place!

Also, renames the base.wait_for_not_running function to be
called wait_for_deleted (that is what we are really doing here)

Change-Id: I4486995a7d9028d876366e8d7a2c845520319b86

2 files changed, 7 insertions, 5 deletions

diff --git a/smoketests/base.py b/smoketests/base.py
index 93f78f5dc..69222878b 100644
--- a/smoketests/base.py
+++ b/smoketests/base.py
@@ -72,11 +72,13 @@ class SmokeTestCase(unittest.TestCase):
         else:
             return False
 
-    def wait_for_not_running(self, instance, tries=60, wait=1):
-        """Wait for instance to not be running"""
+    def wait_for_deleted(self, instance, tries=60, wait=1):
+        """Wait for instance to be deleted"""
         for x in xrange(tries):
-            instance.update()
-            if not instance.state.startswith('running'):
+            try:
+                #NOTE(dprince): raises exception when instance id disappears
+                instance.update(validate=True)
+            except ValueError:
                 return True
             time.sleep(wait)
         else:
diff --git a/smoketests/test_netadmin.py b/smoketests/test_netadmin.py
index 4215f705d..72f892568 100644
--- a/smoketests/test_netadmin.py
+++ b/smoketests/test_netadmin.py
@@ -196,7 +196,7 @@ class SecurityGroupTests(base.UserSmokeTestCase):
         self.conn.disassociate_address(self.data['public_ip'])
         self.conn.delete_key_pair(TEST_KEY)
         self.conn.terminate_instances([self.data['instance'].id])
-        self.wait_for_not_running(self.data['instance'])
+        self.wait_for_deleted(self.data['instance'])
         self.conn.delete_security_group(TEST_GROUP)
         groups = self.conn.get_all_security_groups()
         self.assertFalse(TEST_GROUP in [group.name for group in groups])