Fix test cases in integrated.test_multiprocess_api

Fixes bug #1098043

Be patient in test_multiprocess_api while waiting service to be spawned or
killed, because:
    * Service spawning has a limit of one process a second.
    * A test server in the cloud might have limited resources,
      the time there might be several times slower.

Change-Id: If75e6dfa5d769985a6c1b46461f25485e2a996ad

1 files changed, 26 insertions, 28 deletions

diff --git a/nova/tests/integrated/test_multiprocess_api.py b/nova/tests/integrated/test_multiprocess_api.py
index 4f07d9de9..5a82e0033 100644
--- a/nova/tests/integrated/test_multiprocess_api.py
+++ b/nova/tests/integrated/test_multiprocess_api.py
@@ -71,18 +71,24 @@ class MultiprocessWSGITest(integrated_helpers._IntegratedTestBase):
 
         self.pid = pid
 
-        # Wait for up to a second for workers to get started
-        start = time.time()
-        while time.time() - start < 1:
-            workers = self._get_workers()
-            if len(workers) == self.workers:
-                break
-
-            time.sleep(.1)
+        # Wait at most 10 seconds to spawn workers
+        cond = lambda: self.workers == len(self._get_workers())
+        timeout = 10
+        self._wait(cond, timeout)
 
+        workers = self._get_workers()
         self.assertEqual(len(workers), self.workers)
         return workers
 
+    def _wait(self, cond, timeout):
+        start = time.time()
+        while True:
+            if cond():
+                break
+            if time.time() - start > timeout:
+                break
+            time.sleep(.1)
+
     def tearDown(self):
         if self.pid:
             # Make sure all processes are stopped
@@ -114,18 +120,14 @@ class MultiprocessWSGITest(integrated_helpers._IntegratedTestBase):
         LOG.info('pid of first child is %s' % start_workers[0])
         os.kill(start_workers[0], signal.SIGTERM)
 
-        # loop and check if new worker is spawned (for 1 second max)
-        start = time.time()
-        while time.time() - start < 1:
-            end_workers = self._get_workers()
-            LOG.info('workers: %r' % end_workers)
-
-            if start_workers != end_workers:
-                break
-
-            time.sleep(.1)
+        # Wait at most 5 seconds to respawn a worker
+        cond = lambda: start_workers != self._get_workers()
+        timeout = 5
+        self._wait(cond, timeout)
 
         # Make sure worker pids don't match
+        end_workers = self._get_workers()
+        LOG.info('workers: %r' % end_workers)
         self.assertNotEqual(start_workers, end_workers)
 
         # check if api service still works
@@ -141,17 +143,13 @@ class MultiprocessWSGITest(integrated_helpers._IntegratedTestBase):
 
         os.kill(self.pid, sig)
 
-        # loop and check if all processes are killed (for 1 second max)
-        start = time.time()
-        while time.time() - start < 1:
-            workers = self._get_workers()
-            LOG.info('workers: %r' % workers)
-
-            if not workers:
-                break
-
-            time.sleep(.1)
+        # Wait at most 5 seconds to kill all workers
+        cond = lambda: not self._get_workers()
+        timeout = 5
+        self._wait(cond, timeout)
 
+        workers = self._get_workers()
+        LOG.info('workers: %r' % workers)
         self.assertFalse(workers, 'No OS processes left.')
 
     def test_terminate_sigkill(self):