Comments Incorporated for Bug800759

3 files changed, 163 insertions, 14 deletions

diff --git a/Authors b/Authors
index c3a65f1b4..67cd38bcf 100644
--- a/Authors
+++ b/Authors
@@ -93,3 +93,4 @@ Yoshiaki Tamura <yoshi@midokura.jp>
 Youcef Laribi <Youcef.Laribi@eu.citrix.com>
 Yuriy Taraday <yorik.sar@gmail.com>
 Zhixue Wu <Zhixue.Wu@citrix.com>
+Devendra Modium <dmodium@isi.edu>
diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
index e0daa01e2..2a0816118 100644
--- a/nova/scheduler/zone_manager.py
+++ b/nova/scheduler/zone_manager.py
@@ -109,7 +109,6 @@ class ZoneManager(object):
         self.last_zone_db_check = datetime.datetime.min
         self.zone_states = {}  # { <zone_id> : ZoneState }
         self.service_states = {}  # { <host> : { <service> : { cap k : v }}}
-        self.service_time_stamp = {}  # reported time
         self.green_pool = greenpool.GreenPool()
 
     def get_zone_list(self):
@@ -126,19 +125,28 @@ class ZoneManager(object):
         # But it's likely to change once we understand what the Best-Match
         # code will need better.
         combined = {}  # { <service>_<cap> : (min, max), ... }
-        allowed_time_diff = FLAGS.periodic_interval * 3
+        stale_host_services = {} # { host1 : [svc1, svc2], host2 :[svc1]}
         for host, host_dict in hosts_dict.iteritems():
-            if (utils.utcnow() - self.service_time_stamp[host]) <= \
-                datetime.timedelta(seconds=allowed_time_diff):
-                for service_name, service_dict in host_dict.iteritems():
-                    for cap, value in service_dict.iteritems():
-                        key = "%s_%s" % (service_name, cap)
-                        min_value, max_value = combined.get(key, \
-                            (value, value))
-                        min_value = min(min_value, value)
-                        max_value = max(max_value, value)
-                        combined[key] = (min_value, max_value)
-
+            for service_name, service_dict in host_dict.iteritems():
+
+                #Check if the service capabilities became stale
+                if self.host_service_caps_stale(host, service_name):
+                    if host not in stale_host_services:
+                        stale_host_services[host] = []  # Adding host key once
+                    stale_host_services[host].append(service_name)
+                    continue
+                for cap, value in service_dict.iteritems():
+                    if cap == "timestamp": # Timestamp is not needed
+                        continue
+                    key = "%s_%s" % (service_name, cap)
+                    min_value, max_value = combined.get(key, \
+                        (value, value))
+                    min_value = min(min_value, value)
+                    max_value = max(max_value, value)
+                    combined[key] = (min_value, max_value)
+
+        # Delete the expired host services
+        self.delete_expired_host_services(stale_host_services)
         return combined
 
     def _refresh_from_db(self, context):
@@ -177,6 +185,25 @@ class ZoneManager(object):
         logging.debug(_("Received %(service_name)s service update from "
                             "%(host)s: %(capabilities)s") % locals())
         service_caps = self.service_states.get(host, {})
+        capabilities["timestamp"] = utils.utcnow() # Reported time
         service_caps[service_name] = capabilities
         self.service_states[host] = service_caps
-        self.service_time_stamp[host] = utils.utcnow()
+
+    def host_service_caps_stale(self, host, service):
+        """Check if host service capabilites are not recent enough."""
+        allowed_time_diff = FLAGS.periodic_interval * 3
+        caps = self.service_states[host][service]
+        if (utils.utcnow() - caps["timestamp"]) <= \
+            datetime.timedelta(seconds=allowed_time_diff):
+            return False
+        return True
+
+    def delete_expired_host_services(self, host_services_dict):
+        """Delete all the inactive host services information."""
+        for host, services in host_services_dict.iteritems():
+            service_caps = self.service_states[host]
+            for service in services:
+                del service_caps[service]
+                if len(service_caps) == 0: # Delete host if no services
+                    del self.service_states[host]
+
diff --git a/nova/tests/test_zones.py b/nova/tests/test_zones.py
index e132809dc..f4661a328 100644
--- a/nova/tests/test_zones.py
+++ b/nova/tests/test_zones.py
@@ -198,3 +198,124 @@ class ZoneManagerTestCase(test.TestCase):
         self.assertEquals(zone_state.attempt, 3)
         self.assertFalse(zone_state.is_active)
         self.assertEquals(zone_state.name, None)
+
+    def test_host_service_caps_stale(self):
+        zm = zone_manager.ZoneManager()
+        expiry_time = (FLAGS.periodic_interval * 3) + 1
+
+        # services just updated capabilities 
+        zm.update_service_capabilities("svc1", "host1", dict(a=1, b=2))
+        zm.update_service_capabilities("svc2", "host1", dict(a=3, b=4))
+        self.assertFalse(zm.host_service_caps_stale("host1", "svc1"))
+        self.assertFalse(zm.host_service_caps_stale("host1", "svc2"))
+
+        # Both services became stale
+        time_future = utils.utcnow() + datetime.timedelta(seconds=expiry_time)
+        utils.set_time_override(time_future)
+        self.assertTrue(zm.host_service_caps_stale("host1", "svc1"))
+        self.assertTrue(zm.host_service_caps_stale("host1", "svc2"))
+
+        # One service became stale
+        utils.clear_time_override()
+        caps = zm.service_states["host1"]["svc1"]
+        caps["timestamp"] = utils.utcnow() - \
+                               datetime.timedelta(seconds=expiry_time)
+        self.assertTrue(zm.host_service_caps_stale("host1", "svc1"))
+        self.assertFalse(zm.host_service_caps_stale("host1", "svc2"))
+ 
+    def test_delete_expired_host_services(self):
+        zm = zone_manager.ZoneManager()
+
+        # Delete one service in a host
+        zm.update_service_capabilities("svc1", "host1", dict(a=1, b=2))
+        zm.update_service_capabilities("svc2", "host1", dict(a=3, b=4))
+        stale_host_services = {"host1": ["svc1"]}
+        zm.delete_expired_host_services(stale_host_services)
+        self.assertFalse("svc1" in zm.service_states["host1"])
+        self.assertTrue("svc2" in zm.service_states["host1"])
+
+        # Delete all services in a host
+        zm.update_service_capabilities("svc1", "host1", dict(a=1, b=2))
+        stale_host_services = {"host1": ["svc1", "svc2"]}
+        zm.delete_expired_host_services(stale_host_services)
+        self.assertFalse("host1" in zm.service_states)
+
+        # Delete one service per host
+        zm.update_service_capabilities("svc1", "host1", dict(a=1, b=2))
+        zm.update_service_capabilities("svc1", "host2", dict(a=3, b=4))
+        stale_host_services = {"host1": ["svc1"], "host2": ["svc1"]}
+        zm.delete_expired_host_services(stale_host_services)
+        self.assertFalse("host1" in zm.service_states)
+        self.assertFalse("host2" in zm.service_states)
+ 
+    def test_get_zone_capabilities(self):
+        zm = zone_manager.ZoneManager()
+        expiry_time = (FLAGS.periodic_interval * 3) + 1
+
+        # Service capabilities recent
+        zm.update_service_capabilities("svc1", "host1", dict(a=1, b=2))
+        caps = zm.get_zone_capabilities(None)
+        self.assertEquals(caps, dict(svc1_a=(1, 1), svc1_b=(2, 2)))
+
+        # Service capabilities stale
+        time_future = utils.utcnow() + datetime.timedelta(seconds=expiry_time)
+        utils.set_time_override(time_future)
+        caps = zm.get_zone_capabilities(None)
+        self.assertEquals(caps, {})
+
+        # Both host service capabilities recent
+        utils.clear_time_override()
+        zm.update_service_capabilities("svc1", "host1", dict(a=1, b=2))
+        zm.update_service_capabilities("svc1", "host2", dict(a=3, b=4))
+        caps = zm.get_zone_capabilities(None)
+        self.assertEquals(caps, dict(svc1_a=(1, 3), svc1_b=(2, 4)))
+
+        # One host service capabilities become stale
+        serv_caps = zm.service_states["host1"]["svc1"]
+        serv_caps["timestamp"] = utils.utcnow() - \
+                               datetime.timedelta(seconds=expiry_time)
+        caps = zm.get_zone_capabilities(None)
+        self.assertEquals(caps, dict(svc1_a=(3, 3), svc1_b=(4, 4)))
+
+        # Multiple services per host 
+        zm.update_service_capabilities("svc1", "host1", dict(a=1, b=2))
+        zm.update_service_capabilities("svc2", "host1", dict(a=5, b=6))
+        zm.update_service_capabilities("svc2", "host2", dict(a=7, b=8))
+        caps = zm.get_zone_capabilities(None)
+        self.assertEquals(caps, dict(svc1_a=(1, 3), svc1_b=(2, 4),
+                                     svc2_a=(5, 7), svc2_b=(6, 8)))
+
+        # Two host services among four become stale 
+        serv_caps_1 = zm.service_states["host1"]["svc2"]
+        serv_caps_1["timestamp"] = utils.utcnow() - \
+                               datetime.timedelta(seconds=expiry_time)
+        serv_caps_2 = zm.service_states["host2"]["svc1"]
+        serv_caps_2["timestamp"] = utils.utcnow() - \
+                               datetime.timedelta(seconds=expiry_time)
+        caps = zm.get_zone_capabilities(None)
+        self.assertEquals(caps, dict(svc1_a=(1, 1), svc1_b=(2, 2),
+                                     svc2_a=(7, 7), svc2_b=(8, 8)))
+
+        # Three host services among four become stale 
+        zm.update_service_capabilities("svc2", "host1", dict(a=5, b=6))
+        zm.update_service_capabilities("svc1", "host2", dict(a=3, b=4))
+        serv_caps_1 = zm.service_states["host1"]["svc2"]
+        serv_caps_1["timestamp"] = utils.utcnow() - \
+                               datetime.timedelta(seconds=expiry_time)
+        serv_caps_2 = zm.service_states["host2"]["svc1"]
+        serv_caps_2["timestamp"] = utils.utcnow() - \
+                               datetime.timedelta(seconds=expiry_time)
+        serv_caps_3 = zm.service_states["host2"]["svc2"]
+        serv_caps_3["timestamp"] = utils.utcnow() - \
+                               datetime.timedelta(seconds=expiry_time)
+        caps = zm.get_zone_capabilities(None)
+        self.assertEquals(caps, dict(svc1_a=(1, 1), svc1_b=(2, 2)))
+
+        # All the host services  become stale 
+        zm.update_service_capabilities("svc2", "host1", dict(a=5, b=6))
+        zm.update_service_capabilities("svc1", "host2", dict(a=3, b=4))
+        zm.update_service_capabilities("svc2", "host2", dict(a=7, b=8))
+        utils.set_time_override(time_future)
+        caps = zm.get_zone_capabilities(None)
+        self.assertEquals(caps, {})
+