Merge "Cell weighing class to handle mute child cells"

2 files changed, 124 insertions, 0 deletions

diff --git a/nova/cells/weights/mute_child.py b/nova/cells/weights/mute_child.py
new file mode 100644
index 000000000..b8fd0570b
--- /dev/null
+++ b/nova/cells/weights/mute_child.py
@@ -0,0 +1,72 @@
+# Copyright (c) 2013 Rackspace Hosting
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+If a child cell hasn't sent capacity or capability updates in a while,
+downgrade its likelihood of being chosen for scheduling requests.
+"""
+
+from oslo.config import cfg
+
+from nova.cells import weights
+from nova.openstack.common import log as logging
+from nova.openstack.common import timeutils
+
+LOG = logging.getLogger(__name__)
+
+mute_weigher_opts = [
+        cfg.FloatOpt('mute_weight_multiplier',
+                default=-10.0,
+                help='Multiplier used to weigh mute children.  (The value '
+                     'should be negative.)'),
+        cfg.FloatOpt('mute_weight_value',
+                default=1000.0,
+                help='Weight value assigned to mute children.  (The value '
+                     'should be positive.)'),
+         cfg.IntOpt("mute_child_interval",
+                default=300,
+                help='Number of seconds after which a lack of capability and '
+                     'capacity updates signals the child cell is to be '
+                     'treated as a mute.')
+]
+
+CONF = cfg.CONF
+CONF.register_opts(mute_weigher_opts, group='cells')
+
+
+class MuteChildWeigher(weights.BaseCellWeigher):
+    """If a child cell hasn't been heard from, greatly lower its selection
+    weight.
+    """
+
+    def _weight_multiplier(self):
+        # negative multiplier => lower weight
+        return CONF.cells.mute_weight_multiplier
+
+    def _weigh_object(self, cell, weight_properties):
+        """Check cell against the last_seen timestamp that indicates the time
+        that the most recent capability or capacity update was received from
+        the given cell."""
+
+        last_seen = cell.last_seen
+        secs = CONF.cells.mute_child_interval
+
+        if timeutils.is_older_than(last_seen, secs):
+            # yep, that's a mute child;  recommend highly that it be skipped!
+            LOG.warn(_("%(cell)s has not been seen since %(last_seen)s and is "
+                       "being treated as mute.") % locals())
+            return CONF.cells.mute_weight_value
+        else:
+            return 0
diff --git a/nova/tests/cells/test_cells_weights.py b/nova/tests/cells/test_cells_weights.py
index ca01e9939..38618bc64 100644
--- a/nova/tests/cells/test_cells_weights.py
+++ b/nova/tests/cells/test_cells_weights.py
@@ -18,8 +18,11 @@ Unit Tests for testing the cells weight algorithms.
 Cells with higher weights should be given priority for new builds.
 """
 
+import datetime
+
 from nova.cells import state
 from nova.cells import weights
+from nova.openstack.common import timeutils
 from nova import test
 
 
@@ -163,3 +166,52 @@ class WeightOffsetWeigherTestClass(_WeigherTestClass):
         expected_cells = [cells[2], cells[3], cells[0], cells[1]]
         resulting_cells = [weighed_cell.obj for weighed_cell in weighed_cells]
         self.assertEqual(expected_cells, resulting_cells)
+
+
+class MuteWeigherTestClass(_WeigherTestClass):
+    weigher_cls_name = 'nova.cells.weights.mute_child.MuteChildWeigher'
+
+    def setUp(self):
+        super(MuteWeigherTestClass, self).setUp()
+        self.flags(mute_weight_multiplier=-10.0, mute_child_interval=100,
+                   mute_weight_value=1000.0, group='cells')
+
+        self.now = timeutils.utcnow()
+        timeutils.set_time_override(self.now)
+
+        self.cells = _get_fake_cells()
+        for cell in self.cells:
+            cell.last_seen = self.now
+
+    def tearDown(self):
+        super(MuteWeigherTestClass, self).tearDown()
+        timeutils.clear_time_override()
+
+    def test_non_mute(self):
+        weight_properties = {}
+        weighed_cells = self._get_weighed_cells(self.cells, weight_properties)
+        self.assertEqual(len(weighed_cells), 4)
+
+        for weighed_cell in weighed_cells:
+            self.assertEqual(0, weighed_cell.weight)
+
+    def test_mutes(self):
+        # make 2 of them mute:
+        self.cells[0].last_seen = (self.cells[0].last_seen -
+                                   datetime.timedelta(seconds=200))
+        self.cells[1].last_seen = (self.cells[1].last_seen -
+                                   datetime.timedelta(seconds=200))
+
+        weight_properties = {}
+        weighed_cells = self._get_weighed_cells(self.cells, weight_properties)
+        self.assertEqual(len(weighed_cells), 4)
+
+        for i in range(2):
+            weighed_cell = weighed_cells.pop(0)
+            self.assertEqual(0, weighed_cell.weight)
+            self.assertIn(weighed_cell.obj.name, ['cell3', 'cell4'])
+
+        for i in range(2):
+            weighed_cell = weighed_cells.pop(0)
+            self.assertEqual(1000 * -10.0, weighed_cell.weight)
+            self.assertIn(weighed_cell.obj.name, ['cell1', 'cell2'])