Compute restart causes period of network 'blackout'

Fixes bug 1034401

When a compute service is restarted each instance running on the
host has its iptables rules built and applied sequentially during
the host init stage. The impact of this, especially on a host
running many instances, can be observed as a period where some
instances are not accessible as the existing iptables rules have
been torn down and not yet re-applied.

The presented work-around for this is a configurable/flagged deferred
mode that prevents the application of the iptables rules until all
instances on the host had been initialised then the rules for all
instances are applied all at once preventing a 'blackout' period.

Change-Id: I0da90d07e54225fb63f3884897fb00a6027cd537

3 files changed, 28 insertions, 0 deletions

diff --git a/nova/virt/driver.py b/nova/virt/driver.py
index eab0c1c1d..4199df552 100644
--- a/nova/virt/driver.py
+++ b/nova/virt/driver.py
@@ -488,6 +488,14 @@ class ComputeDriver(object):
         # TODO(Vek): Need to pass context in for access to auth_token
         raise NotImplementedError()
 
+    def filter_defer_apply_on(self):
+        """Defer application of IPTables rules"""
+        pass
+
+    def filter_defer_apply_off(self):
+        """Turn off deferral of IPTables rules and apply the rules now"""
+        pass
+
     def unfilter_instance(self, instance, network_info):
         """Stop filtering instance"""
         # TODO(Vek): Need to pass context in for access to auth_token
diff --git a/nova/virt/firewall.py b/nova/virt/firewall.py
index 2afb8b6cf..b4d27218e 100644
--- a/nova/virt/firewall.py
+++ b/nova/virt/firewall.py
@@ -47,6 +47,14 @@ class FirewallDriver(object):
         At this point, the instance isn't running yet."""
         raise NotImplementedError()
 
+    def filter_defer_apply_on(self):
+        """Defer application of IPTables rules"""
+        pass
+
+    def filter_defer_apply_off(self):
+        """Turn off deferral of IPTables rules and apply the rules now"""
+        pass
+
     def unfilter_instance(self, instance, network_info):
         """Stop filtering instance"""
         raise NotImplementedError()
@@ -128,6 +136,12 @@ class IptablesFirewallDriver(FirewallDriver):
         """No-op. Everything is done in prepare_instance_filter."""
         pass
 
+    def filter_defer_apply_on(self):
+        self.iptables.defer_apply_on()
+
+    def filter_defer_apply_off(self):
+        self.iptables.defer_apply_off()
+
     def unfilter_instance(self, instance, network_info):
         # make sure this is legacy nw_info
         network_info = self._handle_network_info_model(network_info)
diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py
index 63d92af56..d9ed8a7d5 100644
--- a/nova/virt/libvirt/driver.py
+++ b/nova/virt/libvirt/driver.py
@@ -2435,6 +2435,12 @@ class LibvirtDriver(driver.ComputeDriver):
                 raise exception.NovaException(msg % instance_ref["name"])
             time.sleep(1)
 
+    def filter_defer_apply_on(self):
+        self.firewall_driver.filter_defer_apply_on()
+
+    def filter_defer_apply_off(self):
+        self.firewall_driver.filter_defer_apply_off()
+
     def live_migration(self, ctxt, instance_ref, dest,
                        post_method, recover_method, block_migration=False):
         """Spawning live_migration operation for distributing high-load.