Add trust level cache to trusted_filter

    Currently each time the trusted filter is called to check host_pass, it
polls Open Attestation Service to get the trusted level for the host. This
solution is not good on scalability.
    With a cache for the host trust level, trusted filter don't need to
consult OAT service if the cache is still valid, thus improves the
scalability. The cache is setup when the first time the filter consulting
the OAT service.
    The trusted_filters are reinitialized for each VM that is
scheduled, thus the cache is valid only for one VM instance scheduing.

    Implements blueprint trusted-filter-cache

Change-Id: I613da53876e6cd548566e27ab1693287023e9861
Signed-off-by: Yunhong, Jiang <yunhong.jiang@intel.com>

2 files changed, 200 insertions, 39 deletions

diff --git a/nova/scheduler/filters/trusted_filter.py b/nova/scheduler/filters/trusted_filter.py
index 4d0f2305f..302d2b3a8 100644
--- a/nova/scheduler/filters/trusted_filter.py
+++ b/nova/scheduler/filters/trusted_filter.py
@@ -48,9 +48,12 @@ import httplib
 import socket
 import ssl
 
+from nova import context
+from nova import db
 from nova.openstack.common import cfg
 from nova.openstack.common import jsonutils
 from nova.openstack.common import log as logging
+from nova.openstack.common import timeutils
 from nova.scheduler import filters
 
 
@@ -78,6 +81,9 @@ trusted_opts = [
                deprecated_name='auth_blob',
                default=None,
                help='attestation authorization blob - must change'),
+    cfg.IntOpt('attestation_auth_timeout',
+               default=60,
+               help='Attestation status cache valid period length'),
 ]
 
 CONF = cfg.CONF
@@ -119,7 +125,7 @@ class HTTPSClientAuthConnection(httplib.HTTPSConnection):
                                     cert_reqs=ssl.CERT_REQUIRED)
 
 
-class AttestationService(httplib.HTTPSConnection):
+class AttestationService(object):
     # Provide access wrapper to attestation server to get integrity report.
 
     def __init__(self):
@@ -156,10 +162,10 @@ class AttestationService(httplib.HTTPSConnection):
         except (socket.error, IOError) as e:
             return IOError, None
 
-    def _request(self, cmd, subcmd, host):
+    def _request(self, cmd, subcmd, hosts):
         body = {}
-        body['count'] = 1
-        body['hosts'] = host
+        body['count'] = len(hosts)
+        body['hosts'] = hosts
         cooked = jsonutils.dumps(body)
         headers = {}
         headers['content-type'] = 'application/json'
@@ -173,39 +179,124 @@ class AttestationService(httplib.HTTPSConnection):
         else:
             return status, None
 
-    def _check_trust(self, data, host):
-        for item in data:
-            for state in item['hosts']:
-                if state['host_name'] == host:
-                    return state['trust_lvl']
-        return ""
+    def do_attestation(self, hosts):
+        """Attests compute nodes through OAT service.
 
-    def do_attestation(self, host):
-        state = []
-        status, data = self._request("POST", "PollHosts", host)
-        if status != httplib.OK:
-            return {}
-        state.append(data)
-        return self._check_trust(state, host)
+        :param hosts: hosts list to be attested
+        :returns: dictionary for trust level and validate time
+        """
+        result = None
 
+        status, data = self._request("POST", "PollHosts", hosts)
+        if data != None:
+            result = data.get('hosts')
 
-class TrustedFilter(filters.BaseHostFilter):
-    """Trusted filter to support Trusted Compute Pools."""
+        return result
+
+
+class ComputeAttestationCache(object):
+    """Cache for compute node attestation
+
+    Cache compute node's trust level for sometime,
+    if the cache is out of date, poll OAT service to flush the
+    cache.
+
+    OAT service may have cache also. OAT service's cache valid time
+    should be set shorter than trusted filter's cache valid time.
+    """
 
     def __init__(self):
-        self.attestation_service = AttestationService()
+        self.attestservice = AttestationService()
+        self.compute_nodes = {}
+        admin = context.get_admin_context()
+
+        # Fetch compute node list to initialize the compute_nodes,
+        # so that we don't need poll OAT service one by one for each
+        # host in the first round that scheduler invokes us.
+        computes = db.compute_node_get_all(admin)
+        for compute in computes:
+            service = compute['service']
+            if not service:
+                LOG.warn(_("No service for compute ID %s") % compute['id'])
+                continue
+            host = service['host']
+            self._init_cache_entry(host)
+
+    def _cache_valid(self, host):
+        cachevalid = False
+        if host in self.compute_nodes:
+            node_stats = self.compute_nodes.get(host)
+            if not timeutils.is_older_than(
+                             node_stats['vtime'],
+                             CONF.trusted_computing.attestation_auth_timeout):
+                cachevalid = True
+        return cachevalid
+
+    def _init_cache_entry(self, host):
+        self.compute_nodes[host] = {
+            'trust_lvl': 'unknown',
+            'vtime': timeutils.normalize_time(
+                        timeutils.parse_isotime("1970-01-01T00:00:00Z"))}
+
+    def _invalidate_caches(self):
+        for host in self.compute_nodes:
+            self._init_cache_entry(host)
+
+    def _update_cache_entry(self, state):
+        entry = {}
+
+        host = state['host_name']
+        entry['trust_lvl'] = state['trust_lvl']
 
-    def _is_trusted(self, host, trust):
-        level = self.attestation_service.do_attestation(host)
-        LOG.debug(_("TCP: trust state of "
-                    "%(host)s:%(level)s(%(trust)s)") % locals())
+        try:
+            # Normalize as naive object to interoperate with utcnow().
+            entry['vtime'] = timeutils.normalize_time(
+                            timeutils.parse_isotime(state['vtime']))
+        except ValueError:
+            # Mark the system as un-trusted if get invalid vtime.
+            entry['trust_lvl'] = 'unknown'
+            entry['vtime'] = timeutils.utcnow()
+
+        self.compute_nodes[host] = entry
+
+    def _update_cache(self):
+        self._invalidate_caches()
+        states = self.attestservice.do_attestation(self.compute_nodes.keys())
+        if states is None:
+            return
+        for state in states:
+            self._update_cache_entry(state)
+
+    def get_host_attestation(self, host):
+        """Check host's trust level."""
+        if not host in self.compute_nodes:
+            self._init_cache_entry(host)
+        if not self._cache_valid(host):
+            self._update_cache()
+        level = self.compute_nodes.get(host).get('trust_lvl')
+        return level
+
+
+class ComputeAttestation(object):
+    def __init__(self):
+        self.caches = ComputeAttestationCache()
+
+    def is_trusted(self, host, trust):
+        level = self.caches.get_host_attestation(host)
         return trust == level
 
+
+class TrustedFilter(filters.BaseHostFilter):
+    """Trusted filter to support Trusted Compute Pools."""
+
+    def __init__(self):
+        self.compute_attestation = ComputeAttestation()
+
     def host_passes(self, host_state, filter_properties):
         instance = filter_properties.get('instance_type', {})
         extra = instance.get('extra_specs', {})
         trust = extra.get('trust:trusted_host')
         host = host_state.host
         if trust:
-            return self._is_trusted(host, trust)
+            return self.compute_attestation.is_trusted(host, trust)
         return True
diff --git a/nova/tests/scheduler/test_host_filters.py b/nova/tests/scheduler/test_host_filters.py
index 9f7f189cc..f8b9f9296 100644
--- a/nova/tests/scheduler/test_host_filters.py
+++ b/nova/tests/scheduler/test_host_filters.py
@@ -22,6 +22,7 @@ from nova import context
 from nova import db
 from nova.openstack.common import cfg
 from nova.openstack.common import jsonutils
+from nova.openstack.common import timeutils
 from nova.scheduler import filters
 from nova.scheduler.filters import extra_specs_ops
 from nova.scheduler.filters.trusted_filter import AttestationService
@@ -233,11 +234,13 @@ class HostFiltersTestCase(test.TestCase):
 
     def fake_oat_request(self, *args, **kwargs):
         """Stubs out the response from OAT service."""
-        return httplib.OK, jsonutils.loads(self.oat_data)
+        self.oat_attested = True
+        return httplib.OK, self.oat_data
 
     def setUp(self):
         super(HostFiltersTestCase, self).setUp()
         self.oat_data = ''
+        self.oat_attested = False
         self.stubs = stubout.StubOutForTesting()
         self.stubs.Set(AttestationService, '_request', self.fake_oat_request)
         self.context = context.RequestContext('fake', 'fake')
@@ -1147,54 +1150,121 @@ class HostFiltersTestCase(test.TestCase):
     def test_trusted_filter_default_passes(self):
         self._stub_service_is_up(True)
         filt_cls = self.class_map['TrustedFilter']()
-        filter_properties = {'instance_type': {'memory_mb': 1024}}
+        filter_properties = {'context': self.context.elevated(),
+                             'instance_type': {'memory_mb': 1024}}
         host = fakes.FakeHostState('host1', 'node1', {})
         self.assertTrue(filt_cls.host_passes(host, filter_properties))
 
     def test_trusted_filter_trusted_and_trusted_passes(self):
-        self.oat_data =\
-            '{"hosts":[{"host_name":"host1","trust_lvl":"trusted"}]}'
+        self.oat_data = {"hosts": [{"host_name": "host1",
+                                   "trust_lvl": "trusted",
+                                   "vtime": timeutils.isotime()}]}
         self._stub_service_is_up(True)
         filt_cls = self.class_map['TrustedFilter']()
         extra_specs = {'trust:trusted_host': 'trusted'}
-        filter_properties = {'instance_type': {'memory_mb': 1024,
+        filter_properties = {'context': self.context.elevated(),
+                             'instance_type': {'memory_mb': 1024,
                                                'extra_specs': extra_specs}}
         host = fakes.FakeHostState('host1', 'node1', {})
         self.assertTrue(filt_cls.host_passes(host, filter_properties))
 
     def test_trusted_filter_trusted_and_untrusted_fails(self):
-        self.oat_data =\
-            '{"hosts":[{"host_name":"host1","trust_lvl":"untrusted"}]}'
+        self.oat_data = {"hosts": [{"host_name": "host1",
+                                    "trust_lvl": "untrusted",
+                                    "vtime": timeutils.isotime()}]}
         self._stub_service_is_up(True)
         filt_cls = self.class_map['TrustedFilter']()
         extra_specs = {'trust:trusted_host': 'trusted'}
-        filter_properties = {'instance_type': {'memory_mb': 1024,
+        filter_properties = {'context': self.context.elevated(),
+                             'instance_type': {'memory_mb': 1024,
                                                'extra_specs': extra_specs}}
         host = fakes.FakeHostState('host1', 'node1', {})
         self.assertFalse(filt_cls.host_passes(host, filter_properties))
 
     def test_trusted_filter_untrusted_and_trusted_fails(self):
-        self.oat_data =\
-            '{"hosts":[{"host_name":"host1","trust_lvl":"trusted"}]}'
+        self.oat_data = {"hosts": [{"host_name": "host1",
+                                    "trust_lvl": "trusted",
+                                    "vtime": timeutils.isotime()}]}
         self._stub_service_is_up(True)
         filt_cls = self.class_map['TrustedFilter']()
         extra_specs = {'trust:trusted_host': 'untrusted'}
-        filter_properties = {'instance_type': {'memory_mb': 1024,
+        filter_properties = {'context': self.context.elevated(),
+                             'instance_type': {'memory_mb': 1024,
                                                'extra_specs': extra_specs}}
         host = fakes.FakeHostState('host1', 'node1', {})
         self.assertFalse(filt_cls.host_passes(host, filter_properties))
 
     def test_trusted_filter_untrusted_and_untrusted_passes(self):
-        self.oat_data =\
-            '{"hosts":[{"host_name":"host1","trust_lvl":"untrusted"}]}'
+        self.oat_data = {"hosts": [{"host_name": "host1",
+                                    "trust_lvl": "untrusted",
+                                    "vtime":timeutils.isotime()}]}
         self._stub_service_is_up(True)
         filt_cls = self.class_map['TrustedFilter']()
         extra_specs = {'trust:trusted_host': 'untrusted'}
-        filter_properties = {'instance_type': {'memory_mb': 1024,
+        filter_properties = {'context': self.context.elevated(),
+                             'instance_type': {'memory_mb': 1024,
                                                'extra_specs': extra_specs}}
         host = fakes.FakeHostState('host1', 'node1', {})
         self.assertTrue(filt_cls.host_passes(host, filter_properties))
 
+    def test_trusted_filter_update_cache(self):
+        self.oat_data = {"hosts": [{"host_name":
+                                    "host1", "trust_lvl": "untrusted",
+                                    "vtime": timeutils.isotime()}]}
+
+        filt_cls = self.class_map['TrustedFilter']()
+        extra_specs = {'trust:trusted_host': 'untrusted'}
+        filter_properties = {'context': self.context.elevated(),
+                             'instance_type': {'memory_mb': 1024,
+                                               'extra_specs': extra_specs}}
+        host = fakes.FakeHostState('host1', 'node1', {})
+
+        filt_cls.host_passes(host, filter_properties)     # Fill the caches
+
+        self.oat_attested = False
+        filt_cls.host_passes(host, filter_properties)
+        self.assertFalse(self.oat_attested)
+
+        self.oat_attested = False
+
+        timeutils.set_time_override(timeutils.utcnow())
+        timeutils.advance_time_seconds(
+            CONF.trusted_computing.attestation_auth_timeout + 80)
+        filt_cls.host_passes(host, filter_properties)
+        self.assertTrue(self.oat_attested)
+
+        timeutils.clear_time_override()
+
+    def test_trusted_filter_update_cache_timezone(self):
+        self.oat_data = {"hosts": [{"host_name": "host1",
+                                    "trust_lvl": "untrusted",
+                                    "vtime": "2012-09-09T05:10:40-04:00"}]}
+
+        filt_cls = self.class_map['TrustedFilter']()
+        extra_specs = {'trust:trusted_host': 'untrusted'}
+        filter_properties = {'context': self.context.elevated(),
+                             'instance_type': {'memory_mb': 1024,
+                                               'extra_specs': extra_specs}}
+        host = fakes.FakeHostState('host1', 'node1', {})
+
+        timeutils.set_time_override(
+            timeutils.normalize_time(
+                timeutils.parse_isotime("2012-09-09T09:10:40Z")))
+
+        filt_cls.host_passes(host, filter_properties)     # Fill the caches
+
+        self.oat_attested = False
+        filt_cls.host_passes(host, filter_properties)
+        self.assertFalse(self.oat_attested)
+
+        self.oat_attested = False
+        timeutils.advance_time_seconds(
+            CONF.trusted_computing.attestation_auth_timeout - 10)
+        filt_cls.host_passes(host, filter_properties)
+        self.assertFalse(self.oat_attested)
+
+        timeutils.clear_time_override()
+
     def test_core_filter_passes(self):
         filt_cls = self.class_map['CoreFilter']()
         filter_properties = {'instance_type': {'vcpus': 1}}