Merge trunk and address ttx's review comments

5 files changed, 76 insertions, 244 deletions

diff --git a/Authors b/Authors
index 565444ee1..fa38ef0b1 100644
--- a/Authors
+++ b/Authors
@@ -6,6 +6,7 @@ Chris Behrens <cbehrens@codestud.com>
 Chmouel Boudjnah <chmouel@chmouel.com>
 Dean Troyer <dtroyer@gmail.com>
 Devin Carlen <devin.carlen@gmail.com>
+Ed Leafe <ed@leafe.com>
 Eldar Nugaev <enugaev@griddynamics.com>
 Eric Day <eday@oddments.org>
 Ewan Mellor <ewan.mellor@citrix.com>
@@ -14,6 +15,7 @@ Jay Pipes <jaypipes@gmail.com>
 Jesse Andrews <anotherjesse@gmail.com>
 Joe Heck <heckj@mac.com>
 Joel Moore <joelbm24@gmail.com>
+Jonathan Bryce <jbryce@jbryce.com>
 Josh Kearney <josh.kearney@rackspace.com>
 Joshua McKenty <jmckenty@gmail.com>
 Justin Santa Barbara <justin@fathomdb.com>
diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py
index cdab96b79..33cd03430 100644
--- a/nova/auth/fakeldap.py
+++ b/nova/auth/fakeldap.py
@@ -15,7 +15,7 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-"""Fake LDAP server for test harness, backs to ReDIS.
+"""Fake LDAP server for test harness.
 
 This class does very little error checking, and knows nothing about ldap
 class definitions.  It implements the minimum emulation of the python ldap
@@ -23,20 +23,11 @@ library to work with nova.
 
 """
 
+import fnmatch
 import json
-import redis
 
-from nova import flags
 
-FLAGS = flags.FLAGS
-flags.DEFINE_string('redis_host', '127.0.0.1',
-                    'Host that redis is running on.')
-flags.DEFINE_integer('redis_port', 6379,
-                    'Port that redis is running on.')
-flags.DEFINE_integer('redis_db', 0, 'Multiple DB keeps tests away')
-
-
-class Redis(object):
+class Store(object):
     def __init__(self):
         if hasattr(self.__class__, '_instance'):
             raise Exception(_('Attempted to instantiate singleton'))
@@ -44,13 +35,53 @@ class Redis(object):
     @classmethod
     def instance(cls):
         if not hasattr(cls, '_instance'):
-            inst = redis.Redis(host=FLAGS.redis_host,
-                               port=FLAGS.redis_port,
-                               db=FLAGS.redis_db)
-            cls._instance = inst
+            cls._instance = _StorageDict()
         return cls._instance
 
 
+class _StorageDict(dict):
+    def keys(self, pat=None):
+        ret = super(_StorageDict, self).keys()
+        if pat is not None:
+            ret = fnmatch.filter(ret, pat)
+        return ret
+
+    def delete(self, key):
+        try:
+            del self[key]
+        except KeyError:
+            pass
+
+    def flushdb(self):
+        self.clear()
+
+    def hgetall(self, key):
+        """Returns the hash for the given key; creates
+        the hash if the key doesn't exist."""
+        try:
+            return self[key]
+        except KeyError:
+            self[key] = {}
+            return self[key]
+
+    def hget(self, key, field):
+        hashdict = self.hgetall(key)
+        try:
+            return hashdict[field]
+        except KeyError:
+            hashdict[field] = {}
+            return hashdict[field]
+
+    def hset(self, key, field, val):
+        hashdict = self.hgetall(key)
+        hashdict[field] = val
+
+    def hmset(self, key, value_dict):
+        hashdict = self.hgetall(key)
+        for field, val in value_dict.items():
+            hashdict[field] = val
+
+
 SCOPE_BASE = 0
 SCOPE_ONELEVEL = 1  # Not implemented
 SCOPE_SUBTREE = 2
@@ -169,8 +200,6 @@ def _to_json(unencoded):
 
 
 class FakeLDAP(object):
-    #TODO(vish): refactor this class to use a wrapper instead of accessing
-    #            redis directly
     """Fake LDAP connection."""
 
     def simple_bind_s(self, dn, password):
@@ -183,14 +212,13 @@ class FakeLDAP(object):
 
     def add_s(self, dn, attr):
         """Add an object with the specified attributes at dn."""
-        key = "%s%s" % (self.__redis_prefix, dn)
-
+        key = "%s%s" % (self.__prefix, dn)
         value_dict = dict([(k, _to_json(v)) for k, v in attr])
-        Redis.instance().hmset(key, value_dict)
+        Store.instance().hmset(key, value_dict)
 
     def delete_s(self, dn):
         """Remove the ldap object at specified dn."""
-        Redis.instance().delete("%s%s" % (self.__redis_prefix, dn))
+        Store.instance().delete("%s%s" % (self.__prefix, dn))
 
     def modify_s(self, dn, attrs):
         """Modify the object at dn using the attribute list.
@@ -201,18 +229,18 @@ class FakeLDAP(object):
             ([MOD_ADD | MOD_DELETE | MOD_REPACE], attribute, value)
 
         """
-        redis = Redis.instance()
-        key = "%s%s" % (self.__redis_prefix, dn)
+        store = Store.instance()
+        key = "%s%s" % (self.__prefix, dn)
 
         for cmd, k, v in attrs:
-            values = _from_json(redis.hget(key, k))
+            values = _from_json(store.hget(key, k))
             if cmd == MOD_ADD:
                 values.append(v)
             elif cmd == MOD_REPLACE:
                 values = [v]
             else:
                 values.remove(v)
-            values = redis.hset(key, k, _to_json(values))
+            values = store.hset(key, k, _to_json(values))
 
     def search_s(self, dn, scope, query=None, fields=None):
         """Search for all matching objects under dn using the query.
@@ -226,16 +254,17 @@ class FakeLDAP(object):
         """
         if scope != SCOPE_BASE and scope != SCOPE_SUBTREE:
             raise NotImplementedError(str(scope))
-        redis = Redis.instance()
+        store = Store.instance()
         if scope == SCOPE_BASE:
-            keys = ["%s%s" % (self.__redis_prefix, dn)]
+            keys = ["%s%s" % (self.__prefix, dn)]
         else:
-            keys = redis.keys("%s*%s" % (self.__redis_prefix, dn))
+            keys = store.keys("%s*%s" % (self.__prefix, dn))
+
         objects = []
         for key in keys:
-            # get the attributes from redis
-            attrs = redis.hgetall(key)
-            # turn the values from redis into lists
+            # get the attributes from the store
+            attrs = store.hgetall(key)
+            # turn the values from the store into lists
             # pylint: disable-msg=E1103
             attrs = dict([(k, _from_json(v))
                           for k, v in attrs.iteritems()])
@@ -244,13 +273,13 @@ class FakeLDAP(object):
                 # filter the attributes by fields
                 attrs = dict([(k, v) for k, v in attrs.iteritems()
                               if not fields or k in fields])
-                objects.append((key[len(self.__redis_prefix):], attrs))
+                objects.append((key[len(self.__prefix):], attrs))
             # pylint: enable-msg=E1103
         if objects == []:
             raise NO_SUCH_OBJECT()
         return objects
 
     @property
-    def __redis_prefix(self):  # pylint: disable-msg=R0201
-        """Get the prefix to use for all redis keys."""
+    def __prefix(self):  # pylint: disable-msg=R0201
+        """Get the prefix to use for all keys."""
         return 'ldap:'
diff --git a/nova/service.py b/nova/service.py
index ac30aaceb..f1f90742f 100644
--- a/nova/service.py
+++ b/nova/service.py
@@ -151,7 +151,7 @@ class Service(object):
             report_interval = FLAGS.report_interval
         if not periodic_interval:
             periodic_interval = FLAGS.periodic_interval
-        logging.warn("Starting %s node", topic)
+        logging.warn(_("Starting %s node"), topic)
         service_obj = cls(host, binary, topic, manager,
                           report_interval, periodic_interval)
 
@@ -163,7 +163,7 @@ class Service(object):
         try:
             db.service_destroy(context.get_admin_context(), self.service_id)
         except exception.NotFound:
-            logging.warn("Service killed that has no database entry")
+            logging.warn(_("Service killed that has no database entry"))
 
     def stop(self):
         for x in self.timers:
@@ -184,8 +184,8 @@ class Service(object):
             try:
                 service_ref = db.service_get(ctxt, self.service_id)
             except exception.NotFound:
-                logging.debug("The service database object disappeared, "
-                              "Recreating it.")
+                logging.debug(_("The service database object disappeared, "
+                                "Recreating it."))
                 self._create_service_ref(ctxt)
                 service_ref = db.service_get(ctxt, self.service_id)
 
@@ -196,13 +196,13 @@ class Service(object):
             # TODO(termie): make this pattern be more elegant.
             if getattr(self, "model_disconnected", False):
                 self.model_disconnected = False
-                logging.error("Recovered model server connection!")
+                logging.error(_("Recovered model server connection!"))
 
         # TODO(vish): this should probably only catch connection errors
         except Exception:  # pylint: disable-msg=W0702
             if not getattr(self, "model_disconnected", False):
                 self.model_disconnected = True
-                logging.exception("model server went away")
+                logging.exception(_("model server went away"))
 
 
 def serve(*services):
@@ -221,7 +221,7 @@ def serve(*services):
     else:
         logging.getLogger().setLevel(logging.WARNING)
 
-    logging.debug("Full set of FLAGS:")
+    logging.debug(_("Full set of FLAGS:"))
     for flag in FLAGS:
         logging.debug("%s : %s" % (flag, FLAGS.get(flag, None)))
 
diff --git a/nova/service.py.THIS b/nova/service.py.THIS
deleted file mode 100644
index 348b1d192..000000000
--- a/nova/service.py.THIS
+++ /dev/null
@@ -1,195 +0,0 @@
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright 2010 United States Government as represented by the
-# Administrator of the National Aeronautics and Space Administration.
-# All Rights Reserved.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""
-A service is a very thin wrapper around a Manager object.  It exposes the
-manager's public methods to other components of the system via rpc.  It will
-report state periodically to the database and is responsible for initiating
-any periodic tasts that need to be executed on a given host.
-
-This module contains Service, a generic baseclass for all workers.
-"""
-
-import inspect
-import logging
-import os
-
-from twisted.internet import defer
-from twisted.internet import task
-from twisted.application import service
-
-from nova import context
-from nova import db
-from nova import exception
-from nova import flags
-from nova import rpc
-from nova import utils
-
-
-FLAGS = flags.FLAGS
-flags.DEFINE_integer('report_interval', 10,
-                     'seconds between nodes reporting state to datastore',
-                     lower_bound=1)
-
-flags.DEFINE_integer('periodic_interval', 60,
-                     'seconds between running periodic tasks',
-                     lower_bound=1)
-
-
-class Service(object, service.Service):
-    """Base class for workers that run on hosts."""
-
-    def __init__(self, host, binary, topic, manager, report_interval=None,
-                 periodic_interval=None, *args, **kwargs):
-        self.host = host
-        self.binary = binary
-        self.topic = topic
-        self.manager_class_name = manager
-        self.report_interval = report_interval
-        self.periodic_interval = periodic_interval
-        super(Service, self).__init__(*args, **kwargs)
-        self.saved_args, self.saved_kwargs = args, kwargs
-
-    def startService(self):  # pylint: disable-msg C0103
-        manager_class = utils.import_class(self.manager_class_name)
-        self.manager = manager_class(host=self.host, *self.saved_args,
-                                                     **self.saved_kwargs)
-        self.manager.init_host()
-        self.model_disconnected = False
-        ctxt = context.get_admin_context()
-        try:
-            service_ref = db.service_get_by_args(ctxt,
-                                                 self.host,
-                                                 self.binary)
-            self.service_id = service_ref['id']
-        except exception.NotFound:
-            self._create_service_ref(ctxt)
-
-        conn = rpc.Connection.instance()
-        if self.report_interval:
-            consumer_all = rpc.AdapterConsumer(
-                    connection=conn,
-                    topic=self.topic,
-                    proxy=self)
-            consumer_node = rpc.AdapterConsumer(
-                    connection=conn,
-                    topic='%s.%s' % (self.topic, self.host),
-                    proxy=self)
-
-            consumer_all.attach_to_twisted()
-            consumer_node.attach_to_twisted()
-
-            pulse = task.LoopingCall(self.report_state)
-            pulse.start(interval=self.report_interval, now=False)
-
-        if self.periodic_interval:
-            pulse = task.LoopingCall(self.periodic_tasks)
-            pulse.start(interval=self.periodic_interval, now=False)
-
-    def _create_service_ref(self, context):
-        service_ref = db.service_create(context,
-                                        {'host': self.host,
-                                         'binary': self.binary,
-                                         'topic': self.topic,
-                                         'report_count': 0})
-        self.service_id = service_ref['id']
-
-    def __getattr__(self, key):
-        manager = self.__dict__.get('manager', None)
-        return getattr(manager, key)
-
-    @classmethod
-    def create(cls,
-               host=None,
-               binary=None,
-               topic=None,
-               manager=None,
-               report_interval=None,
-               periodic_interval=None):
-        """Instantiates class and passes back application object.
-
-        Args:
-            host, defaults to FLAGS.host
-            binary, defaults to basename of executable
-            topic, defaults to bin_name - "nova-" part
-            manager, defaults to FLAGS.<topic>_manager
-            report_interval, defaults to FLAGS.report_interval
-            periodic_interval, defaults to FLAGS.periodic_interval
-        """
-        if not host:
-            host = FLAGS.host
-        if not binary:
-            binary = os.path.basename(inspect.stack()[-1][1])
-        if not topic:
-            topic = binary.rpartition("nova-")[2]
-        if not manager:
-            manager = FLAGS.get('%s_manager' % topic, None)
-        if not report_interval:
-            report_interval = FLAGS.report_interval
-        if not periodic_interval:
-            periodic_interval = FLAGS.periodic_interval
-        logging.warn(_("Starting %s node"), topic)
-        service_obj = cls(host, binary, topic, manager,
-                          report_interval, periodic_interval)
-
-        # This is the parent service that twistd will be looking for when it
-        # parses this file, return it so that we can get it into globals.
-        application = service.Application(binary)
-        service_obj.setServiceParent(application)
-        return application
-
-    def kill(self):
-        """Destroy the service object in the datastore"""
-        try:
-            db.service_destroy(context.get_admin_context(), self.service_id)
-        except exception.NotFound:
-            logging.warn(_("Service killed that has no database entry"))
-
-    @defer.inlineCallbacks
-    def periodic_tasks(self):
-        """Tasks to be run at a periodic interval"""
-        yield self.manager.periodic_tasks(context.get_admin_context())
-
-    @defer.inlineCallbacks
-    def report_state(self):
-        """Update the state of this service in the datastore."""
-        ctxt = context.get_admin_context()
-        try:
-            try:
-                service_ref = db.service_get(ctxt, self.service_id)
-            except exception.NotFound:
-                logging.debug(_("The service database object disappeared, "
-                              "Recreating it."))
-                self._create_service_ref(ctxt)
-                service_ref = db.service_get(ctxt, self.service_id)
-
-            db.service_update(ctxt,
-                             self.service_id,
-                             {'report_count': service_ref['report_count'] + 1})
-
-            # TODO(termie): make this pattern be more elegant.
-            if getattr(self, "model_disconnected", False):
-                self.model_disconnected = False
-                logging.error(_("Recovered model server connection!"))
-
-        # TODO(vish): this should probably only catch connection errors
-        except Exception:  # pylint: disable-msg=W0702
-            if not getattr(self, "model_disconnected", False):
-                self.model_disconnected = True
-                logging.exception(_("model server went away"))
-        yield
diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py
index 4508d6721..61ae43fb1 100644
--- a/nova/tests/auth_unittest.py
+++ b/nova/tests/auth_unittest.py
@@ -333,14 +333,10 @@ class AuthManagerLdapTestCase(AuthManagerTestCase, test.TestCase):
         AuthManagerTestCase.__init__(self)
         test.TestCase.__init__(self, *args, **kwargs)
         import nova.auth.fakeldap as fakeldap
-        FLAGS.redis_db = 8
         if FLAGS.flush_db:
-            logging.info("Flushing redis datastore")
-            try:
-                r = fakeldap.Redis.instance()
-                r.flushdb()
-            except:
-                self.skip = True
+            logging.info("Flushing datastore")
+            r = fakeldap.Store.instance()
+            r.flushdb()
 
 
 class AuthManagerDbTestCase(AuthManagerTestCase, test.TestCase):