Updated logging and parameterization for bin scripts

8 files changed, 146 insertions, 80 deletions

diff --git a/README.md b/README.md
index 4b9c2222..f23f538a 100644
--- a/README.md
+++ b/README.md
@@ -78,23 +78,12 @@ A sample client that gets a token from Keystone and then uses it to call Echo (a
 
     $ cd echo/echo
     $ python echo_client.py
-    Note: this requires tests data. See section TESTING for initializing data
+    Note: this requires test data. See section TESTING for initializing data
 
 
-ADDITIONAL INFORMATION:
------------------------
-
-Configuration:
-Keystone gets its configuration from command-line parameters or a .conf file. The file can be provided explicitely
-on the command line otherwise the following logic applies (the conf file in use will be output to help
-in troubleshooting:
 
-1. config.py takes the config file from <topdir>/etc/keystone.conf
-2. If the keystone package is also intalled on the system,
-    /etc/keystone.conf or /etc/keystone/keystone.conf have higher priority than <top_dir>/etc/keystone.conf.
-
-
-Testing:
+TESTING:
+--------
 After starting keystone a keystone.db sqlite database should be created in the keystone folder.
 
 Add test data to the database:
@@ -111,6 +100,7 @@ To run client demo (with all auth middleware running locally on sample service):
     $ python echo/echo/echo_client.py
 
 To run unit tests:
+
 * go to unit test/unit directory
 * run tests: python test_keystone
 
@@ -129,12 +119,26 @@ For more on unit testing please refer
 
 To perform contract validation and load testing, use SoapUI (for now).
 
+
 Using SOAPUI:
 
-Download [SOAPUI](http://sourceforge.net/projects/soapui/files/):
+First, download [SOAPUI](http://sourceforge.net/projects/soapui/files/):
 
 To Test Keystone Service:
 
 * File->Import Project
 * Select tests/IdentitySOAPUI.xml
 * Double click on "Keystone Tests" and press the green play (>) button
+
+
+ADDITIONAL INFORMATION:
+-----------------------
+
+Configuration:
+Keystone gets its configuration from command-line parameters or a .conf file. The file can be provided explicitely
+on the command line otherwise the following logic applies (the conf file in use will be output to help
+in troubleshooting:
+
+1. config.py takes the config file from <topdir>/etc/keystone.conf
+2. If the keystone package is also intalled on the system,
+    /etc/keystone.conf or /etc/keystone/keystone.conf have higher priority than <top_dir>/etc/keystone.conf.
diff --git a/bin/keystone b/bin/keystone
index 69b666d6..8c0ab71e 100755
--- a/bin/keystone
+++ b/bin/keystone
@@ -40,41 +40,41 @@ from keystone.common import config
 from keystone.common import wsgi
 
 
-def create_options(parser):
-    """
-    Sets up the CLI and config-file options that may be
-    parsed and program commands.
-
-    :param parser: The option parser
-    """
-    config.add_common_options(parser)
-    config.add_log_options(parser)
-
-
 if __name__ == '__main__':
-    oparser = optparse.OptionParser(version='%%prog %s'
+    # Parse options (command-line arguments and config file)
+    parser = optparse.OptionParser(version='%%prog %s'
                                     % version.version_string())
-    create_options(oparser)
-    (options, args) = config.parse_options(oparser)
+    common_group = config.add_common_options(parser)
+    config.add_log_options(parser)
+    # Handle a special argument to support starting two endpoints
+    common_group.add_option('-a', '--admin-port', default=8081,
+                            dest="admin_port", metavar="PORT",
+                     help = "specifies port for Admin API to listen"
+                            "on (default is 8080)")
+    (options, args) = config.parse_options(parser)
+
+    # Start services
     print "Using config file:", config.find_config_file(options, args)
     try:
+        # Load Service API server
         conf, app = config.load_paste_app('server', options, args)
         server = wsgi.Server()
         server.start(app, int(conf['bind_port']), conf['bind_host'])
         print "Service API listening on %s:%s" % (conf['bind_host'],
                                               conf['bind_port'])
 
+        # Load Admin API server
         admin_conf, admin_app = config.load_paste_app('admin', options, args)
         admin_server = wsgi.Server()
-        if conf['bind_port'] == admin_conf['bind_port']:
-            admin_bind = admin_conf['bind_port'] + 1
-        else:
-            admin_bind = admin_conf['bind_port']
+        admin_bind = options.get('admin_port') or admin_conf.get('bind_port')
+        if conf['bind_port'] == admin_bind:
+            admin_bind += 1
         admin_server.start(admin_app, int(admin_bind),
                            admin_conf['bind_host'])
         print "Admin API listening on %s:%s" % (admin_conf['bind_host'],
-                                              admin_conf['bind_port'])
+                                              admin_bind)
 
+        # Wait until done
         server.wait()
     except RuntimeError, e:
         sys.exit("ERROR: %s" % e)
diff --git a/bin/keystone-admin b/bin/keystone-admin
index 5250d138..4be0483c 100755
--- a/bin/keystone-admin
+++ b/bin/keystone-admin
@@ -39,30 +39,29 @@ from keystone.common import config
 from keystone.common import wsgi
 
 
-def create_options(parser):
-    """
-    Sets up the CLI and config-file options that may be
-    parsed and program commands.
-
-    :param parser: The option parser
-    """
-    config.add_common_options(parser)
-    config.add_log_options(parser)
-
-
 if __name__ == '__main__':
-    oparser = optparse.OptionParser(version='%%prog %s'
+    # Parse options (command-line arguments and config file)
+    parser = optparse.OptionParser(version='%%prog %s'
                                     % version.version_string())
-    create_options(oparser)
-    (options, args) = config.parse_options(oparser)
-    print "Using config file:", config.find_config_file(options, args)
+    common_group = config.add_common_options(parser)
+    config.add_log_options(parser)
+    # Handle a special argument to support starting two endpoints
+    common_group.add_option('-a', '--admin-port', default=8081,
+                            dest="admin_port", metavar="PORT",
+                     help = "specifies port for Admin API to listen"
+                            "on (default is 8080)")
+    (options, args) = config.parse_options(parser)
 
+    # Start services
+    print "Using config file:", config.find_config_file(options, args)
     try:
+        # Load Service API server
         conf, app = config.load_paste_app('admin', options, args)
         server = wsgi.Server()
         server.start(app, int(conf['bind_port']), conf['bind_host'])
         print "Admin API listening on %s:%s" % (conf['bind_host'],
                                               conf['bind_port'])
+
         server.wait()
     except RuntimeError, e:
         sys.exit("ERROR: %s" % e)
diff --git a/bin/keystone-auth b/bin/keystone-auth
index 9bbaaf8a..2c934630 100755
--- a/bin/keystone-auth
+++ b/bin/keystone-auth
@@ -39,25 +39,19 @@ from keystone.common import config
 from keystone.common import wsgi
 
 
-def create_options(parser):
-    """
-    Sets up the CLI and config-file options that may be
-    parsed and program commands.
-
-    :param parser: The option parser
-    """
-    config.add_common_options(parser)
-    config.add_log_options(parser)
-
 
 if __name__ == '__main__':
-    oparser = optparse.OptionParser(version='%%prog %s'
+    # Parse options (command-line arguments and config file)
+    parser = optparse.OptionParser(version='%%prog %s'
                                     % version.version_string())
-    create_options(oparser)
-    (options, args) = config.parse_options(oparser)
-    print "Using config file:", config.find_config_file(options, args)
+    common_group = config.add_common_options(parser)
+    config.add_log_options(parser)
+    (options, args) = config.parse_options(parser)
 
+    # Start services
+    print "Using config file:", config.find_config_file(options, args)
     try:
+        # Load Service API server
         conf, app = config.load_paste_app('server', options, args)
         server = wsgi.Server()
         server.start(app, int(conf['bind_port']), conf['bind_host'])
diff --git a/etc/keystone.conf b/etc/keystone.conf
index b6b0a66d..e9c59046 100644
--- a/etc/keystone.conf
+++ b/etc/keystone.conf
@@ -1,9 +1,9 @@
 [DEFAULT]
 # Show more verbose log output (sets INFO log level output)
-verbose = True
+verbose = False
 
 # Show debugging output in logs (sets DEBUG log level output)
-debug = True
+debug = False
 
 # Which backend store should Keystone use by default.
 # Default: 'sqlite'
@@ -44,4 +44,3 @@ bind_host = 0.0.0.0
 
 # Port the bind the API server to
 bind_port = 8080
-
diff --git a/keystone/common/config.py b/keystone/common/config.py
index 78f05cdc..7831e687 100644
--- a/keystone/common/config.py
+++ b/keystone/common/config.py
@@ -75,14 +75,23 @@ def add_common_options(parser):
                      help="Print more verbose output")
     group.add_option('-d', '--debug', default=False, dest="debug",
                      action="store_true",
-                     help="Print debugging output")
-    group.add_option('--config-file', default=None, metavar="PATH",
+                     help="Print debugging output to console")
+    group.add_option('-c', '--config-file', default=None, metavar="PATH",
                      help="Path to the config file to use. When not specified "
                           "(the default), we generally look at the first "
                           "argument specified to be a config file, and if "
                           "that is also missing, we search standard "
                           "directories for a config file.")
+    group.add_option('-p', '--port', '--bind-port', default=8080,
+                     dest="bind_port",
+                     help="specifies port to listen on (default is 8080)")
+    group.add_option('--host', '--bind-host',
+                     default="0.0.0.0", dest="bind_host",
+                     help="specifies host address to listen on "
+                            "(default is all or 0.0.0.0)")
+
     parser.add_option_group(group)
+    return group
 
 
 def add_log_options(parser):
@@ -112,7 +121,30 @@ def add_log_options(parser):
     group.add_option("--log-dir", default=None,
                       help="(Optional) The directory to keep log files in "
                            "(will be prepended to --logfile)")
+
     parser.add_option_group(group)
+    return group
+
+
+def add_console_handler(logger):
+    # add a Handler which writes INFO messages or higher to sys.stderr
+    # which is often the console
+    console = None
+    for console in logger.handlers:
+        if isinstance(console, logging.StreamHandler):
+            break
+
+    if not console:
+        console = logging.StreamHandler()
+        console.setLevel(logging.INFO)
+        # set a format which is simpler for console use
+        formatter = logging.Formatter("%(name)-12s: "
+                                      "%(levelname)-8s %(message)s")
+        # tell the handler to use this format
+        console.setFormatter(formatter)
+        # add the handler to the root logger
+        logger.addHandler(console)
+    return console
 
 
 def setup_logging(options, conf):
@@ -138,6 +170,7 @@ def setup_logging(options, conf):
     root_logger = logging.root
     if debug:
         root_logger.setLevel(logging.DEBUG)
+        add_console_handler(root_logger)
     elif verbose:
         root_logger.setLevel(logging.INFO)
     else:
@@ -163,7 +196,6 @@ def setup_logging(options, conf):
             logfile = os.path.join(logdir, logfile)
         logfile = logging.FileHandler(logfile)
         logfile.setFormatter(formatter)
-        logfile.setFormatter(formatter)
         root_logger.addHandler(logfile)
     else:
         handler = logging.StreamHandler(sys.stdout)
@@ -295,15 +327,15 @@ def load_paste_app(app_name, options, args):
         # Log the options used when starting if we're in debug mode...
         if debug:
             logger = logging.getLogger(app_name)
-            logger.debug("*" * 80)
-            logger.debug("Configuration options gathered from config file:")
-            logger.debug(conf_file)
-            logger.debug("================================================")
+            logger.info("*" * 50)
+            logger.info("Configuration options gathered from config file:")
+            logger.info(conf_file)
+            logger.info("================================================")
             items = dict([(k, v) for k, v in conf.items()
                           if k not in ('__file__', 'here')])
             for key, value in sorted(items.items()):
-                logger.debug("%(key)-30s %(value)s" % locals())
-            logger.debug("*" * 80)
+                logger.info("%(key)-20s %(value)s" % locals())
+            logger.info("*" * 50)
         app = deploy.loadapp("config:%s" % conf_file, name=app_name)
     except (LookupError, ImportError), e:
         raise RuntimeError("Unable to load %(app_name)s from "
diff --git a/keystone/common/template.py b/keystone/common/template.py
index 58c8bd6e..f8412d62 100644
--- a/keystone/common/template.py
+++ b/keystone/common/template.py
@@ -1,3 +1,22 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# Template library copied from bottle: http://bottlepy.org/
 #
 # Copyright (c) 2011, Marcel Hellkamp.
 #
@@ -19,7 +38,6 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #
-# original code copied from bottle.py
 
 
 import cgi
diff --git a/keystone/common/wsgi.py b/keystone/common/wsgi.py
index 0a4f1564..eab7307b 100755
--- a/keystone/common/wsgi.py
+++ b/keystone/common/wsgi.py
@@ -34,19 +34,39 @@ import routes.middleware
 import webob.dec
 import webob.exc
 
+def add_console_handler(logger, level):
+    # add a Handler which writes INFO messages or higher to sys.stderr
+    # which is often the console
+    console = None
+    for console in logger.handlers:
+        if isinstance(console, logging.StreamHandler):
+            break
+            
+    if not console:
+        console = logging.StreamHandler()
+        console.setLevel(level)
+        # set a format which is simpler for console use
+        formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
+        # tell the handler to use this format
+        console.setFormatter(formatter)
+        # add the handler to the root logger
+        logger.addHandler(console)
+    elif console.level != level:
+        console.setLevel(level)
+    return console
 
 class WritableLogger(object):
     """A thin wrapper that responds to `write` and logs."""
 
     def __init__(self, logger, level=logging.DEBUG):
+        print level, logging.DEBUG
         self.logger = logger
         self.level = level
+        if level == logging.DEBUG:
+            add_console_handler(logger, level)
 
     def write(self, msg):
         self.logger.log(self.level, msg.strip("\n"))
-        #TODO(Ziad): remove this when we get stable
-        if self.level == logging.DEBUG:
-            print msg.strip("\n")
 
 
 def run_server(application, port):
@@ -77,7 +97,7 @@ class Server(object):
         """Start a WSGI server in a new green thread."""
         logger = logging.getLogger('eventlet.wsgi.server')
         eventlet.wsgi.server(socket, application, custom_pool=self.pool,
-                             log=WritableLogger(logger))
+                             log=WritableLogger(logger, logging.root.level))
 
 
 class Middleware(object):