adding docs to test classes,

updating run_tests.sh to match reality
adding debug middleware factory
adding docs on enabling debug middleware
resolving pep8 issues

blueprint keystone-documentation

Change-Id: If16e908f21082bab770b19e1aa384fccc7ee5643

8 files changed, 217 insertions, 20 deletions

diff --git a/doc/source/developing.rst b/doc/source/developing.rst
new file mode 100644
index 00000000..20a82bb9
--- /dev/null
+++ b/doc/source/developing.rst
@@ -0,0 +1,139 @@
+..
+      Copyright 2011 OpenStack, LLC
+      All Rights Reserved.
+
+      Licensed under the Apache License, Version 2.0 (the "License"); you may
+      not use this file except in compliance with the License. You may obtain
+      a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+      WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+      License for the specific language governing permissions and limitations
+      under the License.
+
+========================
+Developing with Keystone
+========================
+
+Get your development environment set up according to :doc:`setup`.
+
+Running a development instance
+==============================
+
+Setting up a virtualenv
+-----------------------
+
+We recommend establishing a virtualenv to run keystone within. To establish
+this environment, use the command::
+
+    $> python tools/install_venv.py
+
+This will create a local virtual environment in the directory ``.keystone-venv``.
+Once created, you can activate this virtualenv for your current shell using:
+
+    $> source .keystone-venv/bin/activate
+
+The virtual environment can be disabled using the command::
+
+    $> deactivate
+
+You can also use ``tools\with_venv.sh`` to prefix commands so that they run
+within the virtual environment. For more information on virtual environments,
+see virtualenv_.
+
+.. _virtualenv: http://www.virtualenv.org/
+
+Running Keystone
+----------------
+
+To run the keystone Admin and API server instances, use::
+
+    $> tools/with_venv.sh bin/keystone
+
+Running a demo service that uses Keystone
+-----------------------------------------
+To run client demo (with all auth middleware running locally on sample service):
+
+    $> tools/with_venv.sh examples/echo/bin/echod
+
+which spins up a simple "echo" service on port 8090. To use a simple echo client:
+
+    $> python examples/echo/echo_client.py
+
+Interacting with Keystone
+=========================
+
+You can interact with Keystone through the command line using :doc:`keystone-manage`
+which allows you to establish tenants, users, etc.
+
+You can also interact with Keystone through it's REST API. There is a python
+keystone client library python-keystoneclient_ which interacts exclusively through
+the REST API.
+
+.. _python-keystoneclient: https://github.com/4P/python-keystoneclient
+
+The easiest way to establish some base information in Keystone to interact with is
+to invoke::
+
+    $> tools/with_venv.sh bin/sampledata
+
+You can see the details of what that creates in ``keystone/test/sampledata.py``
+
+interacting with keystone using curl
+------------------------------------
+
+Get an unscoped token::
+
+    $> curl -d '{"auth": {"passwordCredentials": {"username": "joeuser", "password": "secrete"}}}' -H "Content-type: application/json" http://localhost:5000/v2.0/tokens
+
+Get a token for a tenant::
+
+    $> curl -d '{"auth": {"passwordCredentials": {"username": "joeuser", "password": "secrete"}, "tenantName": "customer-x"}}' -H "Content-type: application/json" http://localhost:5000/v2.0/tokens
+
+Get an admin token::
+
+    $> curl -d '{"auth": {"passwordCredentials": {"username": "admin", "password": "secrete"}}}' -H "Content-type: application/json" http://localhost:35357/v2.0/tokens
+
+Get a list of tenants using the admin token::
+
+    $> curl -d '{"auth": {"passwordCredentials": {"username": "admin", "password": "secrete"}}}' -H "Content-type: application/json" http://localhost:35357/v2.0/tokens
+
+Enabling debugging middleware
+-----------------------------
+
+You can enable a huge amount of additional data (debugging information) about
+the request and repsonse objects flowing through Keystone using the debugging
+WSGI middleware.
+
+To enable this, just modify the pipelines in ``etc/keystone.conf``, changing::
+
+    [pipeline:admin]
+    pipeline =
+        urlrewritefilter
+        admin_api
+
+    [pipeline:keystone-legacy-auth]
+    pipeline =
+        urlrewritefilter
+        legacy_auth
+        RAX-KEY-extension
+        service_api
+
+to::
+
+    [pipeline:admin]
+    pipeline =
+        debug
+        urlrewritefilter
+        admin_api
+
+    [pipeline:keystone-legacy-auth]
+    pipeline =
+        debug
+        urlrewritefilter
+        legacy_auth
+        RAX-KEY-extension
+        service_api
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 85d19565..3d2b74a5 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -71,6 +71,7 @@ Developer Docs
 .. toctree::
     :maxdepth: 1
 
+    developing
     architecture
     sourcecode/autoindex
 
diff --git a/etc/keystone.conf b/etc/keystone.conf
index cb942723..e3ce48b7 100644
--- a/etc/keystone.conf
+++ b/etc/keystone.conf
@@ -62,12 +62,12 @@ sql_idle_timeout = 30
 
 [pipeline:admin]
 pipeline =
-	urlrewritefilter
-	admin_api
+    urlrewritefilter
+    admin_api
 
 [pipeline:keystone-legacy-auth]
 pipeline =
-	urlrewritefilter
+    urlrewritefilter
     legacy_auth
     RAX-KEY-extension
     service_api
@@ -86,3 +86,6 @@ paste.filter_factory = keystone.frontends.legacy_token_auth:filter_factory
 
 [filter:RAX-KEY-extension]
 paste.filter_factory = keystone.contrib.extensions.service.raxkey.frontend:filter_factory
+
+[filter:debug]
+paste.filter_factory = keystone.common.wsgi:debug_filter_factory
diff --git a/keystone/common/wsgi.py b/keystone/common/wsgi.py
index 735fef59..fa83b7d1 100755
--- a/keystone/common/wsgi.py
+++ b/keystone/common/wsgi.py
@@ -183,6 +183,14 @@ class Debug(Middleware):
         print
 
 
+def debug_filter_factory(global_conf):
+    """Filter factor to easily insert a debugging middleware into the
+    paste.deploy pipeline"""
+    def filter(app):
+        return Debug(app)
+    return filter
+
+
 class Router(object):
     """
     WSGI middleware that maps incoming requests to WSGI apps.
diff --git a/keystone/test/__init__.py b/keystone/test/__init__.py
index 9b8c924f..e701a931 100644
--- a/keystone/test/__init__.py
+++ b/keystone/test/__init__.py
@@ -46,6 +46,18 @@ def execute(cmd, raise_error=True):
 
 
 class KeystoneTest(object):
+    """Primary test class for invoking keystone tests. Controls
+    initialization of environment with temporary configuration files,
+    starts keystone admin and service API WSIG servers, and then uses
+    :py:mod:`unittest2` to discover and iterate over existing tests.
+
+    :py:class:`keystone.test.KeystoneTest` is expected to be
+    subclassed and invoked in ``run_tests.py`` where subclasses define
+    a config_name (that matches a template existing in
+    ``keystone/test/etc``) and test_files (that are cleared at the
+    end of test execution from the temporary space used to run these
+    tests).
+    """
     CONF_PARAMS = {'test_dir': TEST_DIR}
 
     def clear_database(self):
diff --git a/keystone/test/functional/common.py b/keystone/test/functional/common.py
index 9a065742..ac3a31e4 100644
--- a/keystone/test/functional/common.py
+++ b/keystone/test/functional/common.py
@@ -6,7 +6,14 @@ from xml.etree import ElementTree
 
 
 class HttpTestCase(unittest.TestCase):
-    """Performs generic HTTP request testing"""
+    """Performs generic HTTP request testing.
+
+    Defines a ``request`` method for use in test cases that makes
+    HTTP requests, and two new asserts:
+
+    * assertResponseSuccessful
+    * assertResponseStatus
+    """
 
     def request(self, host='127.0.0.1', port=80, method='GET', path='/',
             headers=None, body=None, assert_status=None):
@@ -38,13 +45,29 @@ class HttpTestCase(unittest.TestCase):
         return response
 
     def assertResponseSuccessful(self, response):
-        """Asserts that a status code lies inside the 2xx range"""
+        """Asserts that a status code lies inside the 2xx range
+
+        :param response: :py:class:`httplib.HTTPResponse` to be
+        verified to have a status code between 200 and 299.
+
+        example::
+
+            >>> self.assertResponseSuccessful(response, 203)
+        """
         self.assertTrue(response.status >= 200 and response.status <= 299,
             'Status code %d is outside of the expected range (2xx)\n\n%s' %
             (response.status, response.body))
 
     def assertResponseStatus(self, response, assert_status):
-        """Asserts a specific status code on the response"""
+        """Asserts a specific status code on the response
+
+        :param response: :py:class:`httplib.HTTPResponse`
+        :param assert_status: The specific ``status`` result expected
+
+        example::
+
+            >>> self.assertResponseStatus(response, 203)
+        """
         self.assertEqual(response.status, assert_status,
             'Status code %s is not %s, as expected)\n\n%s' %
             (response.status, assert_status, response.body))
@@ -104,11 +127,27 @@ class RestfulTestCase(HttpTestCase):
 
     @staticmethod
     def _encode_json(data):
-        """Returns a JSON-encoded string of the given python dictionary"""
+        """Returns a JSON-encoded string of the given python dictionary
+
+        :param data: python object to be encoded into JSON
+        :returns: string of JSON encoded data
+        """
         return json.dumps(data)
 
     def _decode_response_body(self, response):
-        """Detects response body type, and attempts to decode it"""
+        """Detects response body type, and attempts to decode it
+
+        :param response: :py:class:`httplib.HTTPResponse`
+        :returns: response object with additions:
+
+        If context type is application/json, the response will have an
+        additional attribute ``json`` that will have the decoded JSON
+        result (typically a dict)
+
+        If context type is application/xml, the response will have an
+        additional attribute ``xml`` that will have the an ElementTree
+        result.
+        """
         if response.body != None and response.body.strip():
             if 'application/json' in response.getheader('Content-Type', ''):
                 response.json = self._decode_json(response.body)
diff --git a/run_tests.py b/run_tests.py
index fde2512e..d799a30c 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -5,16 +5,19 @@ from keystone.test import KeystoneTest
 
 
 class SQLTest(KeystoneTest):
+    """Test defined using only SQLAlchemy back-end"""
     config_name = 'sql.conf.template'
     test_files = ('keystone.db',)
 
 
 class MemcacheTest(KeystoneTest):
+    """Test defined using only SQLAlchemy and Memcache back-end"""
     config_name = 'memcache.conf.template'
     test_files = ('keystone.db',)
 
 
 class LDAPTest(KeystoneTest):
+    """Test defined using only SQLAlchemy and LDAP back-end"""
     config_name = 'ldap.conf.template'
     test_files = ('keystone.db', 'ldap.db', 'ldap.db.db',)
 
diff --git a/run_tests.sh b/run_tests.sh
index 447e3f28..8d04e9e2 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -7,7 +7,6 @@ function usage {
   echo "  -V, --virtual-env        Always use virtualenv.  Install automatically if not present"
   echo "  -N, --no-virtual-env     Don't use virtualenv.  Run tests in local environment"
   echo "  -f, --force              Force a clean re-build of the virtual environment. Useful when dependencies have been added."
-  echo "  --unittests-only         Run unit tests only, exclude functional tests."
   echo "  --with-coverage          Runs tests with python code coverage (useful for jenkins)"
   echo "                             Note: cannot be used in combination --with-progress"
   echo "  --with-progress          Runs tests with progress (useful for developers)"
@@ -30,8 +29,7 @@ function process_option {
     -p|--pep8) let just_pep8=1;;
     -l|--pylint) let just_pylint=1; let never_venv=0;;
     -f|--force) let force=1;;
-    --unittests-only) noseargs="$noseargs --exclude-dir=keystone/tests/functional --exclude-dir=keystone/tests/system";;
-    *) noseargs="$noseargs $1"
+    *) addlargs="$addlargs $1"
   esac
 }
 
@@ -40,10 +38,11 @@ with_venv=tools/with_venv.sh
 always_venv=0
 never_venv=0
 force=0
-noseargs=
+addlargs=
 wrapper=""
 just_pep8=0
 just_pylint=0
+RUNTESTS="python run_tests.py $addlargs"
 
 for arg in "$@"; do
   process_option $arg
@@ -51,7 +50,7 @@ done
 
 function run_tests {
   # Just run the test suites in current environment
-  ${wrapper} $NOSETESTS
+  ${wrapper} $RUNTESTS
 }
 
 function run_pep8 {
@@ -71,9 +70,6 @@ function run_pylint {
   echo "Run 'pylint $PYLINT_OPTIONS $PYLINT_INCLUDE' for a full report."
 }
 
-
-NOSETESTS="python run_tests.py $noseargs"
-
 if [ $never_venv -eq 0 ]
 then
   # Remove the virtual environment if --force used
@@ -111,7 +107,3 @@ if [ $just_pylint -eq 1 ]; then
 fi
 
 run_tests || exit
-
-#if [ -z "$noseargs" ]; then
-#  run_pep8
-#fi