Finished fist draft of plugin tutorial in ipalib/__init__.py docstring

5 files changed, 650 insertions, 17 deletions

diff --git a/ipalib/__init__.py b/ipalib/__init__.py
index 5cc4c121..4db6a04f 100644
--- a/ipalib/__init__.py
+++ b/ipalib/__init__.py
@@ -17,17 +17,622 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 
-"""
+
+'''
 Package containing core library.
 
-To learn about the ``ipalib`` library, you should read the code in this order:
+=============================
+ Tutorial for Plugin Authors
+=============================
+
+This tutorial gives a broad learn-by-doing introduction to writing plugins
+for freeIPA v2.  As not to overwhelm the reader, it does not cover every
+detail, but it does provides enough to get one started and is heavily
+cross-referenced with further documentation that (hopefully) fills in the
+missing details.
+
+Where the documentation has left the reader confused, the many built-in
+plugins in `ipalib.plugins` and `ipa_server.plugins` provide real-life
+examples of how to write good plugins.
+
+*Note:*
+
+    This tutorial, along with all the Python docstrings in freeIPA v2,
+    uses the *reStructuredText* markup language.  For documentation on
+    reStructuredText, see:
+
+        http://docutils.sourceforge.net/rst.html
+
+    For documentation on using reStructuredText markup with epydoc, see:
+
+        http://epydoc.sourceforge.net/manual-othermarkup.html
+
+
+------------------------------------
+First steps: A simple command plugin
+------------------------------------
+
+Our first example will create the most basic command plugin possible.  A
+command plugin simultaneously adds a new command that can be called through
+the command-line ``ipa`` script *and* adds a new XML-RPC method... the two are
+one in the same, simply invoked in different ways.
+
+All plugins must subclass from `plugable.Plugin`, and furthermore, must
+subclass from one of the base classes allowed by the `plugable.API` instance
+returned by the `get_standard_api()` function.
+
+To be a command plugin, your plugin must subclass from `frontend.Command`.
+Creating a basic plugin involves two steps, defining the class and then
+registering the class:
+
+>>> from ipalib import Command, get_standard_api
+>>> api = get_standard_api()
+>>> class my_command(Command): # Step 1, define class
+...     """My example plugin."""
+...
+>>> api.register(my_command) # Step 2, register class
+
+Notice that we are registering the ``my_command`` class itself and not an
+instance thereof.
+
+Until `plugable.API.finalize()` is called, your plugin class has not been
+instantiated nor the does the ``Command`` namespace yet exist.  For example:
+
+>>> hasattr(api, 'Command')
+False
+>>> api.finalize()
+>>> hasattr(api.Command, 'my_command')
+True
+>>> api.Command.my_command.doc
+'My example plugin.'
+
+Notice that your plugin instance in accessed through an attribute named
+'my_command', the same name as your plugin class name.
+
+
+------------------------------
+Make your command do something
+------------------------------
+
+This simplest way to make your example command plugin do something is to
+implement a ``run()`` method, like this:
+
+>>> class my_command(Command):
+...     """My example plugin with run()."""
+...
+...     def run(self):
+...         return 'My run() method was called!'
+...
+>>> api = get_standard_api()
+>>> api.register(my_command)
+>>> api.finalize()
+>>> api.Command.my_command() # Call your plugin
+'My run() method was called!'
+
+When `frontend.Command.__call__()` is called, it first validates any arguments
+and options your command plugin takes (if any) and then calls its ``run()``
+method.
+
+
+------------------------
+Forwarding vs. execution
+------------------------
+
+However, unlike the example above, a typical command plugin will implement an
+``execute()`` method instead of a ``run()`` method.  Your command plugin can
+be loaded in two distinct contexts:
+
+    1. In a *client* context - Your command plugin is only used to validate
+       any arguments and options it takes, and then ``self.forward()`` is
+       called, which forwards the call over XML-RPC to an IPA server where
+       the actual work is done.
+
+    2. In a *server* context - Your same command plugin validates any
+       arguments and options it takes, and then ``self.execute()`` is called,
+       which you should implement to perform whatever work your plugin does.
+
+The base `frontend.Command.run()` method simply dispatches the call to
+``self.execute()`` if ``self.env.in_server`` is True, or otherwise
+dispatches the call to ``self.forward()``.
+
+For example, say you have a command plugin like this:
+
+>>> class my_command(Command):
+...     """Forwarding vs. execution."""
+...
+...     def forward(self):
+...         return 'in_server=%r; forward() was called.' % self.env.in_server
+...
+...     def execute(self):
+...         return 'in_server=%r; execute() was called.' % self.env.in_server
+...
+
+If ``my_command`` is loaded in a *client* context, ``forward()`` will be
+called:
+
+>>> api = get_standard_api()
+>>> api.env.in_server = False # run() will dispatch to forward()
+>>> api.register(my_command)
+>>> api.finalize()
+>>> api.Command.my_command() # Call your command plugin
+'in_server=False; forward() was called.'
+
+On the other hand, if ``my_command`` is loaded in a *server* context,
+``execute()`` will be called:
+
+>>> api = get_standard_api()
+>>> api.env.in_server = True # run() will dispatch to execute()
+>>> api.register(my_command)
+>>> api.finalize()
+>>> api.Command.my_command() # Call your command plugin
+'in_server=True; execute() was called.'
+
+Normally there should be no reason to override `frontend.Command.forward()`,
+but, as above, it can be done for demonstration purposes.  In contrast, there
+*is* a reason you might want to override `frontend.Command.run()`: if it only
+makes sense to execute your command locally, if it should never be forwarded
+to the server.  In this case, you should implement your *do-stuff* in the
+``run()`` method instead of in the ``execute()`` method.
+
+For example, the ``ipa`` command line script has a ``help`` command
+(`ipalib.cli.help`) that is specific to the command-line-interface and should
+never be forwarded to the server.
+
+
+---------------
+Backend plugins
+---------------
+
+There are two types of plugins:
+
+    1. *Frontend plugins* - These are loaded in both the *client* and *server*
+       contexts.  These need to be installed with any application built atop
+       the `ipalib` library.  The built-in frontend plugins can be found in
+       `ipalib.plugins`.  The ``my_command`` example above is a frontend
+       plugin.
+
+    2. *Backend plugins* - These are only loaded in a *server* context and
+       only need to be installed on the IPA server.  The built-in backend
+       plugins can be found in `ipa_server.plugins`.
+
+Backend plugins should provide a set of methods that standardize how IPA
+interacts with some external system or library.  For example, all interaction
+with LDAP is done through the ``ldap`` backend plugin defined in
+`ipa_server.plugins.b_ldap`.  As a good rule of thumb, anytime you need to
+import some package that is not part of the Python standard library, you
+should probably interact with that package via a corresponding backend
+plugin you implement.
+
+Backend plugins are much more free-form than command plugins.  Aside from a
+few reserved attribute names, you can define arbitrary public methods on your
+backend plugin (in contrast, frontend plugins get wrapped in a
+`plugable.PluginProxy`, which allow access to only specific attributes on the
+frontend plugin).
+
+Here is a simple example:
+
+>>> from ipalib import Backend
+>>> class my_backend(Backend):
+...     """My example backend plugin."""
+...
+...     def do_stuff(self):
+...         """Part of your API."""
+...         return 'Stuff got done.'
+...
+>>> api = get_standard_api()
+>>> api.register(my_backend)
+>>> api.finalize()
+>>> api.Backend.my_backend.do_stuff()
+'Stuff got done.'
+
+
+-------------------------------
+How your command should do work
+-------------------------------
+
+We now return to our ``my_command`` plugin example.
+
+Plugins are separated into frontend and backend plugins so that there are not
+unnecessary dependencies required by an application that only uses `ipalib` and
+its built-in frontend plugins (and then forwards over XML-RPC for execution).
+
+But how do we avoid introducing additional dependencies?  For example, the
+``user_add`` command needs to talk to LDAP to add the user, yet we want to
+somehow load the ``user_add`` plugin on client machines without requiring the
+``python-ldap`` package (Python bindings to openldap) to be installed.  To
+answer that, we consult our golden rule:
+
+  **The golden rule:** A command plugin should implement its ``execute()``
+  method strictly via calls to methods on one or more backend plugins.
+
+So the module containing the ``user_add`` command does not itself import the
+Python LDAP bindings, only the module containing the ``ldap`` backend plugin
+does that, and the backend plugins are only installed on the server.  The
+``user_add.execute()`` method, which is only called when in a server context,
+is implemented as a series of calls to methods on the ``ldap`` backend plugin.
+
+When `plugable.Plugin.set_api()` is called, each plugin stores a reference to
+the `plugable.API` instance it has been loaded into.  So your plugin can
+access the ``my_backend`` plugin as ``self.api.Backend.my_backend``.
+
+Additionally, convenience attributes are set for each namespace, so your
+plugin can also access the ``my_backend`` plugin as simply
+``self.Backend.my_backend``.
+
+This next example will tie everything together.  First we create our backend
+plugin:
+
+>>> api = get_standard_api()
+>>> api.env.in_server = True # We want to execute, not forward
+>>> class my_backend(Backend):
+...     """My example backend plugin."""
+...
+...     def do_stuff(self):
+...         """my_command.execute() calls this."""
+...         return 'my_backend.do_stuff() indeed did do stuff!'
+...
+>>> api.register(my_backend)
+
+Second, we have our frontend plugin, the command:
+
+>>> class my_command(Command):
+...     """My example command plugin."""
+...
+...     def execute(self):
+...         """Implemented against Backend.my_backend"""
+...         return self.Backend.my_backend.do_stuff()
+...
+>>> api.register(my_command)
+
+Lastly, we call ``api.finalize()`` and see what happens when we call
+``my_command()``:
+
+>>> api.finalize()
+>>> api.Command.my_command()
+'my_backend.do_stuff() indeed did do stuff!'
+
+When not in a server context, ``my_command.execute()`` never gets called, so
+it never tries to access the non-existent backend plugin at
+``self.Backend.my_backend.``  To emphasize this point, here is one last
+example:
+
+>>> api = get_standard_api()
+>>> api.env.in_server = False # We want to forward, not execute
+>>> class my_command(Command):
+...     """My example command plugin."""
+...
+...     def execute(self):
+...         """Same as above."""
+...         return self.Backend.my_backend.do_stuff()
+...
+...     def forward(self):
+...         return 'Just my_command.forward() getting called here.'
+...
+>>> api.register(my_command)
+>>> api.finalize()
+
+Notice that the ``my_backend`` plugin has certainly not be registered:
+
+>>> hasattr(api.Backend, 'my_backend')
+False
+
+And yet we can call ``my_command()``:
+
+>>> api.Command.my_command()
+'Just my_command.forward() getting called here.'
+
+
+----------------------------------------
+Calling other commands from your command
+----------------------------------------
+
+It can be useful to have your ``execute()`` method call other command plugins.
+Among other things, this allows for meta-commands that conveniently call
+several other commands in a single operation.  For example:
+
+>>> api = get_standard_api()
+>>> api.env.in_server = True # We want to execute, not forward
+>>> class meta_command(Command):
+...     """My meta-command plugin."""
+...
+...     def execute(self):
+...         """Calls command_1(), command_2()"""
+...         return '%s; %s.' % (
+...             self.Command.command_1(),
+...             self.Command.command_2()
+...         )
+>>> class command_1(Command):
+...     def execute(self):
+...         return 'command_1.execute() called'
+...
+>>> class command_2(Command):
+...     def execute(self):
+...         return 'command_2.execute() called'
+...
+>>> api.register(meta_command)
+>>> api.register(command_1)
+>>> api.register(command_2)
+>>> api.finalize()
+>>> api.Command.meta_command()
+'command_1.execute() called; command_2.execute() called.'
+
+Because this is quite useful, we are going to revise our golden rule somewhat:
 
-    1. Get the big picture from some actual plugins, like `plugins.f_user`.
+  **The revised golden rule:** A command plugin should implement its
+  ``execute()`` method strictly via what it can access through ``self.api``,
+  most likely via the backend plugins in ``self.api.Backend`` (which can also
+  be conveniently accessed as ``self.Backend``).
 
-    2. Learn about the base classes for frontend plugins in `frontend`.
 
-    3. Learn about the core plugin framework in `plugable`.
-"""
+-----------------------------------------------
+Defining arguments and options for your command
+-----------------------------------------------
+
+You can define a command can accept arbitrary arguments and options.
+For example:
+
+>>> from ipalib import Param
+>>> class nudge(Command):
+...     """Takes one argument, one option"""
+...
+...     takes_args = ['programmer']
+...
+...     takes_options = [Param('stuff', default=u'documentation')]
+...
+...     def execute(self, programmer, **kw):
+...         return '%s, go write more %s!' % (programmer, kw['stuff'])
+...
+>>> api = get_standard_api()
+>>> api.env.in_server = True
+>>> api.register(nudge)
+>>> api.finalize()
+>>> api.Command.nudge('Jason')
+u'Jason, go write more documentation!'
+>>> api.Command.nudge('Jason', stuff='unit tests')
+u'Jason, go write more unit tests!'
+
+The ``args`` and ``options`` attributes are `plugable.NameSpace` instances
+containing a command's arguments and options, respectively, as you can see:
+
+>>> list(api.Command.nudge.args) # Iterates through argument names
+['programmer']
+>>> api.Command.nudge.args.programmer
+Param('programmer', Unicode())
+>>> list(api.Command.nudge.options) # Iterates through option names
+['stuff']
+>>> api.Command.nudge.options.stuff
+Param('stuff', Unicode())
+>>> api.Command.nudge.options.stuff.default
+u'documentation'
+
+The arguments and options must not contain colliding names.  They are both
+merged together into the ``params`` attribute, another `plugable.NameSpace`
+instance, as you can see:
+
+>>> api.Command.nudge.params
+NameSpace(<2 members>, sort=False)
+>>> list(api.Command.nudge.params) # Iterates through the param names
+['programmer', 'stuff']
+
+When calling a command, its positional arguments can also be provided as
+keyword arguments, and in any order.  For example:
+
+>>> api.Command.nudge(stuff='lines of code', programmer='Jason')
+u'Jason, go write more lines of code!'
+
+When a command plugin is called, the values supplied for its parameters are
+put through a sophisticated processing pipeline that includes steps for
+normalization, type conversion, validation, and dynamically constructing
+the defaults for missing values.  The details wont be covered here; however,
+here is a quick teaser:
+
+>>> from ipalib import Int
+>>> class create_player(Command):
+...     takes_options = [
+...         'first',
+...         'last',
+...         Param('nick',
+...             normalize=lambda value: value.lower(),
+...             default_from=lambda first, last: first[0] + last,
+...         ),
+...         Param('points', type=Int(), default=0),
+...     ]
+...
+>>> cp = create_player()
+>>> cp.finalize()
+>>> cp.convert(points=" 1000  ")
+{'points': 1000}
+>>> cp.normalize(nick=u'NickName')
+{'nick': u'nickname'}
+>>> cp.get_default(first='Jason', last='DeRose')
+{'nick': u'jderose', 'points': 0}
+
+For the full details on the parameter system, see the
+`frontend.parse_param_spec()` function, and the `frontend.Param` and
+`frontend.Command` classes.
+
+
+------------------------
+Logging from your plugin
+------------------------
+
+After `plugable.Plugin.set_api()` is called, your plugin will have a
+``self.log`` attribute.  Plugins should only log through this attribute.
+For example:
+
+>>> class paint_house(Command):
+...
+...     takes_args = ['color']
+...
+...     def execute(self, color):
+...         """Uses self.log.error()"""
+...         if color not in ('red', 'blue', 'green'):
+...             self.log.error("I don't have %s paint!", color) # Log error
+...             return
+...         return 'I painted the house %s.' % color
+...
+
+Some basic knowledge of the Python ``logging`` module might be helpful. See:
+
+    http://www.python.org/doc/2.5.2/lib/module-logging.html
+
+The important thing to remember is that your plugin should not configure
+logging itself, but should instead simply use the ``self.log`` logger.
+
+Also see the `plugable.API.bootstrap()` method for details on how the logging
+is configured.
+
+
+---------------------
+Environment variables
+---------------------
+
+Plugins access various environment variables and run-time information through
+``self.api.env`` (for convenience, ``self.env`` is equivalent).
+
+When you create a fresh `plugable.API` instance, its ``env`` attribute is
+likewise a freshly created `config.Env` instance, which will already be
+populated with certain run-time information.  For example:
+
+>>> api = get_standard_api()
+>>> list(api.env)
+['bin', 'dot_ipa', 'home', 'ipalib', 'mode', 'script', 'site_packages']
+
+Here is a quick overview of the run-time information:
+
+=============  ================================  =======================
+Key            Source or example value           Description
+=============  ================================  =======================
+bin            /usr/bin                          Dir. containing script
+dot_ipa        ~/.ipa                            User config directory
+home           os.environ['HOME']                User home dir.
+ipalib         .../site-packages/ipalib          Dir. of ipalib package
+mode           'production' or 'unit_test'       The mode ipalib is in
+script         sys.argv[0]                       Path of script
+site_packages  /usr/lib/python2.5/site-packages  Dir. containing ipalib/
+=============  ================================  =======================
+
+After `plugable.API.bootstrap()` has been called, the env instance will be
+populated with all the environment information used by the built-in plugins.
+This will typically be called before any plugins are registered.  For example:
+
+>>> len(api.env)
+7
+>>> api.bootstrap(in_server=True) # We want to execute, not forward
+>>> len(api.env)
+33
+
+If your plugin requires new environment variables *and* will be included in
+the freeIPA built-in plugins, you should add the defaults for your variables
+in `ipalib.constants.DEFAULT_CONFIG`.  Also, you should consider whether your
+new environment variables should have any auto-magic logic to determine their
+values if they haven't already been set by the time `config.Env._bootstrap()`,
+`config.Env._finalize_core()`, or `config.Env._finalize()` is called.
+
+On the other hand, if your plugin requires new environment variables and will
+be installed in a 3rd-party package, your plugin should set these variables
+in the module it is defined in.
+
+`config.Env` values work on a first-one-wins basis... after a value has been
+set, it can not be overridden with a new value.  As any variables can be set
+using the command-line ``-e`` global option or set in a configuration file,
+your module must check whether a variable has already been set before
+setting its default value.  For example:
+
+>>> if 'message_of_the_day' not in api.env:
+...     api.env.message_of_the_day = 'Hello, world!'
+...
+
+Your plugin can access any environment variables via ``self.env``.
+For example:
+
+>>> class motd(Command):
+...     """Print message of the day."""
+...
+...     def execute(self):
+...         return self.env.message_of_the_day
+...
+>>> api.register(motd)
+>>> api.finalize()
+>>> api.Command.motd()
+'Hello, world!'
+
+Also see the `plugable.API.bootstrap_with_global_options()` method.
+
+
+---------------------------------------------
+Indispensable ipa script commands and options
+---------------------------------------------
+
+The ``console`` command will launch a custom interactive Python interpreter
+session.  The global environment will have an ``api`` variable, which is the
+standard `plugable.API` instance found at ``ipalib.api``.  All plugins will
+have been loaded (well, except the backend plugins if ``in_server`` is False)
+and ``api`` will be fully initialized.  To launch the console from within the
+top-level directory in the the source tree, just run ``ipa console`` from a
+terminal, like this:
+
+    ::
+
+    $ ./ipa console
+
+By default, ``in_server`` is False.  If you want to start the console in a
+server context (so that all the backend plugins are loaded), you can use the
+``-e`` option to set the ``in_server`` environment variable, like this:
+
+    ::
+
+    $ ./ipa -e in_server=True console
+
+You can specify multiple environment variables by including the ``-e`` option
+multiple times, like this:
+
+    ::
+
+    $ ./ipa -e in_server=True -e mode=dummy console
+
+The space after the ``-e`` is optional.  This is equivalent to the above command:
+
+    ::
+
+    $ ./ipa -ein_server=True -emode=dummy console
+
+The ``env`` command will print out the full environment in key=value pairs,
+like this:
+
+    ::
+
+    $ ./ipa env
+
+If you use the ``--server`` option, it will forward the call to the server
+over XML-RPC and print out what the environment is on the server, like this:
+
+    ::
+
+    $ ./ipa env --server
+
+The ``plugins`` command will show details of all the plugin that are loaded,
+like this:
+
+    ::
+
+    $ ./ipa plugins
+
+
+----------------
+Learning more...
+----------------
+
+To learn more about writing plugins, you should:
+
+    1. Look at some of the built-in plugins, like the frontend plugins in
+       `ipalib.plugins.f_user` and the backend plugins in
+       `ipa_server.plugins.b_ldap`.
+
+    2. Learn about the base classes for frontend plugins in `ipalib.frontend`.
+
+    3. Learn about the core plugin framework in `ipalib.plugable`.
+'''
 
 import plugable
 from backend import Backend, Context
@@ -35,11 +640,34 @@ from frontend import Command, Object, Method, Property, Application
 from ipa_types import Bool, Int, Unicode, Enum
 from frontend import Param, DefaultFrom
 
-def get_standard_api():
-    return plugable.API(
+def get_standard_api(mode='dummy'):
+    """
+    Return standard `plugable.API` instance.
+
+    This standard instance allows plugins that subclass from the following
+    base classes:
+
+        - `frontend.Command`
+
+        - `frontend.Object`
+
+        - `frontend.Method`
+
+        - `frontend.Property`
+
+        - `frontend.Application`
+
+        - `backend.Backend`
+
+        - `backend.Context`
+    """
+    api = plugable.API(
         Command, Object, Method, Property, Application,
         Backend, Context,
     )
+    if mode is not None:
+        api.env.mode = mode
+    return api
 
 
-api = get_standard_api()
+api = get_standard_api(mode=None)
diff --git a/ipalib/config.py b/ipalib/config.py
index 02a3fadd..aa7d9cdf 100644
--- a/ipalib/config.py
+++ b/ipalib/config.py
@@ -202,8 +202,8 @@ class Env(object):
         """
         self.__doing('_finalize_core')
         self.__do_if_not_done('_bootstrap')
-        self._merge_config(self.conf)
-        if self.conf_default != self.conf:
+        if self.__d.get('mode', None) != 'dummy':
+            self._merge_config(self.conf)
             self._merge_config(self.conf_default)
         if 'in_server' not in self:
             self.in_server = (self.context == 'server')
@@ -335,7 +335,13 @@ class Env(object):
         """
         return key in self.__d
 
-    def __iter__(self): # Fix
+    def __len__(self):
+        """
+        Return number of variables currently set.
+        """
+        return len(self.__d)
+
+    def __iter__(self):
         """
         Iterate through keys in ascending order.
         """
diff --git a/ipalib/plugable.py b/ipalib/plugable.py
index 5e0611f9..d65a83e2 100644
--- a/ipalib/plugable.py
+++ b/ipalib/plugable.py
@@ -808,7 +808,7 @@ class API(DictProxy):
         log.addHandler(stderr)
 
         # Add file handler:
-        if self.env.mode == 'unit_test':
+        if self.env.mode in ('dummy', 'unit_test'):
             return # But not if in unit-test mode
         log_dir = path.dirname(self.env.log)
         if not path.isdir(log_dir):
@@ -860,7 +860,7 @@ class API(DictProxy):
         """
         self.__doing('load_plugins')
         self.__do_if_not_done('bootstrap')
-        if self.env.mode == 'unit_test':
+        if self.env.mode in ('dummy', 'unit_test'):
             return
         util.import_plugins_subpackage('ipalib')
         if self.env.in_server:
diff --git a/make-test b/make-test
index 2d47707c..f916cc3f 100755
--- a/make-test
+++ b/make-test
@@ -11,7 +11,7 @@ do
     if [[ -f $executable ]]; then
         echo "[ $name: Starting tests... ]"
         ((runs += 1))
-        if $executable /usr/bin/nosetests -v --with-doctest
+        if $executable /usr/bin/nosetests -v --with-doctest --stop
         then
             echo "[ $name: Tests OK ]"
         else
diff --git a/tests/util.py b/tests/util.py
index aa0299fd..d7aaae89 100644
--- a/tests/util.py
+++ b/tests/util.py
@@ -219,8 +219,7 @@ def get_api(**kw):
     instance and a `TempHome` instance.
     """
     home = TempHome()
-    api = ipalib.get_standard_api()
-    api.env.mode = 'unit_test'
+    api = ipalib.get_standard_api(mode='unit_test')
     api.env.in_tree = True
     for (key, value) in kw.iteritems():
         api.env[key] = value