Add client capabilities, enable messages

The API version the client sends can now be used to check what the client
expects or is capable of.

All version tests IPA does will be be named and listed in one module,
ipalib.capabilities, which includes a function to test a specific capability
against an API version.
Similarly to Python's __future__ module, capabilities.py also serves as
documentation of backwards-incompatible changes to the API.

The first capability to be defined is "messages". Recent enough clients can
accept a list of warnings or other info under the "messages" key in the
result dict.

If a JSON client does not send the API version, it is assumed this is a testing
client (e.g. curl from the command line). Such a client "has" all capabilities,
but it will always receive a warning mentioning that forward compatibility
is not guaranteed.
If a XML client does not send the API version, it is assumed it uses the API
version before capabilities were introduced. (This is to keep backwards
compatibility with clients containing bug https://fedorahosted.org/freeipa/ticket/3294)

Whenever a capability is added, the API version must be incremented.
To ensure that, capabilities are written to API.txt and checked by
`makeapi --validate`.

Design page: http://freeipa.org/page/V3/Messages
Ticket: https://fedorahosted.org/freeipa/ticket/2732

13 files changed, 232 insertions, 16 deletions

diff --git a/API.txt b/API.txt
index a5eaaa834..c0800e959 100644
--- a/API.txt
+++ b/API.txt
@@ -3587,3 +3587,4 @@ option: Str('version?', exclude='webui')
 output: Output('result', <type 'bool'>, None)
 output: Output('summary', (<type 'unicode'>, <type 'NoneType'>), None)
 output: Output('value', <type 'unicode'>, None)
+capability: messages 2.52
diff --git a/VERSION b/VERSION
index 27fbc08d1..ed8cbfd76 100644
--- a/VERSION
+++ b/VERSION
@@ -89,4 +89,4 @@ IPA_DATA_VERSION=20100614120000
 #                                                      #
 ########################################################
 IPA_API_VERSION_MAJOR=2
-IPA_API_VERSION_MINOR=51
+IPA_API_VERSION_MINOR=52
diff --git a/ipalib/capabilities.py b/ipalib/capabilities.py
new file mode 100644
index 000000000..751b93e2b
--- /dev/null
+++ b/ipalib/capabilities.py
@@ -0,0 +1,50 @@
+# Authors:
+#   Petr Viktorin <pviktori@redhat.com>
+#
+# Copyright (C) 2012  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""List of, and utilities for working with, client capabilities by API version
+
+The API version is given in ipapython.version.API_VERSION.
+
+This module defines a dict, ``capabilities``, that maps feature names to API
+versions they were introduced in.
+"""
+
+from distutils import version
+
+VERSION_WITHOUT_CAPABILITIES = u'2.51'
+
+capabilities = dict(
+    # messages: Server output may include an extra key, "messages", that
+    # contains a list of warnings and other messages.
+    # http://freeipa.org/page/V3/Messages
+    messages=u'2.52',
+
+)
+
+
+def client_has_capability(client_version, capability):
+    """Determine whether the client has the given capability
+
+    :param capability: Name of the capability to test
+    :param client_version: The API version string reported by the client
+    """
+
+    version_tuple = version.LooseVersion(client_version)
+
+    return version_tuple >= version.LooseVersion(capabilities[capability])
diff --git a/ipalib/frontend.py b/ipalib/frontend.py
index c27ff1389..06259823c 100644
--- a/ipalib/frontend.py
+++ b/ipalib/frontend.py
@@ -23,18 +23,19 @@ Base classes for all front-end plugins.
 
 import re
 import inspect
+from distutils import version
+
+from ipapython.version import API_VERSION
+from ipapython.ipa_log_manager import root_logger
 from base import lock, check_name, NameSpace
 from plugable import Plugin, is_production_mode
 from parameters import create_param, parse_param_spec, Param, Str, Flag, Password
 from output import Output, Entry, ListOfEntries
 from text import _, ngettext
-
 from errors import (ZeroArgumentError, MaxArgumentError, OverlapError,
-    RequiresRoot, VersionError, RequirementError, OptionError)
-from errors import InvocationError
+    RequiresRoot, VersionError, RequirementError, OptionError, InvocationError)
 from constants import TYPE_ERROR
-from ipapython.version import API_VERSION
-from distutils import version
+from ipalib import messages
 
 
 RULE_FLAG = 'validation_rule'
@@ -740,11 +741,17 @@ class Command(HasParam):
         performs is executed remotely.
         """
         if self.api.env.in_server:
-            if 'version' in options:
+            version_provided = 'version' in options
+            if version_provided:
                 self.verify_client_version(options['version'])
             else:
                 options['version'] = API_VERSION
-            return self.execute(*args, **options)
+            result = self.execute(*args, **options)
+            if not version_provided:
+                messages.add_message(
+                    API_VERSION, result,
+                    messages.VersionMissing(server_version=API_VERSION))
+            return result
         return self.forward(*args, **options)
 
     def execute(self, *args, **kw):
@@ -914,7 +921,7 @@ class Command(HasParam):
                 nice, dict, type(output), output)
             )
         expected_set = set(self.output)
-        actual_set = set(output)
+        actual_set = set(output) - set(['messages'])
         if expected_set != actual_set:
             missing = expected_set - actual_set
             if missing:
@@ -945,6 +952,21 @@ class Command(HasParam):
                 continue
             yield param
 
+    def log_messages(self, output, logger):
+        logger_functions = dict(
+            debug=logger.debug,
+            info=logger.info,
+            warning=logger.warning,
+            error=logger.error,
+        )
+        for message in output.get('messages', ()):
+            try:
+                function = logger_functions[message['type']]
+            except KeyError:
+                logger.error('Server sent a message with a wrong type')
+                function = logger.error
+            function(message.get('message'))
+
     def output_for_cli(self, textui, output, *args, **options):
         """
         Generic output method. Prints values the output argument according
@@ -963,6 +985,8 @@ class Command(HasParam):
 
         rv = 0
 
+        self.log_messages(output, root_logger)
+
         order = [p.name for p in self.output_params()]
         if options.get('all', False):
             order.insert(0, 'dn')
diff --git a/ipalib/messages.py b/ipalib/messages.py
index 619e81d53..e5b76a526 100644
--- a/ipalib/messages.py
+++ b/ipalib/messages.py
@@ -35,6 +35,12 @@ from inspect import isclass
 from ipalib.constants import TYPE_ERROR
 from ipalib.text import _ as ugettext
 from ipalib.text import Gettext, NGettext
+from ipalib.capabilities import client_has_capability
+
+
+def add_message(version, result, message):
+    if client_has_capability(version, 'messages'):
+        result.setdefault('messages', []).append(message.to_dict())
 
 
 def process_message_arguments(obj, format=None, message=None, **kw):
diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py
index 581c30b4c..203825ea0 100644
--- a/ipaserver/rpcserver.py
+++ b/ipaserver/rpcserver.py
@@ -35,7 +35,7 @@ import urlparse
 import time
 import json
 
-from ipalib import plugable
+from ipalib import plugable, capabilities
 from ipalib.backend import Executioner
 from ipalib.errors import PublicError, InternalError, CommandError, JSONError, ConversionError, CCacheError, RefererError, InvalidSessionPassword, NotFound, ACIError, ExecutionError
 from ipalib.request import context, Connection, destroy_context
@@ -731,6 +731,11 @@ class xmlserver(WSGIExecutioner, HTTP_Status, KerberosSession):
     def unmarshal(self, data):
         (params, name) = xml_loads(data)
         (args, options) = params_2_args_options(params)
+        if 'version' not in options:
+            # Keep backwards compatibility with client containing
+            # bug https://fedorahosted.org/freeipa/ticket/3294:
+            # If `version` is not given in XML-RPC, assume an old version
+            options['version'] = capabilities.VERSION_WITHOUT_CAPABILITIES
         return (name, args, options, None)
 
     def marshal(self, result, error, _id=None):
diff --git a/makeapi b/makeapi
index 8981a97c2..86907bdbc 100755
--- a/makeapi
+++ b/makeapi
@@ -32,6 +32,7 @@ from ipalib import api
 from ipalib.parameters import Param
 from ipalib.output import Output
 from ipalib.text import Gettext, NGettext
+from ipalib.capabilities import capabilities
 
 API_FILE='API.txt'
 
@@ -211,6 +212,9 @@ def make_api():
             fd.write('option: %s\n' % param_repr(o))
         for o in sorted(cmd.output(), key=operator.attrgetter('name')):
             fd.write('output: %s\n' % param_repr(o))
+    for name, version in sorted(
+            capabilities.items(), key=lambda (k, v): (v, k)):
+        fd.write('capability: %s %s\n' % (name, version))
     fd.close()
 
     return 0
@@ -288,6 +292,7 @@ def validate_api():
 
     # First run through the file and compare it to the API
     existing_cmds = []
+    existing_capabilities = set()
     cmd = None
     for line in lines:
         line = line.strip()
@@ -370,6 +375,20 @@ def validate_api():
                 output = find_name(line)
                 print "Option '%s' in command '%s' in API file not found" % (output, name)
                 rval |= API_FILE_DIFFERENCE
+        if line.startswith('capability:'):
+            cap, version = line.replace('capability: ', '').split(' ', 1)
+            existing_capabilities.add(cap)
+            try:
+                expected_version = str(capabilities[cap])
+            except KeyError:
+                print "Capability '%s' in API file not found" % cap
+                rval |= API_FILE_DIFFERENCE
+            else:
+                if version != expected_version:
+                    print (
+                        "Capability '%s' in API file doesn't match. Got %s, "
+                        "expected %s.") % (cap, version, expected_version)
+                    rval |= API_FILE_DIFFERENCE
 
     if cmd:
         if not _finalize_command_validation(cmd, found_args, expected_args,
@@ -383,6 +402,11 @@ def validate_api():
             print "Command %s in ipalib, not in API" % cmd.name
             rval |= API_NEW_COMMAND
 
+    for cap in capabilities:
+        if cap not in existing_capabilities:
+            print "Capability %s in ipalib, not in API" % cap
+            rval |= API_FILE_DIFFERENCE
+
     return rval
 
 def main():
diff --git a/tests/test_ipalib/test_capabilities.py b/tests/test_ipalib/test_capabilities.py
new file mode 100644
index 000000000..21e53c2dc
--- /dev/null
+++ b/tests/test_ipalib/test_capabilities.py
@@ -0,0 +1,33 @@
+# Authors:
+#   Petr Viktorin <pviktori@redhat.com>
+#
+# Copyright (C) 2012  Red Hat
+# see file 'COPYING' for use and warranty information
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Test the `ipalib.errors` module.
+"""
+
+from ipalib.capabilities import capabilities, client_has_capability
+
+
+def test_client_has_capability():
+    assert capabilities['messages'] == u'2.52'
+    assert client_has_capability(u'2.52', 'messages')
+    assert client_has_capability(u'2.60', 'messages')
+    assert client_has_capability(u'3.0', 'messages')
+    assert not client_has_capability(u'2.11', 'messages')
+    assert not client_has_capability(u'0.1', 'messages')
diff --git a/tests/test_ipalib/test_frontend.py b/tests/test_ipalib/test_frontend.py
index 4b4735599..3a540608d 100644
--- a/tests/test_ipalib/test_frontend.py
+++ b/tests/test_ipalib/test_frontend.py
@@ -27,7 +27,7 @@ from tests.util import assert_equal
 from ipalib.constants import TYPE_ERROR
 from ipalib.base import NameSpace
 from ipalib import frontend, backend, plugable, errors, parameters, config
-from ipalib import output
+from ipalib import output, messages
 from ipalib.parameters import Str
 from ipapython.version import API_VERSION
 
@@ -619,6 +619,47 @@ class test_Command(ClassChecker):
         assert o.run.im_func is self.cls.run.im_func
         assert ('forward', args, kw) == o.run(*args, **kw)
 
+    def test_messages(self):
+        """
+        Test correct handling of messages
+        """
+        class TestMessage(messages.PublicMessage):
+            type = 'info'
+            format = 'This is a message.'
+            errno = 1234
+
+        class my_cmd(self.cls):
+            def execute(self, *args, **kw):
+                result = {'name': 'execute'}
+                messages.add_message(kw['version'], result, TestMessage())
+                return result
+
+            def forward(self, *args, **kw):
+                result = {'name': 'forward'}
+                messages.add_message(kw['version'], result, TestMessage())
+                return result
+
+        args = ('Hello,', 'world,')
+        kw = dict(how_are='you', on_this='fine day?', version=API_VERSION)
+
+        expected = [TestMessage().to_dict()]
+
+        # Test in server context:
+        (api, home) = create_test_api(in_server=True)
+        api.finalize()
+        o = my_cmd()
+        o.set_api(api)
+        assert o.run.im_func is self.cls.run.im_func
+        assert {'name': 'execute', 'messages': expected} == o.run(*args, **kw)
+
+        # Test in non-server context
+        (api, home) = create_test_api(in_server=False)
+        api.finalize()
+        o = my_cmd()
+        o.set_api(api)
+        assert o.run.im_func is self.cls.run.im_func
+        assert {'name': 'forward', 'messages': expected} == o.run(*args, **kw)
+
     def test_validate_output_basic(self):
         """
         Test the `ipalib.frontend.Command.validate_output` method.
diff --git a/tests/test_ipalib/test_messages.py b/tests/test_ipalib/test_messages.py
index d6a4b9aa1..ebc400ee2 100644
--- a/tests/test_ipalib/test_messages.py
+++ b/tests/test_ipalib/test_messages.py
@@ -22,6 +22,7 @@ Test the `ipalib.messages` module.
 """
 
 from ipalib import messages
+from ipalib.capabilities import capabilities
 from tests.test_ipalib import test_errors
 
 
@@ -58,3 +59,31 @@ def test_to_dict():
     )
 
     assert HelloMessage(greeting='Hello', object='world').to_dict() == expected
+
+
+def test_add_message():
+    result = {}
+
+    assert capabilities['messages'] == u'2.52'
+
+    messages.add_message(u'2.52', result,
+                         HelloMessage(greeting='Hello', object='world'))
+    messages.add_message(u'2.1', result,
+                         HelloMessage(greeting="'Lo", object='version'))
+    messages.add_message(u'2.60', result,
+                         HelloMessage(greeting='Hi', object='version'))
+
+    assert result == {'messages': [
+        dict(
+            name='HelloMessage',
+            type='info',
+            message='Hello, world!',
+            code=1234,
+        ),
+        dict(
+            name='HelloMessage',
+            type='info',
+            message='Hi, version!',
+            code=1234,
+        )
+    ]}
diff --git a/tests/test_xmlrpc/test_ping_plugin.py b/tests/test_xmlrpc/test_ping_plugin.py
index 284aed54f..3673b436f 100644
--- a/tests/test_xmlrpc/test_ping_plugin.py
+++ b/tests/test_xmlrpc/test_ping_plugin.py
@@ -21,9 +21,10 @@
 Test the `ipalib/plugins/ping.py` module, and XML-RPC in general.
 """
 
-from ipalib import api, errors, _
-from tests.util import assert_equal, Fuzzy
+from ipalib import api, errors, messages, _
+from tests.util import Fuzzy
 from xmlrpc_test import Declarative
+from ipapython.version import API_VERSION
 
 
 class test_ping(Declarative):
diff --git a/tests/test_xmlrpc/test_user_plugin.py b/tests/test_xmlrpc/test_user_plugin.py
index 50630a0f9..a61db23d5 100644
--- a/tests/test_xmlrpc/test_user_plugin.py
+++ b/tests/test_xmlrpc/test_user_plugin.py
@@ -23,11 +23,12 @@
 Test the `ipalib/plugins/user.py` module.
 """
 
-from ipalib import api, errors
+from ipalib import api, errors, messages
 from tests.test_xmlrpc import objectclasses
 from tests.util import assert_equal, assert_not_equal
 from xmlrpc_test import Declarative, fuzzy_digits, fuzzy_uuid, fuzzy_password, fuzzy_string, fuzzy_dergeneralizedtime
 from ipapython.dn import DN
+from ipapython.version import API_VERSION
 
 user1=u'tuser1'
 user2=u'tuser2'
diff --git a/tests/test_xmlrpc/xmlrpc_test.py b/tests/test_xmlrpc/xmlrpc_test.py
index 0a046b454..89a653088 100644
--- a/tests/test_xmlrpc/xmlrpc_test.py
+++ b/tests/test_xmlrpc/xmlrpc_test.py
@@ -25,9 +25,9 @@ import sys
 import socket
 import nose
 from tests.util import assert_deepequal, Fuzzy
-from ipalib import api, request
-from ipalib import errors
+from ipalib import api, request, errors
 from ipalib.x509 import valid_issuer
+from ipapython.version import API_VERSION
 
 
 # Matches a gidnumber like '1391016742'
@@ -271,6 +271,7 @@ class Declarative(XMLRPC_test):
 
     def check(self, nice, desc, command, expected, extra_check=None):
         (cmd, args, options) = command
+        options.setdefault('version', API_VERSION)
         if cmd not in api.Command:
             raise nose.SkipTest('%r not in api.Command' % cmd)
         if isinstance(expected, errors.PublicError):