From 1e836d2d0c8916f5b8a352cc8395048f1147554d Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 19 Dec 2012 04:25:24 -0500 Subject: Switch client to JSON-RPC Modify ipalib.rpc to support JSON-RPC in addition to XML-RPC. This is done by subclassing and extending xmlrpclib, because our existing code relies on xmlrpclib internals. The URI to use is given in the new jsonrpc_uri env variable. When it is not given, it is generated from xmlrpc_uri by replacing /xml with /json. The rpc_json_uri env variable existed before, but was unused, undocumented and not set the install scripts. This patch removes it in favor of jsonrpc_uri (for consistency with xmlrpc_uri). Add the rpc_protocol env variable to control the protocol IPA uses. rpc_protocol defaults to 'jsonrpc', but may be changed to 'xmlrpc'. Make backend.Executioner and tests use the backend specified by rpc_protocol. For compatibility with unwrap_xml, decoding JSON now gives tuples instead of lists. Design: http://freeipa.org/page/V3/JSON-RPC Ticket: https://fedorahosted.org/freeipa/ticket/3299 --- doc/examples/python-api.py | 4 +- ipa-client/ipa-install/ipa-client-automount | 2 +- ipa-client/ipa-install/ipa-client-install | 14 +- ipa-client/man/default.conf.5 | 8 +- ipalib/backend.py | 2 +- ipalib/config.py | 21 +- ipalib/constants.py | 7 +- ipalib/frontend.py | 6 +- ipalib/plugins/rpcclient.py | 50 +++++ ipalib/plugins/xmlclient.py | 29 --- ipalib/rpc.py | 269 ++++++++++++++++++++++---- ipaserver/advise/base.py | 4 +- ipaserver/rpcserver.py | 113 ++--------- ipatests/test_cmdline/test_cli.py | 4 +- ipatests/test_ipaserver/test_rpcserver.py | 4 +- ipatests/test_xmlrpc/test_dns_plugin.py | 4 +- ipatests/test_xmlrpc/test_external_members.py | 4 +- ipatests/test_xmlrpc/test_trust_plugin.py | 4 +- ipatests/test_xmlrpc/xmlrpc_test.py | 8 +- 19 files changed, 355 insertions(+), 202 deletions(-) create mode 100644 ipalib/plugins/rpcclient.py delete mode 100644 ipalib/plugins/xmlclient.py diff --git a/doc/examples/python-api.py b/doc/examples/python-api.py index 60578e80..129c56d9 100755 --- a/doc/examples/python-api.py +++ b/doc/examples/python-api.py @@ -34,14 +34,14 @@ api.bootstrap_with_global_options(context='example') api.finalize() # You will need to create a connection. If you're in_server, call -# Backend.ldap.connect(), otherwise Backend.xmlclient.connect(). +# Backend.ldap.connect(), otherwise Backend.rpcclient.connect(). if api.env.in_server: api.Backend.ldap2.connect( ccache=api.Backend.krb.default_ccname() ) else: - api.Backend.xmlclient.connect() + api.Backend.rpcclient.connect() # Now that you're connected, you can make calls to api.Command.whatever(): diff --git a/ipa-client/ipa-install/ipa-client-automount b/ipa-client/ipa-install/ipa-client-automount index 2ce31dcb..076bf081 100755 --- a/ipa-client/ipa-install/ipa-client-automount +++ b/ipa-client/ipa-install/ipa-client-automount @@ -436,7 +436,7 @@ def main(): sys.exit("Failed to obtain host TGT.") # Now we have a TGT, connect to IPA try: - api.Backend.xmlclient.connect() + api.Backend.rpcclient.connect() except errors.KerberosError, e: sys.exit('Cannot connect to the server due to ' + str(e)) try: diff --git a/ipa-client/ipa-install/ipa-client-install b/ipa-client/ipa-install/ipa-client-install index e79cb48b..c74e6840 100755 --- a/ipa-client/ipa-install/ipa-client-install +++ b/ipa-client/ipa-install/ipa-client-install @@ -1492,7 +1492,7 @@ def update_ssh_keys(server, hostname, ssh_dir, create_sshfp): try: # Use the RPC directly so older servers are supported - api.Backend.xmlclient.forward( + api.Backend.rpcclient.forward( 'host_mod', unicode(hostname), ipasshpubkey=[pk.openssh() for pk in pubkeys], @@ -2458,19 +2458,19 @@ def install(options, env, fstore, statestore): # Now, let's try to connect to the server's XML-RPC interface connected = False try: - api.Backend.xmlclient.connect() + api.Backend.rpcclient.connect() connected = True root_logger.debug('Try RPC connection') - api.Backend.xmlclient.forward('ping') + api.Backend.rpcclient.forward('ping') except errors.KerberosError, e: if connected: - api.Backend.xmlclient.disconnect() + api.Backend.rpcclient.disconnect() root_logger.info('Cannot connect to the server due to ' + 'Kerberos error: %s. Trying with delegate=True', str(e)) try: - api.Backend.xmlclient.connect(delegate=True) + api.Backend.rpcclient.connect(delegate=True) root_logger.debug('Try RPC connection') - api.Backend.xmlclient.forward('ping') + api.Backend.rpcclient.forward('ping') root_logger.info('Connection with delegate=True successful') @@ -2493,7 +2493,7 @@ def install(options, env, fstore, statestore): return CLIENT_INSTALL_ERROR # Use the RPC directly so older servers are supported - result = api.Backend.xmlclient.forward( + result = api.Backend.rpcclient.forward( 'env', server=True, version=u'2.0', diff --git a/ipa-client/man/default.conf.5 b/ipa-client/man/default.conf.5 index 9e87bb7c..5d5a48db 100644 --- a/ipa-client/man/default.conf.5 +++ b/ipa-client/man/default.conf.5 @@ -179,7 +179,13 @@ Used internally in the IPA source package to verify that the API has not changed When True provides more information. Specifically this sets the global log level to "info". .TP .B xmlrpc_uri -Specifies the URI of the XML\-RPC server for a client. This is used by IPA and some external tools as well, such as ipa\-getcert. e.g. https://ipa.example.com/ipa/xml +Specifies the URI of the XML\-RPC server for a client. This may be used by IPA, and is used by some external tools, such as ipa\-getcert. Example: https://ipa.example.com/ipa/xml +.TP +.B jsonrpc_uri +Specifies the URI of the JSON server for a client. This is used by IPA. If not given, it is derived from xmlrpc_uri. Example: https://ipa.example.com/ipa/json +.TP +.B rpc_protocol +Specifies the type of RPC calls IPA makes: 'jsonrpc' or 'xmlrpc'. Defaults to 'jsonrpc'. .TP The following define the containers for the IPA server. Containers define where in the DIT that objects can be found. The full location is the value of container + basedn. container_accounts: cn=accounts diff --git a/ipalib/backend.py b/ipalib/backend.py index 7be38ecc..b9426423 100644 --- a/ipalib/backend.py +++ b/ipalib/backend.py @@ -113,7 +113,7 @@ class Executioner(Backend): if self.env.in_server: self.Backend.ldap2.connect(ccache=ccache) else: - self.Backend.xmlclient.connect(verbose=(self.env.verbose >= 2), + self.Backend.rpcclient.connect(verbose=(self.env.verbose >= 2), fallback=self.env.fallback, delegate=self.env.delegate) if client_ip is not None: setattr(context, "client_ip", client_ip) diff --git a/ipalib/config.py b/ipalib/config.py index 3c9aeaa2..f86c0a5e 100644 --- a/ipalib/config.py +++ b/ipalib/config.py @@ -29,17 +29,17 @@ of the process. For the per-request thread-local information, see `ipalib.request`. """ +import urlparse from ConfigParser import RawConfigParser, ParsingError from types import NoneType import os from os import path import sys -from socket import getfqdn from ipapython.dn import DN from base import check_name from constants import CONFIG_SECTION -from constants import TYPE_ERROR, OVERRIDE_ERROR, SET_ERROR, DEL_ERROR +from constants import OVERRIDE_ERROR, SET_ERROR, DEL_ERROR class Env(object): @@ -514,8 +514,8 @@ class Env(object): ``self.conf_default`` (if it exists) by calling `Env._merge_from_file()`. - 4. Intelligently fill-in the *in_server* , *logdir*, and *log* - variables if they haven't already been set. + 4. Intelligently fill-in the *in_server* , *logdir*, *log*, and + *jsonrpc_uri* variables if they haven't already been set. 5. Merge-in the variables in ``defaults`` by calling `Env._merge()`. In normal circumstances ``defaults`` will simply be those @@ -556,6 +556,19 @@ class Env(object): if 'log' not in self: self.log = self._join('logdir', '%s.log' % self.context) + # Derive jsonrpc_uri from xmlrpc_uri + if 'jsonrpc_uri' not in self: + if 'xmlrpc_uri' in self: + xmlrpc_uri = self.xmlrpc_uri + else: + xmlrpc_uri = defaults.get('xmlrpc_uri') + if xmlrpc_uri: + (scheme, netloc, uripath, params, query, fragment + ) = urlparse.urlparse(xmlrpc_uri) + uripath = uripath.replace('/xml', '/json', 1) + self.jsonrpc_uri = urlparse.urlunparse(( + scheme, netloc, uripath, params, query, fragment)) + self._merge(**defaults) def _finalize(self, **lastchance): diff --git a/ipalib/constants.py b/ipalib/constants.py index 79885a33..5f9f03a4 100644 --- a/ipalib/constants.py +++ b/ipalib/constants.py @@ -111,10 +111,12 @@ DEFAULT_CONFIG = ( ('container_otp', DN(('cn', 'otp'))), # Ports, hosts, and URIs: - # FIXME: let's renamed xmlrpc_uri to rpc_xml_uri ('xmlrpc_uri', 'http://localhost:8888/ipa/xml'), - ('rpc_json_uri', 'http://localhost:8888/ipa/json'), + # jsonrpc_uri is set in Env._finalize_core() ('ldap_uri', 'ldap://localhost:389'), + + ('rpc_protocol', 'jsonrpc'), + # Time to wait for a service to start, in seconds ('startup_timeout', 120), @@ -199,5 +201,6 @@ DEFAULT_CONFIG = ( ('in_server', object), # Whether or not running in-server (bool) ('logdir', object), # Directory containing log files ('log', object), # Path to context specific log file + ('jsonrpc_uri', object), # derived from xmlrpc_uri in Env._finalize_core() ) diff --git a/ipalib/frontend.py b/ipalib/frontend.py index f478ef09..4d0333e0 100644 --- a/ipalib/frontend.py +++ b/ipalib/frontend.py @@ -742,7 +742,7 @@ class Command(HasParam): actually work this command performs is executed locally. If running in a non-server context, `Command.forward` is called, - which forwards this call over XML-RPC to the exact same command + which forwards this call over RPC to the exact same command on the nearest IPA server and the actual work this command performs is executed remotely. """ @@ -777,9 +777,9 @@ class Command(HasParam): def forward(self, *args, **kw): """ - Forward call over XML-RPC to this same command on server. + Forward call over RPC to this same command on server. """ - return self.Backend.xmlclient.forward(self.name, *args, **kw) + return self.Backend.rpcclient.forward(self.name, *args, **kw) def _on_finalize(self): """ diff --git a/ipalib/plugins/rpcclient.py b/ipalib/plugins/rpcclient.py new file mode 100644 index 00000000..6010b8dd --- /dev/null +++ b/ipalib/plugins/rpcclient.py @@ -0,0 +1,50 @@ +# Authors: +# Jason Gerard DeRose +# Rob Crittenden +# Petr Viktorin +# +# Copyright (C) 2008-2013 Red Hat +# see file 'COPYING' for use and warranty information +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +RPC client plugins. +""" + +from ipalib import api + +if 'in_server' in api.env and api.env.in_server is False: + from ipalib.rpc import xmlclient, jsonclient + api.register(xmlclient) + api.register(jsonclient) + + # FIXME: api.register only looks at the class name, so we need to create + # trivial subclasses with the desired name. + if api.env.rpc_protocol == 'xmlrpc': + + class rpcclient(xmlclient): + """xmlclient renamed to 'rpcclient'""" + pass + api.register(rpcclient) + + elif api.env.rpc_protocol == 'jsonrpc': + + class rpcclient(jsonclient): + """jsonclient renamed to 'rpcclient'""" + pass + api.register(rpcclient) + + else: + raise ValueError('unknown rpc_protocol: %s' % api.env.rpc_protocol) diff --git a/ipalib/plugins/xmlclient.py b/ipalib/plugins/xmlclient.py deleted file mode 100644 index 21ba47e2..00000000 --- a/ipalib/plugins/xmlclient.py +++ /dev/null @@ -1,29 +0,0 @@ -# Authors: -# Jason Gerard DeRose -# Rob Crittenden -# -# Copyright (C) 2008 Red Hat -# see file 'COPYING' for use and warranty information -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -""" -XML-RPC client plugin. -""" - -from ipalib import api - -if 'in_server' in api.env and api.env.in_server is False: - from ipalib.rpc import xmlclient - api.register(xmlclient) diff --git a/ipalib/rpc.py b/ipalib/rpc.py index 6d86f224..3d6cc3f2 100644 --- a/ipalib/rpc.py +++ b/ipalib/rpc.py @@ -32,20 +32,25 @@ Also see the `ipaserver.rpcserver` module. from types import NoneType from decimal import Decimal -import threading import sys import os -import errno import locale -import datetime +import base64 +import urllib +import json +import socket +from urllib2 import urlparse + from xmlrpclib import (Binary, Fault, dumps, loads, ServerProxy, Transport, ProtocolError, MININT, MAXINT) import kerberos from dns import resolver, rdatatype from dns.exception import DNSException +from nss.error import NSPRError from ipalib.backend import Connectible -from ipalib.errors import public_errors, PublicError, UnknownError, NetworkError, KerberosError, XMLRPCMarshallError +from ipalib.errors import (public_errors, UnknownError, NetworkError, + KerberosError, XMLRPCMarshallError, JSONError, ConversionError) from ipalib import errors from ipalib.request import context, Connection from ipalib.util import get_current_principal @@ -54,12 +59,7 @@ from ipapython import ipautil from ipapython import kernel_keyring from ipapython.cookie import Cookie from ipalib.text import _ - -import httplib -import socket from ipapython.nsslib import NSSHTTPS, NSSConnection -from nss.error import NSPRError -from urllib2 import urlparse from ipalib.krb_utils import KRB5KDC_ERR_S_PRINCIPAL_UNKNOWN, KRB5KRB_AP_ERR_TKT_EXPIRED, \ KRB5_FCC_PERM, KRB5_FCC_NOFILE, KRB5_CC_FORMAT, KRB5_REALM_CANT_RESOLVE from ipapython.dn import DN @@ -67,6 +67,8 @@ from ipapython.dn import DN COOKIE_NAME = 'ipa_session' KEYRING_COOKIE_NAME = '%s_cookie:%%s' % COOKIE_NAME +errors_by_code = dict((e.errno, e) for e in public_errors) + def client_session_keyring_keyname(principal): ''' @@ -228,6 +230,84 @@ def xml_dumps(params, methodname=None, methodresponse=False, encoding='UTF-8'): ) +def json_encode_binary(val): + ''' + JSON cannot encode binary values. We encode binary values in Python str + objects and text in Python unicode objects. In order to allow a binary + object to be passed through JSON we base64 encode it thus converting it to + text which JSON can transport. To assure we recognize the value is a base64 + encoded representation of the original binary value and not confuse it with + other text we convert the binary value to a dict in this form: + + {'__base64__' : base64_encoding_of_binary_value} + + This modification of the original input value cannot be done "in place" as + one might first assume (e.g. replacing any binary items in a container + (e.g. list, tuple, dict) with the base64 dict because the container might be + an immutable object (i.e. a tuple). Therefore this function returns a copy + of any container objects it encounters with tuples replaced by lists. This + is O.K. because the JSON encoding will map both lists and tuples to JSON + arrays. + ''' + + if isinstance(val, dict): + new_dict = {} + for k, v in val.items(): + new_dict[k] = json_encode_binary(v) + return new_dict + elif isinstance(val, (list, tuple)): + new_list = [json_encode_binary(v) for v in val] + return new_list + elif isinstance(val, str): + return {'__base64__': base64.b64encode(val)} + elif isinstance(val, Decimal): + return {'__base64__': base64.b64encode(str(val))} + elif isinstance(val, DN): + return str(val) + else: + return val + + +def json_decode_binary(val): + ''' + JSON cannot transport binary data. In order to transport binary data we + convert binary data to a form like this: + + {'__base64__' : base64_encoding_of_binary_value} + + see json_encode_binary() + + After JSON had decoded the JSON stream back into a Python object we must + recursively scan the object looking for any dicts which might represent + binary values and replace the dict containing the base64 encoding of the + binary value with the decoded binary value. Unlike the encoding problem + where the input might consist of immutable object, all JSON decoded + container are mutable so the conversion could be done in place. However we + don't modify objects in place because of side effects which may be + dangerous. Thus we elect to spend a few more cycles and avoid the + possibility of unintended side effects in favor of robustness. + ''' + + if isinstance(val, dict): + if '__base64__' in val: + return base64.b64decode(val['__base64__']) + else: + return dict((k, json_decode_binary(v)) for k, v in val.items()) + elif isinstance(val, list): + return tuple(json_decode_binary(v) for v in val) + else: + if isinstance(val, basestring): + try: + return val.decode('utf-8') + except UnicodeDecodeError: + raise ConversionError( + name=val, + error='incorrect type' + ) + else: + return val + + def decode_fault(e, encoding='UTF-8'): assert isinstance(e, Fault) if type(e.faultString) is str: @@ -265,10 +345,48 @@ def xml_loads(data, encoding='UTF-8'): raise decode_fault(e) -class LanguageAwareTransport(Transport): +class DummyParser(object): + def __init__(self): + self.data = '' + + def feed(self, data): + self.data += data + + def close(self): + return self.data + + +class MultiProtocolTransport(Transport): + """Transport that handles both XML-RPC and JSON""" + def __init__(self, protocol): + Transport.__init__(self) + self.protocol = protocol + + def getparser(self): + if self.protocol == 'json': + parser = DummyParser() + return parser, parser + else: + return Transport.getparser(self) + + def send_content(self, connection, request_body): + if self.protocol == 'json': + connection.putheader("Content-Type", "application/json") + else: + connection.putheader("Content-Type", "text/xml") + + # gzip compression would be set up here, but we have it turned off + # (encode_threshold is None) + + connection.putheader("Content-Length", str(len(request_body))) + connection.endheaders(request_body) + + +class LanguageAwareTransport(MultiProtocolTransport): """Transport sending Accept-Language header""" def get_host_info(self, host): - (host, extra_headers, x509) = Transport.get_host_info(self, host) + host, extra_headers, x509 = MultiProtocolTransport.get_host_info( + self, host) try: lang = locale.setlocale(locale.LC_ALL, '').split('.')[0].lower() @@ -468,23 +586,27 @@ class DelegatedKerbTransport(KerbTransport): flags = kerberos.GSS_C_DELEG_FLAG | kerberos.GSS_C_MUTUAL_FLAG | \ kerberos.GSS_C_SEQUENCE_FLAG -class xmlclient(Connectible): + +class RPCClient(Connectible): """ Forwarding backend plugin for XML-RPC client. Also see the `ipaserver.rpcserver.xmlserver` plugin. """ - def __init__(self): - super(xmlclient, self).__init__() - self.__errors = dict((e.errno, e) for e in public_errors) + # Values to set on subclasses: + session_path = None + server_proxy_class = ServerProxy + protocol = None + env_rpc_uri_key = None - def get_url_list(self, xmlrpc_uri): + def get_url_list(self, rpc_uri): """ Create a list of urls consisting of the available IPA servers. """ # the configured URL defines what we use for the discovered servers - (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(xmlrpc_uri) + (scheme, netloc, path, params, query, fragment + ) = urlparse.urlparse(rpc_uri) servers = [] name = '_ldap._tcp.%s.' % self.env.domain @@ -500,7 +622,7 @@ class xmlclient(Connectible): servers = list(set(servers)) # the list/set conversion won't preserve order so stick in the # local config file version here. - cfg_server = xmlrpc_uri + cfg_server = rpc_uri if cfg_server in servers: # make sure the configured master server is there just once and # it is the first one @@ -593,7 +715,7 @@ class xmlclient(Connectible): # Form the session URL by substituting the session path into the original URL scheme, netloc, path, params, query, fragment = urlparse.urlparse(original_url) - path = '/ipa/session/xml' + path = self.session_path session_url = urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) return session_url @@ -601,31 +723,32 @@ class xmlclient(Connectible): def create_connection(self, ccache=None, verbose=False, fallback=True, delegate=False): try: - xmlrpc_uri = self.env.xmlrpc_uri + rpc_uri = self.env[self.env_rpc_uri_key] principal = get_current_principal() setattr(context, 'principal', principal) # We have a session cookie, try using the session URI to see if it # is still valid if not delegate: - xmlrpc_uri = self.apply_session_cookie(xmlrpc_uri) + rpc_uri = self.apply_session_cookie(rpc_uri) except ValueError: # No session key, do full Kerberos auth pass - urls = self.get_url_list(xmlrpc_uri) + urls = self.get_url_list(rpc_uri) serverproxy = None for url in urls: kw = dict(allow_none=True, encoding='UTF-8') kw['verbose'] = verbose if url.startswith('https://'): if delegate: - kw['transport'] = DelegatedKerbTransport() + transport_class = DelegatedKerbTransport else: - kw['transport'] = KerbTransport() + transport_class = KerbTransport else: - kw['transport'] = LanguageAwareTransport() + transport_class = LanguageAwareTransport + kw['transport'] = transport_class(protocol=self.protocol) self.log.debug('trying %s' % url) setattr(context, 'request_url', url) - serverproxy = ServerProxy(url, **kw) + serverproxy = self.server_proxy_class(url, **kw) if len(urls) == 1: # if we have only 1 server and then let the # main requester handle any errors. This also means it @@ -634,11 +757,11 @@ class xmlclient(Connectible): try: command = getattr(serverproxy, 'ping') try: - response = command() + response = command([], {}) except Fault, e: e = decode_fault(e) - if e.faultCode in self.__errors: - error = self.__errors[e.faultCode] + if e.faultCode in errors_by_code: + error = errors_by_code[e.faultCode] raise error(message=e.faultString) else: raise UnknownError( @@ -683,6 +806,12 @@ class xmlclient(Connectible): conn = conn.conn._ServerProxy__transport conn.close() + def _call_command(self, command, params): + """Call the command with given params""" + # For XML, this method will wrap/unwrap binary values + # For JSON we do that in the proxy + return command(*params) + def forward(self, name, *args, **kw): """ Forward call to command named ``name`` over XML-RPC. @@ -699,18 +828,18 @@ class xmlclient(Connectible): '%s.forward(): %r not in api.Command' % (self.name, name) ) server = getattr(context, 'request_url', None) - self.debug("Forwarding '%s' to server '%s'", name, server) + self.debug("Forwarding '%s' to %s server '%s'", + name, self.protocol, server) command = getattr(self.conn, name) params = [args, kw] try: - response = command(*xml_wrap(params)) - return xml_unwrap(response) + return self._call_command(command, params) except Fault, e: e = decode_fault(e) self.debug('Caught fault %d from server %s: %s', e.faultCode, server, e.faultString) - if e.faultCode in self.__errors: - error = self.__errors[e.faultCode] + if e.faultCode in errors_by_code: + error = errors_by_code[e.faultCode] raise error(message=e.faultString) raise UnknownError( code=e.faultCode, @@ -756,3 +885,75 @@ class xmlclient(Connectible): raise NetworkError(uri=server, error=str(e)) except (OverflowError, TypeError), e: raise XMLRPCMarshallError(error=str(e)) + + +class xmlclient(RPCClient): + session_path = '/ipa/session/xml' + server_proxy_class = ServerProxy + protocol = 'xml' + env_rpc_uri_key = 'xmlrpc_uri' + + def _call_command(self, command, params): + params = xml_wrap(params) + result = command(*params) + return xml_unwrap(result) + + +class JSONServerProxy(object): + def __init__(self, uri, transport, encoding, verbose, allow_none): + type, uri = urllib.splittype(uri) + if type not in ("http", "https"): + raise IOError("unsupported XML-RPC protocol") + self.__host, self.__handler = urllib.splithost(uri) + self.__transport = transport + + assert encoding == 'UTF-8' + assert allow_none + self.__verbose = verbose + + # FIXME: Some of our code requires ServerProxy internals. + # But, xmlrpclib.ServerProxy's _ServerProxy__transport can be accessed + # by calling serverproxy('transport') + self._ServerProxy__transport = transport + + def __request(self, name, args): + payload = {'method': unicode(name), 'params': args, 'id': 0} + + response = self.__transport.request( + self.__host, + self.__handler, + json.dumps(json_encode_binary(payload)), + verbose=self.__verbose, + ) + + try: + response = json_decode_binary(json.loads(response)) + except ValueError, e: + raise JSONError(str(e)) + + error = response.get('error') + if error: + try: + error_class = errors_by_code[error['code']] + except KeyError: + raise UnknownError( + code=error.get('code'), + error=error.get('message'), + server=self.__host, + ) + else: + raise error_class(message=error['message']) + + return response['result'] + + def __getattr__(self, name): + def _call(*args): + return self.__request(name, args) + return _call + + +class jsonclient(RPCClient): + session_path = '/ipa/session/json' + server_proxy_class = JSONServerProxy + protocol = 'json' + env_rpc_uri_key = 'jsonrpc_uri' diff --git a/ipaserver/advise/base.py b/ipaserver/advise/base.py index 92dbb4e9..abaf708d 100644 --- a/ipaserver/advise/base.py +++ b/ipaserver/advise/base.py @@ -162,9 +162,9 @@ class IpaAdvise(admintool.AdminTool): advice.set_options(self.options) # Print out the actual advice - api.Backend.xmlclient.connect() + api.Backend.rpcclient.connect() advice.get_info() - api.Backend.xmlclient.disconnect() + api.Backend.rpcclient.disconnect() for line in advice.log.content: print line diff --git a/ipaserver/rpcserver.py b/ipaserver/rpcserver.py index 0ec7b02d..49643059 100644 --- a/ipaserver/rpcserver.py +++ b/ipaserver/rpcserver.py @@ -25,27 +25,29 @@ Also see the `ipalib.rpc` module. from xml.sax.saxutils import escape from xmlrpclib import Fault -from wsgiref.util import shift_path_info -import base64 import os -import string import datetime -from decimal import Decimal import urlparse import time import json from ipalib import plugable, capabilities from ipalib.backend import Executioner -from ipalib.errors import PublicError, InternalError, CommandError, JSONError, ConversionError, CCacheError, RefererError, InvalidSessionPassword, NotFound, ACIError, ExecutionError -from ipalib.request import context, Connection, destroy_context -from ipalib.rpc import xml_dumps, xml_loads +from ipalib.errors import (PublicError, InternalError, CommandError, JSONError, + CCacheError, RefererError, InvalidSessionPassword, NotFound, ACIError, + ExecutionError) +from ipalib.request import context, destroy_context +from ipalib.rpc import (xml_dumps, xml_loads, + json_encode_binary, json_decode_binary) from ipalib.util import parse_time_duration, normalize_name from ipapython.dn import DN from ipaserver.plugins.ldap2 import ldap2 -from ipalib.session import session_mgr, AuthManager, get_ipa_ccache_name, load_ccache_data, bind_ipa_ccache, release_ipa_ccache, fmt_time, default_max_session_duration +from ipalib.session import (session_mgr, AuthManager, get_ipa_ccache_name, + load_ccache_data, bind_ipa_ccache, release_ipa_ccache, fmt_time, + default_max_session_duration) from ipalib.backend import Backend -from ipalib.krb_utils import krb5_parse_ccache, KRB5_CCache, krb_ticket_expiration_threshold, krb5_format_principal_name +from ipalib.krb_utils import ( + KRB5_CCache, krb_ticket_expiration_threshold, krb5_format_principal_name) from ipapython import ipautil from ipapython.version import VERSION from ipalib.text import _ @@ -397,99 +399,6 @@ class WSGIExecutioner(Executioner): raise NotImplementedError('%s.marshal()' % self.fullname) -def json_encode_binary(val): - ''' - JSON cannot encode binary values. We encode binary values in Python str - objects and text in Python unicode objects. In order to allow a binary - object to be passed through JSON we base64 encode it thus converting it to - text which JSON can transport. To assure we recognize the value is a base64 - encoded representation of the original binary value and not confuse it with - other text we convert the binary value to a dict in this form: - - {'__base64__' : base64_encoding_of_binary_value} - - This modification of the original input value cannot be done "in place" as - one might first assume (e.g. replacing any binary items in a container - (e.g. list, tuple, dict) with the base64 dict because the container might be - an immutable object (i.e. a tuple). Therefore this function returns a copy - of any container objects it encounters with tuples replaced by lists. This - is O.K. because the JSON encoding will map both lists and tuples to JSON - arrays. - ''' - - if isinstance(val, dict): - new_dict = {} - for k,v in val.items(): - new_dict[k] = json_encode_binary(v) - return new_dict - elif isinstance(val, (list, tuple)): - new_list = [json_encode_binary(v) for v in val] - return new_list - elif isinstance(val, str): - return {'__base64__' : base64.b64encode(val)} - elif isinstance(val, Decimal): - return {'__base64__' : base64.b64encode(str(val))} - elif isinstance(val, DN): - return str(val) - else: - return val - -def json_decode_binary(val): - ''' - JSON cannot transport binary data. In order to transport binary data we - convert binary data to a form like this: - - {'__base64__' : base64_encoding_of_binary_value} - - see json_encode_binary() - - After JSON had decoded the JSON stream back into a Python object we must - recursively scan the object looking for any dicts which might represent - binary values and replace the dict containing the base64 encoding of the - binary value with the decoded binary value. Unlike the encoding problem - where the input might consist of immutable object, all JSON decoded - container are mutable so the conversion could be done in place. However we - don't modify objects in place because of side effects which may be - dangerous. Thus we elect to spend a few more cycles and avoid the - possibility of unintended side effects in favor of robustness. - ''' - - if isinstance(val, dict): - if val.has_key('__base64__'): - return base64.b64decode(val['__base64__']) - else: - new_dict = {} - for k,v in val.items(): - if isinstance(v, dict) and v.has_key('__base64__'): - new_dict[k] = base64.b64decode(v['__base64__']) - else: - new_dict[k] = json_decode_binary(v) - return new_dict - elif isinstance(val, list): - new_list = [] - n = len(val) - i = 0 - while i < n: - v = val[i] - if isinstance(v, dict) and v.has_key('__base64__'): - binary_val = base64.b64decode(v['__base64__']) - new_list.append(binary_val) - else: - new_list.append(json_decode_binary(v)) - i += 1 - return new_list - else: - if isinstance(val, basestring): - try: - return val.decode('utf-8') - except UnicodeDecodeError: - raise ConversionError( - name=val, - error='incorrect type' - ) - else: - return val - class jsonserver(WSGIExecutioner, HTTP_Status): """ JSON RPC server. diff --git a/ipatests/test_cmdline/test_cli.py b/ipatests/test_cmdline/test_cli.py index 4c76e181..489d2ceb 100644 --- a/ipatests/test_cmdline/test_cli.py +++ b/ipatests/test_cmdline/test_cli.py @@ -25,8 +25,8 @@ class TestCLIParsing(object): def run_command(self, command_name, **kw): """Run a command on the server""" - if not api.Backend.xmlclient.isconnected(): - api.Backend.xmlclient.connect(fallback=False) + if not api.Backend.rpcclient.isconnected(): + api.Backend.rpcclient.connect(fallback=False) try: api.Command[command_name](**kw) except errors.NetworkError: diff --git a/ipatests/test_ipaserver/test_rpcserver.py b/ipatests/test_ipaserver/test_rpcserver.py index bd567384..08d773c3 100644 --- a/ipatests/test_ipaserver/test_rpcserver.py +++ b/ipatests/test_ipaserver/test_rpcserver.py @@ -241,7 +241,7 @@ class test_jsonserver(PluginTester): assert unicode(e.error) == 'params[1] (aka options) must be a dict' # Test with valid values: - args = [u'jdoe'] + args = (u'jdoe', ) options = dict(givenname=u'John', sn='Doe') - d = dict(method=u'user_add', params=[args, options], id=18) + d = dict(method=u'user_add', params=(args, options), id=18) assert o.unmarshal(json.dumps(d)) == (u'user_add', args, options, 18) diff --git a/ipatests/test_xmlrpc/test_dns_plugin.py b/ipatests/test_xmlrpc/test_dns_plugin.py index 1bfaee71..81e8e4ed 100644 --- a/ipatests/test_xmlrpc/test_dns_plugin.py +++ b/ipatests/test_xmlrpc/test_dns_plugin.py @@ -63,8 +63,8 @@ class test_dns(Declarative): def setUpClass(cls): super(test_dns, cls).setUpClass() - if not api.Backend.xmlclient.isconnected(): - api.Backend.xmlclient.connect(fallback=False) + if not api.Backend.rpcclient.isconnected(): + api.Backend.rpcclient.connect(fallback=False) try: api.Command['dnszone_add'](dnszone1, idnssoamname = dnszone1_mname, diff --git a/ipatests/test_xmlrpc/test_external_members.py b/ipatests/test_xmlrpc/test_external_members.py index 112470dc..a2128cb7 100644 --- a/ipatests/test_xmlrpc/test_external_members.py +++ b/ipatests/test_xmlrpc/test_external_members.py @@ -45,8 +45,8 @@ class test_external_members(Declarative): @classmethod def setUpClass(cls): super(test_external_members, cls).setUpClass() - if not api.Backend.xmlclient.isconnected(): - api.Backend.xmlclient.connect(fallback=False) + if not api.Backend.rpcclient.isconnected(): + api.Backend.rpcclient.connect(fallback=False) trusts = api.Command['trust_find']() if trusts['count'] == 0: diff --git a/ipatests/test_xmlrpc/test_trust_plugin.py b/ipatests/test_xmlrpc/test_trust_plugin.py index 0223e8b3..5260b406 100644 --- a/ipatests/test_xmlrpc/test_trust_plugin.py +++ b/ipatests/test_xmlrpc/test_trust_plugin.py @@ -41,8 +41,8 @@ class test_trustconfig(Declarative): @classmethod def setUpClass(cls): super(test_trustconfig, cls).setUpClass() - if not api.Backend.xmlclient.isconnected(): - api.Backend.xmlclient.connect(fallback=False) + if not api.Backend.rpcclient.isconnected(): + api.Backend.rpcclient.connect(fallback=False) try: api.Command['trustconfig_show'](trust_type=u'ad') except errors.NotFound: diff --git a/ipatests/test_xmlrpc/xmlrpc_test.py b/ipatests/test_xmlrpc/xmlrpc_test.py index 2d12bcb3..04d69475 100644 --- a/ipatests/test_xmlrpc/xmlrpc_test.py +++ b/ipatests/test_xmlrpc/xmlrpc_test.py @@ -86,8 +86,8 @@ def fuzzy_set_ci(s): return Fuzzy(test=lambda other: set(x.lower() for x in other) == set(y.lower() for y in s)) try: - if not api.Backend.xmlclient.isconnected(): - api.Backend.xmlclient.connect(fallback=False) + if not api.Backend.rpcclient.isconnected(): + api.Backend.rpcclient.connect(fallback=False) res = api.Command['user_show'](u'notfound') except errors.NetworkError: server_available = False @@ -163,8 +163,8 @@ class XMLRPC_test(object): (cls.__module__, api.env.xmlrpc_uri)) def setUp(self): - if not api.Backend.xmlclient.isconnected(): - api.Backend.xmlclient.connect(fallback=False) + if not api.Backend.rpcclient.isconnected(): + api.Backend.rpcclient.connect(fallback=False) def tearDown(self): """ -- cgit