This patch wraps binary data in an xmlrpclib Binary object. This

removes the need for LDIF conversion.  It will make TurboGears direct
code faster, but should keep xmlrpc about the same speed.

The patch also swaps out ldap.cidict for the IPA CIDict class.  IPA code
should only use the CIDict class now.

7 files changed, 131 insertions, 85 deletions

diff --git a/ipa-python/entity.py b/ipa-python/entity.py
index 9153c4159..2f0629cda 100644
--- a/ipa-python/entity.py
+++ b/ipa-python/entity.py
@@ -3,6 +3,8 @@ import ldif
 import re
 import cStringIO
 
+import ipa.ipautil
+
 def utf8_encode_value(value):
     if isinstance(value,unicode):
         return value.encode('utf-8')
@@ -22,8 +24,8 @@ class Entity:
     In python-ldap, entries are returned as a list of 2-tuples.
     Instance variables:
     dn - string - the string DN of the entry
-    data - cidict - case insensitive dict of the attributes and values
-    orig_data - cidict - case insentiive dict of the original attributes and values"""
+    data - CIDict - case insensitive dict of the attributes and values
+    orig_data - CIDict - case insentiive dict of the original attributes and values"""
 
     def __init__(self,entrydata=None):
         """data is the raw data returned from the python-ldap result method,
@@ -32,19 +34,19 @@ class Entity:
         if entrydata:
             if isinstance(entrydata,tuple):
                 self.dn = entrydata[0]
-                self.data = ldap.cidict.cidict(entrydata[1])
+                self.data = ipa.ipautil.CIDict(entrydata[1])
             elif isinstance(entrydata,str) or isinstance(entrydata,unicode):
                 self.dn = entrydata
-                self.data = ldap.cidict.cidict()
+                self.data = ipa.ipautil.CIDict()
             elif isinstance(entrydata,dict):
                 self.dn = entrydata['dn']
                 del entrydata['dn']
-                self.data = ldap.cidict.cidict(entrydata)
+                self.data = ipa.ipautil.CIDict(entrydata)
         else:
             self.dn = ''
-            self.data = ldap.cidict.cidict()
+            self.data = ipa.ipautil.CIDict()
 
-        self.orig_data = dict(self.data)
+        self.orig_data = ipa.ipautil.CIDict(self.data)
 
     def __nonzero__(self):
         """This allows us to do tests like if entry: returns false if there is no data,
@@ -112,9 +114,7 @@ class Entity:
     def toDict(self):
         """Convert the attrs and values to a dict. The dict is keyed on the
         attribute name.  The value is either single value or a list of values."""
-        result = {}
-        for k in self.data.keys():
-            result[k] = self.data[k]
+        result = ipa.ipautil.CIDict(self.data)
         result['dn'] = self.dn
         return result
 
@@ -124,9 +124,7 @@ class Entity:
 
     def origDataDict(self):
         """Returns a dict of the original values of the user.  Used for updates."""
-        result = {}
-        for k in self.orig_data.keys():
-            result[k] = self.orig_data[k]
+        result = ipa.ipautil.CIDict(self.orig_data)
         result['dn'] = self.dn
         return result
 
diff --git a/ipa-python/ipaclient.py b/ipa-python/ipaclient.py
index 28573acad..71def70fd 100644
--- a/ipa-python/ipaclient.py
+++ b/ipa-python/ipaclient.py
@@ -30,14 +30,6 @@ import group
 import ipa
 import config
 
-def cidict_to_dict(cid):
-    """Convert a cidict to a standard dict for sending across the wire"""
-    newdict = {}
-    kindex = cid.keys()
-    for dkey in kindex:
-        newdict[dkey] = cid[dkey]
-    return newdict
-
 class IPAClient:
 
     def __init__(self,local=None):
@@ -89,7 +81,7 @@ class IPAClient:
         result = self.transport.get_all_users()
 
         all_users = []
-        for (attrs) in result:
+        for attrs in result:
             if attrs is not None:
                 all_users.append(user.User(attrs))
 
@@ -107,7 +99,7 @@ class IPAClient:
         result = self.transport.find_users(criteria, sattrs)
 
         users = []
-        for (attrs) in result:
+        for attrs in result:
             if attrs is not None:
                 users.append(user.User(attrs))
 
@@ -165,7 +157,7 @@ class IPAClient:
         result = self.transport.find_groups(criteria, sattrs)
 
         groups = []
-        for (attrs) in result:
+        for attrs in result:
             if attrs is not None:
                 groups.append(group.Group(attrs))
 
diff --git a/ipa-python/ipautil.py b/ipa-python/ipautil.py
index 51337a8eb..74f7cfff6 100644
--- a/ipa-python/ipautil.py
+++ b/ipa-python/ipautil.py
@@ -18,6 +18,8 @@
 #
 
 from string import lower
+import re
+import xmlrpclib
 
 class CIDict(dict):
     """
@@ -106,3 +108,65 @@ class CIDict(dict):
         return (key,value)
 
 
+#
+# The safe_string_re regexp and needs_base64 function are extracted from the
+# python-ldap ldif module, which was
+# written by Michael Stroeder <michael@stroeder.com>
+# http://python-ldap.sourceforge.net
+#
+# It was extracted because ipaldap.py is naughtily reaching into the ldif
+# module and squashing this regexp.
+#
+SAFE_STRING_PATTERN = '(^(\000|\n|\r| |:|<)|[\000\n\r\200-\377]+|[ ]+$)'
+safe_string_re = re.compile(SAFE_STRING_PATTERN)
+
+def needs_base64(s):
+  """
+  returns 1 if s has to be base-64 encoded because of special chars
+  """
+  return not safe_string_re.search(s) is None
+
+
+def wrap_binary_data(data):
+    """Converts all binary data strings into Binary objects for transport
+       back over xmlrpc."""
+    if isinstance(data, str):
+        if needs_base64(data):
+            return xmlrpclib.Binary(data)
+        else:
+            return data
+    elif isinstance(data, list) or isinstance(data,tuple):
+        retval = []
+        for value in data:
+            retval.append(wrap_binary_data(value))
+        return retval
+    elif isinstance(data, dict):
+        retval = {}
+        for (k,v) in data.iteritems():
+            retval[k] = wrap_binary_data(v)
+        return retval
+    else:
+        return data
+
+
+def unwrap_binary_data(data):
+    """Converts all Binary objects back into strings."""
+    if isinstance(data, xmlrpclib.Binary):
+        # The data is decoded by the xmlproxy, but is stored
+        # in a binary object for us.
+        return str(data)
+    elif isinstance(data, str):
+        return data
+    elif isinstance(data, list) or isinstance(data,tuple):
+        retval = []
+        for value in data:
+            retval.append(unwrap_binary_data(value))
+        return retval
+    elif isinstance(data, dict):
+        retval = {}
+        for (k,v) in data.iteritems():
+            retval[k] = unwrap_binary_data(v)
+        return retval
+    else:
+        return data
+
diff --git a/ipa-python/rpcclient.py b/ipa-python/rpcclient.py
index 2301edde8..3e5bb113a 100644
--- a/ipa-python/rpcclient.py
+++ b/ipa-python/rpcclient.py
@@ -29,7 +29,7 @@ import os
 import base64
 import user
 import ipa
-from ipa import ipaerror
+from ipa import ipaerror, ipautil
 
 # Some errors to catch
 # http://cvs.fedora.redhat.com/viewcvs/ldapserver/ldap/servers/plugins/pam_passthru/README?root=dirsec&rev=1.6&view=auto
@@ -83,7 +83,7 @@ class RPCClient:
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
 
-        return result
+        return ipautil.unwrap_binary_data(result)
         
     def get_user_by_dn(self,dn,sattrs=None):
         """Get a specific user. If sattrs is not None then only those
@@ -99,7 +99,7 @@ class RPCClient:
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
 
-        return result
+        return ipautil.unwrap_binary_data(result)
 
     def add_user(self,user,user_container=None):
         """Add a new user. Takes as input a dict where the key is the
@@ -111,13 +111,14 @@ class RPCClient:
             user_container = "__NONE__"
     
         try:
-            result = server.add_user(user, user_container)
+            result = server.add_user(ipautil.wrap_binary_data(user),
+                    user_container)
         except xmlrpclib.Fault, fault:
             raise ipaerror.gen_exception(fault.faultCode, fault.faultString)
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
 
-        return result
+        return ipautil.unwrap_binary_data(result)
         
     def get_add_schema(self):
         """Get the list of attributes we need to ask when adding a new
@@ -134,7 +135,7 @@ class RPCClient:
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
       
-        return result
+        return ipautil.unwrap_binary_data(result)
     
     def get_all_users (self):
         """Return a list containing a User object for each existing user."""
@@ -147,7 +148,7 @@ class RPCClient:
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
     
-        return result
+        return ipautil.unwrap_binary_data(result)
 
     def find_users (self, criteria, sattrs=None):
         """Return a list containing a User object for each user that matches
@@ -164,20 +165,21 @@ class RPCClient:
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
     
-        return result
+        return ipautil.unwrap_binary_data(result)
 
     def update_user(self,olduser,newuser):
         """Update an existing user. olduser and newuser are dicts of attributes"""
         server = self.setup_server()
     
         try:
-            result = server.update_user(olduser, newuser)
+            result = server.update_user(ipautil.wrap_binary_data(olduser),
+                    ipautil.wrap_binary_data(newuser))
         except xmlrpclib.Fault, fault:
             raise ipaerror.gen_exception(fault.faultCode, fault.faultString)
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
 
-        return result
+        return ipautil.unwrap_binary_data(result)
 
     def mark_user_deleted(self,uid):
         """Mark a user as deleted/inactive"""
@@ -190,7 +192,7 @@ class RPCClient:
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
 
-        return result
+        return ipautil.unwrap_binary_data(result)
 
 # Group support
         
@@ -208,7 +210,7 @@ class RPCClient:
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
 
-        return result
+        return ipautil.unwrap_binary_data(result)
         
     def get_group_by_dn(self,dn,sattrs=None):
         """Get a specific group. If sattrs is not None then only those
@@ -224,7 +226,7 @@ class RPCClient:
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
 
-        return result
+        return ipautil.unwrap_binary_data(result)
 
     def add_group(self,group,group_container=None):
         """Add a new group. Takes as input a dict where the key is the
@@ -236,7 +238,8 @@ class RPCClient:
             group_container = "__NONE__"
     
         try:
-            result = server.add_group(group, group_container)
+            result = server.add_group(ipautil.wrap_binary_data(group),
+                    group_container)
         except xmlrpclib.Fault, fault:
             raise ipaerror.gen_exception(fault.faultCode, fault.faultString)
         except socket.error, (value, msg):
@@ -257,7 +260,7 @@ class RPCClient:
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
     
-        return result
+        return ipautil.unwrap_binary_data(result)
 
     def add_user_to_group(self, user, group):
         """Add a user to an existing group.
@@ -266,13 +269,14 @@ class RPCClient:
         """
         server = self.setup_server()
         try:
-            result = server.add_user_to_group(user, group)
+            result = server.add_user_to_group(ipautil.wrap_binary_data(user),
+                    ipautil.wrap_binary_data(group))
         except xmlrpclib.Fault, fault:
             raise ipaerror.gen_exception(fault.faultCode, fault.faultString)
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
     
-        return result
+        return ipautil.unwrap_binary_data(result)
 
     def add_users_to_group(self, users, group):
         """Add several users to an existing group.
@@ -283,13 +287,14 @@ class RPCClient:
         """
         server = self.setup_server()
         try:
-            result = server.add_users_to_group(users, group)
+            result = server.add_users_to_group(ipautil.wrap_binary_data(users),
+                    ipautil.wrap_binary_data(group))
         except xmlrpclib.Fault, fault:
             raise ipaerror.gen_exception(fault.faultCode, fault.faultString)
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
     
-        return result
+        return ipautil.unwrap_binary_data(result)
 
     def remove_user_from_group(self, user, group):
         """Remove a user from an existing group.
@@ -298,13 +303,14 @@ class RPCClient:
         """
         server = self.setup_server()
         try:
-            result = server.remove_user_from_group(user, group)
+            result = server.remove_user_from_group(ipautil.wrap_binary_data(user),
+                    ipautil.wrap_binary_data(group))
         except xmlrpclib.Fault, fault:
             raise ipaerror.gen_exception(fault.faultCode, fault.faultString)
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
     
-        return result
+        return ipautil.unwrap_binary_data(result)
 
     def remove_users_from_group(self, users, group):
         """Remove several users from an existing group.
@@ -315,23 +321,26 @@ class RPCClient:
         """
         server = self.setup_server()
         try:
-            result = server.remove_users_from_group(users, group)
+            result = server.remove_users_from_group(
+                    ipautil.wrap_binary_data(users),
+                    ipautil.wrap_binary_data(group))
         except xmlrpclib.Fault, fault:
             raise ipaerror.gen_exception(fault.faultCode, fault.faultString)
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
     
-        return result
+        return ipautil.unwrap_binary_data(result)
 
     def update_group(self,oldgroup,newgroup):
         """Update an existing group. oldgroup and newgroup are dicts of attributes"""
         server = self.setup_server()
     
         try:
-            result = server.update_group(oldgroup, newgroup)
+            result = server.update_group(ipautil.wrap_binary_data(oldgroup),
+                    ipautil.wrap_binary_data(newgroup))
         except xmlrpclib.Fault, fault:
             raise ipaerror.gen_exception(fault.faultCode, fault.faultString)
         except socket.error, (value, msg):
             raise xmlrpclib.Fault(value, msg)
 
-        return result
+        return ipautil.unwrap_binary_data(result)
diff --git a/ipa-server/ipaserver/ipaldap.py b/ipa-server/ipaserver/ipaldap.py
index 0b2a152e9..344e6dc3a 100644
--- a/ipa-server/ipaserver/ipaldap.py
+++ b/ipa-server/ipaserver/ipaldap.py
@@ -39,7 +39,7 @@ from ldap.modlist import modifyModlist
 
 from ldap.ldapobject import SimpleLDAPObject
 
-from ipa import ipaerror
+from ipa import ipaerror, ipautil
 
 class Entry:
     """This class represents an LDAP Entry object.  An LDAP entry consists of a DN
@@ -47,7 +47,7 @@ class Entry:
     values.  In python-ldap, entries are returned as a list of 2-tuples.
     Instance variables:
     dn - string - the string DN of the entry
-    data - cidict - case insensitive dict of the attributes and values"""
+    data - CIDict - case insensitive dict of the attributes and values"""
 
     def __init__(self,entrydata):
         """data is the raw data returned from the python-ldap result method, which is
@@ -56,13 +56,13 @@ class Entry:
         if entrydata:
             if isinstance(entrydata,tuple):
                 self.dn = entrydata[0]
-                self.data = ldap.cidict.cidict(entrydata[1])
+                self.data = ipautil.CIDict(entrydata[1])
             elif isinstance(entrydata,str) or isinstance(entrydata,unicode):
                 self.dn = entrydata
-                self.data = ldap.cidict.cidict()
+                self.data = ipautil.CIDict()
         else:
             self.dn = ''
-            self.data = ldap.cidict.cidict()
+            self.data = ipautil.CIDict()
 
     def __nonzero__(self):
         """This allows us to do tests like if entry: returns false if there is no data,
diff --git a/ipa-server/xmlrpc-server/funcs.py b/ipa-server/xmlrpc-server/funcs.py
index a00498333..fe48a1ffa 100644
--- a/ipa-server/xmlrpc-server/funcs.py
+++ b/ipa-server/xmlrpc-server/funcs.py
@@ -98,36 +98,19 @@ class IPAServer:
         return "dn:" + ent.dn
 
     def convert_entry(self, ent):
-    
-        # Convert to LDIF
-        entry = str(ent) 
+        entry = dict(ent.data)
+        entry['dn'] = ent.dn
+        # For now convert single entry lists to a string for the ui.
+        # TODO: we need to deal with multi-values better
+        for key,value in entry.iteritems():
+            if isinstance(value,list) or isinstance(value,tuple):
+                if len(value) == 0:
+                    entry[key] = ''
+                elif len(value) == 1:
+                    entry[key] = value[0]
+        return entry
 
-        # Strip off any junk
-        entry = entry.strip()
 
-        # Don't need to identify binary fields and this breaks the parser so
-        # remove double colons
-        entry = entry.replace('::', ':')
-        specs = [spec.split(':') for spec in entry.split('\n')]
-    
-        # Convert into a dict. We need to handle multi-valued attributes as well
-        # so we'll convert those into lists.
-        obj={}
-        for (k,v) in specs:
-            k = k.lower()
-            if obj.get(k) is not None:
-                if isinstance(obj[k],list):
-                    obj[k].append(v.strip())
-                else:
-                    first = obj[k]
-                    obj[k] = []
-                    obj[k].append(first)
-                    obj[k].append(v.strip())
-            else:
-                    obj[k] = v.strip()
-    
-        return obj 
-    
     def __get_entry (self, base, filter, sattrs=None, opts=None):
         """Get a specific entry. Return as a dict of values.
            Multi-valued fields are represented as lists.
diff --git a/ipa-server/xmlrpc-server/ipaxmlrpc.py b/ipa-server/xmlrpc-server/ipaxmlrpc.py
index 9ef1e1b0e..16ced2cda 100644
--- a/ipa-server/xmlrpc-server/ipaxmlrpc.py
+++ b/ipa-server/xmlrpc-server/ipaxmlrpc.py
@@ -35,7 +35,7 @@ from mod_python import apache
 
 import ipaserver
 import funcs
-from ipa import ipaerror
+from ipa import ipaerror, ipautil
 import ldap
 
 import string
@@ -173,14 +173,14 @@ class ModXMLRPCRequestHandler(object):
         if func is None:
              raise Fault(1, "Invalid method: %s" % method)
 
-        args = list(params)
+        args = list(ipautil.unwrap_binary_data(params))
         for i in range(len(args)):
           if args[i] == '__NONE__':
               args[i] = None 
 
         ret = func(*args)
 
-        return ret
+        return ipautil.wrap_binary_data(ret)
 
     def multiCall(self, calls):
         """Execute a multicall.  Execute each method call in the calls list, collecting