py3: decode bytes for json.loads()

In py 3.5 json.loads requires to have string as input, all bytes must be
decoded.

Note: python 3.6 supports bytes for json.loads()

https://fedorahosted.org/freeipa/ticket/4985

Reviewed-By: Christian Heimes <cheimes@redhat.com>
Reviewed-By: Jan Cholasta <jcholast@redhat.com>

5 files changed, 59 insertions, 6 deletions

diff --git a/ipaclient/plugins/vault.py b/ipaclient/plugins/vault.py
index 29157c7e1..9efb1f109 100644
--- a/ipaclient/plugins/vault.py
+++ b/ipaclient/plugins/vault.py
@@ -969,7 +969,7 @@ class vault_retrieve(Local):
         json_vault_data = decoding_ctx.cipher_op(wrapped_vault_data)\
             + decoding_ctx.digest_final()
 
-        vault_data = json.loads(json_vault_data)
+        vault_data = json.loads(json_vault_data.decode('utf-8'))
         data = base64.b64decode(vault_data[u'data'].encode('utf-8'))
 
         encrypted_key = None
diff --git a/ipalib/rpc.py b/ipalib/rpc.py
index 921f5cb4b..fb739f8f4 100644
--- a/ipalib/rpc.py
+++ b/ipalib/rpc.py
@@ -1101,7 +1101,8 @@ class JSONServerProxy(object):
         )
 
         try:
-            response = json_decode_binary(json.loads(response.decode('ascii')))
+            response = json_decode_binary(
+                json.loads(response.decode('utf-8')))
         except ValueError as e:
             raise JSONError(error=str(e))
 
diff --git a/ipapython/dogtag.py b/ipapython/dogtag.py
index c6a8346d9..01fc5cbcf 100644
--- a/ipapython/dogtag.py
+++ b/ipapython/dogtag.py
@@ -209,6 +209,7 @@ def _httplib_request(
         http_body = res.read()
         conn.close()
     except Exception as e:
+        root_logger.exception("httplib request failed:")
         raise NetworkError(uri=uri, error=str(e))
 
     root_logger.debug('response status %d',    http_status)
diff --git a/ipapython/ipautil.py b/ipapython/ipautil.py
index f2b3d7419..c8f87ef5b 100644
--- a/ipapython/ipautil.py
+++ b/ipapython/ipautil.py
@@ -19,6 +19,7 @@
 
 from __future__ import print_function
 
+import codecs
 import string
 import tempfile
 import subprocess
@@ -1361,6 +1362,55 @@ def escape_seq(seq, *args):
     return tuple(a.replace(seq, u'\\{}'.format(seq)) for a in args)
 
 
+def decode_json(data):
+    """Decode JSON bytes to string with proper encoding
+
+    Only for supporting Py 3.5
+
+    Py 3.6 supports bytes as parameter for json.load, we can drop this when
+    there is no need for python 3.5 anymore
+
+    Code from:
+        https://bugs.python.org/file43513/json_detect_encoding_3.patch
+
+    :param data: JSON bytes
+    :return: return JSON string
+    """
+
+    def detect_encoding(b):
+        bstartswith = b.startswith
+        if bstartswith((codecs.BOM_UTF32_BE, codecs.BOM_UTF32_LE)):
+            return 'utf-32'
+        if bstartswith((codecs.BOM_UTF16_BE, codecs.BOM_UTF16_LE)):
+            return 'utf-16'
+        if bstartswith(codecs.BOM_UTF8):
+            return 'utf-8-sig'
+
+        if len(b) >= 4:
+            if not b[0]:
+                # 00 00 -- -- - utf-32-be
+                # 00 XX -- -- - utf-16-be
+                return 'utf-16-be' if b[1] else 'utf-32-be'
+            if not b[1]:
+                # XX 00 00 00 - utf-32-le
+                # XX 00 XX XX - utf-16-le
+                return 'utf-16-le' if b[2] or b[3] else 'utf-32-le'
+        elif len(b) == 2:
+            if not b[0]:
+                # 00 XX - utf-16-be
+                return 'utf-16-be'
+            if not b[1]:
+                # XX 00 - utf-16-le
+                return 'utf-16-le'
+        # default
+        return 'utf-8'
+
+    if isinstance(data, six.text_type):
+        return data
+
+    return data.decode(detect_encoding(data), 'surrogatepass')
+
+
 class APIVersion(tuple):
     """API version parser and handler
 
diff --git a/ipaserver/plugins/dogtag.py b/ipaserver/plugins/dogtag.py
index 142f838e1..fbfe60840 100644
--- a/ipaserver/plugins/dogtag.py
+++ b/ipaserver/plugins/dogtag.py
@@ -1232,7 +1232,8 @@ class RestClient(Backend):
     @staticmethod
     def _parse_dogtag_error(body):
         try:
-            return pki.PKIException.from_json(json.loads(body))
+            return pki.PKIException.from_json(
+                json.loads(ipautil.decode_json(body)))
         except Exception:
             return None
 
@@ -1667,7 +1668,7 @@ class ra(rabase.rabase, RestClient):
         )
 
         try:
-            resp_obj = json.loads(http_body)
+            resp_obj = json.loads(ipautil.decode_json(http_body))
         except ValueError:
             raise errors.RemoteRetrieveError(reason=_("Response from CA was not valid JSON"))
 
@@ -2115,7 +2116,7 @@ class ra_lightweight_ca(RestClient):
             body=json.dumps({"parentID": "host-authority", "dn": unicode(dn)}),
         )
         try:
-            return json.loads(resp_body)
+            return json.loads(ipautil.decode_json(resp_body))
         except Exception as e:
             self.log.debug(e, exc_info=True)
             raise errors.RemoteRetrieveError(
@@ -2125,7 +2126,7 @@ class ra_lightweight_ca(RestClient):
         _status, _resp_headers, resp_body = self._ssldo(
             'GET', ca_id, headers={'Accept': 'application/json'})
         try:
-            return json.loads(resp_body)
+            return json.loads(ipautil.decode_json(resp_body))
         except Exception as e:
             self.log.debug(e, exc_info=True)
             raise errors.RemoteRetrieveError(