Use pycurl to get plain URLs, follow redirects

urlllib seems to urlunescape HTTP location headers to get the url to
follow. If this URL contains spaces, a bad request is created.
Python bug report:
http://bugs.python.org/issue6557

1 files changed, 22 insertions, 3 deletions

diff --git a/cnucnu/helper.py b/cnucnu/helper.py
index f2d61de..b2340b8 100644
--- a/cnucnu/helper.py
+++ b/cnucnu/helper.py
@@ -22,9 +22,24 @@ pp = pprint_module.PrettyPrinter(indent=4)
 pprint = pp.pprint
 
 def get_html(url):
-    import urllib
-    res = urllib.urlopen(url)
-    return res.read()
+    import pycurl
+    import StringIO
+
+    c = pycurl.Curl()
+    c.setopt(pycurl.URL, url.encode("ascii"))
+
+    res = StringIO.StringIO()
+
+    c.setopt(pycurl.WRITEFUNCTION, res.write)
+    c.setopt(pycurl.FOLLOWLOCATION, 1)
+    c.setopt(pycurl.MAXREDIRS, 10)
+
+    c.perform()
+    c.close()
+    data = res.getvalue()
+    res.close()
+
+    return data
 
 def rpm_cmp(v1, v2):
     import rpm
@@ -63,6 +78,10 @@ def secure_download(url, cainfo=""):
 
     c.setopt(pycurl.WRITEFUNCTION, res.write)
 
+    # follow up to 10 http location: headers
+    c.setopt(pycurl.FOLLOWLOCATION, 1)
+    c.setopt(pycurl.MAXREDIRS, 10)
+
     c.perform()
     c.close()
     data = res.getvalue()