From b21bdb45133d109ff1b73b524ff3f0a29a3eaefe Mon Sep 17 00:00:00 2001 From: Izhar Firdaus Date: Thu, 25 Dec 2008 14:03:30 +0800 Subject: - added fastestmirror patch - added AxelGrabber URLGrabber --- customgrabber.py | 52 ++++++++++++++++++++++++ fastestmirror-repomd-download-speed-timing.patch | 33 +++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 customgrabber.py create mode 100644 fastestmirror-repomd-download-speed-timing.patch diff --git a/customgrabber.py b/customgrabber.py new file mode 100644 index 0000000..04647bf --- /dev/null +++ b/customgrabber.py @@ -0,0 +1,52 @@ +# file: /usr/lib/python2.5/site-packages/urlgrabber/customgrabber.py + +import grabber, sys, os +import subprocess +import urllib2 + +def get_filesize(url): + usock = urllib2.urlopen(url) + size = usock.info().get('Content-Length') + if size is None: + size = 0 + return float(size) + +class AxelGrabber(grabber.URLGrabber): + def urlgrab(self, url, filename=None, **kwargs): + """grab the file at and make a local copy at + If filename is none, the basename of the url is used. + urlgrab returns the filename of the local file, which may be + different from the passed-in filename if copy_local == 0. + """ + + opts = self.opts.derive(**kwargs) + (url,parts) = opts.urlparser.parse(url, opts) + (scheme, host, path, parm, query, frag) = parts + + + def retryfunc(opts, url, filename): + fsize = get_filesize(url) + if (fsize/1024) < 100: + parts = 1 + elif (fsize/1024) < 500: + parts = 2 + elif (fsize/1024/1024) < 1: + parts = 3 + elif (fsize/1024/1024) < 5: + parts = 4 + elif (fsize/1024/1024) < 10: + parts = 6 + elif (fsize/1024/1024) < 15: + parts = 8 + else: + parts = 10 + if os.path.exists(filename): + if not os.path.exists("%s.st" % filename): + os.unlink(filename) + p = subprocess.Popen(['/usr/bin/axel','-n','%s' % parts,'-a','-o',filename,url],stdout=sys.stdout,stderr=sys.stderr) + o = p.wait() + if o: + raise grabber.URLGrabError(-1) + return filename + + return self._retry(opts, retryfunc, url, filename) diff --git a/fastestmirror-repomd-download-speed-timing.patch b/fastestmirror-repomd-download-speed-timing.patch new file mode 100644 index 0000000..64f76e7 --- /dev/null +++ b/fastestmirror-repomd-download-speed-timing.patch @@ -0,0 +1,33 @@ +--- fastestmirror.py.old 2008-11-12 23:03:22.000000000 +0800 ++++ fastestmirror.py 2008-12-25 14:01:01.000000000 +0800 +@@ -45,6 +45,7 @@ + import urlparse + import datetime + import threading ++import urllib2 + + from yum.plugins import TYPE_CORE + +@@ -464,15 +465,14 @@ + if self.host in ("127.0.0.1", "::1", "localhost"): + result = 0 + else: +- sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +- uPort = string.find(self.host,":") +- if uPort > 0: +- self.port = int(self.host[uPort+1:]) +- self.host = self.host[:uPort] + time_before = time.time() +- sock.connect((self.host, self.port)) +- result = time.time() - time_before +- sock.close() ++ usock = urllib2.urlopen("%s/repodata/repomd.xml" % self.mirror) ++ fsize = len(usock.read()) ++ elapsed = time.time() - time_before ++ result = (elapsed*10000) / fsize ++ if 1: ++ print "Timed %s %.1fKBps" % (self.host,fsize/elapsed/1024.0) ++ + self.parent._add_result(self.mirror, self.host, result) + except: + if verbose: -- cgit