1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
#! /usr/bin/python -tt
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
#
# Copyright Hedayat Vatankhah, 2012.
#
# Author: Hedayat Vatankhah <hedayat.fwd@gmail.com>
#
# * Used the code of original external downloader of urlgrabber:
# Copyright 2011-2012 Zdenek Pavlas
import time, errno, sys
import os
import shutil
import yum
import subprocess
from urlgrabber.grabber import _ExternalDownloader
from urlgrabber.grabber import \
_readlines, URLGrabberOptions, _loads, \
URLGrabError
# variables
global_cache_dir='/var/cache/yum'
downloader_common_args = []
# downloader functions
def _getAria2CArgs(urls, remote_path, local_path, proxies = None):
args = []
args += downloader_common_args
args.append("--dir={0}/yfd/".format(os.path.dirname(local_path)))
args.append("--split={0}".format(max(5, len(urls))))
if proxies:
for (scheme, proxy) in proxies.items():
if not proxy.startswith('ftp://'):
args.append("--{0}-proxy={1}".format(scheme, proxy))
for url in urls:
args.append(url + remote_path)
return args;
def downloadFile(urls, remote_path, local_path, proxies = None):
if not urls[0].startswith("file:"):
args = _getAria2CArgs(urls, remote_path, local_path)
popen = subprocess.Popen(args, stdout=2)
popen.wait()
i=0
while not os.path.exists(local_path):
time.sleep(0.1)
i+=1
if i > 50:
os.write(2, "Error in downloading file: {0}".format(local_path))
raise URLGrabError(2, "Download failed")
else:
shutil.copyfile(urls[0][len("file:"):]+remote_path, local_path)
def configure():
global global_cache_dir
global downloader_common_args
global_cache_dir = sys.argv[1]
downloader_common_args = sys.argv[2:]
def write(fmt, *arg):
try: os.write(1, fmt % arg)
except OSError, e:
if e.args[0] != errno.EPIPE: raise
sys.exit(1)
class ProxyProgress:
def start(self, *d1, **d2):
self.next_update = 0
def update(self, _amount_read):
t = time.time()
if t < self.next_update: return
self.next_update = t + 0.31
write('%d %d\n', self._id, _amount_read)
def main():
import signal
signal.signal(signal.SIGINT, lambda n, f: sys.exit(1))
configure()
cnt = 0
while True:
lines = _readlines(0)
if not lines: break
for line in lines:
cnt += 1
opts = URLGrabberOptions()
opts._id = cnt
for k in line.split(' '):
k, v = k.split('=', 1)
setattr(opts, k, _loads(v))
if opts.progress_obj:
opts.progress_obj = ProxyProgress()
opts.progress_obj._id = cnt
dlsz = dltm = 0
try:
downloadFile(opts.urls, opts.relative_url, opts.filename)
size = 0
# dlsz = fo._tm_last[0] - fo._tm_first[0]
# dltm = fo._tm_last[1] - fo._tm_first[1]
ug_err = 'OK'
except URLGrabError, e:
size = 0
ug_err = '%d %d %s' % (e.errno, getattr(e, 'code', 0), e.strerror)
write('%d %d %d %.3f %s\n', opts._id, size, dlsz, dltm, ug_err)
if __name__ == '__main__':
main()
|
