Show First 20 Lines • Show All 59 Lines • ▼ Show 20 Line(s)
60	def _same_length(filepath, url):	60		def _same_length(filepath, url):
61	'''Determine whether a local file and a file referred by HTTP URL have the	61		'''Determine whether a local file and a file referred by HTTP URL have the
62	same length. If any exception occurs, ``False`` is returned.	62		same length. If any exception occurs, ``False`` is returned.
63	:rtype: bool	63		:rtype: bool
64	'''	64		'''
65	try:	65		try:
66	local_size = os.path.getsize(filepath)	66		local_size = os.path.getsize(filepath)
67		67
68	remote_file = _get_session().head(url, timeout=REQUESTS_TIMEOUT)	68		remote_file = _get_session().head(url, timeout=REQUESTS_TIMEOUT, allow_redirects=True)
69	remote_size = int(remote_file.headers.get('Content-Length'))	69		remote_size = int(remote_file.headers.get('Content-Length'))
70		70
71	return local_size == remote_size	71		return local_size == remote_size
72	except:	72		except:
73	return False	73		return False
74		74
75		75
76	def download(url, dirname, filename=None, cachedir=None):	76		def download(url, dirname, filename=None, cachedir=None):
▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Line(s)		148	def _download(url, dest, timeout=REQUESTS_TIMEOUT, chunk_size=REQUESTS_CHUNK_SIZE):
152	:param str url: an url to be downloaded	152		:param str url: an url to be downloaded
153	:param str dest: a destination of the downloaded file	153		:param str dest: a destination of the downloaded file
154	:param float timeout: how long to wait for the server to send data before giving up	154		:param float timeout: how long to wait for the server to send data before giving up
155	:param int chunk_size: chunk size of the file downloaded (loaded into memory) at a time	155		:param int chunk_size: chunk size of the file downloaded (loaded into memory) at a time
156	:raise requests.exceptions.RequestException: if the download request failed	156		:raise requests.exceptions.RequestException: if the download request failed
157	"""	157		"""
158		158
159	r = _get_session().get(url, timeout=timeout, stream=True)	159		r = _get_session().get(url, timeout=timeout, stream=True)
		160
		161		# if there was a download error (HTTP 4xx or 5xx), raise an error
		162		r.raise_for_status()
		163
160	with open(dest, 'wb') as f:	164		with open(dest, 'wb') as f:
161	total_len = r.headers.get('Content-Length')	165		total_len = r.headers.get('Content-Length')
162		166
163	if total_len is None:	167		if total_len is None:
164	for chunk in r.iter_content(chunk_size):	168		for chunk in r.iter_content(chunk_size):
165	# filter out keep-alive new lines	169		# filter out keep-alive new lines
166	if chunk:	170		if chunk:
167	f.write(chunk)	171		f.write(chunk)
Show All 40 Lines

file_utils: raise error for failed downloads, use redirectsClosedPublicActions

Details

Diff Detail

Revision Contents

Diff 2358

libtaskotron/file_utils.py

file_utils: raise error for failed downloads, use redirects
ClosedPublic
Actions