1 from __future__ import division, unicode_literals
2
3 import os
4 import re
5 import sys
6 import time
7 import random
8
9 from ..compat import compat_os_name
10 from ..utils import (
11 encodeFilename,
12 error_to_compat_str,
13 decodeArgument,
14 format_bytes,
15 timeconvert,
16 )
17
18
19 class FileDownloader(object):
20 """File Downloader class.
21
22 File downloader objects are the ones responsible of downloading the
23 actual video file and writing it to disk.
24
25 File downloaders accept a lot of parameters. In order not to saturate
26 the object constructor with arguments, it receives a dictionary of
27 options instead.
28
29 Available options:
30
31 verbose: Print additional info to stdout.
32 quiet: Do not print messages to stdout.
33 ratelimit: Download speed limit, in bytes/sec.
34 retries: Number of times to retry for HTTP error 5xx
35 buffersize: Size of download buffer in bytes.
36 noresizebuffer: Do not automatically resize the download buffer.
37 continuedl: Try to continue downloads if possible.
38 noprogress: Do not print the progress bar.
39 logtostderr: Log messages to stderr instead of stdout.
40 consoletitle: Display progress in console window's titlebar.
41 nopart: Do not use temporary .part files.
42 updatetime: Use the Last-modified header to set output file timestamps.
43 test: Download only first bytes to test the downloader.
44 min_filesize: Skip files smaller than this size
45 max_filesize: Skip files larger than this size
46 xattr_set_filesize: Set ytdl.filesize user xattribute with expected size.
47 (experimental)
48 external_downloader_args: A list of additional command-line arguments for the
49 external downloader.
50 hls_use_mpegts: Use the mpegts container for HLS videos.
51
52 Subclasses of this one must re-define the real_download method.
53 """
54
55 _TEST_FILE_SIZE = 10241
56 params = None
57
58 def __init__(self, ydl, params):
59 """Create a FileDownloader object with the given options."""
60 self.ydl = ydl
61 self._progress_hooks = []
62 self.params = params
63 self.add_progress_hook(self.report_progress)
64
65 @staticmethod
66 def format_seconds(seconds):
67 (mins, secs) = divmod(seconds, 60)
68 (hours, mins) = divmod(mins, 60)
69 if hours > 99:
70 return '--:--:--'
71 if hours == 0:
72 return '%02d:%02d' % (mins, secs)
73 else:
74 return '%02d:%02d:%02d' % (hours, mins, secs)
75
76 @staticmethod
77 def calc_percent(byte_counter, data_len):
78 if data_len is None:
79 return None
80 return float(byte_counter) / float(data_len) * 100.0
81
82 @staticmethod
83 def format_percent(percent):
84 if percent is None:
85 return '---.-%'
86 return '%6s' % ('%3.1f%%' % percent)
87
88 @staticmethod
89 def calc_eta(start, now, total, current):
90 if total is None:
91 return None
92 if now is None:
93 now = time.time()
94 dif = now - start
95 if current == 0 or dif < 0.001: # One millisecond
96 return None
97 rate = float(current) / dif
98 return int((float(total) - float(current)) / rate)
99
100 @staticmethod
101 def format_eta(eta):
102 if eta is None:
103 return '--:--'
104 return FileDownloader.format_seconds(eta)
105
106 @staticmethod
107 def calc_speed(start, now, bytes):
108 dif = now - start
109 if bytes == 0 or dif < 0.001: # One millisecond
110 return None
111 return float(bytes) / dif
112
113 @staticmethod
114 def format_speed(speed):
115 if speed is None:
116 return '%10s' % '---b/s'
117 return '%10s' % ('%s/s' % format_bytes(speed))
118
119 @staticmethod
120 def format_retries(retries):
121 return 'inf' if retries == float('inf') else '%.0f' % retries
122
123 @staticmethod
124 def best_block_size(elapsed_time, bytes):
125 new_min = max(bytes / 2.0, 1.0)
126 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
127 if elapsed_time < 0.001:
128 return int(new_max)
129 rate = bytes / elapsed_time
130 if rate > new_max:
131 return int(new_max)
132 if rate < new_min:
133 return int(new_min)
134 return int(rate)
135
136 @staticmethod
137 def parse_bytes(bytestr):
138 """Parse a string indicating a byte quantity into an integer."""
139 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
140 if matchobj is None:
141 return None
142 number = float(matchobj.group(1))
143 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
144 return int(round(number * multiplier))
145
146 def to_screen(self, *args, **kargs):
147 self.ydl.to_screen(*args, **kargs)
148
149 def to_stderr(self, message):
150 self.ydl.to_screen(message)
151
152 def to_console_title(self, message):
153 self.ydl.to_console_title(message)
154
155 def trouble(self, *args, **kargs):
156 self.ydl.trouble(*args, **kargs)
157
158 def report_warning(self, *args, **kargs):
159 self.ydl.report_warning(*args, **kargs)
160
161 def report_error(self, *args, **kargs):
162 self.ydl.report_error(*args, **kargs)
163
164 def slow_down(self, start_time, now, byte_counter):
165 """Sleep if the download speed is over the rate limit."""
166 rate_limit = self.params.get('ratelimit')
167 if rate_limit is None or byte_counter == 0:
168 return
169 if now is None:
170 now = time.time()
171 elapsed = now - start_time
172 if elapsed <= 0.0:
173 return
174 speed = float(byte_counter) / elapsed
175 if speed > rate_limit:
176 time.sleep(max((byte_counter // rate_limit) - elapsed, 0))
177
178 def temp_name(self, filename):
179 """Returns a temporary filename for the given filename."""
180 if self.params.get('nopart', False) or filename == '-' or \
181 (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))):
182 return filename
183 return filename + '.part'
184
185 def undo_temp_name(self, filename):
186 if filename.endswith('.part'):
187 return filename[:-len('.part')]
188 return filename
189
190 def try_rename(self, old_filename, new_filename):
191 try:
192 if old_filename == new_filename:
193 return
194 os.rename(encodeFilename(old_filename), encodeFilename(new_filename))
195 except (IOError, OSError) as err:
196 self.report_error('unable to rename file: %s' % error_to_compat_str(err))
197
198 def try_utime(self, filename, last_modified_hdr):
199 """Try to set the last-modified time of the given file."""
200 if last_modified_hdr is None:
201 return
202 if not os.path.isfile(encodeFilename(filename)):
203 return
204 timestr = last_modified_hdr
205 if timestr is None:
206 return
207 filetime = timeconvert(timestr)
208 if filetime is None:
209 return filetime
210 # Ignore obviously invalid dates
211 if filetime == 0:
212 return
213 try:
214 os.utime(filename, (time.time(), filetime))
215 except Exception:
216 pass
217 return filetime
218
219 def report_destination(self, filename):
220 """Report destination filename."""
221 self.to_screen('[download] Destination: ' + filename)
222
223 def _report_progress_status(self, msg, is_last_line=False):
224 fullmsg = '[download] ' + msg
225 if self.params.get('progress_with_newline', False):
226 self.to_screen(fullmsg)
227 else:
228 if compat_os_name == 'nt':
229 prev_len = getattr(self, '_report_progress_prev_line_length',
230 0)
231 if prev_len > len(fullmsg):
232 fullmsg += ' ' * (prev_len - len(fullmsg))
233 self._report_progress_prev_line_length = len(fullmsg)
234 clear_line = '\r'
235 else:
236 clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r')
237 self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line)
238 self.to_console_title('youtube-dl ' + msg)
239
240 def report_progress(self, s):
241 if s['status'] == 'finished':
242 if self.params.get('noprogress', False):
243 self.to_screen('[download] Download completed')
244 else:
245 s['_total_bytes_str'] = format_bytes(s['total_bytes'])
246 if s.get('elapsed') is not None:
247 s['_elapsed_str'] = self.format_seconds(s['elapsed'])
248 msg_template = '100%% of %(_total_bytes_str)s in %(_elapsed_str)s'
249 else:
250 msg_template = '100%% of %(_total_bytes_str)s'
251 self._report_progress_status(
252 msg_template % s, is_last_line=True)
253
254 if self.params.get('noprogress'):
255 return
256
257 if s['status'] != 'downloading':
258 return
259
260 if s.get('eta') is not None:
261 s['_eta_str'] = self.format_eta(s['eta'])
262 else:
263 s['_eta_str'] = 'Unknown ETA'
264
265 if s.get('total_bytes') and s.get('downloaded_bytes') is not None:
266 s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes'])
267 elif s.get('total_bytes_estimate') and s.get('downloaded_bytes') is not None:
268 s['_percent_str'] = self.format_percent(100 * s['downloaded_bytes'] / s['total_bytes_estimate'])
269 else:
270 if s.get('downloaded_bytes') == 0:
271 s['_percent_str'] = self.format_percent(0)
272 else:
273 s['_percent_str'] = 'Unknown %'
274
275 if s.get('speed') is not None:
276 s['_speed_str'] = self.format_speed(s['speed'])
277 else:
278 s['_speed_str'] = 'Unknown speed'
279
280 if s.get('total_bytes') is not None:
281 s['_total_bytes_str'] = format_bytes(s['total_bytes'])
282 msg_template = '%(_percent_str)s of %(_total_bytes_str)s at %(_speed_str)s ETA %(_eta_str)s'
283 elif s.get('total_bytes_estimate') is not None:
284 s['_total_bytes_estimate_str'] = format_bytes(s['total_bytes_estimate'])
285 msg_template = '%(_percent_str)s of ~%(_total_bytes_estimate_str)s at %(_speed_str)s ETA %(_eta_str)s'
286 else:
287 if s.get('downloaded_bytes') is not None:
288 s['_downloaded_bytes_str'] = format_bytes(s['downloaded_bytes'])
289 if s.get('elapsed'):
290 s['_elapsed_str'] = self.format_seconds(s['elapsed'])
291 msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s (%(_elapsed_str)s)'
292 else:
293 msg_template = '%(_downloaded_bytes_str)s at %(_speed_str)s'
294 else:
295 msg_template = '%(_percent_str)s % at %(_speed_str)s ETA %(_eta_str)s'
296
297 self._report_progress_status(msg_template % s)
298
299 def report_resuming_byte(self, resume_len):
300 """Report attempt to resume at given byte."""
301 self.to_screen('[download] Resuming download at byte %s' % resume_len)
302
303 def report_retry(self, count, retries):
304 """Report retry in case of HTTP error 5xx"""
305 self.to_screen(
306 '[download] Got server HTTP error. Retrying (attempt %d of %s)...'
307 % (count, self.format_retries(retries)))
308
309 def report_file_already_downloaded(self, file_name):
310 """Report file has already been fully downloaded."""
311 try:
312 self.to_screen('[download] %s has already been downloaded' % file_name)
313 except UnicodeEncodeError:
314 self.to_screen('[download] The file has already been downloaded')
315
316 def report_unable_to_resume(self):
317 """Report it was impossible to resume download."""
318 self.to_screen('[download] Unable to resume')
319
320 def download(self, filename, info_dict):
321 """Download to a filename using the info from info_dict
322 Return True on success and False otherwise
323 """
324
325 nooverwrites_and_exists = (
326 self.params.get('nooverwrites', False) and
327 os.path.exists(encodeFilename(filename))
328 )
329
330 continuedl_and_exists = (
331 self.params.get('continuedl', True) and
332 os.path.isfile(encodeFilename(filename)) and
333 not self.params.get('nopart', False)
334 )
335
336 # Check file already present
337 if filename != '-' and (nooverwrites_and_exists or continuedl_and_exists):
338 self.report_file_already_downloaded(filename)
339 self._hook_progress({
340 'filename': filename,
341 'status': 'finished',
342 'total_bytes': os.path.getsize(encodeFilename(filename)),
343 })
344 return True
345
346 min_sleep_interval = self.params.get('sleep_interval')
347 if min_sleep_interval:
348 max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval)
349 sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval)
350 self.to_screen('[download] Sleeping %s seconds...' % sleep_interval)
351 time.sleep(sleep_interval)
352
353 return self.real_download(filename, info_dict)
354
355 def real_download(self, filename, info_dict):
356 """Real download process. Redefine in subclasses."""
357 raise NotImplementedError('This method must be implemented by subclasses')
358
359 def _hook_progress(self, status):
360 for ph in self._progress_hooks:
361 ph(status)
362
363 def add_progress_hook(self, ph):
364 # See YoutubeDl.py (search for progress_hooks) for a description of
365 # this interface
366 self._progress_hooks.append(ph)
367
368 def _debug_cmd(self, args, exe=None):
369 if not self.params.get('verbose', False):
370 return
371
372 str_args = [decodeArgument(a) for a in args]
373
374 if exe is None:
375 exe = os.path.basename(str_args[0])
376
377 try:
378 import pipes
379 shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
380 except ImportError:
381 shell_quote = repr
382 self.to_screen('[debug] %s command line: %s' % (
383 exe, shell_quote(str_args)))
|
