1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 import gzip
5 import io
6 import json
7 import locale
8 import os
9 import re
10 import sys
11 import traceback
12 import zlib
13 import email.utils
14 import json
15 import datetime
16
17 try:
18 import urllib.request as compat_urllib_request
19 except ImportError: # Python 2
20 import urllib2 as compat_urllib_request
21
22 try:
23 import urllib.error as compat_urllib_error
24 except ImportError: # Python 2
25 import urllib2 as compat_urllib_error
26
27 try:
28 import urllib.parse as compat_urllib_parse
29 except ImportError: # Python 2
30 import urllib as compat_urllib_parse
31
32 try:
33 from urllib.parse import urlparse as compat_urllib_parse_urlparse
34 except ImportError: # Python 2
35 from urlparse import urlparse as compat_urllib_parse_urlparse
36
37 try:
38 import http.cookiejar as compat_cookiejar
39 except ImportError: # Python 2
40 import cookielib as compat_cookiejar
41
42 try:
43 import html.entities as compat_html_entities
44 except ImportError: # Python 2
45 import htmlentitydefs as compat_html_entities
46
47 try:
48 import html.parser as compat_html_parser
49 except ImportError: # Python 2
50 import HTMLParser as compat_html_parser
51
52 try:
53 import http.client as compat_http_client
54 except ImportError: # Python 2
55 import httplib as compat_http_client
56
57 try:
58 from subprocess import DEVNULL
59 compat_subprocess_get_DEVNULL = lambda: DEVNULL
60 except ImportError:
61 compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
62
63 try:
64 from urllib.parse import parse_qs as compat_parse_qs
65 except ImportError: # Python 2
66 # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
67 # Python 2's version is apparently totally broken
68 def _unquote(string, encoding='utf-8', errors='replace'):
69 if string == '':
70 return string
71 res = string.split('%')
72 if len(res) == 1:
73 return string
74 if encoding is None:
75 encoding = 'utf-8'
76 if errors is None:
77 errors = 'replace'
78 # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
79 pct_sequence = b''
80 string = res[0]
81 for item in res[1:]:
82 try:
83 if not item:
84 raise ValueError
85 pct_sequence += item[:2].decode('hex')
86 rest = item[2:]
87 if not rest:
88 # This segment was just a single percent-encoded character.
89 # May be part of a sequence of code units, so delay decoding.
90 # (Stored in pct_sequence).
91 continue
92 except ValueError:
93 rest = '%' + item
94 # Encountered non-percent-encoded characters. Flush the current
95 # pct_sequence.
96 string += pct_sequence.decode(encoding, errors) + rest
97 pct_sequence = b''
98 if pct_sequence:
99 # Flush the final pct_sequence
100 string += pct_sequence.decode(encoding, errors)
101 return string
102
103 def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
104 encoding='utf-8', errors='replace'):
105 qs, _coerce_result = qs, unicode
106 pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
107 r = []
108 for name_value in pairs:
109 if not name_value and not strict_parsing:
110 continue
111 nv = name_value.split('=', 1)
112 if len(nv) != 2:
113 if strict_parsing:
114 raise ValueError("bad query field: %r" % (name_value,))
115 # Handle case of a control-name with no equal sign
116 if keep_blank_values:
117 nv.append('')
118 else:
119 continue
120 if len(nv[1]) or keep_blank_values:
121 name = nv[0].replace('+', ' ')
122 name = _unquote(name, encoding=encoding, errors=errors)
123 name = _coerce_result(name)
124 value = nv[1].replace('+', ' ')
125 value = _unquote(value, encoding=encoding, errors=errors)
126 value = _coerce_result(value)
127 r.append((name, value))
128 return r
129
130 def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
131 encoding='utf-8', errors='replace'):
132 parsed_result = {}
133 pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
134 encoding=encoding, errors=errors)
135 for name, value in pairs:
136 if name in parsed_result:
137 parsed_result[name].append(value)
138 else:
139 parsed_result[name] = [value]
140 return parsed_result
141
142 try:
143 compat_str = unicode # Python 2
144 except NameError:
145 compat_str = str
146
147 try:
148 compat_chr = unichr # Python 2
149 except NameError:
150 compat_chr = chr
151
152 std_headers = {
153 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
154 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
155 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
156 'Accept-Encoding': 'gzip, deflate',
157 'Accept-Language': 'en-us,en;q=0.5',
158 }
159
160 def preferredencoding():
161 """Get preferred encoding.
162
163 Returns the best encoding scheme for the system, based on
164 locale.getpreferredencoding() and some further tweaks.
165 """
166 try:
167 pref = locale.getpreferredencoding()
168 u'TEST'.encode(pref)
169 except:
170 pref = 'UTF-8'
171
172 return pref
173
174 if sys.version_info < (3,0):
175 def compat_print(s):
176 print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
177 else:
178 def compat_print(s):
179 assert type(s) == type(u'')
180 print(s)
181
182 # In Python 2.x, json.dump expects a bytestream.
183 # In Python 3.x, it writes to a character stream
184 if sys.version_info < (3,0):
185 def write_json_file(obj, fn):
186 with open(fn, 'wb') as f:
187 json.dump(obj, f)
188 else:
189 def write_json_file(obj, fn):
190 with open(fn, 'w', encoding='utf-8') as f:
191 json.dump(obj, f)
192
193 def htmlentity_transform(matchobj):
194 """Transforms an HTML entity to a character.
195
196 This function receives a match object and is intended to be used with
197 the re.sub() function.
198 """
199 entity = matchobj.group(1)
200
201 # Known non-numeric HTML entity
202 if entity in compat_html_entities.name2codepoint:
203 return compat_chr(compat_html_entities.name2codepoint[entity])
204
205 mobj = re.match(u'(?u)#(x?\\d+)', entity)
206 if mobj is not None:
207 numstr = mobj.group(1)
208 if numstr.startswith(u'x'):
209 base = 16
210 numstr = u'0%s' % numstr
211 else:
212 base = 10
213 return compat_chr(int(numstr, base))
214
215 # Unknown entity in name, return its literal representation
216 return (u'&%s;' % entity)
217
218 compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
219 class AttrParser(compat_html_parser.HTMLParser):
220 """Modified HTMLParser that isolates a tag with the specified attribute"""
221 def __init__(self, attribute, value):
222 self.attribute = attribute
223 self.value = value
224 self.result = None
225 self.started = False
226 self.depth = {}
227 self.html = None
228 self.watch_startpos = False
229 self.error_count = 0
230 compat_html_parser.HTMLParser.__init__(self)
231
232 def error(self, message):
233 if self.error_count > 10 or self.started:
234 raise compat_html_parser.HTMLParseError(message, self.getpos())
235 self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
236 self.error_count += 1
237 self.goahead(1)
238
239 def loads(self, html):
240 self.html = html
241 self.feed(html)
242 self.close()
243
244 def handle_starttag(self, tag, attrs):
245 attrs = dict(attrs)
246 if self.started:
247 self.find_startpos(None)
248 if self.attribute in attrs and attrs[self.attribute] == self.value:
249 self.result = [tag]
250 self.started = True
251 self.watch_startpos = True
252 if self.started:
253 if not tag in self.depth: self.depth[tag] = 0
254 self.depth[tag] += 1
255
256 def handle_endtag(self, tag):
257 if self.started:
258 if tag in self.depth: self.depth[tag] -= 1
259 if self.depth[self.result[0]] == 0:
260 self.started = False
261 self.result.append(self.getpos())
262
263 def find_startpos(self, x):
264 """Needed to put the start position of the result (self.result[1])
265 after the opening tag with the requested id"""
266 if self.watch_startpos:
267 self.watch_startpos = False
268 self.result.append(self.getpos())
269 handle_entityref = handle_charref = handle_data = handle_comment = \
270 handle_decl = handle_pi = unknown_decl = find_startpos
271
272 def get_result(self):
273 if self.result is None:
274 return None
275 if len(self.result) != 3:
276 return None
277 lines = self.html.split('\n')
278 lines = lines[self.result[1][0]-1:self.result[2][0]]
279 lines[0] = lines[0][self.result[1][1]:]
280 if len(lines) == 1:
281 lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
282 lines[-1] = lines[-1][:self.result[2][1]]
283 return '\n'.join(lines).strip()
284 # Hack for https://github.com/rg3/youtube-dl/issues/662
285 if sys.version_info < (2, 7, 3):
286 AttrParser.parse_endtag = (lambda self, i:
287 i + len("</scr'+'ipt>")
288 if self.rawdata[i:].startswith("</scr'+'ipt>")
289 else compat_html_parser.HTMLParser.parse_endtag(self, i))
290
291 def get_element_by_id(id, html):
292 """Return the content of the tag with the specified ID in the passed HTML document"""
293 return get_element_by_attribute("id", id, html)
294
295 def get_element_by_attribute(attribute, value, html):
296 """Return the content of the tag with the specified attribute in the passed HTML document"""
297 parser = AttrParser(attribute, value)
298 try:
299 parser.loads(html)
300 except compat_html_parser.HTMLParseError:
301 pass
302 return parser.get_result()
303
304
305 def clean_html(html):
306 """Clean an HTML snippet into a readable string"""
307 # Newline vs <br />
308 html = html.replace('\n', ' ')
309 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
310 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
311 # Strip html tags
312 html = re.sub('<.*?>', '', html)
313 # Replace html entities
314 html = unescapeHTML(html)
315 return html.strip()
316
317
318 def sanitize_open(filename, open_mode):
319 """Try to open the given filename, and slightly tweak it if this fails.
320
321 Attempts to open the given filename. If this fails, it tries to change
322 the filename slightly, step by step, until it's either able to open it
323 or it fails and raises a final exception, like the standard open()
324 function.
325
326 It returns the tuple (stream, definitive_file_name).
327 """
328 try:
329 if filename == u'-':
330 if sys.platform == 'win32':
331 import msvcrt
332 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
333 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
334 stream = open(encodeFilename(filename), open_mode)
335 return (stream, filename)
336 except (IOError, OSError) as err:
337 # In case of error, try to remove win32 forbidden chars
338 filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)
339
340 # An exception here should be caught in the caller
341 stream = open(encodeFilename(filename), open_mode)
342 return (stream, filename)
343
344
345 def timeconvert(timestr):
346 """Convert RFC 2822 defined time string into system timestamp"""
347 timestamp = None
348 timetuple = email.utils.parsedate_tz(timestr)
349 if timetuple is not None:
350 timestamp = email.utils.mktime_tz(timetuple)
351 return timestamp
352
353 def sanitize_filename(s, restricted=False, is_id=False):
354 """Sanitizes a string so it could be used as part of a filename.
355 If restricted is set, use a stricter subset of allowed characters.
356 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
357 """
358 def replace_insane(char):
359 if char == '?' or ord(char) < 32 or ord(char) == 127:
360 return ''
361 elif char == '"':
362 return '' if restricted else '\''
363 elif char == ':':
364 return '_-' if restricted else ' -'
365 elif char in '\\/|*<>':
366 return '_'
367 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
368 return '_'
369 if restricted and ord(char) > 127:
370 return '_'
371 return char
372
373 result = u''.join(map(replace_insane, s))
374 if not is_id:
375 while '__' in result:
376 result = result.replace('__', '_')
377 result = result.strip('_')
378 # Common case of "Foreign band name - English song title"
379 if restricted and result.startswith('-_'):
380 result = result[2:]
381 if not result:
382 result = '_'
383 return result
384
385 def orderedSet(iterable):
386 """ Remove all duplicates from the input iterable """
387 res = []
388 for el in iterable:
389 if el not in res:
390 res.append(el)
391 return res
392
393 def unescapeHTML(s):
394 """
395 @param s a string
396 """
397 assert type(s) == type(u'')
398
399 result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
400 return result
401
402 def encodeFilename(s):
403 """
404 @param s The name of the file
405 """
406
407 assert type(s) == type(u'')
408
409 # Python 3 has a Unicode API
410 if sys.version_info >= (3, 0):
411 return s
412
413 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
414 # Pass u'' directly to use Unicode APIs on Windows 2000 and up
415 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
416 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
417 return s
418 else:
419 encoding = sys.getfilesystemencoding()
420 if encoding is None:
421 encoding = 'utf-8'
422 return s.encode(encoding, 'ignore')
423
424 def decodeOption(optval):
425 if optval is None:
426 return optval
427 if isinstance(optval, bytes):
428 optval = optval.decode(preferredencoding())
429
430 assert isinstance(optval, compat_str)
431 return optval
432
433 class ExtractorError(Exception):
434 """Error during info extraction."""
435 def __init__(self, msg, tb=None):
436 """ tb, if given, is the original traceback (so that it can be printed out). """
437 super(ExtractorError, self).__init__(msg)
438 self.traceback = tb
439 self.exc_info = sys.exc_info() # preserve original exception
440
441 def format_traceback(self):
442 if self.traceback is None:
443 return None
444 return u''.join(traceback.format_tb(self.traceback))
445
446
447 class DownloadError(Exception):
448 """Download Error exception.
449
450 This exception may be thrown by FileDownloader objects if they are not
451 configured to continue on errors. They will contain the appropriate
452 error message.
453 """
454 def __init__(self, msg, exc_info=None):
455 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
456 super(DownloadError, self).__init__(msg)
457 self.exc_info = exc_info
458
459
460 class SameFileError(Exception):
461 """Same File exception.
462
463 This exception will be thrown by FileDownloader objects if they detect
464 multiple files would have to be downloaded to the same file on disk.
465 """
466 pass
467
468
469 class PostProcessingError(Exception):
470 """Post Processing exception.
471
472 This exception may be raised by PostProcessor's .run() method to
473 indicate an error in the postprocessing task.
474 """
475 def __init__(self, msg):
476 self.msg = msg
477
478 class MaxDownloadsReached(Exception):
479 """ --max-downloads limit has been reached. """
480 pass
481
482
483 class UnavailableVideoError(Exception):
484 """Unavailable Format exception.
485
486 This exception will be thrown when a video is requested
487 in a format that is not available for that video.
488 """
489 pass
490
491
492 class ContentTooShortError(Exception):
493 """Content Too Short exception.
494
495 This exception may be raised by FileDownloader objects when a file they
496 download is too small for what the server announced first, indicating
497 the connection was probably interrupted.
498 """
499 # Both in bytes
500 downloaded = None
501 expected = None
502
503 def __init__(self, downloaded, expected):
504 self.downloaded = downloaded
505 self.expected = expected
506
507 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
508 """Handler for HTTP requests and responses.
509
510 This class, when installed with an OpenerDirector, automatically adds
511 the standard headers to every HTTP request and handles gzipped and
512 deflated responses from web servers. If compression is to be avoided in
513 a particular request, the original request in the program code only has
514 to include the HTTP header "Youtubedl-No-Compression", which will be
515 removed before making the real request.
516
517 Part of this code was copied from:
518
519 http://techknack.net/python-urllib2-handlers/
520
521 Andrew Rowls, the author of that code, agreed to release it to the
522 public domain.
523 """
524
525 @staticmethod
526 def deflate(data):
527 try:
528 return zlib.decompress(data, -zlib.MAX_WBITS)
529 except zlib.error:
530 return zlib.decompress(data)
531
532 @staticmethod
533 def addinfourl_wrapper(stream, headers, url, code):
534 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
535 return compat_urllib_request.addinfourl(stream, headers, url, code)
536 ret = compat_urllib_request.addinfourl(stream, headers, url)
537 ret.code = code
538 return ret
539
540 def http_request(self, req):
541 for h,v in std_headers.items():
542 if h in req.headers:
543 del req.headers[h]
544 req.add_header(h, v)
545 if 'Youtubedl-no-compression' in req.headers:
546 if 'Accept-encoding' in req.headers:
547 del req.headers['Accept-encoding']
548 del req.headers['Youtubedl-no-compression']
549 if 'Youtubedl-user-agent' in req.headers:
550 if 'User-agent' in req.headers:
551 del req.headers['User-agent']
552 req.headers['User-agent'] = req.headers['Youtubedl-user-agent']
553 del req.headers['Youtubedl-user-agent']
554 return req
555
556 def http_response(self, req, resp):
557 old_resp = resp
558 # gzip
559 if resp.headers.get('Content-encoding', '') == 'gzip':
560 gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
561 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
562 resp.msg = old_resp.msg
563 # deflate
564 if resp.headers.get('Content-encoding', '') == 'deflate':
565 gz = io.BytesIO(self.deflate(resp.read()))
566 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
567 resp.msg = old_resp.msg
568 return resp
569
570 https_request = http_request
571 https_response = http_response
572
573 def unified_strdate(date_str):
574 """Return a string with the date in the format YYYYMMDD"""
575 upload_date = None
576 #Replace commas
577 date_str = date_str.replace(',',' ')
578 # %z (UTC offset) is only supported in python>=3.2
579 date_str = re.sub(r' (\+|-)[\d]*$', '', date_str)
580 format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y', '%Y-%m-%d', '%d/%m/%Y', '%Y/%m/%d %H:%M:%S']
581 for expression in format_expressions:
582 try:
583 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
584 except:
585 pass
586 return upload_date
587
588 def date_from_str(date_str):
589 """Return a datetime object from a string in the format YYYYMMDD"""
590 return datetime.datetime.strptime(date_str, "%Y%m%d").date()
591
592 class DateRange(object):
593 """Represents a time interval between two dates"""
594 def __init__(self, start=None, end=None):
595 """start and end must be strings in the format accepted by date"""
596 if start is not None:
597 self.start = date_from_str(start)
598 else:
599 self.start = datetime.datetime.min.date()
600 if end is not None:
601 self.end = date_from_str(end)
602 else:
603 self.end = datetime.datetime.max.date()
604 if self.start >= self.end:
605 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
606 @classmethod
607 def day(cls, day):
608 """Returns a range that only contains the given day"""
609 return cls(day,day)
610 def __contains__(self, date):
611 """Check if the date is in the range"""
612 date = date_from_str(date)
613 return self.start <= date and date <= self.end
614 def __str__(self):
615 return '%s - %s' % ( self.start.isoformat(), self.end.isoformat())
|
