1 # coding: utf-8
2
3 from __future__ import unicode_literals
4
5 import os
6 import re
7 import sys
8
9 from .common import InfoExtractor
10 from .youtube import YoutubeIE
11 from ..compat import (
12 compat_etree_fromstring,
13 compat_str,
14 compat_urllib_parse_unquote,
15 compat_urlparse,
16 compat_xml_parse_error,
17 )
18 from ..utils import (
19 determine_ext,
20 ExtractorError,
21 float_or_none,
22 HEADRequest,
23 int_or_none,
24 is_html,
25 js_to_json,
26 KNOWN_EXTENSIONS,
27 merge_dicts,
28 mimetype2ext,
29 orderedSet,
30 parse_duration,
31 sanitized_Request,
32 smuggle_url,
33 unescapeHTML,
34 unified_timestamp,
35 unsmuggle_url,
36 UnsupportedError,
37 url_or_none,
38 xpath_attr,
39 xpath_text,
40 xpath_with_ns,
41 )
42 from .commonprotocols import RtmpIE
43 from .brightcove import (
44 BrightcoveLegacyIE,
45 BrightcoveNewIE,
46 )
47 from .nexx import (
48 NexxIE,
49 NexxEmbedIE,
50 )
51 from .nbc import NBCSportsVPlayerIE
52 from .ooyala import OoyalaIE
53 from .rutv import RUTVIE
54 from .tvc import TVCIE
55 from .sportbox import SportBoxIE
56 from .myvi import MyviIE
57 from .condenast import CondeNastIE
58 from .udn import UDNEmbedIE
59 from .senateisvp import SenateISVPIE
60 from .svt import SVTIE
61 from .pornhub import PornHubIE
62 from .xhamster import XHamsterEmbedIE
63 from .tnaflix import TNAFlixNetworkEmbedIE
64 from .drtuber import DrTuberIE
65 from .redtube import RedTubeIE
66 from .tube8 import Tube8IE
67 from .mofosex import MofosexEmbedIE
68 from .spankwire import SpankwireIE
69 from .youporn import YouPornIE
70 from .vimeo import (
71 VimeoIE,
72 VHXEmbedIE,
73 )
74 from .dailymotion import DailymotionIE
75 from .dailymail import DailyMailIE
76 from .onionstudios import OnionStudiosIE
77 from .viewlift import ViewLiftEmbedIE
78 from .mtv import MTVServicesEmbeddedIE
79 from .pladform import PladformIE
80 from .videomore import VideomoreIE
81 from .webcaster import WebcasterFeedIE
82 from .googledrive import GoogleDriveIE
83 from .jwplatform import JWPlatformIE
84 from .digiteka import DigitekaIE
85 from .arkena import ArkenaIE
86 from .instagram import InstagramIE
87 from .threeqsdn import ThreeQSDNIE
88 from .theplatform import ThePlatformIE
89 from .kaltura import KalturaIE
90 from .eagleplatform import EaglePlatformIE
91 from .facebook import FacebookIE
92 from .soundcloud import SoundcloudEmbedIE
93 from .tunein import TuneInBaseIE
94 from .vbox7 import Vbox7IE
95 from .dbtv import DBTVIE
96 from .piksel import PikselIE
97 from .videa import VideaIE
98 from .twentymin import TwentyMinutenIE
99 from .ustream import UstreamIE
100 from .arte import ArteTVEmbedIE
101 from .videopress import VideoPressIE
102 from .rutube import RutubeIE
103 from .limelight import LimelightBaseIE
104 from .anvato import AnvatoIE
105 from .washingtonpost import WashingtonPostIE
106 from .wistia import WistiaIE
107 from .mediaset import MediasetIE
108 from .joj import JojIE
109 from .megaphone import MegaphoneIE
110 from .vzaar import VzaarIE
111 from .channel9 import Channel9IE
112 from .vshare import VShareIE
113 from .mediasite import MediasiteIE
114 from .springboardplatform import SpringboardPlatformIE
115 from .yapfiles import YapFilesIE
116 from .vice import ViceIE
117 from .xfileshare import XFileShareIE
118 from .cloudflarestream import CloudflareStreamIE
119 from .peertube import PeerTubeIE
120 from .teachable import TeachableIE
121 from .indavideo import IndavideoEmbedIE
122 from .apa import APAIE
123 from .foxnews import FoxNewsIE
124 from .viqeo import ViqeoIE
125 from .expressen import ExpressenIE
126 from .zype import ZypeIE
127 from .odnoklassniki import OdnoklassnikiIE
128 from .vk import VKIE
129 from .kinja import KinjaEmbedIE
130 from .arcpublishing import ArcPublishingIE
131 from .medialaan import MedialaanIE
132 from .simplecast import SimplecastIE
133
134
135 class GenericIE(InfoExtractor):
136 IE_DESC = 'Generic downloader that works on some sites'
137 _VALID_URL = r'.*'
138 IE_NAME = 'generic'
139 _TESTS = [
140 # Direct link to a video
141 {
142 'url': 'http://media.w3.org/2010/05/sintel/trailer.mp4',
143 'md5': '67d406c2bcb6af27fa886f31aa934bbe',
144 'info_dict': {
145 'id': 'trailer',
146 'ext': 'mp4',
147 'title': 'trailer',
148 'upload_date': '20100513',
149 }
150 },
151 # Direct link to media delivered compressed (until Accept-Encoding is *)
152 {
153 'url': 'http://calimero.tk/muzik/FictionJunction-Parallel_Hearts.flac',
154 'md5': '128c42e68b13950268b648275386fc74',
155 'info_dict': {
156 'id': 'FictionJunction-Parallel_Hearts',
157 'ext': 'flac',
158 'title': 'FictionJunction-Parallel_Hearts',
159 'upload_date': '20140522',
160 },
161 'expected_warnings': [
162 'URL could be a direct video link, returning it as such.'
163 ],
164 'skip': 'URL invalid',
165 },
166 # Direct download with broken HEAD
167 {
168 'url': 'http://ai-radio.org:8000/radio.opus',
169 'info_dict': {
170 'id': 'radio',
171 'ext': 'opus',
172 'title': 'radio',
173 },
174 'params': {
175 'skip_download': True, # infinite live stream
176 },
177 'expected_warnings': [
178 r'501.*Not Implemented',
179 r'400.*Bad Request',
180 ],
181 },
182 # Direct link with incorrect MIME type
183 {
184 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
185 'md5': '4ccbebe5f36706d85221f204d7eb5913',
186 'info_dict': {
187 'url': 'http://ftp.nluug.nl/video/nluug/2014-11-20_nj14/zaal-2/5_Lennart_Poettering_-_Systemd.webm',
188 'id': '5_Lennart_Poettering_-_Systemd',
189 'ext': 'webm',
190 'title': '5_Lennart_Poettering_-_Systemd',
191 'upload_date': '20141120',
192 },
193 'expected_warnings': [
194 'URL could be a direct video link, returning it as such.'
195 ]
196 },
197 # RSS feed
198 {
199 'url': 'http://phihag.de/2014/youtube-dl/rss2.xml',
200 'info_dict': {
201 'id': 'http://phihag.de/2014/youtube-dl/rss2.xml',
202 'title': 'Zero Punctuation',
203 'description': 're:.*groundbreaking video review series.*'
204 },
205 'playlist_mincount': 11,
206 },
207 # RSS feed with enclosure
208 {
209 'url': 'http://podcastfeeds.nbcnews.com/audio/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
210 'info_dict': {
211 'id': 'http://podcastfeeds.nbcnews.com/nbcnews/video/podcast/MSNBC-MADDOW-NETCAST-M4V.xml',
212 'title': 'MSNBC Rachel Maddow (video)',
213 'description': 're:.*her unique approach to storytelling.*',
214 },
215 'playlist': [{
216 'info_dict': {
217 'ext': 'mov',
218 'id': 'pdv_maddow_netcast_mov-12-04-2020-224335',
219 'title': 're:MSNBC Rachel Maddow',
220 'description': 're:.*her unique approach to storytelling.*',
221 'timestamp': int,
222 'upload_date': compat_str,
223 'duration': float,
224 },
225 }],
226 },
227 # RSS feed with item with description and thumbnails
228 {
229 'url': 'https://anchor.fm/s/dd00e14/podcast/rss',
230 'info_dict': {
231 'id': 'https://anchor.fm/s/dd00e14/podcast/rss',
232 'title': 're:.*100% Hydrogen.*',
233 'description': 're:.*In this episode.*',
234 },
235 'playlist': [{
236 'info_dict': {
237 'ext': 'm4a',
238 'id': 'c1c879525ce2cb640b344507e682c36d',
239 'title': 're:Hydrogen!',
240 'description': 're:.*In this episode we are going.*',
241 'timestamp': 1567977776,
242 'upload_date': '20190908',
243 'duration': 459,
244 'thumbnail': r're:^https?://.*\.jpg$',
245 'episode_number': 1,
246 'season_number': 1,
247 'age_limit': 0,
248 },
249 }],
250 'params': {
251 'skip_download': True,
252 },
253 },
254 # RSS feed with enclosures and unsupported link URLs
255 {
256 'url': 'http://www.hellointernet.fm/podcast?format=rss',
257 'info_dict': {
258 'id': 'http://www.hellointernet.fm/podcast?format=rss',
259 'description': 'CGP Grey and Brady Haran talk about YouTube, life, work, whatever.',
260 'title': 'Hello Internet',
261 },
262 'playlist_mincount': 100,
263 },
264 # SMIL from http://videolectures.net/promogram_igor_mekjavic_eng
265 {
266 'url': 'http://videolectures.net/promogram_igor_mekjavic_eng/video/1/smil.xml',
267 'info_dict': {
268 'id': 'smil',
269 'ext': 'mp4',
270 'title': 'Automatics, robotics and biocybernetics',
271 'description': 'md5:815fc1deb6b3a2bff99de2d5325be482',
272 'upload_date': '20130627',
273 'formats': 'mincount:16',
274 'subtitles': 'mincount:1',
275 },
276 'params': {
277 'force_generic_extractor': True,
278 'skip_download': True,
279 },
280 },
281 # SMIL from http://www1.wdr.de/mediathek/video/livestream/index.html
282 {
283 'url': 'http://metafilegenerator.de/WDR/WDR_FS/hds/hds.smil',
284 'info_dict': {
285 'id': 'hds',
286 'ext': 'flv',
287 'title': 'hds',
288 'formats': 'mincount:1',
289 },
290 'params': {
291 'skip_download': True,
292 },
293 },
294 # SMIL from https://www.restudy.dk/video/play/id/1637
295 {
296 'url': 'https://www.restudy.dk/awsmedia/SmilDirectory/video_1637.xml',
297 'info_dict': {
298 'id': 'video_1637',
299 'ext': 'flv',
300 'title': 'video_1637',
301 'formats': 'mincount:3',
302 },
303 'params': {
304 'skip_download': True,
305 },
306 },
307 # SMIL from http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm
308 {
309 'url': 'http://services.media.howstuffworks.com/videos/450221/smil-service.smil',
310 'info_dict': {
311 'id': 'smil-service',
312 'ext': 'flv',
313 'title': 'smil-service',
314 'formats': 'mincount:1',
315 },
316 'params': {
317 'skip_download': True,
318 },
319 },
320 # SMIL from http://new.livestream.com/CoheedandCambria/WebsterHall/videos/4719370
321 {
322 'url': 'http://api.new.livestream.com/accounts/1570303/events/1585861/videos/4719370.smil',
323 'info_dict': {
324 'id': '4719370',
325 'ext': 'mp4',
326 'title': '571de1fd-47bc-48db-abf9-238872a58d1f',
327 'formats': 'mincount:3',
328 },
329 'params': {
330 'skip_download': True,
331 },
332 },
333 # XSPF playlist from http://www.telegraaf.nl/tv/nieuws/binnenland/24353229/__Tikibad_ontruimd_wegens_brand__.html
334 {
335 'url': 'http://www.telegraaf.nl/xml/playlist/2015/8/7/mZlp2ctYIUEB.xspf',
336 'info_dict': {
337 'id': 'mZlp2ctYIUEB',
338 'ext': 'mp4',
339 'title': 'Tikibad ontruimd wegens brand',
340 'description': 'md5:05ca046ff47b931f9b04855015e163a4',
341 'thumbnail': r're:^https?://.*\.jpg$',
342 'duration': 33,
343 },
344 'params': {
345 'skip_download': True,
346 },
347 },
348 # MPD from http://dash-mse-test.appspot.com/media.html
349 {
350 'url': 'http://yt-dash-mse-test.commondatastorage.googleapis.com/media/car-20120827-manifest.mpd',
351 'md5': '4b57baab2e30d6eb3a6a09f0ba57ef53',
352 'info_dict': {
353 'id': 'car-20120827-manifest',
354 'ext': 'mp4',
355 'title': 'car-20120827-manifest',
356 'formats': 'mincount:9',
357 'upload_date': '20130904',
358 },
359 'params': {
360 'format': 'bestvideo',
361 },
362 },
363 # m3u8 served with Content-Type: audio/x-mpegURL; charset=utf-8
364 {
365 'url': 'http://once.unicornmedia.com/now/master/playlist/bb0b18ba-64f5-4b1b-a29f-0ac252f06b68/77a785f3-5188-4806-b788-0893a61634ed/93677179-2d99-4ef4-9e17-fe70d49abfbf/content.m3u8',
366 'info_dict': {
367 'id': 'content',
368 'ext': 'mp4',
369 'title': 'content',
370 'formats': 'mincount:8',
371 },
372 'params': {
373 # m3u8 downloads
374 'skip_download': True,
375 },
376 'skip': 'video gone',
377 },
378 # m3u8 served with Content-Type: text/plain
379 {
380 'url': 'http://www.nacentapps.com/m3u8/index.m3u8',
381 'info_dict': {
382 'id': 'index',
383 'ext': 'mp4',
384 'title': 'index',
385 'upload_date': '20140720',
386 'formats': 'mincount:11',
387 },
388 'params': {
389 # m3u8 downloads
390 'skip_download': True,
391 },
392 'skip': 'video gone',
393 },
394 # google redirect
395 {
396 'url': 'http://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&cad=rja&ved=0CCUQtwIwAA&url=http%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DcmQHVoWB5FY&ei=F-sNU-LLCaXk4QT52ICQBQ&usg=AFQjCNEw4hL29zgOohLXvpJ-Bdh2bils1Q&bvm=bv.61965928,d.bGE',
397 'info_dict': {
398 'id': 'cmQHVoWB5FY',
399 'ext': 'mp4',
400 'upload_date': '20130224',
401 'uploader_id': 'TheVerge',
402 'description': r're:^Chris Ziegler takes a look at the\.*',
403 'uploader': 'The Verge',
404 'title': 'First Firefox OS phones side-by-side',
405 },
406 'params': {
407 'skip_download': False,
408 }
409 },
410 {
411 # redirect in Refresh HTTP header
412 'url': 'https://www.facebook.com/l.php?u=https%3A%2F%2Fwww.youtube.com%2Fwatch%3Fv%3DpO8h3EaFRdo&h=TAQHsoToz&enc=AZN16h-b6o4Zq9pZkCCdOLNKMN96BbGMNtcFwHSaazus4JHT_MFYkAA-WARTX2kvsCIdlAIyHZjl6d33ILIJU7Jzwk_K3mcenAXoAzBNoZDI_Q7EXGDJnIhrGkLXo_LJ_pAa2Jzbx17UHMd3jAs--6j2zaeto5w9RTn8T_1kKg3fdC5WPX9Dbb18vzH7YFX0eSJmoa6SP114rvlkw6pkS1-T&s=1',
413 'info_dict': {
414 'id': 'pO8h3EaFRdo',
415 'ext': 'mp4',
416 'title': 'Tripeo Boiler Room x Dekmantel Festival DJ Set',
417 'description': 'md5:6294cc1af09c4049e0652b51a2df10d5',
418 'upload_date': '20150917',
419 'uploader_id': 'brtvofficial',
420 'uploader': 'Boiler Room',
421 },
422 'params': {
423 'skip_download': False,
424 },
425 },
426 {
427 'url': 'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
428 'md5': '85b90ccc9d73b4acd9138d3af4c27f89',
429 'info_dict': {
430 'id': '13601338388002',
431 'ext': 'mp4',
432 'uploader': 'www.hodiho.fr',
433 'title': 'R\u00e9gis plante sa Jeep',
434 }
435 },
436 # bandcamp page with custom domain
437 {
438 'add_ie': ['Bandcamp'],
439 'url': 'http://bronyrock.com/track/the-pony-mash',
440 'info_dict': {
441 'id': '3235767654',
442 'ext': 'mp3',
443 'title': 'The Pony Mash',
444 'uploader': 'M_Pallante',
445 },
446 'skip': 'There is a limit of 200 free downloads / month for the test song',
447 },
448 {
449 # embedded brightcove video
450 # it also tests brightcove videos that need to set the 'Referer'
451 # in the http requests
452 'add_ie': ['BrightcoveLegacy'],
453 'url': 'http://www.bfmtv.com/video/bfmbusiness/cours-bourse/cours-bourse-l-analyse-technique-154522/',
454 'info_dict': {
455 'id': '2765128793001',
456 'ext': 'mp4',
457 'title': 'Le cours de bourse : l’analyse technique',
458 'description': 'md5:7e9ad046e968cb2d1114004aba466fd9',
459 'uploader': 'BFM BUSINESS',
460 },
461 'params': {
462 'skip_download': True,
463 },
464 },
465 {
466 # embedded with itemprop embedURL and video id spelled as `idVideo`
467 'add_id': ['BrightcoveLegacy'],
468 'url': 'http://bfmbusiness.bfmtv.com/mediaplayer/chroniques/olivier-delamarche/',
469 'info_dict': {
470 'id': '5255628253001',
471 'ext': 'mp4',
472 'title': 'md5:37c519b1128915607601e75a87995fc0',
473 'description': 'md5:37f7f888b434bb8f8cc8dbd4f7a4cf26',
474 'uploader': 'BFM BUSINESS',
475 'uploader_id': '876450612001',
476 'timestamp': 1482255315,
477 'upload_date': '20161220',
478 },
479 'params': {
480 'skip_download': True,
481 },
482 },
483 {
484 # https://github.com/ytdl-org/youtube-dl/issues/2253
485 'url': 'http://bcove.me/i6nfkrc3',
486 'md5': '0ba9446db037002366bab3b3eb30c88c',
487 'info_dict': {
488 'id': '3101154703001',
489 'ext': 'mp4',
490 'title': 'Still no power',
491 'uploader': 'thestar.com',
492 'description': 'Mississauga resident David Farmer is still out of power as a result of the ice storm a month ago. To keep the house warm, Farmer cuts wood from his property for a wood burning stove downstairs.',
493 },
494 'add_ie': ['BrightcoveLegacy'],
495 'skip': 'video gone',
496 },
497 {
498 'url': 'http://www.championat.com/video/football/v/87/87499.html',
499 'md5': 'fb973ecf6e4a78a67453647444222983',
500 'info_dict': {
501 'id': '3414141473001',
502 'ext': 'mp4',
503 'title': 'Видео. Удаление Дзагоева (ЦСКА)',
504 'description': 'Онлайн-трансляция матча ЦСКА - "Волга"',
505 'uploader': 'Championat',
506 },
507 },
508 {
509 # https://github.com/ytdl-org/youtube-dl/issues/3541
510 'add_ie': ['BrightcoveLegacy'],
511 'url': 'http://www.kijk.nl/sbs6/leermijvrouwenkennen/videos/jqMiXKAYan2S/aflevering-1',
512 'info_dict': {
513 'id': '3866516442001',
514 'ext': 'mp4',
515 'title': 'Leer mij vrouwen kennen: Aflevering 1',
516 'description': 'Leer mij vrouwen kennen: Aflevering 1',
517 'uploader': 'SBS Broadcasting',
518 },
519 'skip': 'Restricted to Netherlands',
520 'params': {
521 'skip_download': True, # m3u8 download
522 },
523 },
524 {
525 # Brightcove video in <iframe>
526 'url': 'http://www.un.org/chinese/News/story.asp?NewsID=27724',
527 'md5': '36d74ef5e37c8b4a2ce92880d208b968',
528 'info_dict': {
529 'id': '5360463607001',
530 'ext': 'mp4',
531 'title': '叙利亚失明儿童在废墟上演唱《心跳》 呼吁获得正常童年生活',
532 'description': '联合国儿童基金会中东和北非区域大使、作曲家扎德·迪拉尼（Zade Dirani）在3月15日叙利亚冲突爆发7周年纪念日之际发布了为叙利亚谱写的歌曲《心跳》（HEARTBEAT），为受到六年冲突影响的叙利亚儿童发出强烈呐喊，呼吁世界做出共同努力，使叙利亚儿童重新获得享有正常童年生活的权利。',
533 'uploader': 'United Nations',
534 'uploader_id': '1362235914001',
535 'timestamp': 1489593889,
536 'upload_date': '20170315',
537 },
538 'add_ie': ['BrightcoveLegacy'],
539 },
540 {
541 # Brightcove with alternative playerID key
542 'url': 'http://www.nature.com/nmeth/journal/v9/n7/fig_tab/nmeth.2062_SV1.html',
543 'info_dict': {
544 'id': 'nmeth.2062_SV1',
545 'title': 'Simultaneous multiview imaging of the Drosophila syncytial blastoderm : Quantitative high-speed imaging of entire developing embryos with simultaneous multiview light-sheet microscopy : Nature Methods : Nature Research',
546 },
547 'playlist': [{
548 'info_dict': {
549 'id': '2228375078001',
550 'ext': 'mp4',
551 'title': 'nmeth.2062-sv1',
552 'description': 'nmeth.2062-sv1',
553 'timestamp': 1363357591,
554 'upload_date': '20130315',
555 'uploader': 'Nature Publishing Group',
556 'uploader_id': '1964492299001',
557 },
558 }],
559 },
560 {
561 # Brightcove with UUID in videoPlayer
562 'url': 'http://www8.hp.com/cn/zh/home.html',
563 'info_dict': {
564 'id': '5255815316001',
565 'ext': 'mp4',
566 'title': 'Sprocket Video - China',
567 'description': 'Sprocket Video - China',
568 'uploader': 'HP-Video Gallery',
569 'timestamp': 1482263210,
570 'upload_date': '20161220',
571 'uploader_id': '1107601872001',
572 },
573 'params': {
574 'skip_download': True, # m3u8 download
575 },
576 'skip': 'video rotates...weekly?',
577 },
578 {
579 # Brightcove:new type [2].
580 'url': 'http://www.delawaresportszone.com/video-st-thomas-more-earns-first-trip-to-basketball-semis',
581 'md5': '2b35148fcf48da41c9fb4591650784f3',
582 'info_dict': {
583 'id': '5348741021001',
584 'ext': 'mp4',
585 'upload_date': '20170306',
586 'uploader_id': '4191638492001',
587 'timestamp': 1488769918,
588 'title': 'VIDEO: St. Thomas More earns first trip to basketball semis',
589
590 },
591 },
592 {
593 # Alternative brightcove <video> attributes
594 'url': 'http://www.programme-tv.net/videos/extraits/81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche/',
595 'info_dict': {
596 'id': '81095-guillaume-canet-evoque-les-rumeurs-d-infidelite-de-marion-cotillard-avec-brad-pitt-dans-vivement-dimanche',
597 'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche, Extraits : toutes les vidéos avec Télé-Loisirs",
598 },
599 'playlist': [{
600 'md5': '732d22ba3d33f2f3fc253c39f8f36523',
601 'info_dict': {
602 'id': '5311302538001',
603 'ext': 'mp4',
604 'title': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche",
605 'description': "Guillaume Canet évoque les rumeurs d'infidélité de Marion Cotillard avec Brad Pitt dans Vivement Dimanche (France 2, 5 février 2017)",
606 'timestamp': 1486321708,
607 'upload_date': '20170205',
608 'uploader_id': '800000640001',
609 },
610 'only_matching': True,
611 }],
612 },
613 {
614 # Brightcove with UUID in videoPlayer
615 'url': 'http://www8.hp.com/cn/zh/home.html',
616 'info_dict': {
617 'id': '5255815316001',
618 'ext': 'mp4',
619 'title': 'Sprocket Video - China',
620 'description': 'Sprocket Video - China',
621 'uploader': 'HP-Video Gallery',
622 'timestamp': 1482263210,
623 'upload_date': '20161220',
624 'uploader_id': '1107601872001',
625 },
626 'params': {
627 'skip_download': True, # m3u8 download
628 },
629 },
630 # ooyala video
631 {
632 'url': 'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219',
633 'md5': '166dd577b433b4d4ebfee10b0824d8ff',
634 'info_dict': {
635 'id': 'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ',
636 'ext': 'mp4',
637 'title': '2cc213299525360.mov', # that's what we get
638 'duration': 238.231,
639 },
640 'add_ie': ['Ooyala'],
641 },
642 {
643 # ooyala video embedded with http://player.ooyala.com/iframe.js
644 'url': 'http://www.macrumors.com/2015/07/24/steve-jobs-the-man-in-the-machine-first-trailer/',
645 'info_dict': {
646 'id': 'p0MGJndjoG5SOKqO_hZJuZFPB-Tr5VgB',
647 'ext': 'mp4',
648 'title': '"Steve Jobs: Man in the Machine" trailer',
649 'description': 'The first trailer for the Alex Gibney documentary "Steve Jobs: Man in the Machine."',
650 'duration': 135.427,
651 },
652 'params': {
653 'skip_download': True,
654 },
655 'skip': 'movie expired',
656 },
657 # ooyala video embedded with http://player.ooyala.com/static/v4/production/latest/core.min.js
658 {
659 'url': 'http://wnep.com/2017/07/22/steampunk-fest-comes-to-honesdale/',
660 'info_dict': {
661 'id': 'lwYWYxYzE6V5uJMjNGyKtwwiw9ZJD7t2',
662 'ext': 'mp4',
663 'title': 'Steampunk Fest Comes to Honesdale',
664 'duration': 43.276,
665 },
666 'params': {
667 'skip_download': True,
668 }
669 },
670 # embed.ly video
671 {
672 'url': 'http://www.tested.com/science/weird/460206-tested-grinding-coffee-2000-frames-second/',
673 'info_dict': {
674 'id': '9ODmcdjQcHQ',
675 'ext': 'mp4',
676 'title': 'Tested: Grinding Coffee at 2000 Frames Per Second',
677 'upload_date': '20140225',
678 'description': 'md5:06a40fbf30b220468f1e0957c0f558ff',
679 'uploader': 'Tested',
680 'uploader_id': 'testedcom',
681 },
682 # No need to test YoutubeIE here
683 'params': {
684 'skip_download': True,
685 },
686 },
687 # funnyordie embed
688 {
689 'url': 'http://www.theguardian.com/world/2014/mar/11/obama-zach-galifianakis-between-two-ferns',
690 'info_dict': {
691 'id': '18e820ec3f',
692 'ext': 'mp4',
693 'title': 'Between Two Ferns with Zach Galifianakis: President Barack Obama',
694 'description': 'Episode 18: President Barack Obama sits down with Zach Galifianakis for his most memorable interview yet.',
695 },
696 # HEAD requests lead to endless 301, while GET is OK
697 'expected_warnings': ['301'],
698 },
699 # RUTV embed
700 {
701 'url': 'http://www.rg.ru/2014/03/15/reg-dfo/anklav-anons.html',
702 'info_dict': {
703 'id': '776940',
704 'ext': 'mp4',
705 'title': 'Охотское море стало целиком российским',
706 'description': 'md5:5ed62483b14663e2a95ebbe115eb8f43',
707 },
708 'params': {
709 # m3u8 download
710 'skip_download': True,
711 },
712 },
713 # TVC embed
714 {
715 'url': 'http://sch1298sz.mskobr.ru/dou_edu/karamel_ki/filial_galleries/video/iframe_src_http_tvc_ru_video_iframe_id_55304_isplay_false_acc_video_id_channel_brand_id_11_show_episodes_episode_id_32307_frameb/',
716 'info_dict': {
717 'id': '55304',
718 'ext': 'mp4',
719 'title': 'Дошкольное воспитание',
720 },
721 },
722 # SportBox embed
723 {
724 'url': 'http://www.vestifinance.ru/articles/25753',
725 'info_dict': {
726 'id': '25753',
727 'title': 'Прямые трансляции с Форума-выставки "Госзаказ-2013"',
728 },
729 'playlist': [{
730 'info_dict': {
731 'id': '370908',
732 'title': 'Госзаказ. День 3',
733 'ext': 'mp4',
734 }
735 }, {
736 'info_dict': {
737 'id': '370905',
738 'title': 'Госзаказ. День 2',
739 'ext': 'mp4',
740 }
741 }, {
742 'info_dict': {
743 'id': '370902',
744 'title': 'Госзаказ. День 1',
745 'ext': 'mp4',
746 }
747 }],
748 'params': {
749 # m3u8 download
750 'skip_download': True,
751 },
752 },
753 # Myvi.ru embed
754 {
755 'url': 'http://www.kinomyvi.tv/news/detail/Pervij-dublirovannij-trejler--Uzhastikov-_nOw1',
756 'info_dict': {
757 'id': 'f4dafcad-ff21-423d-89b5-146cfd89fa1e',
758 'ext': 'mp4',
759 'title': 'Ужастики, русский трейлер (2015)',
760 'thumbnail': r're:^https?://.*\.jpg$',
761 'duration': 153,
762 }
763 },
764 # XHamster embed
765 {
766 'url': 'http://www.numisc.com/forum/showthread.php?11696-FM15-which-pumiscer-was-this-%28-vid-%29-%28-alfa-as-fuck-srx-%29&s=711f5db534502e22260dec8c5e2d66d8',
767 'info_dict': {
768 'id': 'showthread',
769 'title': '[NSFL] [FM15] which pumiscer was this ( vid ) ( alfa as fuck srx )',
770 },
771 'playlist_mincount': 7,
772 # This forum does not allow <iframe> syntaxes anymore
773 # Now HTML tags are displayed as-is
774 'skip': 'No videos on this page',
775 },
776 # Embedded TED video
777 {
778 'url': 'http://en.support.wordpress.com/videos/ted-talks/',
779 'md5': '65fdff94098e4a607385a60c5177c638',
780 'info_dict': {
781 'id': '1969',
782 'ext': 'mp4',
783 'title': 'Hidden miracles of the natural world',
784 'uploader': 'Louie Schwartzberg',
785 'description': 'md5:8145d19d320ff3e52f28401f4c4283b9',
786 }
787 },
788 # nowvideo embed hidden behind percent encoding
789 {
790 'url': 'http://www.waoanime.tv/the-super-dimension-fortress-macross-episode-1/',
791 'md5': '2baf4ddd70f697d94b1c18cf796d5107',
792 'info_dict': {
793 'id': '06e53103ca9aa',
794 'ext': 'flv',
795 'title': 'Macross Episode 001 Watch Macross Episode 001 onl',
796 'description': 'No description',
797 },
798 },
799 # arte embed
800 {
801 'url': 'http://www.tv-replay.fr/redirection/20-03-14/x-enius-arte-10753389.html',
802 'md5': '7653032cbb25bf6c80d80f217055fa43',
803 'info_dict': {
804 'id': '048195-004_PLUS7-F',
805 'ext': 'flv',
806 'title': 'X:enius',
807 'description': 'md5:d5fdf32ef6613cdbfd516ae658abf168',
808 'upload_date': '20140320',
809 },
810 'params': {
811 'skip_download': 'Requires rtmpdump'
812 },
813 'skip': 'video gone',
814 },
815 # francetv embed
816 {
817 'url': 'http://www.tsprod.com/replay-du-concert-alcaline-de-calogero',
818 'info_dict': {
819 'id': 'EV_30231',
820 'ext': 'mp4',
821 'title': 'Alcaline, le concert avec Calogero',
822 'description': 'md5:61f08036dcc8f47e9cfc33aed08ffaff',
823 'upload_date': '20150226',
824 'timestamp': 1424989860,
825 'duration': 5400,
826 },
827 'params': {
828 # m3u8 downloads
829 'skip_download': True,
830 },
831 'expected_warnings': [
832 'Forbidden'
833 ]
834 },
835 # Condé Nast embed
836 {
837 'url': 'http://www.wired.com/2014/04/honda-asimo/',
838 'md5': 'ba0dfe966fa007657bd1443ee672db0f',
839 'info_dict': {
840 'id': '53501be369702d3275860000',
841 'ext': 'mp4',
842 'title': 'Honda’s New Asimo Robot Is More Human Than Ever',
843 }
844 },
845 # Dailymotion embed
846 {
847 'url': 'http://www.spi0n.com/zap-spi0n-com-n216/',
848 'md5': '441aeeb82eb72c422c7f14ec533999cd',
849 'info_dict': {
850 'id': 'k2mm4bCdJ6CQ2i7c8o2',
851 'ext': 'mp4',
852 'title': 'Le Zap de Spi0n n°216 - Zapping du Web',
853 'description': 'md5:faf028e48a461b8b7fad38f1e104b119',
854 'uploader': 'Spi0n',
855 'uploader_id': 'xgditw',
856 'upload_date': '20140425',
857 'timestamp': 1398441542,
858 },
859 'add_ie': ['Dailymotion'],
860 },
861 # DailyMail embed
862 {
863 'url': 'http://www.bumm.sk/krimi/2017/07/05/biztonsagi-kamera-buktatta-le-az-agg-ferfit-utlegelo-apolot',
864 'info_dict': {
865 'id': '1495629',
866 'ext': 'mp4',
867 'title': 'Care worker punches elderly dementia patient in head 11 times',
868 'description': 'md5:3a743dee84e57e48ec68bf67113199a5',
869 },
870 'add_ie': ['DailyMail'],
871 'params': {
872 'skip_download': True,
873 },
874 },
875 # YouTube embed
876 {
877 'url': 'http://www.badzine.de/ansicht/datum/2014/06/09/so-funktioniert-die-neue-englische-badminton-liga.html',
878 'info_dict': {
879 'id': 'FXRb4ykk4S0',
880 'ext': 'mp4',
881 'title': 'The NBL Auction 2014',
882 'uploader': 'BADMINTON England',
883 'uploader_id': 'BADMINTONEvents',
884 'upload_date': '20140603',
885 'description': 'md5:9ef128a69f1e262a700ed83edb163a73',
886 },
887 'add_ie': ['Youtube'],
888 'params': {
889 'skip_download': True,
890 }
891 },
892 # MTVServices embed
893 {
894 'url': 'http://www.vulture.com/2016/06/new-key-peele-sketches-released.html',
895 'md5': 'ca1aef97695ef2c1d6973256a57e5252',
896 'info_dict': {
897 'id': '769f7ec0-0692-4d62-9b45-0d88074bffc1',
898 'ext': 'mp4',
899 'title': 'Key and Peele|October 10, 2012|2|203|Liam Neesons - Uncensored',
900 'description': 'Two valets share their love for movie star Liam Neesons.',
901 'timestamp': 1349922600,
902 'upload_date': '20121011',
903 },
904 },
905 # YouTube embed via <data-embed-url="">
906 {
907 'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM',
908 'info_dict': {
909 'id': '4vAffPZIT44',
910 'ext': 'mp4',
911 'title': 'Asphalt 8: Airborne - Update - Welcome to Dubai!',
912 'uploader': 'Gameloft',
913 'uploader_id': 'gameloft',
914 'upload_date': '20140828',
915 'description': 'md5:c80da9ed3d83ae6d1876c834de03e1c4',
916 },
917 'params': {
918 'skip_download': True,
919 }
920 },
921 # YouTube <object> embed
922 {
923 'url': 'http://www.improbable.com/2017/04/03/untrained-modern-youths-and-ancient-masters-in-selfie-portraits/',
924 'md5': '516718101ec834f74318df76259fb3cc',
925 'info_dict': {
926 'id': 'msN87y-iEx0',
927 'ext': 'webm',
928 'title': 'Feynman: Mirrors FUN TO IMAGINE 6',
929 'upload_date': '20080526',
930 'description': 'md5:0ffc78ea3f01b2e2c247d5f8d1d3c18d',
931 'uploader': 'Christopher Sykes',
932 'uploader_id': 'ChristopherJSykes',
933 },
934 'add_ie': ['Youtube'],
935 },
936 # Camtasia studio
937 {
938 'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/',
939 'playlist': [{
940 'md5': '0c5e352edabf715d762b0ad4e6d9ee67',
941 'info_dict': {
942 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
943 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1',
944 'ext': 'flv',
945 'duration': 2235.90,
946 }
947 }, {
948 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63',
949 'info_dict': {
950 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP',
951 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip',
952 'ext': 'flv',
953 'duration': 2235.93,
954 }
955 }],
956 'info_dict': {
957 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final',
958 }
959 },
960 # Flowplayer
961 {
962 'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html',
963 'md5': '9d65602bf31c6e20014319c7d07fba27',
964 'info_dict': {
965 'id': '5123ea6d5e5a7',
966 'ext': 'mp4',
967 'age_limit': 18,
968 'uploader': 'www.handjobhub.com',
969 'title': 'Busty Blonde Siri Tit Fuck While Wank at HandjobHub.com',
970 }
971 },
972 # Multiple brightcove videos
973 # https://github.com/ytdl-org/youtube-dl/issues/2283
974 {
975 'url': 'http://www.newyorker.com/online/blogs/newsdesk/2014/01/always-never-nuclear-command-and-control.html',
976 'info_dict': {
977 'id': 'always-never',
978 'title': 'Always / Never - The New Yorker',
979 },
980 'playlist_count': 3,
981 'params': {
982 'extract_flat': False,
983 'skip_download': True,
984 }
985 },
986 # MLB embed
987 {
988 'url': 'http://umpire-empire.com/index.php/topic/58125-laz-decides-no-thats-low/',
989 'md5': '96f09a37e44da40dd083e12d9a683327',
990 'info_dict': {
991 'id': '33322633',
992 'ext': 'mp4',
993 'title': 'Ump changes call to ball',
994 'description': 'md5:71c11215384298a172a6dcb4c2e20685',
995 'duration': 48,
996 'timestamp': 1401537900,
997 'upload_date': '20140531',
998 'thumbnail': r're:^https?://.*\.jpg$',
999 },
1000 },
1001 # Wistia embed
1002 {
1003 'url': 'http://study.com/academy/lesson/north-american-exploration-failed-colonies-of-spain-france-england.html#lesson',
1004 'md5': '1953f3a698ab51cfc948ed3992a0b7ff',
1005 'info_dict': {
1006 'id': '6e2wtrbdaf',
1007 'ext': 'mov',
1008 'title': 'paywall_north-american-exploration-failed-colonies-of-spain-france-england',
1009 'description': 'a Paywall Videos video from Remilon',
1010 'duration': 644.072,
1011 'uploader': 'study.com',
1012 'timestamp': 1459678540,
1013 'upload_date': '20160403',
1014 'filesize': 24687186,
1015 },
1016 },
1017 {
1018 'url': 'http://thoughtworks.wistia.com/medias/uxjb0lwrcz',
1019 'md5': 'baf49c2baa8a7de5f3fc145a8506dcd4',
1020 'info_dict': {
1021 'id': 'uxjb0lwrcz',
1022 'ext': 'mp4',
1023 'title': 'Conversation about Hexagonal Rails Part 1',
1024 'description': 'a Martin Fowler video from ThoughtWorks',
1025 'duration': 1715.0,
1026 'uploader': 'thoughtworks.wistia.com',
1027 'timestamp': 1401832161,
1028 'upload_date': '20140603',
1029 },
1030 },
1031 # Wistia standard embed (async)
1032 {
1033 'url': 'https://www.getdrip.com/university/brennan-dunn-drip-workshop/',
1034 'info_dict': {
1035 'id': '807fafadvk',
1036 'ext': 'mp4',
1037 'title': 'Drip Brennan Dunn Workshop',
1038 'description': 'a JV Webinars video from getdrip-1',
1039 'duration': 4986.95,
1040 'timestamp': 1463607249,
1041 'upload_date': '20160518',
1042 },
1043 'params': {
1044 'skip_download': True,
1045 }
1046 },
1047 # Soundcloud embed
1048 {
1049 'url': 'http://nakedsecurity.sophos.com/2014/10/29/sscc-171-are-you-sure-that-1234-is-a-bad-password-podcast/',
1050 'info_dict': {
1051 'id': '174391317',
1052 'ext': 'mp3',
1053 'description': 'md5:ff867d6b555488ad3c52572bb33d432c',
1054 'uploader': 'Sophos Security',
1055 'title': 'Chet Chat 171 - Oct 29, 2014',
1056 'upload_date': '20141029',
1057 }
1058 },
1059 # Soundcloud multiple embeds
1060 {
1061 'url': 'http://www.guitarplayer.com/lessons/1014/legato-workout-one-hour-to-more-fluid-performance---tab/52809',
1062 'info_dict': {
1063 'id': '52809',
1064 'title': 'Guitar Essentials: Legato Workout—One-Hour to Fluid Performance | TAB + AUDIO',
1065 },
1066 'playlist_mincount': 7,
1067 },
1068 # TuneIn station embed
1069 {
1070 'url': 'http://radiocnrv.com/promouvoir-radio-cnrv/',
1071 'info_dict': {
1072 'id': '204146',
1073 'ext': 'mp3',
1074 'title': 'CNRV',
1075 'location': 'Paris, France',
1076 'is_live': True,
1077 },
1078 'params': {
1079 # Live stream
1080 'skip_download': True,
1081 },
1082 },
1083 # Livestream embed
1084 {
1085 'url': 'http://www.esa.int/Our_Activities/Space_Science/Rosetta/Philae_comet_touch-down_webcast',
1086 'info_dict': {
1087 'id': '67864563',
1088 'ext': 'flv',
1089 'upload_date': '20141112',
1090 'title': 'Rosetta #CometLanding webcast HL 10',
1091 }
1092 },
1093 # Another Livestream embed, without 'new.' in URL
1094 {
1095 'url': 'https://www.freespeech.org/',
1096 'info_dict': {
1097 'id': '123537347',
1098 'ext': 'mp4',
1099 'title': 're:^FSTV [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
1100 },
1101 'params': {
1102 # Live stream
1103 'skip_download': True,
1104 },
1105 },
1106 # LazyYT
1107 {
1108 'url': 'https://skiplagged.com/',
1109 'info_dict': {
1110 'id': 'skiplagged',
1111 'title': 'Skiplagged: The smart way to find cheap flights',
1112 },
1113 'playlist_mincount': 1,
1114 'add_ie': ['Youtube'],
1115 },
1116 # Cinchcast embed
1117 {
1118 'url': 'http://undergroundwellness.com/podcasts/306-5-steps-to-permanent-gut-healing/',
1119 'info_dict': {
1120 'id': '7141703',
1121 'ext': 'mp3',
1122 'upload_date': '20141126',
1123 'title': 'Jack Tips: 5 Steps to Permanent Gut Healing',
1124 }
1125 },
1126 # Cinerama player
1127 {
1128 'url': 'http://www.abc.net.au/7.30/content/2015/s4164797.htm',
1129 'info_dict': {
1130 'id': '730m_DandD_1901_512k',
1131 'ext': 'mp4',
1132 'uploader': 'www.abc.net.au',
1133 'title': 'Game of Thrones with dice - Dungeons and Dragons fantasy role-playing game gets new life - 19/01/2015',
1134 }
1135 },
1136 # embedded viddler video
1137 {
1138 'url': 'http://deadspin.com/i-cant-stop-watching-john-wall-chop-the-nuggets-with-th-1681801597',
1139 'info_dict': {
1140 'id': '4d03aad9',
1141 'ext': 'mp4',
1142 'uploader': 'deadspin',
1143 'title': 'WALL-TO-GORTAT',
1144 'timestamp': 1422285291,
1145 'upload_date': '20150126',
1146 },
1147 'add_ie': ['Viddler'],
1148 },
1149 # Libsyn embed
1150 {
1151 'url': 'http://thedailyshow.cc.com/podcast/episodetwelve',
1152 'info_dict': {
1153 'id': '3377616',
1154 'ext': 'mp3',
1155 'title': "The Daily Show Podcast without Jon Stewart - Episode 12: Bassem Youssef: Egypt's Jon Stewart",
1156 'description': 'md5:601cb790edd05908957dae8aaa866465',
1157 'upload_date': '20150220',
1158 },
1159 'skip': 'All The Daily Show URLs now redirect to http://www.cc.com/shows/',
1160 },
1161 # jwplayer YouTube
1162 {
1163 'url': 'http://media.nationalarchives.gov.uk/index.php/webinar-using-discovery-national-archives-online-catalogue/',
1164 'info_dict': {
1165 'id': 'Mrj4DVp2zeA',
1166 'ext': 'mp4',
1167 'upload_date': '20150212',
1168 'uploader': 'The National Archives UK',
1169 'description': 'md5:8078af856dca76edc42910b61273dbbf',
1170 'uploader_id': 'NationalArchives08',
1171 'title': 'Webinar: Using Discovery, The National Archives’ online catalogue',
1172 },
1173 },
1174 # jwplayer rtmp
1175 {
1176 'url': 'http://www.suffolk.edu/sjc/live.php',
1177 'info_dict': {
1178 'id': 'live',
1179 'ext': 'flv',
1180 'title': 'Massachusetts Supreme Judicial Court Oral Arguments',
1181 'uploader': 'www.suffolk.edu',
1182 },
1183 'params': {
1184 'skip_download': True,
1185 },
1186 'skip': 'Only has video a few mornings per month, see http://www.suffolk.edu/sjc/',
1187 },
1188 # Complex jwplayer
1189 {
1190 'url': 'http://www.indiedb.com/games/king-machine/videos',
1191 'info_dict': {
1192 'id': 'videos',
1193 'ext': 'mp4',
1194 'title': 'king machine trailer 1',
1195 'description': 'Browse King Machine videos & audio for sweet media. Your eyes will thank you.',
1196 'thumbnail': r're:^https?://.*\.jpg$',
1197 },
1198 },
1199 {
1200 # JWPlayer config passed as variable
1201 'url': 'http://www.txxx.com/videos/3326530/ariele/',
1202 'info_dict': {
1203 'id': '3326530_hq',
1204 'ext': 'mp4',
1205 'title': 'ARIELE | Tube Cup',
1206 'uploader': 'www.txxx.com',
1207 'age_limit': 18,
1208 },
1209 'params': {
1210 'skip_download': True,
1211 }
1212 },
1213 {
1214 # JWPlatform iframe
1215 'url': 'https://www.mediaite.com/tv/dem-senator-claims-gary-cohn-faked-a-bad-connection-during-trump-call-to-get-him-off-the-phone/',
1216 'md5': 'ca00a040364b5b439230e7ebfd02c4e9',
1217 'info_dict': {
1218 'id': 'O0c5JcKT',
1219 'ext': 'mp4',
1220 'upload_date': '20171122',
1221 'timestamp': 1511366290,
1222 'title': 'Dem Senator Claims Gary Cohn Faked a Bad Connection During Trump Call to Get Him Off the Phone',
1223 },
1224 'add_ie': [JWPlatformIE.ie_key()],
1225 },
1226 {
1227 # Video.js embed, multiple formats
1228 'url': 'http://ortcam.com/solidworks-урок-6-настройка-чертежа_33f9b7351.html',
1229 'info_dict': {
1230 'id': 'yygqldloqIk',
1231 'ext': 'mp4',
1232 'title': 'SolidWorks. Урок 6 Настройка чертежа',
1233 'description': 'md5:baf95267792646afdbf030e4d06b2ab3',
1234 'upload_date': '20130314',
1235 'uploader': 'PROстое3D',
1236 'uploader_id': 'PROstoe3D',
1237 },
1238 'params': {
1239 'skip_download': True,
1240 },
1241 },
1242 {
1243 # Video.js embed, single format
1244 'url': 'https://www.vooplayer.com/v3/watch/watch.php?v=NzgwNTg=',
1245 'info_dict': {
1246 'id': 'watch',
1247 'ext': 'mp4',
1248 'title': 'Step 1 - Good Foundation',
1249 'description': 'md5:d1e7ff33a29fc3eb1673d6c270d344f4',
1250 },
1251 'params': {
1252 'skip_download': True,
1253 },
1254 },
1255 # rtl.nl embed
1256 {
1257 'url': 'http://www.rtlnieuws.nl/nieuws/buitenland/aanslagen-kopenhagen',
1258 'playlist_mincount': 5,
1259 'info_dict': {
1260 'id': 'aanslagen-kopenhagen',
1261 'title': 'Aanslagen Kopenhagen',
1262 }
1263 },
1264 # Zapiks embed
1265 {
1266 'url': 'http://www.skipass.com/news/116090-bon-appetit-s5ep3-baqueira-mi-cor.html',
1267 'info_dict': {
1268 'id': '118046',
1269 'ext': 'mp4',
1270 'title': 'EP3S5 - Bon Appétit - Baqueira Mi Corazon !',
1271 }
1272 },
1273 # Kaltura embed (different embed code)
1274 {
1275 'url': 'http://www.premierchristianradio.com/Shows/Saturday/Unbelievable/Conference-Videos/Os-Guinness-Is-It-Fools-Talk-Unbelievable-Conference-2014',
1276 'info_dict': {
1277 'id': '1_a52wc67y',
1278 'ext': 'flv',
1279 'upload_date': '20150127',
1280 'uploader_id': 'PremierMedia',
1281 'timestamp': int,
1282 'title': 'Os Guinness // Is It Fools Talk? // Unbelievable? Conference 2014',
1283 },
1284 },
1285 # Kaltura embed with single quotes
1286 {
1287 'url': 'http://fod.infobase.com/p_ViewPlaylist.aspx?AssignmentID=NUN8ZY',
1288 'info_dict': {
1289 'id': '0_izeg5utt',
1290 'ext': 'mp4',
1291 'title': '35871',
1292 'timestamp': 1355743100,
1293 'upload_date': '20121217',
1294 'uploader_id': 'cplapp@learn360.com',
1295 },
1296 'add_ie': ['Kaltura'],
1297 },
1298 {
1299 # Kaltura embedded via quoted entry_id
1300 'url': 'https://www.oreilly.com/ideas/my-cloud-makes-pretty-pictures',
1301 'info_dict': {
1302 'id': '0_utuok90b',
1303 'ext': 'mp4',
1304 'title': '06_matthew_brender_raj_dutt',
1305 'timestamp': 1466638791,
1306 'upload_date': '20160622',
1307 },
1308 'add_ie': ['Kaltura'],
1309 'expected_warnings': [
1310 'Could not send HEAD request'
1311 ],
1312 'params': {
1313 'skip_download': True,
1314 }
1315 },
1316 {
1317 # Kaltura embedded, some fileExt broken (#11480)
1318 'url': 'http://www.cornell.edu/video/nima-arkani-hamed-standard-models-of-particle-physics',
1319 'info_dict': {
1320 'id': '1_sgtvehim',
1321 'ext': 'mp4',
1322 'title': 'Our "Standard Models" of particle physics and cosmology',
1323 'description': 'md5:67ea74807b8c4fea92a6f38d6d323861',
1324 'timestamp': 1321158993,
1325 'upload_date': '20111113',
1326 'uploader_id': 'kps1',
1327 },
1328 'add_ie': ['Kaltura'],
1329 },
1330 {
1331 # Kaltura iframe embed
1332 'url': 'http://www.gsd.harvard.edu/event/i-m-pei-a-centennial-celebration/',
1333 'md5': 'ae5ace8eb09dc1a35d03b579a9c2cc44',
1334 'info_dict': {
1335 'id': '0_f2cfbpwy',
1336 'ext': 'mp4',
1337 'title': 'I. M. Pei: A Centennial Celebration',
1338 'description': 'md5:1db8f40c69edc46ca180ba30c567f37c',
1339 'upload_date': '20170403',
1340 'uploader_id': 'batchUser',
1341 'timestamp': 1491232186,
1342 },
1343 'add_ie': ['Kaltura'],
1344 },
1345 {
1346 # Kaltura iframe embed, more sophisticated
1347 'url': 'http://www.cns.nyu.edu/~eero/math-tools/Videos/lecture-05sep2017.html',
1348 'info_dict': {
1349 'id': '1_9gzouybz',
1350 'ext': 'mp4',
1351 'title': 'lecture-05sep2017',
1352 'description': 'md5:40f347d91fd4ba047e511c5321064b49',
1353 'upload_date': '20170913',
1354 'uploader_id': 'eps2',
1355 'timestamp': 1505340777,
1356 },
1357 'params': {
1358 'skip_download': True,
1359 },
1360 'add_ie': ['Kaltura'],
1361 },
1362 {
1363 # meta twitter:player
1364 'url': 'http://thechive.com/2017/12/08/all-i-want-for-christmas-is-more-twerk/',
1365 'info_dict': {
1366 'id': '0_01b42zps',
1367 'ext': 'mp4',
1368 'title': 'Main Twerk (Video)',
1369 'upload_date': '20171208',
1370 'uploader_id': 'sebastian.salinas@thechive.com',
1371 'timestamp': 1512713057,
1372 },
1373 'params': {
1374 'skip_download': True,
1375 },
1376 'add_ie': ['Kaltura'],
1377 },
1378 # referrer protected EaglePlatform embed
1379 {
1380 'url': 'https://tvrain.ru/lite/teleshow/kak_vse_nachinalos/namin-418921/',
1381 'info_dict': {
1382 'id': '582306',
1383 'ext': 'mp4',
1384 'title': 'Стас Намин: «Мы нарушили девственность Кремля»',
1385 'thumbnail': r're:^https?://.*\.jpg$',
1386 'duration': 3382,
1387 'view_count': int,
1388 },
1389 'params': {
1390 'skip_download': True,
1391 },
1392 },
1393 # ClipYou (EaglePlatform) embed (custom URL)
1394 {
1395 'url': 'http://muz-tv.ru/play/7129/',
1396 # Not checking MD5 as sometimes the direct HTTP link results in 404 and HLS is used
1397 'info_dict': {
1398 'id': '12820',
1399 'ext': 'mp4',
1400 'title': "'O Sole Mio",
1401 'thumbnail': r're:^https?://.*\.jpg$',
1402 'duration': 216,
1403 'view_count': int,
1404 },
1405 'params': {
1406 'skip_download': True,
1407 },
1408 'skip': 'This video is unavailable.',
1409 },
1410 # Pladform embed
1411 {
1412 'url': 'http://muz-tv.ru/kinozal/view/7400/',
1413 'info_dict': {
1414 'id': '100183293',
1415 'ext': 'mp4',
1416 'title': 'Тайны перевала Дятлова • 1 серия 2 часть',
1417 'description': 'Документальный сериал-расследование одной из самых жутких тайн ХХ века',
1418 'thumbnail': r're:^https?://.*\.jpg$',
1419 'duration': 694,
1420 'age_limit': 0,
1421 },
1422 'skip': 'HTTP Error 404: Not Found',
1423 },
1424 # Playwire embed
1425 {
1426 'url': 'http://www.cinemablend.com/new/First-Joe-Dirt-2-Trailer-Teaser-Stupid-Greatness-70874.html',
1427 'info_dict': {
1428 'id': '3519514',
1429 'ext': 'mp4',
1430 'title': 'Joe Dirt 2 Beautiful Loser Teaser Trailer',
1431 'thumbnail': r're:^https?://.*\.png$',
1432 'duration': 45.115,
1433 },
1434 },
1435 # 5min embed
1436 {
1437 'url': 'http://techcrunch.com/video/facebook-creates-on-this-day-crunch-report/518726732/',
1438 'md5': '4c6f127a30736b59b3e2c19234ee2bf7',
1439 'info_dict': {
1440 'id': '518726732',
1441 'ext': 'mp4',
1442 'title': 'Facebook Creates "On This Day" | Crunch Report',
1443 'description': 'Amazon updates Fire TV line, Tesla\'s Model X spotted in the wild',
1444 'timestamp': 1427237531,
1445 'uploader': 'Crunch Report',
1446 'upload_date': '20150324',
1447 },
1448 'params': {
1449 # m3u8 download
1450 'skip_download': True,
1451 },
1452 },
1453 # Crooks and Liars embed
1454 {
1455 'url': 'http://crooksandliars.com/2015/04/fox-friends-says-protecting-atheists',
1456 'info_dict': {
1457 'id': '8RUoRhRi',
1458 'ext': 'mp4',
1459 'title': "Fox & Friends Says Protecting Atheists From Discrimination Is Anti-Christian!",
1460 'description': 'md5:e1a46ad1650e3a5ec7196d432799127f',
1461 'timestamp': 1428207000,
1462 'upload_date': '20150405',
1463 'uploader': 'Heather',
1464 },
1465 },
1466 # Crooks and Liars external embed
1467 {
1468 'url': 'http://theothermccain.com/2010/02/02/video-proves-that-bill-kristol-has-been-watching-glenn-beck/comment-page-1/',
1469 'info_dict': {
1470 'id': 'MTE3MjUtMzQ2MzA',
1471 'ext': 'mp4',
1472 'title': 'md5:5e3662a81a4014d24c250d76d41a08d5',
1473 'description': 'md5:9b8e9542d6c3c5de42d6451b7d780cec',
1474 'timestamp': 1265032391,
1475 'upload_date': '20100201',
1476 'uploader': 'Heather',
1477 },
1478 },
1479 # NBC Sports vplayer embed
1480 {
1481 'url': 'http://www.riderfans.com/forum/showthread.php?121827-Freeman&s=e98fa1ea6dc08e886b1678d35212494a',
1482 'info_dict': {
1483 'id': 'ln7x1qSThw4k',
1484 'ext': 'flv',
1485 'title': "PFT Live: New leader in the 'new-look' defense",
1486 'description': 'md5:65a19b4bbfb3b0c0c5768bed1dfad74e',
1487 'uploader': 'NBCU-SPORTS',
1488 'upload_date': '20140107',
1489 'timestamp': 1389118457,
1490 },
1491 'skip': 'Invalid Page URL',
1492 },
1493 # NBC News embed
1494 {
1495 'url': 'http://www.vulture.com/2016/06/letterman-couldnt-care-less-about-late-night.html',
1496 'md5': '1aa589c675898ae6d37a17913cf68d66',
1497 'info_dict': {
1498 'id': 'x_dtl_oa_LettermanliftPR_160608',
1499 'ext': 'mp4',
1500 'title': 'David Letterman: A Preview',
1501 'description': 'A preview of Tom Brokaw\'s interview with David Letterman as part of the On Assignment series powered by Dateline. Airs Sunday June 12 at 7/6c.',
1502 'upload_date': '20160609',
1503 'timestamp': 1465431544,
1504 'uploader': 'NBCU-NEWS',
1505 },
1506 },
1507 # UDN embed
1508 {
1509 'url': 'https://video.udn.com/news/300346',
1510 'md5': 'fd2060e988c326991037b9aff9df21a6',
1511 'info_dict': {
1512 'id': '300346',
1513 'ext': 'mp4',
1514 'title': '中一中男師變性 全校師生力挺',
1515 'thumbnail': r're:^https?://.*\.jpg$',
1516 },
1517 'params': {
1518 # m3u8 download
1519 'skip_download': True,
1520 },
1521 'expected_warnings': ['Failed to parse JSON Expecting value'],
1522 },
1523 # Brightcove URL in single quotes
1524 {
1525 'url': 'http://www.sportsnet.ca/baseball/mlb/sn-presents-russell-martin-world-citizen/',
1526 'md5': '4ae374f1f8b91c889c4b9203c8c752af',
1527 'info_dict': {
1528 'id': '4255764656001',
1529 'ext': 'mp4',
1530 'title': 'SN Presents: Russell Martin, World Citizen',
1531 'description': 'To understand why he was the Toronto Blue Jays’ top off-season priority is to appreciate his background and upbringing in Montreal, where he first developed his baseball skills. Written and narrated by Stephen Brunt.',
1532 'uploader': 'Rogers Sportsnet',
1533 'uploader_id': '1704050871',
1534 'upload_date': '20150525',
1535 'timestamp': 1432570283,
1536 },
1537 },
1538 # Kinja embed
1539 {
1540 'url': 'http://www.clickhole.com/video/dont-understand-bitcoin-man-will-mumble-explanatio-2537',
1541 'info_dict': {
1542 'id': '106351',
1543 'ext': 'mp4',
1544 'title': 'Don’t Understand Bitcoin? This Man Will Mumble An Explanation At You',
1545 'description': 'Migrated from OnionStudios',
1546 'thumbnail': r're:^https?://.*\.jpe?g$',
1547 'uploader': 'clickhole',
1548 'upload_date': '20150527',
1549 'timestamp': 1432744860,
1550 }
1551 },
1552 # SnagFilms embed
1553 {
1554 'url': 'http://whilewewatch.blogspot.ru/2012/06/whilewewatch-whilewewatch-gripping.html',
1555 'info_dict': {
1556 'id': '74849a00-85a9-11e1-9660-123139220831',
1557 'ext': 'mp4',
1558 'title': '#whilewewatch',
1559 }
1560 },
1561 # AdobeTVVideo embed
1562 {
1563 'url': 'https://helpx.adobe.com/acrobat/how-to/new-experience-acrobat-dc.html?set=acrobat--get-started--essential-beginners',
1564 'md5': '43662b577c018ad707a63766462b1e87',
1565 'info_dict': {
1566 'id': '2456',
1567 'ext': 'mp4',
1568 'title': 'New experience with Acrobat DC',
1569 'description': 'New experience with Acrobat DC',
1570 'duration': 248.667,
1571 },
1572 },
1573 # BrightcoveInPageEmbed embed
1574 {
1575 'url': 'http://www.geekandsundry.com/tabletop-bonus-wils-final-thoughts-on-dread/',
1576 'info_dict': {
1577 'id': '4238694884001',
1578 'ext': 'flv',
1579 'title': 'Tabletop: Dread, Last Thoughts',
1580 'description': 'Tabletop: Dread, Last Thoughts',
1581 'duration': 51690,
1582 },
1583 },
1584 # Brightcove embed, with no valid 'renditions' but valid 'IOSRenditions'
1585 # This video can't be played in browsers if Flash disabled and UA set to iPhone, which is actually a false alarm
1586 {
1587 'url': 'https://dl.dropboxusercontent.com/u/29092637/interview.html',
1588 'info_dict': {
1589 'id': '4785848093001',
1590 'ext': 'mp4',
1591 'title': 'The Cardinal Pell Interview',
1592 'description': 'Sky News Contributor Andrew Bolt interviews George Pell in Rome, following the Cardinal\'s evidence before the Royal Commission into Child Abuse. ',
1593 'uploader': 'GlobeCast Australia - GlobeStream',
1594 'uploader_id': '2733773828001',
1595 'upload_date': '20160304',
1596 'timestamp': 1457083087,
1597 },
1598 'params': {
1599 # m3u8 downloads
1600 'skip_download': True,
1601 },
1602 },
1603 {
1604 # Brightcove embed with whitespace around attribute names
1605 'url': 'http://www.stack.com/video/3167554373001/learn-to-hit-open-three-pointers-with-damian-lillard-s-baseline-drift-drill',
1606 'info_dict': {
1607 'id': '3167554373001',
1608 'ext': 'mp4',
1609 'title': "Learn to Hit Open Three-Pointers With Damian Lillard's Baseline Drift Drill",
1610 'description': 'md5:57bacb0e0f29349de4972bfda3191713',
1611 'uploader_id': '1079349493',
1612 'upload_date': '20140207',
1613 'timestamp': 1391810548,
1614 },
1615 'params': {
1616 'skip_download': True,
1617 },
1618 },
1619 # Another form of arte.tv embed
1620 {
1621 'url': 'http://www.tv-replay.fr/redirection/09-04-16/arte-reportage-arte-11508975.html',
1622 'md5': '850bfe45417ddf221288c88a0cffe2e2',
1623 'info_dict': {
1624 'id': '030273-562_PLUS7-F',
1625 'ext': 'mp4',
1626 'title': 'ARTE Reportage - Nulle part, en France',
1627 'description': 'md5:e3a0e8868ed7303ed509b9e3af2b870d',
1628 'upload_date': '20160409',
1629 },
1630 },
1631 # Duplicated embedded video URLs
1632 {
1633 'url': 'http://www.hudl.com/athlete/2538180/highlights/149298443',
1634 'info_dict': {
1635 'id': '149298443_480_16c25b74_2',
1636 'ext': 'mp4',
1637 'title': 'vs. Blue Orange Spring Game',
1638 'uploader': 'www.hudl.com',
1639 },
1640 },
1641 # twitter:player:stream embed
1642 {
1643 'url': 'http://www.rtl.be/info/video/589263.aspx?CategoryID=288',
1644 'info_dict': {
1645 'id': 'master',
1646 'ext': 'mp4',
1647 'title': 'Une nouvelle espèce de dinosaure découverte en Argentine',
1648 'uploader': 'www.rtl.be',
1649 },
1650 'params': {
1651 # m3u8 downloads
1652 'skip_download': True,
1653 },
1654 },
1655 # twitter:player embed
1656 {
1657 'url': 'http://www.theatlantic.com/video/index/484130/what-do-black-holes-sound-like/',
1658 'md5': 'a3e0df96369831de324f0778e126653c',
1659 'info_dict': {
1660 'id': '4909620399001',
1661 'ext': 'mp4',
1662 'title': 'What Do Black Holes Sound Like?',
1663 'description': 'what do black holes sound like',
1664 'upload_date': '20160524',
1665 'uploader_id': '29913724001',
1666 'timestamp': 1464107587,
1667 'uploader': 'TheAtlantic',
1668 },
1669 'add_ie': ['BrightcoveLegacy'],
1670 },
1671 # Facebook <iframe> embed
1672 {
1673 'url': 'https://www.hostblogger.de/blog/archives/6181-Auto-jagt-Betonmischer.html',
1674 'md5': 'fbcde74f534176ecb015849146dd3aee',
1675 'info_dict': {
1676 'id': '599637780109885',
1677 'ext': 'mp4',
1678 'title': 'Facebook video #599637780109885',
1679 },
1680 },
1681 # Facebook <iframe> embed, plugin video
1682 {
1683 'url': 'http://5pillarsuk.com/2017/06/07/tariq-ramadan-disagrees-with-pr-exercise-by-imams-refusing-funeral-prayers-for-london-attackers/',
1684 'info_dict': {
1685 'id': '1754168231264132',
1686 'ext': 'mp4',
1687 'title': 'About the Imams and Religious leaders refusing to perform funeral prayers for...',
1688 'uploader': 'Tariq Ramadan (official)',
1689 'timestamp': 1496758379,
1690 'upload_date': '20170606',
1691 },
1692 'params': {
1693 'skip_download': True,
1694 },
1695 },
1696 # Facebook API embed
1697 {
1698 'url': 'http://www.lothype.com/blue-stars-2016-preview-standstill-full-show/',
1699 'md5': 'a47372ee61b39a7b90287094d447d94e',
1700 'info_dict': {
1701 'id': '10153467542406923',
1702 'ext': 'mp4',
1703 'title': 'Facebook video #10153467542406923',
1704 },
1705 },
1706 # Wordpress "YouTube Video Importer" plugin
1707 {
1708 'url': 'http://www.lothype.com/blue-devils-drumline-stanford-lot-2016/',
1709 'md5': 'd16797741b560b485194eddda8121b48',
1710 'info_dict': {
1711 'id': 'HNTXWDXV9Is',
1712 'ext': 'mp4',
1713 'title': 'Blue Devils Drumline Stanford lot 2016',
1714 'upload_date': '20160627',
1715 'uploader_id': 'GENOCIDE8GENERAL10',
1716 'uploader': 'cylus cyrus',
1717 },
1718 },
1719 {
1720 # video stored on custom kaltura server
1721 'url': 'http://www.expansion.com/multimedia/videos.html?media=EQcM30NHIPv',
1722 'md5': '537617d06e64dfed891fa1593c4b30cc',
1723 'info_dict': {
1724 'id': '0_1iotm5bh',
1725 'ext': 'mp4',
1726 'title': 'Elecciones británicas: 5 lecciones para Rajoy',
1727 'description': 'md5:435a89d68b9760b92ce67ed227055f16',
1728 'uploader_id': 'videos.expansion@el-mundo.net',
1729 'upload_date': '20150429',
1730 'timestamp': 1430303472,
1731 },
1732 'add_ie': ['Kaltura'],
1733 },
1734 {
1735 # multiple kaltura embeds, nsfw
1736 'url': 'https://www.quartier-rouge.be/prive/femmes/kamila-avec-video-jaime-sadomie.html',
1737 'info_dict': {
1738 'id': 'kamila-avec-video-jaime-sadomie',
1739 'title': "Kamila avec vídeo “J'aime sadomie”",
1740 },
1741 'playlist_count': 8,
1742 },
1743 {
1744 # Non-standard Vimeo embed
1745 'url': 'https://openclassrooms.com/courses/understanding-the-web',
1746 'md5': '64d86f1c7d369afd9a78b38cbb88d80a',
1747 'info_dict': {
1748 'id': '148867247',
1749 'ext': 'mp4',
1750 'title': 'Understanding the web - Teaser',
1751 'description': 'This is "Understanding the web - Teaser" by openclassrooms on Vimeo, the home for high quality videos and the people who love them.',
1752 'upload_date': '20151214',
1753 'uploader': 'OpenClassrooms',
1754 'uploader_id': 'openclassrooms',
1755 },
1756 'add_ie': ['Vimeo'],
1757 },
1758 {
1759 # generic vimeo embed that requires original URL passed as Referer
1760 'url': 'http://racing4everyone.eu/2016/07/30/formula-1-2016-round12-germany/',
1761 'only_matching': True,
1762 },
1763 {
1764 'url': 'https://support.arkena.com/display/PLAY/Ways+to+embed+your+video',
1765 'md5': 'b96f2f71b359a8ecd05ce4e1daa72365',
1766 'info_dict': {
1767 'id': 'b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe',
1768 'ext': 'mp4',
1769 'title': 'Big Buck Bunny',
1770 'description': 'Royalty free test video',
1771 'timestamp': 1432816365,
1772 'upload_date': '20150528',
1773 'is_live': False,
1774 },
1775 'params': {
1776 'skip_download': True,
1777 },
1778 'add_ie': [ArkenaIE.ie_key()],
1779 },
1780 {
1781 'url': 'http://nova.bg/news/view/2016/08/16/156543/%D0%BD%D0%B0-%D0%BA%D0%BE%D1%81%D1%8A%D0%BC-%D0%BE%D1%82-%D0%B2%D0%B7%D1%80%D0%B8%D0%B2-%D0%BE%D1%82%D1%86%D0%B5%D0%BF%D0%B8%D1%85%D0%B0-%D1%86%D1%8F%D0%BB-%D0%BA%D0%B2%D0%B0%D1%80%D1%82%D0%B0%D0%BB-%D0%B7%D0%B0%D1%80%D0%B0%D0%B4%D0%B8-%D0%B8%D0%B7%D1%82%D0%B8%D1%87%D0%B0%D0%BD%D0%B5-%D0%BD%D0%B0-%D0%B3%D0%B0%D0%B7-%D0%B2-%D0%BF%D0%BB%D0%BE%D0%B2%D0%B4%D0%B8%D0%B2/',
1782 'info_dict': {
1783 'id': '1c7141f46c',
1784 'ext': 'mp4',
1785 'title': 'НА КОСЪМ ОТ ВЗРИВ: Изтичане на газ на бензиностанция в Пловдив',
1786 },
1787 'params': {
1788 'skip_download': True,
1789 },
1790 'add_ie': [Vbox7IE.ie_key()],
1791 },
1792 {
1793 # DBTV embeds
1794 'url': 'http://www.dagbladet.no/2016/02/23/nyheter/nordlys/ski/troms/ver/43254897/',
1795 'info_dict': {
1796 'id': '43254897',
1797 'title': 'Etter ett års planlegging, klaffet endelig alt: - Jeg måtte ta en liten dans',
1798 },
1799 'playlist_mincount': 3,
1800 },
1801 {
1802 # Videa embeds
1803 'url': 'http://forum.dvdtalk.com/movie-talk/623756-deleted-magic-star-wars-ot-deleted-alt-scenes-docu-style.html',
1804 'info_dict': {
1805 'id': '623756-deleted-magic-star-wars-ot-deleted-alt-scenes-docu-style',
1806 'title': 'Deleted Magic - Star Wars: OT Deleted / Alt. Scenes Docu. Style - DVD Talk Forum',
1807 },
1808 'playlist_mincount': 2,
1809 },
1810 {
1811 # 20 minuten embed
1812 'url': 'http://www.20min.ch/schweiz/news/story/So-kommen-Sie-bei-Eis-und-Schnee-sicher-an-27032552',
1813 'info_dict': {
1814 'id': '523629',
1815 'ext': 'mp4',
1816 'title': 'So kommen Sie bei Eis und Schnee sicher an',
1817 'description': 'md5:117c212f64b25e3d95747e5276863f7d',
1818 },
1819 'params': {
1820 'skip_download': True,
1821 },
1822 'add_ie': [TwentyMinutenIE.ie_key()],
1823 },
1824 {
1825 # VideoPress embed
1826 'url': 'https://en.support.wordpress.com/videopress/',
1827 'info_dict': {
1828 'id': 'OcobLTqC',
1829 'ext': 'm4v',
1830 'title': 'IMG_5786',
1831 'timestamp': 1435711927,
1832 'upload_date': '20150701',
1833 },
1834 'params': {
1835 'skip_download': True,
1836 },
1837 'add_ie': [VideoPressIE.ie_key()],
1838 },
1839 {
1840 # Rutube embed
1841 'url': 'http://magazzino.friday.ru/videos/vipuski/kazan-2',
1842 'info_dict': {
1843 'id': '9b3d5bee0a8740bf70dfd29d3ea43541',
1844 'ext': 'flv',
1845 'title': 'Магаззино: Казань 2',
1846 'description': 'md5:99bccdfac2269f0e8fdbc4bbc9db184a',
1847 'uploader': 'Магаззино',
1848 'upload_date': '20170228',
1849 'uploader_id': '996642',
1850 },
1851 'params': {
1852 'skip_download': True,
1853 },
1854 'add_ie': [RutubeIE.ie_key()],
1855 },
1856 {
1857 # ThePlatform embedded with whitespaces in URLs
1858 'url': 'http://www.golfchannel.com/topics/shows/golftalkcentral.htm',
1859 'only_matching': True,
1860 },
1861 {
1862 # Senate ISVP iframe https
1863 'url': 'https://www.hsgac.senate.gov/hearings/canadas-fast-track-refugee-plan-unanswered-questions-and-implications-for-us-national-security',
1864 'md5': 'fb8c70b0b515e5037981a2492099aab8',
1865 'info_dict': {
1866 'id': 'govtaff020316',
1867 'ext': 'mp4',
1868 'title': 'Integrated Senate Video Player',
1869 },
1870 'add_ie': [SenateISVPIE.ie_key()],
1871 },
1872 {
1873 # Limelight embeds (1 channel embed + 4 media embeds)
1874 'url': 'http://www.sedona.com/FacilitatorTraining2017',
1875 'info_dict': {
1876 'id': 'FacilitatorTraining2017',
1877 'title': 'Facilitator Training 2017',
1878 },
1879 'playlist_mincount': 5,
1880 },
1881 {
1882 # Limelight embed (LimelightPlayerUtil.embed)
1883 'url': 'https://tv5.ca/videos?v=xuu8qowr291ri',
1884 'info_dict': {
1885 'id': '95d035dc5c8a401588e9c0e6bd1e9c92',
1886 'ext': 'mp4',
1887 'title': '07448641',
1888 'timestamp': 1499890639,
1889 'upload_date': '20170712',
1890 },
1891 'params': {
1892 'skip_download': True,
1893 },
1894 'add_ie': ['LimelightMedia'],
1895 },
1896 {
1897 'url': 'http://kron4.com/2017/04/28/standoff-with-walnut-creek-murder-suspect-ends-with-arrest/',
1898 'info_dict': {
1899 'id': 'standoff-with-walnut-creek-murder-suspect-ends-with-arrest',
1900 'title': 'Standoff with Walnut Creek murder suspect ends',
1901 'description': 'md5:3ccc48a60fc9441eeccfc9c469ebf788',
1902 },
1903 'playlist_mincount': 4,
1904 },
1905 {
1906 # WashingtonPost embed
1907 'url': 'http://www.vanityfair.com/hollywood/2017/04/donald-trump-tv-pitches',
1908 'info_dict': {
1909 'id': '8caf6e88-d0ec-11e5-90d3-34c2c42653ac',
1910 'ext': 'mp4',
1911 'title': "No one has seen the drama series based on Trump's life \u2014 until now",
1912 'description': 'Donald Trump wanted a weekly TV drama based on his life. It never aired. But The Washington Post recently obtained a scene from the pilot script — and enlisted actors.',
1913 'timestamp': 1455216756,
1914 'uploader': 'The Washington Post',
1915 'upload_date': '20160211',
1916 },
1917 'add_ie': [WashingtonPostIE.ie_key()],
1918 },
1919 {
1920 # Mediaset embed
1921 'url': 'http://www.tgcom24.mediaset.it/politica/serracchiani-voglio-vivere-in-una-societa-aperta-reazioni-sproporzionate-_3071354-201702a.shtml',
1922 'info_dict': {
1923 'id': '720642',
1924 'ext': 'mp4',
1925 'title': 'Serracchiani: "Voglio vivere in una società aperta, con tutela del patto di fiducia"',
1926 },
1927 'params': {
1928 'skip_download': True,
1929 },
1930 'add_ie': [MediasetIE.ie_key()],
1931 },
1932 {
1933 # JOJ.sk embeds
1934 'url': 'https://www.noviny.sk/slovensko/238543-slovenskom-sa-prehnala-vlna-silnych-burok',
1935 'info_dict': {
1936 'id': '238543-slovenskom-sa-prehnala-vlna-silnych-burok',
1937 'title': 'Slovenskom sa prehnala vlna silných búrok',
1938 },
1939 'playlist_mincount': 5,
1940 'add_ie': [JojIE.ie_key()],
1941 },
1942 {
1943 # AMP embed (see https://www.ampproject.org/docs/reference/components/amp-video)
1944 'url': 'https://tvrain.ru/amp/418921/',
1945 'md5': 'cc00413936695987e8de148b67d14f1d',
1946 'info_dict': {
1947 'id': '418921',
1948 'ext': 'mp4',
1949 'title': 'Стас Намин: «Мы нарушили девственность Кремля»',
1950 },
1951 },
1952 {
1953 # vzaar embed
1954 'url': 'http://help.vzaar.com/article/165-embedding-video',
1955 'md5': '7e3919d9d2620b89e3e00bec7fe8c9d4',
1956 'info_dict': {
1957 'id': '8707641',
1958 'ext': 'mp4',
1959 'title': 'Building A Business Online: Principal Chairs Q & A',
1960 },
1961 },
1962 {
1963 # multiple HTML5 videos on one page
1964 'url': 'https://www.paragon-software.com/home/rk-free/keyscenarios.html',
1965 'info_dict': {
1966 'id': 'keyscenarios',
1967 'title': 'Rescue Kit 14 Free Edition - Getting started',
1968 },
1969 'playlist_count': 4,
1970 },
1971 {
1972 # vshare embed
1973 'url': 'https://youtube-dl-demo.neocities.org/vshare.html',
1974 'md5': '17b39f55b5497ae8b59f5fbce8e35886',
1975 'info_dict': {
1976 'id': '0f64ce6',
1977 'title': 'vl14062007715967',
1978 'ext': 'mp4',
1979 }
1980 },
1981 {
1982 'url': 'http://www.heidelberg-laureate-forum.org/blog/video/lecture-friday-september-23-2016-sir-c-antony-r-hoare/',
1983 'md5': 'aecd089f55b1cb5a59032cb049d3a356',
1984 'info_dict': {
1985 'id': '90227f51a80c4d8f86c345a7fa62bd9a1d',
1986 'ext': 'mp4',
1987 'title': 'Lecture: Friday, September 23, 2016 - Sir Tony Hoare',
1988 'description': 'md5:5a51db84a62def7b7054df2ade403c6c',
1989 'timestamp': 1474354800,
1990 'upload_date': '20160920',
1991 }
1992 },
1993 {
1994 'url': 'http://www.kidzworld.com/article/30935-trolls-the-beat-goes-on-interview-skylar-astin-and-amanda-leighton',
1995 'info_dict': {
1996 'id': '1731611',
1997 'ext': 'mp4',
1998 'title': 'Official Trailer | TROLLS: THE BEAT GOES ON!',
1999 'description': 'md5:eb5f23826a027ba95277d105f248b825',
2000 'timestamp': 1516100691,
2001 'upload_date': '20180116',
2002 },
2003 'params': {
2004 'skip_download': True,
2005 },
2006 'add_ie': [SpringboardPlatformIE.ie_key()],
2007 },
2008 {
2009 'url': 'https://www.yapfiles.ru/show/1872528/690b05d3054d2dbe1e69523aa21bb3b1.mp4.html',
2010 'info_dict': {
2011 'id': 'vMDE4NzI1Mjgt690b',
2012 'ext': 'mp4',
2013 'title': 'Котята',
2014 },
2015 'add_ie': [YapFilesIE.ie_key()],
2016 'params': {
2017 'skip_download': True,
2018 },
2019 },
2020 {
2021 # CloudflareStream embed
2022 'url': 'https://www.cloudflare.com/products/cloudflare-stream/',
2023 'info_dict': {
2024 'id': '31c9291ab41fac05471db4e73aa11717',
2025 'ext': 'mp4',
2026 'title': '31c9291ab41fac05471db4e73aa11717',
2027 },
2028 'add_ie': [CloudflareStreamIE.ie_key()],
2029 'params': {
2030 'skip_download': True,
2031 },
2032 },
2033 {
2034 # PeerTube embed
2035 'url': 'https://joinpeertube.org/fr/home/',
2036 'info_dict': {
2037 'id': 'home',
2038 'title': 'Reprenez le contrôle de vos vidéos ! #JoinPeertube',
2039 },
2040 'playlist_count': 2,
2041 },
2042 {
2043 # Indavideo embed
2044 'url': 'https://streetkitchen.hu/receptek/igy_kell_otthon_hamburgert_sutni/',
2045 'info_dict': {
2046 'id': '1693903',
2047 'ext': 'mp4',
2048 'title': 'Így kell otthon hamburgert sütni',
2049 'description': 'md5:f5a730ecf900a5c852e1e00540bbb0f7',
2050 'timestamp': 1426330212,
2051 'upload_date': '20150314',
2052 'uploader': 'StreetKitchen',
2053 'uploader_id': '546363',
2054 },
2055 'add_ie': [IndavideoEmbedIE.ie_key()],
2056 'params': {
2057 'skip_download': True,
2058 },
2059 },
2060 {
2061 # APA embed via JWPlatform embed
2062 'url': 'http://www.vol.at/blue-man-group/5593454',
2063 'info_dict': {
2064 'id': 'jjv85FdZ',
2065 'ext': 'mp4',
2066 'title': '"Blau ist mysteriös": Die Blue Man Group im Interview',
2067 'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
2068 'thumbnail': r're:^https?://.*\.jpg$',
2069 'duration': 254,
2070 'timestamp': 1519211149,
2071 'upload_date': '20180221',
2072 },
2073 'params': {
2074 'skip_download': True,
2075 },
2076 },
2077 {
2078 'url': 'http://share-videos.se/auto/video/83645793?uid=13',
2079 'md5': 'b68d276de422ab07ee1d49388103f457',
2080 'info_dict': {
2081 'id': '83645793',
2082 'title': 'Lock up and get excited',
2083 'ext': 'mp4'
2084 },
2085 'skip': 'TODO: fix nested playlists processing in tests',
2086 },
2087 {
2088 # Viqeo embeds
2089 'url': 'https://viqeo.tv/',
2090 'info_dict': {
2091 'id': 'viqeo',
2092 'title': 'All-new video platform',
2093 },
2094 'playlist_count': 6,
2095 },
2096 {
2097 # Squarespace video embed, 2019-08-28
2098 'url': 'http://ootboxford.com',
2099 'info_dict': {
2100 'id': 'Tc7b_JGdZfw',
2101 'title': 'Out of the Blue, at Childish Things 10',
2102 'ext': 'mp4',
2103 'description': 'md5:a83d0026666cf5ee970f8bd1cfd69c7f',
2104 'uploader_id': 'helendouglashouse',
2105 'uploader': 'Helen & Douglas House',
2106 'upload_date': '20140328',
2107 },
2108 'params': {
2109 'skip_download': True,
2110 },
2111 },
2112 # {
2113 # # Zype embed
2114 # 'url': 'https://www.cookscountry.com/episode/554-smoky-barbecue-favorites',
2115 # 'info_dict': {
2116 # 'id': '5b400b834b32992a310622b9',
2117 # 'ext': 'mp4',
2118 # 'title': 'Smoky Barbecue Favorites',
2119 # 'thumbnail': r're:^https?://.*\.jpe?g',
2120 # 'description': 'md5:5ff01e76316bd8d46508af26dc86023b',
2121 # 'upload_date': '20170909',
2122 # 'timestamp': 1504915200,
2123 # },
2124 # 'add_ie': [ZypeIE.ie_key()],
2125 # 'params': {
2126 # 'skip_download': True,
2127 # },
2128 # },
2129 {
2130 # videojs embed
2131 'url': 'https://video.sibnet.ru/shell.php?videoid=3422904',
2132 'info_dict': {
2133 'id': 'shell',
2134 'ext': 'mp4',
2135 'title': 'Доставщик пиццы спросил разрешения сыграть на фортепиано',
2136 'description': 'md5:89209cdc587dab1e4a090453dbaa2cb1',
2137 'thumbnail': r're:^https?://.*\.jpg$',
2138 },
2139 'params': {
2140 'skip_download': True,
2141 },
2142 'expected_warnings': ['Failed to download MPD manifest'],
2143 },
2144 {
2145 # DailyMotion embed with DM.player
2146 'url': 'https://www.beinsports.com/us/copa-del-rey/video/the-locker-room-valencia-beat-barca-in-copa/1203804',
2147 'info_dict': {
2148 'id': 'k6aKkGHd9FJs4mtJN39',
2149 'ext': 'mp4',
2150 'title': 'The Locker Room: Valencia Beat Barca In Copa del Rey Final',
2151 'description': 'This video is private.',
2152 'uploader_id': 'x1jf30l',
2153 'uploader': 'beIN SPORTS USA',
2154 'upload_date': '20190528',
2155 'timestamp': 1559062971,
2156 },
2157 'params': {
2158 'skip_download': True,
2159 },
2160 },
2161 # {
2162 # # TODO: find another test
2163 # # http://schema.org/VideoObject
2164 # 'url': 'https://flipagram.com/f/nyvTSJMKId',
2165 # 'md5': '888dcf08b7ea671381f00fab74692755',
2166 # 'info_dict': {
2167 # 'id': 'nyvTSJMKId',
2168 # 'ext': 'mp4',
2169 # 'title': 'Flipagram by sjuria101 featuring Midnight Memories by One Direction',
2170 # 'description': '#love for cats.',
2171 # 'timestamp': 1461244995,
2172 # 'upload_date': '20160421',
2173 # },
2174 # 'params': {
2175 # 'force_generic_extractor': True,
2176 # },
2177 # },
2178 {
2179 # VHX Embed
2180 'url': 'https://demo.vhx.tv/category-c/videos/file-example-mp4-480-1-5mg-copy',
2181 'info_dict': {
2182 'id': '858208',
2183 'ext': 'mp4',
2184 'title': 'Untitled',
2185 'uploader_id': 'user80538407',
2186 'uploader': 'OTT Videos',
2187 },
2188 },
2189 {
2190 # ArcPublishing PoWa video player
2191 'url': 'https://www.adn.com/politics/2020/11/02/video-senate-candidates-campaign-in-anchorage-on-eve-of-election-day/',
2192 'md5': 'b03b2fac8680e1e5a7cc81a5c27e71b3',
2193 'info_dict': {
2194 'id': '8c99cb6e-b29c-4bc9-9173-7bf9979225ab',
2195 'ext': 'mp4',
2196 'title': 'Senate candidates wave to voters on Anchorage streets',
2197 'description': 'md5:91f51a6511f090617353dc720318b20e',
2198 'timestamp': 1604378735,
2199 'upload_date': '20201103',
2200 'duration': 1581,
2201 },
2202 },
2203 {
2204 # MyChannels SDK embed
2205 # https://www.24kitchen.nl/populair/deskundige-dit-waarom-sommigen-gevoelig-zijn-voor-voedselallergieen
2206 'url': 'https://www.demorgen.be/nieuws/burgemeester-rotterdam-richt-zich-in-videoboodschap-tot-relschoppers-voelt-het-goed~b0bcfd741/',
2207 'md5': '90c0699c37006ef18e198c032d81739c',
2208 'info_dict': {
2209 'id': '194165',
2210 'ext': 'mp4',
2211 'title': 'Burgemeester Aboutaleb spreekt relschoppers toe',
2212 'timestamp': 1611740340,
2213 'upload_date': '20210127',
2214 'duration': 159,
2215 },
2216 },
2217 {
2218 # Simplecast player embed
2219 'url': 'https://www.bio.org/podcast',
2220 'info_dict': {
2221 'id': 'podcast',
2222 'title': 'I AM BIO Podcast | BIO',
2223 },
2224 'playlist_mincount': 52,
2225 },
2226 {
2227 # Sibnet embed (https://help.sibnet.ru/?sibnet_video_embed)
2228 'url': 'https://phpbb3.x-tk.ru/bbcode-video-sibnet-t24.html',
2229 'only_matching': True,
2230 },
2231 ]
2232
2233 def report_following_redirect(self, new_url):
2234 """Report information extraction."""
2235 self._downloader.to_screen('[redirect] Following redirect to %s' % new_url)
2236
2237 def _extract_rss(self, url, video_id, doc):
2238 playlist_title = doc.find('./channel/title').text
2239 playlist_desc_el = doc.find('./channel/description')
2240 playlist_desc = None if playlist_desc_el is None else playlist_desc_el.text
2241
2242 NS_MAP = {
2243 'itunes': 'http://www.itunes.com/dtds/podcast-1.0.dtd',
2244 }
2245
2246 entries = []
2247 for it in doc.findall('./channel/item'):
2248 next_url = None
2249 enclosure_nodes = it.findall('./enclosure')
2250 for e in enclosure_nodes:
2251 next_url = e.attrib.get('url')
2252 if next_url:
2253 break
2254
2255 if not next_url:
2256 next_url = xpath_text(it, 'link', fatal=False)
2257
2258 if not next_url:
2259 continue
2260
2261 def itunes(key):
2262 return xpath_text(
2263 it, xpath_with_ns('./itunes:%s' % key, NS_MAP),
2264 default=None)
2265
2266 duration = itunes('duration')
2267 explicit = (itunes('explicit') or '').lower()
2268 if explicit in ('true', 'yes'):
2269 age_limit = 18
2270 elif explicit in ('false', 'no'):
2271 age_limit = 0
2272 else:
2273 age_limit = None
2274
2275 entries.append({
2276 '_type': 'url_transparent',
2277 'url': next_url,
2278 'title': it.find('title').text,
2279 'description': xpath_text(it, 'description', default=None),
2280 'timestamp': unified_timestamp(
2281 xpath_text(it, 'pubDate', default=None)),
2282 'duration': int_or_none(duration) or parse_duration(duration),
2283 'thumbnail': url_or_none(xpath_attr(it, xpath_with_ns('./itunes:image', NS_MAP), 'href')),
2284 'episode': itunes('title'),
2285 'episode_number': int_or_none(itunes('episode')),
2286 'season_number': int_or_none(itunes('season')),
2287 'age_limit': age_limit,
2288 })
2289
2290 return {
2291 '_type': 'playlist',
2292 'id': url,
2293 'title': playlist_title,
2294 'description': playlist_desc,
2295 'entries': entries,
2296 }
2297
2298 def _extract_camtasia(self, url, video_id, webpage):
2299 """ Returns None if no camtasia video can be found. """
2300
2301 camtasia_cfg = self._search_regex(
2302 r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);',
2303 webpage, 'camtasia configuration file', default=None)
2304 if camtasia_cfg is None:
2305 return None
2306
2307 title = self._html_search_meta('DC.title', webpage, fatal=True)
2308
2309 camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg)
2310 camtasia_cfg = self._download_xml(
2311 camtasia_url, video_id,
2312 note='Downloading camtasia configuration',
2313 errnote='Failed to download camtasia configuration')
2314 fileset_node = camtasia_cfg.find('./playlist/array/fileset')
2315
2316 entries = []
2317 for n in fileset_node.getchildren():
2318 url_n = n.find('./uri')
2319 if url_n is None:
2320 continue
2321
2322 entries.append({
2323 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0],
2324 'title': '%s - %s' % (title, n.tag),
2325 'url': compat_urlparse.urljoin(url, url_n.text),
2326 'duration': float_or_none(n.find('./duration').text),
2327 })
2328
2329 return {
2330 '_type': 'playlist',
2331 'entries': entries,
2332 'title': title,
2333 }
2334
2335 def _real_extract(self, url):
2336 if url.startswith('//'):
2337 return self.url_result(self.http_scheme() + url)
2338
2339 parsed_url = compat_urlparse.urlparse(url)
2340 if not parsed_url.scheme:
2341 default_search = self._downloader.params.get('default_search')
2342 if default_search is None:
2343 default_search = 'fixup_error'
2344
2345 if default_search in ('auto', 'auto_warning', 'fixup_error'):
2346 if re.match(r'^[^\s/]+\.[^\s/]+/', url):
2347 self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http')
2348 return self.url_result('http://' + url)
2349 elif default_search != 'fixup_error':
2350 if default_search == 'auto_warning':
2351 if re.match(r'^(?:url|URL)$', url):
2352 raise ExtractorError(
2353 'Invalid URL: %r . Call youtube-dl like this: youtube-dl -v "https://www.youtube.com/watch?v=BaW_jenozKc" ' % url,
2354 expected=True)
2355 else:
2356 self._downloader.report_warning(
2357 'Falling back to youtube search for %s . Set --default-search "auto" to suppress this warning.' % url)
2358 return self.url_result('ytsearch:' + url)
2359
2360 if default_search in ('error', 'fixup_error'):
2361 raise ExtractorError(
2362 '%r is not a valid URL. '
2363 'Set --default-search "ytsearch" (or run youtube-dl "ytsearch:%s" ) to search YouTube'
2364 % (url, url), expected=True)
2365 else:
2366 if ':' not in default_search:
2367 default_search += ':'
2368 return self.url_result(default_search + url)
2369
2370 url, smuggled_data = unsmuggle_url(url)
2371 force_videoid = None
2372 is_intentional = smuggled_data and smuggled_data.get('to_generic')
2373 if smuggled_data and 'force_videoid' in smuggled_data:
2374 force_videoid = smuggled_data['force_videoid']
2375 video_id = force_videoid
2376 else:
2377 video_id = self._generic_id(url)
2378
2379 self.to_screen('%s: Requesting header' % video_id)
2380
2381 head_req = HEADRequest(url)
2382 head_response = self._request_webpage(
2383 head_req, video_id,
2384 note=False, errnote='Could not send HEAD request to %s' % url,
2385 fatal=False)
2386
2387 if head_response is not False:
2388 # Check for redirect
2389 new_url = head_response.geturl()
2390 if url != new_url:
2391 self.report_following_redirect(new_url)
2392 if force_videoid:
2393 new_url = smuggle_url(
2394 new_url, {'force_videoid': force_videoid})
2395 return self.url_result(new_url)
2396
2397 full_response = None
2398 if head_response is False:
2399 request = sanitized_Request(url)
2400 request.add_header('Accept-Encoding', '*')
2401 full_response = self._request_webpage(request, video_id)
2402 head_response = full_response
2403
2404 info_dict = {
2405 'id': video_id,
2406 'title': self._generic_title(url),
2407 'timestamp': unified_timestamp(head_response.headers.get('Last-Modified'))
2408 }
2409
2410 # Check for direct link to a video
2411 content_type = head_response.headers.get('Content-Type', '').lower()
2412 m = re.match(r'^(?P<type>audio|video|application(?=/(?:ogg$|(?:vnd\.apple\.|x-)?mpegurl)))/(?P<format_id>[^;\s]+)', content_type)
2413 if m:
2414 format_id = compat_str(m.group('format_id'))
2415 if format_id.endswith('mpegurl'):
2416 formats = self._extract_m3u8_formats(url, video_id, 'mp4')
2417 elif format_id == 'f4m':
2418 formats = self._extract_f4m_formats(url, video_id)
2419 else:
2420 formats = [{
2421 'format_id': format_id,
2422 'url': url,
2423 'vcodec': 'none' if m.group('type') == 'audio' else None
2424 }]
2425 info_dict['direct'] = True
2426 self._sort_formats(formats)
2427 info_dict['formats'] = formats
2428 return info_dict
2429
2430 if not self._downloader.params.get('test', False) and not is_intentional:
2431 force = self._downloader.params.get('force_generic_extractor', False)
2432 self._downloader.report_warning(
2433 '%s on generic information extractor.' % ('Forcing' if force else 'Falling back'))
2434
2435 if not full_response:
2436 request = sanitized_Request(url)
2437 # Some webservers may serve compressed content of rather big size (e.g. gzipped flac)
2438 # making it impossible to download only chunk of the file (yet we need only 512kB to
2439 # test whether it's HTML or not). According to youtube-dl default Accept-Encoding
2440 # that will always result in downloading the whole file that is not desirable.
2441 # Therefore for extraction pass we have to override Accept-Encoding to any in order
2442 # to accept raw bytes and being able to download only a chunk.
2443 # It may probably better to solve this by checking Content-Type for application/octet-stream
2444 # after HEAD request finishes, but not sure if we can rely on this.
2445 request.add_header('Accept-Encoding', '*')
2446 full_response = self._request_webpage(request, video_id)
2447
2448 first_bytes = full_response.read(512)
2449
2450 # Is it an M3U playlist?
2451 if first_bytes.startswith(b'#EXTM3U'):
2452 info_dict['formats'] = self._extract_m3u8_formats(url, video_id, 'mp4')
2453 self._sort_formats(info_dict['formats'])
2454 return info_dict
2455
2456 # Maybe it's a direct link to a video?
2457 # Be careful not to download the whole thing!
2458 if not is_html(first_bytes):
2459 self._downloader.report_warning(
2460 'URL could be a direct video link, returning it as such.')
2461 info_dict.update({
2462 'direct': True,
2463 'url': url,
2464 })
2465 return info_dict
2466
2467 webpage = self._webpage_read_content(
2468 full_response, url, video_id, prefix=first_bytes)
2469
2470 if '<title>DPG Media Privacy Gate</title>' in webpage:
2471 webpage = self._download_webpage(url, video_id)
2472
2473 self.report_extraction(video_id)
2474
2475 # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest?
2476 try:
2477 doc = compat_etree_fromstring(webpage.encode('utf-8'))
2478 if doc.tag == 'rss':
2479 return self._extract_rss(url, video_id, doc)
2480 elif doc.tag == 'SmoothStreamingMedia':
2481 info_dict['formats'] = self._parse_ism_formats(doc, url)
2482 self._sort_formats(info_dict['formats'])
2483 return info_dict
2484 elif re.match(r'^(?:{[^}]+})?smil$', doc.tag):
2485 smil = self._parse_smil(doc, url, video_id)
2486 self._sort_formats(smil['formats'])
2487 return smil
2488 elif doc.tag == '{http://xspf.org/ns/0/}playlist':
2489 return self.playlist_result(
2490 self._parse_xspf(
2491 doc, video_id, xspf_url=url,
2492 xspf_base_url=full_response.geturl()),
2493 video_id)
2494 elif re.match(r'(?i)^(?:{[^}]+})?MPD$', doc.tag):
2495 info_dict['formats'] = self._parse_mpd_formats(
2496 doc,
2497 mpd_base_url=full_response.geturl().rpartition('/')[0],
2498 mpd_url=url)
2499 self._sort_formats(info_dict['formats'])
2500 return info_dict
2501 elif re.match(r'^{http://ns\.adobe\.com/f4m/[12]\.0}manifest$', doc.tag):
2502 info_dict['formats'] = self._parse_f4m_formats(doc, url, video_id)
2503 self._sort_formats(info_dict['formats'])
2504 return info_dict
2505 except compat_xml_parse_error:
2506 pass
2507
2508 # Is it a Camtasia project?
2509 camtasia_res = self._extract_camtasia(url, video_id, webpage)
2510 if camtasia_res is not None:
2511 return camtasia_res
2512
2513 # Sometimes embedded video player is hidden behind percent encoding
2514 # (e.g. https://github.com/ytdl-org/youtube-dl/issues/2448)
2515 # Unescaping the whole page allows to handle those cases in a generic way
2516 # FIXME: unescaping the whole page may break URLs, commenting out for now.
2517 # There probably should be a second run of generic extractor on unescaped webpage.
2518 # webpage = compat_urllib_parse_unquote(webpage)
2519
2520 # Unescape squarespace embeds to be detected by generic extractor,
2521 # see https://github.com/ytdl-org/youtube-dl/issues/21294
2522 webpage = re.sub(
2523 r'<div[^>]+class=[^>]*?\bsqs-video-wrapper\b[^>]*>',
2524 lambda x: unescapeHTML(x.group(0)), webpage)
2525
2526 # it's tempting to parse this further, but you would
2527 # have to take into account all the variations like
2528 # Video Title - Site Name
2529 # Site Name | Video Title
2530 # Video Title - Tagline | Site Name
2531 # and so on and so forth; it's just not practical
2532 video_title = self._og_search_title(
2533 webpage, default=None) or self._html_search_regex(
2534 r'(?s)<title>(.*?)</title>', webpage, 'video title',
2535 default='video')
2536
2537 # Try to detect age limit automatically
2538 age_limit = self._rta_search(webpage)
2539 # And then there are the jokers who advertise that they use RTA,
2540 # but actually don't.
2541 AGE_LIMIT_MARKERS = [
2542 r'Proudly Labeled <a href="http://www\.rtalabel\.org/" title="Restricted to Adults">RTA</a>',
2543 ]
2544 if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
2545 age_limit = 18
2546
2547 # video uploader is domain name
2548 video_uploader = self._search_regex(
2549 r'^(?:https?://)?([^/]*)/.*', url, 'video uploader')
2550
2551 video_description = self._og_search_description(webpage, default=None)
2552 video_thumbnail = self._og_search_thumbnail(webpage, default=None)
2553
2554 info_dict.update({
2555 'title': video_title,
2556 'description': video_description,
2557 'thumbnail': video_thumbnail,
2558 'age_limit': age_limit,
2559 })
2560
2561 # Look for Brightcove Legacy Studio embeds
2562 bc_urls = BrightcoveLegacyIE._extract_brightcove_urls(webpage)
2563 if bc_urls:
2564 entries = [{
2565 '_type': 'url',
2566 'url': smuggle_url(bc_url, {'Referer': url}),
2567 'ie_key': 'BrightcoveLegacy'
2568 } for bc_url in bc_urls]
2569
2570 return {
2571 '_type': 'playlist',
2572 'title': video_title,
2573 'id': video_id,
2574 'entries': entries,
2575 }
2576
2577 # Look for Brightcove New Studio embeds
2578 bc_urls = BrightcoveNewIE._extract_urls(self, webpage)
2579 if bc_urls:
2580 return self.playlist_from_matches(
2581 bc_urls, video_id, video_title,
2582 getter=lambda x: smuggle_url(x, {'referrer': url}),
2583 ie='BrightcoveNew')
2584
2585 # Look for Nexx embeds
2586 nexx_urls = NexxIE._extract_urls(webpage)
2587 if nexx_urls:
2588 return self.playlist_from_matches(nexx_urls, video_id, video_title, ie=NexxIE.ie_key())
2589
2590 # Look for Nexx iFrame embeds
2591 nexx_embed_urls = NexxEmbedIE._extract_urls(webpage)
2592 if nexx_embed_urls:
2593 return self.playlist_from_matches(nexx_embed_urls, video_id, video_title, ie=NexxEmbedIE.ie_key())
2594
2595 # Look for ThePlatform embeds
2596 tp_urls = ThePlatformIE._extract_urls(webpage)
2597 if tp_urls:
2598 return self.playlist_from_matches(tp_urls, video_id, video_title, ie='ThePlatform')
2599
2600 arc_urls = ArcPublishingIE._extract_urls(webpage)
2601 if arc_urls:
2602 return self.playlist_from_matches(arc_urls, video_id, video_title, ie=ArcPublishingIE.ie_key())
2603
2604 mychannels_urls = MedialaanIE._extract_urls(webpage)
2605 if mychannels_urls:
2606 return self.playlist_from_matches(
2607 mychannels_urls, video_id, video_title, ie=MedialaanIE.ie_key())
2608
2609 # Look for embedded rtl.nl player
2610 matches = re.findall(
2611 r'<iframe[^>]+?src="((?:https?:)?//(?:(?:www|static)\.)?rtl\.nl/(?:system/videoplayer/[^"]+(?:video_)?)?embed[^"]+)"',
2612 webpage)
2613 if matches:
2614 return self.playlist_from_matches(matches, video_id, video_title, ie='RtlNl')
2615
2616 vimeo_urls = VimeoIE._extract_urls(url, webpage)
2617 if vimeo_urls:
2618 return self.playlist_from_matches(vimeo_urls, video_id, video_title, ie=VimeoIE.ie_key())
2619
2620 vhx_url = VHXEmbedIE._extract_url(webpage)
2621 if vhx_url:
2622 return self.url_result(vhx_url, VHXEmbedIE.ie_key())
2623
2624 vid_me_embed_url = self._search_regex(
2625 r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
2626 webpage, 'vid.me embed', default=None)
2627 if vid_me_embed_url is not None:
2628 return self.url_result(vid_me_embed_url, 'Vidme')
2629
2630 # Look for YouTube embeds
2631 youtube_urls = YoutubeIE._extract_urls(webpage)
2632 if youtube_urls:
2633 return self.playlist_from_matches(
2634 youtube_urls, video_id, video_title, ie=YoutubeIE.ie_key())
2635
2636 matches = DailymotionIE._extract_urls(webpage)
2637 if matches:
2638 return self.playlist_from_matches(matches, video_id, video_title)
2639
2640 # Look for embedded Dailymotion playlist player (#3822)
2641 m = re.search(
2642 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.[a-z]{2,3}/widget/jukebox\?.+?)\1', webpage)
2643 if m:
2644 playlists = re.findall(
2645 r'list\[\]=/playlist/([^/]+)/', unescapeHTML(m.group('url')))
2646 if playlists:
2647 return self.playlist_from_matches(
2648 playlists, video_id, video_title, lambda p: '//dailymotion.com/playlist/%s' % p)
2649
2650 # Look for DailyMail embeds
2651 dailymail_urls = DailyMailIE._extract_urls(webpage)
2652 if dailymail_urls:
2653 return self.playlist_from_matches(
2654 dailymail_urls, video_id, video_title, ie=DailyMailIE.ie_key())
2655
2656 # Look for Teachable embeds, must be before Wistia
2657 teachable_url = TeachableIE._extract_url(webpage, url)
2658 if teachable_url:
2659 return self.url_result(teachable_url)
2660
2661 # Look for embedded Wistia player
2662 wistia_urls = WistiaIE._extract_urls(webpage)
2663 if wistia_urls:
2664 playlist = self.playlist_from_matches(wistia_urls, video_id, video_title, ie=WistiaIE.ie_key())
2665 for entry in playlist['entries']:
2666 entry.update({
2667 '_type': 'url_transparent',
2668 'uploader': video_uploader,
2669 })
2670 return playlist
2671
2672 # Look for SVT player
2673 svt_url = SVTIE._extract_url(webpage)
2674 if svt_url:
2675 return self.url_result(svt_url, 'SVT')
2676
2677 # Look for Bandcamp pages with custom domain
2678 mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
2679 if mobj is not None:
2680 burl = unescapeHTML(mobj.group(1))
2681 # Don't set the extractor because it can be a track url or an album
2682 return self.url_result(burl)
2683
2684 # Look for embedded Vevo player
2685 mobj = re.search(
2686 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage)
2687 if mobj is not None:
2688 return self.url_result(mobj.group('url'))
2689
2690 # Look for embedded Viddler player
2691 mobj = re.search(
2692 r'<(?:iframe[^>]+?src|param[^>]+?value)=(["\'])(?P<url>(?:https?:)?//(?:www\.)?viddler\.com/(?:embed|player)/.+?)\1',
2693 webpage)
2694 if mobj is not None:
2695 return self.url_result(mobj.group('url'))
2696
2697 # Look for NYTimes player
2698 mobj = re.search(
2699 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>',
2700 webpage)
2701 if mobj is not None:
2702 return self.url_result(mobj.group('url'))
2703
2704 # Look for Libsyn player
2705 mobj = re.search(
2706 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//html5-player\.libsyn\.com/embed/.+?)\1', webpage)
2707 if mobj is not None:
2708 return self.url_result(mobj.group('url'))
2709
2710 # Look for Ooyala videos
2711 mobj = (re.search(r'player\.ooyala\.com/[^"?]+[?#][^"]*?(?:embedCode|ec)=(?P<ec>[^"&]+)', webpage)
2712 or re.search(r'OO\.Player\.create\([\'"].*?[\'"],\s*[\'"](?P<ec>.{32})[\'"]', webpage)
2713 or re.search(r'OO\.Player\.create\.apply\(\s*OO\.Player\s*,\s*op\(\s*\[\s*[\'"][^\'"]*[\'"]\s*,\s*[\'"](?P<ec>.{32})[\'"]', webpage)
2714 or re.search(r'SBN\.VideoLinkset\.ooyala\([\'"](?P<ec>.{32})[\'"]\)', webpage)
2715 or re.search(r'data-ooyala-video-id\s*=\s*[\'"](?P<ec>.{32})[\'"]', webpage))
2716 if mobj is not None:
2717 embed_token = self._search_regex(
2718 r'embedToken[\'"]?\s*:\s*[\'"]([^\'"]+)',
2719 webpage, 'ooyala embed token', default=None)
2720 return OoyalaIE._build_url_result(smuggle_url(
2721 mobj.group('ec'), {
2722 'domain': url,
2723 'embed_token': embed_token,
2724 }))
2725
2726 # Look for multiple Ooyala embeds on SBN network websites
2727 mobj = re.search(r'SBN\.VideoLinkset\.entryGroup\((\[.*?\])', webpage)
2728 if mobj is not None:
2729 embeds = self._parse_json(mobj.group(1), video_id, fatal=False)
2730 if embeds:
2731 return self.playlist_from_matches(
2732 embeds, video_id, video_title,
2733 getter=lambda v: OoyalaIE._url_for_embed_code(smuggle_url(v['provider_video_id'], {'domain': url})), ie='Ooyala')
2734
2735 # Look for Aparat videos
2736 mobj = re.search(r'<iframe .*?src="(http://www\.aparat\.com/video/[^"]+)"', webpage)
2737 if mobj is not None:
2738 return self.url_result(mobj.group(1), 'Aparat')
2739
2740 # Look for MPORA videos
2741 mobj = re.search(r'<iframe .*?src="(http://mpora\.(?:com|de)/videos/[^"]+)"', webpage)
2742 if mobj is not None:
2743 return self.url_result(mobj.group(1), 'Mpora')
2744
2745 # Look for embedded Facebook player
2746 facebook_urls = FacebookIE._extract_urls(webpage)
2747 if facebook_urls:
2748 return self.playlist_from_matches(facebook_urls, video_id, video_title)
2749
2750 # Look for embedded VK player
2751 mobj = re.search(r'<iframe[^>]+?src=(["\'])(?P<url>https?://vk\.com/video_ext\.php.+?)\1', webpage)
2752 if mobj is not None:
2753 return self.url_result(mobj.group('url'), 'VK')
2754
2755 # Look for embedded Odnoklassniki player
2756 odnoklassniki_url = OdnoklassnikiIE._extract_url(webpage)
2757 if odnoklassniki_url:
2758 return self.url_result(odnoklassniki_url, OdnoklassnikiIE.ie_key())
2759
2760 # Look for sibnet embedded player
2761 sibnet_urls = VKIE._extract_sibnet_urls(webpage)
2762 if sibnet_urls:
2763 return self.playlist_from_matches(sibnet_urls, video_id, video_title)
2764
2765 # Look for embedded ivi player
2766 mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
2767 if mobj is not None:
2768 return self.url_result(mobj.group('url'), 'Ivi')
2769
2770 # Look for embedded Huffington Post player
2771 mobj = re.search(
2772 r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed\.live\.huffingtonpost\.com/.+?)\1', webpage)
2773 if mobj is not None:
2774 return self.url_result(mobj.group('url'), 'HuffPost')
2775
2776 # Look for embed.ly
2777 mobj = re.search(r'class=["\']embedly-card["\'][^>]href=["\'](?P<url>[^"\']+)', webpage)
2778 if mobj is not None:
2779 return self.url_result(mobj.group('url'))
2780 mobj = re.search(r'class=["\']embedly-embed["\'][^>]src=["\'][^"\']*url=(?P<url>[^&]+)', webpage)
2781 if mobj is not None:
2782 return self.url_result(compat_urllib_parse_unquote(mobj.group('url')))
2783
2784 # Look for funnyordie embed
2785 matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage)
2786 if matches:
2787 return self.playlist_from_matches(
2788 matches, video_id, video_title, getter=unescapeHTML, ie='FunnyOrDie')
2789
2790 # Look for Simplecast embeds
2791 simplecast_urls = SimplecastIE._extract_urls(webpage)
2792 if simplecast_urls:
2793 return self.playlist_from_matches(
2794 simplecast_urls, video_id, video_title)
2795
2796 # Look for BBC iPlayer embed
2797 matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
2798 if matches:
2799 return self.playlist_from_matches(matches, video_id, video_title, ie='BBCCoUk')
2800
2801 # Look for embedded RUTV player
2802 rutv_url = RUTVIE._extract_url(webpage)
2803 if rutv_url:
2804 return self.url_result(rutv_url, 'RUTV')
2805
2806 # Look for embedded TVC player
2807 tvc_url = TVCIE._extract_url(webpage)
2808 if tvc_url:
2809 return self.url_result(tvc_url, 'TVC')
2810
2811 # Look for embedded SportBox player
2812 sportbox_urls = SportBoxIE._extract_urls(webpage)
2813 if sportbox_urls:
2814 return self.playlist_from_matches(sportbox_urls, video_id, video_title, ie=SportBoxIE.ie_key())
2815
2816 # Look for embedded XHamster player
2817 xhamster_urls = XHamsterEmbedIE._extract_urls(webpage)
2818 if xhamster_urls:
2819 return self.playlist_from_matches(xhamster_urls, video_id, video_title, ie='XHamsterEmbed')
2820
2821 # Look for embedded TNAFlixNetwork player
2822 tnaflix_urls = TNAFlixNetworkEmbedIE._extract_urls(webpage)
2823 if tnaflix_urls:
2824 return self.playlist_from_matches(tnaflix_urls, video_id, video_title, ie=TNAFlixNetworkEmbedIE.ie_key())
2825
2826 # Look for embedded PornHub player
2827 pornhub_urls = PornHubIE._extract_urls(webpage)
2828 if pornhub_urls:
2829 return self.playlist_from_matches(pornhub_urls, video_id, video_title, ie=PornHubIE.ie_key())
2830
2831 # Look for embedded DrTuber player
2832 drtuber_urls = DrTuberIE._extract_urls(webpage)
2833 if drtuber_urls:
2834 return self.playlist_from_matches(drtuber_urls, video_id, video_title, ie=DrTuberIE.ie_key())
2835
2836 # Look for embedded RedTube player
2837 redtube_urls = RedTubeIE._extract_urls(webpage)
2838 if redtube_urls:
2839 return self.playlist_from_matches(redtube_urls, video_id, video_title, ie=RedTubeIE.ie_key())
2840
2841 # Look for embedded Tube8 player
2842 tube8_urls = Tube8IE._extract_urls(webpage)
2843 if tube8_urls:
2844 return self.playlist_from_matches(tube8_urls, video_id, video_title, ie=Tube8IE.ie_key())
2845
2846 # Look for embedded Mofosex player
2847 mofosex_urls = MofosexEmbedIE._extract_urls(webpage)
2848 if mofosex_urls:
2849 return self.playlist_from_matches(mofosex_urls, video_id, video_title, ie=MofosexEmbedIE.ie_key())
2850
2851 # Look for embedded Spankwire player
2852 spankwire_urls = SpankwireIE._extract_urls(webpage)
2853 if spankwire_urls:
2854 return self.playlist_from_matches(spankwire_urls, video_id, video_title, ie=SpankwireIE.ie_key())
2855
2856 # Look for embedded YouPorn player
2857 youporn_urls = YouPornIE._extract_urls(webpage)
2858 if youporn_urls:
2859 return self.playlist_from_matches(youporn_urls, video_id, video_title, ie=YouPornIE.ie_key())
2860
2861 # Look for embedded Tvigle player
2862 mobj = re.search(
2863 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//cloud\.tvigle\.ru/video/.+?)\1', webpage)
2864 if mobj is not None:
2865 return self.url_result(mobj.group('url'), 'Tvigle')
2866
2867 # Look for embedded TED player
2868 mobj = re.search(
2869 r'<iframe[^>]+?src=(["\'])(?P<url>https?://embed(?:-ssl)?\.ted\.com/.+?)\1', webpage)
2870 if mobj is not None:
2871 return self.url_result(mobj.group('url'), 'TED')
2872
2873 # Look for embedded Ustream videos
2874 ustream_url = UstreamIE._extract_url(webpage)
2875 if ustream_url:
2876 return self.url_result(ustream_url, UstreamIE.ie_key())
2877
2878 # Look for embedded arte.tv player
2879 arte_urls = ArteTVEmbedIE._extract_urls(webpage)
2880 if arte_urls:
2881 return self.playlist_from_matches(arte_urls, video_id, video_title)
2882
2883 # Look for embedded francetv player
2884 mobj = re.search(
2885 r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?://)?embed\.francetv\.fr/\?ue=.+?)\1',
2886 webpage)
2887 if mobj is not None:
2888 return self.url_result(mobj.group('url'))
2889
2890 # Look for embedded Myvi.ru player
2891 myvi_url = MyviIE._extract_url(webpage)
2892 if myvi_url:
2893 return self.url_result(myvi_url)
2894
2895 # Look for embedded soundcloud player
2896 soundcloud_urls = SoundcloudEmbedIE._extract_urls(webpage)
2897 if soundcloud_urls:
2898 return self.playlist_from_matches(soundcloud_urls, video_id, video_title, getter=unescapeHTML)
2899
2900 # Look for tunein player
2901 tunein_urls = TuneInBaseIE._extract_urls(webpage)
2902 if tunein_urls:
2903 return self.playlist_from_matches(tunein_urls, video_id, video_title)
2904
2905 # Look for embedded mtvservices player
2906 mtvservices_url = MTVServicesEmbeddedIE._extract_url(webpage)
2907 if mtvservices_url:
2908 return self.url_result(mtvservices_url, ie='MTVServicesEmbedded')
2909
2910 # Look for embedded yahoo player
2911 mobj = re.search(
2912 r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1',
2913 webpage)
2914 if mobj is not None:
2915 return self.url_result(mobj.group('url'), 'Yahoo')
2916
2917 # Look for embedded sbs.com.au player
2918 mobj = re.search(
2919 r'''(?x)
2920 (?:
2921 <meta\s+property="og:video"\s+content=|
2922 <iframe[^>]+?src=
2923 )
2924 (["\'])(?P<url>https?://(?:www\.)?sbs\.com\.au/ondemand/video/.+?)\1''',
2925 webpage)
2926 if mobj is not None:
2927 return self.url_result(mobj.group('url'), 'SBS')
2928
2929 # Look for embedded Cinchcast player
2930 mobj = re.search(
2931 r'<iframe[^>]+?src=(["\'])(?P<url>https?://player\.cinchcast\.com/.+?)\1',
2932 webpage)
2933 if mobj is not None:
2934 return self.url_result(mobj.group('url'), 'Cinchcast')
2935
2936 mobj = re.search(
2937 r'<iframe[^>]+?src=(["\'])(?P<url>https?://m(?:lb)?\.mlb\.com/shared/video/embed/embed\.html\?.+?)\1',
2938 webpage)
2939 if not mobj:
2940 mobj = re.search(
2941 r'data-video-link=["\'](?P<url>http://m\.mlb\.com/video/[^"\']+)',
2942 webpage)
2943 if mobj is not None:
2944 return self.url_result(mobj.group('url'), 'MLB')
2945
2946 mobj = re.search(
2947 r'<(?:iframe|script)[^>]+?src=(["\'])(?P<url>%s)\1' % CondeNastIE.EMBED_URL,
2948 webpage)
2949 if mobj is not None:
2950 return self.url_result(self._proto_relative_url(mobj.group('url'), scheme='http:'), 'CondeNast')
2951
2952 mobj = re.search(
2953 r'<iframe[^>]+src="(?P<url>https?://(?:new\.)?livestream\.com/[^"]+/player[^"]+)"',
2954 webpage)
2955 if mobj is not None:
2956 return self.url_result(mobj.group('url'), 'Livestream')
2957
2958 # Look for Zapiks embed
2959 mobj = re.search(
2960 r'<iframe[^>]+src="(?P<url>https?://(?:www\.)?zapiks\.fr/index\.php\?.+?)"', webpage)
2961 if mobj is not None:
2962 return self.url_result(mobj.group('url'), 'Zapiks')
2963
2964 # Look for Kaltura embeds
2965 kaltura_urls = KalturaIE._extract_urls(webpage)
2966 if kaltura_urls:
2967 return self.playlist_from_matches(
2968 kaltura_urls, video_id, video_title,
2969 getter=lambda x: smuggle_url(x, {'source_url': url}),
2970 ie=KalturaIE.ie_key())
2971
2972 # Look for EaglePlatform embeds
2973 eagleplatform_url = EaglePlatformIE._extract_url(webpage)
2974 if eagleplatform_url:
2975 return self.url_result(smuggle_url(eagleplatform_url, {'referrer': url}), EaglePlatformIE.ie_key())
2976
2977 # Look for ClipYou (uses EaglePlatform) embeds
2978 mobj = re.search(
2979 r'<iframe[^>]+src="https?://(?P<host>media\.clipyou\.ru)/index/player\?.*\brecord_id=(?P<id>\d+).*"', webpage)
2980 if mobj is not None:
2981 return self.url_result('eagleplatform:%(host)s:%(id)s' % mobj.groupdict(), 'EaglePlatform')
2982
2983 # Look for Pladform embeds
2984 pladform_url = PladformIE._extract_url(webpage)
2985 if pladform_url:
2986 return self.url_result(pladform_url)
2987
2988 # Look for Videomore embeds
2989 videomore_url = VideomoreIE._extract_url(webpage)
2990 if videomore_url:
2991 return self.url_result(videomore_url)
2992
2993 # Look for Webcaster embeds
2994 webcaster_url = WebcasterFeedIE._extract_url(self, webpage)
2995 if webcaster_url:
2996 return self.url_result(webcaster_url, ie=WebcasterFeedIE.ie_key())
2997
2998 # Look for Playwire embeds
2999 mobj = re.search(
3000 r'<script[^>]+data-config=(["\'])(?P<url>(?:https?:)?//config\.playwire\.com/.+?)\1', webpage)
3001 if mobj is not None:
3002 return self.url_result(mobj.group('url'))
3003
3004 # Look for 5min embeds
3005 mobj = re.search(
3006 r'<meta[^>]+property="og:video"[^>]+content="https?://embed\.5min\.com/(?P<id>[0-9]+)/?', webpage)
3007 if mobj is not None:
3008 return self.url_result('5min:%s' % mobj.group('id'), 'FiveMin')
3009
3010 # Look for Crooks and Liars embeds
3011 mobj = re.search(
3012 r'<(?:iframe[^>]+src|param[^>]+value)=(["\'])(?P<url>(?:https?:)?//embed\.crooksandliars\.com/(?:embed|v)/.+?)\1', webpage)
3013 if mobj is not None:
3014 return self.url_result(mobj.group('url'))
3015
3016 # Look for NBC Sports VPlayer embeds
3017 nbc_sports_url = NBCSportsVPlayerIE._extract_url(webpage)
3018 if nbc_sports_url:
3019 return self.url_result(nbc_sports_url, 'NBCSportsVPlayer')
3020
3021 # Look for NBC News embeds
3022 nbc_news_embed_url = re.search(
3023 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//www\.nbcnews\.com/widget/video-embed/[^"\']+)\1', webpage)
3024 if nbc_news_embed_url:
3025 return self.url_result(nbc_news_embed_url.group('url'), 'NBCNews')
3026
3027 # Look for Google Drive embeds
3028 google_drive_url = GoogleDriveIE._extract_url(webpage)
3029 if google_drive_url:
3030 return self.url_result(google_drive_url, 'GoogleDrive')
3031
3032 # Look for UDN embeds
3033 mobj = re.search(
3034 r'<iframe[^>]+src="(?:https?:)?(?P<url>%s)"' % UDNEmbedIE._PROTOCOL_RELATIVE_VALID_URL, webpage)
3035 if mobj is not None:
3036 return self.url_result(
3037 compat_urlparse.urljoin(url, mobj.group('url')), 'UDNEmbed')
3038
3039 # Look for Senate ISVP iframe
3040 senate_isvp_url = SenateISVPIE._search_iframe_url(webpage)
3041 if senate_isvp_url:
3042 return self.url_result(senate_isvp_url, 'SenateISVP')
3043
3044 # Look for Kinja embeds
3045 kinja_embed_urls = KinjaEmbedIE._extract_urls(webpage, url)
3046 if kinja_embed_urls:
3047 return self.playlist_from_matches(
3048 kinja_embed_urls, video_id, video_title)
3049
3050 # Look for OnionStudios embeds
3051 onionstudios_url = OnionStudiosIE._extract_url(webpage)
3052 if onionstudios_url:
3053 return self.url_result(onionstudios_url)
3054
3055 # Look for ViewLift embeds
3056 viewlift_url = ViewLiftEmbedIE._extract_url(webpage)
3057 if viewlift_url:
3058 return self.url_result(viewlift_url)
3059
3060 # Look for JWPlatform embeds
3061 jwplatform_urls = JWPlatformIE._extract_urls(webpage)
3062 if jwplatform_urls:
3063 return self.playlist_from_matches(jwplatform_urls, video_id, video_title, ie=JWPlatformIE.ie_key())
3064
3065 # Look for Digiteka embeds
3066 digiteka_url = DigitekaIE._extract_url(webpage)
3067 if digiteka_url:
3068 return self.url_result(self._proto_relative_url(digiteka_url), DigitekaIE.ie_key())
3069
3070 # Look for Arkena embeds
3071 arkena_url = ArkenaIE._extract_url(webpage)
3072 if arkena_url:
3073 return self.url_result(arkena_url, ArkenaIE.ie_key())
3074
3075 # Look for Piksel embeds
3076 piksel_url = PikselIE._extract_url(webpage)
3077 if piksel_url:
3078 return self.url_result(piksel_url, PikselIE.ie_key())
3079
3080 # Look for Limelight embeds
3081 limelight_urls = LimelightBaseIE._extract_urls(webpage, url)
3082 if limelight_urls:
3083 return self.playlist_result(
3084 limelight_urls, video_id, video_title, video_description)
3085
3086 # Look for Anvato embeds
3087 anvato_urls = AnvatoIE._extract_urls(self, webpage, video_id)
3088 if anvato_urls:
3089 return self.playlist_result(
3090 anvato_urls, video_id, video_title, video_description)
3091
3092 # Look for AdobeTVVideo embeds
3093 mobj = re.search(
3094 r'<iframe[^>]+src=[\'"]((?:https?:)?//video\.tv\.adobe\.com/v/\d+[^"]+)[\'"]',
3095 webpage)
3096 if mobj is not None:
3097 return self.url_result(
3098 self._proto_relative_url(unescapeHTML(mobj.group(1))),
3099 'AdobeTVVideo')
3100
3101 # Look for Vine embeds
3102 mobj = re.search(
3103 r'<iframe[^>]+src=[\'"]((?:https?:)?//(?:www\.)?vine\.co/v/[^/]+/embed/(?:simple|postcard))',
3104 webpage)
3105 if mobj is not None:
3106 return self.url_result(
3107 self._proto_relative_url(unescapeHTML(mobj.group(1))), 'Vine')
3108
3109 # Look for VODPlatform embeds
3110 mobj = re.search(
3111 r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//(?:(?:www\.)?vod-platform\.net|embed\.kwikmotion\.com)/[eE]mbed/.+?)\1',
3112 webpage)
3113 if mobj is not None:
3114 return self.url_result(
3115 self._proto_relative_url(unescapeHTML(mobj.group('url'))), 'VODPlatform')
3116
3117 # Look for Mangomolo embeds
3118 mobj = re.search(
3119 r'''(?x)<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//
3120 (?:
3121 admin\.mangomolo\.com/analytics/index\.php/customers/embed|
3122 player\.mangomolo\.com/v1
3123 )/
3124 (?:
3125 video\?.*?\bid=(?P<video_id>\d+)|
3126 (?:index|live)\?.*?\bchannelid=(?P<channel_id>(?:[A-Za-z0-9+/=]|%2B|%2F|%3D)+)
3127 ).+?)\1''', webpage)
3128 if mobj is not None:
3129 info = {
3130 '_type': 'url_transparent',
3131 'url': self._proto_relative_url(unescapeHTML(mobj.group('url'))),
3132 'title': video_title,
3133 'description': video_description,
3134 'thumbnail': video_thumbnail,
3135 'uploader': video_uploader,
3136 }
3137 video_id = mobj.group('video_id')
3138 if video_id:
3139 info.update({
3140 'ie_key': 'MangomoloVideo',
3141 'id': video_id,
3142 })
3143 else:
3144 info.update({
3145 'ie_key': 'MangomoloLive',
3146 'id': mobj.group('channel_id'),
3147 })
3148 return info
3149
3150 # Look for Instagram embeds
3151 instagram_embed_url = InstagramIE._extract_embed_url(webpage)
3152 if instagram_embed_url is not None:
3153 return self.url_result(
3154 self._proto_relative_url(instagram_embed_url), InstagramIE.ie_key())
3155
3156 # Look for 3Q SDN embeds
3157 threeqsdn_url = ThreeQSDNIE._extract_url(webpage)
3158 if threeqsdn_url:
3159 return {
3160 '_type': 'url_transparent',
3161 'ie_key': ThreeQSDNIE.ie_key(),
3162 'url': self._proto_relative_url(threeqsdn_url),
3163 'title': video_title,
3164 'description': video_description,
3165 'thumbnail': video_thumbnail,
3166 'uploader': video_uploader,
3167 }
3168
3169 # Look for VBOX7 embeds
3170 vbox7_url = Vbox7IE._extract_url(webpage)
3171 if vbox7_url:
3172 return self.url_result(vbox7_url, Vbox7IE.ie_key())
3173
3174 # Look for DBTV embeds
3175 dbtv_urls = DBTVIE._extract_urls(webpage)
3176 if dbtv_urls:
3177 return self.playlist_from_matches(dbtv_urls, video_id, video_title, ie=DBTVIE.ie_key())
3178
3179 # Look for Videa embeds
3180 videa_urls = VideaIE._extract_urls(webpage)
3181 if videa_urls:
3182 return self.playlist_from_matches(videa_urls, video_id, video_title, ie=VideaIE.ie_key())
3183
3184 # Look for 20 minuten embeds
3185 twentymin_urls = TwentyMinutenIE._extract_urls(webpage)
3186 if twentymin_urls:
3187 return self.playlist_from_matches(
3188 twentymin_urls, video_id, video_title, ie=TwentyMinutenIE.ie_key())
3189
3190 # Look for VideoPress embeds
3191 videopress_urls = VideoPressIE._extract_urls(webpage)
3192 if videopress_urls:
3193 return self.playlist_from_matches(
3194 videopress_urls, video_id, video_title, ie=VideoPressIE.ie_key())
3195
3196 # Look for Rutube embeds
3197 rutube_urls = RutubeIE._extract_urls(webpage)
3198 if rutube_urls:
3199 return self.playlist_from_matches(
3200 rutube_urls, video_id, video_title, ie=RutubeIE.ie_key())
3201
3202 # Look for WashingtonPost embeds
3203 wapo_urls = WashingtonPostIE._extract_urls(webpage)
3204 if wapo_urls:
3205 return self.playlist_from_matches(
3206 wapo_urls, video_id, video_title, ie=WashingtonPostIE.ie_key())
3207
3208 # Look for Mediaset embeds
3209 mediaset_urls = MediasetIE._extract_urls(self, webpage)
3210 if mediaset_urls:
3211 return self.playlist_from_matches(
3212 mediaset_urls, video_id, video_title, ie=MediasetIE.ie_key())
3213
3214 # Look for JOJ.sk embeds
3215 joj_urls = JojIE._extract_urls(webpage)
3216 if joj_urls:
3217 return self.playlist_from_matches(
3218 joj_urls, video_id, video_title, ie=JojIE.ie_key())
3219
3220 # Look for megaphone.fm embeds
3221 mpfn_urls = MegaphoneIE._extract_urls(webpage)
3222 if mpfn_urls:
3223 return self.playlist_from_matches(
3224 mpfn_urls, video_id, video_title, ie=MegaphoneIE.ie_key())
3225
3226 # Look for vzaar embeds
3227 vzaar_urls = VzaarIE._extract_urls(webpage)
3228 if vzaar_urls:
3229 return self.playlist_from_matches(
3230 vzaar_urls, video_id, video_title, ie=VzaarIE.ie_key())
3231
3232 channel9_urls = Channel9IE._extract_urls(webpage)
3233 if channel9_urls:
3234 return self.playlist_from_matches(
3235 channel9_urls, video_id, video_title, ie=Channel9IE.ie_key())
3236
3237 vshare_urls = VShareIE._extract_urls(webpage)
3238 if vshare_urls:
3239 return self.playlist_from_matches(
3240 vshare_urls, video_id, video_title, ie=VShareIE.ie_key())
3241
3242 # Look for Mediasite embeds
3243 mediasite_urls = MediasiteIE._extract_urls(webpage)
3244 if mediasite_urls:
3245 entries = [
3246 self.url_result(smuggle_url(
3247 compat_urlparse.urljoin(url, mediasite_url),
3248 {'UrlReferrer': url}), ie=MediasiteIE.ie_key())
3249 for mediasite_url in mediasite_urls]
3250 return self.playlist_result(entries, video_id, video_title)
3251
3252 springboardplatform_urls = SpringboardPlatformIE._extract_urls(webpage)
3253 if springboardplatform_urls:
3254 return self.playlist_from_matches(
3255 springboardplatform_urls, video_id, video_title,
3256 ie=SpringboardPlatformIE.ie_key())
3257
3258 yapfiles_urls = YapFilesIE._extract_urls(webpage)
3259 if yapfiles_urls:
3260 return self.playlist_from_matches(
3261 yapfiles_urls, video_id, video_title, ie=YapFilesIE.ie_key())
3262
3263 vice_urls = ViceIE._extract_urls(webpage)
3264 if vice_urls:
3265 return self.playlist_from_matches(
3266 vice_urls, video_id, video_title, ie=ViceIE.ie_key())
3267
3268 xfileshare_urls = XFileShareIE._extract_urls(webpage)
3269 if xfileshare_urls:
3270 return self.playlist_from_matches(
3271 xfileshare_urls, video_id, video_title, ie=XFileShareIE.ie_key())
3272
3273 cloudflarestream_urls = CloudflareStreamIE._extract_urls(webpage)
3274 if cloudflarestream_urls:
3275 return self.playlist_from_matches(
3276 cloudflarestream_urls, video_id, video_title, ie=CloudflareStreamIE.ie_key())
3277
3278 peertube_urls = PeerTubeIE._extract_urls(webpage, url)
3279 if peertube_urls:
3280 return self.playlist_from_matches(
3281 peertube_urls, video_id, video_title, ie=PeerTubeIE.ie_key())
3282
3283 indavideo_urls = IndavideoEmbedIE._extract_urls(webpage)
3284 if indavideo_urls:
3285 return self.playlist_from_matches(
3286 indavideo_urls, video_id, video_title, ie=IndavideoEmbedIE.ie_key())
3287
3288 apa_urls = APAIE._extract_urls(webpage)
3289 if apa_urls:
3290 return self.playlist_from_matches(
3291 apa_urls, video_id, video_title, ie=APAIE.ie_key())
3292
3293 foxnews_urls = FoxNewsIE._extract_urls(webpage)
3294 if foxnews_urls:
3295 return self.playlist_from_matches(
3296 foxnews_urls, video_id, video_title, ie=FoxNewsIE.ie_key())
3297
3298 sharevideos_urls = [sharevideos_mobj.group('url') for sharevideos_mobj in re.finditer(
3299 r'<iframe[^>]+?\bsrc\s*=\s*(["\'])(?P<url>(?:https?:)?//embed\.share-videos\.se/auto/embed/\d+\?.*?\buid=\d+.*?)\1',
3300 webpage)]
3301 if sharevideos_urls:
3302 return self.playlist_from_matches(
3303 sharevideos_urls, video_id, video_title)
3304
3305 viqeo_urls = ViqeoIE._extract_urls(webpage)
3306 if viqeo_urls:
3307 return self.playlist_from_matches(
3308 viqeo_urls, video_id, video_title, ie=ViqeoIE.ie_key())
3309
3310 expressen_urls = ExpressenIE._extract_urls(webpage)
3311 if expressen_urls:
3312 return self.playlist_from_matches(
3313 expressen_urls, video_id, video_title, ie=ExpressenIE.ie_key())
3314
3315 zype_urls = ZypeIE._extract_urls(webpage)
3316 if zype_urls:
3317 return self.playlist_from_matches(
3318 zype_urls, video_id, video_title, ie=ZypeIE.ie_key())
3319
3320 # Look for HTML5 media
3321 entries = self._parse_html5_media_entries(url, webpage, video_id, m3u8_id='hls')
3322 if entries:
3323 if len(entries) == 1:
3324 entries[0].update({
3325 'id': video_id,
3326 'title': video_title,
3327 })
3328 else:
3329 for num, entry in enumerate(entries, start=1):
3330 entry.update({
3331 'id': '%s-%s' % (video_id, num),
3332 'title': '%s (%d)' % (video_title, num),
3333 })
3334 for entry in entries:
3335 self._sort_formats(entry['formats'])
3336 return self.playlist_result(entries, video_id, video_title)
3337
3338 jwplayer_data = self._find_jwplayer_data(
3339 webpage, video_id, transform_source=js_to_json)
3340 if jwplayer_data:
3341 try:
3342 info = self._parse_jwplayer_data(
3343 jwplayer_data, video_id, require_title=False, base_url=url)
3344 return merge_dicts(info, info_dict)
3345 except ExtractorError:
3346 # See https://github.com/ytdl-org/youtube-dl/pull/16735
3347 pass
3348
3349 # Video.js embed
3350 mobj = re.search(
3351 r'(?s)\bvideojs\s*\(.+?\.src\s*\(\s*((?:\[.+?\]|{.+?}))\s*\)\s*;',
3352 webpage)
3353 if mobj is not None:
3354 sources = self._parse_json(
3355 mobj.group(1), video_id, transform_source=js_to_json,
3356 fatal=False) or []
3357 if not isinstance(sources, list):
3358 sources = [sources]
3359 formats = []
3360 for source in sources:
3361 src = source.get('src')
3362 if not src or not isinstance(src, compat_str):
3363 continue
3364 src = compat_urlparse.urljoin(url, src)
3365 src_type = source.get('type')
3366 if isinstance(src_type, compat_str):
3367 src_type = src_type.lower()
3368 ext = determine_ext(src).lower()
3369 if src_type == 'video/youtube':
3370 return self.url_result(src, YoutubeIE.ie_key())
3371 if src_type == 'application/dash+xml' or ext == 'mpd':
3372 formats.extend(self._extract_mpd_formats(
3373 src, video_id, mpd_id='dash', fatal=False))
3374 elif src_type == 'application/x-mpegurl' or ext == 'm3u8':
3375 formats.extend(self._extract_m3u8_formats(
3376 src, video_id, 'mp4', entry_protocol='m3u8_native',
3377 m3u8_id='hls', fatal=False))
3378 else:
3379 formats.append({
3380 'url': src,
3381 'ext': (mimetype2ext(src_type)
3382 or ext if ext in KNOWN_EXTENSIONS else 'mp4'),
3383 'http_headers': {
3384 'Referer': full_response.geturl(),
3385 },
3386 })
3387 if formats:
3388 self._sort_formats(formats)
3389 info_dict['formats'] = formats
3390 return info_dict
3391
3392 # Looking for http://schema.org/VideoObject
3393 json_ld = self._search_json_ld(
3394 webpage, video_id, default={}, expected_type='VideoObject')
3395 if json_ld.get('url'):
3396 return merge_dicts(json_ld, info_dict)
3397
3398 def check_video(vurl):
3399 if YoutubeIE.suitable(vurl):
3400 return True
3401 if RtmpIE.suitable(vurl):
3402 return True
3403 vpath = compat_urlparse.urlparse(vurl).path
3404 vext = determine_ext(vpath)
3405 return '.' in vpath and vext not in ('swf', 'png', 'jpg', 'srt', 'sbv', 'sub', 'vtt', 'ttml', 'js', 'xml')
3406
3407 def filter_video(urls):
3408 return list(filter(check_video, urls))
3409
3410 # Start with something easy: JW Player in SWFObject
3411 found = filter_video(re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage))
3412 if not found:
3413 # Look for gorilla-vid style embedding
3414 found = filter_video(re.findall(r'''(?sx)
3415 (?:
3416 jw_plugins|
3417 JWPlayerOptions|
3418 jwplayer\s*\(\s*["'][^'"]+["']\s*\)\s*\.setup
3419 )
3420 .*?
3421 ['"]?file['"]?\s*:\s*["\'](.*?)["\']''', webpage))
3422 if not found:
3423 # Broaden the search a little bit
3424 found = filter_video(re.findall(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage))
3425 if not found:
3426 # Broaden the findall a little bit: JWPlayer JS loader
3427 found = filter_video(re.findall(
3428 r'[^A-Za-z0-9]?(?:file|video_url)["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage))
3429 if not found:
3430 # Flow player
3431 found = filter_video(re.findall(r'''(?xs)
3432 flowplayer\("[^"]+",\s*
3433 \{[^}]+?\}\s*,
3434 \s*\{[^}]+? ["']?clip["']?\s*:\s*\{\s*
3435 ["']?url["']?\s*:\s*["']([^"']+)["']
3436 ''', webpage))
3437 if not found:
3438 # Cinerama player
3439 found = re.findall(
3440 r"cinerama\.embedPlayer\(\s*\'[^']+\',\s*'([^']+)'", webpage)
3441 if not found:
3442 # Try to find twitter cards info
3443 # twitter:player:stream should be checked before twitter:player since
3444 # it is expected to contain a raw stream (see
3445 # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
3446 found = filter_video(re.findall(
3447 r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage))
3448 if not found:
3449 # We look for Open Graph info:
3450 # We have to match any number spaces between elements, some sites try to align them (eg.: statigr.am)
3451 m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
3452 # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
3453 if m_video_type is not None:
3454 found = filter_video(re.findall(r'<meta.*?property="og:(?:video|audio)".*?content="(.*?)"', webpage))
3455 if not found:
3456 REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
3457 found = re.search(
3458 r'(?i)<meta\s+(?=(?:[a-z-]+="[^"]+"\s+)*http-equiv="refresh")'
3459 r'(?:[a-z-]+="[^"]+"\s+)*?content="%s' % REDIRECT_REGEX,
3460 webpage)
3461 if not found:
3462 # Look also in Refresh HTTP header
3463 refresh_header = head_response.headers.get('Refresh')
3464 if refresh_header:
3465 # In python 2 response HTTP headers are bytestrings
3466 if sys.version_info < (3, 0) and isinstance(refresh_header, str):
3467 refresh_header = refresh_header.decode('iso-8859-1')
3468 found = re.search(REDIRECT_REGEX, refresh_header)
3469 if found:
3470 new_url = compat_urlparse.urljoin(url, unescapeHTML(found.group(1)))
3471 if new_url != url:
3472 self.report_following_redirect(new_url)
3473 return {
3474 '_type': 'url',
3475 'url': new_url,
3476 }
3477 else:
3478 found = None
3479
3480 if not found:
3481 # twitter:player is a https URL to iframe player that may or may not
3482 # be supported by youtube-dl thus this is checked the very last (see
3483 # https://dev.twitter.com/cards/types/player#On_twitter.com_via_desktop_browser)
3484 embed_url = self._html_search_meta('twitter:player', webpage, default=None)
3485 if embed_url and embed_url != url:
3486 return self.url_result(embed_url)
3487
3488 if not found:
3489 raise UnsupportedError(url)
3490
3491 entries = []
3492 for video_url in orderedSet(found):
3493 video_url = unescapeHTML(video_url)
3494 video_url = video_url.replace('\\/', '/')
3495 video_url = compat_urlparse.urljoin(url, video_url)
3496 video_id = compat_urllib_parse_unquote(os.path.basename(video_url))
3497
3498 # Sometimes, jwplayer extraction will result in a YouTube URL
3499 if YoutubeIE.suitable(video_url):
3500 entries.append(self.url_result(video_url, 'Youtube'))
3501 continue
3502
3503 # here's a fun little line of code for you:
3504 video_id = os.path.splitext(video_id)[0]
3505
3506 entry_info_dict = {
3507 'id': video_id,
3508 'uploader': video_uploader,
3509 'title': video_title,
3510 'age_limit': age_limit,
3511 }
3512
3513 if RtmpIE.suitable(video_url):
3514 entry_info_dict.update({
3515 '_type': 'url_transparent',
3516 'ie_key': RtmpIE.ie_key(),
3517 'url': video_url,
3518 })
3519 entries.append(entry_info_dict)
3520 continue
3521
3522 ext = determine_ext(video_url)
3523 if ext == 'smil':
3524 entry_info_dict['formats'] = self._extract_smil_formats(video_url, video_id)
3525 elif ext == 'xspf':
3526 return self.playlist_result(self._extract_xspf_playlist(video_url, video_id), video_id)
3527 elif ext == 'm3u8':
3528 entry_info_dict['formats'] = self._extract_m3u8_formats(video_url, video_id, ext='mp4')
3529 elif ext == 'mpd':
3530 entry_info_dict['formats'] = self._extract_mpd_formats(video_url, video_id)
3531 elif ext == 'f4m':
3532 entry_info_dict['formats'] = self._extract_f4m_formats(video_url, video_id)
3533 elif re.search(r'(?i)\.(?:ism|smil)/manifest', video_url) and video_url != url:
3534 # Just matching .ism/manifest is not enough to be reliably sure
3535 # whether it's actually an ISM manifest or some other streaming
3536 # manifest since there are various streaming URL formats
3537 # possible (see [1]) as well as some other shenanigans like
3538 # .smil/manifest URLs that actually serve an ISM (see [2]) and
3539 # so on.
3540 # Thus the most reasonable way to solve this is to delegate
3541 # to generic extractor in order to look into the contents of
3542 # the manifest itself.
3543 # 1. https://azure.microsoft.com/en-us/documentation/articles/media-services-deliver-content-overview/#streaming-url-formats
3544 # 2. https://svs.itworkscdn.net/lbcivod/smil:itwfcdn/lbci/170976.smil/Manifest
3545 entry_info_dict = self.url_result(
3546 smuggle_url(video_url, {'to_generic': True}),
3547 GenericIE.ie_key())
3548 else:
3549 entry_info_dict['url'] = video_url
3550
3551 if entry_info_dict.get('formats'):
3552 self._sort_formats(entry_info_dict['formats'])
3553
3554 entries.append(entry_info_dict)
3555
3556 if len(entries) == 1:
3557 return entries[0]
3558 else:
3559 for num, e in enumerate(entries, start=1):
3560 # 'url' results don't have a title
3561 if e.get('title') is not None:
3562 e['title'] = '%s (%d)' % (e['title'], num)
3563 return {
3564 '_type': 'playlist',
3565 'entries': entries,
3566 }
|
