diff options
Diffstat (limited to 'youtube_dl/extractor')
| -rw-r--r-- | youtube_dl/extractor/arte.py | 10 | ||||
| -rw-r--r-- | youtube_dl/extractor/facebook.py | 22 | ||||
| -rw-r--r-- | youtube_dl/extractor/jukebox.py | 75 | ||||
| -rw-r--r-- | youtube_dl/extractor/myvideo.py | 101 | ||||
| -rw-r--r-- | youtube_dl/extractor/photobucket.py | 69 | ||||
| -rw-r--r-- | youtube_dl/extractor/pornhub.py | 2 | ||||
| -rw-r--r-- | youtube_dl/extractor/xnxx.py | 46 |
7 files changed, 134 insertions, 191 deletions
diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 7cf3785ac..d194f2564 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -72,18 +72,22 @@ class ArteTvIE(InfoExtractor): return self._extract_liveweb(url, name, lang) if re.search(self._LIVE_URL, url) is not None: - raise ExtractorError(u'Arte live streams are not yet supported, sorry') + raise ExtractorError('Arte live streams are not yet supported, sorry') # self.extractLiveStream(url) # return + raise ExtractorError('No video found') + def _extract_video(self, url, video_id, lang): """Extract from videos.arte.tv""" ref_xml_url = url.replace('/videos/', '/do_delegate/videos/') ref_xml_url = ref_xml_url.replace('.html', ',view,asPlayerXml.xml') - ref_xml_doc = self._download_xml(ref_xml_url, video_id, note=u'Downloading metadata') + ref_xml_doc = self._download_xml( + ref_xml_url, video_id, note='Downloading metadata') config_node = find_xpath_attr(ref_xml_doc, './/video', 'lang', lang) config_xml_url = config_node.attrib['ref'] - config_xml = self._download_webpage(config_xml_url, video_id, note=u'Downloading configuration') + config_xml = self._download_webpage( + config_xml_url, video_id, note='Downloading configuration') video_urls = list(re.finditer(r'<url quality="(?P<quality>.*?)">(?P<url>.*?)</url>', config_xml)) def _key(m): diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py index 848a2c263..a713628b2 100644 --- a/youtube_dl/extractor/facebook.py +++ b/youtube_dl/extractor/facebook.py @@ -18,10 +18,8 @@ from ..utils import ( class FacebookIE(InfoExtractor): - """Information Extractor for Facebook""" - _VALID_URL = r'''(?x) - (?:https?://)?(?:\w+\.)?facebook\.com/ + https?://(?:\w+\.)?facebook\.com/ (?:[^#?]*\#!/)? (?:video/video\.php|photo\.php|video/embed)\?(?:.*?) (?:v|video_id)=(?P<id>[0-9]+) @@ -37,14 +35,10 @@ class FacebookIE(InfoExtractor): 'id': '120708114770723', 'ext': 'mp4', 'duration': 279, - 'title': 'PEOPLE ARE AWESOME 2013' + 'title': 'PEOPLE ARE AWESOME 2013', } } - def report_login(self): - """Report attempt to log in.""" - self.to_screen('Logging in') - def _login(self): (useremail, password) = self._get_login_info() if useremail is None: @@ -101,8 +95,6 @@ class FacebookIE(InfoExtractor): def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError('Invalid URL: %s' % url) video_id = mobj.group('id') url = 'https://www.facebook.com/video/video.php?v=%s' % video_id @@ -128,18 +120,14 @@ class FacebookIE(InfoExtractor): video_url = video_data['sd_src'] if not video_url: raise ExtractorError('Cannot find video URL') - video_duration = int(video_data['video_duration']) - thumbnail = video_data['thumbnail_src'] video_title = self._html_search_regex( r'<h2 class="uiHeaderTitle">([^<]*)</h2>', webpage, 'title') - info = { + return { 'id': video_id, 'title': video_title, 'url': video_url, - 'ext': 'mp4', - 'duration': video_duration, - 'thumbnail': thumbnail, + 'duration': int(video_data['video_duration']), + 'thumbnail': video_data['thumbnail_src'], } - return [info] diff --git a/youtube_dl/extractor/jukebox.py b/youtube_dl/extractor/jukebox.py index 592c64e1d..f06dcb05e 100644 --- a/youtube_dl/extractor/jukebox.py +++ b/youtube_dl/extractor/jukebox.py @@ -1,56 +1,61 @@ -# coding: utf-8 +from __future__ import unicode_literals + import re from .common import InfoExtractor from ..utils import ( ExtractorError, + RegexNotFoundError, unescapeHTML, ) + class JukeboxIE(InfoExtractor): _VALID_URL = r'^http://www\.jukebox?\..+?\/.+[,](?P<video_id>[a-z0-9\-]+)\.html' - _IFRAME = r'<iframe .*src="(?P<iframe>[^"]*)".*>' - _VIDEO_URL = r'"config":{"file":"(?P<video_url>http:[^"]+[.](?P<video_ext>[^.?]+)[?]mdtk=[0-9]+)"' - _TITLE = r'<h1 class="inline">(?P<title>[^<]+)</h1>.*<span id="infos_article_artist">(?P<artist>[^<]+)</span>' - _IS_YOUTUBE = r'config":{"file":"(?P<youtube_url>http:[\\][/][\\][/]www[.]youtube[.]com[\\][/]watch[?]v=[^"]+)"' + _TEST = { + 'url': 'http://www.jukebox.es/kosheen/videoclip,pride,r303r.html', + 'md5': '5dc6477e74b1e37042ac5acedd8413e5', + 'info_dict': { + 'id': 'r303r', + 'ext': 'flv', + 'title': 'Kosheen-En Vivo Pride', + 'uploader': 'Kosheen', + }, + } def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('video_id') html = self._download_webpage(url, video_id) - - mobj = re.search(self._IFRAME, html) - if mobj is None: - raise ExtractorError(u'Cannot extract iframe url') - iframe_url = unescapeHTML(mobj.group('iframe')) + iframe_url = unescapeHTML(self._search_regex(r'<iframe .*src="([^"]*)"', html, 'iframe url')) iframe_html = self._download_webpage(iframe_url, video_id, 'Downloading iframe') - mobj = re.search(r'class="jkb_waiting"', iframe_html) - if mobj is not None: - raise ExtractorError(u'Video is not available(in your country?)!') + if re.search(r'class="jkb_waiting"', iframe_html) is not None: + raise ExtractorError('Video is not available(in your country?)!') self.report_extraction(video_id) - mobj = re.search(self._VIDEO_URL, iframe_html) - if mobj is None: - mobj = re.search(self._IS_YOUTUBE, iframe_html) - if mobj is None: - raise ExtractorError(u'Cannot extract video url') - youtube_url = unescapeHTML(mobj.group('youtube_url')).replace('\/','/') - self.to_screen(u'Youtube video detected') - return self.url_result(youtube_url,ie='Youtube') - video_url = unescapeHTML(mobj.group('video_url')).replace('\/','/') - video_ext = unescapeHTML(mobj.group('video_ext')) - - mobj = re.search(self._TITLE, html) - if mobj is None: - raise ExtractorError(u'Cannot extract title') - title = unescapeHTML(mobj.group('title')) - artist = unescapeHTML(mobj.group('artist')) - - return [{'id': video_id, - 'url': video_url, - 'title': artist + '-' + title, - 'ext': video_ext - }] + try: + video_url = self._search_regex(r'"config":{"file":"(?P<video_url>http:[^"]+\?mdtk=[0-9]+)"', + iframe_html, 'video url') + video_url = unescapeHTML(video_url).replace('\/', '/') + except RegexNotFoundError: + youtube_url = self._search_regex( + r'config":{"file":"(http:\\/\\/www\.youtube\.com\\/watch\?v=[^"]+)"', + iframe_html, 'youtube url') + youtube_url = unescapeHTML(youtube_url).replace('\/', '/') + self.to_screen('Youtube video detected') + return self.url_result(youtube_url, ie='Youtube') + + title = self._html_search_regex(r'<h1 class="inline">([^<]+)</h1>', + html, 'title') + artist = self._html_search_regex(r'<span id="infos_article_artist">([^<]+)</span>', + html, 'artist') + + return { + 'id': video_id, + 'url': video_url, + 'title': artist + '-' + title, + 'uploader': artist, + } diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py index 6d35c7861..ccb5959c4 100644 --- a/youtube_dl/extractor/myvideo.py +++ b/youtube_dl/extractor/myvideo.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import binascii import base64 import hashlib @@ -14,18 +16,16 @@ from ..utils import ( ) - class MyVideoIE(InfoExtractor): - """Information Extractor for myvideo.de.""" - - _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/([0-9]+)/([^?/]+).*' - IE_NAME = u'myvideo' + _VALID_URL = r'http://(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/(?P<id>[0-9]+)/[^?/]+.*' + IE_NAME = 'myvideo' _TEST = { - u'url': u'http://www.myvideo.de/watch/8229274/bowling_fail_or_win', - u'file': u'8229274.flv', - u'md5': u'2d2753e8130479ba2cb7e0a37002053e', - u'info_dict': { - u"title": u"bowling-fail-or-win" + 'url': 'http://www.myvideo.de/watch/8229274/bowling_fail_or_win', + 'md5': '2d2753e8130479ba2cb7e0a37002053e', + 'info_dict': { + 'id': '8229274', + 'ext': 'flv', + 'title': 'bowling-fail-or-win', } } @@ -53,10 +53,7 @@ class MyVideoIE(InfoExtractor): def _real_extract(self,url): mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'invalid URL: %s' % url) - - video_id = mobj.group(1) + video_id = mobj.group('id') GK = ( b'WXpnME1EZGhNRGhpTTJNM01XVmhOREU0WldNNVpHTTJOakpt' @@ -74,37 +71,33 @@ class MyVideoIE(InfoExtractor): video_url = mobj.group(1) + '.flv' video_title = self._html_search_regex('<title>([^<]+)</title>', - webpage, u'title') - - video_ext = self._search_regex('[.](.+?)$', video_url, u'extension') + webpage, 'title') - return [{ - 'id': video_id, - 'url': video_url, - 'uploader': None, - 'upload_date': None, - 'title': video_title, - 'ext': video_ext, - }] + return { + 'id': video_id, + 'url': video_url, + 'title': video_title, + } mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage) if mobj is not None: request = compat_urllib_request.Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '') response = self._download_webpage(request, video_id, - u'Downloading video info') + 'Downloading video info') info = json.loads(base64.b64decode(response).decode('utf-8')) - return {'id': video_id, - 'title': info['title'], - 'url': info['streaming_url'].replace('rtmpe', 'rtmpt'), - 'play_path': info['filename'], - 'ext': 'flv', - 'thumbnail': info['thumbnail'][0]['url'], - } + return { + 'id': video_id, + 'title': info['title'], + 'url': info['streaming_url'].replace('rtmpe', 'rtmpt'), + 'play_path': info['filename'], + 'ext': 'flv', + 'thumbnail': info['thumbnail'][0]['url'], + } # try encxml mobj = re.search('var flashvars={(.+?)}', webpage) if mobj is None: - raise ExtractorError(u'Unable to extract video') + raise ExtractorError('Unable to extract video') params = {} encxml = '' @@ -118,7 +111,7 @@ class MyVideoIE(InfoExtractor): params['domain'] = 'www.myvideo.de' xmldata_url = '%s?%s' % (encxml, compat_urllib_parse.urlencode(params)) if 'flash_playertype=MTV' in xmldata_url: - self._downloader.report_warning(u'avoiding MTV player') + self._downloader.report_warning('avoiding MTV player') xmldata_url = ( 'http://www.myvideo.de/dynamic/get_player_video_xml.php' '?flash_playertype=D&ID=%s&_countlimit=4&autorun=yes' @@ -144,7 +137,7 @@ class MyVideoIE(InfoExtractor): video_url = compat_urllib_parse.unquote(mobj.group(1)) if 'myvideo2flash' in video_url: self.report_warning( - u'Rewriting URL to use unencrypted rtmp:// ...', + 'Rewriting URL to use unencrypted rtmp:// ...', video_id) video_url = video_url.replace('rtmpe://', 'rtmp://') @@ -152,39 +145,31 @@ class MyVideoIE(InfoExtractor): # extract non rtmp videos mobj = re.search('path=\'(http.*?)\' source=\'(.*?)\'', dec_data) if mobj is None: - raise ExtractorError(u'unable to extract url') + raise ExtractorError('unable to extract url') video_url = compat_urllib_parse.unquote(mobj.group(1)) + compat_urllib_parse.unquote(mobj.group(2)) - video_file = self._search_regex('source=\'(.*?)\'', dec_data, u'video file') + video_file = self._search_regex('source=\'(.*?)\'', dec_data, 'video file') video_file = compat_urllib_parse.unquote(video_file) if not video_file.endswith('f4m'): ppath, prefix = video_file.split('.') video_playpath = '%s:%s' % (prefix, ppath) - video_hls_playlist = '' else: video_playpath = '' - video_hls_playlist = ( - video_file - ).replace('.f4m', '.m3u8') - video_swfobj = self._search_regex('swfobject.embedSWF\(\'(.+?)\'', webpage, u'swfobj') + video_swfobj = self._search_regex('swfobject.embedSWF\(\'(.+?)\'', webpage, 'swfobj') video_swfobj = compat_urllib_parse.unquote(video_swfobj) video_title = self._html_search_regex("<h1(?: class='globalHd')?>(.*?)</h1>", - webpage, u'title') - - return [{ - 'id': video_id, - 'url': video_url, - 'tc_url': video_url, - 'uploader': None, - 'upload_date': None, - 'title': video_title, - 'ext': u'flv', - 'play_path': video_playpath, - 'video_file': video_file, - 'video_hls_playlist': video_hls_playlist, - 'player_url': video_swfobj, - }] + webpage, 'title') + + return { + 'id': video_id, + 'url': video_url, + 'tc_url': video_url, + 'title': video_title, + 'ext': 'flv', + 'play_path': video_playpath, + 'player_url': video_swfobj, + } diff --git a/youtube_dl/extractor/photobucket.py b/youtube_dl/extractor/photobucket.py index 305b79773..58863e382 100644 --- a/youtube_dl/extractor/photobucket.py +++ b/youtube_dl/extractor/photobucket.py @@ -1,76 +1,43 @@ +from __future__ import unicode_literals + import datetime import json import re from .common import InfoExtractor -from ..utils import ( - ExtractorError, -) class PhotobucketIE(InfoExtractor): - """Information extractor for photobucket.com.""" - - # TODO: the original _VALID_URL was: - # r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)' - # Check if it's necessary to keep the old extracion process - _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P<id>.*)\.(?P<ext>(flv)|(mp4))' - IE_NAME = u'photobucket' + _VALID_URL = r'http://(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P<id>.*)\.(?P<ext>(flv)|(mp4))' _TEST = { u'url': u'http://media.photobucket.com/user/rachaneronas/media/TiredofLinkBuildingTryBacklinkMyDomaincom_zpsc0c3b9fa.mp4.html?filters[term]=search&filters[primary]=videos&filters[secondary]=images&sort=1&o=0', u'file': u'zpsc0c3b9fa.mp4', u'md5': u'7dabfb92b0a31f6c16cebc0f8e60ff99', u'info_dict': { - u"upload_date": u"20130504", - u"uploader": u"rachaneronas", - u"title": u"Tired of Link Building? Try BacklinkMyDomain.com!" + 'upload_date': '20130504', + 'uploader': 'rachaneronas', + 'title': 'Tired of Link Building? Try BacklinkMyDomain.com!', } } def _real_extract(self, url): - # Extract id from URL mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - video_id = mobj.group('id') - video_extension = mobj.group('ext') - # Retrieve video webpage to extract further information webpage = self._download_webpage(url, video_id) # Extract URL, uploader, and title from webpage self.report_extraction(video_id) - # We try first by looking the javascript code: - mobj = re.search(r'Pb\.Data\.Shared\.put\(Pb\.Data\.Shared\.MEDIA, (?P<json>.*?)\);', webpage) - if mobj is not None: - info = json.loads(mobj.group('json')) - return [{ - 'id': video_id, - 'url': info[u'downloadUrl'], - 'uploader': info[u'username'], - 'upload_date': datetime.date.fromtimestamp(info[u'creationDate']).strftime('%Y%m%d'), - 'title': info[u'title'], - 'ext': video_extension, - 'thumbnail': info[u'thumbUrl'], - }] - - # We try looking in other parts of the webpage - video_url = self._search_regex(r'<link rel="video_src" href=".*\?file=([^"]+)" />', - webpage, u'video URL') - - mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage) - if mobj is None: - raise ExtractorError(u'Unable to extract title') - video_title = mobj.group(1).decode('utf-8') - video_uploader = mobj.group(2).decode('utf-8') - - return [{ - 'id': video_id.decode('utf-8'), - 'url': video_url.decode('utf-8'), - 'uploader': video_uploader, - 'upload_date': None, - 'title': video_title, - 'ext': video_extension.decode('utf-8'), - }] + info_json = self._search_regex(r'Pb\.Data\.Shared\.put\(Pb\.Data\.Shared\.MEDIA, (.*?)\);', + webpage, 'info json') + info = json.loads(info_json) + return { + 'id': video_id, + 'url': info[u'downloadUrl'], + 'uploader': info[u'username'], + 'upload_date': datetime.date.fromtimestamp(info[u'creationDate']).strftime('%Y%m%d'), + 'title': info[u'title'], + 'ext': video_extension, + 'thumbnail': info[u'thumbUrl'], + } diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py index fdda69f33..834fe7266 100644 --- a/youtube_dl/extractor/pornhub.py +++ b/youtube_dl/extractor/pornhub.py @@ -44,7 +44,7 @@ class PornHubIE(InfoExtractor): video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage))) if webpage.find('"encrypted":true') != -1: - password = self._html_search_regex(r'"video_title":"([^"]+)', webpage, 'password').replace('+', ' ') + password = compat_urllib_parse.unquote_plus(self._html_search_regex(r'"video_title":"([^"]+)', webpage, 'password')) video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls)) formats = [] diff --git a/youtube_dl/extractor/xnxx.py b/youtube_dl/extractor/xnxx.py index 1177a4b14..7a73b2430 100644 --- a/youtube_dl/extractor/xnxx.py +++ b/youtube_dl/extractor/xnxx.py @@ -1,55 +1,49 @@ +# encoding: utf-8 +from __future__ import unicode_literals + import re from .common import InfoExtractor from ..utils import ( compat_urllib_parse, - - ExtractorError, ) class XNXXIE(InfoExtractor): - _VALID_URL = r'^(?:https?://)?(?:video|www)\.xnxx\.com/video([0-9]+)/(.*)' - VIDEO_URL_RE = r'flv_url=(.*?)&' - VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM' - VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&' + _VALID_URL = r'^https?://(?:video|www)\.xnxx\.com/video(?P<id>[0-9]+)/(.*)' _TEST = { - u'url': u'http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_', - u'file': u'1135332.flv', - u'md5': u'0831677e2b4761795f68d417e0b7b445', - u'info_dict': { - u"title": u"lida \u00bb Naked Funny Actress (5)", - u"age_limit": 18, + 'url': 'http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_', + 'md5': '0831677e2b4761795f68d417e0b7b445', + 'info_dict': { + 'id': '1135332', + 'ext': 'flv', + 'title': 'lida ยป Naked Funny Actress (5)', + 'age_limit': 18, } } def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - if mobj is None: - raise ExtractorError(u'Invalid URL: %s' % url) - video_id = mobj.group(1) + video_id = mobj.group('id') # Get webpage content webpage = self._download_webpage(url, video_id) - video_url = self._search_regex(self.VIDEO_URL_RE, - webpage, u'video URL') + video_url = self._search_regex(r'flv_url=(.*?)&', + webpage, 'video URL') video_url = compat_urllib_parse.unquote(video_url) - video_title = self._html_search_regex(self.VIDEO_TITLE_RE, - webpage, u'title') + video_title = self._html_search_regex(r'<title>(.*?)\s+-\s+XNXX.COM', + webpage, 'title') - video_thumbnail = self._search_regex(self.VIDEO_THUMB_RE, - webpage, u'thumbnail', fatal=False) + video_thumbnail = self._search_regex(r'url_bigthumb=(.*?)&', + webpage, 'thumbnail', fatal=False) - return [{ + return { 'id': video_id, 'url': video_url, - 'uploader': None, - 'upload_date': None, 'title': video_title, 'ext': 'flv', 'thumbnail': video_thumbnail, - 'description': None, 'age_limit': 18, - }] + } |