aboutsummaryrefslogtreecommitdiff
path: root/youtube_dl/extractor
diff options
context:
space:
mode:
Diffstat (limited to 'youtube_dl/extractor')
-rw-r--r--youtube_dl/extractor/addanime.py10
-rw-r--r--youtube_dl/extractor/exfm.py14
-rw-r--r--youtube_dl/extractor/facebook.py71
-rw-r--r--youtube_dl/extractor/faz.py6
-rw-r--r--youtube_dl/extractor/generic.py20
-rw-r--r--youtube_dl/extractor/rtlnow.py11
-rw-r--r--youtube_dl/extractor/xhamster.py52
7 files changed, 117 insertions, 67 deletions
diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py
index 82a785a19..465df8cf0 100644
--- a/youtube_dl/extractor/addanime.py
+++ b/youtube_dl/extractor/addanime.py
@@ -17,8 +17,8 @@ class AddAnimeIE(InfoExtractor):
IE_NAME = u'AddAnime'
_TEST = {
u'url': u'http://www.add-anime.net/watch_video.php?v=24MR3YO5SAS9',
- u'file': u'24MR3YO5SAS9.flv',
- u'md5': u'1036a0e0cd307b95bd8a8c3a5c8cfaf1',
+ u'file': u'24MR3YO5SAS9.mp4',
+ u'md5': u'72954ea10bc979ab5e2eb288b21425a0',
u'info_dict': {
u"description": u"One Piece 606",
u"title": u"One Piece 606"
@@ -60,8 +60,10 @@ class AddAnimeIE(InfoExtractor):
note=u'Confirming after redirect')
webpage = self._download_webpage(url, video_id)
- video_url = self._search_regex(r"var normal_video_file = '(.*?)';",
+ video_url = self._search_regex(r"var (?:hq|normal)_video_file = '(.*?)';",
webpage, u'video file URL')
+
+ video_extension = video_url[-3:] # mp4 or flv ?
video_title = self._og_search_title(webpage)
video_description = self._og_search_description(webpage)
@@ -69,7 +71,7 @@ class AddAnimeIE(InfoExtractor):
'_type': 'video',
'id': video_id,
'url': video_url,
- 'ext': 'flv',
+ 'ext': video_extension,
'title': video_title,
'description': video_description
}
diff --git a/youtube_dl/extractor/exfm.py b/youtube_dl/extractor/exfm.py
index 3443f19c5..c74556579 100644
--- a/youtube_dl/extractor/exfm.py
+++ b/youtube_dl/extractor/exfm.py
@@ -11,14 +11,14 @@ class ExfmIE(InfoExtractor):
_SOUNDCLOUD_URL = r'(?:http://)?(?:www\.)?api\.soundcloud.com/tracks/([^/]+)/stream'
_TESTS = [
{
- u'url': u'http://ex.fm/song/1bgtzg',
- u'file': u'95223130.mp3',
- u'md5': u'8a7967a3fef10e59a1d6f86240fd41cf',
+ u'url': u'http://ex.fm/song/eh359',
+ u'file': u'44216187.mp3',
+ u'md5': u'e45513df5631e6d760970b14cc0c11e7',
u'info_dict': {
- u"title": u"We Can't Stop - Miley Cyrus",
- u"uploader": u"Miley Cyrus",
- u'upload_date': u'20130603',
- u'description': u'Download "We Can\'t Stop" \r\niTunes: http://smarturl.it/WeCantStop?IQid=SC\r\nAmazon: http://smarturl.it/WeCantStopAMZ?IQid=SC',
+ u"title": u"Test House \"Love Is Not Enough\" (Extended Mix) DeadJournalist Exclusive",
+ u"uploader": u"deadjournalist",
+ u'upload_date': u'20120424',
+ u'description': u'Test House \"Love Is Not Enough\" (Extended Mix) DeadJournalist Exclusive',
},
u'note': u'Soundcloud song',
},
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 9d1bc0751..f8bdfc2d3 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -19,7 +19,8 @@ class FacebookIE(InfoExtractor):
"""Information Extractor for Facebook"""
_VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
- _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
+ _LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
+ _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
_NETRC_MACHINE = 'facebook'
IE_NAME = u'facebook'
_TEST = {
@@ -36,50 +37,56 @@ class FacebookIE(InfoExtractor):
"""Report attempt to log in."""
self.to_screen(u'Logging in')
- def _real_initialize(self):
- if self._downloader is None:
- return
-
- useremail = None
- password = None
- downloader_params = self._downloader.params
-
- # Attempt to use provided username and password or .netrc data
- if downloader_params.get('username', None) is not None:
- useremail = downloader_params['username']
- password = downloader_params['password']
- elif downloader_params.get('usenetrc', False):
- try:
- info = netrc.netrc().authenticators(self._NETRC_MACHINE)
- if info is not None:
- useremail = info[0]
- password = info[2]
- else:
- raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
- except (IOError, netrc.NetrcParseError) as err:
- self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err))
- return
-
+ def _login(self):
+ (useremail, password) = self._get_login_info()
if useremail is None:
return
- # Log in
+ login_page_req = compat_urllib_request.Request(self._LOGIN_URL)
+ login_page_req.add_header('Cookie', 'locale=en_US')
+ self.report_login()
+ login_page = self._download_webpage(login_page_req, None, note=False,
+ errnote=u'Unable to download login page')
+ lsd = self._search_regex(r'"lsd":"(\w*?)"', login_page, u'lsd')
+ lgnrnd = self._search_regex(r'name="lgnrnd" value="([^"]*?)"', login_page, u'lgnrnd')
+
login_form = {
'email': useremail,
'pass': password,
- 'login': 'Log+In'
+ 'lsd': lsd,
+ 'lgnrnd': lgnrnd,
+ 'next': 'http://facebook.com/home.php',
+ 'default_persistent': '0',
+ 'legacy_return': '1',
+ 'timezone': '-60',
+ 'trynum': '1',
}
request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+ request.add_header('Content-Type', 'application/x-www-form-urlencoded')
try:
- self.report_login()
login_results = compat_urllib_request.urlopen(request).read()
if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
self._downloader.report_warning(u'unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
return
+
+ check_form = {
+ 'fb_dtsg': self._search_regex(r'"fb_dtsg":"(.*?)"', login_results, u'fb_dtsg'),
+ 'nh': self._search_regex(r'name="nh" value="(\w*?)"', login_results, u'nh'),
+ 'name_action_selected': 'dont_save',
+ 'submit[Continue]': self._search_regex(r'<input value="(.*?)" name="submit\[Continue\]"', login_results, u'continue'),
+ }
+ check_req = compat_urllib_request.Request(self._CHECKPOINT_URL, compat_urllib_parse.urlencode(check_form))
+ check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ check_response = compat_urllib_request.urlopen(check_req).read()
+ if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
+ self._downloader.report_warning(u'Unable to confirm login, you have to login in your brower and authorize the login.')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.report_warning(u'unable to log in: %s' % compat_str(err))
return
+ def _real_initialize(self):
+ self._login()
+
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
if mobj is None:
@@ -93,7 +100,13 @@ class FacebookIE(InfoExtractor):
AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});'
m = re.search(re.escape(BEFORE) + '(.*?)' + re.escape(AFTER), webpage)
if not m:
- raise ExtractorError(u'Cannot parse data')
+ m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
+ if m_msg is not None:
+ raise ExtractorError(
+ u'The video is not available, Facebook said: "%s"' % m_msg.group(1),
+ expected=True)
+ else:
+ raise ExtractorError(u'Cannot parse data')
data = dict(json.loads(m.group(1)))
params_raw = compat_urllib_parse.unquote(data['params'])
params = json.loads(params_raw)
diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index deaa4ed2d..89ed08db4 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -5,8 +5,6 @@ import xml.etree.ElementTree
from .common import InfoExtractor
from ..utils import (
determine_ext,
- clean_html,
- get_element_by_attribute,
)
@@ -47,12 +45,12 @@ class FazIE(InfoExtractor):
'format_id': code.lower(),
})
- descr_html = get_element_by_attribute('class', 'Content Copy', webpage)
+ descr = self._html_search_regex(r'<p class="Content Copy">(.*?)</p>', webpage, u'description')
info = {
'id': video_id,
'title': self._og_search_title(webpage),
'formats': formats,
- 'description': clean_html(descr_html),
+ 'description': descr,
'thumbnail': config.find('STILL/STILL_BIG').text,
}
# TODO: Remove when #980 has been merged
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 69e0a7bd2..2c8fcf5ae 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -25,7 +25,7 @@ class GenericIE(InfoExtractor):
{
u'url': u'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
u'file': u'13601338388002.mp4',
- u'md5': u'85b90ccc9d73b4acd9138d3af4c27f89',
+ u'md5': u'6e15c93721d7ec9e9ca3fdbf07982cfd',
u'info_dict': {
u"uploader": u"www.hodiho.fr",
u"title": u"R\u00e9gis plante sa Jeep"
@@ -41,7 +41,17 @@ class GenericIE(InfoExtractor):
u"uploader_id": u"skillsmatter",
u"uploader": u"Skills Matter",
}
- }
+ },
+ # bandcamp page with custom domain
+ {
+ u'url': u'http://bronyrock.com/track/the-pony-mash',
+ u'file': u'3235767654.mp3',
+ u'info_dict': {
+ u'title': u'The Pony Mash',
+ u'uploader': u'M_Pallante',
+ },
+ u'skip': u'There is a limit of 200 free downloads / month for the test song',
+ },
]
def report_download_webpage(self, video_id):
@@ -155,6 +165,12 @@ class GenericIE(InfoExtractor):
surl = unescapeHTML(mobj.group(1))
return self.url_result(surl, 'Youtube')
+ # Look for Bandcamp pages with custom domain
+ mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
+ if mobj is not None:
+ burl = unescapeHTML(mobj.group(1))
+ return self.url_result(burl, 'Bandcamp')
+
# Start with something easy: JW Player in SWFObject
mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
if mobj is None:
diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index d1b08c9bc..9ac7c3be8 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -63,13 +63,12 @@ class RTLnowIE(InfoExtractor):
},
},
{
- u'url': u'http://www.rtlnitronow.de/recht-ordnung/lebensmittelkontrolle-erlangenordnungsamt-berlin.php?film_id=127367&player=1&season=1',
- u'file': u'127367.flv',
+ u'url': u'http://www.rtlnitronow.de/recht-ordnung/stadtpolizei-frankfurt-gerichtsvollzieher-leipzig.php?film_id=129679&player=1&season=1',
+ u'file': u'129679.flv',
u'info_dict': {
- u'upload_date': u'20130926',
- u'title': u'Recht & Ordnung - Lebensmittelkontrolle Erlangen/Ordnungsamt...',
- u'description': u'Lebensmittelkontrolle Erlangen/Ordnungsamt Berlin',
- u'thumbnail': u'http://autoimg.static-fra.de/nitronow/344787/1500x1500/image2.jpg',
+ u'upload_date': u'20131016',
+ u'title': u'Recht & Ordnung - Stadtpolizei Frankfurt/ Gerichtsvollzieher...',
+ u'description': u'Stadtpolizei Frankfurt/ Gerichtsvollzieher Leipzig',
},
u'params': {
u'skip_download': True,
diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 81c4be326..7444d3393 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -36,21 +36,25 @@ class XHamsterIE(InfoExtractor):
}]
def _real_extract(self,url):
+ def extract_video_url(webpage):
+ mobj = re.search(r'\'srv\': \'(?P<server>[^\']*)\',\s*\'file\': \'(?P<file>[^\']+)\',', webpage)
+ if mobj is None:
+ raise ExtractorError(u'Unable to extract media URL')
+ if len(mobj.group('server')) == 0:
+ return compat_urllib_parse.unquote(mobj.group('file'))
+ else:
+ return mobj.group('server')+'/key='+mobj.group('file')
+
+ def is_hd(webpage):
+ return webpage.find('<div class=\'icon iconHD\'>') != -1
+
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
seo = mobj.group('seo')
- mrss_url = 'http://xhamster.com/movies/%s/%s.html?hd' % (video_id, seo)
+ mrss_url = 'http://xhamster.com/movies/%s/%s.html' % (video_id, seo)
webpage = self._download_webpage(mrss_url, video_id)
- mobj = re.search(r'\'srv\': \'(?P<server>[^\']*)\',\s*\'file\': \'(?P<file>[^\']+)\',', webpage)
- if mobj is None:
- raise ExtractorError(u'Unable to extract media URL')
- if len(mobj.group('server')) == 0:
- video_url = compat_urllib_parse.unquote(mobj.group('file'))
- else:
- video_url = mobj.group('server')+'/key='+mobj.group('file')
-
video_title = self._html_search_regex(r'<title>(?P<title>.+?) - xHamster\.com</title>',
webpage, u'title')
@@ -76,14 +80,32 @@ class XHamsterIE(InfoExtractor):
age_limit = self._rta_search(webpage)
- return [{
- 'id': video_id,
- 'url': video_url,
- 'ext': determine_ext(video_url),
- 'title': video_title,
+ video_url = extract_video_url(webpage)
+ hd = is_hd(webpage)
+ formats = [{
+ 'url': video_url,
+ 'ext': determine_ext(video_url),
+ 'format': 'hd' if hd else 'sd',
+ 'format_id': 'hd' if hd else 'sd',
+ }]
+ if not hd:
+ webpage = self._download_webpage(mrss_url+'?hd', video_id)
+ if is_hd(webpage):
+ video_url = extract_video_url(webpage)
+ formats.append({
+ 'url': video_url,
+ 'ext': determine_ext(video_url),
+ 'format': 'hd',
+ 'format_id': 'hd',
+ })
+
+ return {
+ 'id': video_id,
+ 'title': video_title,
+ 'formats': formats,
'description': video_description,
'upload_date': video_upload_date,
'uploader_id': video_uploader_id,
'thumbnail': video_thumbnail,
'age_limit': age_limit,
- }]
+ }
generated by cgit v1.2.3 (git 2.26.2) at 2024-11-02 04:14:17 +0000