aboutsummaryrefslogtreecommitdiff
path: root/youtube_dl
diff options
context:
space:
mode:
Diffstat (limited to 'youtube_dl')
-rw-r--r--youtube_dl/extractor/facebook.py71
-rw-r--r--youtube_dl/extractor/faz.py6
-rw-r--r--youtube_dl/extractor/generic.py20
-rw-r--r--youtube_dl/extractor/rtlnow.py11
4 files changed, 67 insertions, 41 deletions
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 9d1bc0751..f8bdfc2d3 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -19,7 +19,8 @@ class FacebookIE(InfoExtractor):
"""Information Extractor for Facebook"""
_VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
- _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
+ _LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1'
+ _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1'
_NETRC_MACHINE = 'facebook'
IE_NAME = u'facebook'
_TEST = {
@@ -36,50 +37,56 @@ class FacebookIE(InfoExtractor):
"""Report attempt to log in."""
self.to_screen(u'Logging in')
- def _real_initialize(self):
- if self._downloader is None:
- return
-
- useremail = None
- password = None
- downloader_params = self._downloader.params
-
- # Attempt to use provided username and password or .netrc data
- if downloader_params.get('username', None) is not None:
- useremail = downloader_params['username']
- password = downloader_params['password']
- elif downloader_params.get('usenetrc', False):
- try:
- info = netrc.netrc().authenticators(self._NETRC_MACHINE)
- if info is not None:
- useremail = info[0]
- password = info[2]
- else:
- raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
- except (IOError, netrc.NetrcParseError) as err:
- self._downloader.report_warning(u'parsing .netrc: %s' % compat_str(err))
- return
-
+ def _login(self):
+ (useremail, password) = self._get_login_info()
if useremail is None:
return
- # Log in
+ login_page_req = compat_urllib_request.Request(self._LOGIN_URL)
+ login_page_req.add_header('Cookie', 'locale=en_US')
+ self.report_login()
+ login_page = self._download_webpage(login_page_req, None, note=False,
+ errnote=u'Unable to download login page')
+ lsd = self._search_regex(r'"lsd":"(\w*?)"', login_page, u'lsd')
+ lgnrnd = self._search_regex(r'name="lgnrnd" value="([^"]*?)"', login_page, u'lgnrnd')
+
login_form = {
'email': useremail,
'pass': password,
- 'login': 'Log+In'
+ 'lsd': lsd,
+ 'lgnrnd': lgnrnd,
+ 'next': 'http://facebook.com/home.php',
+ 'default_persistent': '0',
+ 'legacy_return': '1',
+ 'timezone': '-60',
+ 'trynum': '1',
}
request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
+ request.add_header('Content-Type', 'application/x-www-form-urlencoded')
try:
- self.report_login()
login_results = compat_urllib_request.urlopen(request).read()
if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
self._downloader.report_warning(u'unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
return
+
+ check_form = {
+ 'fb_dtsg': self._search_regex(r'"fb_dtsg":"(.*?)"', login_results, u'fb_dtsg'),
+ 'nh': self._search_regex(r'name="nh" value="(\w*?)"', login_results, u'nh'),
+ 'name_action_selected': 'dont_save',
+ 'submit[Continue]': self._search_regex(r'<input value="(.*?)" name="submit\[Continue\]"', login_results, u'continue'),
+ }
+ check_req = compat_urllib_request.Request(self._CHECKPOINT_URL, compat_urllib_parse.urlencode(check_form))
+ check_req.add_header('Content-Type', 'application/x-www-form-urlencoded')
+ check_response = compat_urllib_request.urlopen(check_req).read()
+ if re.search(r'id="checkpointSubmitButton"', check_response) is not None:
+ self._downloader.report_warning(u'Unable to confirm login, you have to login in your brower and authorize the login.')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.report_warning(u'unable to log in: %s' % compat_str(err))
return
+ def _real_initialize(self):
+ self._login()
+
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
if mobj is None:
@@ -93,7 +100,13 @@ class FacebookIE(InfoExtractor):
AFTER = '.forEach(function(variable) {swf.addVariable(variable[0], variable[1]);});'
m = re.search(re.escape(BEFORE) + '(.*?)' + re.escape(AFTER), webpage)
if not m:
- raise ExtractorError(u'Cannot parse data')
+ m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
+ if m_msg is not None:
+ raise ExtractorError(
+ u'The video is not available, Facebook said: "%s"' % m_msg.group(1),
+ expected=True)
+ else:
+ raise ExtractorError(u'Cannot parse data')
data = dict(json.loads(m.group(1)))
params_raw = compat_urllib_parse.unquote(data['params'])
params = json.loads(params_raw)
diff --git a/youtube_dl/extractor/faz.py b/youtube_dl/extractor/faz.py
index deaa4ed2d..89ed08db4 100644
--- a/youtube_dl/extractor/faz.py
+++ b/youtube_dl/extractor/faz.py
@@ -5,8 +5,6 @@ import xml.etree.ElementTree
from .common import InfoExtractor
from ..utils import (
determine_ext,
- clean_html,
- get_element_by_attribute,
)
@@ -47,12 +45,12 @@ class FazIE(InfoExtractor):
'format_id': code.lower(),
})
- descr_html = get_element_by_attribute('class', 'Content Copy', webpage)
+ descr = self._html_search_regex(r'<p class="Content Copy">(.*?)</p>', webpage, u'description')
info = {
'id': video_id,
'title': self._og_search_title(webpage),
'formats': formats,
- 'description': clean_html(descr_html),
+ 'description': descr,
'thumbnail': config.find('STILL/STILL_BIG').text,
}
# TODO: Remove when #980 has been merged
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 69e0a7bd2..2c8fcf5ae 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -25,7 +25,7 @@ class GenericIE(InfoExtractor):
{
u'url': u'http://www.hodiho.fr/2013/02/regis-plante-sa-jeep.html',
u'file': u'13601338388002.mp4',
- u'md5': u'85b90ccc9d73b4acd9138d3af4c27f89',
+ u'md5': u'6e15c93721d7ec9e9ca3fdbf07982cfd',
u'info_dict': {
u"uploader": u"www.hodiho.fr",
u"title": u"R\u00e9gis plante sa Jeep"
@@ -41,7 +41,17 @@ class GenericIE(InfoExtractor):
u"uploader_id": u"skillsmatter",
u"uploader": u"Skills Matter",
}
- }
+ },
+ # bandcamp page with custom domain
+ {
+ u'url': u'http://bronyrock.com/track/the-pony-mash',
+ u'file': u'3235767654.mp3',
+ u'info_dict': {
+ u'title': u'The Pony Mash',
+ u'uploader': u'M_Pallante',
+ },
+ u'skip': u'There is a limit of 200 free downloads / month for the test song',
+ },
]
def report_download_webpage(self, video_id):
@@ -155,6 +165,12 @@ class GenericIE(InfoExtractor):
surl = unescapeHTML(mobj.group(1))
return self.url_result(surl, 'Youtube')
+ # Look for Bandcamp pages with custom domain
+ mobj = re.search(r'<meta property="og:url"[^>]*?content="(.*?bandcamp\.com.*?)"', webpage)
+ if mobj is not None:
+ burl = unescapeHTML(mobj.group(1))
+ return self.url_result(burl, 'Bandcamp')
+
# Start with something easy: JW Player in SWFObject
mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
if mobj is None:
diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py
index d1b08c9bc..9ac7c3be8 100644
--- a/youtube_dl/extractor/rtlnow.py
+++ b/youtube_dl/extractor/rtlnow.py
@@ -63,13 +63,12 @@ class RTLnowIE(InfoExtractor):
},
},
{
- u'url': u'http://www.rtlnitronow.de/recht-ordnung/lebensmittelkontrolle-erlangenordnungsamt-berlin.php?film_id=127367&player=1&season=1',
- u'file': u'127367.flv',
+ u'url': u'http://www.rtlnitronow.de/recht-ordnung/stadtpolizei-frankfurt-gerichtsvollzieher-leipzig.php?film_id=129679&player=1&season=1',
+ u'file': u'129679.flv',
u'info_dict': {
- u'upload_date': u'20130926',
- u'title': u'Recht & Ordnung - Lebensmittelkontrolle Erlangen/Ordnungsamt...',
- u'description': u'Lebensmittelkontrolle Erlangen/Ordnungsamt Berlin',
- u'thumbnail': u'http://autoimg.static-fra.de/nitronow/344787/1500x1500/image2.jpg',
+ u'upload_date': u'20131016',
+ u'title': u'Recht & Ordnung - Stadtpolizei Frankfurt/ Gerichtsvollzieher...',
+ u'description': u'Stadtpolizei Frankfurt/ Gerichtsvollzieher Leipzig',
},
u'params': {
u'skip_download': True,
generated by cgit v1.2.3 (git 2.26.2) at 2024-09-01 13:18:13 +0000