aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSergey M․ <dstftw@gmail.com>2018-06-02 01:32:18 +0700
committerSergey M․ <dstftw@gmail.com>2018-06-02 01:42:05 +0700
commit9b89daefa6fc3dbfce0d725283ecef753a8603ba (patch)
tree27d3fe5e83f5f366d5e62bac0f559c3adfc7b19b
parent9d082e7cb81c63dc5c2e616cd7ca7237a5e0d642 (diff)
downloadyoutube-dl-9b89daefa6fc3dbfce0d725283ecef753a8603ba.tar.xz
[facebook] Improve extraction (closes #16554)
-rw-r--r--youtube_dl/extractor/facebook.py62
1 files changed, 35 insertions, 27 deletions
diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index 8bbca4f56..8a9ed96c2 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -324,47 +324,56 @@ class FacebookIE(InfoExtractor):
if server_js_data:
video_data = extract_video_data(server_js_data.get('instances', []))
+ def extract_from_jsmods_instances(js_data):
+ if js_data:
+ return extract_video_data(try_get(
+ js_data, lambda x: x['jsmods']['instances'], list) or [])
+
if not video_data:
server_js_data = self._parse_json(
self._search_regex(
r'bigPipe\.onPageletArrive\(({.+?})\)\s*;\s*}\s*\)\s*,\s*["\']onPageletArrive\s+(?:stream_pagelet|pagelet_group_mall|permalink_video_pagelet)',
webpage, 'js data', default='{}'),
video_id, transform_source=js_to_json, fatal=False)
- if server_js_data:
- video_data = extract_video_data(try_get(
- server_js_data, lambda x: x['jsmods']['instances'],
- list) or [])
+ video_data = extract_from_jsmods_instances(server_js_data)
if not video_data:
- # video info not in first request, do a secondary request using tahoe player specific url
+ if not fatal_if_no_video:
+ return webpage, False
+ m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
+ if m_msg is not None:
+ raise ExtractorError(
+ 'The video is not available, Facebook said: "%s"' % m_msg.group(1),
+ expected=True)
+ elif '>You must log in to continue' in webpage:
+ self.raise_login_required()
+
+ # Video info not in first request, do a secondary request using
+ # tahoe player specific URL
tahoe_data = self._download_webpage(
self._VIDEO_PAGE_TAHOE_TEMPLATE % video_id, video_id,
data=urlencode_postdata({
'__user': 0,
'__a': 1,
- '__pc': self._search_regex(r'"pkg_cohort":"(.*?)"', webpage, 'pkg cohort', default='PHASED:DEFAULT'),
- '__rev': self._search_regex(r'"client_revision":(\d+),', webpage, 'client revision', default=3944515),
+ '__pc': self._search_regex(
+ r'pkg_cohort["\']\s*:\s*["\'](.+?)["\']', webpage,
+ 'pkg cohort', default='PHASED:DEFAULT'),
+ '__rev': self._search_regex(
+ r'client_revision["\']\s*:\s*(\d+),', webpage,
+ 'client revision', default='3944515'),
}),
headers={
'Content-Type': 'application/x-www-form-urlencoded',
})
- tahoe_js_data = self._parse_json(self._search_regex(
- r'for \(;;\);(.+)', tahoe_data,
- 'tahoe js data', default='{}'), video_id, fatal=False)
- video_data = extract_video_data(tahoe_js_data.get('jsmods', {}).get('instances', []))
+ tahoe_js_data = self._parse_json(
+ self._search_regex(
+ r'for\s+\(\s*;\s*;\s*\)\s*;(.+)', tahoe_data,
+ 'tahoe js data', default='{}'),
+ video_id, fatal=False)
+ video_data = extract_from_jsmods_instances(tahoe_js_data)
if not video_data:
- if not fatal_if_no_video:
- return webpage, False
- m_msg = re.search(r'class="[^"]*uiInterstitialContent[^"]*"><div>(.*?)</div>', webpage)
- if m_msg is not None:
- raise ExtractorError(
- 'The video is not available, Facebook said: "%s"' % m_msg.group(1),
- expected=True)
- elif '>You must log in to continue' in webpage:
- self.raise_login_required()
- else:
- raise ExtractorError('Cannot parse data')
+ raise ExtractorError('Cannot parse data')
formats = []
for f in video_data:
@@ -408,11 +417,10 @@ class FacebookIE(InfoExtractor):
video_title = limit_length(video_title, 80)
else:
video_title = 'Facebook video #%s' % video_id
- uploader = clean_html(get_element_by_id('fbPhotoPageAuthorName', webpage))
- if not uploader:
- uploader = self._search_regex(
- [r'ownerName\s*:\s*"([^"]+)"', r'property="og:title"\s*content="(.*?)"'],
- webpage, 'uploader', fatal=False)
+ uploader = clean_html(get_element_by_id(
+ 'fbPhotoPageAuthorName', webpage)) or self._search_regex(
+ r'ownerName\s*:\s*"([^"]+)"', webpage, 'uploader',
+ fatal=False) or self._og_search_title(webpage, fatal=False)
timestamp = int_or_none(self._search_regex(
r'<abbr[^>]+data-utime=["\'](\d+)', webpage,
'timestamp', default=None))