diff options
| -rw-r--r-- | youtube_dl/extractor/__init__.py | 1 | ||||
| -rw-r--r-- | youtube_dl/extractor/common.py | 7 | ||||
| -rw-r--r-- | youtube_dl/extractor/nerdist.py | 80 | ||||
| -rw-r--r-- | youtube_dl/extractor/normalboots.py | 27 |
4 files changed, 100 insertions, 15 deletions
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 205bd4338..5dcb14feb 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -285,6 +285,7 @@ from .ndr import NDRIE from .ndtv import NDTVIE from .netzkino import NetzkinoIE from .nerdcubed import NerdCubedFeedIE +from .nerdist import NerdistIE from .newgrounds import NewgroundsIE from .newstube import NewstubeIE from .nextmedia import ( diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 8b4ef3f09..9e517e3ac 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -860,10 +860,13 @@ class InfoExtractor(object): return formats # TODO: improve extraction - def _extract_smil_formats(self, smil_url, video_id): + def _extract_smil_formats(self, smil_url, video_id, fatal=True): smil = self._download_xml( smil_url, video_id, 'Downloading SMIL file', - 'Unable to download SMIL file') + 'Unable to download SMIL file', fatal=fatal) + if smil is False: + assert not fatal + return [] base = smil.find('./head/meta').get('base') diff --git a/youtube_dl/extractor/nerdist.py b/youtube_dl/extractor/nerdist.py new file mode 100644 index 000000000..c6dc34be4 --- /dev/null +++ b/youtube_dl/extractor/nerdist.py @@ -0,0 +1,80 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +from .common import InfoExtractor + +from ..utils import ( + determine_ext, + parse_iso8601, + xpath_text, +) + + +class NerdistIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?nerdist\.com/vepisode/(?P<id>[^/?#]+)' + _TEST = { + 'url': 'http://www.nerdist.com/vepisode/exclusive-which-dc-characters-w', + 'md5': '3698ed582931b90d9e81e02e26e89f23', + 'info_dict': { + 'display_id': 'exclusive-which-dc-characters-w', + 'id': 'RPHpvJyr', + 'ext': 'mp4', + 'title': 'Your TEEN TITANS Revealed! Who\'s on the show?', + 'thumbnail': 're:^https?://.*/thumbs/.*\.jpg$', + 'description': 'Exclusive: Find out which DC Comics superheroes will star in TEEN TITANS Live-Action TV Show on Nerdist News with Jessica Chobot!', + 'uploader': 'Eric Diaz', + 'upload_date': '20150202', + 'timestamp': 1422892808, + } + } + + def _real_extract(self, url): + display_id = self._match_id(url) + webpage = self._download_webpage(url, display_id) + + video_id = self._search_regex( + r'''(?x)<script\s+(?:type="text/javascript"\s+)? + src="https?://content\.nerdist\.com/players/([a-zA-Z0-9_]+)-''', + webpage, 'video ID') + timestamp = parse_iso8601(self._html_search_meta( + 'shareaholic:article_published_time', webpage, 'upload date')) + uploader = self._html_search_meta( + 'shareaholic:article_author_name', webpage, 'article author') + + doc = self._download_xml( + 'http://content.nerdist.com/jw6/%s.xml' % video_id, video_id) + video_info = doc.find('.//item') + title = xpath_text(video_info, './title', fatal=True) + description = xpath_text(video_info, './description') + thumbnail = xpath_text( + video_info, './{http://rss.jwpcdn.com/}image', 'thumbnail') + + formats = [] + for source in video_info.findall('./{http://rss.jwpcdn.com/}source'): + vurl = source.attrib['file'] + ext = determine_ext(vurl) + if ext == 'm3u8': + formats.extend(self._extract_m3u8_formats( + vurl, video_id, entry_protocol='m3u8_native', ext='mp4', + preference=0)) + elif ext == 'smil': + formats.extend(self._extract_smil_formats( + vurl, video_id, fatal=False + )) + else: + formats.append({ + 'format_id': ext, + 'url': vurl, + }) + self._sort_formats(formats) + + return { + 'id': video_id, + 'display_id': display_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'timestamp': timestamp, + 'formats': formats, + 'uploader': uploader, + } diff --git a/youtube_dl/extractor/normalboots.py b/youtube_dl/extractor/normalboots.py index c13ff0d65..5952d136f 100644 --- a/youtube_dl/extractor/normalboots.py +++ b/youtube_dl/extractor/normalboots.py @@ -1,8 +1,6 @@ # encoding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor from ..utils import ( @@ -11,7 +9,7 @@ from ..utils import ( class NormalbootsIE(InfoExtractor): - _VALID_URL = r'http://(?:www\.)?normalboots\.com/video/(?P<videoid>[0-9a-z-]*)/?$' + _VALID_URL = r'http://(?:www\.)?normalboots\.com/video/(?P<id>[0-9a-z-]*)/?$' _TEST = { 'url': 'http://normalboots.com/video/home-alone-games-jontron/', 'md5': '8bf6de238915dd501105b44ef5f1e0f6', @@ -30,19 +28,22 @@ class NormalbootsIE(InfoExtractor): } def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - video_id = mobj.group('videoid') - + video_id = self._match_id(url) webpage = self._download_webpage(url, video_id) - video_uploader = self._html_search_regex(r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>', - webpage, 'uploader') - raw_upload_date = self._html_search_regex('<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>', - webpage, 'date') - video_upload_date = unified_strdate(raw_upload_date) - player_url = self._html_search_regex(r'<iframe\swidth="[0-9]+"\sheight="[0-9]+"\ssrc="(?P<url>[\S]+)"', webpage, 'url') + video_uploader = self._html_search_regex( + r'Posted\sby\s<a\shref="[A-Za-z0-9/]*">(?P<uploader>[A-Za-z]*)\s</a>', + webpage, 'uploader', fatal=False) + video_upload_date = unified_strdate(self._html_search_regex( + r'<span style="text-transform:uppercase; font-size:inherit;">[A-Za-z]+, (?P<date>.*)</span>', + webpage, 'date', fatal=False)) + + player_url = self._html_search_regex( + r'<iframe\swidth="[0-9]+"\sheight="[0-9]+"\ssrc="(?P<url>[\S]+)"', + webpage, 'player url') player_page = self._download_webpage(player_url, video_id) - video_url = self._html_search_regex(r"file:\s'(?P<file>[^']+\.mp4)'", player_page, 'file') + video_url = self._html_search_regex( + r"file:\s'(?P<file>[^']+\.mp4)'", player_page, 'file') return { 'id': video_id, |