diff options
| -rw-r--r-- | youtube_dl/downloader/__init__.py | 3 | ||||
| -rw-r--r-- | youtube_dl/downloader/hls.py | 47 | ||||
| -rw-r--r-- | youtube_dl/extractor/common.py | 16 | ||||
| -rw-r--r-- | youtube_dl/extractor/vevo.py | 40 |
4 files changed, 102 insertions, 4 deletions
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py index 4ea5811a5..3f941596e 100644 --- a/youtube_dl/downloader/__init__.py +++ b/youtube_dl/downloader/__init__.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from .common import FileDownloader from .hls import HlsFD +from .hls import NativeHlsFD from .http import HttpFD from .mplayer import MplayerFD from .rtmp import RtmpFD @@ -19,6 +20,8 @@ def get_suitable_downloader(info_dict): if url.startswith('rtmp'): return RtmpFD + if protocol == 'm3u8_native': + return NativeHlsFD if (protocol == 'm3u8') or (protocol is None and determine_ext(url) == 'm3u8'): return HlsFD if url.startswith('mms') or url.startswith('rtsp'): diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py index 32852f333..8040bdf08 100644 --- a/youtube_dl/downloader/hls.py +++ b/youtube_dl/downloader/hls.py @@ -1,8 +1,12 @@ +from __future__ import unicode_literals + import os +import re import subprocess from .common import FileDownloader from ..utils import ( + compat_urlparse, check_executable, encodeFilename, ) @@ -43,3 +47,46 @@ class HlsFD(FileDownloader): self.to_stderr(u"\n") self.report_error(u'%s exited with code %d' % (program, retval)) return False + + +class NativeHlsFD(FileDownloader): + """ A more limited implementation that does not require ffmpeg """ + + def real_download(self, filename, info_dict): + url = info_dict['url'] + self.report_destination(filename) + tmpfilename = self.temp_name(filename) + + self.to_screen( + '[hlsnative] %s: Downloading m3u8 manifest' % info_dict['id']) + data = self.ydl.urlopen(url).read() + s = data.decode('utf-8', 'ignore') + segment_urls = [] + for line in s.splitlines(): + line = line.strip() + if line and not line.startswith('#'): + segment_url = ( + line + if re.match(r'^https?://', line) + else compat_urlparse.urljoin(url, line)) + segment_urls.append(segment_url) + + byte_counter = 0 + with open(tmpfilename, 'wb') as outf: + for i, segurl in enumerate(segment_urls): + segment = self.ydl.urlopen(segurl).read() + outf.write(segment) + byte_counter += len(segment) + self.to_screen( + '[hlsnative] %s: Downloading segment %d / %d' % + (info_dict['id'], i + 1, len(segment_urls))) + + self._hook_progress({ + 'downloaded_bytes': byte_counter, + 'total_bytes': byte_counter, + 'filename': filename, + 'status': 'finished', + }) + self.try_rename(tmpfilename, filename) + return True + diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 9c30a1d33..60cab6f4e 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -15,6 +15,7 @@ from ..utils import ( compat_http_client, compat_urllib_error, compat_urllib_parse_urlparse, + compat_urlparse, compat_str, clean_html, @@ -640,7 +641,9 @@ class InfoExtractor(object): return formats - def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None): + def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None, + entry_protocol='m3u8', preference=None): + formats = [{ 'format_id': 'm3u8-meta', 'url': m3u8_url, @@ -651,6 +654,11 @@ class InfoExtractor(object): 'format_note': 'Quality selection URL', }] + format_url = lambda u: ( + u + if re.match(r'^https?://', u) + else compat_urlparse.urljoin(m3u8_url, u)) + m3u8_doc = self._download_webpage(m3u8_url, video_id) last_info = None kv_rex = re.compile( @@ -667,15 +675,17 @@ class InfoExtractor(object): continue else: if last_info is None: - formats.append({'url': line}) + formats.append({'url': format_url(line)}) continue tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000) f = { 'format_id': 'm3u8-%d' % (tbr if tbr else len(formats)), - 'url': line.strip(), + 'url': format_url(line.strip()), 'tbr': tbr, 'ext': ext, + 'protocol': entry_protocol, + 'preference': preference, } codecs = last_info.get('CODECS') if codecs: diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index d2ffd1b6b..5e54a35d4 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -6,6 +6,7 @@ import xml.etree.ElementTree from .common import InfoExtractor from ..utils import ( compat_HTTPError, + compat_urllib_request, ExtractorError, ) @@ -69,6 +70,21 @@ class VevoIE(InfoExtractor): }] _SMIL_BASE_URL = 'http://smil.lvl3.vevo.com/' + def _real_initialize(self): + req = compat_urllib_request.Request( + 'http://www.vevo.com/auth', data=b'') + webpage = self._download_webpage( + req, None, + note='Retrieving oauth token', + errnote='Unable to retrieve oauth token', + fatal=False) + if webpage is False: + self._oauth_token = None + else: + self._oauth_token = self._search_regex( + r'access_token":\s*"([^"]+)"', + webpage, 'access token', fatal=False) + def _formats_from_json(self, video_info): last_version = {'version': -1} for version in video_info['videoVersions']: @@ -129,6 +145,26 @@ class VevoIE(InfoExtractor): }) return formats + def _download_api_formats(self, video_id): + if not self._oauth_token: + self._downloader.report_warning( + 'No oauth token available, skipping API HLS download') + return [] + + api_url = 'https://apiv2.vevo.com/video/%s/streams/hls?token=%s' % ( + video_id, self._oauth_token) + api_data = self._download_json( + api_url, video_id, + note='Downloading HLS formats', + errnote='Failed to download HLS format list', fatal=False) + if api_data is None: + return [] + + m3u8_url = api_data[0]['url'] + return self._extract_m3u8_formats( + m3u8_url, video_id, entry_protocol='m3u8_native', ext='mp4', + preference=0) + def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') @@ -152,6 +188,9 @@ class VevoIE(InfoExtractor): else: age_limit = None + # Download via HLS API + formats.extend(self._download_api_formats(video_id)) + # Download SMIL smil_blocks = sorted(( f for f in video_info['videoVersions'] @@ -166,7 +205,6 @@ class VevoIE(InfoExtractor): fatal=False) if smil_url_m is not None: smil_url = smil_url_m - try: smil_xml = self._download_webpage(smil_url, video_id, 'Downloading SMIL info') |