diff options
Diffstat (limited to 'youtube_dl/downloader')
-rw-r--r-- | youtube_dl/downloader/__init__.py | 3 | ||||
-rw-r--r-- | youtube_dl/downloader/common.py | 1 | ||||
-rw-r--r-- | youtube_dl/downloader/f4m.py | 5 | ||||
-rw-r--r-- | youtube_dl/downloader/hls.py | 59 | ||||
-rw-r--r-- | youtube_dl/downloader/http.py | 2 |
5 files changed, 68 insertions, 2 deletions
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py index 4ea5811a5..3f941596e 100644 --- a/youtube_dl/downloader/__init__.py +++ b/youtube_dl/downloader/__init__.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from .common import FileDownloader from .hls import HlsFD +from .hls import NativeHlsFD from .http import HttpFD from .mplayer import MplayerFD from .rtmp import RtmpFD @@ -19,6 +20,8 @@ def get_suitable_downloader(info_dict): if url.startswith('rtmp'): return RtmpFD + if protocol == 'm3u8_native': + return NativeHlsFD if (protocol == 'm3u8') or (protocol is None and determine_ext(url) == 'm3u8'): return HlsFD if url.startswith('mms') or url.startswith('rtsp'): diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 9ce97f5fe..f85f0c94e 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -42,6 +42,7 @@ class FileDownloader(object): Subclasses of this one must re-define the real_download method. """ + _TEST_FILE_SIZE = 10241 params = None def __init__(self, ydl, params): diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py index 71353f607..b3be16ff1 100644 --- a/youtube_dl/downloader/f4m.py +++ b/youtube_dl/downloader/f4m.py @@ -16,6 +16,7 @@ from ..utils import ( format_bytes, encodeFilename, sanitize_open, + xpath_text, ) @@ -251,6 +252,8 @@ class F4mFD(FileDownloader): # We only download the first fragment fragments_list = fragments_list[:1] total_frags = len(fragments_list) + # For some akamai manifests we'll need to add a query to the fragment url + akamai_pv = xpath_text(doc, _add_ns('pv-2.0')) tmpfilename = self.temp_name(filename) (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb') @@ -290,6 +293,8 @@ class F4mFD(FileDownloader): for (seg_i, frag_i) in fragments_list: name = 'Seg%d-Frag%d' % (seg_i, frag_i) url = base_url + name + if akamai_pv: + url += '?' + akamai_pv.strip(';') frag_filename = '%s-%s' % (tmpfilename, name) success = http_dl.download(frag_filename, {'url': url}) if not success: diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py index 32852f333..68eafa403 100644 --- a/youtube_dl/downloader/hls.py +++ b/youtube_dl/downloader/hls.py @@ -1,8 +1,13 @@ +from __future__ import unicode_literals + import os +import re import subprocess from .common import FileDownloader from ..utils import ( + compat_urlparse, + compat_urllib_request, check_executable, encodeFilename, ) @@ -43,3 +48,57 @@ class HlsFD(FileDownloader): self.to_stderr(u"\n") self.report_error(u'%s exited with code %d' % (program, retval)) return False + + +class NativeHlsFD(FileDownloader): + """ A more limited implementation that does not require ffmpeg """ + + def real_download(self, filename, info_dict): + url = info_dict['url'] + self.report_destination(filename) + tmpfilename = self.temp_name(filename) + + self.to_screen( + '[hlsnative] %s: Downloading m3u8 manifest' % info_dict['id']) + data = self.ydl.urlopen(url).read() + s = data.decode('utf-8', 'ignore') + segment_urls = [] + for line in s.splitlines(): + line = line.strip() + if line and not line.startswith('#'): + segment_url = ( + line + if re.match(r'^https?://', line) + else compat_urlparse.urljoin(url, line)) + segment_urls.append(segment_url) + + is_test = self.params.get('test', False) + remaining_bytes = self._TEST_FILE_SIZE if is_test else None + byte_counter = 0 + with open(tmpfilename, 'wb') as outf: + for i, segurl in enumerate(segment_urls): + self.to_screen( + '[hlsnative] %s: Downloading segment %d / %d' % + (info_dict['id'], i + 1, len(segment_urls))) + seg_req = compat_urllib_request.Request(segurl) + if remaining_bytes is not None: + seg_req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1)) + + segment = self.ydl.urlopen(seg_req).read() + if remaining_bytes is not None: + segment = segment[:remaining_bytes] + remaining_bytes -= len(segment) + outf.write(segment) + byte_counter += len(segment) + if remaining_bytes is not None and remaining_bytes <= 0: + break + + self._hook_progress({ + 'downloaded_bytes': byte_counter, + 'total_bytes': byte_counter, + 'filename': filename, + 'status': 'finished', + }) + self.try_rename(tmpfilename, filename) + return True + diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index 6caf7451e..f62555ce0 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -14,8 +14,6 @@ from ..utils import ( class HttpFD(FileDownloader): - _TEST_FILE_SIZE = 10241 - def real_download(self, filename, info_dict): url = info_dict['url'] tmpfilename = self.temp_name(filename) |