aboutsummaryrefslogtreecommitdiff
path: root/youtube_dl/downloader
diff options
context:
space:
mode:
Diffstat (limited to 'youtube_dl/downloader')
-rw-r--r--youtube_dl/downloader/__init__.py8
-rw-r--r--youtube_dl/downloader/common.py13
-rw-r--r--youtube_dl/downloader/dash.py66
-rw-r--r--youtube_dl/downloader/external.py27
-rw-r--r--youtube_dl/downloader/f4m.py2
-rw-r--r--youtube_dl/downloader/http.py9
-rw-r--r--youtube_dl/downloader/rtmp.py23
-rw-r--r--youtube_dl/downloader/rtsp.py (renamed from youtube_dl/downloader/mplayer.py)20
8 files changed, 115 insertions, 53 deletions
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 9fb66e2f7..dccc59212 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -6,8 +6,9 @@ from .f4m import F4mFD
from .hls import HlsFD
from .hls import NativeHlsFD
from .http import HttpFD
-from .mplayer import MplayerFD
+from .rtsp import RtspFD
from .rtmp import RtmpFD
+from .dash import DashSegmentsFD
from ..utils import (
determine_protocol,
@@ -17,9 +18,10 @@ PROTOCOL_MAP = {
'rtmp': RtmpFD,
'm3u8_native': NativeHlsFD,
'm3u8': HlsFD,
- 'mms': MplayerFD,
- 'rtsp': MplayerFD,
+ 'mms': RtspFD,
+ 'rtsp': RtspFD,
'f4m': F4mFD,
+ 'http_dash_segments': DashSegmentsFD,
}
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index a0fc5ead0..97e755d4b 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -8,6 +8,7 @@ import time
from ..compat import compat_str
from ..utils import (
encodeFilename,
+ decodeArgument,
format_bytes,
timeconvert,
)
@@ -353,19 +354,15 @@ class FileDownloader(object):
# this interface
self._progress_hooks.append(ph)
- def _debug_cmd(self, args, subprocess_encoding, exe=None):
+ def _debug_cmd(self, args, exe=None):
if not self.params.get('verbose', False):
return
+ str_args = [decodeArgument(a) for a in args]
+
if exe is None:
- exe = os.path.basename(args[0])
+ exe = os.path.basename(str_args[0])
- if subprocess_encoding:
- str_args = [
- a.decode(subprocess_encoding) if isinstance(a, bytes) else a
- for a in args]
- else:
- str_args = args
try:
import pipes
shell_quote = lambda args: ' '.join(map(pipes.quote, str_args))
diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
new file mode 100644
index 000000000..a4685d307
--- /dev/null
+++ b/youtube_dl/downloader/dash.py
@@ -0,0 +1,66 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import FileDownloader
+from ..compat import compat_urllib_request
+
+
+class DashSegmentsFD(FileDownloader):
+ """
+ Download segments in a DASH manifest
+ """
+ def real_download(self, filename, info_dict):
+ self.report_destination(filename)
+ tmpfilename = self.temp_name(filename)
+ base_url = info_dict['url']
+ segment_urls = info_dict['segment_urls']
+
+ is_test = self.params.get('test', False)
+ remaining_bytes = self._TEST_FILE_SIZE if is_test else None
+ byte_counter = 0
+
+ def append_url_to_file(outf, target_url, target_name, remaining_bytes=None):
+ self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name))
+ req = compat_urllib_request.Request(target_url)
+ if remaining_bytes is not None:
+ req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1))
+
+ data = self.ydl.urlopen(req).read()
+
+ if remaining_bytes is not None:
+ data = data[:remaining_bytes]
+
+ outf.write(data)
+ return len(data)
+
+ def combine_url(base_url, target_url):
+ if re.match(r'^https?://', target_url):
+ return target_url
+ return '%s/%s' % (base_url, target_url)
+
+ with open(tmpfilename, 'wb') as outf:
+ append_url_to_file(
+ outf, combine_url(base_url, info_dict['initialization_url']),
+ 'initialization segment')
+ for i, segment_url in enumerate(segment_urls):
+ segment_len = append_url_to_file(
+ outf, combine_url(base_url, segment_url),
+ 'segment %d / %d' % (i + 1, len(segment_urls)),
+ remaining_bytes)
+ byte_counter += segment_len
+ if remaining_bytes is not None:
+ remaining_bytes -= segment_len
+ if remaining_bytes <= 0:
+ break
+
+ self.try_rename(tmpfilename, filename)
+
+ self._hook_progress({
+ 'downloaded_bytes': byte_counter,
+ 'total_bytes': byte_counter,
+ 'filename': filename,
+ 'status': 'finished',
+ })
+
+ return True
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 1673b2382..1d5cc9904 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -2,11 +2,11 @@ from __future__ import unicode_literals
import os.path
import subprocess
-import sys
from .common import FileDownloader
from ..utils import (
encodeFilename,
+ encodeArgument,
)
@@ -60,17 +60,9 @@ class ExternalFD(FileDownloader):
def _call_downloader(self, tmpfilename, info_dict):
""" Either overwrite this or implement _make_cmd """
- cmd = self._make_cmd(tmpfilename, info_dict)
-
- if sys.platform == 'win32' and sys.version_info < (3, 0):
- # Windows subprocess module does not actually support Unicode
- # on Python 2.x
- # See http://stackoverflow.com/a/9951851/35070
- subprocess_encoding = sys.getfilesystemencoding()
- cmd = [a.encode(subprocess_encoding, 'ignore') for a in cmd]
- else:
- subprocess_encoding = None
- self._debug_cmd(cmd, subprocess_encoding)
+ cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
+
+ self._debug_cmd(cmd)
p = subprocess.Popen(
cmd, stderr=subprocess.PIPE)
@@ -117,6 +109,14 @@ class Aria2cFD(ExternalFD):
cmd += ['--', info_dict['url']]
return cmd
+
+class HttpieFD(ExternalFD):
+ def _make_cmd(self, tmpfilename, info_dict):
+ cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
+ for key, val in info_dict['http_headers'].items():
+ cmd += ['%s:%s' % (key, val)]
+ return cmd
+
_BY_NAME = dict(
(klass.get_basename(), klass)
for name, klass in globals().items()
@@ -131,5 +131,6 @@ def list_external_downloaders():
def get_external_downloader(external_downloader):
""" Given the name of the executable, see whether we support the given
downloader . """
- bn = os.path.basename(external_downloader)
+ # Drop .exe extension on Windows
+ bn = os.path.splitext(os.path.basename(external_downloader))[0]
return _BY_NAME[bn]
diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py
index 4ab000d67..b1a858c45 100644
--- a/youtube_dl/downloader/f4m.py
+++ b/youtube_dl/downloader/f4m.py
@@ -389,6 +389,8 @@ class F4mFD(FileDownloader):
url = base_url + name
if akamai_pv:
url += '?' + akamai_pv.strip(';')
+ if info_dict.get('extra_param_to_segment_url'):
+ url += info_dict.get('extra_param_to_segment_url')
frag_filename = '%s-%s' % (tmpfilename, name)
try:
success = http_dl.download(frag_filename, {'url': url})
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index d136bebd1..b7f144af9 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -28,13 +28,8 @@ class HttpFD(FileDownloader):
add_headers = info_dict.get('http_headers')
if add_headers:
headers.update(add_headers)
- data = info_dict.get('http_post_data')
- http_method = info_dict.get('http_method')
- basic_request = compat_urllib_request.Request(url, data, headers)
- request = compat_urllib_request.Request(url, data, headers)
- if http_method is not None:
- basic_request.get_method = lambda: http_method
- request.get_method = lambda: http_method
+ basic_request = compat_urllib_request.Request(url, None, headers)
+ request = compat_urllib_request.Request(url, None, headers)
is_test = self.params.get('test', False)
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index ddf5724ae..7d19bb808 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
import os
import re
import subprocess
-import sys
import time
from .common import FileDownloader
@@ -11,6 +10,7 @@ from ..compat import compat_str
from ..utils import (
check_executable,
encodeFilename,
+ encodeArgument,
get_exe_version,
)
@@ -121,7 +121,7 @@ class RtmpFD(FileDownloader):
# possible. This is part of rtmpdump's normal usage, AFAIK.
basic_args = [
'rtmpdump', '--verbose', '-r', url,
- '-o', encodeFilename(tmpfilename, True)]
+ '-o', tmpfilename]
if player_url is not None:
basic_args += ['--swfVfy', player_url]
if page_url is not None:
@@ -131,7 +131,7 @@ class RtmpFD(FileDownloader):
if play_path is not None:
basic_args += ['--playpath', play_path]
if tc_url is not None:
- basic_args += ['--tcUrl', url]
+ basic_args += ['--tcUrl', tc_url]
if test:
basic_args += ['--stop', '1']
if flash_version is not None:
@@ -154,16 +154,9 @@ class RtmpFD(FileDownloader):
if not live and continue_dl:
args += ['--skip', '1']
- if sys.platform == 'win32' and sys.version_info < (3, 0):
- # Windows subprocess module does not actually support Unicode
- # on Python 2.x
- # See http://stackoverflow.com/a/9951851/35070
- subprocess_encoding = sys.getfilesystemencoding()
- args = [a.encode(subprocess_encoding, 'ignore') for a in args]
- else:
- subprocess_encoding = None
+ args = [encodeArgument(a) for a in args]
- self._debug_cmd(args, subprocess_encoding, exe='rtmpdump')
+ self._debug_cmd(args, exe='rtmpdump')
RD_SUCCESS = 0
RD_FAILED = 1
@@ -180,7 +173,11 @@ class RtmpFD(FileDownloader):
prevsize = os.path.getsize(encodeFilename(tmpfilename))
self.to_screen('[rtmpdump] %s bytes' % prevsize)
time.sleep(5.0) # This seems to be needed
- retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == RD_FAILED])
+ args = basic_args + ['--resume']
+ if retval == RD_FAILED:
+ args += ['--skip', '1']
+ args = [encodeArgument(a) for a in args]
+ retval = run_rtmpdump(args)
cursize = os.path.getsize(encodeFilename(tmpfilename))
if prevsize == cursize and retval == RD_FAILED:
break
diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/rtsp.py
index 72cef30ea..3eb29526c 100644
--- a/youtube_dl/downloader/mplayer.py
+++ b/youtube_dl/downloader/rtsp.py
@@ -10,21 +10,23 @@ from ..utils import (
)
-class MplayerFD(FileDownloader):
+class RtspFD(FileDownloader):
def real_download(self, filename, info_dict):
url = info_dict['url']
self.report_destination(filename)
tmpfilename = self.temp_name(filename)
- args = [
- 'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
- '-dumpstream', '-dumpfile', tmpfilename, url]
- # Check for mplayer first
- if not check_executable('mplayer', ['-h']):
- self.report_error('MMS or RTSP download detected but "%s" could not be run' % args[0])
+ if check_executable('mplayer', ['-h']):
+ args = [
+ 'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy',
+ '-dumpstream', '-dumpfile', tmpfilename, url]
+ elif check_executable('mpv', ['-h']):
+ args = [
+ 'mpv', '-really-quiet', '--vo=null', '--stream-dump=' + tmpfilename, url]
+ else:
+ self.report_error('MMS or RTSP download detected but neither "mplayer" nor "mpv" could be run. Please install any.')
return False
- # Download using mplayer.
retval = subprocess.call(args)
if retval == 0:
fsize = os.path.getsize(encodeFilename(tmpfilename))
@@ -39,5 +41,5 @@ class MplayerFD(FileDownloader):
return True
else:
self.to_stderr('\n')
- self.report_error('mplayer exited with code %d' % retval)
+ self.report_error('%s exited with code %d' % (args[0], retval))
return False
generated by cgit v1.2.3 (git 2.26.2) at 2025-04-09 20:50:48 +0000