diff options
| -rwxr-xr-x | youtube_dl/YoutubeDL.py | 1 | ||||
| -rw-r--r-- | youtube_dl/__init__.py | 1 | ||||
| -rw-r--r-- | youtube_dl/downloader/__init__.py | 9 | ||||
| -rw-r--r-- | youtube_dl/downloader/common.py | 21 | ||||
| -rw-r--r-- | youtube_dl/downloader/external.py | 131 | ||||
| -rw-r--r-- | youtube_dl/downloader/rtmp.py | 14 | ||||
| -rw-r--r-- | youtube_dl/options.py | 6 |
7 files changed, 169 insertions, 14 deletions
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index e61e6c2a7..54e732943 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -219,6 +219,7 @@ class YoutubeDL(object): call_home: Boolean, true iff we are allowed to contact the youtube-dl servers for debugging. sleep_interval: Number of seconds to sleep before each download. + external_downloader: Executable of the external downloader to call. The following parameters are not used by YoutubeDL itself, they are used by diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 7bd7295e2..3fc7dc5c2 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -330,6 +330,7 @@ def _real_main(argv=None): 'source_address': opts.source_address, 'call_home': opts.call_home, 'sleep_interval': opts.sleep_interval, + 'external_downloader': opts.external_downloader, } with YoutubeDL(ydl_opts) as ydl: diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py index 2aca3cab5..eff1122c5 100644 --- a/youtube_dl/downloader/__init__.py +++ b/youtube_dl/downloader/__init__.py @@ -1,12 +1,13 @@ from __future__ import unicode_literals from .common import FileDownloader +from .external import get_external_downloader +from .f4m import F4mFD from .hls import HlsFD from .hls import NativeHlsFD from .http import HttpFD from .mplayer import MplayerFD from .rtmp import RtmpFD -from .f4m import F4mFD from ..utils import ( determine_protocol, @@ -27,6 +28,12 @@ def get_suitable_downloader(info_dict, params={}): protocol = determine_protocol(info_dict) info_dict['protocol'] = protocol + external_downloader = params.get('external_downloader') + if external_downloader is not None: + ed = get_external_downloader(external_downloader) + if ed.supports(info_dict): + return ed + return PROTOCOL_MAP.get(protocol, HttpFD) diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 82c917d92..c35c42c1d 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -325,3 +325,24 @@ class FileDownloader(object): # See YoutubeDl.py (search for progress_hooks) for a description of # this interface self._progress_hooks.append(ph) + + def _debug_cmd(self, args, subprocess_encoding, exe=None): + if not self.params.get('verbose', False): + return + + if exe is None: + exe = os.path.basename(args[0]) + + if subprocess_encoding: + str_args = [ + a.decode(subprocess_encoding) if isinstance(a, bytes) else a + for a in args] + else: + str_args = args + try: + import pipes + shell_quote = lambda args: ' '.join(map(pipes.quote, str_args)) + except ImportError: + shell_quote = repr + self.to_screen('[debug] %s command line: %s' % ( + exe, shell_quote(str_args))) diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py new file mode 100644 index 000000000..c05596255 --- /dev/null +++ b/youtube_dl/downloader/external.py @@ -0,0 +1,131 @@ +from __future__ import unicode_literals + +import os.path +import subprocess +import sys + +from .common import FileDownloader +from ..utils import ( + encodeFilename, + std_headers, +) + + +class ExternalFD(FileDownloader): + def real_download(self, filename, info_dict): + self.report_destination(filename) + tmpfilename = self.temp_name(filename) + + retval = self._call_downloader(tmpfilename, info_dict) + if retval == 0: + fsize = os.path.getsize(encodeFilename(tmpfilename)) + self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize)) + self.try_rename(tmpfilename, filename) + self._hook_progress({ + 'downloaded_bytes': fsize, + 'total_bytes': fsize, + 'filename': filename, + 'status': 'finished', + }) + return True + else: + self.to_stderr('\n') + self.report_error('%s exited with code %d' % ( + self.get_basename(), retval)) + return False + + @classmethod + def get_basename(cls): + return cls.__name__[:-2].lower() + + @property + def exe(self): + return self.params.get('external_downloader') + + @classmethod + def supports(cls, info_dict): + return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps') + + def _calc_headers(self, info_dict): + res = std_headers.copy() + + ua = info_dict.get('user_agent') + if ua is not None: + res['User-Agent'] = ua + + cookies = self._calc_cookies(info_dict) + if cookies: + res['Cookie'] = cookies + + return res + + def _calc_cookies(self, info_dict): + class _PseudoRequest(object): + def __init__(self, url): + self.url = url + self.headers = {} + self.unverifiable = False + + def add_unredirected_header(self, k, v): + self.headers[k] = v + + def get_full_url(self): + return self.url + + def is_unverifiable(self): + return self.unverifiable + + def has_header(self, h): + return h in self.headers + + pr = _PseudoRequest(info_dict['url']) + self.ydl.cookiejar.add_cookie_header(pr) + return pr.headers.get('Cookie') + + def _call_downloader(self, tmpfilename, info_dict): + """ Either overwrite this or implement _make_cmd """ + cmd = self._make_cmd(tmpfilename, info_dict) + + if sys.platform == 'win32' and sys.version_info < (3, 0): + # Windows subprocess module does not actually support Unicode + # on Python 2.x + # See http://stackoverflow.com/a/9951851/35070 + subprocess_encoding = sys.getfilesystemencoding() + cmd = [a.encode(subprocess_encoding, 'ignore') for a in cmd] + else: + subprocess_encoding = None + self._debug_cmd(cmd, subprocess_encoding) + + p = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if p.returncode != 0: + self.to_stderr(stderr) + return p.returncode + + +class WgetFD(ExternalFD): + def _make_cmd(self, tmpfilename, info_dict): + cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies'] + for key, val in self._calc_headers(info_dict).items(): + cmd += ['--header', '%s: %s' % (key, val)] + cmd += ['--', info_dict['url']] + return cmd + + +_BY_NAME = dict( + (klass.get_basename(), klass) + for name, klass in globals().items() + if name.endswith('FD') and name != 'ExternalFD' +) + + +def list_external_downloaders(): + return sorted(_BY_NAME.keys()) + + +def get_external_downloader(external_downloader): + """ Given the name of the executable, see whether we support the given + downloader . """ + bn = os.path.basename(external_downloader) + return _BY_NAME[bn] diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py index 5346cb9a0..6dbbc053c 100644 --- a/youtube_dl/downloader/rtmp.py +++ b/youtube_dl/downloader/rtmp.py @@ -152,19 +152,7 @@ class RtmpFD(FileDownloader): else: subprocess_encoding = None - if self.params.get('verbose', False): - if subprocess_encoding: - str_args = [ - a.decode(subprocess_encoding) if isinstance(a, bytes) else a - for a in args] - else: - str_args = args - try: - import pipes - shell_quote = lambda args: ' '.join(map(pipes.quote, str_args)) - except ImportError: - shell_quote = repr - self.to_screen('[debug] rtmpdump command line: ' + shell_quote(str_args)) + self._debug_cmd(args, subprocess_encoding, exe='rtmpdump') RD_SUCCESS = 0 RD_FAILED = 1 diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 262c60013..b38b8349f 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -5,6 +5,7 @@ import optparse import shlex import sys +from .downloader.external import list_external_downloaders from .compat import ( compat_expanduser, compat_getenv, @@ -389,6 +390,11 @@ def parseOpts(overrideArguments=None): '--playlist-reverse', action='store_true', help='Download playlist videos in reverse order') + downloader.add_option( + '--external-downloader', + dest='external_downloader', metavar='COMMAND', + help='(experimental) Use the specified external downloader. ' + 'Currently supports %s' % ','.join(list_external_downloaders())) workarounds = optparse.OptionGroup(parser, 'Workarounds') workarounds.add_option( |