diff options
Diffstat (limited to 'youtube_dl/downloader')
| -rw-r--r-- | youtube_dl/downloader/__init__.py | 47 | ||||
| -rw-r--r-- | youtube_dl/downloader/common.py | 114 | ||||
| -rw-r--r-- | youtube_dl/downloader/external.py | 117 | ||||
| -rw-r--r-- | youtube_dl/downloader/f4m.py | 82 | ||||
| -rw-r--r-- | youtube_dl/downloader/hls.py | 23 | ||||
| -rw-r--r-- | youtube_dl/downloader/http.py | 58 | ||||
| -rw-r--r-- | youtube_dl/downloader/mplayer.py | 19 | ||||
| -rw-r--r-- | youtube_dl/downloader/rtmp.py | 36 |
8 files changed, 345 insertions, 151 deletions
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py index 3f941596e..eff1122c5 100644 --- a/youtube_dl/downloader/__init__.py +++ b/youtube_dl/downloader/__init__.py @@ -1,32 +1,43 @@ from __future__ import unicode_literals from .common import FileDownloader +from .external import get_external_downloader +from .f4m import F4mFD from .hls import HlsFD from .hls import NativeHlsFD from .http import HttpFD from .mplayer import MplayerFD from .rtmp import RtmpFD -from .f4m import F4mFD from ..utils import ( - determine_ext, + determine_protocol, ) +PROTOCOL_MAP = { + 'rtmp': RtmpFD, + 'm3u8_native': NativeHlsFD, + 'm3u8': HlsFD, + 'mms': MplayerFD, + 'rtsp': MplayerFD, + 'f4m': F4mFD, +} -def get_suitable_downloader(info_dict): + +def get_suitable_downloader(info_dict, params={}): """Get the downloader class that can handle the info dict.""" - url = info_dict['url'] - protocol = info_dict.get('protocol') - - if url.startswith('rtmp'): - return RtmpFD - if protocol == 'm3u8_native': - return NativeHlsFD - if (protocol == 'm3u8') or (protocol is None and determine_ext(url) == 'm3u8'): - return HlsFD - if url.startswith('mms') or url.startswith('rtsp'): - return MplayerFD - if determine_ext(url) == 'f4m': - return F4mFD - else: - return HttpFD + protocol = determine_protocol(info_dict) + info_dict['protocol'] = protocol + + external_downloader = params.get('external_downloader') + if external_downloader is not None: + ed = get_external_downloader(external_downloader) + if ed.supports(info_dict): + return ed + + return PROTOCOL_MAP.get(protocol, HttpFD) + + +__all__ = [ + 'get_suitable_downloader', + 'FileDownloader', +] diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index f85f0c94e..c35c42c1d 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -1,10 +1,12 @@ +from __future__ import unicode_literals + import os import re import sys import time +from ..compat import compat_str from ..utils import ( - compat_str, encodeFilename, format_bytes, timeconvert, @@ -78,8 +80,10 @@ class FileDownloader(object): def calc_eta(start, now, total, current): if total is None: return None + if now is None: + now = time.time() dif = now - start - if current == 0 or dif < 0.001: # One millisecond + if current == 0 or dif < 0.001: # One millisecond return None rate = float(current) / dif return int((float(total) - float(current)) / rate) @@ -93,7 +97,7 @@ class FileDownloader(object): @staticmethod def calc_speed(start, now, bytes): dif = now - start - if bytes == 0 or dif < 0.001: # One millisecond + if bytes == 0 or dif < 0.001: # One millisecond return None return float(bytes) / dif @@ -106,7 +110,7 @@ class FileDownloader(object): @staticmethod def best_block_size(elapsed_time, bytes): new_min = max(bytes / 2.0, 1.0) - new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB + new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB if elapsed_time < 0.001: return int(new_max) rate = bytes / elapsed_time @@ -144,29 +148,30 @@ class FileDownloader(object): def report_error(self, *args, **kargs): self.ydl.report_error(*args, **kargs) - def slow_down(self, start_time, byte_counter): + def slow_down(self, start_time, now, byte_counter): """Sleep if the download speed is over the rate limit.""" rate_limit = self.params.get('ratelimit', None) if rate_limit is None or byte_counter == 0: return - now = time.time() + if now is None: + now = time.time() elapsed = now - start_time if elapsed <= 0.0: return speed = float(byte_counter) / elapsed if speed > rate_limit: - time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit) + time.sleep(max((byte_counter // rate_limit) - elapsed, 0)) def temp_name(self, filename): """Returns a temporary filename for the given filename.""" - if self.params.get('nopart', False) or filename == u'-' or \ + if self.params.get('nopart', False) or filename == '-' or \ (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): return filename - return filename + u'.part' + return filename + '.part' def undo_temp_name(self, filename): - if filename.endswith(u'.part'): - return filename[:-len(u'.part')] + if filename.endswith('.part'): + return filename[:-len('.part')] return filename def try_rename(self, old_filename, new_filename): @@ -175,7 +180,7 @@ class FileDownloader(object): return os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) except (IOError, OSError) as err: - self.report_error(u'unable to rename file: %s' % compat_str(err)) + self.report_error('unable to rename file: %s' % compat_str(err)) def try_utime(self, filename, last_modified_hdr): """Try to set the last-modified time of the given file.""" @@ -200,10 +205,10 @@ class FileDownloader(object): def report_destination(self, filename): """Report destination filename.""" - self.to_screen(u'[download] Destination: ' + filename) + self.to_screen('[download] Destination: ' + filename) def _report_progress_status(self, msg, is_last_line=False): - fullmsg = u'[download] ' + msg + fullmsg = '[download] ' + msg if self.params.get('progress_with_newline', False): self.to_screen(fullmsg) else: @@ -211,13 +216,13 @@ class FileDownloader(object): prev_len = getattr(self, '_report_progress_prev_line_length', 0) if prev_len > len(fullmsg): - fullmsg += u' ' * (prev_len - len(fullmsg)) + fullmsg += ' ' * (prev_len - len(fullmsg)) self._report_progress_prev_line_length = len(fullmsg) - clear_line = u'\r' + clear_line = '\r' else: - clear_line = (u'\r\x1b[K' if sys.stderr.isatty() else u'\r') + clear_line = ('\r\x1b[K' if sys.stderr.isatty() else '\r') self.to_screen(clear_line + fullmsg, skip_eol=not is_last_line) - self.to_console_title(u'youtube-dl ' + msg) + self.to_console_title('youtube-dl ' + msg) def report_progress(self, percent, data_len_str, speed, eta): """Report download progress.""" @@ -233,7 +238,7 @@ class FileDownloader(object): percent_str = 'Unknown %' speed_str = self.format_speed(speed) - msg = (u'%s of %s at %s ETA %s' % + msg = ('%s of %s at %s ETA %s' % (percent_str, data_len_str, speed_str, eta_str)) self._report_progress_status(msg) @@ -243,44 +248,56 @@ class FileDownloader(object): downloaded_str = format_bytes(downloaded_data_len) speed_str = self.format_speed(speed) elapsed_str = FileDownloader.format_seconds(elapsed) - msg = u'%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str) + msg = '%s at %s (%s)' % (downloaded_str, speed_str, elapsed_str) self._report_progress_status(msg) def report_finish(self, data_len_str, tot_time): """Report download finished.""" if self.params.get('noprogress', False): - self.to_screen(u'[download] Download completed') + self.to_screen('[download] Download completed') else: self._report_progress_status( - (u'100%% of %s in %s' % + ('100%% of %s in %s' % (data_len_str, self.format_seconds(tot_time))), is_last_line=True) def report_resuming_byte(self, resume_len): """Report attempt to resume at given byte.""" - self.to_screen(u'[download] Resuming download at byte %s' % resume_len) + self.to_screen('[download] Resuming download at byte %s' % resume_len) def report_retry(self, count, retries): """Report retry in case of HTTP error 5xx""" - self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) + self.to_screen('[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries)) def report_file_already_downloaded(self, file_name): """Report file has already been fully downloaded.""" try: - self.to_screen(u'[download] %s has already been downloaded' % file_name) + self.to_screen('[download] %s has already been downloaded' % file_name) except UnicodeEncodeError: - self.to_screen(u'[download] The file has already been downloaded') + self.to_screen('[download] The file has already been downloaded') def report_unable_to_resume(self): """Report it was impossible to resume download.""" - self.to_screen(u'[download] Unable to resume') + self.to_screen('[download] Unable to resume') def download(self, filename, info_dict): """Download to a filename using the info from info_dict Return True on success and False otherwise """ + + nooverwrites_and_exists = ( + self.params.get('nooverwrites', False) + and os.path.exists(encodeFilename(filename)) + ) + + continuedl_and_exists = ( + self.params.get('continuedl', False) + and os.path.isfile(encodeFilename(filename)) + and not self.params.get('nopart', False) + ) + # Check file already present - if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False): + if filename != '-' and nooverwrites_and_exists or continuedl_and_exists: self.report_file_already_downloaded(filename) self._hook_progress({ 'filename': filename, @@ -289,30 +306,43 @@ class FileDownloader(object): }) return True + sleep_interval = self.params.get('sleep_interval') + if sleep_interval: + self.to_screen('[download] Sleeping %s seconds...' % sleep_interval) + time.sleep(sleep_interval) + return self.real_download(filename, info_dict) def real_download(self, filename, info_dict): """Real download process. Redefine in subclasses.""" - raise NotImplementedError(u'This method must be implemented by subclasses') + raise NotImplementedError('This method must be implemented by subclasses') def _hook_progress(self, status): for ph in self._progress_hooks: ph(status) def add_progress_hook(self, ph): - """ ph gets called on download progress, with a dictionary with the entries - * filename: The final filename - * status: One of "downloading" and "finished" + # See YoutubeDl.py (search for progress_hooks) for a description of + # this interface + self._progress_hooks.append(ph) - It can also have some of the following entries: + def _debug_cmd(self, args, subprocess_encoding, exe=None): + if not self.params.get('verbose', False): + return - * downloaded_bytes: Bytes on disks - * total_bytes: Total bytes, None if unknown - * tmpfilename: The filename we're currently writing to - * eta: The estimated time in seconds, None if unknown - * speed: The download speed in bytes/second, None if unknown + if exe is None: + exe = os.path.basename(args[0]) - Hooks are guaranteed to be called at least once (with status "finished") - if the download is successful. - """ - self._progress_hooks.append(ph) + if subprocess_encoding: + str_args = [ + a.decode(subprocess_encoding) if isinstance(a, bytes) else a + for a in args] + else: + str_args = args + try: + import pipes + shell_quote = lambda args: ' '.join(map(pipes.quote, str_args)) + except ImportError: + shell_quote = repr + self.to_screen('[debug] %s command line: %s' % ( + exe, shell_quote(str_args))) diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py new file mode 100644 index 000000000..af9fdba75 --- /dev/null +++ b/youtube_dl/downloader/external.py @@ -0,0 +1,117 @@ +from __future__ import unicode_literals + +import os.path +import subprocess +import sys + +from .common import FileDownloader +from ..utils import ( + encodeFilename, +) + + +class ExternalFD(FileDownloader): + def real_download(self, filename, info_dict): + self.report_destination(filename) + tmpfilename = self.temp_name(filename) + + retval = self._call_downloader(tmpfilename, info_dict) + if retval == 0: + fsize = os.path.getsize(encodeFilename(tmpfilename)) + self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize)) + self.try_rename(tmpfilename, filename) + self._hook_progress({ + 'downloaded_bytes': fsize, + 'total_bytes': fsize, + 'filename': filename, + 'status': 'finished', + }) + return True + else: + self.to_stderr('\n') + self.report_error('%s exited with code %d' % ( + self.get_basename(), retval)) + return False + + @classmethod + def get_basename(cls): + return cls.__name__[:-2].lower() + + @property + def exe(self): + return self.params.get('external_downloader') + + @classmethod + def supports(cls, info_dict): + return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps') + + def _call_downloader(self, tmpfilename, info_dict): + """ Either overwrite this or implement _make_cmd """ + cmd = self._make_cmd(tmpfilename, info_dict) + + if sys.platform == 'win32' and sys.version_info < (3, 0): + # Windows subprocess module does not actually support Unicode + # on Python 2.x + # See http://stackoverflow.com/a/9951851/35070 + subprocess_encoding = sys.getfilesystemencoding() + cmd = [a.encode(subprocess_encoding, 'ignore') for a in cmd] + else: + subprocess_encoding = None + self._debug_cmd(cmd, subprocess_encoding) + + p = subprocess.Popen( + cmd, stderr=subprocess.PIPE) + _, stderr = p.communicate() + if p.returncode != 0: + self.to_stderr(stderr) + return p.returncode + + +class CurlFD(ExternalFD): + def _make_cmd(self, tmpfilename, info_dict): + cmd = [self.exe, '-o', tmpfilename] + for key, val in info_dict['http_headers'].items(): + cmd += ['--header', '%s: %s' % (key, val)] + cmd += ['--', info_dict['url']] + return cmd + + +class WgetFD(ExternalFD): + def _make_cmd(self, tmpfilename, info_dict): + cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies'] + for key, val in info_dict['http_headers'].items(): + cmd += ['--header', '%s: %s' % (key, val)] + cmd += ['--', info_dict['url']] + return cmd + + +class Aria2cFD(ExternalFD): + def _make_cmd(self, tmpfilename, info_dict): + cmd = [ + self.exe, '-c', + '--min-split-size', '1M', '--max-connection-per-server', '4'] + dn = os.path.dirname(tmpfilename) + if dn: + cmd += ['--dir', dn] + cmd += ['--out', os.path.basename(tmpfilename)] + for key, val in info_dict['http_headers'].items(): + cmd += ['--header', '%s: %s' % (key, val)] + cmd += ['--', info_dict['url']] + return cmd + +_BY_NAME = dict( + (klass.get_basename(), klass) + for name, klass in globals().items() + if name.endswith('FD') and name != 'ExternalFD' +) + + +def list_external_downloaders(): + return sorted(_BY_NAME.keys()) + + +def get_external_downloader(external_downloader): + """ Given the name of the executable, see whether we support the given + downloader . """ + bn = os.path.basename(external_downloader) + return _BY_NAME[bn] diff --git a/youtube_dl/downloader/f4m.py b/youtube_dl/downloader/f4m.py index b3be16ff1..c68b2c303 100644 --- a/youtube_dl/downloader/f4m.py +++ b/youtube_dl/downloader/f4m.py @@ -9,10 +9,12 @@ import xml.etree.ElementTree as etree from .common import FileDownloader from .http import HttpFD +from ..compat import ( + compat_urlparse, +) from ..utils import ( struct_pack, struct_unpack, - compat_urlparse, format_bytes, encodeFilename, sanitize_open, @@ -55,7 +57,7 @@ class FlvReader(io.BytesIO): if size == 1: real_size = self.read_unsigned_long_long() header_end = 16 - return real_size, box_type, self.read(real_size-header_end) + return real_size, box_type, self.read(real_size - header_end) def read_asrt(self): # version @@ -175,34 +177,43 @@ def build_fragments_list(boot_info): """ Return a list of (segment, fragment) for each fragment in the video """ res = [] segment_run_table = boot_info['segments'][0] - # I've only found videos with one segment - segment_run_entry = segment_run_table['segment_run'][0] - n_frags = segment_run_entry[1] fragment_run_entry_table = boot_info['fragments'][0]['fragments'] first_frag_number = fragment_run_entry_table[0]['first'] - for (i, frag_number) in zip(range(1, n_frags+1), itertools.count(first_frag_number)): - res.append((1, frag_number)) + fragments_counter = itertools.count(first_frag_number) + for segment, fragments_count in segment_run_table['segment_run']: + for _ in range(fragments_count): + res.append((segment, next(fragments_counter))) return res -def write_flv_header(stream, metadata): - """Writes the FLV header and the metadata to stream""" +def write_unsigned_int(stream, val): + stream.write(struct_pack('!I', val)) + + +def write_unsigned_int_24(stream, val): + stream.write(struct_pack('!I', val)[1:]) + + +def write_flv_header(stream): + """Writes the FLV header to stream""" # FLV header stream.write(b'FLV\x01') stream.write(b'\x05') stream.write(b'\x00\x00\x00\x09') - # FLV File body stream.write(b'\x00\x00\x00\x00') - # FLVTAG - # Script data - stream.write(b'\x12') - # Size of the metadata with 3 bytes - stream.write(struct_pack('!L', len(metadata))[1:]) - stream.write(b'\x00\x00\x00\x00\x00\x00\x00') - stream.write(metadata) - # Magic numbers extracted from the output files produced by AdobeHDS.php - #(https://github.com/K-S-V/Scripts) - stream.write(b'\x00\x00\x01\x73') + + +def write_metadata_tag(stream, metadata): + """Writes optional metadata tag to stream""" + SCRIPT_TAG = b'\x12' + FLV_TAG_HEADER_LEN = 11 + + if metadata: + stream.write(SCRIPT_TAG) + write_unsigned_int_24(stream, len(metadata)) + stream.write(b'\x00\x00\x00\x00\x00\x00\x00') + stream.write(metadata) + write_unsigned_int(stream, FLV_TAG_HEADER_LEN + len(metadata)) def _add_ns(prop): @@ -225,13 +236,16 @@ class F4mFD(FileDownloader): self.to_screen('[download] Downloading f4m manifest') manifest = self.ydl.urlopen(man_url).read() self.report_destination(filename) - http_dl = HttpQuietDownloader(self.ydl, + http_dl = HttpQuietDownloader( + self.ydl, { 'continuedl': True, 'quiet': True, 'noprogress': True, + 'ratelimit': self.params.get('ratelimit', None), 'test': self.params.get('test', False), - }) + } + ) doc = etree.fromstring(manifest) formats = [(int(f.attrib.get('bitrate', -1)), f) for f in doc.findall(_add_ns('media'))] @@ -244,9 +258,20 @@ class F4mFD(FileDownloader): lambda f: int(f[0]) == requested_bitrate, formats))[0] base_url = compat_urlparse.urljoin(man_url, media.attrib['url']) - bootstrap = base64.b64decode(doc.find(_add_ns('bootstrapInfo')).text) - metadata = base64.b64decode(media.find(_add_ns('metadata')).text) + bootstrap_node = doc.find(_add_ns('bootstrapInfo')) + if bootstrap_node.text is None: + bootstrap_url = compat_urlparse.urljoin( + base_url, bootstrap_node.attrib['url']) + bootstrap = self.ydl.urlopen(bootstrap_url).read() + else: + bootstrap = base64.b64decode(bootstrap_node.text) + metadata_node = media.find(_add_ns('metadata')) + if metadata_node is not None: + metadata = base64.b64decode(metadata_node.text) + else: + metadata = None boot_info = read_bootstrap_info(bootstrap) + fragments_list = build_fragments_list(boot_info) if self.params.get('test', False): # We only download the first fragment @@ -257,7 +282,8 @@ class F4mFD(FileDownloader): tmpfilename = self.temp_name(filename) (dest_stream, tmpfilename) = sanitize_open(tmpfilename, 'wb') - write_flv_header(dest_stream, metadata) + write_flv_header(dest_stream) + write_metadata_tag(dest_stream, metadata) # This dict stores the download progress, it's updated by the progress # hook @@ -270,7 +296,7 @@ class F4mFD(FileDownloader): def frag_progress_hook(status): frag_total_bytes = status.get('total_bytes', 0) estimated_size = (state['downloaded_bytes'] + - (total_frags - state['frag_counter']) * frag_total_bytes) + (total_frags - state['frag_counter']) * frag_total_bytes) if status['status'] == 'finished': state['downloaded_bytes'] += frag_total_bytes state['frag_counter'] += 1 @@ -280,13 +306,13 @@ class F4mFD(FileDownloader): frag_downloaded_bytes = status['downloaded_bytes'] byte_counter = state['downloaded_bytes'] + frag_downloaded_bytes frag_progress = self.calc_percent(frag_downloaded_bytes, - frag_total_bytes) + frag_total_bytes) progress = self.calc_percent(state['frag_counter'], total_frags) progress += frag_progress / float(total_frags) eta = self.calc_eta(start, time.time(), estimated_size, byte_counter) self.report_progress(progress, format_bytes(estimated_size), - status.get('speed'), eta) + status.get('speed'), eta) http_dl.add_progress_hook(frag_progress_hook) frags_filenames = [] diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py index 68eafa403..aa58b52ab 100644 --- a/youtube_dl/downloader/hls.py +++ b/youtube_dl/downloader/hls.py @@ -4,11 +4,13 @@ import os import re import subprocess +from ..postprocessor.ffmpeg import FFmpegPostProcessor from .common import FileDownloader -from ..utils import ( +from ..compat import ( compat_urlparse, compat_urllib_request, - check_executable, +) +from ..utils import ( encodeFilename, ) @@ -24,18 +26,18 @@ class HlsFD(FileDownloader): '-bsf:a', 'aac_adtstoasc', encodeFilename(tmpfilename, for_subprocess=True)] - for program in ['avconv', 'ffmpeg']: - if check_executable(program, ['-version']): - break - else: - self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found. Please install one.') + ffpp = FFmpegPostProcessor(downloader=self) + program = ffpp._executable + if program is None: + self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.') return False + ffpp.check_version() cmd = [program] + args retval = subprocess.call(cmd) if retval == 0: fsize = os.path.getsize(encodeFilename(tmpfilename)) - self.to_screen(u'\r[%s] %s bytes' % (cmd[0], fsize)) + self.to_screen('\r[%s] %s bytes' % (cmd[0], fsize)) self.try_rename(tmpfilename, filename) self._hook_progress({ 'downloaded_bytes': fsize, @@ -45,8 +47,8 @@ class HlsFD(FileDownloader): }) return True else: - self.to_stderr(u"\n") - self.report_error(u'%s exited with code %d' % (program, retval)) + self.to_stderr('\n') + self.report_error('%s exited with code %d' % (program, retval)) return False @@ -101,4 +103,3 @@ class NativeHlsFD(FileDownloader): }) self.try_rename(tmpfilename, filename) return True - diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index f62555ce0..4db50ee90 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -1,12 +1,15 @@ +from __future__ import unicode_literals + import os import time from .common import FileDownloader -from ..utils import ( +from ..compat import ( compat_urllib_request, compat_urllib_error, +) +from ..utils import ( ContentTooShortError, - encodeFilename, sanitize_open, format_bytes, @@ -21,10 +24,6 @@ class HttpFD(FileDownloader): # Do not include the Accept-Encoding header headers = {'Youtubedl-no-compression': 'True'} - if 'user_agent' in info_dict: - headers['Youtubedl-user-agent'] = info_dict['user_agent'] - if 'http_referer' in info_dict: - headers['Referer'] = info_dict['http_referer'] add_headers = info_dict.get('http_headers') if add_headers: headers.update(add_headers) @@ -106,7 +105,7 @@ class HttpFD(FileDownloader): self.report_retry(count, retries) if count > retries: - self.report_error(u'giving up after %s retries' % retries) + self.report_error('giving up after %s retries' % retries) return False data_len = data.info().get('Content-length', None) @@ -124,26 +123,31 @@ class HttpFD(FileDownloader): min_data_len = self.params.get("min_filesize", None) max_data_len = self.params.get("max_filesize", None) if min_data_len is not None and data_len < min_data_len: - self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len)) + self.to_screen('\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len)) return False if max_data_len is not None and data_len > max_data_len: - self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len)) + self.to_screen('\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len)) return False data_len_str = format_bytes(data_len) byte_counter = 0 + resume_len block_size = self.params.get('buffersize', 1024) start = time.time() + + # measure time over whole while-loop, so slow_down() and best_block_size() work together properly + now = None # needed for slow_down() in the first loop run + before = start # start measuring while True: + # Download and write - before = time.time() data_block = data.read(block_size if not is_test else min(block_size, data_len - byte_counter)) - after = time.time() + byte_counter += len(data_block) + + # exit loop when download is finished if len(data_block) == 0: break - byte_counter += len(data_block) - # Open file just in time + # Open destination file just in time if stream is None: try: (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) @@ -151,19 +155,30 @@ class HttpFD(FileDownloader): filename = self.undo_temp_name(tmpfilename) self.report_destination(filename) except (OSError, IOError) as err: - self.report_error(u'unable to open for writing: %s' % str(err)) + self.report_error('unable to open for writing: %s' % str(err)) return False try: stream.write(data_block) except (IOError, OSError) as err: - self.to_stderr(u"\n") - self.report_error(u'unable to write data: %s' % str(err)) + self.to_stderr('\n') + self.report_error('unable to write data: %s' % str(err)) return False + + # Apply rate limit + self.slow_down(start, now, byte_counter - resume_len) + + # end measuring of one loop run + now = time.time() + after = now + + # Adjust block size if not self.params.get('noresizebuffer', False): block_size = self.best_block_size(after - before, len(data_block)) + before = after + # Progress message - speed = self.calc_speed(start, time.time(), byte_counter - resume_len) + speed = self.calc_speed(start, now, byte_counter - resume_len) if data_len is None: eta = percent = None else: @@ -184,14 +199,11 @@ class HttpFD(FileDownloader): if is_test and byte_counter == data_len: break - # Apply rate limit - self.slow_down(start, byte_counter - resume_len) - if stream is None: - self.to_stderr(u"\n") - self.report_error(u'Did not get any data blocks') + self.to_stderr('\n') + self.report_error('Did not get any data blocks') return False - if tmpfilename != u'-': + if tmpfilename != '-': stream.close() self.report_finish(data_len_str, (time.time() - start)) if data_len is not None and byte_counter != data_len: diff --git a/youtube_dl/downloader/mplayer.py b/youtube_dl/downloader/mplayer.py index 4de7f15f4..72cef30ea 100644 --- a/youtube_dl/downloader/mplayer.py +++ b/youtube_dl/downloader/mplayer.py @@ -1,8 +1,11 @@ +from __future__ import unicode_literals + import os import subprocess from .common import FileDownloader from ..utils import ( + check_executable, encodeFilename, ) @@ -13,19 +16,19 @@ class MplayerFD(FileDownloader): self.report_destination(filename) tmpfilename = self.temp_name(filename) - args = ['mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy', '-dumpstream', '-dumpfile', tmpfilename, url] + args = [ + 'mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy', + '-dumpstream', '-dumpfile', tmpfilename, url] # Check for mplayer first - try: - subprocess.call(['mplayer', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) - except (OSError, IOError): - self.report_error(u'MMS or RTSP download detected but "%s" could not be run' % args[0]) + if not check_executable('mplayer', ['-h']): + self.report_error('MMS or RTSP download detected but "%s" could not be run' % args[0]) return False # Download using mplayer. retval = subprocess.call(args) if retval == 0: fsize = os.path.getsize(encodeFilename(tmpfilename)) - self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize)) + self.to_screen('\r[%s] %s bytes' % (args[0], fsize)) self.try_rename(tmpfilename, filename) self._hook_progress({ 'downloaded_bytes': fsize, @@ -35,6 +38,6 @@ class MplayerFD(FileDownloader): }) return True else: - self.to_stderr(u"\n") - self.report_error(u'mplayer exited with code %d' % retval) + self.to_stderr('\n') + self.report_error('mplayer exited with code %d' % retval) return False diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py index 5eb108302..6dbbc053c 100644 --- a/youtube_dl/downloader/rtmp.py +++ b/youtube_dl/downloader/rtmp.py @@ -7,14 +7,20 @@ import sys import time from .common import FileDownloader +from ..compat import compat_str from ..utils import ( check_executable, - compat_str, encodeFilename, format_bytes, + get_exe_version, ) +def rtmpdump_version(): + return get_exe_version( + 'rtmpdump', ['--help'], r'(?i)RTMPDump\s*v?([0-9a-zA-Z._-]+)') + + class RtmpFD(FileDownloader): def real_download(self, filename, info_dict): def run_rtmpdump(args): @@ -40,13 +46,13 @@ class RtmpFD(FileDownloader): continue mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line) if mobj: - downloaded_data_len = int(float(mobj.group(1))*1024) + downloaded_data_len = int(float(mobj.group(1)) * 1024) percent = float(mobj.group(2)) if not resume_percent: resume_percent = percent resume_downloaded_data_len = downloaded_data_len - eta = self.calc_eta(start, time.time(), 100-resume_percent, percent-resume_percent) - speed = self.calc_speed(start, time.time(), downloaded_data_len-resume_downloaded_data_len) + eta = self.calc_eta(start, time.time(), 100 - resume_percent, percent - resume_percent) + speed = self.calc_speed(start, time.time(), downloaded_data_len - resume_downloaded_data_len) data_len = None if percent > 0: data_len = int(downloaded_data_len * 100 / percent) @@ -66,7 +72,7 @@ class RtmpFD(FileDownloader): # no percent for live streams mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line) if mobj: - downloaded_data_len = int(float(mobj.group(1))*1024) + downloaded_data_len = int(float(mobj.group(1)) * 1024) time_now = time.time() speed = self.calc_speed(start, time_now, downloaded_data_len) self.report_progress_live_stream(downloaded_data_len, speed, time_now - start) @@ -82,7 +88,7 @@ class RtmpFD(FileDownloader): if not cursor_in_new_line: self.to_screen('') cursor_in_new_line = True - self.to_screen('[rtmpdump] '+line) + self.to_screen('[rtmpdump] ' + line) proc.wait() if not cursor_in_new_line: self.to_screen('') @@ -146,19 +152,7 @@ class RtmpFD(FileDownloader): else: subprocess_encoding = None - if self.params.get('verbose', False): - if subprocess_encoding: - str_args = [ - a.decode(subprocess_encoding) if isinstance(a, bytes) else a - for a in args] - else: - str_args = args - try: - import pipes - shell_quote = lambda args: ' '.join(map(pipes.quote, str_args)) - except ImportError: - shell_quote = repr - self.to_screen('[debug] rtmpdump command line: ' + shell_quote(str_args)) + self._debug_cmd(args, subprocess_encoding, exe='rtmpdump') RD_SUCCESS = 0 RD_FAILED = 1 @@ -174,12 +168,12 @@ class RtmpFD(FileDownloader): while (retval == RD_INCOMPLETE or retval == RD_FAILED) and not test and not live: prevsize = os.path.getsize(encodeFilename(tmpfilename)) self.to_screen('[rtmpdump] %s bytes' % prevsize) - time.sleep(5.0) # This seems to be needed + time.sleep(5.0) # This seems to be needed retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == RD_FAILED]) cursize = os.path.getsize(encodeFilename(tmpfilename)) if prevsize == cursize and retval == RD_FAILED: break - # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those + # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those if prevsize == cursize and retval == RD_INCOMPLETE and cursize > 1024: self.to_screen('[rtmpdump] Could not download the whole video. This can happen for some advertisements.') retval = RD_SUCCESS |