diff options
Diffstat (limited to 'youtube_dl/YoutubeDL.py')
| -rw-r--r-- | youtube_dl/YoutubeDL.py | 355 |
1 files changed, 207 insertions, 148 deletions
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 2a078adfb..5748ceaf3 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals import collections import errno @@ -51,9 +51,11 @@ from .utils import ( write_json_file, write_string, YoutubeDLHandler, + prepend_extension, ) from .extractor import get_info_extractor, gen_extractors -from .FileDownloader import FileDownloader +from .downloader import get_suitable_downloader +from .PostProcessor import FFmpegMergerPP from .version import __version__ @@ -148,6 +150,7 @@ class YoutubeDL(object): socket_timeout: Time to wait for unresponsive hosts, in seconds bidi_workaround: Work around buggy terminals without bidirectional text support, using fridibi + debug_printtraffic:Print out sent and received HTTP traffic The following parameters are not used by YoutubeDL itself, they are used by the FileDownloader: @@ -164,6 +167,8 @@ class YoutubeDL(object): def __init__(self, params=None): """Create a FileDownloader object with the given options.""" + if params is None: + params = {} self._ies = [] self._ies_instances = {} self._pps = [] @@ -172,7 +177,7 @@ class YoutubeDL(object): self._num_downloads = 0 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)] self._err_file = sys.stderr - self.params = {} if params is None else params + self.params = params if params.get('bidi_workaround', False): try: @@ -183,15 +188,21 @@ class YoutubeDL(object): width_args = [] else: width_args = ['-w', str(width)] - self._fribidi = subprocess.Popen( - ['fribidi', '-c', 'UTF-8'] + width_args, + sp_kwargs = dict( stdin=subprocess.PIPE, stdout=slave, stderr=self._err_file) - self._fribidi_channel = os.fdopen(master, 'rb') + try: + self._output_process = subprocess.Popen( + ['bidiv'] + width_args, **sp_kwargs + ) + except OSError: + self._output_process = subprocess.Popen( + ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs) + self._output_channel = os.fdopen(master, 'rb') except OSError as ose: if ose.errno == 2: - self.report_warning(u'Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.') + self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.') else: raise @@ -200,15 +211,13 @@ class YoutubeDL(object): and not params['restrictfilenames']): # On Python 3, the Unicode filesystem API will throw errors (#1474) self.report_warning( - u'Assuming --restrict-filenames since file system encoding ' - u'cannot encode all charactes. ' - u'Set the LC_ALL environment variable to fix this.') + 'Assuming --restrict-filenames since file system encoding ' + 'cannot encode all charactes. ' + 'Set the LC_ALL environment variable to fix this.') self.params['restrictfilenames'] = True - self.fd = FileDownloader(self, self.params) - if '%(stitle)s' in self.params.get('outtmpl', ''): - self.report_warning(u'%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') + self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.') self._setup_opener() @@ -242,17 +251,22 @@ class YoutubeDL(object): self._pps.append(pp) pp.set_downloader(self) + def add_progress_hook(self, ph): + """Add the progress hook (currently only for the file downloader)""" + self._progress_hooks.append(ph) + def _bidi_workaround(self, message): - if not hasattr(self, '_fribidi_channel'): + if not hasattr(self, '_output_channel'): return message - assert type(message) == type(u'') - line_count = message.count(u'\n') + 1 - self._fribidi.stdin.write((message + u'\n').encode('utf-8')) - self._fribidi.stdin.flush() - res = u''.join(self._fribidi_channel.readline().decode('utf-8') + assert hasattr(self, '_output_process') + assert type(message) == type('') + line_count = message.count('\n') + 1 + self._output_process.stdin.write((message + '\n').encode('utf-8')) + self._output_process.stdin.flush() + res = ''.join(self._output_channel.readline().decode('utf-8') for _ in range(line_count)) - return res[:-len(u'\n')] + return res[:-len('\n')] def to_screen(self, message, skip_eol=False): """Print message to stdout if not in quiet mode.""" @@ -264,19 +278,19 @@ class YoutubeDL(object): self.params['logger'].debug(message) elif not check_quiet or not self.params.get('quiet', False): message = self._bidi_workaround(message) - terminator = [u'\n', u''][skip_eol] + terminator = ['\n', ''][skip_eol] output = message + terminator write_string(output, self._screen_file) def to_stderr(self, message): """Print message to stderr.""" - assert type(message) == type(u'') + assert type(message) == type('') if self.params.get('logger'): self.params['logger'].error(message) else: message = self._bidi_workaround(message) - output = message + u'\n' + output = message + '\n' write_string(output, self._err_file) def to_console_title(self, message): @@ -287,21 +301,21 @@ class YoutubeDL(object): # already of type unicode() ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message)) elif 'TERM' in os.environ: - write_string(u'\033]0;%s\007' % message, self._screen_file) + write_string('\033]0;%s\007' % message, self._screen_file) def save_console_title(self): if not self.params.get('consoletitle', False): return if 'TERM' in os.environ: # Save the title on stack - write_string(u'\033[22;0t', self._screen_file) + write_string('\033[22;0t', self._screen_file) def restore_console_title(self): if not self.params.get('consoletitle', False): return if 'TERM' in os.environ: # Restore the title from stack - write_string(u'\033[23;0t', self._screen_file) + write_string('\033[23;0t', self._screen_file) def __enter__(self): self.save_console_title() @@ -327,13 +341,13 @@ class YoutubeDL(object): if self.params.get('verbose'): if tb is None: if sys.exc_info()[0]: # if .trouble has been called from an except block - tb = u'' + tb = '' if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]: - tb += u''.join(traceback.format_exception(*sys.exc_info()[1].exc_info)) + tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info)) tb += compat_str(traceback.format_exc()) else: tb_data = traceback.format_list(traceback.extract_stack()) - tb = u''.join(tb_data) + tb = ''.join(tb_data) self.to_stderr(tb) if not self.params.get('ignoreerrors', False): if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]: @@ -349,10 +363,10 @@ class YoutubeDL(object): If stderr is a tty file the 'WARNING:' will be colored ''' if self._err_file.isatty() and os.name != 'nt': - _msg_header = u'\033[0;33mWARNING:\033[0m' + _msg_header = '\033[0;33mWARNING:\033[0m' else: - _msg_header = u'WARNING:' - warning_message = u'%s %s' % (_msg_header, message) + _msg_header = 'WARNING:' + warning_message = '%s %s' % (_msg_header, message) self.to_stderr(warning_message) def report_error(self, message, tb=None): @@ -361,18 +375,18 @@ class YoutubeDL(object): in red if stderr is a tty file. ''' if self._err_file.isatty() and os.name != 'nt': - _msg_header = u'\033[0;31mERROR:\033[0m' + _msg_header = '\033[0;31mERROR:\033[0m' else: - _msg_header = u'ERROR:' - error_message = u'%s %s' % (_msg_header, message) + _msg_header = 'ERROR:' + error_message = '%s %s' % (_msg_header, message) self.trouble(error_message, tb) def report_file_already_downloaded(self, file_name): """Report file has already been fully downloaded.""" try: - self.to_screen(u'[download] %s has already been downloaded' % file_name) + self.to_screen('[download] %s has already been downloaded' % file_name) except UnicodeEncodeError: - self.to_screen(u'[download] The file has already been downloaded') + self.to_screen('[download] The file has already been downloaded') def increment_downloads(self): """Increment the ordinal that assigns a number to each file.""" @@ -387,61 +401,61 @@ class YoutubeDL(object): autonumber_size = self.params.get('autonumber_size') if autonumber_size is None: autonumber_size = 5 - autonumber_templ = u'%0' + str(autonumber_size) + u'd' + autonumber_templ = '%0' + str(autonumber_size) + 'd' template_dict['autonumber'] = autonumber_templ % self._num_downloads if template_dict.get('playlist_index') is not None: - template_dict['playlist_index'] = u'%05d' % template_dict['playlist_index'] + template_dict['playlist_index'] = '%05d' % template_dict['playlist_index'] sanitize = lambda k, v: sanitize_filename( compat_str(v), restricted=self.params.get('restrictfilenames'), - is_id=(k == u'id')) + is_id=(k == 'id')) template_dict = dict((k, sanitize(k, v)) for k, v in template_dict.items() if v is not None) - template_dict = collections.defaultdict(lambda: u'NA', template_dict) + template_dict = collections.defaultdict(lambda: 'NA', template_dict) tmpl = os.path.expanduser(self.params['outtmpl']) filename = tmpl % template_dict return filename except ValueError as err: - self.report_error(u'Error in output template: ' + str(err) + u' (encoding: ' + repr(preferredencoding()) + ')') + self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')') return None def _match_entry(self, info_dict): """ Returns None iff the file should be downloaded """ - video_title = info_dict.get('title', info_dict.get('id', u'video')) + video_title = info_dict.get('title', info_dict.get('id', 'video')) if 'title' in info_dict: # This can happen when we're just evaluating the playlist title = info_dict['title'] matchtitle = self.params.get('matchtitle', False) if matchtitle: if not re.search(matchtitle, title, re.IGNORECASE): - return u'"' + title + '" title did not match pattern "' + matchtitle + '"' + return '"' + title + '" title did not match pattern "' + matchtitle + '"' rejecttitle = self.params.get('rejecttitle', False) if rejecttitle: if re.search(rejecttitle, title, re.IGNORECASE): - return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' + return '"' + title + '" title matched reject pattern "' + rejecttitle + '"' date = info_dict.get('upload_date', None) if date is not None: dateRange = self.params.get('daterange', DateRange()) if date not in dateRange: - return u'%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange) + return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange) view_count = info_dict.get('view_count', None) if view_count is not None: min_views = self.params.get('min_views') if min_views is not None and view_count < min_views: - return u'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views) + return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views) max_views = self.params.get('max_views') if max_views is not None and view_count > max_views: - return u'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views) + return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views) age_limit = self.params.get('age_limit') if age_limit is not None: if age_limit < info_dict.get('age_limit', 0): - return u'Skipping "' + title + '" because it is age restricted' + return 'Skipping "' + title + '" because it is age restricted' if self.in_download_archive(info_dict): - return u'%s has already been recorded in archive' % video_title + return '%s has already been recorded in archive' % video_title return None @staticmethod @@ -468,8 +482,8 @@ class YoutubeDL(object): continue if not ie.working(): - self.report_warning(u'The program functionality for this site has been marked as broken, ' - u'and will probably not work.') + self.report_warning('The program functionality for this site has been marked as broken, ' + 'and will probably not work.') try: ie_result = ie.extract(url) @@ -502,7 +516,7 @@ class YoutubeDL(object): else: raise else: - self.report_error(u'no suitable InfoExtractor: %s' % url) + self.report_error('no suitable InfoExtractor: %s' % url) def process_ie_result(self, ie_result, download=True, extra_info={}): """ @@ -533,7 +547,7 @@ class YoutubeDL(object): def make_result(embedded_info): new_result = ie_result.copy() for f in ('_type', 'url', 'ext', 'player_url', 'formats', - 'entries', 'urlhandle', 'ie_key', 'duration', + 'entries', 'ie_key', 'duration', 'subtitles', 'annotations', 'format', 'thumbnail', 'thumbnails'): if f in new_result: @@ -553,7 +567,7 @@ class YoutubeDL(object): elif result_type == 'playlist': # We process each entry in the playlist playlist = ie_result.get('title', None) or ie_result.get('id', None) - self.to_screen(u'[download] Downloading playlist: %s' % playlist) + self.to_screen('[download] Downloading playlist: %s' % playlist) playlist_results = [] @@ -568,11 +582,11 @@ class YoutubeDL(object): n_entries = len(entries) self.to_screen( - u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" % + "[%s] playlist '%s': Collected %d video ids (downloading %d of them)" % (ie_result['extractor'], playlist, n_all_entries, n_entries)) for i, entry in enumerate(entries, 1): - self.to_screen(u'[download] Downloading video #%s of %s' % (i, n_entries)) + self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries)) extra = { 'playlist': playlist, 'playlist_index': i + playliststart, @@ -584,7 +598,7 @@ class YoutubeDL(object): reason = self._match_entry(entry) if reason is not None: - self.to_screen(u'[download] ' + reason) + self.to_screen('[download] ' + reason) continue entry_result = self.process_ie_result(entry, @@ -617,7 +631,7 @@ class YoutubeDL(object): elif format_spec == 'worst': return available_formats[0] else: - extensions = [u'mp4', u'flv', u'webm', u'3gp'] + extensions = ['mp4', 'flv', 'webm', '3gp'] if format_spec in extensions: filter_f = lambda f: f['ext'] == format_spec else: @@ -636,7 +650,7 @@ class YoutubeDL(object): info_dict['playlist_index'] = None # This extractors handle format selection themselves - if info_dict['extractor'] in [u'youtube', u'Youku']: + if info_dict['extractor'] in ['Youku']: if download: self.process_info(info_dict) return info_dict @@ -653,33 +667,32 @@ class YoutubeDL(object): if format.get('format_id') is None: format['format_id'] = compat_str(i) if format.get('format') is None: - format['format'] = u'{id} - {res}{note}'.format( + format['format'] = '{id} - {res}{note}'.format( id=format['format_id'], res=self.format_resolution(format), - note=u' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '', + note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '', ) # Automatically determine file extension if missing if 'ext' not in format: format['ext'] = determine_ext(format['url']) - if self.params.get('listformats', None): - self.list_formats(info_dict) - return - format_limit = self.params.get('format_limit', None) if format_limit: formats = list(takewhile_inclusive( lambda f: f['format_id'] != format_limit, formats )) - if self.params.get('prefer_free_formats'): - def _free_formats_key(f): - try: - ext_ord = [u'flv', u'mp4', u'webm'].index(f['ext']) - except ValueError: - ext_ord = -1 - # We only compare the extension if they have the same height and width - return (f.get('height'), f.get('width'), ext_ord) - formats = sorted(formats, key=_free_formats_key) + + # TODO Central sorting goes here + + if formats[0] is not info_dict: + # only set the 'formats' fields if the original info_dict list them + # otherwise we end up with a circular reference, the first (and unique) + # element in the 'formats' field in info_dict is info_dict itself, + # wich can't be exported to json + info_dict['formats'] = formats + if self.params.get('listformats', None): + self.list_formats(info_dict) + return req_format = self.params.get('format', 'best') if req_format is None: @@ -689,21 +702,35 @@ class YoutubeDL(object): if req_format in ('-1', 'all'): formats_to_download = formats else: - # We can accept formats requestd in the format: 34/5/best, we pick + # We can accept formats requested in the format: 34/5/best, we pick # the first that is available, starting from left req_formats = req_format.split('/') for rf in req_formats: - selected_format = self.select_format(rf, formats) + if re.match(r'.+?\+.+?', rf) is not None: + # Two formats have been requested like '137+139' + format_1, format_2 = rf.split('+') + formats_info = (self.select_format(format_1, formats), + self.select_format(format_2, formats)) + if all(formats_info): + selected_format = { + 'requested_formats': formats_info, + 'format': rf, + 'ext': formats_info[0]['ext'], + } + else: + selected_format = None + else: + selected_format = self.select_format(rf, formats) if selected_format is not None: formats_to_download = [selected_format] break if not formats_to_download: - raise ExtractorError(u'requested format not available', + raise ExtractorError('requested format not available', expected=True) if download: if len(formats_to_download) > 1: - self.to_screen(u'[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download))) + self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download))) for format in formats_to_download: new_info = dict(info_dict) new_info.update(format) @@ -721,7 +748,7 @@ class YoutubeDL(object): info_dict['fulltitle'] = info_dict['title'] if len(info_dict['title']) > 200: - info_dict['title'] = info_dict['title'][:197] + u'...' + info_dict['title'] = info_dict['title'][:197] + '...' # Keep for backwards compatibility info_dict['stitle'] = info_dict['title'] @@ -731,7 +758,7 @@ class YoutubeDL(object): reason = self._match_entry(info_dict) if reason is not None: - self.to_screen(u'[download] ' + reason) + self.to_screen('[download] ' + reason) return max_downloads = self.params.get('max_downloads') @@ -748,7 +775,7 @@ class YoutubeDL(object): self.to_stdout(info_dict['id']) if self.params.get('forceurl', False): # For RTMP URLs, also include the playpath - self.to_stdout(info_dict['url'] + info_dict.get('play_path', u'')) + self.to_stdout(info_dict['url'] + info_dict.get('play_path', '')) if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None: self.to_stdout(info_dict['thumbnail']) if self.params.get('forcedescription', False) and info_dict.get('description') is not None: @@ -775,37 +802,37 @@ class YoutubeDL(object): if dn != '' and not os.path.exists(dn): os.makedirs(dn) except (OSError, IOError) as err: - self.report_error(u'unable to create directory ' + compat_str(err)) + self.report_error('unable to create directory ' + compat_str(err)) return if self.params.get('writedescription', False): - descfn = filename + u'.description' + descfn = filename + '.description' if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)): - self.to_screen(u'[info] Video description is already present') + self.to_screen('[info] Video description is already present') else: try: - self.to_screen(u'[info] Writing video description to: ' + descfn) + self.to_screen('[info] Writing video description to: ' + descfn) with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile: descfile.write(info_dict['description']) except (KeyError, TypeError): - self.report_warning(u'There\'s no description to write.') + self.report_warning('There\'s no description to write.') except (OSError, IOError): - self.report_error(u'Cannot write description file ' + descfn) + self.report_error('Cannot write description file ' + descfn) return if self.params.get('writeannotations', False): - annofn = filename + u'.annotations.xml' + annofn = filename + '.annotations.xml' if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)): - self.to_screen(u'[info] Video annotations are already present') + self.to_screen('[info] Video annotations are already present') else: try: - self.to_screen(u'[info] Writing video annotations to: ' + annofn) + self.to_screen('[info] Writing video annotations to: ' + annofn) with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile: annofile.write(info_dict['annotations']) except (KeyError, TypeError): - self.report_warning(u'There are no annotations to write.') + self.report_warning('There are no annotations to write.') except (OSError, IOError): - self.report_error(u'Cannot write annotations file: ' + annofn) + self.report_error('Cannot write annotations file: ' + annofn) return subtitles_are_requested = any([self.params.get('writesubtitles', False), @@ -823,46 +850,45 @@ class YoutubeDL(object): try: sub_filename = subtitles_filename(filename, sub_lang, sub_format) if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)): - self.to_screen(u'[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format)) + self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format)) else: - self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename) + self.to_screen('[info] Writing video subtitles to: ' + sub_filename) with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: subfile.write(sub) except (OSError, IOError): - self.report_error(u'Cannot write subtitles file ' + descfn) + self.report_error('Cannot write subtitles file ' + descfn) return if self.params.get('writeinfojson', False): - infofn = os.path.splitext(filename)[0] + u'.info.json' + infofn = os.path.splitext(filename)[0] + '.info.json' if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): - self.to_screen(u'[info] Video description metadata is already present') + self.to_screen('[info] Video description metadata is already present') else: - self.to_screen(u'[info] Writing video description metadata as JSON to: ' + infofn) + self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn) try: - json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle']) - write_json_file(json_info_dict, encodeFilename(infofn)) + write_json_file(info_dict, encodeFilename(infofn)) except (OSError, IOError): - self.report_error(u'Cannot write metadata to JSON file ' + infofn) + self.report_error('Cannot write metadata to JSON file ' + infofn) return if self.params.get('writethumbnail', False): if info_dict.get('thumbnail') is not None: - thumb_format = determine_ext(info_dict['thumbnail'], u'jpg') - thumb_filename = os.path.splitext(filename)[0] + u'.' + thumb_format + thumb_format = determine_ext(info_dict['thumbnail'], 'jpg') + thumb_filename = os.path.splitext(filename)[0] + '.' + thumb_format if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)): - self.to_screen(u'[%s] %s: Thumbnail is already present' % + self.to_screen('[%s] %s: Thumbnail is already present' % (info_dict['extractor'], info_dict['id'])) else: - self.to_screen(u'[%s] %s: Downloading thumbnail ...' % + self.to_screen('[%s] %s: Downloading thumbnail ...' % (info_dict['extractor'], info_dict['id'])) try: uf = compat_urllib_request.urlopen(info_dict['thumbnail']) with open(thumb_filename, 'wb') as thumbf: shutil.copyfileobj(uf, thumbf) - self.to_screen(u'[%s] %s: Writing thumbnail to: %s' % + self.to_screen('[%s] %s: Writing thumbnail to: %s' % (info_dict['extractor'], info_dict['id'], thumb_filename)) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self.report_warning(u'Unable to download thumbnail "%s": %s' % + self.report_warning('Unable to download thumbnail "%s": %s' % (info_dict['thumbnail'], compat_str(err))) if not self.params.get('skip_download', False): @@ -870,21 +896,41 @@ class YoutubeDL(object): success = True else: try: - success = self.fd._do_download(filename, info_dict) + def dl(name, info): + fd = get_suitable_downloader(info)(self, self.params) + for ph in self._progress_hooks: + fd.add_progress_hook(ph) + return fd.download(name, info) + if info_dict.get('requested_formats') is not None: + downloaded = [] + success = True + for f in info_dict['requested_formats']: + new_info = dict(info_dict) + new_info.update(f) + fname = self.prepare_filename(new_info) + fname = prepend_extension(fname, 'f%s' % f['format_id']) + downloaded.append(fname) + partial_success = dl(fname, new_info) + success = success and partial_success + info_dict['__postprocessors'] = [FFmpegMergerPP(self)] + info_dict['__files_to_merge'] = downloaded + else: + # Just a single file + success = dl(filename, info_dict) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self.report_error(u'unable to download video data: %s' % str(err)) + self.report_error('unable to download video data: %s' % str(err)) return except (OSError, IOError) as err: raise UnavailableVideoError(err) except (ContentTooShortError, ) as err: - self.report_error(u'content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) + self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded)) return if success: try: self.post_process(filename, info_dict) except (PostProcessingError) as err: - self.report_error(u'postprocessing: %s' % str(err)) + self.report_error('postprocessing: %s' % str(err)) return self.record_download_archive(info_dict) @@ -901,9 +947,9 @@ class YoutubeDL(object): #It also downloads the videos self.extract_info(url) except UnavailableVideoError: - self.report_error(u'unable to download video') + self.report_error('unable to download video') except MaxDownloadsReached: - self.to_screen(u'[info] Maximum number of downloaded files reached.') + self.to_screen('[info] Maximum number of downloaded files reached.') raise return self._download_retcode @@ -916,7 +962,7 @@ class YoutubeDL(object): except DownloadError: webpage_url = info.get('webpage_url') if webpage_url is not None: - self.report_warning(u'The info failed to download, trying with "%s"' % webpage_url) + self.report_warning('The info failed to download, trying with "%s"' % webpage_url) return self.download([webpage_url]) else: raise @@ -927,7 +973,11 @@ class YoutubeDL(object): info = dict(ie_info) info['filepath'] = filename keep_video = None - for pp in self._pps: + pps_chain = [] + if ie_info.get('__postprocessors') is not None: + pps_chain.extend(ie_info['__postprocessors']) + pps_chain.extend(self._pps) + for pp in pps_chain: try: keep_video_wish, new_info = pp.run(info) if keep_video_wish is not None: @@ -940,10 +990,10 @@ class YoutubeDL(object): self.report_error(e.msg) if keep_video is False and not self.params.get('keepvideo', False): try: - self.to_screen(u'Deleting original file %s (pass -k to keep)' % filename) + self.to_screen('Deleting original file %s (pass -k to keep)' % filename) os.remove(encodeFilename(filename)) except (IOError, OSError): - self.report_warning(u'Unable to remove downloaded video file') + self.report_warning('Unable to remove downloaded video file') def _make_archive_id(self, info_dict): # Future-proof against any change in case @@ -954,7 +1004,7 @@ class YoutubeDL(object): extractor = info_dict.get('ie_key') # key in a playlist if extractor is None: return None # Incomplete video information - return extractor.lower() + u' ' + info_dict['id'] + return extractor.lower() + ' ' + info_dict['id'] def in_download_archive(self, info_dict): fn = self.params.get('download_archive') @@ -982,53 +1032,59 @@ class YoutubeDL(object): vid_id = self._make_archive_id(info_dict) assert vid_id with locked_file(fn, 'a', encoding='utf-8') as archive_file: - archive_file.write(vid_id + u'\n') + archive_file.write(vid_id + '\n') @staticmethod def format_resolution(format, default='unknown'): if format.get('vcodec') == 'none': return 'audio only' - if format.get('_resolution') is not None: - return format['_resolution'] + if format.get('resolution') is not None: + return format['resolution'] if format.get('height') is not None: if format.get('width') is not None: - res = u'%sx%s' % (format['width'], format['height']) + res = '%sx%s' % (format['width'], format['height']) else: - res = u'%sp' % format['height'] + res = '%sp' % format['height'] + elif format.get('width') is not None: + res = '?x%d' % format['width'] else: res = default return res def list_formats(self, info_dict): def format_note(fdict): - res = u'' + res = '' + if fdict.get('ext') in ['f4f', 'f4m']: + res += '(unsupported) ' if fdict.get('format_note') is not None: - res += fdict['format_note'] + u' ' + res += fdict['format_note'] + ' ' + if fdict.get('tbr') is not None: + res += '%4dk ' % fdict['tbr'] if (fdict.get('vcodec') is not None and fdict.get('vcodec') != 'none'): - res += u'%-5s' % fdict['vcodec'] - elif fdict.get('vbr') is not None: - res += u'video' + res += '%-5s@' % fdict['vcodec'] + elif fdict.get('vbr') is not None and fdict.get('abr') is not None: + res += 'video@' if fdict.get('vbr') is not None: - res += u'@%4dk' % fdict['vbr'] + res += '%4dk' % fdict['vbr'] if fdict.get('acodec') is not None: if res: - res += u', ' - res += u'%-5s' % fdict['acodec'] + res += ', ' + res += '%-5s' % fdict['acodec'] elif fdict.get('abr') is not None: if res: - res += u', ' + res += ', ' res += 'audio' if fdict.get('abr') is not None: - res += u'@%3dk' % fdict['abr'] + res += '@%3dk' % fdict['abr'] if fdict.get('filesize') is not None: if res: - res += u', ' + res += ', ' res += format_bytes(fdict['filesize']) return res def line(format, idlen=20): - return ((u'%-' + compat_str(idlen + 1) + u's%-10s%-12s%s') % ( + return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % ( format['format_id'], format['ext'], self.format_resolution(format), @@ -1036,7 +1092,7 @@ class YoutubeDL(object): )) formats = info_dict.get('formats', [info_dict]) - idlen = max(len(u'format code'), + idlen = max(len('format code'), max(len(f['format_id']) for f in formats)) formats_s = [line(f, idlen) for f in formats] if len(formats) > 1: @@ -1044,10 +1100,10 @@ class YoutubeDL(object): formats_s[-1] += (' ' if format_note(formats[-1]) else '') + '(best)' header_line = line({ - 'format_id': u'format code', 'ext': u'extension', - '_resolution': u'resolution', 'format_note': u'note'}, idlen=idlen) - self.to_screen(u'[info] Available formats for %s:\n%s\n%s' % - (info_dict['id'], header_line, u"\n".join(formats_s))) + 'format_id': 'format code', 'ext': 'extension', + 'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen) + self.to_screen('[info] Available formats for %s:\n%s\n%s' % + (info_dict['id'], header_line, '\n'.join(formats_s))) def urlopen(self, req): """ Start an HTTP download """ @@ -1056,7 +1112,7 @@ class YoutubeDL(object): def print_debug_header(self): if not self.params.get('verbose'): return - write_string(u'[debug] youtube-dl version ' + __version__ + u'\n') + write_string('[debug] youtube-dl version ' + __version__ + '\n') try: sp = subprocess.Popen( ['git', 'rev-parse', '--short', 'HEAD'], @@ -1065,20 +1121,20 @@ class YoutubeDL(object): out, err = sp.communicate() out = out.decode().strip() if re.match('[0-9a-f]+', out): - write_string(u'[debug] Git HEAD: ' + out + u'\n') + write_string('[debug] Git HEAD: ' + out + '\n') except: try: sys.exc_clear() except: pass - write_string(u'[debug] Python version %s - %s' % - (platform.python_version(), platform_name()) + u'\n') + write_string('[debug] Python version %s - %s' % + (platform.python_version(), platform_name()) + '\n') proxy_map = {} for handler in self._opener.handlers: if hasattr(handler, 'proxies'): proxy_map.update(handler.proxies) - write_string(u'[debug] Proxy map: ' + compat_str(proxy_map) + u'\n') + write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n') def _setup_opener(self): timeout_val = self.params.get('socket_timeout') @@ -1108,10 +1164,13 @@ class YoutubeDL(object): if 'http' in proxies and 'https' not in proxies: proxies['https'] = proxies['http'] proxy_handler = compat_urllib_request.ProxyHandler(proxies) + + debuglevel = 1 if self.params.get('debug_printtraffic') else 0 https_handler = make_HTTPS_handler( - self.params.get('nocheckcertificate', False)) + self.params.get('nocheckcertificate', False), debuglevel=debuglevel) + ydlh = YoutubeDLHandler(debuglevel=debuglevel) opener = compat_urllib_request.build_opener( - https_handler, proxy_handler, cookie_processor, YoutubeDLHandler()) + https_handler, proxy_handler, cookie_processor, ydlh) # Delete the default user-agent header, which would otherwise apply in # cases where our custom HTTP handler doesn't come into play # (See https://github.com/rg3/youtube-dl/issues/1309 for details) |