diff options
Diffstat (limited to 'youtube_dl/downloader/dash.py')
| -rw-r--r-- | youtube_dl/downloader/dash.py | 113 |
1 files changed, 65 insertions, 48 deletions
diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py index 535f2a7fc..f3c058879 100644 --- a/youtube_dl/downloader/dash.py +++ b/youtube_dl/downloader/dash.py @@ -1,66 +1,83 @@ from __future__ import unicode_literals -import re +import itertools -from .common import FileDownloader -from ..utils import sanitized_Request +from .fragment import FragmentFD +from ..compat import compat_urllib_error +from ..utils import ( + DownloadError, + urljoin, +) -class DashSegmentsFD(FileDownloader): +class DashSegmentsFD(FragmentFD): """ Download segments in a DASH manifest """ - def real_download(self, filename, info_dict): - self.report_destination(filename) - tmpfilename = self.temp_name(filename) - base_url = info_dict['url'] - segment_urls = info_dict['segment_urls'] - - is_test = self.params.get('test', False) - remaining_bytes = self._TEST_FILE_SIZE if is_test else None - byte_counter = 0 - def append_url_to_file(outf, target_url, target_name, remaining_bytes=None): - self.to_screen('[DashSegments] %s: Downloading %s' % (info_dict['id'], target_name)) - req = sanitized_Request(target_url) - if remaining_bytes is not None: - req.add_header('Range', 'bytes=0-%d' % (remaining_bytes - 1)) + FD_NAME = 'dashsegments' - data = self.ydl.urlopen(req).read() + def real_download(self, filename, info_dict): + fragment_base_url = info_dict.get('fragment_base_url') + fragments = info_dict['fragments'][:1] if self.params.get( + 'test', False) else info_dict['fragments'] - if remaining_bytes is not None: - data = data[:remaining_bytes] + ctx = { + 'filename': filename, + 'total_frags': len(fragments), + } - outf.write(data) - return len(data) + self._prepare_and_start_frag_download(ctx) - def combine_url(base_url, target_url): - if re.match(r'^https?://', target_url): - return target_url - return '%s%s%s' % (base_url, '' if base_url.endswith('/') else '/', target_url) + fragment_retries = self.params.get('fragment_retries', 0) + skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True) - with open(tmpfilename, 'wb') as outf: - append_url_to_file( - outf, combine_url(base_url, info_dict['initialization_url']), - 'initialization segment') - for i, segment_url in enumerate(segment_urls): - segment_len = append_url_to_file( - outf, combine_url(base_url, segment_url), - 'segment %d / %d' % (i + 1, len(segment_urls)), - remaining_bytes) - byte_counter += segment_len - if remaining_bytes is not None: - remaining_bytes -= segment_len - if remaining_bytes <= 0: - break + for frag_index, fragment in enumerate(fragments, 1): + if frag_index <= ctx['fragment_index']: + continue + success = False + # In DASH, the first segment contains necessary headers to + # generate a valid MP4 file, so always abort for the first segment + fatal = frag_index == 1 or not skip_unavailable_fragments + fragment_url = fragment.get('url') + if not fragment_url: + assert fragment_base_url + fragment_url = urljoin(fragment_base_url, fragment['path']) + headers = info_dict.get('http_headers') + fragment_range = fragment.get('range') + if fragment_range: + headers = headers.copy() if headers else {} + headers['Range'] = 'bytes=%s' % (fragment_range,) + for count in itertools.count(): + try: + success, frag_content = self._download_fragment(ctx, fragment_url, info_dict, headers) + if not success: + return False + self._append_fragment(ctx, frag_content) + except compat_urllib_error.HTTPError as err: + # YouTube may often return 404 HTTP error for a fragment causing the + # whole download to fail. However if the same fragment is immediately + # retried with the same request data this usually succeeds (1-2 attempts + # is usually enough) thus allowing to download the whole file successfully. + # To be future-proof we will retry all fragments that fail with any + # HTTP error. + if count < fragment_retries: + self.report_retry_fragment(err, frag_index, count + 1, fragment_retries) + continue + except DownloadError: + # Don't retry fragment if error occurred during HTTP downloading + # itself since it has its own retry settings + if fatal: + raise + break - self.try_rename(tmpfilename, filename) + if not success: + if not fatal: + self.report_skip_fragment(frag_index) + continue + self.report_error('giving up after %s fragment retries' % count) + return False - self._hook_progress({ - 'downloaded_bytes': byte_counter, - 'total_bytes': byte_counter, - 'filename': filename, - 'status': 'finished', - }) + self._finish_frag_download(ctx) return True |