diff options
| author | Sergey M․ <dstftw@gmail.com> | 2020-12-29 02:11:48 +0700 |
|---|---|---|
| committer | Sergey M․ <dstftw@gmail.com> | 2020-12-29 02:11:48 +0700 |
| commit | f1bc56c99bac05dccb01c1b68ef778eb08fbcb71 (patch) | |
| tree | e610646fb42ea37c27214bed3d0f4838cb1df318 | |
| parent | 64e419bd7386b2a16a3c2e7ac5da30427afe856d (diff) | |
| download | youtube-dl-f1bc56c99bac05dccb01c1b68ef778eb08fbcb71.tar.xz | |
[youtube:tab] Restore retry on browse requests (closes #27313, closes #27564)
| -rw-r--r-- | youtube_dl/extractor/youtube.py | 23 |
1 files changed, 19 insertions, 4 deletions
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 5ef58d730..77f128285 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -16,6 +16,7 @@ from ..jsinterp import JSInterpreter from ..swfinterp import SWFInterpreter from ..compat import ( compat_chr, + compat_HTTPError, compat_parse_qs, compat_urllib_parse_unquote, compat_urllib_parse_unquote_plus, @@ -3009,10 +3010,24 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): for page_num in itertools.count(1): if not continuation: break - browse = self._download_json( - 'https://www.youtube.com/browse_ajax', None, - 'Downloading page %d' % page_num, - headers=headers, query=continuation, fatal=False) + count = 0 + retries = 3 + while count <= retries: + try: + # Downloading page may result in intermittent 5xx HTTP error + # that is usually worked around with a retry + browse = self._download_json( + 'https://www.youtube.com/browse_ajax', None, + 'Downloading page %d%s' + % (page_num, ' (retry #%d)' % count if count else ''), + headers=headers, query=continuation) + break + except ExtractorError as e: + if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503): + count += 1 + if count <= retries: + continue + raise if not browse: break response = try_get(browse, lambda x: x[1]['response'], dict) |