From 37419b4f9937f11ed3ca3545a32ed3451eb734ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Fri, 12 Sep 2014 23:20:17 +0700 Subject: [YoutubeDL] Escape non-ASCII characters in URLs urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991) Working around by replacing request's original URL with escaped one --- youtube_dl/YoutubeDL.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'youtube_dl/YoutubeDL.py') diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 553bf559b..99cd05e65 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -28,6 +28,7 @@ from .utils import ( compat_str, compat_urllib_error, compat_urllib_request, + compat_urllib_parse_urlparse, ContentTooShortError, date_from_str, DateRange, @@ -1241,6 +1242,31 @@ class YoutubeDL(object): def urlopen(self, req): """ Start an HTTP download """ + + # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991) + # Working around by replacing request's original URL with escaped one + + url = req if isinstance(req, compat_str) else req.get_full_url() + + def escape(component): + return compat_cookiejar.escape_path(component.encode('utf-8')) + + url_parsed = compat_urllib_parse_urlparse(url) + url_escaped = url_parsed._replace( + path=escape(url_parsed.path), + query=escape(url_parsed.query), + fragment=escape(url_parsed.fragment) + ).geturl() + + # Substitute URL if any change after escaping + if url != url_escaped: + if isinstance(req, compat_str): + req = url_escaped + else: + req = compat_urllib_request.Request( + url_escaped, data=req.data, headers=req.headers, + origin_req_host=req.origin_req_host, unverifiable=req.unverifiable) + return self._opener.open(req, timeout=self._socket_timeout) def print_debug_header(self): -- cgit v1.2.3 From d05cfe06006c4a44032e95dde047d5e12be8674c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 13 Sep 2014 20:59:16 +0700 Subject: [YoutubeDL/utils] Clarify rationale for URL escaping in comment, move escape routines to utils and add some tests --- youtube_dl/YoutubeDL.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) (limited to 'youtube_dl/YoutubeDL.py') diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 99cd05e65..9519594c9 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -28,7 +28,7 @@ from .utils import ( compat_str, compat_urllib_error, compat_urllib_request, - compat_urllib_parse_urlparse, + escape_url, ContentTooShortError, date_from_str, DateRange, @@ -1243,20 +1243,14 @@ class YoutubeDL(object): def urlopen(self, req): """ Start an HTTP download """ + # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not + # always respected by websites, some tend to give out URLs with non percent-encoded + # non-ASCII characters (see telemb.py, ard.py [#3412]) # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991) - # Working around by replacing request's original URL with escaped one - + # To work around aforementioned issue we will replace request's original URL with + # percent-encoded one url = req if isinstance(req, compat_str) else req.get_full_url() - - def escape(component): - return compat_cookiejar.escape_path(component.encode('utf-8')) - - url_parsed = compat_urllib_parse_urlparse(url) - url_escaped = url_parsed._replace( - path=escape(url_parsed.path), - query=escape(url_parsed.query), - fragment=escape(url_parsed.fragment) - ).geturl() + url_escaped = escape_url(url) # Substitute URL if any change after escaping if url != url_escaped: -- cgit v1.2.3 From e2e5dae64da60c37af65c7cffd18475a30fcbad3 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Thu, 18 Sep 2014 18:40:19 +0200 Subject: Add -f m4a --- youtube_dl/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube_dl/YoutubeDL.py') diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 9519594c9..eaba40bf2 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -708,7 +708,7 @@ class YoutubeDL(object): if video_formats: return video_formats[0] else: - extensions = ['mp4', 'flv', 'webm', '3gp'] + extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a'] if format_spec in extensions: filter_f = lambda f: f['ext'] == format_spec else: -- cgit v1.2.3 From 1de33fafd94c7e0d4ccede711ef7f13bd3e2301b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Thu, 18 Sep 2014 18:43:49 +0200 Subject: [YoutubeDL] Allow downloading multiple formats with , --- youtube_dl/YoutubeDL.py | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) (limited to 'youtube_dl/YoutubeDL.py') diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index eaba40bf2..a1713dc5a 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -809,28 +809,29 @@ class YoutubeDL(object): if req_format in ('-1', 'all'): formats_to_download = formats else: - # We can accept formats requested in the format: 34/5/best, we pick - # the first that is available, starting from left - req_formats = req_format.split('/') - for rf in req_formats: - if re.match(r'.+?\+.+?', rf) is not None: - # Two formats have been requested like '137+139' - format_1, format_2 = rf.split('+') - formats_info = (self.select_format(format_1, formats), - self.select_format(format_2, formats)) - if all(formats_info): - selected_format = { - 'requested_formats': formats_info, - 'format': rf, - 'ext': formats_info[0]['ext'], - } + for rfstr in req_format.split(','): + # We can accept formats requested in the format: 34/5/best, we pick + # the first that is available, starting from left + req_formats = rfstr.split('/') + for rf in req_formats: + if re.match(r'.+?\+.+?', rf) is not None: + # Two formats have been requested like '137+139' + format_1, format_2 = rf.split('+') + formats_info = (self.select_format(format_1, formats), + self.select_format(format_2, formats)) + if all(formats_info): + selected_format = { + 'requested_formats': formats_info, + 'format': rf, + 'ext': formats_info[0]['ext'], + } + else: + selected_format = None else: - selected_format = None - else: - selected_format = self.select_format(rf, formats) - if selected_format is not None: - formats_to_download = [selected_format] - break + selected_format = self.select_format(rf, formats) + if selected_format is not None: + formats_to_download.append(selected_format) + break if not formats_to_download: raise ExtractorError('requested format not available', expected=True) -- cgit v1.2.3