From aa37e3d486f52b8c7a22dd5255469292a6a6afb9 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Fri, 12 Sep 2014 07:50:31 +0200 Subject: [utils] Default SSL to TLS. (Fixes #3727) On 2.x, we now try TLS first, and fall back to the compat 23 (basically anything) afterwards. On 3.4+, we now use the proper function so that we get all the latest security configurations. We allow SSLv3 though for the time being, since a lot of older pages use that. On 3.3, we default to SSLv23 (basically "anything, including TLS") because that has the widest compatibility. --- youtube_dl/utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'youtube_dl/utils.py') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 0bc410e91..d920c65a4 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -617,7 +617,7 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs): self.sock = sock self._tunnel() try: - self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv3) + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_TLSv1) except ssl.SSLError: self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, ssl_version=ssl.PROTOCOL_SSLv23) @@ -625,8 +625,14 @@ def make_HTTPS_handler(opts_no_check_certificate, **kwargs): def https_open(self, req): return self.do_open(HTTPSConnectionV3, req) return HTTPSHandlerV3(**kwargs) - else: - context = ssl.SSLContext(ssl.PROTOCOL_SSLv3) + elif hasattr(ssl, 'create_default_context'): # Python >= 3.4 + context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + context.options &= ~ssl.OP_NO_SSLv3 # Allow older, not-as-secure SSLv3 + if opts_no_check_certificate: + context.verify_mode = ssl.CERT_NONE + return compat_urllib_request.HTTPSHandler(context=context, **kwargs) + else: # Python < 3.4 + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.verify_mode = (ssl.CERT_NONE if opts_no_check_certificate else ssl.CERT_REQUIRED) -- cgit v1.2.3 From 4eefbfdbfd472398ed5e40b13d20e3a92f837f52 Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 13 Sep 2014 08:34:15 +0200 Subject: [utils] Fix find_xpath_attr on 2.6 --- youtube_dl/utils.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'youtube_dl/utils.py') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index d920c65a4..8828161e5 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -280,6 +280,11 @@ if sys.version_info >= (2, 7): return node.find(expr) else: def find_xpath_attr(node, xpath, key, val): + # Here comes the crazy part: In 2.6, if the xpath is a unicode, + # .//node does not match if a node is a direct child of . ! + if isinstance(xpath, unicode): + xpath = xpath.encode('ascii') + for f in node.findall(xpath): if f.attrib.get(key) == val: return f -- cgit v1.2.3 From bf0ff93277ba36fbda70223ca7e78b5132e54ddf Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 13 Sep 2014 09:09:55 +0200 Subject: [ard] Make more robust against missing thumbnails I cannot reproduce this error, it's from travis. --- youtube_dl/utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'youtube_dl/utils.py') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 8828161e5..7536b3b36 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -304,6 +304,17 @@ def xpath_with_ns(path, ns_map): return '/'.join(replaced) +def xpath_text(node, xpath, name=None, fatal=False): + n = node.find(xpath) + if n is None: + if fatal: + name = xpath if name is None else name + raise ExtractorError('Could not find XML element %s' % name) + else: + return None + return n.text + + compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix class BaseHTMLParser(compat_html_parser.HTMLParser): def __init(self): -- cgit v1.2.3 From d74bebd50263a2f744595b9a54825914bc07657b Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Sat, 13 Sep 2014 09:11:14 +0200 Subject: [utils] Apply 2.6 xpath craziness This fixes ARD on 2.6 --- youtube_dl/utils.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'youtube_dl/utils.py') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 7536b3b36..247788078 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -305,6 +305,9 @@ def xpath_with_ns(path, ns_map): def xpath_text(node, xpath, name=None, fatal=False): + if sys.version_info < (2, 7): # Crazy 2.6 + xpath = xpath.encode('ascii') + n = node.find(xpath) if n is None: if fatal: -- cgit v1.2.3 From d05cfe06006c4a44032e95dde047d5e12be8674c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 13 Sep 2014 20:59:16 +0700 Subject: [YoutubeDL/utils] Clarify rationale for URL escaping in comment, move escape routines to utils and add some tests --- youtube_dl/utils.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'youtube_dl/utils.py') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index d920c65a4..9124c3621 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1418,6 +1418,24 @@ def uppercase_escape(s): lambda m: unicode_escape(m.group(0))[0], s) + +def escape_rfc3986(s): + """Escape non-ASCII characters as suggested by RFC 3986""" + if sys.version_info < (3, 0) and isinstance(s, unicode): + s = s.encode('utf-8') + return compat_urllib_parse.quote(s, "%/;:@&=+$,!~*'()?#[]") #"%/;:@&=+$,!~*'()?#[]+" #?#[]+ + + +def escape_url(url): + """Escape URL as suggested by RFC 3986""" + url_parsed = compat_urllib_parse_urlparse(url) + return url_parsed._replace( + path=escape_rfc3986(url_parsed.path), + params=escape_rfc3986(url_parsed.params), + query=escape_rfc3986(url_parsed.query), + fragment=escape_rfc3986(url_parsed.fragment) + ).geturl() + try: struct.pack(u'!I', 0) except TypeError: -- cgit v1.2.3 From 984e8e14ea266d406c253098f953e727ca8c19c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Sat, 13 Sep 2014 21:08:04 +0700 Subject: [utils] Remove debug garbage --- youtube_dl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube_dl/utils.py') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 9124c3621..e924b1688 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1423,7 +1423,7 @@ def escape_rfc3986(s): """Escape non-ASCII characters as suggested by RFC 3986""" if sys.version_info < (3, 0) and isinstance(s, unicode): s = s.encode('utf-8') - return compat_urllib_parse.quote(s, "%/;:@&=+$,!~*'()?#[]") #"%/;:@&=+$,!~*'()?#[]+" #?#[]+ + return compat_urllib_parse.quote(s, "%/;:@&=+$,!~*'()?#[]") def escape_url(url): -- cgit v1.2.3 From a020a0dc20ced6468ec46214c394f6f360735b1d Mon Sep 17 00:00:00 2001 From: Philipp Hagemeister Date: Mon, 15 Sep 2014 15:10:24 +0200 Subject: [facebook] Fix support for untitled videos (Fixes #3757) --- youtube_dl/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'youtube_dl/utils.py') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 247788078..3ac0f1f54 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1571,3 +1571,13 @@ except AttributeError: if ret: raise subprocess.CalledProcessError(ret, p.args, output=output) return output + + +def limit_length(s, length): + """ Add ellipses to overly long strings """ + if s is None: + return None + ELLIPSES = '...' + if len(s) > length: + return s[:length - len(ELLIPSES)] + ELLIPSES + return s -- cgit v1.2.3