diff options
| -rw-r--r-- | test/test_compat.py | 20 | ||||
| -rw-r--r-- | youtube_dl/compat.py | 91 | ||||
| -rw-r--r-- | youtube_dl/extractor/generic.py | 2 |
3 files changed, 84 insertions, 29 deletions
diff --git a/test/test_compat.py b/test/test_compat.py index 1eb454e06..431e6bdf1 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -14,6 +14,7 @@ from youtube_dl.utils import get_filesystem_encoding from youtube_dl.compat import ( compat_getenv, compat_expanduser, + compat_urllib_parse_unquote, ) @@ -42,5 +43,24 @@ class TestCompat(unittest.TestCase): dir(youtube_dl.compat))) - set(['unicode_literals']) self.assertEqual(all_names, sorted(present_names)) + def test_compat_urllib_parse_unquote(self): + test_strings = [ + ['''''', ''''''], + ['''津波''', '''%E6%B4%A5%E6%B3%A2'''], + ['''津波''', str('%E6%B4%A5%E6%B3%A2')], + ['''<meta property="og:description" content="▁▂▃▄%▅▆▇█" /> +%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''', + '''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" /> +%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''], + ['''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%Things%''', + '''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%'''] + ] + for test in test_strings: + strutf = test[0] + strurlenc = test[1] + strurldec = compat_urllib_parse_unquote(strurlenc) + self.assertEqual(strutf, strurldec) + self.assertEqual(strutf, compat_urllib_parse_unquote(strurlenc)) + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index c3783337a..9e506352f 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -77,40 +77,73 @@ except ImportError: try: from urllib.parse import unquote as compat_urllib_parse_unquote except ImportError: - def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): - if string == '': + def compat_urllib_parse_unquote_to_bytes(string): + """unquote_to_bytes('abc%20def') -> b'abc def'.""" + # Note: strings are encoded as UTF-8. This is only an issue if it contains + # unescaped non-ASCII characters, which URIs should not. + if not string: + # Is it a string-like object? + string.split + return b'' + if isinstance(string, str): + string = string.encode('utf-8') + # string = encode('utf-8') + + # python3 -> 2: must implicitly convert to bits + bits = bytes(string).split(b'%') + + if len(bits) == 1: return string - res = string.split('%') - if len(res) == 1: + res = [bits[0]] + append = res.append + + for item in bits[1:]: + if item == '': + append(b'%') + continue + try: + append(item[:2].decode('hex')) + append(item[2:]) + except: + append(b'%') + append(item) + return b''.join(res) + + compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)') + + def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): + """Replace %xx escapes by their single-character equivalent. The optional + encoding and errors parameters specify how to decode percent-encoded + sequences into Unicode characters, as accepted by the bytes.decode() + method. + By default, percent-encoded sequences are decoded with UTF-8, and invalid + sequences are replaced by a placeholder character. + + unquote('abc%20def') -> 'abc def'. + """ + + if '%' not in string: + string.split return string if encoding is None: encoding = 'utf-8' if errors is None: errors = 'replace' - # pct_sequence: contiguous sequence of percent-encoded bytes, decoded - pct_sequence = b'' - string = res[0] - for item in res[1:]: - try: - if not item: - raise ValueError - pct_sequence += item[:2].decode('hex') - rest = item[2:] - if not rest: - # This segment was just a single percent-encoded character. - # May be part of a sequence of code units, so delay decoding. - # (Stored in pct_sequence). - continue - except ValueError: - rest = '%' + item - # Encountered non-percent-encoded characters. Flush the current - # pct_sequence. - string += pct_sequence.decode(encoding, errors) + rest - pct_sequence = b'' - if pct_sequence: - # Flush the final pct_sequence - string += pct_sequence.decode(encoding, errors) - return string + + bits = compat_urllib_parse_asciire.split(string) + res = [bits[0]] + append = res.append + for i in range(1, len(bits), 2): + foo = compat_urllib_parse_unquote_to_bytes(bits[i]) + foo = foo.decode(encoding, errors) + append(foo) + + if bits[i + 1]: + bar = bits[i + 1] + if not isinstance(bar, unicode): + bar = bar.decode('utf-8') + append(bar) + return ''.join(res) try: compat_str = unicode # Python 2 @@ -421,7 +454,9 @@ __all__ = [ 'compat_subprocess_get_DEVNULL', 'compat_urllib_error', 'compat_urllib_parse', + 'compat_urllib_parse_asciire', 'compat_urllib_parse_unquote', + 'compat_urllib_parse_unquote_to_bytes', 'compat_urllib_parse_urlparse', 'compat_urllib_request', 'compat_urlparse', diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 392ad3648..fc1bf2b6e 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -1115,7 +1115,7 @@ class GenericIE(InfoExtractor): # Sometimes embedded video player is hidden behind percent encoding # (e.g. https://github.com/rg3/youtube-dl/issues/2448) # Unescaping the whole page allows to handle those cases in a generic way - webpage = compat_urllib_parse.unquote(webpage) + webpage = compat_urllib_parse_unquote(webpage) # it's tempting to parse this further, but you would # have to take into account all the variations like |