diff options
| -rw-r--r-- | test/test_compat.py | 44 | ||||
| -rw-r--r-- | test/test_utils.py | 14 | ||||
| -rwxr-xr-x | youtube_dl/YoutubeDL.py | 4 | ||||
| -rw-r--r-- | youtube_dl/__init__.py | 4 | ||||
| -rw-r--r-- | youtube_dl/cache.py | 6 | ||||
| -rw-r--r-- | youtube_dl/compat.py | 314 | ||||
| -rw-r--r-- | youtube_dl/extractor/addanime.py | 5 | ||||
| -rw-r--r-- | youtube_dl/extractor/cloudy.py | 10 | ||||
| -rw-r--r-- | youtube_dl/extractor/common.py | 5 | ||||
| -rw-r--r-- | youtube_dl/extractor/crunchyroll.py | 1 | ||||
| -rw-r--r-- | youtube_dl/extractor/dropbox.py | 3 | ||||
| -rw-r--r-- | youtube_dl/extractor/facebook.py | 4 | ||||
| -rw-r--r-- | youtube_dl/extractor/generic.py | 5 | ||||
| -rw-r--r-- | youtube_dl/extractor/globo.py | 8 | ||||
| -rw-r--r-- | youtube_dl/extractor/laola1tv.py | 1 | ||||
| -rw-r--r-- | youtube_dl/extractor/myvideo.py | 5 | ||||
| -rw-r--r-- | youtube_dl/extractor/ro220.py | 2 | ||||
| -rw-r--r-- | youtube_dl/extractor/vimeo.py | 4 | ||||
| -rw-r--r-- | youtube_dl/options.py | 4 | ||||
| -rw-r--r-- | youtube_dl/postprocessor/atomicparsley.py | 5 | ||||
| -rw-r--r-- | youtube_dl/postprocessor/execafterdownload.py | 6 | ||||
| -rw-r--r-- | youtube_dl/postprocessor/ffmpeg.py | 5 | ||||
| -rw-r--r-- | youtube_dl/postprocessor/xattrpp.py | 4 | ||||
| -rw-r--r-- | youtube_dl/utils.py | 290 |
24 files changed, 426 insertions, 327 deletions
diff --git a/test/test_compat.py b/test/test_compat.py new file mode 100644 index 000000000..d1a33ddc9 --- /dev/null +++ b/test/test_compat.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# coding: utf-8 + +from __future__ import unicode_literals + +# Allow direct execution +import os +import sys +import unittest +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +from youtube_dl.utils import get_filesystem_encoding +from youtube_dl.compat import ( + compat_getenv, + compat_expanduser, +) + + +class TestCompat(unittest.TestCase): + def test_compat_getenv(self): + test_str = 'тест' + os.environ['YOUTUBE-DL-TEST'] = ( + test_str if sys.version_info >= (3, 0) + else test_str.encode(get_filesystem_encoding())) + self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str) + + def test_compat_expanduser(self): + test_str = 'C:\Documents and Settings\тест\Application Data' + os.environ['HOME'] = ( + test_str if sys.version_info >= (3, 0) + else test_str.encode(get_filesystem_encoding())) + self.assertEqual(compat_expanduser('~'), test_str) + + def test_all_present(self): + import youtube_dl.compat + all_names = youtube_dl.compat.__all__ + present_names = set(filter( + lambda c: '_' in c and not c.startswith('_'), + dir(youtube_dl.compat))) + self.assertEqual(all_names, sorted(present_names)) + +if __name__ == '__main__': + unittest.main() diff --git a/test/test_utils.py b/test/test_utils.py index 0b31d1a39..338701f4c 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -46,8 +46,6 @@ from youtube_dl.utils import ( escape_url, js_to_json, get_filesystem_encoding, - compat_getenv, - compat_expanduser, ) @@ -359,17 +357,5 @@ class TestUtil(unittest.TestCase): on = js_to_json('{"abc": true}') self.assertEqual(json.loads(on), {'abc': True}) - def test_compat_getenv(self): - test_str = 'тест' - os.environ['YOUTUBE-DL-TEST'] = (test_str if sys.version_info >= (3, 0) - else test_str.encode(get_filesystem_encoding())) - self.assertEqual(compat_getenv('YOUTUBE-DL-TEST'), test_str) - - def test_compat_expanduser(self): - test_str = 'C:\Documents and Settings\тест\Application Data' - os.environ['HOME'] = (test_str if sys.version_info >= (3, 0) - else test_str.encode(get_filesystem_encoding())) - self.assertEqual(compat_expanduser('~'), test_str) - if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index f7c996c84..8732f3db4 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -22,13 +22,15 @@ import traceback if os.name == 'nt': import ctypes -from .utils import ( +from .compat import ( compat_cookiejar, compat_expanduser, compat_http_client, compat_str, compat_urllib_error, compat_urllib_request, +) +from .utils import ( escape_url, ContentTooShortError, date_from_str, diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 3c968082c..685dd8e5e 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -13,10 +13,12 @@ import sys from .options import ( parseOpts, ) -from .utils import ( +from .compat import ( compat_expanduser, compat_getpass, compat_print, +) +from .utils import ( DateRange, DEFAULT_OUTTMPL, decodeOption, diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py index ac5925d32..2d9b426cb 100644 --- a/youtube_dl/cache.py +++ b/youtube_dl/cache.py @@ -8,10 +8,8 @@ import re import shutil import traceback -from .utils import ( - compat_expanduser, - write_json_file, -) +from .compat import compat_expanduser +from .utils import write_json_file class Cache(object): diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py new file mode 100644 index 000000000..08e8d64d7 --- /dev/null +++ b/youtube_dl/compat.py @@ -0,0 +1,314 @@ +import getpass +import os +import subprocess +import sys + + +try: + import urllib.request as compat_urllib_request +except ImportError: # Python 2 + import urllib2 as compat_urllib_request + +try: + import urllib.error as compat_urllib_error +except ImportError: # Python 2 + import urllib2 as compat_urllib_error + +try: + import urllib.parse as compat_urllib_parse +except ImportError: # Python 2 + import urllib as compat_urllib_parse + +try: + from urllib.parse import urlparse as compat_urllib_parse_urlparse +except ImportError: # Python 2 + from urlparse import urlparse as compat_urllib_parse_urlparse + +try: + import urllib.parse as compat_urlparse +except ImportError: # Python 2 + import urlparse as compat_urlparse + +try: + import http.cookiejar as compat_cookiejar +except ImportError: # Python 2 + import cookielib as compat_cookiejar + +try: + import html.entities as compat_html_entities +except ImportError: # Python 2 + import htmlentitydefs as compat_html_entities + +try: + import html.parser as compat_html_parser +except ImportError: # Python 2 + import HTMLParser as compat_html_parser + +try: + import http.client as compat_http_client +except ImportError: # Python 2 + import httplib as compat_http_client + +try: + from urllib.error import HTTPError as compat_HTTPError +except ImportError: # Python 2 + from urllib2 import HTTPError as compat_HTTPError + +try: + from urllib.request import urlretrieve as compat_urlretrieve +except ImportError: # Python 2 + from urllib import urlretrieve as compat_urlretrieve + + +try: + from subprocess import DEVNULL + compat_subprocess_get_DEVNULL = lambda: DEVNULL +except ImportError: + compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w') + +try: + from urllib.parse import unquote as compat_urllib_parse_unquote +except ImportError: + def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): + if string == '': + return string + res = string.split('%') + if len(res) == 1: + return string + if encoding is None: + encoding = 'utf-8' + if errors is None: + errors = 'replace' + # pct_sequence: contiguous sequence of percent-encoded bytes, decoded + pct_sequence = b'' + string = res[0] + for item in res[1:]: + try: + if not item: + raise ValueError + pct_sequence += item[:2].decode('hex') + rest = item[2:] + if not rest: + # This segment was just a single percent-encoded character. + # May be part of a sequence of code units, so delay decoding. + # (Stored in pct_sequence). + continue + except ValueError: + rest = '%' + item + # Encountered non-percent-encoded characters. Flush the current + # pct_sequence. + string += pct_sequence.decode(encoding, errors) + rest + pct_sequence = b'' + if pct_sequence: + # Flush the final pct_sequence + string += pct_sequence.decode(encoding, errors) + return string + + +try: + from urllib.parse import parse_qs as compat_parse_qs +except ImportError: # Python 2 + # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib. + # Python 2's version is apparently totally broken + + def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False, + encoding='utf-8', errors='replace'): + qs, _coerce_result = qs, unicode + pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] + r = [] + for name_value in pairs: + if not name_value and not strict_parsing: + continue + nv = name_value.split('=', 1) + if len(nv) != 2: + if strict_parsing: + raise ValueError("bad query field: %r" % (name_value,)) + # Handle case of a control-name with no equal sign + if keep_blank_values: + nv.append('') + else: + continue + if len(nv[1]) or keep_blank_values: + name = nv[0].replace('+', ' ') + name = compat_urllib_parse_unquote( + name, encoding=encoding, errors=errors) + name = _coerce_result(name) + value = nv[1].replace('+', ' ') + value = compat_urllib_parse_unquote( + value, encoding=encoding, errors=errors) + value = _coerce_result(value) + r.append((name, value)) + return r + + def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False, + encoding='utf-8', errors='replace'): + parsed_result = {} + pairs = _parse_qsl(qs, keep_blank_values, strict_parsing, + encoding=encoding, errors=errors) + for name, value in pairs: + if name in parsed_result: + parsed_result[name].append(value) + else: + parsed_result[name] = [value] + return parsed_result + +try: + compat_str = unicode # Python 2 +except NameError: + compat_str = str + +try: + compat_chr = unichr # Python 2 +except NameError: + compat_chr = chr + +try: + from xml.etree.ElementTree import ParseError as compat_xml_parse_error +except ImportError: # Python 2.6 + from xml.parsers.expat import ExpatError as compat_xml_parse_error + +try: + from shlex import quote as shlex_quote +except ImportError: # Python < 3.3 + def shlex_quote(s): + return "'" + s.replace("'", "'\"'\"'") + "'" + + +def compat_ord(c): + if type(c) is int: return c + else: return ord(c) + + +if sys.version_info >= (3, 0): + compat_getenv = os.getenv + compat_expanduser = os.path.expanduser +else: + # Environment variables should be decoded with filesystem encoding. + # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918) + + def compat_getenv(key, default=None): + from .utils import get_filesystem_encoding + env = os.getenv(key, default) + if env: + env = env.decode(get_filesystem_encoding()) + return env + + # HACK: The default implementations of os.path.expanduser from cpython do not decode + # environment variables with filesystem encoding. We will work around this by + # providing adjusted implementations. + # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib + # for different platforms with correct environment variables decoding. + + if os.name == 'posix': + def compat_expanduser(path): + """Expand ~ and ~user constructions. If user or $HOME is unknown, + do nothing.""" + if not path.startswith('~'): + return path + i = path.find('/', 1) + if i < 0: + i = len(path) + if i == 1: + if 'HOME' not in os.environ: + import pwd + userhome = pwd.getpwuid(os.getuid()).pw_dir + else: + userhome = compat_getenv('HOME') + else: + import pwd + try: + pwent = pwd.getpwnam(path[1:i]) + except KeyError: + return path + userhome = pwent.pw_dir + userhome = userhome.rstrip('/') + return (userhome + path[i:]) or '/' + elif os.name == 'nt' or os.name == 'ce': + def compat_expanduser(path): + """Expand ~ and ~user constructs. + + If user or $HOME is unknown, do nothing.""" + if path[:1] != '~': + return path + i, n = 1, len(path) + while i < n and path[i] not in '/\\': + i = i + 1 + + if 'HOME' in os.environ: + userhome = compat_getenv('HOME') + elif 'USERPROFILE' in os.environ: + userhome = compat_getenv('USERPROFILE') + elif not 'HOMEPATH' in os.environ: + return path + else: + try: + drive = compat_getenv('HOMEDRIVE') + except KeyError: + drive = '' + userhome = os.path.join(drive, compat_getenv('HOMEPATH')) + + if i != 1: #~user + userhome = os.path.join(os.path.dirname(userhome), path[1:i]) + + return userhome + path[i:] + else: + compat_expanduser = os.path.expanduser + + +if sys.version_info < (3, 0): + def compat_print(s): + from .utils import preferredencoding + print(s.encode(preferredencoding(), 'xmlcharrefreplace')) +else: + def compat_print(s): + assert type(s) == type(u'') + print(s) + + +try: + subprocess_check_output = subprocess.check_output +except AttributeError: + def subprocess_check_output(*args, **kwargs): + assert 'input' not in kwargs + p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs) + output, _ = p.communicate() + ret = p.poll() + if ret: + raise subprocess.CalledProcessError(ret, p.args, output=output) + return output + +if sys.version_info < (3, 0) and sys.platform == 'win32': + def compat_getpass(prompt, *args, **kwargs): + if isinstance(prompt, compat_str): + prompt = prompt.encode(preferredencoding()) + return getpass.getpass(prompt, *args, **kwargs) +else: + compat_getpass = getpass.getpass + + +__all__ = [ + 'compat_HTTPError', + 'compat_chr', + 'compat_cookiejar', + 'compat_expanduser', + 'compat_getenv', + 'compat_getpass', + 'compat_html_entities', + 'compat_html_parser', + 'compat_http_client', + 'compat_ord', + 'compat_parse_qs', + 'compat_print', + 'compat_str', + 'compat_subprocess_get_DEVNULL', + 'compat_urllib_error', + 'compat_urllib_parse', + 'compat_urllib_parse_unquote', + 'compat_urllib_parse_urlparse', + 'compat_urllib_request', + 'compat_urlparse', + 'compat_urlretrieve', + 'compat_xml_parse_error', + 'shlex_quote', + 'subprocess_check_output', +] diff --git a/youtube_dl/extractor/addanime.py b/youtube_dl/extractor/addanime.py index fcf296057..11f149f9e 100644 --- a/youtube_dl/extractor/addanime.py +++ b/youtube_dl/extractor/addanime.py @@ -3,12 +3,13 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..utils import ( +from ..compat import ( compat_HTTPError, compat_str, compat_urllib_parse, compat_urllib_parse_urlparse, - +) +from ..utils import ( ExtractorError, ) diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py index 386f080d2..abf8cc280 100644 --- a/youtube_dl/extractor/cloudy.py +++ b/youtube_dl/extractor/cloudy.py @@ -4,14 +4,16 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..utils import ( - ExtractorError, +from ..compat import ( compat_parse_qs, compat_urllib_parse, - remove_end, - HEADRequest, compat_HTTPError, ) +from ..utils import ( + ExtractorError, + HEADRequest, + remove_end, +) class CloudyIE(InfoExtractor): diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 7e4113213..9e1d62c2b 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -12,13 +12,14 @@ import sys import time import xml.etree.ElementTree -from ..utils import ( +from ..compat import ( compat_http_client, compat_urllib_error, compat_urllib_parse_urlparse, compat_urlparse, compat_str, - +) +from ..utils import ( clean_html, compiled_regex_type, ExtractorError, diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py index cc612d08e..0bd0eccba 100644 --- a/youtube_dl/extractor/crunchyroll.py +++ b/youtube_dl/extractor/crunchyroll.py @@ -17,7 +17,6 @@ from ..utils import ( bytes_to_intlist, intlist_to_bytes, unified_strdate, - clean_html, urlencode_postdata, ) from ..aes import ( diff --git a/youtube_dl/extractor/dropbox.py b/youtube_dl/extractor/dropbox.py index 5f24ac721..aefca848a 100644 --- a/youtube_dl/extractor/dropbox.py +++ b/youtube_dl/extractor/dropbox.py @@ -5,7 +5,8 @@ import os.path import re from .common import InfoExtractor -from ..utils import compat_urllib_parse_unquote, url_basename +from ..compat import compat_urllib_parse_unquote +from ..utils import url_basename class DropboxIE(InfoExtractor): diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py index 3ad993751..104803563 100644 --- a/youtube_dl/extractor/facebook.py +++ b/youtube_dl/extractor/facebook.py @@ -5,12 +5,14 @@ import re import socket from .common import InfoExtractor -from ..utils import ( +from ..compat import ( compat_http_client, compat_str, compat_urllib_error, compat_urllib_parse, compat_urllib_request, +) +from ..utils import ( urlencode_postdata, ExtractorError, limit_length, diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 3882e859c..babd581ea 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -7,11 +7,12 @@ import re from .common import InfoExtractor from .youtube import YoutubeIE -from ..utils import ( +from ..compat import ( compat_urllib_parse, compat_urlparse, compat_xml_parse_error, - +) +from ..utils import ( determine_ext, ExtractorError, float_or_none, diff --git a/youtube_dl/extractor/globo.py b/youtube_dl/extractor/globo.py index 77c3ad4fc..66ca37918 100644 --- a/youtube_dl/extractor/globo.py +++ b/youtube_dl/extractor/globo.py @@ -5,13 +5,15 @@ import random import math from .common import InfoExtractor -from ..utils import ( - ExtractorError, - float_or_none, +from ..compat import ( compat_str, compat_chr, compat_ord, ) +from ..utils import ( + ExtractorError, + float_or_none, +) class GloboIE(InfoExtractor): diff --git a/youtube_dl/extractor/laola1tv.py b/youtube_dl/extractor/laola1tv.py index 263f68773..102e29f7a 100644 --- a/youtube_dl/extractor/laola1tv.py +++ b/youtube_dl/extractor/laola1tv.py @@ -4,6 +4,7 @@ import random import re from .common import InfoExtractor +from ..utils import ExtractorError class Laola1TvIE(InfoExtractor): diff --git a/youtube_dl/extractor/myvideo.py b/youtube_dl/extractor/myvideo.py index ccb5959c4..a89153985 100644 --- a/youtube_dl/extractor/myvideo.py +++ b/youtube_dl/extractor/myvideo.py @@ -7,11 +7,12 @@ import re import json from .common import InfoExtractor -from ..utils import ( +from ..compat import ( compat_ord, compat_urllib_parse, compat_urllib_request, - +) +from ..utils import ( ExtractorError, ) diff --git a/youtube_dl/extractor/ro220.py b/youtube_dl/extractor/ro220.py index 0a3a71448..962b524e9 100644 --- a/youtube_dl/extractor/ro220.py +++ b/youtube_dl/extractor/ro220.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals from .common import InfoExtractor -from ..utils import compat_urllib_parse_unquote +from ..compat import compat_urllib_parse_unquote class Ro220IE(InfoExtractor): diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index d9cad0ea5..c744d4f04 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -7,11 +7,13 @@ import itertools from .common import InfoExtractor from .subtitles import SubtitlesInfoExtractor -from ..utils import ( +from ..compat import ( compat_HTTPError, compat_urllib_parse, compat_urllib_request, compat_urlparse, +) +from ..utils import ( ExtractorError, InAdvancePagedList, int_or_none, diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 98e20d549..997e92ad7 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -5,9 +5,11 @@ import optparse import shlex import sys -from .utils import ( +from .compat import ( compat_expanduser, compat_getenv, +) +from .utils import ( get_term_width, write_string, ) diff --git a/youtube_dl/postprocessor/atomicparsley.py b/youtube_dl/postprocessor/atomicparsley.py index 765b2d9ee..448ccc5f3 100644 --- a/youtube_dl/postprocessor/atomicparsley.py +++ b/youtube_dl/postprocessor/atomicparsley.py @@ -6,10 +6,11 @@ import os import subprocess from .common import PostProcessor - +from ..compat import ( + compat_urlretrieve, +) from ..utils import ( check_executable, - compat_urlretrieve, encodeFilename, PostProcessingError, prepend_extension, diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py index 08419a3d4..baf1b1945 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dl/postprocessor/execafterdownload.py @@ -3,10 +3,8 @@ from __future__ import unicode_literals import subprocess from .common import PostProcessor -from ..utils import ( - shlex_quote, - PostProcessingError, -) +from ..compat import shlex_quote +from ..utils import PostProcessingError class ExecAfterDownloadPP(PostProcessor): diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 338c145fa..f3f2743c0 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -1,5 +1,4 @@ import os -import re import subprocess import sys import time @@ -7,8 +6,10 @@ import time from .common import AudioConversionError, PostProcessor -from ..utils import ( +from ..compat import ( compat_subprocess_get_DEVNULL, +) +from ..utils import ( encodeArgument, encodeFilename, get_exe_version, diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py index f6940940b..b5cae41c8 100644 --- a/youtube_dl/postprocessor/xattrpp.py +++ b/youtube_dl/postprocessor/xattrpp.py @@ -3,10 +3,12 @@ import subprocess import sys from .common import PostProcessor +from ..compat import ( + subprocess_check_output +) from ..utils import ( check_executable, hyphenate_date, - subprocess_check_output ) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index fcfdadeb6..b0255c943 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -29,254 +29,19 @@ import traceback import xml.etree.ElementTree import zlib -try: - import urllib.request as compat_urllib_request -except ImportError: # Python 2 - import urllib2 as compat_urllib_request - -try: - import urllib.error as compat_urllib_error -except ImportError: # Python 2 - import urllib2 as compat_urllib_error - -try: - import urllib.parse as compat_urllib_parse -except ImportError: # Python 2 - import urllib as compat_urllib_parse - -try: - from urllib.parse import urlparse as compat_urllib_parse_urlparse -except ImportError: # Python 2 - from urlparse import urlparse as compat_urllib_parse_urlparse - -try: - import urllib.parse as compat_urlparse -except ImportError: # Python 2 - import urlparse as compat_urlparse - -try: - import http.cookiejar as compat_cookiejar -except ImportError: # Python 2 - import cookielib as compat_cookiejar - -try: - import html.entities as compat_html_entities -except ImportError: # Python 2 - import htmlentitydefs as compat_html_entities - -try: - import html.parser as compat_html_parser -except ImportError: # Python 2 - import HTMLParser as compat_html_parser - -try: - import http.client as compat_http_client -except ImportError: # Python 2 - import httplib as compat_http_client - -try: - from urllib.error import HTTPError as compat_HTTPError -except ImportError: # Python 2 - from urllib2 import HTTPError as compat_HTTPError - -try: - from urllib.request import urlretrieve as compat_urlretrieve -except ImportError: # Python 2 - from urllib import urlretrieve as compat_urlretrieve - - -try: - from subprocess import DEVNULL - compat_subprocess_get_DEVNULL = lambda: DEVNULL -except ImportError: - compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w') - -try: - from urllib.parse import unquote as compat_urllib_parse_unquote -except ImportError: - def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): - if string == '': - return string - res = string.split('%') - if len(res) == 1: - return string - if encoding is None: - encoding = 'utf-8' - if errors is None: - errors = 'replace' - # pct_sequence: contiguous sequence of percent-encoded bytes, decoded - pct_sequence = b'' - string = res[0] - for item in res[1:]: - try: - if not item: - raise ValueError - pct_sequence += item[:2].decode('hex') - rest = item[2:] - if not rest: - # This segment was just a single percent-encoded character. - # May be part of a sequence of code units, so delay decoding. - # (Stored in pct_sequence). - continue - except ValueError: - rest = '%' + item - # Encountered non-percent-encoded characters. Flush the current - # pct_sequence. - string += pct_sequence.decode(encoding, errors) + rest - pct_sequence = b'' - if pct_sequence: - # Flush the final pct_sequence - string += pct_sequence.decode(encoding, errors) - return string - - -try: - from urllib.parse import parse_qs as compat_parse_qs -except ImportError: # Python 2 - # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib. - # Python 2's version is apparently totally broken - - def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace'): - qs, _coerce_result = qs, unicode - pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')] - r = [] - for name_value in pairs: - if not name_value and not strict_parsing: - continue - nv = name_value.split('=', 1) - if len(nv) != 2: - if strict_parsing: - raise ValueError("bad query field: %r" % (name_value,)) - # Handle case of a control-name with no equal sign - if keep_blank_values: - nv.append('') - else: - continue - if len(nv[1]) or keep_blank_values: - name = nv[0].replace('+', ' ') - name = compat_urllib_parse_unquote( - name, encoding=encoding, errors=errors) - name = _coerce_result(name) - value = nv[1].replace('+', ' ') - value = compat_urllib_parse_unquote( - value, encoding=encoding, errors=errors) - value = _coerce_result(value) - r.append((name, value)) - return r - - def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False, - encoding='utf-8', errors='replace'): - parsed_result = {} - pairs = _parse_qsl(qs, keep_blank_values, strict_parsing, - encoding=encoding, errors=errors) - for name, value in pairs: - if name in parsed_result: - parsed_result[name].append(value) - else: - parsed_result[name] = [value] - return parsed_result - -try: - compat_str = unicode # Python 2 -except NameError: - compat_str = str - -try: - compat_chr = unichr # Python 2 -except NameError: - compat_chr = chr - -try: - from xml.etree.ElementTree import ParseError as compat_xml_parse_error -except ImportError: # Python 2.6 - from xml.parsers.expat import ExpatError as compat_xml_parse_error - -try: - from shlex import quote as shlex_quote -except ImportError: # Python < 3.3 - def shlex_quote(s): - return "'" + s.replace("'", "'\"'\"'") + "'" - - -def compat_ord(c): - if type(c) is int: return c - else: return ord(c) - - -if sys.version_info >= (3, 0): - compat_getenv = os.getenv - compat_expanduser = os.path.expanduser -else: - # Environment variables should be decoded with filesystem encoding. - # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918) - - def compat_getenv(key, default=None): - env = os.getenv(key, default) - if env: - env = env.decode(get_filesystem_encoding()) - return env - - # HACK: The default implementations of os.path.expanduser from cpython do not decode - # environment variables with filesystem encoding. We will work around this by - # providing adjusted implementations. - # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib - # for different platforms with correct environment variables decoding. - - if os.name == 'posix': - def compat_expanduser(path): - """Expand ~ and ~user constructions. If user or $HOME is unknown, - do nothing.""" - if not path.startswith('~'): - return path - i = path.find('/', 1) - if i < 0: - i = len(path) - if i == 1: - if 'HOME' not in os.environ: - import pwd - userhome = pwd.getpwuid(os.getuid()).pw_dir - else: - userhome = compat_getenv('HOME') - else: - import pwd - try: - pwent = pwd.getpwnam(path[1:i]) - except KeyError: - return path - userhome = pwent.pw_dir - userhome = userhome.rstrip('/') - return (userhome + path[i:]) or '/' - elif os.name == 'nt' or os.name == 'ce': - def compat_expanduser(path): - """Expand ~ and ~user constructs. - - If user or $HOME is unknown, do nothing.""" - if path[:1] != '~': - return path - i, n = 1, len(path) - while i < n and path[i] not in '/\\': - i = i + 1 - - if 'HOME' in os.environ: - userhome = compat_getenv('HOME') - elif 'USERPROFILE' in os.environ: - userhome = compat_getenv('USERPROFILE') - elif not 'HOMEPATH' in os.environ: - return path - else: - try: - drive = compat_getenv('HOMEDRIVE') - except KeyError: - drive = '' - userhome = os.path.join(drive, compat_getenv('HOMEPATH')) - - if i != 1: #~user - userhome = os.path.join(os.path.dirname(userhome), path[1:i]) - - return userhome + path[i:] - else: - compat_expanduser = os.path.expanduser +from .compat import ( + compat_chr, + compat_getenv, + compat_html_entities, + compat_html_parser, + compat_parse_qs, + compat_str, + compat_urllib_error, + compat_urllib_parse, + compat_urllib_parse_urlparse, + compat_urllib_request, + compat_urlparse, +) # This is not clearly defined otherwise @@ -304,14 +69,6 @@ def preferredencoding(): return pref -if sys.version_info < (3,0): - def compat_print(s): - print(s.encode(preferredencoding(), 'xmlcharrefreplace')) -else: - def compat_print(s): - assert type(s) == type(u'') - print(s) - def write_json_file(obj, fn): """ Encode obj as JSON and write it to fn, atomically """ @@ -1655,15 +1412,6 @@ def parse_xml(s): return tree -if sys.version_info < (3, 0) and sys.platform == 'win32': - def compat_getpass(prompt, *args, **kwargs): - if isinstance(prompt, compat_str): - prompt = prompt.encode(preferredencoding()) - return getpass.getpass(prompt, *args, **kwargs) -else: - compat_getpass = getpass.getpass - - US_RATINGS = { 'G': 0, 'PG': 10, @@ -1721,18 +1469,6 @@ def qualities(quality_ids): DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s' -try: - subprocess_check_output = subprocess.check_output -except AttributeError: - def subprocess_check_output(*args, **kwargs): - assert 'input' not in kwargs - p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs) - output, _ = p.communicate() - ret = p.poll() - if ret: - raise subprocess.CalledProcessError(ret, p.args, output=output) - return output - def limit_length(s, length): """ Add ellipses to overly long strings """ |