diff options
Diffstat (limited to 'youtube_dl')
| -rw-r--r-- | youtube_dl/__init__.py | 4 | ||||
| -rw-r--r-- | youtube_dl/casefold.py | 12 | ||||
| -rw-r--r-- | youtube_dl/compat.py | 192 | ||||
| -rw-r--r-- | youtube_dl/extractor/bokecc.py | 2 | ||||
| -rw-r--r-- | youtube_dl/extractor/cloudy.py | 2 | ||||
| -rw-r--r-- | youtube_dl/extractor/common.py | 2 | ||||
| -rw-r--r-- | youtube_dl/extractor/itv.py | 17 | ||||
| -rw-r--r-- | youtube_dl/extractor/senateisvp.py | 2 | ||||
| -rw-r--r-- | youtube_dl/extractor/youtube.py | 322 | ||||
| -rw-r--r-- | youtube_dl/jsinterp.py | 140 |
10 files changed, 543 insertions, 152 deletions
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 06bdfb689..3c1272e7b 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -18,7 +18,7 @@ from .compat import ( compat_getpass, compat_register_utf8, compat_shlex_split, - workaround_optparse_bug9161, + _workaround_optparse_bug9161, ) from .utils import ( _UnsafeExtensionError, @@ -50,7 +50,7 @@ def _real_main(argv=None): # Compatibility fix for Windows compat_register_utf8() - workaround_optparse_bug9161() + _workaround_optparse_bug9161() setproctitle('youtube-dl') diff --git a/youtube_dl/casefold.py b/youtube_dl/casefold.py index ad9c66f8e..712b2e7fa 100644 --- a/youtube_dl/casefold.py +++ b/youtube_dl/casefold.py @@ -10,9 +10,10 @@ from .compat import ( # https://github.com/unicode-org/icu/blob/main/icu4c/source/data/unidata/CaseFolding.txt # In case newly foldable Unicode characters are defined, paste the new version # of the text inside the ''' marks. -# The text is expected to have only blank lines andlines with 1st character #, +# The text is expected to have only blank lines and lines with 1st character #, # all ignored, and fold definitions like this: -# `from_hex_code; space_separated_to_hex_code_list; comment` +# `from_hex_code; status; space_separated_to_hex_code_list; comment` +# Only `status` C/F are used. _map_str = ''' # CaseFolding-15.0.0.txt @@ -1657,11 +1658,6 @@ _map = dict( del _map_str -def casefold(s): +def _casefold(s): assert isinstance(s, compat_str) return ''.join((_map.get(c, c) for c in s)) - - -__all__ = [ - 'casefold', -] diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index ed1a33cf2..8910a4dac 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -16,7 +16,6 @@ import os import platform import re import shlex -import shutil import socket import struct import subprocess @@ -24,11 +23,15 @@ import sys import types import xml.etree.ElementTree +_IDENTITY = lambda x: x + # naming convention # 'compat_' + Python3_name.replace('.', '_') # other aliases exist for convenience and/or legacy +# wrap disposable test values in type() to reclaim storage -# deal with critical unicode/str things first +# deal with critical unicode/str things first: +# compat_str, compat_basestring, compat_chr try: # Python 2 compat_str, compat_basestring, compat_chr = ( @@ -39,18 +42,23 @@ except NameError: str, (str, bytes), chr ) -# casefold + +# compat_casefold try: compat_str.casefold compat_casefold = lambda s: s.casefold() except AttributeError: - from .casefold import casefold as compat_casefold + from .casefold import _casefold as compat_casefold + +# compat_collections_abc try: import collections.abc as compat_collections_abc except ImportError: import collections as compat_collections_abc + +# compat_urllib_request try: import urllib.request as compat_urllib_request except ImportError: # Python 2 @@ -79,11 +87,15 @@ except TypeError: _add_init_method_arg(compat_urllib_request.Request) del _add_init_method_arg + +# compat_urllib_error try: import urllib.error as compat_urllib_error except ImportError: # Python 2 import urllib2 as compat_urllib_error + +# compat_urllib_parse try: import urllib.parse as compat_urllib_parse except ImportError: # Python 2 @@ -98,17 +110,23 @@ except ImportError: # Python 2 compat_urlparse = compat_urllib_parse compat_urllib_parse_urlparse = compat_urllib_parse.urlparse + +# compat_urllib_response try: import urllib.response as compat_urllib_response except ImportError: # Python 2 import urllib as compat_urllib_response + +# compat_urllib_response.addinfourl try: compat_urllib_response.addinfourl.status except AttributeError: # .getcode() is deprecated in Py 3. compat_urllib_response.addinfourl.status = property(lambda self: self.getcode()) + +# compat_http_cookiejar try: import http.cookiejar as compat_cookiejar except ImportError: # Python 2 @@ -127,12 +145,16 @@ else: compat_cookiejar_Cookie = compat_cookiejar.Cookie compat_http_cookiejar_Cookie = compat_cookiejar_Cookie + +# compat_http_cookies try: import http.cookies as compat_cookies except ImportError: # Python 2 import Cookie as compat_cookies compat_http_cookies = compat_cookies + +# compat_http_cookies_SimpleCookie if sys.version_info[0] == 2 or sys.version_info < (3, 3): class compat_cookies_SimpleCookie(compat_cookies.SimpleCookie): def load(self, rawdata): @@ -155,11 +177,15 @@ else: compat_cookies_SimpleCookie = compat_cookies.SimpleCookie compat_http_cookies_SimpleCookie = compat_cookies_SimpleCookie + +# compat_html_entities, probably useless now try: import html.entities as compat_html_entities except ImportError: # Python 2 import htmlentitydefs as compat_html_entities + +# compat_html_entities_html5 try: # Python >= 3.3 compat_html_entities_html5 = compat_html_entities.html5 except AttributeError: @@ -2408,18 +2434,24 @@ except AttributeError: # Py < 3.1 compat_http_client.HTTPResponse.getcode = lambda self: self.status + +# compat_urllib_HTTPError try: from urllib.error import HTTPError as compat_HTTPError except ImportError: # Python 2 from urllib2 import HTTPError as compat_HTTPError compat_urllib_HTTPError = compat_HTTPError + +# compat_urllib_request_urlretrieve try: from urllib.request import urlretrieve as compat_urlretrieve except ImportError: # Python 2 from urllib import urlretrieve as compat_urlretrieve compat_urllib_request_urlretrieve = compat_urlretrieve + +# compat_html_parser_HTMLParser, compat_html_parser_HTMLParseError try: from HTMLParser import ( HTMLParser as compat_HTMLParser, @@ -2432,22 +2464,33 @@ except ImportError: # Python 3 # HTMLParseError was deprecated in Python 3.3 and removed in # Python 3.5. Introducing dummy exception for Python >3.5 for compatible # and uniform cross-version exception handling + class compat_HTMLParseError(Exception): pass + compat_html_parser_HTMLParser = compat_HTMLParser compat_html_parser_HTMLParseError = compat_HTMLParseError + +# compat_subprocess_get_DEVNULL try: _DEVNULL = subprocess.DEVNULL compat_subprocess_get_DEVNULL = lambda: _DEVNULL except AttributeError: compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w') + +# compat_http_server try: import http.server as compat_http_server except ImportError: import BaseHTTPServer as compat_http_server + +# compat_urllib_parse_unquote_to_bytes, +# compat_urllib_parse_unquote, compat_urllib_parse_unquote_plus, +# compat_urllib_parse_urlencode, +# compat_urllib_parse_parse_qs try: from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes from urllib.parse import unquote as compat_urllib_parse_unquote @@ -2598,6 +2641,8 @@ except ImportError: # Python 2 compat_urllib_parse_parse_qs = compat_parse_qs + +# compat_urllib_request_DataHandler try: from urllib.request import DataHandler as compat_urllib_request_DataHandler except ImportError: # Python < 3.4 @@ -2632,16 +2677,20 @@ except ImportError: # Python < 3.4 return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url) + +# compat_xml_etree_ElementTree_ParseError try: from xml.etree.ElementTree import ParseError as compat_xml_parse_error except ImportError: # Python 2.6 from xml.parsers.expat import ExpatError as compat_xml_parse_error compat_xml_etree_ElementTree_ParseError = compat_xml_parse_error -etree = xml.etree.ElementTree + +# compat_xml_etree_ElementTree_Element +_etree = xml.etree.ElementTree -class _TreeBuilder(etree.TreeBuilder): +class _TreeBuilder(_etree.TreeBuilder): def doctype(self, name, pubid, system): pass @@ -2650,7 +2699,7 @@ try: # xml.etree.ElementTree.Element is a method in Python <=2.6 and # the following will crash with: # TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types - isinstance(None, etree.Element) + isinstance(None, _etree.Element) from xml.etree.ElementTree import Element as compat_etree_Element except TypeError: # Python <=2.6 from xml.etree.ElementTree import _ElementInterface as compat_etree_Element @@ -2658,12 +2707,12 @@ compat_xml_etree_ElementTree_Element = compat_etree_Element if sys.version_info[0] >= 3: def compat_etree_fromstring(text): - return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder())) + return _etree.XML(text, parser=_etree.XMLParser(target=_TreeBuilder())) else: # python 2.x tries to encode unicode strings with ascii (see the # XMLParser._fixtext method) try: - _etree_iter = etree.Element.iter + _etree_iter = _etree.Element.iter except AttributeError: # Python <=2.6 def _etree_iter(root): for el in root.findall('*'): @@ -2675,27 +2724,29 @@ else: # 2.7 source def _XML(text, parser=None): if not parser: - parser = etree.XMLParser(target=_TreeBuilder()) + parser = _etree.XMLParser(target=_TreeBuilder()) parser.feed(text) return parser.close() def _element_factory(*args, **kwargs): - el = etree.Element(*args, **kwargs) + el = _etree.Element(*args, **kwargs) for k, v in el.items(): if isinstance(v, bytes): el.set(k, v.decode('utf-8')) return el def compat_etree_fromstring(text): - doc = _XML(text, parser=etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory))) + doc = _XML(text, parser=_etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory))) for el in _etree_iter(doc): if el.text is not None and isinstance(el.text, bytes): el.text = el.text.decode('utf-8') return doc -if hasattr(etree, 'register_namespace'): - compat_etree_register_namespace = etree.register_namespace -else: + +# compat_xml_etree_register_namespace +try: + compat_etree_register_namespace = _etree.register_namespace +except AttributeError: def compat_etree_register_namespace(prefix, uri): """Register a namespace prefix. The registry is global, and any existing mapping for either the @@ -2704,14 +2755,16 @@ else: attributes in this namespace will be serialized with prefix if possible. ValueError is raised if prefix is reserved or is invalid. """ - if re.match(r"ns\d+$", prefix): - raise ValueError("Prefix format reserved for internal use") - for k, v in list(etree._namespace_map.items()): + if re.match(r'ns\d+$', prefix): + raise ValueError('Prefix format reserved for internal use') + for k, v in list(_etree._namespace_map.items()): if k == uri or v == prefix: - del etree._namespace_map[k] - etree._namespace_map[uri] = prefix + del _etree._namespace_map[k] + _etree._namespace_map[uri] = prefix compat_xml_etree_register_namespace = compat_etree_register_namespace + +# compat_xpath, compat_etree_iterfind if sys.version_info < (2, 7): # Here comes the crazy part: In 2.6, if the xpath is a unicode, # .//node does not match if a node is a direct child of . ! @@ -2898,7 +2951,6 @@ if sys.version_info < (2, 7): def __init__(self, root): self.root = root - ## # Generate all matching objects. def compat_etree_iterfind(elem, path, namespaces=None): @@ -2933,13 +2985,15 @@ if sys.version_info < (2, 7): else: - compat_xpath = lambda xpath: xpath compat_etree_iterfind = lambda element, match: element.iterfind(match) + compat_xpath = _IDENTITY +# compat_os_name compat_os_name = os._name if os.name == 'java' else os.name +# compat_shlex_quote if compat_os_name == 'nt': def compat_shlex_quote(s): return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"') @@ -2954,6 +3008,7 @@ else: return "'" + s.replace("'", "'\"'\"'") + "'" +# compat_shlex.split try: args = shlex.split('中文') assert (isinstance(args, list) @@ -2969,6 +3024,7 @@ except (AssertionError, UnicodeEncodeError): return list(map(lambda s: s.decode('utf-8'), shlex.split(s, comments, posix))) +# compat_ord def compat_ord(c): if isinstance(c, int): return c @@ -2976,6 +3032,7 @@ def compat_ord(c): return ord(c) +# compat_getenv, compat_os_path_expanduser, compat_setenv if sys.version_info >= (3, 0): compat_getenv = os.getenv compat_expanduser = os.path.expanduser @@ -3063,6 +3120,7 @@ else: compat_os_path_expanduser = compat_expanduser +# compat_os_path_realpath if compat_os_name == 'nt' and sys.version_info < (3, 8): # os.path.realpath on Windows does not follow symbolic links # prior to Python 3.8 (see https://bugs.python.org/issue9949) @@ -3076,6 +3134,7 @@ else: compat_os_path_realpath = compat_realpath +# compat_print if sys.version_info < (3, 0): def compat_print(s): from .utils import preferredencoding @@ -3086,6 +3145,7 @@ else: print(s) +# compat_getpass_getpass if sys.version_info < (3, 0) and sys.platform == 'win32': def compat_getpass(prompt, *args, **kwargs): if isinstance(prompt, compat_str): @@ -3098,36 +3158,42 @@ else: compat_getpass_getpass = compat_getpass +# compat_input try: compat_input = raw_input except NameError: # Python 3 compat_input = input +# compat_kwargs # Python < 2.6.5 require kwargs to be bytes try: - def _testfunc(x): - pass - _testfunc(**{'x': 0}) + (lambda x: x)(**{'x': 0}) except TypeError: def compat_kwargs(kwargs): return dict((bytes(k), v) for k, v in kwargs.items()) else: - compat_kwargs = lambda kwargs: kwargs + compat_kwargs = _IDENTITY +# compat_numeric_types try: compat_numeric_types = (int, float, long, complex) except NameError: # Python 3 compat_numeric_types = (int, float, complex) +# compat_integer_types try: compat_integer_types = (int, long) except NameError: # Python 3 compat_integer_types = (int, ) +# compat_int +compat_int = compat_integer_types[-1] + +# compat_socket_create_connection if sys.version_info < (2, 7): def compat_socket_create_connection(address, timeout, source_address=None): host, port = address @@ -3154,6 +3220,7 @@ else: compat_socket_create_connection = socket.create_connection +# compat_contextlib_suppress try: from contextlib import suppress as compat_contextlib_suppress except ImportError: @@ -3196,12 +3263,12 @@ except AttributeError: # repeated .close() is OK, but just in case with compat_contextlib_suppress(EnvironmentError): f.close() - popen.wait() + popen.wait() # Fix https://github.com/ytdl-org/youtube-dl/issues/4223 # See http://bugs.python.org/issue9161 for what is broken -def workaround_optparse_bug9161(): +def _workaround_optparse_bug9161(): op = optparse.OptionParser() og = optparse.OptionGroup(op, 'foo') try: @@ -3220,9 +3287,10 @@ def workaround_optparse_bug9161(): optparse.OptionGroup.add_option = _compat_add_option -if hasattr(shutil, 'get_terminal_size'): # Python >= 3.3 - compat_get_terminal_size = shutil.get_terminal_size -else: +# compat_shutil_get_terminal_size +try: + from shutil import get_terminal_size as compat_get_terminal_size # Python >= 3.3 +except ImportError: _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines']) def compat_get_terminal_size(fallback=(80, 24)): @@ -3252,27 +3320,33 @@ else: columns = _columns if lines is None or lines <= 0: lines = _lines + return _terminal_size(columns, lines) +compat_shutil_get_terminal_size = compat_get_terminal_size + +# compat_itertools_count try: - itertools.count(start=0, step=1) + type(itertools.count(start=0, step=1)) compat_itertools_count = itertools.count -except TypeError: # Python 2.6 +except TypeError: # Python 2.6 lacks step def compat_itertools_count(start=0, step=1): while True: yield start start += step +# compat_tokenize_tokenize if sys.version_info >= (3, 0): from tokenize import tokenize as compat_tokenize_tokenize else: from tokenize import generate_tokens as compat_tokenize_tokenize +# compat_struct_pack, compat_struct_unpack, compat_Struct try: - struct.pack('!I', 0) + type(struct.pack('!I', 0)) except TypeError: # In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument # See https://bugs.python.org/issue19099 @@ -3304,8 +3378,10 @@ else: compat_Struct = struct.Struct -# compat_map/filter() returning an iterator, supposedly the -# same versioning as for zip below +# builtins returning an iterator + +# compat_map, compat_filter +# supposedly the same versioning as for zip below try: from future_builtins import map as compat_map except ImportError: @@ -3322,6 +3398,7 @@ except ImportError: except ImportError: compat_filter = filter +# compat_zip try: from future_builtins import zip as compat_zip except ImportError: # not 2.6+ or is 3.x @@ -3331,6 +3408,7 @@ except ImportError: # not 2.6+ or is 3.x compat_zip = zip +# compat_itertools_zip_longest # method renamed between Py2/3 try: from itertools import zip_longest as compat_itertools_zip_longest @@ -3338,7 +3416,8 @@ except ImportError: from itertools import izip_longest as compat_itertools_zip_longest -# new class in collections +# compat_collections_chain_map +# collections.ChainMap: new class try: from collections import ChainMap as compat_collections_chain_map # Py3.3's ChainMap is deficient @@ -3394,19 +3473,22 @@ except ImportError: def new_child(self, m=None, **kwargs): m = m or {} m.update(kwargs) - return compat_collections_chain_map(m, *self.maps) + # support inheritance ! + return type(self)(m, *self.maps) @property def parents(self): - return compat_collections_chain_map(*(self.maps[1:])) + return type(self)(*(self.maps[1:])) +# compat_re_Pattern, compat_re_Match # Pythons disagree on the type of a pattern (RegexObject, _sre.SRE_Pattern, Pattern, ...?) compat_re_Pattern = type(re.compile('')) # and on the type of a match compat_re_Match = type(re.match('a', 'a')) +# compat_base64_b64decode if sys.version_info < (3, 3): def compat_b64decode(s, *args, **kwargs): if isinstance(s, compat_str): @@ -3418,6 +3500,7 @@ else: compat_base64_b64decode = compat_b64decode +# compat_ctypes_WINFUNCTYPE if platform.python_implementation() == 'PyPy' and sys.pypy_version_info < (5, 4, 0): # PyPy2 prior to version 5.4.0 expects byte strings as Windows function # names, see the original PyPy issue [1] and the youtube-dl one [2]. @@ -3436,6 +3519,7 @@ else: return ctypes.WINFUNCTYPE(*args, **kwargs) +# compat_open if sys.version_info < (3, 0): # open(file, mode='r', buffering=- 1, encoding=None, errors=None, newline=None, closefd=True) not: opener=None def compat_open(file_, *args, **kwargs): @@ -3463,18 +3547,28 @@ except AttributeError: def compat_datetime_timedelta_total_seconds(td): return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 + # optional decompression packages +# compat_brotli # PyPi brotli package implements 'br' Content-Encoding try: import brotli as compat_brotli except ImportError: compat_brotli = None +# compat_ncompress # PyPi ncompress package implements 'compress' Content-Encoding try: import ncompress as compat_ncompress except ImportError: compat_ncompress = None +# compat_zstandard +# PyPi zstandard package implements 'zstd' Content-Encoding (RFC 8878 7.2) +try: + import zstandard as compat_zstandard +except ImportError: + compat_zstandard = None + legacy = [ 'compat_HTMLParseError', @@ -3491,6 +3585,7 @@ legacy = [ 'compat_getpass', 'compat_parse_qs', 'compat_realpath', + 'compat_shlex_split', 'compat_urllib_parse_parse_qs', 'compat_urllib_parse_unquote', 'compat_urllib_parse_unquote_plus', @@ -3504,8 +3599,6 @@ legacy = [ __all__ = [ - 'compat_html_parser_HTMLParseError', - 'compat_html_parser_HTMLParser', 'compat_Struct', 'compat_base64_b64decode', 'compat_basestring', @@ -3514,13 +3607,9 @@ __all__ = [ 'compat_chr', 'compat_collections_abc', 'compat_collections_chain_map', - 'compat_datetime_timedelta_total_seconds', - 'compat_http_cookiejar', - 'compat_http_cookiejar_Cookie', - 'compat_http_cookies', - 'compat_http_cookies_SimpleCookie', 'compat_contextlib_suppress', 'compat_ctypes_WINFUNCTYPE', + 'compat_datetime_timedelta_total_seconds', 'compat_etree_fromstring', 'compat_etree_iterfind', 'compat_filter', @@ -3529,9 +3618,16 @@ __all__ = [ 'compat_getpass_getpass', 'compat_html_entities', 'compat_html_entities_html5', + 'compat_html_parser_HTMLParseError', + 'compat_html_parser_HTMLParser', + 'compat_http_cookiejar', + 'compat_http_cookiejar_Cookie', + 'compat_http_cookies', + 'compat_http_cookies_SimpleCookie', 'compat_http_client', 'compat_http_server', 'compat_input', + 'compat_int', 'compat_integer_types', 'compat_itertools_count', 'compat_itertools_zip_longest', @@ -3550,7 +3646,7 @@ __all__ = [ 'compat_register_utf8', 'compat_setenv', 'compat_shlex_quote', - 'compat_shlex_split', + 'compat_shutil_get_terminal_size', 'compat_socket_create_connection', 'compat_str', 'compat_struct_pack', @@ -3570,5 +3666,5 @@ __all__ = [ 'compat_xml_etree_register_namespace', 'compat_xpath', 'compat_zip', - 'workaround_optparse_bug9161', + 'compat_zstandard', ] diff --git a/youtube_dl/extractor/bokecc.py b/youtube_dl/extractor/bokecc.py index 6017e8344..4b8bef391 100644 --- a/youtube_dl/extractor/bokecc.py +++ b/youtube_dl/extractor/bokecc.py @@ -32,7 +32,7 @@ class BokeCCBaseIE(InfoExtractor): class BokeCCIE(BokeCCBaseIE): - _IE_DESC = 'CC视频' + IE_DESC = 'CC视频' _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)' _TESTS = [{ diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py index 85ca20ecc..d39a9a5c2 100644 --- a/youtube_dl/extractor/cloudy.py +++ b/youtube_dl/extractor/cloudy.py @@ -9,7 +9,7 @@ from ..utils import ( class CloudyIE(InfoExtractor): - _IE_DESC = 'cloudy.ec' + IE_DESC = 'cloudy.ec' _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)' _TESTS = [{ 'url': 'https://www.cloudy.ec/v/af511e2527aac', diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 78704b557..cb67b976d 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -422,6 +422,8 @@ class InfoExtractor(object): _GEO_COUNTRIES = None _GEO_IP_BLOCKS = None _WORKING = True + # supply this in public subclasses: used in supported sites list, etc + # IE_DESC = 'short description of IE' def __init__(self, downloader=None): """Constructor. Receives an optional downloader.""" diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py index c64af3be6..2510ad887 100644 --- a/youtube_dl/extractor/itv.py +++ b/youtube_dl/extractor/itv.py @@ -35,15 +35,6 @@ from ..utils import ( class ITVBaseIE(InfoExtractor): - def _search_nextjs_data(self, webpage, video_id, **kw): - transform_source = kw.pop('transform_source', None) - fatal = kw.pop('fatal', True) - return self._parse_json( - self._search_regex( - r'''<script\b[^>]+\bid=('|")__NEXT_DATA__\1[^>]*>(?P<js>[^<]+)</script>''', - webpage, 'next.js data', group='js', fatal=fatal, **kw), - video_id, transform_source=transform_source, fatal=fatal) - def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True): if errnote is False: return False @@ -109,7 +100,9 @@ class ITVBaseIE(InfoExtractor): class ITVIE(ITVBaseIE): _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:(?P<w>watch)|hub)/[^/]+/(?(w)[\w-]+/)(?P<id>\w+)' - _IE_DESC = 'ITVX' + IE_DESC = 'ITVX' + _WORKING = False + _TESTS = [{ 'note': 'Hub URLs redirect to ITVX', 'url': 'https://www.itv.com/hub/liar/2a4547a0012', @@ -270,7 +263,7 @@ class ITVIE(ITVBaseIE): 'ext': determine_ext(href, 'vtt'), }) - next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default='{}') + next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default={}) video_data.update(traverse_obj(next_data, ('props', 'pageProps', ('title', 'episode')), expected_type=dict)[0] or {}) title = traverse_obj(video_data, 'headerTitle', 'episodeTitle') info = self._og_extract(webpage, require_title=not title) @@ -323,7 +316,7 @@ class ITVIE(ITVBaseIE): class ITVBTCCIE(ITVBaseIE): _VALID_URL = r'https?://(?:www\.)?itv\.com/(?!(?:watch|hub)/)(?:[^/]+/)+(?P<id>[^/?#&]+)' - _IE_DESC = 'ITV articles: News, British Touring Car Championship' + IE_DESC = 'ITV articles: News, British Touring Car Championship' _TESTS = [{ 'note': 'British Touring Car Championship', 'url': 'https://www.itv.com/btcc/articles/btcc-2018-all-the-action-from-brands-hatch', diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py index db5ef8b57..b8ac58713 100644 --- a/youtube_dl/extractor/senateisvp.py +++ b/youtube_dl/extractor/senateisvp.py @@ -47,7 +47,7 @@ class SenateISVPIE(InfoExtractor): ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'], ['arch', '', 'http://ussenate-f.akamaihd.net/'] ] - _IE_NAME = 'senate.gov' + IE_NAME = 'senate.gov' _VALID_URL = r'https?://(?:www\.)?senate\.gov/isvp/?\?(?P<qs>.+)' _TESTS = [{ 'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png', diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 56957a661..ce97fd75b 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -27,11 +27,14 @@ from ..compat import ( ) from ..jsinterp import JSInterpreter from ..utils import ( + bug_reports_message, clean_html, dict_get, error_to_compat_str, ExtractorError, + filter_dict, float_or_none, + get_first, extract_attributes, get_element_by_attribute, int_or_none, @@ -63,6 +66,7 @@ from ..utils import ( url_or_none, urlencode_postdata, urljoin, + variadic, ) @@ -82,9 +86,66 @@ class YoutubeBaseInfoExtractor(InfoExtractor): _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM)' + _INNERTUBE_CLIENTS = { + 'ios': { + 'INNERTUBE_CONTEXT': { + 'client': { + 'clientName': 'IOS', + 'clientVersion': '20.10.4', + 'deviceMake': 'Apple', + 'deviceModel': 'iPhone16,2', + 'userAgent': 'com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)', + 'osName': 'iPhone', + 'osVersion': '18.3.2.22D82', + }, + }, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 5, + 'REQUIRE_JS_PLAYER': False, + 'REQUIRE_PO_TOKEN': True, + }, + # mweb has 'ultralow' formats + # See: https://github.com/yt-dlp/yt-dlp/pull/557 + 'mweb': { + 'INNERTUBE_CONTEXT': { + 'client': { + 'clientName': 'MWEB', + 'clientVersion': '2.20250311.03.00', + # mweb previously did not require PO Token with this UA + 'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)', + }, + }, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 2, + 'REQUIRE_PO_TOKEN': True, + 'SUPPORTS_COOKIES': True, + }, + 'tv': { + 'INNERTUBE_CONTEXT': { + 'client': { + 'clientName': 'TVHTML5', + 'clientVersion': '7.20250312.16.00', + 'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version', + }, + }, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 7, + 'SUPPORTS_COOKIES': True, + }, + 'web': { + 'INNERTUBE_CONTEXT': { + 'client': { + 'clientName': 'WEB', + 'clientVersion': '2.20250312.04.00', + }, + }, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 1, + 'REQUIRE_PO_TOKEN': True, + 'SUPPORTS_COOKIES': True, + }, + } + def _login(self): """ Attempt to log in to YouTube. + True is returned if successful or skipped. False is returned if login failed. @@ -321,19 +382,24 @@ class YoutubeBaseInfoExtractor(InfoExtractor): '{0} {1} {2}'.format(time_now, self._SAPISID, origin).encode('utf-8')).hexdigest() return 'SAPISIDHASH {0}_{1}'.format(time_now, sapisidhash) - def _call_api(self, ep, query, video_id, fatal=True, headers=None): + def _call_api(self, ep, query, video_id, fatal=True, headers=None, + note='Downloading API JSON'): data = self._DEFAULT_API_DATA.copy() data.update(query) real_headers = {'content-type': 'application/json'} if headers: real_headers.update(headers) + # was: 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8' + api_key = self.get_param('youtube_innertube_key') return self._download_json( 'https://www.youtube.com/youtubei/v1/%s' % ep, video_id=video_id, - note='Downloading API JSON', errnote='Unable to download API page', + note=note, errnote='Unable to download API page', data=json.dumps(data).encode('utf8'), fatal=fatal, - headers=real_headers, - query={'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'}) + headers=real_headers, query=filter_dict({ + 'key': api_key, + 'prettyPrint': 'false', + })) def _extract_yt_initial_data(self, video_id, webpage): return self._parse_json( @@ -342,6 +408,22 @@ class YoutubeBaseInfoExtractor(InfoExtractor): self._YT_INITIAL_DATA_RE), webpage, 'yt initial data'), video_id) + def _extract_visitor_data(self, *args): + """ + Extract visitorData from an API response or ytcfg + + Appears to be used to track session state + """ + visitor_data = self.get_param('youtube_visitor_data') + if visitor_data: + return visitor_data + + return get_first( + args, (('VISITOR_DATA', + ('INNERTUBE_CONTEXT', 'client', 'visitorData'), + ('responseContext', 'visitorData')), + T(compat_str))) + def _extract_ytcfg(self, video_id, webpage): return self._parse_json( self._search_regex( @@ -381,6 +463,26 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'uploader': uploader, } + @staticmethod + def _extract_thumbnails(data, *path_list, **kw_final_key): + """ + Extract thumbnails from thumbnails dict + @param path_list: path list to level that contains 'thumbnails' key + """ + final_key = kw_final_key.get('final_key', 'thumbnails') + + return traverse_obj(data, (( + tuple(variadic(path) + (final_key, Ellipsis) + for path in path_list or [()])), { + 'url': ('url', T(url_or_none), + # Sometimes youtube gives a wrong thumbnail URL. See: + # https://github.com/yt-dlp/yt-dlp/issues/233 + # https://github.com/ytdl-org/youtube-dl/issues/28023 + T(lambda u: update_url(u, query=None) if u and 'maxresdefault' in u else u)), + 'height': ('height', T(int_or_none)), + 'width': ('width', T(int_or_none)), + }, T(lambda t: t if t.get('url') else None))) + def _search_results(self, query, params): data = { 'context': { @@ -590,9 +692,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'invidious': '|'.join(_INVIDIOUS_SITES), } _PLAYER_INFO_RE = ( - r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player', - r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$', - r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$', + r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/(?:tv-)?player', + r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias(?:_tce)?\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$', + r'\b(?P<id>vfl[a-zA-Z0-9_-]{6,})\b.*?\.js$', ) _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt') @@ -1524,15 +1626,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor): """ Return a string representation of a signature """ return '.'.join(compat_str(len(part)) for part in example_sig.split('.')) - @classmethod - def _extract_player_info(cls, player_url): - for player_re in cls._PLAYER_INFO_RE: - id_m = re.search(player_re, player_url) - if id_m: - break - else: - raise ExtractorError('Cannot identify player %r' % player_url) - return id_m.group('id') + def _extract_player_info(self, player_url): + try: + return self._search_regex( + self._PLAYER_INFO_RE, player_url, 'player info', group='id') + except ExtractorError as e: + raise ExtractorError( + 'Cannot identify player %r' % (player_url,), cause=e) def _load_player(self, video_id, player_url, fatal=True, player_id=None): if not player_id: @@ -1609,6 +1709,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor): ' return %s\n') % (signature_id_tuple, expr_code) self.to_screen('Extracted signature function:\n' + code) + def _extract_sig_fn(self, jsi, funcname): + var_ay = self._search_regex( + r'''(?x) + (?:\*/|\{|\n|^)\s*(?:'[^']+'\s*;\s*) + (var\s*[\w$]+\s*=\s*(?: + ('|")(?:\\\2|(?!\2).)+\2\s*\.\s*split\(\s*('|")\W+\3\s*\)| + \[\s*(?:('|")(?:\\\4|(?!\4).)*\4\s*(?:(?=\])|,\s*))+\] + ))(?=\s*[,;]) + ''', jsi.code, 'useful values', default='') + + sig_fn = jsi.extract_function_code(funcname) + + if var_ay: + sig_fn = (sig_fn[0], ';\n'.join((var_ay, sig_fn[1]))) + + return sig_fn + def _parse_sig_js(self, jscode): # Examples where `sig` is funcname: # sig=function(a){a=a.split(""); ... ;return a.join("")}; @@ -1634,8 +1751,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor): jscode, 'Initial JS player signature function name', group='sig') jsi = JSInterpreter(jscode) - initial_function = jsi.extract_function(funcname) - return lambda s: initial_function([s]) + + initial_function = self._extract_sig_fn(jsi, funcname) + + func = jsi.extract_function_from_code(*initial_function) + + return lambda s: func([s]) def _cached(self, func, *cache_id): def inner(*args, **kwargs): @@ -1750,12 +1871,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if func_code: return jsi, player_id, func_code + return self._extract_n_function_code_jsi(video_id, jsi, player_id) + + def _extract_n_function_code_jsi(self, video_id, jsi, player_id=None): - func_name = self._extract_n_function_name(jscode) + func_name = self._extract_n_function_name(jsi.code) - func_code = jsi.extract_function_code(func_name) + func_code = self._extract_sig_fn(jsi, func_name) - self.cache.store('youtube-nsig', player_id, func_code) + if player_id: + self.cache.store('youtube-nsig', player_id, func_code) return jsi, player_id, func_code def _extract_n_function_from_code(self, jsi, func_code): @@ -1944,6 +2069,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): player_response = self._extract_yt_initial_variable( webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE, video_id, 'initial player response') + is_live = traverse_obj(player_response, ('videoDetails', 'isLive')) + if False and not player_response: player_response = self._call_api( 'player', {'videoId': video_id}, video_id) @@ -1957,37 +2084,74 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if sts: pb_context['signatureTimestamp'] = sts - query = { - 'playbackContext': { - 'contentPlaybackContext': pb_context, - 'contentCheckOk': True, - 'racyCheckOk': True, - }, - 'context': { - 'client': { - 'clientName': 'MWEB', - 'clientVersion': '2.20241202.07.00', - 'hl': 'en', - 'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)', - 'timeZone': 'UTC', - 'utcOffsetMinutes': 0, - }, - }, - 'videoId': video_id, - } + client_names = traverse_obj(self._INNERTUBE_CLIENTS, ( + T(dict.items), lambda _, k_v: not k_v[1].get('REQUIRE_PO_TOKEN'), + 0))[:1] + if 'web' not in client_names: + # webpage links won't download: ignore links and playability + player_response = filter_dict( + player_response or {}, + lambda k, _: k not in ('streamingData', 'playabilityStatus')) + + if is_live and 'ios' not in client_names: + client_names.append('ios') + headers = { - 'X-YouTube-Client-Name': '2', - 'X-YouTube-Client-Version': '2.20241202.07.00', - 'Origin': origin, 'Sec-Fetch-Mode': 'navigate', - 'User-Agent': query['context']['client']['userAgent'], + 'Origin': origin, + 'X-Goog-Visitor-Id': self._extract_visitor_data(ytcfg) or '', } auth = self._generate_sapisidhash_header(origin) if auth is not None: headers['Authorization'] = auth headers['X-Origin'] = origin - player_response = self._call_api('player', query, video_id, fatal=False, headers=headers) + for client in traverse_obj(self._INNERTUBE_CLIENTS, (client_names, T(dict))): + + query = { + 'playbackContext': { + 'contentPlaybackContext': pb_context, + }, + 'contentCheckOk': True, + 'racyCheckOk': True, + 'context': { + 'client': merge_dicts( + traverse_obj(client, ('INNERTUBE_CONTEXT', 'client')), { + 'hl': 'en', + 'timeZone': 'UTC', + 'utcOffsetMinutes': 0, + }), + }, + 'videoId': video_id, + } + + api_headers = merge_dicts(headers, traverse_obj(client, { + 'X-YouTube-Client-Name': 'INNERTUBE_CONTEXT_CLIENT_NAME', + 'X-YouTube-Client-Version': ( + 'INNERTUBE_CONTEXT', 'client', 'clientVersion'), + 'User-Agent': ( + 'INNERTUBE_CONTEXT', 'client', 'userAgent'), + })) + + api_player_response = self._call_api( + 'player', query, video_id, fatal=False, headers=api_headers, + note=join_nonempty( + 'Downloading', traverse_obj(query, ( + 'context', 'client', 'clientName')), + 'API JSON', delim=' ')) + + hls = traverse_obj( + (player_response, api_player_response), + (Ellipsis, 'streamingData', 'hlsManifestUrl', T(url_or_none))) + if len(hls) == 2 and not hls[0] and hls[1]: + player_response['streamingData']['hlsManifestUrl'] = hls[1] + else: + video_details = merge_dicts(*traverse_obj( + (player_response, api_player_response), + (Ellipsis, 'videoDetails', T(dict)))) + player_response.update(filter_dict( + api_player_response or {}, cndn=lambda k, _: k != 'captions')) + player_response['videoDetails'] = video_details def is_agegated(playability): if not isinstance(playability, dict): @@ -2130,6 +2294,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): itag_qualities = {} q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres']) CHUNK_SIZE = 10 << 20 + is_live = video_details.get('isLive') streaming_data = player_response.get('streamingData') or {} streaming_formats = streaming_data.get('formats') or [] @@ -2274,7 +2439,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): hls_manifest_url = streaming_data.get('hlsManifestUrl') if hls_manifest_url: for f in self._extract_m3u8_formats( - hls_manifest_url, video_id, 'mp4', fatal=False): + hls_manifest_url, video_id, 'mp4', + entry_protocol='m3u8_native', live=is_live, fatal=False): if process_manifest_format( f, 'hls', None, self._search_regex( r'/itag/(\d+)', f['url'], 'itag', default=None)): @@ -2380,8 +2546,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # Strictly de-prioritize damaged formats f['preference'] = -10 - is_live = video_details.get('isLive') - owner_profile_url = self._yt_urljoin(self._extract_author_var( webpage, 'url', videodetails=video_details, metadata=microformat)) @@ -2416,8 +2580,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): } pctr = traverse_obj( - player_response, - ('captions', 'playerCaptionsTracklistRenderer', T(dict))) + (player_response, api_player_response), + (Ellipsis, 'captions', 'playerCaptionsTracklistRenderer', T(dict))) if pctr: def process_language(container, base_url, lang_code, query): lang_subs = [] @@ -2434,19 +2598,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor): def process_subtitles(): subtitles = {} for caption_track in traverse_obj(pctr, ( - 'captionTracks', lambda _, v: v.get('baseUrl'))): + Ellipsis, 'captionTracks', lambda _, v: ( + v.get('baseUrl') and v.get('languageCode')))): + base_url = self._yt_urljoin(caption_track['baseUrl']) if not base_url: continue + lang_code = caption_track['languageCode'] if caption_track.get('kind') != 'asr': - lang_code = caption_track.get('languageCode') - if not lang_code: - continue process_language( subtitles, base_url, lang_code, {}) continue automatic_captions = {} + process_language( + automatic_captions, base_url, lang_code, {}) for translation_language in traverse_obj(pctr, ( - 'translationLanguages', lambda _, v: v.get('languageCode'))): + Ellipsis, 'translationLanguages', lambda _, v: v.get('languageCode'))): translation_language_code = translation_language['languageCode'] process_language( automatic_captions, base_url, translation_language_code, @@ -3065,8 +3231,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): expected_type=txt_or_none) def _grid_entries(self, grid_renderer): - for item in grid_renderer['items']: - if not isinstance(item, dict): + for item in traverse_obj(grid_renderer, ('items', Ellipsis, T(dict))): + lockup_view_model = traverse_obj(item, ('lockupViewModel', T(dict))) + if lockup_view_model: + entry = self._extract_lockup_view_model(lockup_view_model) + if entry: + yield entry continue renderer = self._extract_grid_item_renderer(item) if not isinstance(renderer, dict): @@ -3150,6 +3320,25 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): continue yield self._extract_video(renderer) + def _extract_lockup_view_model(self, view_model): + content_id = view_model.get('contentId') + if not content_id: + return + content_type = view_model.get('contentType') + if content_type not in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'): + self.report_warning( + 'Unsupported lockup view model content type "{0}"{1}'.format(content_type, bug_reports_message()), only_once=True) + return + return merge_dicts(self.url_result( + update_url_query('https://www.youtube.com/playlist', {'list': content_id}), + ie=YoutubeTabIE.ie_key(), video_id=content_id), { + 'title': traverse_obj(view_model, ( + 'metadata', 'lockupMetadataViewModel', 'title', 'content', T(compat_str))), + 'thumbnails': self._extract_thumbnails(view_model, ( + 'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail', + 'thumbnailViewModel', 'image'), final_key='sources'), + }) + def _video_entry(self, video_renderer): video_id = video_renderer.get('videoId') if video_id: @@ -3354,7 +3543,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): if not continuation: break if visitor_data: - headers['x-goog-visitor-id'] = visitor_data + headers['X-Goog-Visitor-Id'] = visitor_data data['continuation'] = continuation['continuation'] data['clickTracking'] = { 'clickTrackingParams': continuation['itct'], @@ -3536,10 +3725,23 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): def _real_extract(self, url): item_id = self._match_id(url) url = update_url(url, netloc='www.youtube.com') - # Handle both video/playlist URLs qs = parse_qs(url) - video_id = qs.get('v', [None])[0] - playlist_id = qs.get('list', [None])[0] + + def qs_get(key, default=None): + return qs.get(key, [default])[-1] + + # Go around for /feeds/videos.xml?playlist_id={pl_id} + if item_id == 'feeds' and '/feeds/videos.xml?' in url: + playlist_id = qs_get('playlist_id') + if playlist_id: + return self.url_result( + update_url_query('https://www.youtube.com/playlist', { + 'list': playlist_id, + }), ie=self.ie_key(), video_id=playlist_id) + + # Handle both video/playlist URLs + video_id = qs_get('v') + playlist_id = qs_get('list') if video_id and playlist_id: if self._downloader.params.get('noplaylist'): self.to_screen('Downloading just video %s because of --no-playlist' % video_id) diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index 7835187f5..69c8f77ca 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -1,10 +1,12 @@ # coding: utf-8 from __future__ import unicode_literals +import calendar import itertools import json import operator import re +import time from functools import update_wrapper, wraps @@ -12,8 +14,10 @@ from .utils import ( error_to_compat_str, ExtractorError, float_or_none, + int_or_none, js_to_json, remove_quotes, + str_or_none, unified_timestamp, variadic, write_string, @@ -24,6 +28,8 @@ from .compat import ( compat_collections_chain_map as ChainMap, compat_contextlib_suppress, compat_filter as filter, + compat_int, + compat_integer_types, compat_itertools_zip_longest as zip_longest, compat_map as map, compat_numeric_types, @@ -70,14 +76,27 @@ class JS_Undefined(object): pass -def _js_bit_op(op): +def _js_bit_op(op, is_shift=False): - def zeroise(x): - return 0 if x in (None, JS_Undefined, _NaN, _Infinity) else x + def zeroise(x, is_shift_arg=False): + if isinstance(x, compat_integer_types): + return (x % 32) if is_shift_arg else (x & 0xffffffff) + try: + x = float(x) + if is_shift_arg: + x = int(x % 32) + elif x < 0: + x = -compat_int(-x % 0xffffffff) + else: + x = compat_int(x % 0xffffffff) + except (ValueError, TypeError): + # also here for int(NaN), including float('inf') % 32 + x = 0 + return x @wraps_op(op) def wrapped(a, b): - return op(zeroise(a), zeroise(b)) & 0xffffffff + return op(zeroise(a), zeroise(b, is_shift)) & 0xffffffff return wrapped @@ -135,6 +154,7 @@ def _js_to_primitive(v): ) +# more exact: yt-dlp/yt-dlp#12110 def _js_toString(v): return ( 'undefined' if v is JS_Undefined @@ -143,7 +163,7 @@ def _js_toString(v): else 'null' if v is None # bool <= int: do this first else ('false', 'true')[v] if isinstance(v, bool) - else '{0:.7f}'.format(v).rstrip('.0') if isinstance(v, compat_numeric_types) + else re.sub(r'(?<=\d)\.?0*$', '', '{0:.7f}'.format(v)) if isinstance(v, compat_numeric_types) else _js_to_primitive(v)) @@ -253,8 +273,8 @@ def _js_typeof(expr): # avoid dict to maintain order # definition None => Defined in JSInterpreter._operator _OPERATORS = ( - ('>>', _js_bit_op(operator.rshift)), - ('<<', _js_bit_op(operator.lshift)), + ('>>', _js_bit_op(operator.rshift, True)), + ('<<', _js_bit_op(operator.lshift, True)), ('+', _js_add), ('-', _js_arith_op(operator.sub)), ('*', _js_arith_op(operator.mul)), @@ -389,6 +409,7 @@ class JSInterpreter(object): class Exception(ExtractorError): def __init__(self, msg, *args, **kwargs): expr = kwargs.pop('expr', None) + msg = str_or_none(msg, default='"None"') if expr is not None: msg = '{0} in: {1!r:.100}'.format(msg.rstrip(), expr) super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs) @@ -416,6 +437,7 @@ class JSInterpreter(object): flags, _ = self.regex_flags(flags) # First, avoid https://github.com/python/cpython/issues/74534 self.__self = None + pattern_txt = str_or_none(pattern_txt) or '(?:)' self.__pattern_txt = pattern_txt.replace('[[', r'[\[') self.__flags = flags @@ -460,6 +482,73 @@ class JSInterpreter(object): flags |= cls.RE_FLAGS[ch] return flags, expr[idx + 1:] + class JS_Date(object): + _t = None + + @staticmethod + def __ymd_etc(*args, **kw_is_utc): + # args: year, monthIndex, day, hours, minutes, seconds, milliseconds + is_utc = kw_is_utc.get('is_utc', False) + + args = list(args[:7]) + args += [0] * (9 - len(args)) + args[1] += 1 # month 0..11 -> 1..12 + ms = args[6] + for i in range(6, 9): + args[i] = -1 # don't know + if is_utc: + args[-1] = 1 + # TODO: [MDN] When a segment overflows or underflows its expected + # range, it usually "carries over to" or "borrows from" the higher segment. + try: + mktime = calendar.timegm if is_utc else time.mktime + return mktime(time.struct_time(args)) * 1000 + ms + except (OverflowError, ValueError): + return None + + @classmethod + def UTC(cls, *args): + t = cls.__ymd_etc(*args, is_utc=True) + return _NaN if t is None else t + + @staticmethod + def parse(date_str, **kw_is_raw): + is_raw = kw_is_raw.get('is_raw', False) + + t = unified_timestamp(str_or_none(date_str), False) + return int(t * 1000) if t is not None else t if is_raw else _NaN + + @staticmethod + def now(**kw_is_raw): + is_raw = kw_is_raw.get('is_raw', False) + + t = time.time() + return int(t * 1000) if t is not None else t if is_raw else _NaN + + def __init__(self, *args): + if not args: + args = [self.now(is_raw=True)] + if len(args) == 1: + if isinstance(args[0], JSInterpreter.JS_Date): + self._t = int_or_none(args[0].valueOf(), default=None) + else: + arg_type = _js_typeof(args[0]) + if arg_type == 'string': + self._t = self.parse(args[0], is_raw=True) + elif arg_type == 'number': + self._t = int(args[0]) + else: + self._t = self.__ymd_etc(*args) + + def toString(self): + try: + return time.strftime('%a %b %0d %Y %H:%M:%S %Z%z', self._t).rstrip() + except TypeError: + return "Invalid Date" + + def valueOf(self): + return _NaN if self._t is None else self._t + @classmethod def __op_chars(cls): op_chars = set(';,[') @@ -584,18 +673,21 @@ class JSInterpreter(object): except Exception as e: raise self.Exception('Failed to evaluate {left_val!r:.50} {op} {right_val!r:.50}'.format(**locals()), expr, cause=e) - def _index(self, obj, idx, allow_undefined=True): + def _index(self, obj, idx, allow_undefined=None): if idx == 'length' and isinstance(obj, list): return len(obj) try: return obj[int(idx)] if isinstance(obj, list) else obj[compat_str(idx)] - except (TypeError, KeyError, IndexError) as e: - if allow_undefined: - # when is not allowed? + except (TypeError, KeyError, IndexError, ValueError) as e: + # allow_undefined is None gives correct behaviour + if allow_undefined or ( + allow_undefined is None and not isinstance(e, TypeError)): return JS_Undefined raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e) def _dump(self, obj, namespace): + if obj is JS_Undefined: + return 'undefined' try: return json.dumps(obj) except TypeError: @@ -700,7 +792,7 @@ class JSInterpreter(object): new_kw, _, obj = expr.partition('new ') if not new_kw: - for klass, konstr in (('Date', lambda x: int(unified_timestamp(x, False) * 1000)), + for klass, konstr in (('Date', lambda *x: self.JS_Date(*x).valueOf()), ('RegExp', self.JS_RegExp), ('Error', self.Exception)): if not obj.startswith(klass + '('): @@ -948,6 +1040,10 @@ class JSInterpreter(object): left_val = self._index(left_val, idx) if isinstance(idx, float): idx = int(idx) + if isinstance(left_val, list) and len(left_val) <= int_or_none(idx, default=-1): + # JS Array is a sparsely assignable list + # TODO: handle extreme sparsity without memory bloat, eg using auxiliary dict + left_val.extend((idx - len(left_val) + 1) * [JS_Undefined]) left_val[idx] = self._operator( m.group('op'), self._index(left_val, idx) if m.group('op') else None, m.group('expr'), expr, local_vars, allow_recursion) @@ -1019,6 +1115,7 @@ class JSInterpreter(object): 'String': compat_str, 'Math': float, 'Array': list, + 'Date': self.JS_Date, } obj = local_vars.get(variable) if obj in (JS_Undefined, None): @@ -1071,6 +1168,8 @@ class JSInterpreter(object): assertion(len(argvals) == 2, 'takes two arguments') return argvals[0] ** argvals[1] raise self.Exception('Unsupported Math method ' + member, expr=expr) + elif obj is self.JS_Date: + return getattr(obj, member)(*argvals) if member == 'split': assertion(len(argvals) <= 2, 'takes at most two arguments') @@ -1111,9 +1210,10 @@ class JSInterpreter(object): elif member == 'join': assertion(isinstance(obj, list), 'must be applied on a list') assertion(len(argvals) <= 1, 'takes at most one argument') - return (',' if len(argvals) == 0 else argvals[0]).join( - ('' if x in (None, JS_Undefined) else _js_toString(x)) - for x in obj) + return (',' if len(argvals) == 0 or argvals[0] in (None, JS_Undefined) + else argvals[0]).join( + ('' if x in (None, JS_Undefined) else _js_toString(x)) + for x in obj) elif member == 'reverse': assertion(not argvals, 'does not take any arguments') obj.reverse() @@ -1271,19 +1371,21 @@ class JSInterpreter(object): code, _ = self._separate_at_paren(func_m.group('code')) # refine the match return self.build_arglist(func_m.group('args')), code - def extract_function(self, funcname): + def extract_function(self, funcname, *global_stack): return function_with_repr( - self.extract_function_from_code(*self.extract_function_code(funcname)), + self.extract_function_from_code(*itertools.chain( + self.extract_function_code(funcname), global_stack)), 'F<%s>' % (funcname,)) def extract_function_from_code(self, argnames, code, *global_stack): local_vars = {} + start = None while True: - mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code) + mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code[start:]) if mobj is None: break - start, body_start = mobj.span() + start, body_start = ((start or 0) + x for x in mobj.span()) body, remaining = self._separate_at_paren(code[body_start - 1:]) name = self._named_object(local_vars, self.extract_function_from_code( [x.strip() for x in mobj.group('args').split(',')], |