[rh:curl_cffi] Support `curl_cffi` 0.10.x (#12670)

Authored by: Grub4K

5 files changed, 71 insertions, 35 deletions

diff --git a/pyproject.toml b/pyproject.toml
index b5d77d35b..5e987a6fd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -55,8 +55,7 @@ default = [
     "websockets>=13.0",
 ]
 curl-cffi = [
-    "curl-cffi==0.5.10; os_name=='nt' and implementation_name=='cpython'",
-    "curl-cffi>=0.5.10,!=0.6.*,<0.7.2; os_name!='nt' and implementation_name=='cpython'",
+    "curl-cffi>=0.5.10,!=0.6.*,!=0.7.*,!=0.8.*,!=0.9.*,<0.11; implementation_name=='cpython'",
 ]
 secretstorage = [
     "cffi",
diff --git a/test/test_networking.py b/test/test_networking.py
index 63914bc4b..3ab60fe83 100644
--- a/test/test_networking.py
+++ b/test/test_networking.py
@@ -614,7 +614,6 @@ class TestHTTPRequestHandler(TestRequestHandlerBase):
                 rh, Request(f'http://127.0.0.1:{self.http_port}/source_address')).read().decode()
             assert source_address == data
 
-    # Not supported by CurlCFFI
     @pytest.mark.skip_handler('CurlCFFI', 'not supported by curl-cffi')
     def test_gzip_trailing_garbage(self, handler):
         with handler() as rh:
diff --git a/yt_dlp/YoutubeDL.py b/yt_dlp/YoutubeDL.py
index 6c0b12e66..63e6e11b2 100644
--- a/yt_dlp/YoutubeDL.py
+++ b/yt_dlp/YoutubeDL.py
@@ -4152,7 +4152,7 @@ class YoutubeDL:
             (target, rh.RH_NAME)
             for rh in self._request_director.handlers.values()
             if isinstance(rh, ImpersonateRequestHandler)
-            for target in rh.supported_targets
+            for target in reversed(rh.supported_targets)
         ]
 
     def _impersonate_target_available(self, target):
diff --git a/yt_dlp/__init__.py b/yt_dlp/__init__.py
index 7d8f10047..714d9ad5c 100644
--- a/yt_dlp/__init__.py
+++ b/yt_dlp/__init__.py
@@ -1021,8 +1021,9 @@ def _real_main(argv=None):
                 # List of simplified targets we know are supported,
                 # to help users know what dependencies may be required.
                 (ImpersonateTarget('chrome'), 'curl_cffi'),
-                (ImpersonateTarget('edge'), 'curl_cffi'),
                 (ImpersonateTarget('safari'), 'curl_cffi'),
+                (ImpersonateTarget('firefox'), 'curl_cffi>=0.10'),
+                (ImpersonateTarget('edge'), 'curl_cffi'),
             ]
 
             available_targets = ydl._get_available_impersonate_targets()
@@ -1038,12 +1039,12 @@ def _real_main(argv=None):
 
             for known_target, known_handler in known_targets:
                 if not any(
-                    known_target in target and handler == known_handler
+                    known_target in target and known_handler.startswith(handler)
                     for target, handler in available_targets
                 ):
-                    rows.append([
+                    rows.insert(0, [
                         ydl._format_out(text, ydl.Styles.SUPPRESS)
-                        for text in make_row(known_target, f'{known_handler} (not available)')
+                        for text in make_row(known_target, f'{known_handler} (unavailable)')
                     ])
 
             ydl.to_screen('[info] Available impersonate targets')
diff --git a/yt_dlp/networking/_curlcffi.py b/yt_dlp/networking/_curlcffi.py
index 0643348e7..c800f2c09 100644
--- a/yt_dlp/networking/_curlcffi.py
+++ b/yt_dlp/networking/_curlcffi.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import io
+import itertools
 import math
 import re
 import urllib.parse
@@ -31,9 +32,9 @@ if curl_cffi is None:
 
 curl_cffi_version = tuple(map(int, re.split(r'[^\d]+', curl_cffi.__version__)[:3]))
 
-if curl_cffi_version != (0, 5, 10) and not ((0, 7, 0) <= curl_cffi_version < (0, 7, 2)):
+if curl_cffi_version != (0, 5, 10) and not (0, 10) <= curl_cffi_version:
     curl_cffi._yt_dlp__version = f'{curl_cffi.__version__} (unsupported)'
-    raise ImportError('Only curl_cffi versions 0.5.10, 0.7.0 and 0.7.1 are supported')
+    raise ImportError('Only curl_cffi versions 0.5.10 and 0.10.x are supported')
 
 import curl_cffi.requests
 from curl_cffi.const import CurlECode, CurlOpt
@@ -97,7 +98,7 @@ class CurlCFFIResponseAdapter(Response):
             return self.fp.read(amt)
         except curl_cffi.requests.errors.RequestsError as e:
             if e.code == CurlECode.PARTIAL_FILE:
-                content_length = int_or_none(e.response.headers.get('Content-Length'))
+                content_length = e.response and int_or_none(e.response.headers.get('Content-Length'))
                 raise IncompleteRead(
                     partial=self.fp.bytes_read,
                     expected=content_length - self.fp.bytes_read if content_length is not None else None,
@@ -105,6 +106,51 @@ class CurlCFFIResponseAdapter(Response):
             raise TransportError(cause=e) from e
 
 
+# See: https://github.com/lexiforest/curl_cffi?tab=readme-ov-file#supported-impersonate-browsers
+#      https://github.com/lexiforest/curl-impersonate?tab=readme-ov-file#supported-browsers
+BROWSER_TARGETS: dict[tuple[int, ...], dict[str, ImpersonateTarget]] = {
+    (0, 5): {
+        'chrome99': ImpersonateTarget('chrome', '99', 'windows', '10'),
+        'chrome99_android': ImpersonateTarget('chrome', '99', 'android', '12'),
+        'chrome100': ImpersonateTarget('chrome', '100', 'windows', '10'),
+        'chrome101': ImpersonateTarget('chrome', '101', 'windows', '10'),
+        'chrome104': ImpersonateTarget('chrome', '104', 'windows', '10'),
+        'chrome107': ImpersonateTarget('chrome', '107', 'windows', '10'),
+        'chrome110': ImpersonateTarget('chrome', '110', 'windows', '10'),
+        'edge99': ImpersonateTarget('edge', '99', 'windows', '10'),
+        'edge101': ImpersonateTarget('edge', '101', 'windows', '10'),
+        'safari15_3': ImpersonateTarget('safari', '15.3', 'macos', '11'),
+        'safari15_5': ImpersonateTarget('safari', '15.5', 'macos', '12'),
+    },
+    (0, 7): {
+        'chrome116': ImpersonateTarget('chrome', '116', 'windows', '10'),
+        'chrome119': ImpersonateTarget('chrome', '119', 'macos', '14'),
+        'chrome120': ImpersonateTarget('chrome', '120', 'macos', '14'),
+        'chrome123': ImpersonateTarget('chrome', '123', 'macos', '14'),
+        'chrome124': ImpersonateTarget('chrome', '124', 'macos', '14'),
+        'safari17_0': ImpersonateTarget('safari', '17.0', 'macos', '14'),
+        'safari17_2_ios': ImpersonateTarget('safari', '17.2', 'ios', '17.2'),
+    },
+    (0, 9): {
+        'safari15_3': ImpersonateTarget('safari', '15.3', 'macos', '14'),
+        'safari15_5': ImpersonateTarget('safari', '15.5', 'macos', '14'),
+        'chrome119': ImpersonateTarget('chrome', '119', 'macos', '14'),
+        'chrome120': ImpersonateTarget('chrome', '120', 'macos', '14'),
+        'chrome123': ImpersonateTarget('chrome', '123', 'macos', '14'),
+        'chrome124': ImpersonateTarget('chrome', '124', 'macos', '14'),
+        'chrome131': ImpersonateTarget('chrome', '131', 'macos', '14'),
+        'chrome131_android': ImpersonateTarget('chrome', '131', 'android', '14'),
+        'chrome133a': ImpersonateTarget('chrome', '133', 'macos', '15'),
+        'firefox133': ImpersonateTarget('firefox', '133', 'macos', '14'),
+        'safari18_0': ImpersonateTarget('safari', '18.0', 'macos', '15'),
+        'safari18_0_ios': ImpersonateTarget('safari', '18.0', 'ios', '18.0'),
+    },
+    (0, 10): {
+        'firefox135': ImpersonateTarget('firefox', '135', 'macos', '14'),
+    },
+}
+
+
 @register_rh
 class CurlCFFIRH(ImpersonateRequestHandler, InstanceStoreMixin):
     RH_NAME = 'curl_cffi'
@@ -112,30 +158,21 @@ class CurlCFFIRH(ImpersonateRequestHandler, InstanceStoreMixin):
     _SUPPORTED_FEATURES = (Features.NO_PROXY, Features.ALL_PROXY)
     _SUPPORTED_PROXY_SCHEMES = ('http', 'https', 'socks4', 'socks4a', 'socks5', 'socks5h')
     _SUPPORTED_IMPERSONATE_TARGET_MAP = {
-        **({
-            ImpersonateTarget('chrome', '124', 'macos', '14'): curl_cffi.requests.BrowserType.chrome124,
-            ImpersonateTarget('chrome', '123', 'macos', '14'): curl_cffi.requests.BrowserType.chrome123,
-            ImpersonateTarget('chrome', '120', 'macos', '14'): curl_cffi.requests.BrowserType.chrome120,
-            ImpersonateTarget('chrome', '119', 'macos', '14'): curl_cffi.requests.BrowserType.chrome119,
-            ImpersonateTarget('chrome', '116', 'windows', '10'): curl_cffi.requests.BrowserType.chrome116,
-        } if curl_cffi_version >= (0, 7, 0) else {}),
-        ImpersonateTarget('chrome', '110', 'windows', '10'): curl_cffi.requests.BrowserType.chrome110,
-        ImpersonateTarget('chrome', '107', 'windows', '10'): curl_cffi.requests.BrowserType.chrome107,
-        ImpersonateTarget('chrome', '104', 'windows', '10'): curl_cffi.requests.BrowserType.chrome104,
-        ImpersonateTarget('chrome', '101', 'windows', '10'): curl_cffi.requests.BrowserType.chrome101,
-        ImpersonateTarget('chrome', '100', 'windows', '10'): curl_cffi.requests.BrowserType.chrome100,
-        ImpersonateTarget('chrome', '99', 'windows', '10'): curl_cffi.requests.BrowserType.chrome99,
-        ImpersonateTarget('edge', '101', 'windows', '10'): curl_cffi.requests.BrowserType.edge101,
-        ImpersonateTarget('edge', '99', 'windows', '10'): curl_cffi.requests.BrowserType.edge99,
-        **({
-            ImpersonateTarget('safari', '17.0', 'macos', '14'): curl_cffi.requests.BrowserType.safari17_0,
-        } if curl_cffi_version >= (0, 7, 0) else {}),
-        ImpersonateTarget('safari', '15.5', 'macos', '12'): curl_cffi.requests.BrowserType.safari15_5,
-        ImpersonateTarget('safari', '15.3', 'macos', '11'): curl_cffi.requests.BrowserType.safari15_3,
-        ImpersonateTarget('chrome', '99', 'android', '12'): curl_cffi.requests.BrowserType.chrome99_android,
-        **({
-            ImpersonateTarget('safari', '17.2', 'ios', '17.2'): curl_cffi.requests.BrowserType.safari17_2_ios,
-        } if curl_cffi_version >= (0, 7, 0) else {}),
+        target: name if curl_cffi_version >= (0, 9) else curl_cffi.requests.BrowserType[name]
+        for name, target in dict(sorted(itertools.chain.from_iterable(
+            targets.items()
+            for version, targets in BROWSER_TARGETS.items()
+            if curl_cffi_version >= version
+        ), key=lambda x: (
+            # deprioritize mobile targets since they give very different behavior
+            x[1].os not in ('ios', 'android'),
+            # prioritize edge < firefox < safari < chrome
+            ('edge', 'firefox', 'safari', 'chrome').index(x[1].client),
+            # prioritize newest version
+            float(x[1].version) if x[1].version else 0,
+            # group by os name
+            x[1].os,
+        ), reverse=True)).items()
     }
 
     def _create_instance(self, cookiejar=None):