aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorcoletdjnz <coletdjnz@protonmail.com>2025-03-13 17:37:33 +1300
committerGitHub <noreply@github.com>2025-03-13 17:37:33 +1300
commit4432a9390c79253ac830702b226d2e558b636725 (patch)
treea410c277cad901bfd9545ab3797132204d4a3ed2
parent05c8023a27dd37c49163c0498bf98e3e3c1cb4b9 (diff)
[ie/youtube] Split into package (#12557)
Authored by: coletdjnz
-rw-r--r--yt_dlp/extractor/youtube/__init__.py50
-rw-r--r--yt_dlp/extractor/youtube/_base.py1145
-rw-r--r--yt_dlp/extractor/youtube/_clip.py66
-rw-r--r--yt_dlp/extractor/youtube/_mistakes.py69
-rw-r--r--yt_dlp/extractor/youtube/_notifications.py98
-rw-r--r--yt_dlp/extractor/youtube/_redirect.py247
-rw-r--r--yt_dlp/extractor/youtube/_search.py167
-rw-r--r--yt_dlp/extractor/youtube/_tab.py2348
-rw-r--r--yt_dlp/extractor/youtube/_video.py (renamed from yt_dlp/extractor/youtube.py)4078
9 files changed, 4203 insertions, 4065 deletions
diff --git a/yt_dlp/extractor/youtube/__init__.py b/yt_dlp/extractor/youtube/__init__.py
new file mode 100644
index 000000000..892d860b0
--- /dev/null
+++ b/yt_dlp/extractor/youtube/__init__.py
@@ -0,0 +1,50 @@
+# flake8: noqa: F401
+from ._base import YoutubeBaseInfoExtractor
+from ._clip import YoutubeClipIE
+from ._mistakes import YoutubeTruncatedIDIE, YoutubeTruncatedURLIE
+from ._notifications import YoutubeNotificationsIE
+from ._redirect import (
+ YoutubeConsentRedirectIE,
+ YoutubeFavouritesIE,
+ YoutubeFeedsInfoExtractor,
+ YoutubeHistoryIE,
+ YoutubeLivestreamEmbedIE,
+ YoutubeRecommendedIE,
+ YoutubeShortsAudioPivotIE,
+ YoutubeSubscriptionsIE,
+ YoutubeWatchLaterIE,
+ YoutubeYtBeIE,
+ YoutubeYtUserIE,
+)
+from ._search import YoutubeMusicSearchURLIE, YoutubeSearchDateIE, YoutubeSearchIE, YoutubeSearchURLIE
+from ._tab import YoutubePlaylistIE, YoutubeTabBaseInfoExtractor, YoutubeTabIE
+from ._video import YoutubeIE
+
+# Hack to allow plugin overrides work
+for _cls in [
+ YoutubeBaseInfoExtractor,
+ YoutubeClipIE,
+ YoutubeTruncatedIDIE,
+ YoutubeTruncatedURLIE,
+ YoutubeNotificationsIE,
+ YoutubeConsentRedirectIE,
+ YoutubeFavouritesIE,
+ YoutubeFeedsInfoExtractor,
+ YoutubeHistoryIE,
+ YoutubeLivestreamEmbedIE,
+ YoutubeRecommendedIE,
+ YoutubeShortsAudioPivotIE,
+ YoutubeSubscriptionsIE,
+ YoutubeWatchLaterIE,
+ YoutubeYtBeIE,
+ YoutubeYtUserIE,
+ YoutubeMusicSearchURLIE,
+ YoutubeSearchDateIE,
+ YoutubeSearchIE,
+ YoutubeSearchURLIE,
+ YoutubePlaylistIE,
+ YoutubeTabBaseInfoExtractor,
+ YoutubeTabIE,
+ YoutubeIE,
+]:
+ _cls.__module__ = 'yt_dlp.extractor.youtube'
diff --git a/yt_dlp/extractor/youtube/_base.py b/yt_dlp/extractor/youtube/_base.py
new file mode 100644
index 000000000..ba28189a6
--- /dev/null
+++ b/yt_dlp/extractor/youtube/_base.py
@@ -0,0 +1,1145 @@
+import calendar
+import copy
+import datetime as dt
+import enum
+import functools
+import hashlib
+import json
+import re
+import time
+import urllib.parse
+
+from ..common import InfoExtractor
+from ...networking.exceptions import HTTPError, network_exceptions
+from ...utils import (
+ ExtractorError,
+ bug_reports_message,
+ datetime_from_str,
+ filter_dict,
+ get_first,
+ int_or_none,
+ is_html,
+ join_nonempty,
+ parse_count,
+ qualities,
+ str_to_int,
+ traverse_obj,
+ try_call,
+ try_get,
+ unified_timestamp,
+ url_or_none,
+ variadic,
+)
+
+
+class _PoTokenContext(enum.Enum):
+ PLAYER = 'player'
+ GVS = 'gvs'
+
+
+# any clients starting with _ cannot be explicitly requested by the user
+INNERTUBE_CLIENTS = {
+ 'web': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'WEB',
+ 'clientVersion': '2.20241126.01.00',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
+ 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
+ 'SUPPORTS_COOKIES': True,
+ },
+ # Safari UA returns pre-merged video+audio 144p/240p/360p/720p/1080p HLS formats
+ 'web_safari': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'WEB',
+ 'clientVersion': '2.20241126.01.00',
+ 'userAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15,gzip(gfe)',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
+ 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
+ 'SUPPORTS_COOKIES': True,
+ },
+ 'web_embedded': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'WEB_EMBEDDED_PLAYER',
+ 'clientVersion': '1.20241201.00.00',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 56,
+ 'SUPPORTS_COOKIES': True,
+ },
+ 'web_music': {
+ 'INNERTUBE_HOST': 'music.youtube.com',
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'WEB_REMIX',
+ 'clientVersion': '1.20241127.01.00',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
+ 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
+ 'SUPPORTS_COOKIES': True,
+ },
+ # This client now requires sign-in for every video
+ 'web_creator': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'WEB_CREATOR',
+ 'clientVersion': '1.20241203.01.00',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
+ 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
+ 'REQUIRE_AUTH': True,
+ 'SUPPORTS_COOKIES': True,
+ },
+ 'android': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'ANDROID',
+ 'clientVersion': '19.44.38',
+ 'androidSdkVersion': 30,
+ 'userAgent': 'com.google.android.youtube/19.44.38 (Linux; U; Android 11) gzip',
+ 'osName': 'Android',
+ 'osVersion': '11',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
+ 'REQUIRE_JS_PLAYER': False,
+ 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
+ },
+ # This client now requires sign-in for every video
+ 'android_music': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'ANDROID_MUSIC',
+ 'clientVersion': '7.27.52',
+ 'androidSdkVersion': 30,
+ 'userAgent': 'com.google.android.apps.youtube.music/7.27.52 (Linux; U; Android 11) gzip',
+ 'osName': 'Android',
+ 'osVersion': '11',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
+ 'REQUIRE_JS_PLAYER': False,
+ 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
+ 'REQUIRE_AUTH': True,
+ },
+ # This client now requires sign-in for every video
+ 'android_creator': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'ANDROID_CREATOR',
+ 'clientVersion': '24.45.100',
+ 'androidSdkVersion': 30,
+ 'userAgent': 'com.google.android.apps.youtube.creator/24.45.100 (Linux; U; Android 11) gzip',
+ 'osName': 'Android',
+ 'osVersion': '11',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
+ 'REQUIRE_JS_PLAYER': False,
+ 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
+ 'REQUIRE_AUTH': True,
+ },
+ # YouTube Kids videos aren't returned on this client for some reason
+ 'android_vr': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'ANDROID_VR',
+ 'clientVersion': '1.60.19',
+ 'deviceMake': 'Oculus',
+ 'deviceModel': 'Quest 3',
+ 'androidSdkVersion': 32,
+ 'userAgent': 'com.google.android.apps.youtube.vr.oculus/1.60.19 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip',
+ 'osName': 'Android',
+ 'osVersion': '12L',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 28,
+ 'REQUIRE_JS_PLAYER': False,
+ },
+ # iOS clients have HLS live streams. Setting device model to get 60fps formats.
+ # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
+ 'ios': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'IOS',
+ 'clientVersion': '20.03.02',
+ 'deviceMake': 'Apple',
+ 'deviceModel': 'iPhone16,2',
+ 'userAgent': 'com.google.ios.youtube/20.03.02 (iPhone16,2; U; CPU iOS 18_2_1 like Mac OS X;)',
+ 'osName': 'iPhone',
+ 'osVersion': '18.2.1.22C161',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
+ 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
+ 'REQUIRE_JS_PLAYER': False,
+ },
+ # This client now requires sign-in for every video
+ 'ios_music': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'IOS_MUSIC',
+ 'clientVersion': '7.27.0',
+ 'deviceMake': 'Apple',
+ 'deviceModel': 'iPhone16,2',
+ 'userAgent': 'com.google.ios.youtubemusic/7.27.0 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)',
+ 'osName': 'iPhone',
+ 'osVersion': '18.1.0.22B83',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
+ 'REQUIRE_JS_PLAYER': False,
+ 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
+ 'REQUIRE_AUTH': True,
+ },
+ # This client now requires sign-in for every video
+ 'ios_creator': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'IOS_CREATOR',
+ 'clientVersion': '24.45.100',
+ 'deviceMake': 'Apple',
+ 'deviceModel': 'iPhone16,2',
+ 'userAgent': 'com.google.ios.ytcreator/24.45.100 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)',
+ 'osName': 'iPhone',
+ 'osVersion': '18.1.0.22B83',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
+ 'REQUIRE_JS_PLAYER': False,
+ 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
+ 'REQUIRE_AUTH': True,
+ },
+ # mweb has 'ultralow' formats
+ # See: https://github.com/yt-dlp/yt-dlp/pull/557
+ 'mweb': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'MWEB',
+ 'clientVersion': '2.20241202.07.00',
+ # mweb previously did not require PO Token with this UA
+ 'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 2,
+ 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
+ 'SUPPORTS_COOKIES': True,
+ },
+ 'tv': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'TVHTML5',
+ 'clientVersion': '7.20250120.19.00',
+ 'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
+ 'SUPPORTS_COOKIES': True,
+ },
+ # This client now requires sign-in for every video
+ # It was previously an age-gate workaround for videos that were `playable_in_embed`
+ # It may still be useful if signed into an EU account that is not age-verified
+ 'tv_embedded': {
+ 'INNERTUBE_CONTEXT': {
+ 'client': {
+ 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
+ 'clientVersion': '2.0',
+ },
+ },
+ 'INNERTUBE_CONTEXT_CLIENT_NAME': 85,
+ 'REQUIRE_AUTH': True,
+ 'SUPPORTS_COOKIES': True,
+ },
+}
+
+
+def _split_innertube_client(client_name):
+ variant, *base = client_name.rsplit('.', 1)
+ if base:
+ return variant, base[0], variant
+ base, *variant = client_name.split('_', 1)
+ return client_name, base, variant[0] if variant else None
+
+
+def short_client_name(client_name):
+ main, *parts = _split_innertube_client(client_name)[0].split('_')
+ return join_nonempty(main[:4], ''.join(x[0] for x in parts)).upper()
+
+
+def build_innertube_clients():
+ THIRD_PARTY = {
+ 'embedUrl': 'https://www.youtube.com/', # Can be any valid URL
+ }
+ BASE_CLIENTS = ('ios', 'web', 'tv', 'mweb', 'android')
+ priority = qualities(BASE_CLIENTS[::-1])
+
+ for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
+ ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
+ ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
+ ytcfg.setdefault('PO_TOKEN_REQUIRED_CONTEXTS', [])
+ ytcfg.setdefault('REQUIRE_AUTH', False)
+ ytcfg.setdefault('SUPPORTS_COOKIES', False)
+ ytcfg.setdefault('PLAYER_PARAMS', None)
+ ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
+
+ _, base_client, variant = _split_innertube_client(client)
+ ytcfg['priority'] = 10 * priority(base_client)
+
+ if variant == 'embedded':
+ ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
+ ytcfg['priority'] -= 2
+ elif variant:
+ ytcfg['priority'] -= 3
+
+
+build_innertube_clients()
+
+
+class BadgeType(enum.Enum):
+ AVAILABILITY_UNLISTED = enum.auto()
+ AVAILABILITY_PRIVATE = enum.auto()
+ AVAILABILITY_PUBLIC = enum.auto()
+ AVAILABILITY_PREMIUM = enum.auto()
+ AVAILABILITY_SUBSCRIPTION = enum.auto()
+ LIVE_NOW = enum.auto()
+ VERIFIED = enum.auto()
+
+
+CONFIGURATION_ARG_KEY = 'youtube'
+
+
+class YoutubeBaseInfoExtractor(InfoExtractor):
+ """Provide base functions for Youtube extractors"""
+
+ _RESERVED_NAMES = (
+ r'channel|c|user|playlist|watch|w|v|embed|e|live|watch_popup|clip|'
+ r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
+ r'browse|oembed|get_video_info|iframe_api|s/player|source|'
+ r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
+
+ _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
+
+ # _NETRC_MACHINE = 'youtube'
+
+ # If True it will raise an error if no login info is provided
+ _LOGIN_REQUIRED = False
+
+ _INVIDIOUS_SITES = (
+ # invidious-redirect websites
+ r'(?:www\.)?redirect\.invidious\.io',
+ r'(?:(?:www|dev)\.)?invidio\.us',
+ # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
+ r'(?:www\.)?invidious\.pussthecat\.org',
+ r'(?:www\.)?invidious\.zee\.li',
+ r'(?:www\.)?invidious\.ethibox\.fr',
+ r'(?:www\.)?iv\.ggtyler\.dev',
+ r'(?:www\.)?inv\.vern\.i2p',
+ r'(?:www\.)?am74vkcrjp2d5v36lcdqgsj2m6x36tbrkhsruoegwfcizzabnfgf5zyd\.onion',
+ r'(?:www\.)?inv\.riverside\.rocks',
+ r'(?:www\.)?invidious\.silur\.me',
+ r'(?:www\.)?inv\.bp\.projectsegfau\.lt',
+ r'(?:www\.)?invidious\.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid\.onion',
+ r'(?:www\.)?invidious\.slipfox\.xyz',
+ r'(?:www\.)?invidious\.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd\.onion',
+ r'(?:www\.)?inv\.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad\.onion',
+ r'(?:www\.)?invidious\.tiekoetter\.com',
+ r'(?:www\.)?iv\.odysfvr23q5wgt7i456o5t3trw2cw5dgn56vbjfbq2m7xsc5vqbqpcyd\.onion',
+ r'(?:www\.)?invidious\.nerdvpn\.de',
+ r'(?:www\.)?invidious\.weblibre\.org',
+ r'(?:www\.)?inv\.odyssey346\.dev',
+ r'(?:www\.)?invidious\.dhusch\.de',
+ r'(?:www\.)?iv\.melmac\.space',
+ r'(?:www\.)?watch\.thekitty\.zone',
+ r'(?:www\.)?invidious\.privacydev\.net',
+ r'(?:www\.)?ng27owmagn5amdm7l5s3rsqxwscl5ynppnis5dqcasogkyxcfqn7psid\.onion',
+ r'(?:www\.)?invidious\.drivet\.xyz',
+ r'(?:www\.)?vid\.priv\.au',
+ r'(?:www\.)?euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd\.onion',
+ r'(?:www\.)?inv\.vern\.cc',
+ r'(?:www\.)?invidious\.esmailelbob\.xyz',
+ r'(?:www\.)?invidious\.sethforprivacy\.com',
+ r'(?:www\.)?yt\.oelrichsgarcia\.de',
+ r'(?:www\.)?yt\.artemislena\.eu',
+ r'(?:www\.)?invidious\.flokinet\.to',
+ r'(?:www\.)?invidious\.baczek\.me',
+ r'(?:www\.)?y\.com\.sb',
+ r'(?:www\.)?invidious\.epicsite\.xyz',
+ r'(?:www\.)?invidious\.lidarshield\.cloud',
+ r'(?:www\.)?yt\.funami\.tech',
+ r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
+ r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
+ r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
+ # youtube-dl invidious instances list
+ r'(?:(?:www|no)\.)?invidiou\.sh',
+ r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
+ r'(?:www\.)?invidious\.kabi\.tk',
+ r'(?:www\.)?invidious\.mastodon\.host',
+ r'(?:www\.)?invidious\.zapashcanon\.fr',
+ r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
+ r'(?:www\.)?invidious\.tinfoil-hat\.net',
+ r'(?:www\.)?invidious\.himiko\.cloud',
+ r'(?:www\.)?invidious\.reallyancient\.tech',
+ r'(?:www\.)?invidious\.tube',
+ r'(?:www\.)?invidiou\.site',
+ r'(?:www\.)?invidious\.site',
+ r'(?:www\.)?invidious\.xyz',
+ r'(?:www\.)?invidious\.nixnet\.xyz',
+ r'(?:www\.)?invidious\.048596\.xyz',
+ r'(?:www\.)?invidious\.drycat\.fr',
+ r'(?:www\.)?inv\.skyn3t\.in',
+ r'(?:www\.)?tube\.poal\.co',
+ r'(?:www\.)?tube\.connect\.cafe',
+ r'(?:www\.)?vid\.wxzm\.sx',
+ r'(?:www\.)?vid\.mint\.lgbt',
+ r'(?:www\.)?vid\.puffyan\.us',
+ r'(?:www\.)?yewtu\.be',
+ r'(?:www\.)?yt\.elukerio\.org',
+ r'(?:www\.)?yt\.lelux\.fi',
+ r'(?:www\.)?invidious\.ggc-project\.de',
+ r'(?:www\.)?yt\.maisputain\.ovh',
+ r'(?:www\.)?ytprivate\.com',
+ r'(?:www\.)?invidious\.13ad\.de',
+ r'(?:www\.)?invidious\.toot\.koeln',
+ r'(?:www\.)?invidious\.fdn\.fr',
+ r'(?:www\.)?watch\.nettohikari\.com',
+ r'(?:www\.)?invidious\.namazso\.eu',
+ r'(?:www\.)?invidious\.silkky\.cloud',
+ r'(?:www\.)?invidious\.exonip\.de',
+ r'(?:www\.)?invidious\.riverside\.rocks',
+ r'(?:www\.)?invidious\.blamefran\.net',
+ r'(?:www\.)?invidious\.moomoo\.de',
+ r'(?:www\.)?ytb\.trom\.tf',
+ r'(?:www\.)?yt\.cyberhost\.uk',
+ r'(?:www\.)?kgg2m7yk5aybusll\.onion',
+ r'(?:www\.)?qklhadlycap4cnod\.onion',
+ r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
+ r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
+ r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
+ r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
+ r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
+ r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
+ r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
+ r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
+ r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
+ r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
+ # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
+ r'(?:www\.)?piped\.kavin\.rocks',
+ r'(?:www\.)?piped\.tokhmi\.xyz',
+ r'(?:www\.)?piped\.syncpundit\.io',
+ r'(?:www\.)?piped\.mha\.fi',
+ r'(?:www\.)?watch\.whatever\.social',
+ r'(?:www\.)?piped\.garudalinux\.org',
+ r'(?:www\.)?piped\.rivo\.lol',
+ r'(?:www\.)?piped-libre\.kavin\.rocks',
+ r'(?:www\.)?yt\.jae\.fi',
+ r'(?:www\.)?piped\.mint\.lgbt',
+ r'(?:www\.)?il\.ax',
+ r'(?:www\.)?piped\.esmailelbob\.xyz',
+ r'(?:www\.)?piped\.projectsegfau\.lt',
+ r'(?:www\.)?piped\.privacydev\.net',
+ r'(?:www\.)?piped\.palveluntarjoaja\.eu',
+ r'(?:www\.)?piped\.smnz\.de',
+ r'(?:www\.)?piped\.adminforge\.de',
+ r'(?:www\.)?watch\.whatevertinfoil\.de',
+ r'(?:www\.)?piped\.qdi\.fi',
+ r'(?:(?:www|cf)\.)?piped\.video',
+ r'(?:www\.)?piped\.aeong\.one',
+ r'(?:www\.)?piped\.moomoo\.me',
+ r'(?:www\.)?piped\.chauvet\.pro',
+ r'(?:www\.)?watch\.leptons\.xyz',
+ r'(?:www\.)?pd\.vern\.cc',
+ r'(?:www\.)?piped\.hostux\.net',
+ r'(?:www\.)?piped\.lunar\.icu',
+ # Hyperpipe instances from https://hyperpipe.codeberg.page/
+ r'(?:www\.)?hyperpipe\.surge\.sh',
+ r'(?:www\.)?hyperpipe\.esmailelbob\.xyz',
+ r'(?:www\.)?listen\.whatever\.social',
+ r'(?:www\.)?music\.adminforge\.de',
+ )
+
+ # extracted from account/account_menu ep
+ # XXX: These are the supported YouTube UI and API languages,
+ # which is slightly different from languages supported for translation in YouTube studio
+ _SUPPORTED_LANG_CODES = [
+ 'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
+ 'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
+ 'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
+ 'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
+ 'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
+ 'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko',
+ ]
+
+ _IGNORED_WARNINGS = {
+ 'Unavailable videos will be hidden during playback',
+ 'Unavailable videos are hidden',
+ }
+
+ _YT_HANDLE_RE = r'@[\w.-]{3,30}' # https://support.google.com/youtube/answer/11585688?hl=en
+ _YT_CHANNEL_UCID_RE = r'UC[\w-]{22}'
+
+ _NETRC_MACHINE = 'youtube'
+
+ def ucid_or_none(self, ucid):
+ return self._search_regex(rf'^({self._YT_CHANNEL_UCID_RE})$', ucid, 'UC-id', default=None)
+
+ def handle_or_none(self, handle):
+ return self._search_regex(rf'^({self._YT_HANDLE_RE})$', urllib.parse.unquote(handle or ''),
+ '@-handle', default=None)
+
+ def handle_from_url(self, url):
+ return self._search_regex(rf'^(?:https?://(?:www\.)?youtube\.com)?/({self._YT_HANDLE_RE})',
+ urllib.parse.unquote(url or ''), 'channel handle', default=None)
+
+ def ucid_from_url(self, url):
+ return self._search_regex(rf'^(?:https?://(?:www\.)?youtube\.com)?/({self._YT_CHANNEL_UCID_RE})',
+ url, 'channel id', default=None)
+
+ @functools.cached_property
+ def _preferred_lang(self):
+ """
+ Returns a language code supported by YouTube for the user preferred language.
+ Returns None if no preferred language set.
+ """
+ preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
+ if not preferred_lang:
+ return
+ if preferred_lang not in self._SUPPORTED_LANG_CODES:
+ raise ExtractorError(
+ f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
+ expected=True)
+ elif preferred_lang != 'en':
+ self.report_warning(
+ f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
+ return preferred_lang
+
+ def _initialize_consent(self):
+ cookies = self._get_cookies('https://www.youtube.com/')
+ if cookies.get('__Secure-3PSID'):
+ return
+ socs = cookies.get('SOCS')
+ if socs and not socs.value.startswith('CAA'): # not consented
+ return
+ self._set_cookie('.youtube.com', 'SOCS', 'CAI', secure=True) # accept all (required for mixes)
+
+ def _initialize_pref(self):
+ cookies = self._get_cookies('https://www.youtube.com/')
+ pref_cookie = cookies.get('PREF')
+ pref = {}
+ if pref_cookie:
+ try:
+ pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
+ except ValueError:
+ self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
+ pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
+ self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
+
+ def _initialize_cookie_auth(self):
+ yt_sapisid, yt_1psapisid, yt_3psapisid = self._get_sid_cookies()
+ if yt_sapisid or yt_1psapisid or yt_3psapisid:
+ self.write_debug('Found YouTube account cookies')
+
+ def _real_initialize(self):
+ self._initialize_pref()
+ self._initialize_consent()
+ self._initialize_cookie_auth()
+ self._check_login_required()
+
+ def _perform_login(self, username, password):
+ if username.startswith('oauth'):
+ raise ExtractorError(
+ f'Login with OAuth is no longer supported. {self._youtube_login_hint}', expected=True)
+
+ self.report_warning(
+ f'Login with password is not supported for YouTube. {self._youtube_login_hint}')
+
+ @property
+ def _youtube_login_hint(self):
+ return (f'{self._login_hint(method="cookies")}. Also see '
+ 'https://github.com/yt-dlp/yt-dlp/wiki/Extractors#exporting-youtube-cookies '
+ 'for tips on effectively exporting YouTube cookies')
+
+ def _check_login_required(self):
+ if self._LOGIN_REQUIRED and not self.is_authenticated:
+ self.raise_login_required(
+ f'Login details are needed to download this content. {self._youtube_login_hint}', method=None)
+
+ _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
+ _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
+
+ def _get_default_ytcfg(self, client='web'):
+ return copy.deepcopy(INNERTUBE_CLIENTS[client])
+
+ def _get_innertube_host(self, client='web'):
+ return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
+
+ def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
+ # try_get but with fallback to default ytcfg client values when present
+ _func = lambda y: try_get(y, getter, expected_type)
+ return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
+
+ def _extract_client_name(self, ytcfg, default_client='web'):
+ return self._ytcfg_get_safe(
+ ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
+ lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
+
+ def _extract_client_version(self, ytcfg, default_client='web'):
+ return self._ytcfg_get_safe(
+ ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
+ lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
+
+ def _select_api_hostname(self, req_api_hostname, default_client=None):
+ return (self._configuration_arg('innertube_host', [''], ie_key=CONFIGURATION_ARG_KEY)[0]
+ or req_api_hostname or self._get_innertube_host(default_client or 'web'))
+
+ def _extract_context(self, ytcfg=None, default_client='web'):
+ context = get_first(
+ (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
+ # Enforce language and tz for extraction
+ client_context = traverse_obj(context, 'client', expected_type=dict, default={})
+ client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
+ return context
+
+ @staticmethod
+ def _make_sid_authorization(scheme, sid, origin, additional_parts):
+ timestamp = str(round(time.time()))
+
+ hash_parts = []
+ if additional_parts:
+ hash_parts.append(':'.join(additional_parts.values()))
+ hash_parts.extend([timestamp, sid, origin])
+ sidhash = hashlib.sha1(' '.join(hash_parts).encode()).hexdigest()
+
+ parts = [timestamp, sidhash]
+ if additional_parts:
+ parts.append(''.join(additional_parts))
+
+ return f'{scheme} {"_".join(parts)}'
+
+ def _get_sid_cookies(self):
+ """
+ Get SAPISID, 1PSAPISID, 3PSAPISID cookie values
+ @returns sapisid, 1psapisid, 3psapisid
+ """
+ yt_cookies = self._get_cookies('https://www.youtube.com')
+ yt_sapisid = try_call(lambda: yt_cookies['SAPISID'].value)
+ yt_3papisid = try_call(lambda: yt_cookies['__Secure-3PAPISID'].value)
+ yt_1papisid = try_call(lambda: yt_cookies['__Secure-1PAPISID'].value)
+
+ # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
+ # YouTube also falls back to __Secure-3PAPISID if SAPISID is missing.
+ # See: https://github.com/yt-dlp/yt-dlp/issues/393
+
+ return yt_sapisid or yt_3papisid, yt_1papisid, yt_3papisid
+
+ def _get_sid_authorization_header(self, origin='https://www.youtube.com', user_session_id=None):
+ """
+ Generate API Session ID Authorization for Innertube requests. Assumes all requests are secure (https).
+ @param origin: Origin URL
+ @param user_session_id: Optional User Session ID
+ @return: Authorization header value
+ """
+
+ authorizations = []
+ additional_parts = {}
+ if user_session_id:
+ additional_parts['u'] = user_session_id
+
+ yt_sapisid, yt_1psapisid, yt_3psapisid = self._get_sid_cookies()
+
+ for scheme, sid in (('SAPISIDHASH', yt_sapisid),
+ ('SAPISID1PHASH', yt_1psapisid),
+ ('SAPISID3PHASH', yt_3psapisid)):
+ if sid:
+ authorizations.append(self._make_sid_authorization(scheme, sid, origin, additional_parts))
+
+ if not authorizations:
+ return None
+
+ return ' '.join(authorizations)
+
+ def _call_api(self, ep, query, video_id, fatal=True, headers=None,
+ note='Downloading API JSON', errnote='Unable to download API page',
+ context=None, api_key=None, api_hostname=None, default_client='web'):
+
+ data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
+ data.update(query)
+ real_headers = self.generate_api_headers(default_client=default_client)
+ real_headers.update({'content-type': 'application/json'})
+ if headers:
+ real_headers.update(headers)
+ return self._download_json(
+ f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
+ video_id=video_id, fatal=fatal, note=note, errnote=errnote,
+ data=json.dumps(data).encode('utf8'), headers=real_headers,
+ query=filter_dict({
+ 'key': self._configuration_arg(
+ 'innertube_key', [api_key], ie_key=CONFIGURATION_ARG_KEY, casesense=True)[0],
+ 'prettyPrint': 'false',
+ }, cndn=lambda _, v: v))
+
+ def extract_yt_initial_data(self, item_id, webpage, fatal=True):
+ return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
+
+ @staticmethod
+ def _extract_session_index(*data):
+ """
+ Index of current account in account list.
+ See: https://github.com/yt-dlp/yt-dlp/pull/519
+ """
+ for ytcfg in data:
+ session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
+ if session_index is not None:
+ return session_index
+
+ @staticmethod
+ def _parse_data_sync_id(data_sync_id):
+ """
+ Parse data_sync_id into delegated_session_id and user_session_id.
+
+ data_sync_id is of the form "delegated_session_id||user_session_id" for secondary channel
+ and just "user_session_id||" for primary channel.
+
+ @param data_sync_id: data_sync_id string
+ @return: Tuple of (delegated_session_id, user_session_id)
+ """
+ if not data_sync_id:
+ return None, None
+ first, _, second = data_sync_id.partition('||')
+ if second:
+ return first, second
+ return None, first
+
+ def _extract_delegated_session_id(self, *args):
+ """
+ Extract current delegated session ID required to download private playlists of secondary channels
+ @params response and/or ytcfg
+ @return: delegated session ID
+ """
+ # ytcfg includes channel_syncid if on secondary channel
+ if delegated_sid := traverse_obj(args, (..., 'DELEGATED_SESSION_ID', {str}, any)):
+ return delegated_sid
+
+ data_sync_id = self._extract_data_sync_id(*args)
+ return self._parse_data_sync_id(data_sync_id)[0]
+
+ def _extract_user_session_id(self, *args):
+ """
+ Extract current user session ID
+ @params response and/or ytcfg
+ @return: user session ID
+ """
+ if user_sid := traverse_obj(args, (..., 'USER_SESSION_ID', {str}, any)):
+ return user_sid
+
+ data_sync_id = self._extract_data_sync_id(*args)
+ return self._parse_data_sync_id(data_sync_id)[1]
+
+ def _extract_data_sync_id(self, *args):
+ """
+ Extract current account dataSyncId.
+ In the format DELEGATED_SESSION_ID||USER_SESSION_ID or USER_SESSION_ID||
+ @params response and/or ytcfg
+ """
+ if data_sync_id := self._configuration_arg('data_sync_id', [None], ie_key=CONFIGURATION_ARG_KEY, casesense=True)[0]:
+ return data_sync_id
+
+ return traverse_obj(
+ args, (..., ('DATASYNC_ID', ('responseContext', 'mainAppWebResponseContext', 'datasyncId')), {str}, any))
+
+ def _extract_visitor_data(self, *args):
+ """
+ Extracts visitorData from an API response or ytcfg
+ Appears to be used to track session state
+ """
+ if visitor_data := self._configuration_arg('visitor_data', [None], ie_key=CONFIGURATION_ARG_KEY, casesense=True)[0]:
+ return visitor_data
+ return get_first(
+ args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
+ expected_type=str)
+
+ @functools.cached_property
+ def is_authenticated(self):
+ return bool(self._get_sid_authorization_header())
+
+ def extract_ytcfg(self, video_id, webpage):
+ if not webpage:
+ return {}
+ return self._parse_json(
+ self._search_regex(
+ r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
+ default='{}'), video_id, fatal=False) or {}
+
+ def _generate_cookie_auth_headers(self, *, ytcfg=None, delegated_session_id=None, user_session_id=None, session_index=None, origin=None, **kwargs):
+ headers = {}
+ delegated_session_id = delegated_session_id or self._extract_delegated_session_id(ytcfg)
+ if delegated_session_id:
+ headers['X-Goog-PageId'] = delegated_session_id
+ if session_index is None:
+ session_index = self._extract_session_index(ytcfg)
+ if delegated_session_id or session_index is not None:
+ headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
+
+ auth = self._get_sid_authorization_header(origin, user_session_id=user_session_id or self._extract_user_session_id(ytcfg))
+ if auth is not None:
+ headers['Authorization'] = auth
+ headers['X-Origin'] = origin
+
+ if traverse_obj(ytcfg, 'LOGGED_IN', expected_type=bool):
+ headers['X-Youtube-Bootstrap-Logged-In'] = 'true'
+
+ return headers
+
+ def generate_api_headers(
+ self, *, ytcfg=None, delegated_session_id=None, user_session_id=None, session_index=None,
+ visitor_data=None, api_hostname=None, default_client='web', **kwargs):
+
+ origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
+ headers = {
+ 'X-YouTube-Client-Name': str(
+ self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
+ 'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
+ 'Origin': origin,
+ 'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
+ 'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client),
+ **self._generate_cookie_auth_headers(
+ ytcfg=ytcfg,
+ delegated_session_id=delegated_session_id,
+ user_session_id=user_session_id,
+ session_index=session_index,
+ origin=origin),
+ }
+ return filter_dict(headers)
+
+ def _download_webpage_with_retries(self, *args, retry_fatal=False, retry_on_status=None, **kwargs):
+ for retry in self.RetryManager(fatal=retry_fatal):
+ try:
+ return self._download_webpage(*args, **kwargs)
+ except ExtractorError as e:
+ if isinstance(e.cause, network_exceptions):
+ if not isinstance(e.cause, HTTPError) or e.cause.status not in (retry_on_status or (403, 429)):
+ retry.error = e
+ continue
+ self._error_or_warning(e, fatal=retry_fatal)
+ break
+
+ def _download_ytcfg(self, client, video_id):
+ url = {
+ 'web': 'https://www.youtube.com',
+ 'web_music': 'https://music.youtube.com',
+ 'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1',
+ 'tv': 'https://www.youtube.com/tv',
+ }.get(client)
+ if not url:
+ return {}
+ webpage = self._download_webpage_with_retries(
+ url, video_id, note=f'Downloading {client.replace("_", " ").strip()} client config',
+ headers=traverse_obj(self._get_default_ytcfg(client), {
+ 'User-Agent': ('INNERTUBE_CONTEXT', 'client', 'userAgent', {str}),
+ }))
+ return self.extract_ytcfg(video_id, webpage) or {}
+
+ @staticmethod
+ def _build_api_continuation_query(continuation, ctp=None):
+ query = {
+ 'continuation': continuation,
+ }
+ # TODO: Inconsistency with clickTrackingParams.
+ # Currently we have a fixed ctp contained within context (from ytcfg)
+ # and a ctp in root query for continuation.
+ if ctp:
+ query['clickTracking'] = {'clickTrackingParams': ctp}
+ return query
+
+ @classmethod
+ def _extract_next_continuation_data(cls, renderer):
+ next_continuation = try_get(
+ renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
+ lambda x: x['continuation']['reloadContinuationData']), dict)
+ if not next_continuation:
+ return
+ continuation = next_continuation.get('continuation')
+ if not continuation:
+ return
+ ctp = next_continuation.get('clickTrackingParams')
+ return cls._build_api_continuation_query(continuation, ctp)
+
+ @classmethod
+ def _extract_continuation_ep_data(cls, continuation_ep: dict):
+ if isinstance(continuation_ep, dict):
+ continuation = try_get(
+ continuation_ep, lambda x: x['continuationCommand']['token'], str)
+ if not continuation:
+ return
+ ctp = continuation_ep.get('clickTrackingParams')
+ return cls._build_api_continuation_query(continuation, ctp)
+
+ @classmethod
+ def _extract_continuation(cls, renderer):
+ next_continuation = cls._extract_next_continuation_data(renderer)
+ if next_continuation:
+ return next_continuation
+
+ return traverse_obj(renderer, (
+ ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
+ ('continuationEndpoint', ('button', 'buttonRenderer', 'command')),
+ ), get_all=False, expected_type=cls._extract_continuation_ep_data)
+
+ @classmethod
+ def _extract_alerts(cls, data):
+ for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
+ if not isinstance(alert_dict, dict):
+ continue
+ for alert in alert_dict.values():
+ alert_type = alert.get('type')
+ if not alert_type:
+ continue
+ message = cls._get_text(alert, 'text')
+ if message:
+ yield alert_type, message
+
+ def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
+ errors, warnings = [], []
+ for alert_type, alert_message in alerts:
+ if alert_type.lower() == 'error' and fatal:
+ errors.append([alert_type, alert_message])
+ elif alert_message not in self._IGNORED_WARNINGS:
+ warnings.append([alert_type, alert_message])
+
+ for alert_type, alert_message in (warnings + errors[:-1]):
+ self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
+ if errors:
+ raise ExtractorError(f'YouTube said: {errors[-1][1]}', expected=expected)
+
+ def _extract_and_report_alerts(self, data, *args, **kwargs):
+ return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
+
+ def _extract_badges(self, badge_list: list):
+ """
+ Extract known BadgeType's from a list of badge renderers.
+ @returns [{'type': BadgeType}]
+ """
+ icon_type_map = {
+ 'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
+ 'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
+ 'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC,
+ 'CHECK_CIRCLE_THICK': BadgeType.VERIFIED,
+ 'OFFICIAL_ARTIST_BADGE': BadgeType.VERIFIED,
+ 'CHECK': BadgeType.VERIFIED,
+ }
+
+ badge_style_map = {
+ 'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
+ 'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
+ 'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW,
+ 'BADGE_STYLE_TYPE_VERIFIED': BadgeType.VERIFIED,
+ 'BADGE_STYLE_TYPE_VERIFIED_ARTIST': BadgeType.VERIFIED,
+ }
+
+ label_map = {
+ 'unlisted': BadgeType.AVAILABILITY_UNLISTED,
+ 'private': BadgeType.AVAILABILITY_PRIVATE,
+ 'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
+ 'live': BadgeType.LIVE_NOW,
+ 'premium': BadgeType.AVAILABILITY_PREMIUM,
+ 'verified': BadgeType.VERIFIED,
+ 'official artist channel': BadgeType.VERIFIED,
+ }
+
+ badges = []
+ for badge in traverse_obj(badge_list, (..., lambda key, _: re.search(r'[bB]adgeRenderer$', key))):
+ badge_type = (
+ icon_type_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
+ or badge_style_map.get(traverse_obj(badge, 'style'))
+ )
+ if badge_type:
+ badges.append({'type': badge_type})
+ continue
+
+ # fallback, won't work in some languages
+ label = traverse_obj(
+ badge, 'label', ('accessibilityData', 'label'), 'tooltip', 'iconTooltip', get_all=False, expected_type=str, default='')
+ for match, label_badge_type in label_map.items():
+ if match in label.lower():
+ badges.append({'type': label_badge_type})
+ break
+
+ return badges
+
+ @staticmethod
+ def _has_badge(badges, badge_type):
+ return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
+
+ @staticmethod
+ def _get_text(data, *path_list, max_runs=None):
+ for path in path_list or [None]:
+ if path is None:
+ obj = [data]
+ else:
+ obj = traverse_obj(data, path, default=[])
+ if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
+ obj = [obj]
+ for item in obj:
+ text = try_get(item, lambda x: x['simpleText'], str)
+ if text:
+ return text
+ runs = try_get(item, lambda x: x['runs'], list) or []
+ if not runs and isinstance(item, list):
+ runs = item
+
+ runs = runs[:min(len(runs), max_runs or len(runs))]
+ text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str))
+ if text:
+ return text
+
+ def _get_count(self, data, *path_list):
+ count_text = self._get_text(data, *path_list) or ''
+ count = parse_count(count_text)
+ if count is None:
+ count = str_to_int(
+ self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
+ return count
+
+ @staticmethod
+ def _extract_thumbnails(data, *path_list, final_key='thumbnails'):
+ """
+ Extract thumbnails from thumbnails dict
+ @param path_list: path list to level that contains 'thumbnails' key
+ """
+ thumbnails = []
+ for path in path_list or [()]:
+ for thumbnail in traverse_obj(data, (*variadic(path), final_key, ...)):
+ thumbnail_url = url_or_none(thumbnail.get('url'))
+ if not thumbnail_url:
+ continue
+ # Sometimes youtube gives a wrong thumbnail URL. See:
+ # https://github.com/yt-dlp/yt-dlp/issues/233
+ # https://github.com/ytdl-org/youtube-dl/issues/28023
+ if 'maxresdefault' in thumbnail_url:
+ thumbnail_url = thumbnail_url.split('?')[0]
+ thumbnails.append({
+ 'url': thumbnail_url,
+ 'height': int_or_none(thumbnail.get('height')),
+ 'width': int_or_none(thumbnail.get('width')),
+ })
+ return thumbnails
+
+ @staticmethod
+ def extract_relative_time(relative_time_text):
+ """
+ Extracts a relative time from string and converts to dt object
+ e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today', '8 yr ago'
+ """
+
+ # XXX: this could be moved to a general function in utils/_utils.py
+ # The relative time text strings are roughly the same as what
+ # Javascript's Intl.RelativeTimeFormat function generates.
+ # See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/RelativeTimeFormat
+ mobj = re.search(
+ r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>sec(?:ond)?|s|min(?:ute)?|h(?:our|r)?|d(?:ay)?|w(?:eek|k)?|mo(?:nth)?|y(?:ear|r)?)s?\s*ago',
+ relative_time_text)
+ if mobj:
+ start = mobj.group('start')
+ if start:
+ return datetime_from_str(start)
+ try:
+ return datetime_from_str('now-{}{}'.format(mobj.group('time'), mobj.group('unit')))
+ except ValueError:
+ return None
+
+ def _parse_time_text(self, text):
+ if not text:
+ return
+ dt_ = self.extract_relative_time(text)
+ timestamp = None
+ if isinstance(dt_, dt.datetime):
+ timestamp = calendar.timegm(dt_.timetuple())
+
+ if timestamp is None:
+ timestamp = (
+ unified_timestamp(text) or unified_timestamp(
+ self._search_regex(
+ (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
+ text.lower(), 'time text', default=None)))
+
+ if text and timestamp is None and self._preferred_lang in (None, 'en'):
+ self.report_warning(
+ f'Cannot parse localized time text "{text}"', only_once=True)
+ return timestamp
+
+ def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
+ ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
+ default_client='web'):
+ raise_for_incomplete = bool(self._configuration_arg('raise_incomplete_data', ie_key=CONFIGURATION_ARG_KEY))
+ # Incomplete Data should be a warning by default when retries are exhausted, while other errors should be fatal.
+ icd_retries = iter(self.RetryManager(fatal=raise_for_incomplete))
+ icd_rm = next(icd_retries)
+ main_retries = iter(self.RetryManager())
+ main_rm = next(main_retries)
+ # Manual retry loop for multiple RetryManagers
+ # The proper RetryManager MUST be advanced after an error
+ # and its result MUST be checked if the manager is non fatal
+ while True:
+ try:
+ response = self._call_api(
+ ep=ep, fatal=True, headers=headers,
+ video_id=item_id, query=query, note=note,
+ context=self._extract_context(ytcfg, default_client),
+ api_hostname=api_hostname, default_client=default_client)
+ except ExtractorError as e:
+ if not isinstance(e.cause, network_exceptions):
+ return self._error_or_warning(e, fatal=fatal)
+ elif not isinstance(e.cause, HTTPError):
+ main_rm.error = e
+ next(main_retries)
+ continue
+
+ first_bytes = e.cause.response.read(512)
+ if not is_html(first_bytes):
+ yt_error = try_get(
+ self._parse_json(
+ self._webpage_read_content(e.cause.response, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
+ lambda x: x['error']['message'], str)
+ if yt_error:
+ self._report_alerts([('ERROR', yt_error)], fatal=False)
+ # Downloading page may result in intermittent 5xx HTTP error
+ # Sometimes a 404 is also received. See: https://github.com/ytdl-org/youtube-dl/issues/28289
+ # We also want to catch all other network exceptions since errors in later pages can be troublesome
+ # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
+ if e.cause.status not in (403, 429):
+ main_rm.error = e
+ next(main_retries)
+ continue
+ return self._error_or_warning(e, fatal=fatal)
+
+ try:
+ self._extract_and_report_alerts(response, only_once=True)
+ except ExtractorError as e:
+ # YouTube's servers may return errors we want to retry on in a 200 OK response
+ # See: https://github.com/yt-dlp/yt-dlp/issues/839
+ if 'unknown error' in e.msg.lower():
+ main_rm.error = e
+ next(main_retries)
+ continue
+ return self._error_or_warning(e, fatal=fatal)
+ # Youtube sometimes sends incomplete data
+ # See: https://github.com/ytdl-org/youtube-dl/issues/28194
+ if not traverse_obj(response, *variadic(check_get_keys)):
+ icd_rm.error = ExtractorError('Incomplete data received', expected=True)
+ should_retry = next(icd_retries, None)
+ if not should_retry:
+ return None
+ continue
+
+ return response
+
+ @staticmethod
+ def is_music_url(url):
+ return re.match(r'(https?://)?music\.youtube\.com/', url) is not None
diff --git a/yt_dlp/extractor/youtube/_clip.py b/yt_dlp/extractor/youtube/_clip.py
new file mode 100644
index 000000000..7d063700e
--- /dev/null
+++ b/yt_dlp/extractor/youtube/_clip.py
@@ -0,0 +1,66 @@
+from ._tab import YoutubeTabBaseInfoExtractor
+from ._video import YoutubeIE
+from ...utils import ExtractorError, traverse_obj
+
+
+class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
+ IE_NAME = 'youtube:clip'
+ _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
+ _TESTS = [{
+ # FIXME: Other metadata should be extracted from the clip, not from the base video
+ 'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
+ 'info_dict': {
+ 'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
+ 'ext': 'mp4',
+ 'section_start': 29.0,
+ 'section_end': 39.7,
+ 'duration': 10.7,
+ 'age_limit': 0,
+ 'availability': 'public',
+ 'categories': ['Gaming'],
+ 'channel': 'Scott The Woz',
+ 'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
+ 'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
+ 'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
+ 'like_count': int,
+ 'playable_in_embed': True,
+ 'tags': 'count:17',
+ 'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
+ 'title': 'Mobile Games on Console - Scott The Woz',
+ 'upload_date': '20210920',
+ 'uploader': 'Scott The Woz',
+ 'uploader_id': '@ScottTheWoz',
+ 'uploader_url': 'https://www.youtube.com/@ScottTheWoz',
+ 'view_count': int,
+ 'live_status': 'not_live',
+ 'channel_follower_count': int,
+ 'chapters': 'count:20',
+ 'comment_count': int,
+ 'heatmap': 'count:100',
+ },
+ }]
+
+ def _real_extract(self, url):
+ clip_id = self._match_id(url)
+ _, data = self._extract_webpage(url, clip_id)
+
+ video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
+ if not video_id:
+ raise ExtractorError('Unable to find video ID')
+
+ clip_data = traverse_obj(data, (
+ 'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
+ 'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
+ 'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
+ 'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
+
+ return {
+ '_type': 'url_transparent',
+ 'url': f'https://www.youtube.com/watch?v={video_id}',
+ 'ie_key': YoutubeIE.ie_key(),
+ 'id': clip_id,
+ 'section_start': int(clip_data['startTimeMs']) / 1000,
+ 'section_end': int(clip_data['endTimeMs']) / 1000,
+ '_format_sort_fields': ( # https protocol is prioritized for ffmpeg compatibility
+ 'proto:https', 'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec', 'channels', 'acodec', 'lang'),
+ }
diff --git a/yt_dlp/extractor/youtube/_mistakes.py b/yt_dlp/extractor/youtube/_mistakes.py
new file mode 100644
index 000000000..c5eb5161c
--- /dev/null
+++ b/yt_dlp/extractor/youtube/_mistakes.py
@@ -0,0 +1,69 @@
+
+from ._base import YoutubeBaseInfoExtractor
+from ...utils import ExtractorError
+
+
+class YoutubeTruncatedURLIE(YoutubeBaseInfoExtractor):
+ IE_NAME = 'youtube:truncated_url'
+ IE_DESC = False # Do not list
+ _VALID_URL = r'''(?x)
+ (?:https?://)?
+ (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
+ (?:watch\?(?:
+ feature=[a-z_]+|
+ annotation_id=annotation_[^&]+|
+ x-yt-cl=[0-9]+|
+ hl=[^&]*|
+ t=[0-9]+
+ )?
+ |
+ attribution_link\?a=[^&]+
+ )
+ $
+ '''
+
+ _TESTS = [{
+ 'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/watch?',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/watch?feature=foo',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/watch?hl=en-GB',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/watch?t=2372',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ raise ExtractorError(
+ 'Did you forget to quote the URL? Remember that & is a meta '
+ 'character in most shells, so you want to put the URL in quotes, '
+ 'like yt-dlp '
+ '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
+ ' or simply yt-dlp BaW_jenozKc .',
+ expected=True)
+
+
+class YoutubeTruncatedIDIE(YoutubeBaseInfoExtractor):
+ IE_NAME = 'youtube:truncated_id'
+ IE_DESC = False # Do not list
+ _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
+
+ _TESTS = [{
+ 'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ raise ExtractorError(
+ f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
+ expected=True)
diff --git a/yt_dlp/extractor/youtube/_notifications.py b/yt_dlp/extractor/youtube/_notifications.py
new file mode 100644
index 000000000..ae55528da
--- /dev/null
+++ b/yt_dlp/extractor/youtube/_notifications.py
@@ -0,0 +1,98 @@
+import itertools
+import re
+
+from ._tab import YoutubeTabBaseInfoExtractor, YoutubeTabIE
+from ._video import YoutubeIE
+from ...utils import traverse_obj
+
+
+class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
+ IE_NAME = 'youtube:notif'
+ IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
+ _VALID_URL = r':ytnotif(?:ication)?s?'
+ _LOGIN_REQUIRED = True
+ _TESTS = [{
+ 'url': ':ytnotif',
+ 'only_matching': True,
+ }, {
+ 'url': ':ytnotifications',
+ 'only_matching': True,
+ }]
+
+ def _extract_notification_menu(self, response, continuation_list):
+ notification_list = traverse_obj(
+ response,
+ ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
+ ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
+ expected_type=list) or []
+ continuation_list[0] = None
+ for item in notification_list:
+ entry = self._extract_notification_renderer(item.get('notificationRenderer'))
+ if entry:
+ yield entry
+ continuation = item.get('continuationItemRenderer')
+ if continuation:
+ continuation_list[0] = continuation
+
+ def _extract_notification_renderer(self, notification):
+ video_id = traverse_obj(
+ notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
+ url = f'https://www.youtube.com/watch?v={video_id}'
+ channel_id = None
+ if not video_id:
+ browse_ep = traverse_obj(
+ notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
+ channel_id = self.ucid_or_none(traverse_obj(browse_ep, 'browseId', expected_type=str))
+ post_id = self._search_regex(
+ r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
+ 'post id', default=None)
+ if not channel_id or not post_id:
+ return
+ # The direct /post url redirects to this in the browser
+ url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
+
+ channel = traverse_obj(
+ notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
+ expected_type=str)
+ notification_title = self._get_text(notification, 'shortMessage')
+ if notification_title:
+ notification_title = notification_title.replace('\xad', '') # remove soft hyphens
+ # TODO: handle recommended videos
+ title = self._search_regex(
+ rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
+ 'video title', default=None)
+ timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
+ if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
+ else None)
+ return {
+ '_type': 'url',
+ 'url': url,
+ 'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
+ 'video_id': video_id,
+ 'title': title,
+ 'channel_id': channel_id,
+ 'channel': channel,
+ 'uploader': channel,
+ 'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
+ 'timestamp': timestamp,
+ }
+
+ def _notification_menu_entries(self, ytcfg):
+ continuation_list = [None]
+ response = None
+ for page in itertools.count(1):
+ ctoken = traverse_obj(
+ continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
+ response = self._extract_response(
+ item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
+ ep='notification/get_notification_menu', check_get_keys='actions',
+ headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
+ yield from self._extract_notification_menu(response, continuation_list)
+ if not continuation_list[0]:
+ break
+
+ def _real_extract(self, url):
+ display_id = 'notifications'
+ ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
+ self._report_playlist_authcheck(ytcfg)
+ return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
diff --git a/yt_dlp/extractor/youtube/_redirect.py b/yt_dlp/extractor/youtube/_redirect.py
new file mode 100644
index 000000000..1908df124
--- /dev/null
+++ b/yt_dlp/extractor/youtube/_redirect.py
@@ -0,0 +1,247 @@
+import base64
+import urllib.parse
+
+from ._base import YoutubeBaseInfoExtractor
+from ._tab import YoutubeTabIE
+from ...utils import ExtractorError, classproperty, parse_qs, update_url_query, url_or_none
+
+
+class YoutubeYtBeIE(YoutubeBaseInfoExtractor):
+ IE_DESC = 'youtu.be'
+ _VALID_URL = rf'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{{11}})/*?.*?\blist=(?P<playlist_id>{YoutubeBaseInfoExtractor._PLAYLIST_ID_RE})'
+ _TESTS = [{
+ 'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
+ 'info_dict': {
+ 'id': 'yeWKywCrFtk',
+ 'ext': 'mp4',
+ 'title': 'Small Scale Baler and Braiding Rugs',
+ 'uploader': 'Backus-Page House Museum',
+ 'uploader_id': '@backuspagemuseum',
+ 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@backuspagemuseum',
+ 'upload_date': '20161008',
+ 'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
+ 'categories': ['Nonprofits & Activism'],
+ 'tags': list,
+ 'like_count': int,
+ 'age_limit': 0,
+ 'playable_in_embed': True,
+ 'thumbnail': r're:^https?://.*\.webp',
+ 'channel': 'Backus-Page House Museum',
+ 'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
+ 'live_status': 'not_live',
+ 'view_count': int,
+ 'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
+ 'availability': 'public',
+ 'duration': 59,
+ 'comment_count': int,
+ 'channel_follower_count': int,
+ },
+ 'params': {
+ 'noplaylist': True,
+ 'skip_download': True,
+ },
+ }, {
+ 'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ mobj = self._match_valid_url(url)
+ video_id = mobj.group('id')
+ playlist_id = mobj.group('playlist_id')
+ return self.url_result(
+ update_url_query('https://www.youtube.com/watch', {
+ 'v': video_id,
+ 'list': playlist_id,
+ 'feature': 'youtu.be',
+ }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
+
+
+class YoutubeLivestreamEmbedIE(YoutubeBaseInfoExtractor):
+ IE_DESC = 'YouTube livestream embeds'
+ _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
+ _TESTS = [{
+ 'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ channel_id = self._match_id(url)
+ return self.url_result(
+ f'https://www.youtube.com/channel/{channel_id}/live',
+ ie=YoutubeTabIE.ie_key(), video_id=channel_id)
+
+
+class YoutubeYtUserIE(YoutubeBaseInfoExtractor):
+ IE_DESC = 'YouTube user videos; "ytuser:" prefix'
+ IE_NAME = 'youtube:user'
+ _VALID_URL = r'ytuser:(?P<id>.+)'
+ _TESTS = [{
+ 'url': 'ytuser:phihag',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ user_id = self._match_id(url)
+ return self.url_result(f'https://www.youtube.com/user/{user_id}', YoutubeTabIE, user_id)
+
+
+class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
+ IE_NAME = 'youtube:favorites'
+ IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
+ _VALID_URL = r':ytfav(?:ou?rite)?s?'
+ _LOGIN_REQUIRED = True
+ _TESTS = [{
+ 'url': ':ytfav',
+ 'only_matching': True,
+ }, {
+ 'url': ':ytfavorites',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ return self.url_result(
+ 'https://www.youtube.com/playlist?list=LL',
+ ie=YoutubeTabIE.ie_key())
+
+
+class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
+ """
+ Base class for feed extractors
+ Subclasses must re-define the _FEED_NAME property.
+ """
+ _LOGIN_REQUIRED = True
+ _FEED_NAME = 'feeds'
+
+ @classproperty
+ def IE_NAME(cls):
+ return f'youtube:{cls._FEED_NAME}'
+
+ def _real_extract(self, url):
+ return self.url_result(
+ f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
+
+
+class YoutubeWatchLaterIE(YoutubeBaseInfoExtractor):
+ IE_NAME = 'youtube:watchlater'
+ IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
+ _VALID_URL = r':ytwatchlater'
+ _TESTS = [{
+ 'url': ':ytwatchlater',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ return self.url_result(
+ 'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
+
+
+class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
+ IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
+ _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
+ _FEED_NAME = 'recommended'
+ _LOGIN_REQUIRED = False
+ _TESTS = [{
+ 'url': ':ytrec',
+ 'only_matching': True,
+ }, {
+ 'url': ':ytrecommended',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://youtube.com',
+ 'only_matching': True,
+ }]
+
+
+class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
+ IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
+ _VALID_URL = r':ytsub(?:scription)?s?'
+ _FEED_NAME = 'subscriptions'
+ _TESTS = [{
+ 'url': ':ytsubs',
+ 'only_matching': True,
+ }, {
+ 'url': ':ytsubscriptions',
+ 'only_matching': True,
+ }]
+
+
+class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
+ IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
+ _VALID_URL = r':ythis(?:tory)?'
+ _FEED_NAME = 'history'
+ _TESTS = [{
+ 'url': ':ythistory',
+ 'only_matching': True,
+ }]
+
+
+class YoutubeShortsAudioPivotIE(YoutubeBaseInfoExtractor):
+ IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
+ IE_NAME = 'youtube:shorts:pivot:audio'
+ _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
+ _TESTS = [{
+ 'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
+ 'only_matching': True,
+ }]
+
+ @staticmethod
+ def _generate_audio_pivot_params(video_id):
+ """
+ Generates sfv_audio_pivot browse params for this video id
+ """
+ pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
+ return urllib.parse.quote(base64.b64encode(pb_params).decode())
+
+ def _real_extract(self, url):
+ video_id = self._match_id(url)
+ return self.url_result(
+ f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
+ ie=YoutubeTabIE)
+
+
+class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
+ IE_NAME = 'youtube:consent'
+ IE_DESC = False # Do not list
+ _VALID_URL = r'https?://consent\.youtube\.com/m\?'
+ _TESTS = [{
+ 'url': 'https://consent.youtube.com/m?continue=https%3A%2F%2Fwww.youtube.com%2Flive%2FqVv6vCqciTM%3Fcbrd%3D1&gl=NL&m=0&pc=yt&hl=en&src=1',
+ 'info_dict': {
+ 'id': 'qVv6vCqciTM',
+ 'ext': 'mp4',
+ 'age_limit': 0,
+ 'uploader_id': '@sana_natori',
+ 'comment_count': int,
+ 'chapters': 'count:13',
+ 'upload_date': '20221223',
+ 'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
+ 'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
+ 'uploader_url': 'https://www.youtube.com/@sana_natori',
+ 'like_count': int,
+ 'release_date': '20221223',
+ 'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
+ 'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
+ 'view_count': int,
+ 'playable_in_embed': True,
+ 'duration': 4438,
+ 'availability': 'public',
+ 'channel_follower_count': int,
+ 'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
+ 'categories': ['Entertainment'],
+ 'live_status': 'was_live',
+ 'release_timestamp': 1671793345,
+ 'channel': 'さなちゃんねる',
+ 'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
+ 'uploader': 'さなちゃんねる',
+ 'channel_is_verified': True,
+ 'heatmap': 'count:100',
+ },
+ 'add_ie': ['Youtube'],
+ 'params': {'skip_download': 'Youtube'},
+ }]
+
+ def _real_extract(self, url):
+ redirect_url = url_or_none(parse_qs(url).get('continue', [None])[-1])
+ if not redirect_url:
+ raise ExtractorError('Invalid cookie consent redirect URL', expected=True)
+ return self.url_result(redirect_url)
diff --git a/yt_dlp/extractor/youtube/_search.py b/yt_dlp/extractor/youtube/_search.py
new file mode 100644
index 000000000..be10a20da
--- /dev/null
+++ b/yt_dlp/extractor/youtube/_search.py
@@ -0,0 +1,167 @@
+import urllib.parse
+
+from ._tab import YoutubeTabBaseInfoExtractor
+from ..common import SearchInfoExtractor
+from ...utils import join_nonempty, parse_qs
+
+
+class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
+ IE_DESC = 'YouTube search'
+ IE_NAME = 'youtube:search'
+ _SEARCH_KEY = 'ytsearch'
+ _SEARCH_PARAMS = 'EgIQAfABAQ==' # Videos only
+ _TESTS = [{
+ 'url': 'ytsearch5:youtube-dl test video',
+ 'playlist_count': 5,
+ 'info_dict': {
+ 'id': 'youtube-dl test video',
+ 'title': 'youtube-dl test video',
+ },
+ }, {
+ 'note': 'Suicide/self-harm search warning',
+ 'url': 'ytsearch1:i hate myself and i wanna die',
+ 'playlist_count': 1,
+ 'info_dict': {
+ 'id': 'i hate myself and i wanna die',
+ 'title': 'i hate myself and i wanna die',
+ },
+ }]
+
+
+class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
+ IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
+ _SEARCH_KEY = 'ytsearchdate'
+ IE_DESC = 'YouTube search, newest videos first'
+ _SEARCH_PARAMS = 'CAISAhAB8AEB' # Videos only, sorted by date
+ _TESTS = [{
+ 'url': 'ytsearchdate5:youtube-dl test video',
+ 'playlist_count': 5,
+ 'info_dict': {
+ 'id': 'youtube-dl test video',
+ 'title': 'youtube-dl test video',
+ },
+ }]
+
+
+class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
+ IE_DESC = 'YouTube search URLs with sorting and filter support'
+ IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
+ _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
+ _TESTS = [{
+ 'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
+ 'playlist_mincount': 5,
+ 'info_dict': {
+ 'id': 'youtube-dl test video',
+ 'title': 'youtube-dl test video',
+ },
+ }, {
+ 'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
+ 'playlist_mincount': 5,
+ 'info_dict': {
+ 'id': 'python',
+ 'title': 'python',
+ },
+ }, {
+ 'url': 'https://www.youtube.com/results?search_query=%23cats',
+ 'playlist_mincount': 1,
+ 'info_dict': {
+ 'id': '#cats',
+ 'title': '#cats',
+ # The test suite does not have support for nested playlists
+ # 'entries': [{
+ # 'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
+ # 'title': '#cats',
+ # }],
+ },
+ }, {
+ # Channel results
+ 'url': 'https://www.youtube.com/results?search_query=kurzgesagt&sp=EgIQAg%253D%253D',
+ 'info_dict': {
+ 'id': 'kurzgesagt',
+ 'title': 'kurzgesagt',
+ },
+ 'playlist': [{
+ 'info_dict': {
+ '_type': 'url',
+ 'id': 'UCsXVk37bltHxD1rDPwtNM8Q',
+ 'url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
+ 'ie_key': 'YoutubeTab',
+ 'channel': 'Kurzgesagt – In a Nutshell',
+ 'description': 'md5:4ae48dfa9505ffc307dad26342d06bfc',
+ 'title': 'Kurzgesagt – In a Nutshell',
+ 'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q',
+ # No longer available for search as it is set to the handle.
+ # 'playlist_count': int,
+ 'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
+ 'thumbnails': list,
+ 'uploader_id': '@kurzgesagt',
+ 'uploader_url': 'https://www.youtube.com/@kurzgesagt',
+ 'uploader': 'Kurzgesagt – In a Nutshell',
+ 'channel_is_verified': True,
+ 'channel_follower_count': int,
+ },
+ }],
+ 'params': {'extract_flat': True, 'playlist_items': '1'},
+ 'playlist_mincount': 1,
+ }, {
+ 'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
+ 'only_matching': True,
+ }]
+
+ def _real_extract(self, url):
+ qs = parse_qs(url)
+ query = (qs.get('search_query') or qs.get('q'))[0]
+ return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
+
+
+class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
+ IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
+ IE_NAME = 'youtube:music:search_url'
+ _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
+ _TESTS = [{
+ 'url': 'https://music.youtube.com/search?q=royalty+free+music',
+ 'playlist_count': 16,
+ 'info_dict': {
+ 'id': 'royalty free music',
+ 'title': 'royalty free music',
+ },
+ }, {
+ 'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
+ 'playlist_mincount': 30,
+ 'info_dict': {
+ 'id': 'royalty free music - songs',
+ 'title': 'royalty free music - songs',
+ },
+ 'params': {'extract_flat': 'in_playlist'},
+ }, {
+ 'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
+ 'playlist_mincount': 30,
+ 'info_dict': {
+ 'id': 'royalty free music - community playlists',
+ 'title': 'royalty free music - community playlists',
+ },
+ 'params': {'extract_flat': 'in_playlist'},
+ }]
+
+ _SECTIONS = {
+ 'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
+ 'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
+ 'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
+ 'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
+ 'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
+ 'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
+ }
+
+ def _real_extract(self, url):
+ qs = parse_qs(url)
+ query = (qs.get('search_query') or qs.get('q'))[0]
+ params = qs.get('sp', (None,))[0]
+ if params:
+ section = next((k for k, v in self._SECTIONS.items() if v == params), params)
+ else:
+ section = urllib.parse.unquote_plus(([*url.split('#'), ''])[1]).lower()
+ params = self._SECTIONS.get(section)
+ if not params:
+ section = None
+ title = join_nonempty(query, section, delim=' - ')
+ return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
diff --git a/yt_dlp/extractor/youtube/_tab.py b/yt_dlp/extractor/youtube/_tab.py
new file mode 100644
index 000000000..122300e60
--- /dev/null
+++ b/yt_dlp/extractor/youtube/_tab.py
@@ -0,0 +1,2348 @@
+import functools
+import itertools
+import re
+import shlex
+import urllib.parse
+
+from ._base import BadgeType, YoutubeBaseInfoExtractor
+from ._video import YoutubeIE
+from ...networking.exceptions import HTTPError, network_exceptions
+from ...utils import (
+ NO_DEFAULT,
+ ExtractorError,
+ UserNotLive,
+ bug_reports_message,
+ format_field,
+ get_first,
+ int_or_none,
+ parse_count,
+ parse_duration,
+ parse_qs,
+ smuggle_url,
+ str_to_int,
+ strftime_or_none,
+ traverse_obj,
+ try_get,
+ unsmuggle_url,
+ update_url_query,
+ url_or_none,
+ urljoin,
+ variadic,
+)
+
+
+class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
+ @staticmethod
+ def passthrough_smuggled_data(func):
+ def _smuggle(info, smuggled_data):
+ if info.get('_type') not in ('url', 'url_transparent'):
+ return info
+ if smuggled_data.get('is_music_url'):
+ parsed_url = urllib.parse.urlparse(info['url'])
+ if parsed_url.netloc in ('www.youtube.com', 'music.youtube.com'):
+ smuggled_data.pop('is_music_url')
+ info['url'] = urllib.parse.urlunparse(parsed_url._replace(netloc='music.youtube.com'))
+ if smuggled_data:
+ info['url'] = smuggle_url(info['url'], smuggled_data)
+ return info
+
+ @functools.wraps(func)
+ def wrapper(self, url):
+ url, smuggled_data = unsmuggle_url(url, {})
+ if self.is_music_url(url):
+ smuggled_data['is_music_url'] = True
+ info_dict = func(self, url, smuggled_data)
+ if smuggled_data:
+ _smuggle(info_dict, smuggled_data)
+ if info_dict.get('entries'):
+ info_dict['entries'] = (_smuggle(i, smuggled_data.copy()) for i in info_dict['entries'])
+ return info_dict
+ return wrapper
+
+ @staticmethod
+ def _extract_basic_item_renderer(item):
+ # Modified from _extract_grid_item_renderer
+ known_basic_renderers = (
+ 'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer',
+ )
+ for key, renderer in item.items():
+ if not isinstance(renderer, dict):
+ continue
+ elif key in known_basic_renderers:
+ return renderer
+ elif key.startswith('grid') and key.endswith('Renderer'):
+ return renderer
+
+ def _extract_video(self, renderer):
+ video_id = renderer.get('videoId')
+
+ reel_header_renderer = traverse_obj(renderer, (
+ 'navigationEndpoint', 'reelWatchEndpoint', 'overlay', 'reelPlayerOverlayRenderer',
+ 'reelPlayerHeaderSupportedRenderers', 'reelPlayerHeaderRenderer'))
+
+ title = self._get_text(renderer, 'title', 'headline') or self._get_text(reel_header_renderer, 'reelTitleText')
+ description = self._get_text(renderer, 'descriptionSnippet')
+
+ duration = int_or_none(renderer.get('lengthSeconds'))
+ if duration is None:
+ duration = parse_duration(self._get_text(
+ renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
+ if duration is None:
+ # XXX: should write a parser to be more general to support more cases (e.g. shorts in shorts tab)
+ duration = parse_duration(self._search_regex(
+ r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
+ traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
+ video_id, default=None, group='duration'))
+
+ channel_id = traverse_obj(
+ renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
+ expected_type=str, get_all=False)
+ if not channel_id:
+ channel_id = traverse_obj(reel_header_renderer, ('channelNavigationEndpoint', 'browseEndpoint', 'browseId'))
+
+ channel_id = self.ucid_or_none(channel_id)
+
+ overlay_style = traverse_obj(
+ renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
+ get_all=False, expected_type=str)
+ badges = self._extract_badges(traverse_obj(renderer, 'badges'))
+ owner_badges = self._extract_badges(traverse_obj(renderer, 'ownerBadges'))
+ navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
+ renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
+ expected_type=str)) or ''
+ url = f'https://www.youtube.com/watch?v={video_id}'
+ if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
+ url = f'https://www.youtube.com/shorts/{video_id}'
+
+ time_text = (self._get_text(renderer, 'publishedTimeText', 'videoInfo')
+ or self._get_text(reel_header_renderer, 'timestampText') or '')
+ scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
+
+ live_status = (
+ 'is_upcoming' if scheduled_timestamp is not None
+ else 'was_live' if 'streamed' in time_text.lower()
+ else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
+ else None)
+
+ # videoInfo is a string like '50K views • 10 years ago'.
+ view_count_text = self._get_text(renderer, 'viewCountText', 'shortViewCountText', 'videoInfo') or ''
+ view_count = (0 if 'no views' in view_count_text.lower()
+ else self._get_count({'simpleText': view_count_text}))
+ view_count_field = 'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count'
+
+ channel = (self._get_text(renderer, 'ownerText', 'shortBylineText')
+ or self._get_text(reel_header_renderer, 'channelTitleText'))
+
+ channel_handle = traverse_obj(renderer, (
+ 'shortBylineText', 'runs', ..., 'navigationEndpoint',
+ (('commandMetadata', 'webCommandMetadata', 'url'), ('browseEndpoint', 'canonicalBaseUrl'))),
+ expected_type=self.handle_from_url, get_all=False)
+ return {
+ '_type': 'url',
+ 'ie_key': YoutubeIE.ie_key(),
+ 'id': video_id,
+ 'url': url,
+ 'title': title,
+ 'description': description,
+ 'duration': duration,
+ 'channel_id': channel_id,
+ 'channel': channel,
+ 'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
+ 'uploader': channel,
+ 'uploader_id': channel_handle,
+ 'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
+ 'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
+ 'timestamp': (self._parse_time_text(time_text)
+ if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
+ else None),
+ 'release_timestamp': scheduled_timestamp,
+ 'availability':
+ 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
+ else self._availability(
+ is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
+ needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
+ needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
+ is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
+ view_count_field: view_count,
+ 'live_status': live_status,
+ 'channel_is_verified': True if self._has_badge(owner_badges, BadgeType.VERIFIED) else None,
+ }
+
+ def _extract_channel_renderer(self, renderer):
+ channel_id = self.ucid_or_none(renderer['channelId'])
+ title = self._get_text(renderer, 'title')
+ channel_url = format_field(channel_id, None, 'https://www.youtube.com/channel/%s', default=None)
+ channel_handle = self.handle_from_url(
+ traverse_obj(renderer, (
+ 'navigationEndpoint', (('commandMetadata', 'webCommandMetadata', 'url'),
+ ('browseEndpoint', 'canonicalBaseUrl')),
+ {str}), get_all=False))
+ if not channel_handle:
+ # As of 2023-06-01, YouTube sets subscriberCountText to the handle in search
+ channel_handle = self.handle_or_none(self._get_text(renderer, 'subscriberCountText'))
+ return {
+ '_type': 'url',
+ 'url': channel_url,
+ 'id': channel_id,
+ 'ie_key': YoutubeTabIE.ie_key(),
+ 'channel': title,
+ 'uploader': title,
+ 'channel_id': channel_id,
+ 'channel_url': channel_url,
+ 'title': title,
+ 'uploader_id': channel_handle,
+ 'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
+ # See above. YouTube sets videoCountText to the subscriber text in search channel renderers.
+ # However, in feed/channels this is set correctly to the subscriber count
+ 'channel_follower_count': traverse_obj(
+ renderer, 'subscriberCountText', 'videoCountText', expected_type=self._get_count),
+ 'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
+ 'playlist_count': (
+ # videoCountText may be the subscriber count
+ self._get_count(renderer, 'videoCountText')
+ if self._get_count(renderer, 'subscriberCountText') is not None else None),
+ 'description': self._get_text(renderer, 'descriptionSnippet'),
+ 'channel_is_verified': True if self._has_badge(
+ self._extract_badges(traverse_obj(renderer, 'ownerBadges')), BadgeType.VERIFIED) else None,
+ }
+
+ def _grid_entries(self, grid_renderer):
+ for item in grid_renderer['items']:
+ if not isinstance(item, dict):
+ continue
+ if lockup_view_model := traverse_obj(item, ('lockupViewModel', {dict})):
+ if entry := self._extract_lockup_view_model(lockup_view_model):
+ yield entry
+ continue
+ renderer = self._extract_basic_item_renderer(item)
+ if not isinstance(renderer, dict):
+ continue
+ title = self._get_text(renderer, 'title')
+
+ # playlist
+ playlist_id = renderer.get('playlistId')
+ if playlist_id:
+ yield self.url_result(
+ f'https://www.youtube.com/playlist?list={playlist_id}',
+ ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
+ video_title=title)
+ continue
+ # video
+ video_id = renderer.get('videoId')
+ if video_id:
+ yield self._extract_video(renderer)
+ continue
+ # channel
+ channel_id = renderer.get('channelId')
+ if channel_id:
+ yield self._extract_channel_renderer(renderer)
+ continue
+ # generic endpoint URL support
+ ep_url = urljoin('https://www.youtube.com/', try_get(
+ renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
+ str))
+ if ep_url:
+
+ for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
+ if ie.suitable(ep_url):
+ yield self.url_result(
+ ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
+ break
+
+ def _music_reponsive_list_entry(self, renderer):
+ video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
+ if video_id:
+ title = traverse_obj(renderer, (
+ 'flexColumns', 0, 'musicResponsiveListItemFlexColumnRenderer',
+ 'text', 'runs', 0, 'text'))
+ return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
+ ie=YoutubeIE.ie_key(), video_id=video_id, title=title)
+ playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
+ if playlist_id:
+ video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
+ if video_id:
+ return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
+ ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
+ return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
+ ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
+ browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
+ if browse_id:
+ return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
+ ie=YoutubeTabIE.ie_key(), video_id=browse_id)
+
+ def _shelf_entries_from_content(self, shelf_renderer):
+ content = shelf_renderer.get('content')
+ if not isinstance(content, dict):
+ return
+ renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
+ if renderer:
+ # TODO: add support for nested playlists so each shelf is processed
+ # as separate playlist
+ # TODO: this includes only first N items
+ yield from self._grid_entries(renderer)
+ renderer = content.get('horizontalListRenderer')
+ if renderer:
+ # TODO: handle case
+ pass
+
+ def _shelf_entries(self, shelf_renderer, skip_channels=False):
+ ep = try_get(
+ shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
+ str)
+ shelf_url = urljoin('https://www.youtube.com', ep)
+ if shelf_url:
+ # Skipping links to another channels, note that checking for
+ # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
+ # will not work
+ if skip_channels and '/channels?' in shelf_url:
+ return
+ title = self._get_text(shelf_renderer, 'title')
+ yield self.url_result(shelf_url, video_title=title)
+ # Shelf may not contain shelf URL, fallback to extraction from content
+ yield from self._shelf_entries_from_content(shelf_renderer)
+
+ def _playlist_entries(self, video_list_renderer):
+ for content in video_list_renderer['contents']:
+ if not isinstance(content, dict):
+ continue
+ renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
+ if not isinstance(renderer, dict):
+ continue
+ video_id = renderer.get('videoId')
+ if not video_id:
+ continue
+ yield self._extract_video(renderer)
+
+ def _extract_lockup_view_model(self, view_model):
+ content_id = view_model.get('contentId')
+ if not content_id:
+ return
+ content_type = view_model.get('contentType')
+ if content_type not in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'):
+ self.report_warning(
+ f'Unsupported lockup view model content type "{content_type}"{bug_reports_message()}', only_once=True)
+ return
+ return self.url_result(
+ f'https://www.youtube.com/playlist?list={content_id}', ie=YoutubeTabIE, video_id=content_id,
+ title=traverse_obj(view_model, (
+ 'metadata', 'lockupMetadataViewModel', 'title', 'content', {str})),
+ thumbnails=self._extract_thumbnails(view_model, (
+ 'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail', 'thumbnailViewModel', 'image'), final_key='sources'))
+
+ def _rich_entries(self, rich_grid_renderer):
+ if lockup_view_model := traverse_obj(rich_grid_renderer, ('content', 'lockupViewModel', {dict})):
+ if entry := self._extract_lockup_view_model(lockup_view_model):
+ yield entry
+ return
+ renderer = traverse_obj(
+ rich_grid_renderer,
+ ('content', ('videoRenderer', 'reelItemRenderer', 'playlistRenderer', 'shortsLockupViewModel'), any)) or {}
+ video_id = renderer.get('videoId')
+ if video_id:
+ yield self._extract_video(renderer)
+ return
+ playlist_id = renderer.get('playlistId')
+ if playlist_id:
+ yield self.url_result(
+ f'https://www.youtube.com/playlist?list={playlist_id}',
+ ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
+ video_title=self._get_text(renderer, 'title'))
+ return
+ # shortsLockupViewModel extraction
+ entity_id = renderer.get('entityId')
+ if entity_id:
+ video_id = traverse_obj(renderer, ('onTap', 'innertubeCommand', 'reelWatchEndpoint', 'videoId', {str}))
+ if not video_id:
+ return
+ yield self.url_result(
+ f'https://www.youtube.com/shorts/{video_id}',
+ ie=YoutubeIE, video_id=video_id,
+ **traverse_obj(renderer, {
+ 'title': ((
+ ('overlayMetadata', 'primaryText', 'content', {str}),
+ ('accessibilityText', {lambda x: re.fullmatch(r'(.+), (?:[\d,.]+(?:[KM]| million)?|No) views? - play Short', x)}, 1)), any),
+ 'view_count': ('overlayMetadata', 'secondaryText', 'content', {parse_count}),
+ }),
+ thumbnails=self._extract_thumbnails(renderer, 'thumbnail', final_key='sources'))
+ return
+
+ def _video_entry(self, video_renderer):
+ video_id = video_renderer.get('videoId')
+ if video_id:
+ return self._extract_video(video_renderer)
+
+ def _hashtag_tile_entry(self, hashtag_tile_renderer):
+ url = urljoin('https://youtube.com', traverse_obj(
+ hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
+ if url:
+ return self.url_result(
+ url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
+
+ def _post_thread_entries(self, post_thread_renderer):
+ post_renderer = try_get(
+ post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
+ if not post_renderer:
+ return
+ # video attachment
+ video_renderer = try_get(
+ post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
+ video_id = video_renderer.get('videoId')
+ if video_id:
+ entry = self._extract_video(video_renderer)
+ if entry:
+ yield entry
+ # playlist attachment
+ playlist_id = try_get(
+ post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
+ if playlist_id:
+ yield self.url_result(
+ f'https://www.youtube.com/playlist?list={playlist_id}',
+ ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
+ # inline video links
+ runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
+ for run in runs:
+ if not isinstance(run, dict):
+ continue
+ ep_url = try_get(
+ run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
+ if not ep_url:
+ continue
+ if not YoutubeIE.suitable(ep_url):
+ continue
+ ep_video_id = YoutubeIE._match_id(ep_url)
+ if video_id == ep_video_id:
+ continue
+ yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
+
+ def _post_thread_continuation_entries(self, post_thread_continuation):
+ contents = post_thread_continuation.get('contents')
+ if not isinstance(contents, list):
+ return
+ for content in contents:
+ renderer = content.get('backstagePostThreadRenderer')
+ if isinstance(renderer, dict):
+ yield from self._post_thread_entries(renderer)
+ continue
+ renderer = content.get('videoRenderer')
+ if isinstance(renderer, dict):
+ yield self._video_entry(renderer)
+
+ r''' # unused
+ def _rich_grid_entries(self, contents):
+ for content in contents:
+ video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
+ if video_renderer:
+ entry = self._video_entry(video_renderer)
+ if entry:
+ yield entry
+ '''
+
+ def _report_history_entries(self, renderer):
+ for url in traverse_obj(renderer, (
+ 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
+ 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
+ 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
+ yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
+
+ def _extract_entries(self, parent_renderer, continuation_list):
+ # continuation_list is modified in-place with continuation_list = [continuation_token]
+ continuation_list[:] = [None]
+ contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
+ for content in contents:
+ if not isinstance(content, dict):
+ continue
+ is_renderer = traverse_obj(
+ content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
+ expected_type=dict)
+ if not is_renderer:
+ if content.get('richItemRenderer'):
+ for entry in self._rich_entries(content['richItemRenderer']):
+ yield entry
+ continuation_list[0] = self._extract_continuation(parent_renderer)
+ elif content.get('reportHistorySectionRenderer'): # https://www.youtube.com/reporthistory
+ table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
+ yield from self._report_history_entries(table)
+ continuation_list[0] = self._extract_continuation(table)
+ continue
+
+ isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
+ for isr_content in isr_contents:
+ if not isinstance(isr_content, dict):
+ continue
+
+ known_renderers = {
+ 'playlistVideoListRenderer': self._playlist_entries,
+ 'gridRenderer': self._grid_entries,
+ 'reelShelfRenderer': self._grid_entries,
+ 'shelfRenderer': self._shelf_entries,
+ 'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
+ 'backstagePostThreadRenderer': self._post_thread_entries,
+ 'videoRenderer': lambda x: [self._video_entry(x)],
+ 'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
+ 'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
+ 'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)],
+ 'richGridRenderer': lambda x: self._extract_entries(x, continuation_list),
+ 'lockupViewModel': lambda x: [self._extract_lockup_view_model(x)],
+ }
+ for key, renderer in isr_content.items():
+ if key not in known_renderers:
+ continue
+ for entry in known_renderers[key](renderer):
+ if entry:
+ yield entry
+ continuation_list[0] = self._extract_continuation(renderer)
+ break
+
+ if not continuation_list[0]:
+ continuation_list[0] = self._extract_continuation(is_renderer)
+
+ if not continuation_list[0]:
+ continuation_list[0] = self._extract_continuation(parent_renderer)
+
+ def _entries(self, tab, item_id, ytcfg, delegated_session_id, visitor_data):
+ continuation_list = [None]
+ extract_entries = lambda x: self._extract_entries(x, continuation_list)
+ tab_content = try_get(tab, lambda x: x['content'], dict)
+ if not tab_content:
+ return
+ parent_renderer = (
+ try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
+ or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
+ yield from extract_entries(parent_renderer)
+ continuation = continuation_list[0]
+ seen_continuations = set()
+ for page_num in itertools.count(1):
+ if not continuation:
+ break
+ continuation_token = continuation.get('continuation')
+ if continuation_token is not None and continuation_token in seen_continuations:
+ self.write_debug('Detected YouTube feed looping - assuming end of feed.')
+ break
+ seen_continuations.add(continuation_token)
+ headers = self.generate_api_headers(
+ ytcfg=ytcfg, delegated_session_id=delegated_session_id, visitor_data=visitor_data)
+ response = self._extract_response(
+ item_id=f'{item_id} page {page_num}',
+ query=continuation, headers=headers, ytcfg=ytcfg,
+ check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
+
+ if not response:
+ break
+ # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
+ # See: https://github.com/ytdl-org/youtube-dl/issues/28702
+ visitor_data = self._extract_visitor_data(response) or visitor_data
+
+ known_renderers = {
+ 'videoRenderer': (self._grid_entries, 'items'), # for membership tab
+ 'gridPlaylistRenderer': (self._grid_entries, 'items'),
+ 'gridVideoRenderer': (self._grid_entries, 'items'),
+ 'gridChannelRenderer': (self._grid_entries, 'items'),
+ 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
+ 'itemSectionRenderer': (extract_entries, 'contents'), # for feeds
+ 'richItemRenderer': (extract_entries, 'contents'), # for hashtag
+ 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
+ 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
+ 'playlistVideoListContinuation': (self._playlist_entries, None),
+ 'gridContinuation': (self._grid_entries, None),
+ 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
+ 'sectionListContinuation': (extract_entries, None), # for feeds
+ }
+
+ continuation_items = traverse_obj(response, (
+ ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
+ 'appendContinuationItemsAction', 'continuationItems',
+ ), 'continuationContents', get_all=False)
+ continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
+
+ video_items_renderer = None
+ for key in continuation_item:
+ if key not in known_renderers:
+ continue
+ func, parent_key = known_renderers[key]
+ video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
+ continuation_list = [None]
+ yield from func(video_items_renderer)
+ continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
+
+ if not video_items_renderer:
+ break
+
+ @staticmethod
+ def _extract_selected_tab(tabs, fatal=True):
+ for tab_renderer in tabs:
+ if tab_renderer.get('selected'):
+ return tab_renderer
+ if fatal:
+ raise ExtractorError('Unable to find selected tab')
+
+ @staticmethod
+ def _extract_tab_renderers(response):
+ return traverse_obj(
+ response, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., ('tabRenderer', 'expandableTabRenderer')), expected_type=dict)
+
+ def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
+ metadata = self._extract_metadata_from_tabs(item_id, data)
+
+ selected_tab = self._extract_selected_tab(tabs)
+ metadata['title'] += format_field(selected_tab, 'title', ' - %s')
+ metadata['title'] += format_field(selected_tab, 'expandedText', ' - %s')
+
+ return self.playlist_result(
+ self._entries(
+ selected_tab, metadata['id'], ytcfg,
+ self._extract_delegated_session_id(ytcfg, data),
+ self._extract_visitor_data(data, ytcfg)),
+ **metadata)
+
+ def _extract_metadata_from_tabs(self, item_id, data):
+ info = {'id': item_id}
+
+ metadata_renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'), expected_type=dict)
+ if metadata_renderer:
+ channel_id = traverse_obj(metadata_renderer, ('externalId', {self.ucid_or_none}),
+ ('channelUrl', {self.ucid_from_url}))
+ info.update({
+ 'channel': metadata_renderer.get('title'),
+ 'channel_id': channel_id,
+ })
+ if info['channel_id']:
+ info['id'] = info['channel_id']
+ else:
+ metadata_renderer = traverse_obj(data, ('metadata', 'playlistMetadataRenderer'), expected_type=dict)
+
+ # pageHeaderViewModel slow rollout began April 2024
+ page_header_view_model = traverse_obj(data, (
+ 'header', 'pageHeaderRenderer', 'content', 'pageHeaderViewModel', {dict}))
+
+ # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
+ # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
+ def _get_uncropped(url):
+ return url_or_none((url or '').split('=')[0] + '=s0')
+
+ avatar_thumbnails = self._extract_thumbnails(metadata_renderer, 'avatar')
+ if avatar_thumbnails:
+ uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
+ if uncropped_avatar:
+ avatar_thumbnails.append({
+ 'url': uncropped_avatar,
+ 'id': 'avatar_uncropped',
+ 'preference': 1,
+ })
+
+ channel_banners = (
+ self._extract_thumbnails(data, ('header', ..., ('banner', 'mobileBanner', 'tvBanner')))
+ or self._extract_thumbnails(
+ page_header_view_model, ('banner', 'imageBannerViewModel', 'image'), final_key='sources'))
+ for banner in channel_banners:
+ banner['preference'] = -10
+
+ if channel_banners:
+ uncropped_banner = _get_uncropped(channel_banners[0]['url'])
+ if uncropped_banner:
+ channel_banners.append({
+ 'url': uncropped_banner,
+ 'id': 'banner_uncropped',
+ 'preference': -5,
+ })
+
+ # Deprecated - remove primary_sidebar_renderer when layout discontinued
+ primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
+ playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer'), expected_type=dict)
+
+ primary_thumbnails = self._extract_thumbnails(
+ primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
+ playlist_thumbnails = self._extract_thumbnails(
+ playlist_header_renderer, ('playlistHeaderBanner', 'heroPlaylistThumbnailRenderer', 'thumbnail'))
+
+ info.update({
+ 'title': (traverse_obj(metadata_renderer, 'title')
+ or self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag'))
+ or info['id']),
+ 'availability': self._extract_availability(data),
+ 'channel_follower_count': (
+ self._get_count(data, ('header', ..., 'subscriberCountText'))
+ or traverse_obj(page_header_view_model, (
+ 'metadata', 'contentMetadataViewModel', 'metadataRows', ..., 'metadataParts',
+ lambda _, v: 'subscribers' in v['text']['content'], 'text', 'content', {parse_count}, any))),
+ 'description': try_get(metadata_renderer, lambda x: x.get('description', '')),
+ 'tags': (traverse_obj(data, ('microformat', 'microformatDataRenderer', 'tags', ..., {str}))
+ or traverse_obj(metadata_renderer, ('keywords', {lambda x: x and shlex.split(x)}, ...))),
+ 'thumbnails': (primary_thumbnails or playlist_thumbnails) + avatar_thumbnails + channel_banners,
+ })
+
+ channel_handle = (
+ traverse_obj(metadata_renderer, (('vanityChannelUrl', ('ownerUrls', ...)), {self.handle_from_url}), get_all=False)
+ or traverse_obj(data, ('header', ..., 'channelHandleText', {self.handle_or_none}), get_all=False))
+
+ if channel_handle:
+ info.update({
+ 'uploader_id': channel_handle,
+ 'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
+ })
+
+ channel_badges = self._extract_badges(traverse_obj(data, ('header', ..., 'badges'), get_all=False))
+ if self._has_badge(channel_badges, BadgeType.VERIFIED):
+ info['channel_is_verified'] = True
+ # Playlist stats is a text runs array containing [video count, view count, last updated].
+ # last updated or (view count and last updated) may be missing.
+ playlist_stats = get_first(
+ (primary_sidebar_renderer, playlist_header_renderer), (('stats', 'briefStats', 'numVideosText'), ))
+
+ last_updated_unix = self._parse_time_text(
+ self._get_text(playlist_stats, 2) # deprecated, remove when old layout discontinued
+ or self._get_text(playlist_header_renderer, ('byline', 1, 'playlistBylineRenderer', 'text')))
+ info['modified_date'] = strftime_or_none(last_updated_unix)
+
+ info['view_count'] = self._get_count(playlist_stats, 1)
+ if info['view_count'] is None: # 0 is allowed
+ info['view_count'] = self._get_count(playlist_header_renderer, 'viewCountText')
+ if info['view_count'] is None:
+ info['view_count'] = self._get_count(data, (
+ 'contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., 'tabRenderer', 'content', 'sectionListRenderer',
+ 'contents', ..., 'itemSectionRenderer', 'contents', ..., 'channelAboutFullMetadataRenderer', 'viewCountText'))
+
+ info['playlist_count'] = self._get_count(playlist_stats, 0)
+ if info['playlist_count'] is None: # 0 is allowed
+ info['playlist_count'] = self._get_count(playlist_header_renderer, ('byline', 0, 'playlistBylineRenderer', 'text'))
+
+ if not info.get('channel_id'):
+ owner = traverse_obj(playlist_header_renderer, 'ownerText')
+ if not owner: # Deprecated
+ owner = traverse_obj(
+ self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer'),
+ ('videoOwner', 'videoOwnerRenderer', 'title'))
+ owner_text = self._get_text(owner)
+ browse_ep = traverse_obj(owner, ('runs', 0, 'navigationEndpoint', 'browseEndpoint')) or {}
+ info.update({
+ 'channel': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
+ 'channel_id': self.ucid_or_none(browse_ep.get('browseId')),
+ 'uploader_id': self.handle_from_url(urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl'))),
+ })
+
+ info.update({
+ 'uploader': info['channel'],
+ 'channel_url': format_field(info.get('channel_id'), None, 'https://www.youtube.com/channel/%s', default=None),
+ 'uploader_url': format_field(info.get('uploader_id'), None, 'https://www.youtube.com/%s', default=None),
+ })
+
+ return info
+
+ def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
+ first_id = last_id = response = None
+ for page_num in itertools.count(1):
+ videos = list(self._playlist_entries(playlist))
+ if not videos:
+ return
+ start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
+ if start >= len(videos):
+ return
+ yield from videos[start:]
+ first_id = first_id or videos[0]['id']
+ last_id = videos[-1]['id']
+ watch_endpoint = try_get(
+ playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
+ headers = self.generate_api_headers(
+ ytcfg=ytcfg, delegated_session_id=self._extract_delegated_session_id(ytcfg, data),
+ visitor_data=self._extract_visitor_data(response, data, ytcfg))
+ query = {
+ 'playlistId': playlist_id,
+ 'videoId': watch_endpoint.get('videoId') or last_id,
+ 'index': watch_endpoint.get('index') or len(videos),
+ 'params': watch_endpoint.get('params') or 'OAE%3D',
+ }
+ response = self._extract_response(
+ item_id=f'{playlist_id} page {page_num}',
+ query=query, ep='next', headers=headers, ytcfg=ytcfg,
+ check_get_keys='contents',
+ )
+ playlist = try_get(
+ response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
+
+ def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
+ title = playlist.get('title') or try_get(
+ data, lambda x: x['titleText']['simpleText'], str)
+ playlist_id = playlist.get('playlistId') or item_id
+
+ # Delegating everything except mix playlists to regular tab-based playlist URL
+ playlist_url = urljoin(url, try_get(
+ playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
+ str))
+
+ # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
+ # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
+ is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
+
+ if playlist_url and playlist_url != url and not is_known_unviewable:
+ return self.url_result(
+ playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
+ video_title=title)
+
+ return self.playlist_result(
+ self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
+ playlist_id=playlist_id, playlist_title=title)
+
+ def _extract_availability(self, data):
+ """
+ Gets the availability of a given playlist/tab.
+ Note: Unless YouTube tells us explicitly, we do not assume it is public
+ @param data: response
+ """
+ sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
+ playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
+ player_header_privacy = playlist_header_renderer.get('privacy')
+
+ badges = self._extract_badges(traverse_obj(sidebar_renderer, 'badges'))
+
+ # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
+ privacy_setting_icon = get_first(
+ (playlist_header_renderer, sidebar_renderer),
+ ('privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
+ lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
+ expected_type=str)
+
+ microformats_is_unlisted = traverse_obj(
+ data, ('microformat', 'microformatDataRenderer', 'unlisted'), expected_type=bool)
+
+ return (
+ 'public' if (
+ self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
+ or player_header_privacy == 'PUBLIC'
+ or privacy_setting_icon == 'PRIVACY_PUBLIC')
+ else self._availability(
+ is_private=(
+ self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
+ or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
+ else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
+ is_unlisted=(
+ self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
+ or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
+ else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None
+ else microformats_is_unlisted if microformats_is_unlisted is not None else None),
+ needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
+ needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
+ needs_auth=False))
+
+ @staticmethod
+ def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
+ sidebar_renderer = try_get(
+ data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
+ for item in sidebar_renderer:
+ renderer = try_get(item, lambda x: x[info_renderer], expected_type)
+ if renderer:
+ return renderer
+
+ def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
+ """
+ Reload playlists with unavailable videos (e.g. private videos, region blocked, etc.)
+ """
+ is_playlist = bool(traverse_obj(
+ data, ('metadata', 'playlistMetadataRenderer'), ('header', 'playlistHeaderRenderer')))
+ if not is_playlist:
+ return
+ headers = self.generate_api_headers(
+ ytcfg=ytcfg, delegated_session_id=self._extract_delegated_session_id(ytcfg, data),
+ visitor_data=self._extract_visitor_data(data, ytcfg))
+ query = {
+ 'params': 'wgYCCAA=',
+ 'browseId': f'VL{item_id}',
+ }
+ return self._extract_response(
+ item_id=item_id, headers=headers, query=query,
+ check_get_keys='contents', fatal=False, ytcfg=ytcfg,
+ note='Redownloading playlist API JSON with unavailable videos')
+
+ @functools.cached_property
+ def skip_webpage(self):
+ return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
+
+ def _extract_webpage(self, url, item_id, fatal=True):
+ webpage, data = None, None
+ for retry in self.RetryManager(fatal=fatal):
+ try:
+ webpage = self._download_webpage(url, item_id, note='Downloading webpage')
+ data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
+ except ExtractorError as e:
+ if isinstance(e.cause, network_exceptions):
+ if not isinstance(e.cause, HTTPError) or e.cause.status not in (403, 429):
+ retry.error = e
+ continue
+ self._error_or_warning(e, fatal=fatal)
+ break
+
+ try:
+ self._extract_and_report_alerts(data)
+ except ExtractorError as e:
+ self._error_or_warning(e, fatal=fatal)
+ break
+
+ # Sometimes youtube returns a webpage with incomplete ytInitialData
+ # See: https://github.com/yt-dlp/yt-dlp/issues/116
+ if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
+ retry.error = ExtractorError('Incomplete yt initial data received')
+ data = None
+ continue
+
+ return webpage, data
+
+ def _report_playlist_authcheck(self, ytcfg, fatal=True):
+ """Use if failed to extract ytcfg (and data) from initial webpage"""
+ if not ytcfg and self.is_authenticated:
+ msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
+ if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
+ raise ExtractorError(
+ f'{msg}. If you are not downloading private content, or '
+ 'your cookies are only for the first account and channel,'
+ ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
+ expected=True)
+ self.report_warning(msg, only_once=True)
+
+ def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
+ data = None
+ if not self.skip_webpage:
+ webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
+ ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
+ # Reject webpage data if redirected to home page without explicitly requesting
+ selected_tab = self._extract_selected_tab(self._extract_tab_renderers(data), fatal=False) or {}
+ if (url != 'https://www.youtube.com/feed/recommended'
+ and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch' # Home page
+ and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
+ msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
+ if fatal:
+ raise ExtractorError(msg, expected=True)
+ self.report_warning(msg, only_once=True)
+ if not data:
+ self._report_playlist_authcheck(ytcfg, fatal=fatal)
+ data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
+ return data, ytcfg
+
+ def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
+ headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
+ resolve_response = self._extract_response(
+ item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
+ ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
+ endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
+ for ep_key, ep in endpoints.items():
+ params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
+ if params:
+ return self._extract_response(
+ item_id=item_id, query=params, ep=ep, headers=headers,
+ ytcfg=ytcfg, fatal=fatal, default_client=default_client,
+ check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
+ err_note = 'Failed to resolve url (does the playlist exist?)'
+ if fatal:
+ raise ExtractorError(err_note, expected=True)
+ self.report_warning(err_note, item_id)
+
+ _SEARCH_PARAMS = None
+
+ def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
+ data = {'query': query}
+ if params is NO_DEFAULT:
+ params = self._SEARCH_PARAMS
+ if params:
+ data['params'] = params
+
+ content_keys = (
+ ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
+ ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
+ # ytmusic search
+ ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
+ ('continuationContents', ),
+ )
+ display_id = f'query "{query}"'
+ check_get_keys = tuple({keys[0] for keys in content_keys})
+ ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
+ self._report_playlist_authcheck(ytcfg, fatal=False)
+
+ continuation_list = [None]
+ search = None
+ for page_num in itertools.count(1):
+ data.update(continuation_list[0] or {})
+ headers = self.generate_api_headers(
+ ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
+ search = self._extract_response(
+ item_id=f'{display_id} page {page_num}', ep='search', query=data,
+ default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
+ slr_contents = traverse_obj(search, *content_keys)
+ yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
+ if not continuation_list[0]:
+ break
+
+
+class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
+ IE_DESC = 'YouTube Tabs'
+ _VALID_URL = r'''(?x:
+ https?://
+ (?!consent\.)(?:\w+\.)?
+ (?:
+ youtube(?:kids)?\.com|
+ {invidious}
+ )/
+ (?:
+ (?P<channel_type>channel|c|user|browse)/|
+ (?P<not_channel>
+ feed/|hashtag/|
+ (?:playlist|watch)\?.*?\blist=
+ )|
+ (?!(?:{reserved_names})\b) # Direct URLs
+ )
+ (?P<id>[^/?\#&]+)
+ )'''.format(
+ reserved_names=YoutubeBaseInfoExtractor._RESERVED_NAMES,
+ invidious='|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
+ )
+ IE_NAME = 'youtube:tab'
+
+ _TESTS = [{
+ 'note': 'playlists, multipage',
+ 'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
+ 'playlist_mincount': 94,
+ 'info_dict': {
+ 'id': 'UCqj7Cz7revf5maW9g5pgNcg',
+ 'title': 'Igor Kleiner Ph.D. - Playlists',
+ 'description': 'md5:15d7dd9e333cb987907fcb0d604b233a',
+ 'uploader': 'Igor Kleiner Ph.D.',
+ 'uploader_id': '@IgorDataScience',
+ 'uploader_url': 'https://www.youtube.com/@IgorDataScience',
+ 'channel': 'Igor Kleiner Ph.D.',
+ 'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
+ 'tags': ['критическое мышление', 'наука просто', 'математика', 'анализ данных'],
+ 'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
+ 'channel_follower_count': int,
+ },
+ }, {
+ 'note': 'playlists, multipage, different order',
+ 'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
+ 'playlist_mincount': 94,
+ 'info_dict': {
+ 'id': 'UCqj7Cz7revf5maW9g5pgNcg',
+ 'title': 'Igor Kleiner Ph.D. - Playlists',
+ 'description': 'md5:15d7dd9e333cb987907fcb0d604b233a',
+ 'uploader': 'Igor Kleiner Ph.D.',
+ 'uploader_id': '@IgorDataScience',
+ 'uploader_url': 'https://www.youtube.com/@IgorDataScience',
+ 'tags': ['критическое мышление', 'наука просто', 'математика', 'анализ данных'],
+ 'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
+ 'channel': 'Igor Kleiner Ph.D.',
+ 'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
+ 'channel_follower_count': int,
+ },
+ }, {
+ 'note': 'playlists, series',
+ 'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
+ 'playlist_mincount': 5,
+ 'info_dict': {
+ 'id': 'UCYO_jab_esuFRV4b17AJtAw',
+ 'title': '3Blue1Brown - Playlists',
+ 'description': 'md5:602e3789e6a0cb7d9d352186b720e395',
+ 'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
+ 'channel': '3Blue1Brown',
+ 'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
+ 'uploader_id': '@3blue1brown',
+ 'uploader_url': 'https://www.youtube.com/@3blue1brown',
+ 'uploader': '3Blue1Brown',
+ 'tags': ['Mathematics'],
+ 'channel_follower_count': int,
+ 'channel_is_verified': True,
+ },
+ }, {
+ 'note': 'playlists, singlepage',
+ 'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
+ 'playlist_mincount': 4,
+ 'info_dict': {
+ 'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
+ 'title': 'ThirstForScience - Playlists',
+ 'description': 'md5:609399d937ea957b0f53cbffb747a14c',
+ 'uploader': 'ThirstForScience',
+ 'uploader_url': 'https://www.youtube.com/@ThirstForScience',
+ 'uploader_id': '@ThirstForScience',
+ 'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
+ 'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
+ 'tags': 'count:12',
+ 'channel': 'ThirstForScience',
+ 'channel_follower_count': int,
+ },
+ }, {
+ 'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
+ 'only_matching': True,
+ }, {
+ 'note': 'basic, single video playlist',
+ 'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
+ 'info_dict': {
+ 'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
+ 'title': 'youtube-dl public playlist',
+ 'description': '',
+ 'tags': [],
+ 'view_count': int,
+ 'modified_date': '20201130',
+ 'channel': 'Sergey M.',
+ 'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
+ 'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
+ 'availability': 'public',
+ 'uploader': 'Sergey M.',
+ 'uploader_url': 'https://www.youtube.com/@sergeym.6173',
+ 'uploader_id': '@sergeym.6173',
+ },
+ 'playlist_count': 1,
+ }, {
+ 'note': 'empty playlist',
+ 'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
+ 'info_dict': {
+ 'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
+ 'title': 'youtube-dl empty playlist',
+ 'tags': [],
+ 'channel': 'Sergey M.',
+ 'description': '',
+ 'modified_date': '20230921',
+ 'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
+ 'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
+ 'availability': 'unlisted',
+ 'uploader_url': 'https://www.youtube.com/@sergeym.6173',
+ 'uploader_id': '@sergeym.6173',
+ 'uploader': 'Sergey M.',
+ },
+ 'playlist_count': 0,
+ }, {
+ 'note': 'Home tab',
+ 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
+ 'info_dict': {
+ 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'lex will - Home',
+ 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+ 'uploader': 'lex will',
+ 'uploader_id': '@lexwill718',
+ 'channel': 'lex will',
+ 'tags': ['bible', 'history', 'prophesy'],
+ 'uploader_url': 'https://www.youtube.com/@lexwill718',
+ 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'channel_follower_count': int,
+ },
+ 'playlist_mincount': 2,
+ }, {
+ 'note': 'Videos tab',
+ 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
+ 'info_dict': {
+ 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'lex will - Videos',
+ 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+ 'uploader': 'lex will',
+ 'uploader_id': '@lexwill718',
+ 'tags': ['bible', 'history', 'prophesy'],
+ 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'uploader_url': 'https://www.youtube.com/@lexwill718',
+ 'channel': 'lex will',
+ 'channel_follower_count': int,
+ },
+ 'playlist_mincount': 975,
+ }, {
+ 'note': 'Videos tab, sorted by popular',
+ 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
+ 'info_dict': {
+ 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'lex will - Videos',
+ 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+ 'uploader': 'lex will',
+ 'uploader_id': '@lexwill718',
+ 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'uploader_url': 'https://www.youtube.com/@lexwill718',
+ 'channel': 'lex will',
+ 'tags': ['bible', 'history', 'prophesy'],
+ 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'channel_follower_count': int,
+ },
+ 'playlist_mincount': 199,
+ }, {
+ 'note': 'Playlists tab',
+ 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
+ 'info_dict': {
+ 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'lex will - Playlists',
+ 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+ 'uploader': 'lex will',
+ 'uploader_id': '@lexwill718',
+ 'uploader_url': 'https://www.youtube.com/@lexwill718',
+ 'channel': 'lex will',
+ 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'tags': ['bible', 'history', 'prophesy'],
+ 'channel_follower_count': int,
+ },
+ 'playlist_mincount': 17,
+ }, {
+ 'note': 'Community tab',
+ 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
+ 'info_dict': {
+ 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'lex will - Community',
+ 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+ 'channel': 'lex will',
+ 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'tags': ['bible', 'history', 'prophesy'],
+ 'channel_follower_count': int,
+ 'uploader_url': 'https://www.youtube.com/@lexwill718',
+ 'uploader_id': '@lexwill718',
+ 'uploader': 'lex will',
+ },
+ 'playlist_mincount': 18,
+ }, {
+ 'note': 'Channels tab',
+ 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
+ 'info_dict': {
+ 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'title': 'lex will - Channels',
+ 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+ 'channel': 'lex will',
+ 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
+ 'tags': ['bible', 'history', 'prophesy'],
+ 'channel_follower_count': int,
+ 'uploader_url': 'https://www.youtube.com/@lexwill718',
+ 'uploader_id': '@lexwill718',
+ 'uploader': 'lex will',
+ },
+ 'playlist_mincount': 12,
+ }, {
+ 'note': 'Search tab',
+ 'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
+ 'playlist_mincount': 40,
+ 'info_dict': {
+ 'id': 'UCYO_jab_esuFRV4b17AJtAw',
+ 'title': '3Blue1Brown - Search - linear algebra',
+ 'description': 'md5:4d1da95432004b7ba840ebc895b6b4c9',
+ 'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
+ 'tags': ['Mathematics'],
+ 'channel': '3Blue1Brown',
+ 'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
+ 'channel_follower_count': int,
+ 'uploader_url': 'https://www.youtube.com/@3blue1brown',
+ 'uploader_id': '@3blue1brown',
+ 'uploader': '3Blue1Brown',
+ 'channel_is_verified': True,
+ },
+ }, {
+ 'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
+ 'only_matching': True,
+ }, {
+ 'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
+ 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
+ 'info_dict': {
+ 'title': '29C3: Not my department',
+ 'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
+ 'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
+ 'tags': [],
+ 'view_count': int,
+ 'modified_date': '20150605',
+ 'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
+ 'channel_url': 'https://www.youtube.com/channel/UCEPzS1rYsrkqzSLNp76nrcg',
+ 'channel': 'Christiaan008',
+ 'availability': 'public',
+ 'uploader_id': '@ChRiStIaAn008',
+ 'uploader': 'Christiaan008',
+ 'uploader_url': 'https://www.youtube.com/@ChRiStIaAn008',
+ },
+ 'playlist_count': 96,
+ }, {
+ 'note': 'Large playlist',
+ 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
+ 'info_dict': {
+ 'title': 'Uploads from Cauchemar',
+ 'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
+ 'channel_url': 'https://www.youtube.com/channel/UCBABnxM4Ar9ten8Mdjj1j0Q',
+ 'tags': [],
+ 'modified_date': r're:\d{8}',
+ 'channel': 'Cauchemar',
+ 'view_count': int,
+ 'description': '',
+ 'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
+ 'availability': 'public',
+ 'uploader_id': '@Cauchemar89',
+ 'uploader': 'Cauchemar',
+ 'uploader_url': 'https://www.youtube.com/@Cauchemar89',
+ },
+ 'playlist_mincount': 1123,
+ 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
+ }, {
+ 'note': 'even larger playlist, 8832 videos',
+ 'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
+ 'only_matching': True,
+ }, {
+ 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
+ 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
+ 'info_dict': {
+ 'title': 'Uploads from Interstellar Movie',
+ 'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
+ 'tags': [],
+ 'view_count': int,
+ 'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
+ 'channel_url': 'https://www.youtube.com/channel/UCXw-G3eDE9trcvY2sBMM_aA',
+ 'channel': 'Interstellar Movie',
+ 'description': '',
+ 'modified_date': r're:\d{8}',
+ 'availability': 'public',
+ 'uploader_id': '@InterstellarMovie',
+ 'uploader': 'Interstellar Movie',
+ 'uploader_url': 'https://www.youtube.com/@InterstellarMovie',
+ },
+ 'playlist_mincount': 21,
+ }, {
+ 'note': 'Playlist with "show unavailable videos" button',
+ 'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
+ 'info_dict': {
+ 'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
+ 'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
+ 'view_count': int,
+ 'channel': 'Phim Siêu Nhân Nhật Bản',
+ 'tags': [],
+ 'description': '',
+ 'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
+ 'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
+ 'modified_date': r're:\d{8}',
+ 'availability': 'public',
+ 'uploader_url': 'https://www.youtube.com/@phimsieunhannhatban',
+ 'uploader_id': '@phimsieunhannhatban',
+ 'uploader': 'Phim Siêu Nhân Nhật Bản',
+ },
+ 'playlist_mincount': 200,
+ 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
+ }, {
+ 'note': 'Playlist with unavailable videos in page 7',
+ 'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
+ 'info_dict': {
+ 'title': 'Uploads from BlankTV',
+ 'id': 'UU8l9frL61Yl5KFOl87nIm2w',
+ 'channel': 'BlankTV',
+ 'channel_url': 'https://www.youtube.com/channel/UC8l9frL61Yl5KFOl87nIm2w',
+ 'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
+ 'view_count': int,
+ 'tags': [],
+ 'modified_date': r're:\d{8}',
+ 'description': '',
+ 'availability': 'public',
+ 'uploader_id': '@blanktv',
+ 'uploader': 'BlankTV',
+ 'uploader_url': 'https://www.youtube.com/@blanktv',
+ },
+ 'playlist_mincount': 1000,
+ 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
+ }, {
+ 'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
+ 'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
+ 'info_dict': {
+ 'title': 'Data Analysis with Dr Mike Pound',
+ 'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
+ 'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
+ 'tags': [],
+ 'view_count': int,
+ 'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
+ 'channel_url': 'https://www.youtube.com/channel/UC9-y-6csu5WGm29I7JiwpnA',
+ 'channel': 'Computerphile',
+ 'availability': 'public',
+ 'modified_date': '20190712',
+ 'uploader_id': '@Computerphile',
+ 'uploader': 'Computerphile',
+ 'uploader_url': 'https://www.youtube.com/@Computerphile',
+ },
+ 'playlist_mincount': 11,
+ }, {
+ 'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
+ 'only_matching': True,
+ }, {
+ 'note': 'Playlist URL that does not actually serve a playlist',
+ 'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
+ 'info_dict': {
+ 'id': 'FqZTN594JQw',
+ 'ext': 'webm',
+ 'title': "Smiley's People 01 detective, Adventure Series, Action",
+ 'upload_date': '20150526',
+ 'license': 'Standard YouTube License',
+ 'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
+ 'categories': ['People & Blogs'],
+ 'tags': list,
+ 'view_count': int,
+ 'like_count': int,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ 'skip': 'This video is not available.',
+ 'add_ie': [YoutubeIE.ie_key()],
+ }, {
+ 'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
+ 'info_dict': {
+ 'id': 'hGkQjiJLjWQ', # This will keep changing
+ 'ext': 'mp4',
+ 'title': str,
+ 'upload_date': r're:\d{8}',
+ 'description': str,
+ 'categories': ['News & Politics'],
+ 'tags': list,
+ 'like_count': int,
+ 'release_timestamp': int,
+ 'channel': 'Sky News',
+ 'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
+ 'age_limit': 0,
+ 'view_count': int,
+ 'thumbnail': r're:https?://i\.ytimg\.com/vi/[^/]+/maxresdefault(?:_live)?\.jpg',
+ 'playable_in_embed': True,
+ 'release_date': r're:\d+',
+ 'availability': 'public',
+ 'live_status': 'is_live',
+ 'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
+ 'channel_follower_count': int,
+ 'concurrent_view_count': int,
+ 'uploader_url': 'https://www.youtube.com/@SkyNews',
+ 'uploader_id': '@SkyNews',
+ 'uploader': 'Sky News',
+ 'channel_is_verified': True,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ 'expected_warnings': ['Ignoring subtitle tracks found in '],
+ }, {
+ 'url': 'https://www.youtube.com/user/TheYoungTurks/live',
+ 'info_dict': {
+ 'id': 'a48o2S1cPoo',
+ 'ext': 'mp4',
+ 'title': 'The Young Turks - Live Main Show',
+ 'upload_date': '20150715',
+ 'license': 'Standard YouTube License',
+ 'description': 'md5:438179573adcdff3c97ebb1ee632b891',
+ 'categories': ['News & Politics'],
+ 'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
+ 'like_count': int,
+ },
+ 'params': {
+ 'skip_download': True,
+ },
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
+ 'only_matching': True,
+ }, {
+ 'note': 'A channel that is not live. Should raise error',
+ 'url': 'https://www.youtube.com/user/numberphile/live',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/feed/trending',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/feed/library',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/feed/history',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/feed/subscriptions',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/feed/watch_later',
+ 'only_matching': True,
+ }, {
+ 'note': 'Recommended - redirects to home page.',
+ 'url': 'https://www.youtube.com/feed/recommended',
+ 'only_matching': True,
+ }, {
+ 'note': 'inline playlist with not always working continuations',
+ 'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/course',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/zsecurity',
+ 'only_matching': True,
+ }, {
+ 'url': 'http://www.youtube.com/NASAgovVideo/videos',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/TheYoungTurks/live',
+ 'only_matching': True,
+ }, {
+ 'url': 'https://www.youtube.com/hashtag/cctv9',
+ 'info_dict': {
+ 'id': 'cctv9',
+ 'title': 'cctv9 - All',
+ 'tags': [],
+ },
+ 'playlist_mincount': 300, # not consistent but should be over 300
+ }, {
+ 'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
+ 'only_matching': True,
+ }, {
+ 'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
+ 'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
+ 'only_matching': True,
+ }, {
+ 'note': '/browse/ should redirect to /channel/',
+ 'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
+ 'only_matching': True,
+ }, {
+ 'note': 'VLPL, should redirect to playlist?list=PL...',
+ 'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
+ 'info_dict': {
+ 'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
+ 'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
+ 'title': 'NCS : All Releases 💿',
+ 'channel_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
+ 'modified_date': r're:\d{8}',
+ 'view_count': int,
+ 'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
+ 'tags': [],
+ 'channel': 'NoCopyrightSounds',
+ 'availability': 'public',
+ 'uploader_url': 'https://www.youtube.com/@NoCopyrightSounds',
+ 'uploader': 'NoCopyrightSounds',
+ 'uploader_id': '@NoCopyrightSounds',
+ },
+ 'playlist_mincount': 166,
+ 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden', 'YouTube Music is not directly supported'],
+ }, {
+ # TODO: fix 'unviewable' issue with this playlist when reloading with unavailable videos
+ 'note': 'Topic, should redirect to playlist?list=UU...',
+ 'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
+ 'info_dict': {
+ 'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
+ 'title': 'Uploads from Royalty Free Music - Topic',
+ 'tags': [],
+ 'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
+ 'channel': 'Royalty Free Music - Topic',
+ 'view_count': int,
+ 'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
+ 'modified_date': r're:\d{8}',
+ 'description': '',
+ 'availability': 'public',
+ 'uploader': 'Royalty Free Music - Topic',
+ },
+ 'playlist_mincount': 101,
+ 'expected_warnings': ['YouTube Music is not directly supported', r'[Uu]navailable videos (are|will be) hidden'],
+ }, {
+ # Destination channel with only a hidden self tab (tab id is UCtFRv9O2AHqOZjjynzrv-xg)
+ # Treat as a general feed
+ 'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
+ 'info_dict': {
+ 'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
+ 'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
+ 'tags': [],
+ },
+ 'playlist_mincount': 9,
+ }, {
+ 'note': 'Youtube music Album',
+ 'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
+ 'info_dict': {
+ 'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
+ 'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
+ 'tags': [],
+ 'view_count': int,
+ 'description': '',
+ 'availability': 'unlisted',
+ 'modified_date': r're:\d{8}',
+ },
+ 'playlist_count': 50,
+ 'expected_warnings': ['YouTube Music is not directly supported'],
+ }, {
+ 'note': 'unlisted single video playlist',
+ 'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
+ 'info_dict': {
+ 'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
+ 'title': 'yt-dlp unlisted playlist test',
+ 'availability': 'unlisted',
+ 'tags': [],
+ 'modified_date': '20220418',
+ 'channel': 'colethedj',
+ 'view_count': int,
+ 'description': '',
+ 'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
+ 'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
+ 'uploader_url': 'https://www.youtube.com/@colethedj1894',
+ 'uploader_id': '@colethedj1894',
+ 'uploader': 'colethedj',
+ },
+ 'playlist': [{
+ 'info_dict': {
+ 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
+ 'id': 'BaW_jenozKc',
+ '_type': 'url',
+ 'ie_key': 'Youtube',
+ 'duration': 10,
+ 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
+ 'channel_url': 'https://www.youtube.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
+ 'view_count': int,
+ 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc',
+ 'channel': 'Philipp Hagemeister',
+ 'uploader_id': '@PhilippHagemeister',
+ 'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
+ 'uploader': 'Philipp Hagemeister',
+ },
+ }],
+ 'playlist_count': 1,
+ 'params': {'extract_flat': True},
+ }, {
+ 'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
+ 'url': 'https://www.youtube.com/feed/recommended',
+ 'info_dict': {
+ 'id': 'recommended',
+ 'title': 'recommended',
+ 'tags': [],
+ },
+ 'playlist_mincount': 50,
+ 'params': {
+ 'skip_download': True,
+ 'extractor_args': {'youtubetab': {'skip': ['webpage']}},
+ },
+ }, {
+ 'note': 'API Fallback: /videos tab, sorted by oldest first',
+ 'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
+ 'info_dict': {
+ 'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
+ 'title': 'Cody\'sLab - Videos',
+ 'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
+ 'channel': 'Cody\'sLab',
+ 'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
+ 'tags': [],
+ 'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
+ 'channel_follower_count': int,
+ },
+ 'playlist_mincount': 650,
+ 'params': {
+ 'skip_download': True,
+ 'extractor_args': {'youtubetab': {'skip': ['webpage']}},
+ },
+ 'skip': 'Query for sorting no longer works',
+ }, {
+ 'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
+ 'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
+ 'info_dict': {
+ 'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
+ 'title': 'Uploads from Royalty Free Music - Topic',
+ 'modified_date': r're:\d{8}',
+ 'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
+ 'description': '',
+ 'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
+ 'tags': [],
+ 'channel': 'Royalty Free Music - Topic',
+ 'view_count': int,
+ 'availability': 'public',
+ 'uploader': 'Royalty Free Music - Topic',
+ },
+ 'playlist_mincount': 101,
+ 'params': {
+ 'skip_download': True,
+ 'extractor_args': {'youtubetab': {'skip': ['webpage']}},
+ },
+ 'expected_warnings': ['YouTube Music is not directly supported', r'[Uu]navailable videos (are|will be) hidden'],
+ }, {
+ 'note': 'non-standard redirect to regional channel',
+ 'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
+ 'only_matching': True,
+ }, {
+ 'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
+ 'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
+ 'info_dict': {
+ 'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
+ 'modified_date': '20220407',
+ 'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
+ 'tags': [],
+ 'availability': 'unlisted',
+ 'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
+ 'channel': 'pukkandan',
+ 'description': 'Test for collaborative playlist',
+ 'title': 'yt-dlp test - collaborative playlist',
+ 'view_count': int,
+ 'uploader_url': 'https://www.youtube.com/@pukkandan',
+ 'uploader_id': '@pukkandan',
+ 'uploader': 'pukkandan',
+ },
+ 'playlist_mincount': 2,
+ }, {
+ 'note': 'translated tab name',
+ 'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
+ 'info_dict': {
+ 'id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'tags': [],
+ 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
+ 'description': 'test description',
+ 'title': 'cole-dlp-test-acc - 再生リスト',
+ 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'channel': 'cole-dlp-test-acc',
+ 'uploader_url': 'https://www.youtube.com/@coletdjnz',
+ 'uploader_id': '@coletdjnz',
+ 'uploader': 'cole-dlp-test-acc',
+ },
+ 'playlist_mincount': 1,
+ 'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
+ 'expected_warnings': ['Preferring "ja"'],
+ }, {
+ # XXX: this should really check flat playlist entries, but the test suite doesn't support that
+ 'note': 'preferred lang set with playlist with translated video titles',
+ 'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
+ 'info_dict': {
+ 'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
+ 'tags': [],
+ 'view_count': int,
+ 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
+ 'channel': 'cole-dlp-test-acc',
+ 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'description': 'test',
+ 'title': 'dlp test playlist',
+ 'availability': 'public',
+ 'uploader_url': 'https://www.youtube.com/@coletdjnz',
+ 'uploader_id': '@coletdjnz',
+ 'uploader': 'cole-dlp-test-acc',
+ },
+ 'playlist_mincount': 1,
+ 'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
+ 'expected_warnings': ['Preferring "ja"'],
+ }, {
+ # shorts audio pivot for 2GtVksBMYFM.
+ 'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
+ 'info_dict': {
+ 'id': 'sfv_audio_pivot',
+ 'title': 'sfv_audio_pivot',
+ 'tags': [],
+ },
+ 'playlist_mincount': 50,
+
+ }, {
+ # Channel with a real live tab (not to be mistaken with streams tab)
+ # Do not treat like it should redirect to live stream
+ 'url': 'https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live',
+ 'info_dict': {
+ 'id': 'UCEH7P7kyJIkS_gJf93VYbmg',
+ 'title': 'UCEH7P7kyJIkS_gJf93VYbmg - Live',
+ 'tags': [],
+ },
+ 'playlist_mincount': 20,
+ }, {
+ # Tab name is not the same as tab id
+ 'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/letsplay',
+ 'info_dict': {
+ 'id': 'UCQvWX73GQygcwXOTSf_VDVg',
+ 'title': 'UCQvWX73GQygcwXOTSf_VDVg - Let\'s play',
+ 'tags': [],
+ },
+ 'playlist_mincount': 8,
+ }, {
+ # Home tab id is literally home. Not to get mistaken with featured
+ 'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/home',
+ 'info_dict': {
+ 'id': 'UCQvWX73GQygcwXOTSf_VDVg',
+ 'title': 'UCQvWX73GQygcwXOTSf_VDVg - Home',
+ 'tags': [],
+ },
+ 'playlist_mincount': 8,
+ }, {
+ # Should get three playlists for videos, shorts and streams tabs
+ 'url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
+ 'info_dict': {
+ 'id': 'UCK9V2B22uJYu3N7eR_BT9QA',
+ 'title': 'Polka Ch. 尾丸ポルカ',
+ 'channel_follower_count': int,
+ 'channel_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
+ 'channel_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
+ 'description': 'md5:49809d8bf9da539bc48ed5d1f83c33f2',
+ 'channel': 'Polka Ch. 尾丸ポルカ',
+ 'tags': 'count:35',
+ 'uploader_url': 'https://www.youtube.com/@OmaruPolka',
+ 'uploader': 'Polka Ch. 尾丸ポルカ',
+ 'uploader_id': '@OmaruPolka',
+ 'channel_is_verified': True,
+ },
+ 'playlist_count': 3,
+ }, {
+ # Shorts tab with channel with handle
+ # TODO: fix channel description
+ 'url': 'https://www.youtube.com/@NotJustBikes/shorts',
+ 'info_dict': {
+ 'id': 'UC0intLFzLaudFG-xAvUEO-A',
+ 'title': 'Not Just Bikes - Shorts',
+ 'tags': 'count:10',
+ 'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
+ 'description': 'md5:5e82545b3a041345927a92d0585df247',
+ 'channel_follower_count': int,
+ 'channel_id': 'UC0intLFzLaudFG-xAvUEO-A',
+ 'channel': 'Not Just Bikes',
+ 'uploader_url': 'https://www.youtube.com/@NotJustBikes',
+ 'uploader': 'Not Just Bikes',
+ 'uploader_id': '@NotJustBikes',
+ 'channel_is_verified': True,
+ },
+ 'playlist_mincount': 10,
+ }, {
+ # Streams tab
+ 'url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig/streams',
+ 'info_dict': {
+ 'id': 'UC3eYAvjCVwNHgkaGbXX3sig',
+ 'title': '中村悠一 - Live',
+ 'tags': 'count:7',
+ 'channel_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
+ 'channel_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
+ 'channel': '中村悠一',
+ 'channel_follower_count': int,
+ 'description': 'md5:e744f6c93dafa7a03c0c6deecb157300',
+ 'uploader_url': 'https://www.youtube.com/@Yuichi-Nakamura',
+ 'uploader_id': '@Yuichi-Nakamura',
+ 'uploader': '中村悠一',
+ },
+ 'playlist_mincount': 60,
+ }, {
+ # Channel with no uploads and hence no videos, streams, shorts tabs or uploads playlist. This should fail.
+ # See test_youtube_lists
+ 'url': 'https://www.youtube.com/channel/UC2yXPzFejc422buOIzn_0CA',
+ 'only_matching': True,
+ }, {
+ # No uploads and no UCID given. Should fail with no uploads error
+ # See test_youtube_lists
+ 'url': 'https://www.youtube.com/news',
+ 'only_matching': True,
+ }, {
+ # No videos tab but has a shorts tab
+ 'url': 'https://www.youtube.com/c/TKFShorts',
+ 'info_dict': {
+ 'id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
+ 'title': 'Shorts Break - Shorts',
+ 'tags': 'count:48',
+ 'channel_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
+ 'channel': 'Shorts Break',
+ 'description': 'md5:6de33c5e7ba686e5f3efd4e19c7ef499',
+ 'channel_follower_count': int,
+ 'channel_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
+ 'uploader_url': 'https://www.youtube.com/@ShortsBreak_Official',
+ 'uploader': 'Shorts Break',
+ 'uploader_id': '@ShortsBreak_Official',
+ },
+ 'playlist_mincount': 30,
+ }, {
+ # Trending Now Tab. tab id is empty
+ 'url': 'https://www.youtube.com/feed/trending',
+ 'info_dict': {
+ 'id': 'trending',
+ 'title': 'trending - Now',
+ 'tags': [],
+ },
+ 'playlist_mincount': 30,
+ }, {
+ # Trending Gaming Tab. tab id is empty
+ 'url': 'https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D',
+ 'info_dict': {
+ 'id': 'trending',
+ 'title': 'trending - Gaming',
+ 'tags': [],
+ },
+ 'playlist_mincount': 30,
+ }, {
+ # Shorts url result in shorts tab
+ # TODO: Fix channel id extraction
+ 'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/shorts',
+ 'info_dict': {
+ 'id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'title': 'cole-dlp-test-acc - Shorts',
+ 'channel': 'cole-dlp-test-acc',
+ 'description': 'test description',
+ 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
+ 'tags': [],
+ 'uploader_url': 'https://www.youtube.com/@coletdjnz',
+ 'uploader_id': '@coletdjnz',
+ 'uploader': 'cole-dlp-test-acc',
+ },
+ 'playlist': [{
+ 'info_dict': {
+ # Channel data is not currently available for short renderers (as of 2023-03-01)
+ '_type': 'url',
+ 'ie_key': 'Youtube',
+ 'url': 'https://www.youtube.com/shorts/sSM9J5YH_60',
+ 'id': 'sSM9J5YH_60',
+ 'title': 'SHORT short',
+ 'view_count': int,
+ 'thumbnails': list,
+ },
+ }],
+ 'params': {'extract_flat': True},
+ }, {
+ # Live video status should be extracted
+ 'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/live',
+ 'info_dict': {
+ 'id': 'UCQvWX73GQygcwXOTSf_VDVg',
+ 'title': 'UCQvWX73GQygcwXOTSf_VDVg - Live', # TODO: should be Minecraft - Live or Minecraft - Topic - Live
+ 'tags': [],
+ },
+ 'playlist': [{
+ 'info_dict': {
+ '_type': 'url',
+ 'ie_key': 'Youtube',
+ 'url': 'startswith:https://www.youtube.com/watch?v=',
+ 'id': str,
+ 'title': str,
+ 'live_status': 'is_live',
+ 'channel_id': str,
+ 'channel_url': str,
+ 'concurrent_view_count': int,
+ 'channel': str,
+ 'uploader': str,
+ 'uploader_url': str,
+ 'uploader_id': str,
+ 'channel_is_verified': bool, # this will keep changing
+ },
+ }],
+ 'params': {'extract_flat': True, 'playlist_items': '1'},
+ 'playlist_mincount': 1,
+ }, {
+ # Channel renderer metadata. Contains number of videos on the channel
+ 'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/channels',
+ 'info_dict': {
+ 'id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'title': 'cole-dlp-test-acc - Channels',
+ 'channel': 'cole-dlp-test-acc',
+ 'description': 'test description',
+ 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
+ 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
+ 'tags': [],
+ 'uploader_url': 'https://www.youtube.com/@coletdjnz',
+ 'uploader_id': '@coletdjnz',
+ 'uploader': 'cole-dlp-test-acc',
+ },
+ 'playlist': [{
+ 'info_dict': {
+ '_type': 'url',
+ 'ie_key': 'YoutubeTab',
+ 'url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
+ 'id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
+ 'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
+ 'title': 'PewDiePie',
+ 'channel': 'PewDiePie',
+ 'channel_url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
+ 'thumbnails': list,
+ 'channel_follower_count': int,
+ 'playlist_count': int,
+ 'uploader': 'PewDiePie',
+ 'uploader_url': 'https://www.youtube.com/@PewDiePie',
+ 'uploader_id': '@PewDiePie',
+ 'channel_is_verified': True,
+ },
+ }],
+ 'params': {'extract_flat': True},
+ }, {
+ 'url': 'https://www.youtube.com/@3blue1brown/about',
+ 'info_dict': {
+ 'id': '@3blue1brown',
+ 'tags': ['Mathematics'],
+ 'title': '3Blue1Brown',
+ 'channel_follower_count': int,
+ 'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
+ 'channel': '3Blue1Brown',
+ 'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
+ 'description': 'md5:4d1da95432004b7ba840ebc895b6b4c9',
+ 'uploader_url': 'https://www.youtube.com/@3blue1brown',
+ 'uploader_id': '@3blue1brown',
+ 'uploader': '3Blue1Brown',
+ 'channel_is_verified': True,
+ },
+ 'playlist_count': 0,
+ }, {
+ # Podcasts tab, with rich entry lockupViewModel
+ 'url': 'https://www.youtube.com/@99percentinvisiblepodcast/podcasts',
+ 'info_dict': {
+ 'id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
+ 'channel_id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
+ 'uploader_url': 'https://www.youtube.com/@99percentinvisiblepodcast',
+ 'description': 'md5:3a0ed38f1ad42a68ef0428c04a15695c',
+ 'title': '99% Invisible - Podcasts',
+ 'uploader': '99% Invisible',
+ 'channel_follower_count': int,
+ 'channel_url': 'https://www.youtube.com/channel/UCVMF2HD4ZgC0QHpU9Yq5Xrw',
+ 'tags': [],
+ 'channel': '99% Invisible',
+ 'uploader_id': '@99percentinvisiblepodcast',
+ },
+ 'playlist_count': 5,
+ }, {
+ # Releases tab, with rich entry playlistRenderers (same as Podcasts tab)
+ 'url': 'https://www.youtube.com/@AHimitsu/releases',
+ 'info_dict': {
+ 'id': 'UCgFwu-j5-xNJml2FtTrrB3A',
+ 'channel': 'A Himitsu',
+ 'uploader_url': 'https://www.youtube.com/@AHimitsu',
+ 'title': 'A Himitsu - Releases',
+ 'uploader_id': '@AHimitsu',
+ 'uploader': 'A Himitsu',
+ 'channel_id': 'UCgFwu-j5-xNJml2FtTrrB3A',
+ 'tags': 'count:12',
+ 'description': 'I make music',
+ 'channel_url': 'https://www.youtube.com/channel/UCgFwu-j5-xNJml2FtTrrB3A',
+ 'channel_follower_count': int,
+ 'channel_is_verified': True,
+ },
+ 'playlist_mincount': 10,
+ }, {
+ # Playlist with only shorts, shown as reel renderers
+ # FIXME: future: YouTube currently doesn't give continuation for this,
+ # may do in future.
+ 'url': 'https://www.youtube.com/playlist?list=UUxqPAgubo4coVn9Lx1FuKcg',
+ 'info_dict': {
+ 'id': 'UUxqPAgubo4coVn9Lx1FuKcg',
+ 'channel_url': 'https://www.youtube.com/channel/UCxqPAgubo4coVn9Lx1FuKcg',
+ 'view_count': int,
+ 'uploader_id': '@BangyShorts',
+ 'description': '',
+ 'uploader_url': 'https://www.youtube.com/@BangyShorts',
+ 'channel_id': 'UCxqPAgubo4coVn9Lx1FuKcg',
+ 'channel': 'Bangy Shorts',
+ 'uploader': 'Bangy Shorts',
+ 'tags': [],
+ 'availability': 'public',
+ 'modified_date': r're:\d{8}',
+ 'title': 'Uploads from Bangy Shorts',
+ },
+ 'playlist_mincount': 100,
+ 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
+ }, {
+ 'note': 'Tags containing spaces',
+ 'url': 'https://www.youtube.com/channel/UC7_YxT-KID8kRbqZo7MyscQ',
+ 'playlist_count': 3,
+ 'info_dict': {
+ 'id': 'UC7_YxT-KID8kRbqZo7MyscQ',
+ 'channel': 'Markiplier',
+ 'channel_id': 'UC7_YxT-KID8kRbqZo7MyscQ',
+ 'title': 'Markiplier',
+ 'channel_follower_count': int,
+ 'description': 'md5:0c010910558658824402809750dc5d97',
+ 'uploader_id': '@markiplier',
+ 'uploader_url': 'https://www.youtube.com/@markiplier',
+ 'uploader': 'Markiplier',
+ 'channel_url': 'https://www.youtube.com/channel/UC7_YxT-KID8kRbqZo7MyscQ',
+ 'channel_is_verified': True,
+ 'tags': ['markiplier', 'comedy', 'gaming', 'funny videos', 'funny moments',
+ 'sketch comedy', 'laughing', 'lets play', 'challenge videos', 'hilarious',
+ 'challenges', 'sketches', 'scary games', 'funny games', 'rage games',
+ 'mark fischbach'],
+ },
+ }]
+
+ @classmethod
+ def suitable(cls, url):
+ return False if YoutubeIE.suitable(url) else super().suitable(url)
+
+ _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/[^?#/]+))?(?P<post>.*)$')
+
+ def _get_url_mobj(self, url):
+ mobj = self._URL_RE.match(url).groupdict()
+ mobj.update((k, '') for k, v in mobj.items() if v is None)
+ return mobj
+
+ def _extract_tab_id_and_name(self, tab, base_url='https://www.youtube.com'):
+ tab_name = (tab.get('title') or '').lower()
+ tab_url = urljoin(base_url, traverse_obj(
+ tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
+
+ tab_id = ((tab_url and self._get_url_mobj(tab_url)['tab'][1:])
+ or traverse_obj(tab, 'tabIdentifier', expected_type=str))
+ if tab_id:
+ return {
+ 'TAB_ID_SPONSORSHIPS': 'membership',
+ }.get(tab_id, tab_id), tab_name
+
+ # Fallback to tab name if we cannot get the tab id.
+ # XXX: should we strip non-ascii letters? e.g. in case of 'let's play' tab example on special gaming channel
+ # Note that in the case of translated tab name this may result in an empty string, which we don't want.
+ if tab_name:
+ self.write_debug(f'Falling back to selected tab name: {tab_name}')
+ return {
+ 'home': 'featured',
+ 'live': 'streams',
+ }.get(tab_name, tab_name), tab_name
+
+ def _has_tab(self, tabs, tab_id):
+ return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
+
+ def _empty_playlist(self, item_id, data):
+ return self.playlist_result([], item_id, **self._extract_metadata_from_tabs(item_id, data))
+
+ @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
+ def _real_extract(self, url, smuggled_data):
+ item_id = self._match_id(url)
+ url = urllib.parse.urlunparse(
+ urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
+ compat_opts = self.get_param('compat_opts', [])
+
+ mobj = self._get_url_mobj(url)
+ pre, tab, post, is_channel = mobj['pre'], mobj['tab'], mobj['post'], not mobj['not_channel']
+ if is_channel and smuggled_data.get('is_music_url'):
+ if item_id[:2] == 'VL': # Youtube music VL channels have an equivalent playlist
+ return self.url_result(
+ f'https://music.youtube.com/playlist?list={item_id[2:]}', YoutubeTabIE, item_id[2:])
+ elif item_id[:2] == 'MP': # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
+ mdata = self._extract_tab_endpoint(
+ f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
+ murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
+ get_all=False, expected_type=str)
+ if not murl:
+ raise ExtractorError('Failed to resolve album to playlist')
+ return self.url_result(murl, YoutubeTabIE)
+ elif mobj['channel_type'] == 'browse': # Youtube music /browse/ should be changed to /channel/
+ return self.url_result(
+ f'https://music.youtube.com/channel/{item_id}{tab}{post}', YoutubeTabIE, item_id)
+
+ original_tab_id, display_id = tab[1:], f'{item_id}{tab}'
+ if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
+ url = f'{pre}/videos{post}'
+ if smuggled_data.get('is_music_url'):
+ self.report_warning(f'YouTube Music is not directly supported. Redirecting to {url}')
+
+ # Handle both video/playlist URLs
+ qs = parse_qs(url)
+ video_id, playlist_id = (traverse_obj(qs, (key, 0)) for key in ('v', 'list'))
+ if not video_id and mobj['not_channel'].startswith('watch'):
+ if not playlist_id:
+ # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
+ raise ExtractorError('A video URL was given without video ID', expected=True)
+ # Common mistake: https://www.youtube.com/watch?list=playlist_id
+ self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
+ return self.url_result(
+ f'https://www.youtube.com/playlist?list={playlist_id}', YoutubeTabIE, playlist_id)
+
+ if not self._yes_playlist(playlist_id, video_id):
+ return self.url_result(
+ f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
+
+ data, ytcfg = self._extract_data(url, display_id)
+
+ # YouTube may provide a non-standard redirect to the regional channel
+ # See: https://github.com/yt-dlp/yt-dlp/issues/2694
+ # https://support.google.com/youtube/answer/2976814#zippy=,conditional-redirects
+ redirect_url = traverse_obj(
+ data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
+ if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
+ redirect_url = ''.join((urljoin('https://www.youtube.com', redirect_url), tab, post))
+ self.to_screen(f'This playlist is likely not available in your region. Following conditional redirect to {redirect_url}')
+ return self.url_result(redirect_url, YoutubeTabIE)
+
+ tabs, extra_tabs = self._extract_tab_renderers(data), []
+ if is_channel and tabs and 'no-youtube-channel-redirect' not in compat_opts:
+ selected_tab = self._extract_selected_tab(tabs)
+ selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url) # NB: Name may be translated
+ self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
+
+ # /about is no longer a tab
+ if original_tab_id == 'about':
+ return self._empty_playlist(item_id, data)
+
+ if not original_tab_id and selected_tab_name:
+ self.to_screen('Downloading all uploads of the channel. '
+ 'To download only the videos in a specific tab, pass the tab\'s URL')
+ if self._has_tab(tabs, 'streams'):
+ extra_tabs.append(''.join((pre, '/streams', post)))
+ if self._has_tab(tabs, 'shorts'):
+ extra_tabs.append(''.join((pre, '/shorts', post)))
+ # XXX: Members-only tab should also be extracted
+
+ if not extra_tabs and selected_tab_id != 'videos':
+ # Channel does not have streams, shorts or videos tabs
+ if item_id[:2] != 'UC':
+ return self._empty_playlist(item_id, data)
+
+ # Topic channels don't have /videos. Use the equivalent playlist instead
+ pl_id = f'UU{item_id[2:]}'
+ pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
+ try:
+ data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
+ except ExtractorError:
+ return self._empty_playlist(item_id, data)
+ else:
+ item_id, url = pl_id, pl_url
+ self.to_screen(
+ f'The channel does not have a videos, shorts, or live tab. Redirecting to playlist {pl_id} instead')
+
+ elif extra_tabs and selected_tab_id != 'videos':
+ # When there are shorts/live tabs but not videos tab
+ url, data = f'{pre}{post}', None
+
+ elif (original_tab_id or 'videos') != selected_tab_id:
+ if original_tab_id == 'live':
+ # Live tab should have redirected to the video
+ # Except in the case the channel has an actual live tab
+ # Example: https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live
+ raise UserNotLive(video_id=item_id)
+ elif selected_tab_name:
+ raise ExtractorError(f'This channel does not have a {original_tab_id} tab', expected=True)
+
+ # For channels such as https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg
+ url = f'{pre}{post}'
+
+ # YouTube sometimes provides a button to reload playlist with unavailable videos.
+ if 'no-youtube-unavailable-videos' not in compat_opts:
+ data = self._reload_with_unavailable_videos(display_id, data, ytcfg) or data
+ self._extract_and_report_alerts(data, only_once=True)
+
+ tabs, entries = self._extract_tab_renderers(data), []
+ if tabs:
+ entries = [self._extract_from_tabs(item_id, ytcfg, data, tabs)]
+ entries[0].update({
+ 'extractor_key': YoutubeTabIE.ie_key(),
+ 'extractor': YoutubeTabIE.IE_NAME,
+ 'webpage_url': url,
+ })
+ if self.get_param('playlist_items') == '0':
+ entries.extend(self.url_result(u, YoutubeTabIE) for u in extra_tabs)
+ else: # Users expect to get all `video_id`s even with `--flat-playlist`. So don't return `url_result`
+ entries.extend(map(self._real_extract, extra_tabs))
+
+ if len(entries) == 1:
+ return entries[0]
+ elif entries:
+ metadata = self._extract_metadata_from_tabs(item_id, data)
+ uploads_url = 'the Uploads (UU) playlist URL'
+ if try_get(metadata, lambda x: x['channel_id'].startswith('UC')):
+ uploads_url = f'https://www.youtube.com/playlist?list=UU{metadata["channel_id"][2:]}'
+ self.to_screen(
+ 'Downloading as multiple playlists, separated by tabs. '
+ f'To download as a single playlist instead, pass {uploads_url}')
+ return self.playlist_result(entries, item_id, **metadata)
+
+ # Inline playlist
+ playlist = traverse_obj(
+ data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
+ if playlist:
+ return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
+
+ video_id = traverse_obj(
+ data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
+ if video_id:
+ if tab != '/live': # live tab is expected to redirect to video
+ self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
+ return self.url_result(f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
+
+ raise ExtractorError('Unable to recognize tab page')
+
+
+# xxx: This is tightly coupled to YoutubeTabBaseInfoExtractor. Should be decoupled at some point
+class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
+ IE_DESC = 'YouTube playlists'
+ _VALID_URL = r'''(?x)(?:
+ (?:https?://)?
+ (?:\w+\.)?
+ (?:
+ (?:
+ youtube(?:kids)?\.com|
+ {invidious}
+ )
+ /.*?\?.*?\blist=
+ )?
+ (?P<id>{playlist_id})
+ )'''.format(
+ playlist_id=YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
+ invidious='|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
+ )
+ IE_NAME = 'youtube:playlist'
+ _TESTS = [{
+ 'note': 'issue #673',
+ 'url': 'PLBB231211A4F62143',
+ 'info_dict': {
+ 'title': '[OLD]Team Fortress 2 (Class-based LP)',
+ 'id': 'PLBB231211A4F62143',
+ 'uploader': 'Wickman',
+ 'uploader_id': '@WickmanVT',
+ 'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
+ 'view_count': int,
+ 'uploader_url': 'https://www.youtube.com/@WickmanVT',
+ 'modified_date': r're:\d{8}',
+ 'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
+ 'channel': 'Wickman',
+ 'tags': [],
+ 'channel_url': 'https://www.youtube.com/channel/UCKSpbfbl5kRQpTdL7kMc-1Q',
+ 'availability': 'public',
+ },
+ 'playlist_mincount': 29,
+ }, {
+ 'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
+ 'info_dict': {
+ 'title': 'YDL_safe_search',
+ 'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
+ },
+ 'playlist_count': 2,
+ 'skip': 'This playlist is private',
+ }, {
+ 'note': 'embedded',
+ 'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
+ 'playlist_count': 4,
+ 'info_dict': {
+ 'title': 'JODA15',
+ 'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
+ 'uploader': 'milan',
+ 'uploader_id': '@milan5503',
+ 'description': '',
+ 'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
+ 'tags': [],
+ 'modified_date': '20140919',
+ 'view_count': int,
+ 'channel': 'milan',
+ 'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
+ 'uploader_url': 'https://www.youtube.com/@milan5503',
+ 'availability': 'public',
+ },
+ 'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden', 'Retrying', 'Giving up'],
+ }, {
+ 'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
+ 'playlist_mincount': 455,
+ 'info_dict': {
+ 'title': '2018 Chinese New Singles (11/6 updated)',
+ 'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
+ 'uploader': 'LBK',
+ 'uploader_id': '@music_king',
+ 'description': 'md5:da521864744d60a198e3a88af4db0d9d',
+ 'channel': 'LBK',
+ 'view_count': int,
+ 'channel_url': 'https://www.youtube.com/channel/UC21nz3_MesPLqtDqwdvnoxA',
+ 'tags': [],
+ 'uploader_url': 'https://www.youtube.com/@music_king',
+ 'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
+ 'modified_date': r're:\d{8}',
+ 'availability': 'public',
+ },
+ 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
+ }, {
+ 'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
+ 'only_matching': True,
+ }, {
+ # music album playlist
+ 'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
+ 'only_matching': True,
+ }]
+
+ @classmethod
+ def suitable(cls, url):
+ if YoutubeTabIE.suitable(url):
+ return False
+ from yt_dlp.utils import parse_qs
+ qs = parse_qs(url)
+ if qs.get('v', [None])[0]:
+ return False
+ return super().suitable(url)
+
+ def _real_extract(self, url):
+ playlist_id = self._match_id(url)
+ is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
+ url = update_url_query(
+ 'https://www.youtube.com/playlist',
+ parse_qs(url) or {'list': playlist_id})
+ if is_music_url:
+ url = smuggle_url(url, {'is_music_url': True})
+ return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
diff --git a/yt_dlp/extractor/youtube.py b/yt_dlp/extractor/youtube/_video.py
index ee4c98473..2b9b89700 100644
--- a/yt_dlp/extractor/youtube.py
+++ b/yt_dlp/extractor/youtube/_video.py
@@ -1,45 +1,43 @@
import base64
import binascii
-import calendar
import collections
-import copy
import datetime as dt
-import enum
import functools
-import hashlib
import itertools
import json
import math
import os.path
import random
import re
-import shlex
import sys
import threading
import time
import traceback
import urllib.parse
-from .common import InfoExtractor, SearchInfoExtractor
-from .openload import PhantomJSwrapper
-from ..jsinterp import JSInterpreter
-from ..networking.exceptions import HTTPError, network_exceptions
-from ..utils import (
+from ._base import (
+ INNERTUBE_CLIENTS,
+ BadgeType,
+ YoutubeBaseInfoExtractor,
+ _PoTokenContext,
+ _split_innertube_client,
+ short_client_name,
+)
+from ..openload import PhantomJSwrapper
+from ...jsinterp import JSInterpreter
+from ...networking.exceptions import HTTPError
+from ...utils import (
NO_DEFAULT,
ExtractorError,
LazyList,
- UserNotLive,
bug_reports_message,
- classproperty,
clean_html,
datetime_from_str,
filesize_from_tbr,
- filter_dict,
float_or_none,
format_field,
get_first,
int_or_none,
- is_html,
join_nonempty,
js_to_json,
mimetype2ext,
@@ -61,7 +59,6 @@ from ..utils import (
try_get,
unescapeHTML,
unified_strdate,
- unified_timestamp,
unsmuggle_url,
update_url_query,
url_or_none,
@@ -74,1211 +71,6 @@ STREAMING_DATA_INITIAL_PO_TOKEN = '__yt_dlp_po_token'
PO_TOKEN_GUIDE_URL = 'https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide'
-class _PoTokenContext(enum.Enum):
- PLAYER = 'player'
- GVS = 'gvs'
-
-
-# any clients starting with _ cannot be explicitly requested by the user
-INNERTUBE_CLIENTS = {
- 'web': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'WEB',
- 'clientVersion': '2.20241126.01.00',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'SUPPORTS_COOKIES': True,
- },
- # Safari UA returns pre-merged video+audio 144p/240p/360p/720p/1080p HLS formats
- 'web_safari': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'WEB',
- 'clientVersion': '2.20241126.01.00',
- 'userAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15,gzip(gfe)',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'SUPPORTS_COOKIES': True,
- },
- 'web_embedded': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'WEB_EMBEDDED_PLAYER',
- 'clientVersion': '1.20241201.00.00',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 56,
- 'SUPPORTS_COOKIES': True,
- },
- 'web_music': {
- 'INNERTUBE_HOST': 'music.youtube.com',
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'WEB_REMIX',
- 'clientVersion': '1.20241127.01.00',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 67,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'SUPPORTS_COOKIES': True,
- },
- # This client now requires sign-in for every video
- 'web_creator': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'WEB_CREATOR',
- 'clientVersion': '1.20241203.01.00',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 62,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'REQUIRE_AUTH': True,
- 'SUPPORTS_COOKIES': True,
- },
- 'android': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'ANDROID',
- 'clientVersion': '19.44.38',
- 'androidSdkVersion': 30,
- 'userAgent': 'com.google.android.youtube/19.44.38 (Linux; U; Android 11) gzip',
- 'osName': 'Android',
- 'osVersion': '11',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 3,
- 'REQUIRE_JS_PLAYER': False,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- },
- # This client now requires sign-in for every video
- 'android_music': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'ANDROID_MUSIC',
- 'clientVersion': '7.27.52',
- 'androidSdkVersion': 30,
- 'userAgent': 'com.google.android.apps.youtube.music/7.27.52 (Linux; U; Android 11) gzip',
- 'osName': 'Android',
- 'osVersion': '11',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 21,
- 'REQUIRE_JS_PLAYER': False,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'REQUIRE_AUTH': True,
- },
- # This client now requires sign-in for every video
- 'android_creator': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'ANDROID_CREATOR',
- 'clientVersion': '24.45.100',
- 'androidSdkVersion': 30,
- 'userAgent': 'com.google.android.apps.youtube.creator/24.45.100 (Linux; U; Android 11) gzip',
- 'osName': 'Android',
- 'osVersion': '11',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 14,
- 'REQUIRE_JS_PLAYER': False,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'REQUIRE_AUTH': True,
- },
- # YouTube Kids videos aren't returned on this client for some reason
- 'android_vr': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'ANDROID_VR',
- 'clientVersion': '1.60.19',
- 'deviceMake': 'Oculus',
- 'deviceModel': 'Quest 3',
- 'androidSdkVersion': 32,
- 'userAgent': 'com.google.android.apps.youtube.vr.oculus/1.60.19 (Linux; U; Android 12L; eureka-user Build/SQ3A.220605.009.A1) gzip',
- 'osName': 'Android',
- 'osVersion': '12L',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 28,
- 'REQUIRE_JS_PLAYER': False,
- },
- # iOS clients have HLS live streams. Setting device model to get 60fps formats.
- # See: https://github.com/TeamNewPipe/NewPipeExtractor/issues/680#issuecomment-1002724558
- 'ios': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'IOS',
- 'clientVersion': '20.03.02',
- 'deviceMake': 'Apple',
- 'deviceModel': 'iPhone16,2',
- 'userAgent': 'com.google.ios.youtube/20.03.02 (iPhone16,2; U; CPU iOS 18_2_1 like Mac OS X;)',
- 'osName': 'iPhone',
- 'osVersion': '18.2.1.22C161',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'REQUIRE_JS_PLAYER': False,
- },
- # This client now requires sign-in for every video
- 'ios_music': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'IOS_MUSIC',
- 'clientVersion': '7.27.0',
- 'deviceMake': 'Apple',
- 'deviceModel': 'iPhone16,2',
- 'userAgent': 'com.google.ios.youtubemusic/7.27.0 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)',
- 'osName': 'iPhone',
- 'osVersion': '18.1.0.22B83',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 26,
- 'REQUIRE_JS_PLAYER': False,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'REQUIRE_AUTH': True,
- },
- # This client now requires sign-in for every video
- 'ios_creator': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'IOS_CREATOR',
- 'clientVersion': '24.45.100',
- 'deviceMake': 'Apple',
- 'deviceModel': 'iPhone16,2',
- 'userAgent': 'com.google.ios.ytcreator/24.45.100 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)',
- 'osName': 'iPhone',
- 'osVersion': '18.1.0.22B83',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 15,
- 'REQUIRE_JS_PLAYER': False,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'REQUIRE_AUTH': True,
- },
- # mweb has 'ultralow' formats
- # See: https://github.com/yt-dlp/yt-dlp/pull/557
- 'mweb': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'MWEB',
- 'clientVersion': '2.20241202.07.00',
- # mweb previously did not require PO Token with this UA
- 'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 2,
- 'PO_TOKEN_REQUIRED_CONTEXTS': [_PoTokenContext.GVS],
- 'SUPPORTS_COOKIES': True,
- },
- 'tv': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'TVHTML5',
- 'clientVersion': '7.20250120.19.00',
- 'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
- 'SUPPORTS_COOKIES': True,
- },
- # This client now requires sign-in for every video
- # It was previously an age-gate workaround for videos that were `playable_in_embed`
- # It may still be useful if signed into an EU account that is not age-verified
- 'tv_embedded': {
- 'INNERTUBE_CONTEXT': {
- 'client': {
- 'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
- 'clientVersion': '2.0',
- },
- },
- 'INNERTUBE_CONTEXT_CLIENT_NAME': 85,
- 'REQUIRE_AUTH': True,
- 'SUPPORTS_COOKIES': True,
- },
-}
-
-
-def _split_innertube_client(client_name):
- variant, *base = client_name.rsplit('.', 1)
- if base:
- return variant, base[0], variant
- base, *variant = client_name.split('_', 1)
- return client_name, base, variant[0] if variant else None
-
-
-def short_client_name(client_name):
- main, *parts = _split_innertube_client(client_name)[0].split('_')
- return join_nonempty(main[:4], ''.join(x[0] for x in parts)).upper()
-
-
-def build_innertube_clients():
- THIRD_PARTY = {
- 'embedUrl': 'https://www.youtube.com/', # Can be any valid URL
- }
- BASE_CLIENTS = ('ios', 'web', 'tv', 'mweb', 'android')
- priority = qualities(BASE_CLIENTS[::-1])
-
- for client, ytcfg in tuple(INNERTUBE_CLIENTS.items()):
- ytcfg.setdefault('INNERTUBE_HOST', 'www.youtube.com')
- ytcfg.setdefault('REQUIRE_JS_PLAYER', True)
- ytcfg.setdefault('PO_TOKEN_REQUIRED_CONTEXTS', [])
- ytcfg.setdefault('REQUIRE_AUTH', False)
- ytcfg.setdefault('SUPPORTS_COOKIES', False)
- ytcfg.setdefault('PLAYER_PARAMS', None)
- ytcfg['INNERTUBE_CONTEXT']['client'].setdefault('hl', 'en')
-
- _, base_client, variant = _split_innertube_client(client)
- ytcfg['priority'] = 10 * priority(base_client)
-
- if variant == 'embedded':
- ytcfg['INNERTUBE_CONTEXT']['thirdParty'] = THIRD_PARTY
- ytcfg['priority'] -= 2
- elif variant:
- ytcfg['priority'] -= 3
-
-
-build_innertube_clients()
-
-
-class BadgeType(enum.Enum):
- AVAILABILITY_UNLISTED = enum.auto()
- AVAILABILITY_PRIVATE = enum.auto()
- AVAILABILITY_PUBLIC = enum.auto()
- AVAILABILITY_PREMIUM = enum.auto()
- AVAILABILITY_SUBSCRIPTION = enum.auto()
- LIVE_NOW = enum.auto()
- VERIFIED = enum.auto()
-
-
-class YoutubeBaseInfoExtractor(InfoExtractor):
- """Provide base functions for Youtube extractors"""
-
- _RESERVED_NAMES = (
- r'channel|c|user|playlist|watch|w|v|embed|e|live|watch_popup|clip|'
- r'shorts|movies|results|search|shared|hashtag|trending|explore|feed|feeds|'
- r'browse|oembed|get_video_info|iframe_api|s/player|source|'
- r'storefront|oops|index|account|t/terms|about|upload|signin|logout')
-
- _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM|WL|LL|LM)'
-
- # _NETRC_MACHINE = 'youtube'
-
- # If True it will raise an error if no login info is provided
- _LOGIN_REQUIRED = False
-
- _INVIDIOUS_SITES = (
- # invidious-redirect websites
- r'(?:www\.)?redirect\.invidious\.io',
- r'(?:(?:www|dev)\.)?invidio\.us',
- # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/docs/instances.md
- r'(?:www\.)?invidious\.pussthecat\.org',
- r'(?:www\.)?invidious\.zee\.li',
- r'(?:www\.)?invidious\.ethibox\.fr',
- r'(?:www\.)?iv\.ggtyler\.dev',
- r'(?:www\.)?inv\.vern\.i2p',
- r'(?:www\.)?am74vkcrjp2d5v36lcdqgsj2m6x36tbrkhsruoegwfcizzabnfgf5zyd\.onion',
- r'(?:www\.)?inv\.riverside\.rocks',
- r'(?:www\.)?invidious\.silur\.me',
- r'(?:www\.)?inv\.bp\.projectsegfau\.lt',
- r'(?:www\.)?invidious\.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid\.onion',
- r'(?:www\.)?invidious\.slipfox\.xyz',
- r'(?:www\.)?invidious\.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd\.onion',
- r'(?:www\.)?inv\.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad\.onion',
- r'(?:www\.)?invidious\.tiekoetter\.com',
- r'(?:www\.)?iv\.odysfvr23q5wgt7i456o5t3trw2cw5dgn56vbjfbq2m7xsc5vqbqpcyd\.onion',
- r'(?:www\.)?invidious\.nerdvpn\.de',
- r'(?:www\.)?invidious\.weblibre\.org',
- r'(?:www\.)?inv\.odyssey346\.dev',
- r'(?:www\.)?invidious\.dhusch\.de',
- r'(?:www\.)?iv\.melmac\.space',
- r'(?:www\.)?watch\.thekitty\.zone',
- r'(?:www\.)?invidious\.privacydev\.net',
- r'(?:www\.)?ng27owmagn5amdm7l5s3rsqxwscl5ynppnis5dqcasogkyxcfqn7psid\.onion',
- r'(?:www\.)?invidious\.drivet\.xyz',
- r'(?:www\.)?vid\.priv\.au',
- r'(?:www\.)?euxxcnhsynwmfidvhjf6uzptsmh4dipkmgdmcmxxuo7tunp3ad2jrwyd\.onion',
- r'(?:www\.)?inv\.vern\.cc',
- r'(?:www\.)?invidious\.esmailelbob\.xyz',
- r'(?:www\.)?invidious\.sethforprivacy\.com',
- r'(?:www\.)?yt\.oelrichsgarcia\.de',
- r'(?:www\.)?yt\.artemislena\.eu',
- r'(?:www\.)?invidious\.flokinet\.to',
- r'(?:www\.)?invidious\.baczek\.me',
- r'(?:www\.)?y\.com\.sb',
- r'(?:www\.)?invidious\.epicsite\.xyz',
- r'(?:www\.)?invidious\.lidarshield\.cloud',
- r'(?:www\.)?yt\.funami\.tech',
- r'(?:www\.)?invidious\.3o7z6yfxhbw7n3za4rss6l434kmv55cgw2vuziwuigpwegswvwzqipyd\.onion',
- r'(?:www\.)?osbivz6guyeahrwp2lnwyjk2xos342h4ocsxyqrlaopqjuhwn2djiiyd\.onion',
- r'(?:www\.)?u2cvlit75owumwpy4dj2hsmvkq7nvrclkpht7xgyye2pyoxhpmclkrad\.onion',
- # youtube-dl invidious instances list
- r'(?:(?:www|no)\.)?invidiou\.sh',
- r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
- r'(?:www\.)?invidious\.kabi\.tk',
- r'(?:www\.)?invidious\.mastodon\.host',
- r'(?:www\.)?invidious\.zapashcanon\.fr',
- r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
- r'(?:www\.)?invidious\.tinfoil-hat\.net',
- r'(?:www\.)?invidious\.himiko\.cloud',
- r'(?:www\.)?invidious\.reallyancient\.tech',
- r'(?:www\.)?invidious\.tube',
- r'(?:www\.)?invidiou\.site',
- r'(?:www\.)?invidious\.site',
- r'(?:www\.)?invidious\.xyz',
- r'(?:www\.)?invidious\.nixnet\.xyz',
- r'(?:www\.)?invidious\.048596\.xyz',
- r'(?:www\.)?invidious\.drycat\.fr',
- r'(?:www\.)?inv\.skyn3t\.in',
- r'(?:www\.)?tube\.poal\.co',
- r'(?:www\.)?tube\.connect\.cafe',
- r'(?:www\.)?vid\.wxzm\.sx',
- r'(?:www\.)?vid\.mint\.lgbt',
- r'(?:www\.)?vid\.puffyan\.us',
- r'(?:www\.)?yewtu\.be',
- r'(?:www\.)?yt\.elukerio\.org',
- r'(?:www\.)?yt\.lelux\.fi',
- r'(?:www\.)?invidious\.ggc-project\.de',
- r'(?:www\.)?yt\.maisputain\.ovh',
- r'(?:www\.)?ytprivate\.com',
- r'(?:www\.)?invidious\.13ad\.de',
- r'(?:www\.)?invidious\.toot\.koeln',
- r'(?:www\.)?invidious\.fdn\.fr',
- r'(?:www\.)?watch\.nettohikari\.com',
- r'(?:www\.)?invidious\.namazso\.eu',
- r'(?:www\.)?invidious\.silkky\.cloud',
- r'(?:www\.)?invidious\.exonip\.de',
- r'(?:www\.)?invidious\.riverside\.rocks',
- r'(?:www\.)?invidious\.blamefran\.net',
- r'(?:www\.)?invidious\.moomoo\.de',
- r'(?:www\.)?ytb\.trom\.tf',
- r'(?:www\.)?yt\.cyberhost\.uk',
- r'(?:www\.)?kgg2m7yk5aybusll\.onion',
- r'(?:www\.)?qklhadlycap4cnod\.onion',
- r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
- r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
- r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
- r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
- r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
- r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
- r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
- r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
- r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
- r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
- # piped instances from https://github.com/TeamPiped/Piped/wiki/Instances
- r'(?:www\.)?piped\.kavin\.rocks',
- r'(?:www\.)?piped\.tokhmi\.xyz',
- r'(?:www\.)?piped\.syncpundit\.io',
- r'(?:www\.)?piped\.mha\.fi',
- r'(?:www\.)?watch\.whatever\.social',
- r'(?:www\.)?piped\.garudalinux\.org',
- r'(?:www\.)?piped\.rivo\.lol',
- r'(?:www\.)?piped-libre\.kavin\.rocks',
- r'(?:www\.)?yt\.jae\.fi',
- r'(?:www\.)?piped\.mint\.lgbt',
- r'(?:www\.)?il\.ax',
- r'(?:www\.)?piped\.esmailelbob\.xyz',
- r'(?:www\.)?piped\.projectsegfau\.lt',
- r'(?:www\.)?piped\.privacydev\.net',
- r'(?:www\.)?piped\.palveluntarjoaja\.eu',
- r'(?:www\.)?piped\.smnz\.de',
- r'(?:www\.)?piped\.adminforge\.de',
- r'(?:www\.)?watch\.whatevertinfoil\.de',
- r'(?:www\.)?piped\.qdi\.fi',
- r'(?:(?:www|cf)\.)?piped\.video',
- r'(?:www\.)?piped\.aeong\.one',
- r'(?:www\.)?piped\.moomoo\.me',
- r'(?:www\.)?piped\.chauvet\.pro',
- r'(?:www\.)?watch\.leptons\.xyz',
- r'(?:www\.)?pd\.vern\.cc',
- r'(?:www\.)?piped\.hostux\.net',
- r'(?:www\.)?piped\.lunar\.icu',
- # Hyperpipe instances from https://hyperpipe.codeberg.page/
- r'(?:www\.)?hyperpipe\.surge\.sh',
- r'(?:www\.)?hyperpipe\.esmailelbob\.xyz',
- r'(?:www\.)?listen\.whatever\.social',
- r'(?:www\.)?music\.adminforge\.de',
- )
-
- # extracted from account/account_menu ep
- # XXX: These are the supported YouTube UI and API languages,
- # which is slightly different from languages supported for translation in YouTube studio
- _SUPPORTED_LANG_CODES = [
- 'af', 'az', 'id', 'ms', 'bs', 'ca', 'cs', 'da', 'de', 'et', 'en-IN', 'en-GB', 'en', 'es',
- 'es-419', 'es-US', 'eu', 'fil', 'fr', 'fr-CA', 'gl', 'hr', 'zu', 'is', 'it', 'sw', 'lv',
- 'lt', 'hu', 'nl', 'no', 'uz', 'pl', 'pt-PT', 'pt', 'ro', 'sq', 'sk', 'sl', 'sr-Latn', 'fi',
- 'sv', 'vi', 'tr', 'be', 'bg', 'ky', 'kk', 'mk', 'mn', 'ru', 'sr', 'uk', 'el', 'hy', 'iw',
- 'ur', 'ar', 'fa', 'ne', 'mr', 'hi', 'as', 'bn', 'pa', 'gu', 'or', 'ta', 'te', 'kn', 'ml',
- 'si', 'th', 'lo', 'my', 'ka', 'am', 'km', 'zh-CN', 'zh-TW', 'zh-HK', 'ja', 'ko',
- ]
-
- _IGNORED_WARNINGS = {
- 'Unavailable videos will be hidden during playback',
- 'Unavailable videos are hidden',
- }
-
- _YT_HANDLE_RE = r'@[\w.-]{3,30}' # https://support.google.com/youtube/answer/11585688?hl=en
- _YT_CHANNEL_UCID_RE = r'UC[\w-]{22}'
-
- _NETRC_MACHINE = 'youtube'
-
- def ucid_or_none(self, ucid):
- return self._search_regex(rf'^({self._YT_CHANNEL_UCID_RE})$', ucid, 'UC-id', default=None)
-
- def handle_or_none(self, handle):
- return self._search_regex(rf'^({self._YT_HANDLE_RE})$', urllib.parse.unquote(handle or ''),
- '@-handle', default=None)
-
- def handle_from_url(self, url):
- return self._search_regex(rf'^(?:https?://(?:www\.)?youtube\.com)?/({self._YT_HANDLE_RE})',
- urllib.parse.unquote(url or ''), 'channel handle', default=None)
-
- def ucid_from_url(self, url):
- return self._search_regex(rf'^(?:https?://(?:www\.)?youtube\.com)?/({self._YT_CHANNEL_UCID_RE})',
- url, 'channel id', default=None)
-
- @functools.cached_property
- def _preferred_lang(self):
- """
- Returns a language code supported by YouTube for the user preferred language.
- Returns None if no preferred language set.
- """
- preferred_lang = self._configuration_arg('lang', ie_key='Youtube', casesense=True, default=[''])[0]
- if not preferred_lang:
- return
- if preferred_lang not in self._SUPPORTED_LANG_CODES:
- raise ExtractorError(
- f'Unsupported language code: {preferred_lang}. Supported language codes (case-sensitive): {join_nonempty(*self._SUPPORTED_LANG_CODES, delim=", ")}.',
- expected=True)
- elif preferred_lang != 'en':
- self.report_warning(
- f'Preferring "{preferred_lang}" translated fields. Note that some metadata extraction may fail or be incorrect.')
- return preferred_lang
-
- def _initialize_consent(self):
- cookies = self._get_cookies('https://www.youtube.com/')
- if cookies.get('__Secure-3PSID'):
- return
- socs = cookies.get('SOCS')
- if socs and not socs.value.startswith('CAA'): # not consented
- return
- self._set_cookie('.youtube.com', 'SOCS', 'CAI', secure=True) # accept all (required for mixes)
-
- def _initialize_pref(self):
- cookies = self._get_cookies('https://www.youtube.com/')
- pref_cookie = cookies.get('PREF')
- pref = {}
- if pref_cookie:
- try:
- pref = dict(urllib.parse.parse_qsl(pref_cookie.value))
- except ValueError:
- self.report_warning('Failed to parse user PREF cookie' + bug_reports_message())
- pref.update({'hl': self._preferred_lang or 'en', 'tz': 'UTC'})
- self._set_cookie('.youtube.com', name='PREF', value=urllib.parse.urlencode(pref))
-
- def _initialize_cookie_auth(self):
- yt_sapisid, yt_1psapisid, yt_3psapisid = self._get_sid_cookies()
- if yt_sapisid or yt_1psapisid or yt_3psapisid:
- self.write_debug('Found YouTube account cookies')
-
- def _real_initialize(self):
- self._initialize_pref()
- self._initialize_consent()
- self._initialize_cookie_auth()
- self._check_login_required()
-
- def _perform_login(self, username, password):
- if username.startswith('oauth'):
- raise ExtractorError(
- f'Login with OAuth is no longer supported. {self._youtube_login_hint}', expected=True)
-
- self.report_warning(
- f'Login with password is not supported for YouTube. {self._youtube_login_hint}')
-
- @property
- def _youtube_login_hint(self):
- return (f'{self._login_hint(method="cookies")}. Also see '
- 'https://github.com/yt-dlp/yt-dlp/wiki/Extractors#exporting-youtube-cookies '
- 'for tips on effectively exporting YouTube cookies')
-
- def _check_login_required(self):
- if self._LOGIN_REQUIRED and not self.is_authenticated:
- self.raise_login_required(
- f'Login details are needed to download this content. {self._youtube_login_hint}', method=None)
-
- _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*='
- _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*='
-
- def _get_default_ytcfg(self, client='web'):
- return copy.deepcopy(INNERTUBE_CLIENTS[client])
-
- def _get_innertube_host(self, client='web'):
- return INNERTUBE_CLIENTS[client]['INNERTUBE_HOST']
-
- def _ytcfg_get_safe(self, ytcfg, getter, expected_type=None, default_client='web'):
- # try_get but with fallback to default ytcfg client values when present
- _func = lambda y: try_get(y, getter, expected_type)
- return _func(ytcfg) or _func(self._get_default_ytcfg(default_client))
-
- def _extract_client_name(self, ytcfg, default_client='web'):
- return self._ytcfg_get_safe(
- ytcfg, (lambda x: x['INNERTUBE_CLIENT_NAME'],
- lambda x: x['INNERTUBE_CONTEXT']['client']['clientName']), str, default_client)
-
- def _extract_client_version(self, ytcfg, default_client='web'):
- return self._ytcfg_get_safe(
- ytcfg, (lambda x: x['INNERTUBE_CLIENT_VERSION'],
- lambda x: x['INNERTUBE_CONTEXT']['client']['clientVersion']), str, default_client)
-
- def _select_api_hostname(self, req_api_hostname, default_client=None):
- return (self._configuration_arg('innertube_host', [''], ie_key=YoutubeIE.ie_key())[0]
- or req_api_hostname or self._get_innertube_host(default_client or 'web'))
-
- def _extract_context(self, ytcfg=None, default_client='web'):
- context = get_first(
- (ytcfg, self._get_default_ytcfg(default_client)), 'INNERTUBE_CONTEXT', expected_type=dict)
- # Enforce language and tz for extraction
- client_context = traverse_obj(context, 'client', expected_type=dict, default={})
- client_context.update({'hl': self._preferred_lang or 'en', 'timeZone': 'UTC', 'utcOffsetMinutes': 0})
- return context
-
- @staticmethod
- def _make_sid_authorization(scheme, sid, origin, additional_parts):
- timestamp = str(round(time.time()))
-
- hash_parts = []
- if additional_parts:
- hash_parts.append(':'.join(additional_parts.values()))
- hash_parts.extend([timestamp, sid, origin])
- sidhash = hashlib.sha1(' '.join(hash_parts).encode()).hexdigest()
-
- parts = [timestamp, sidhash]
- if additional_parts:
- parts.append(''.join(additional_parts))
-
- return f'{scheme} {"_".join(parts)}'
-
- def _get_sid_cookies(self):
- """
- Get SAPISID, 1PSAPISID, 3PSAPISID cookie values
- @returns sapisid, 1psapisid, 3psapisid
- """
- yt_cookies = self._get_cookies('https://www.youtube.com')
- yt_sapisid = try_call(lambda: yt_cookies['SAPISID'].value)
- yt_3papisid = try_call(lambda: yt_cookies['__Secure-3PAPISID'].value)
- yt_1papisid = try_call(lambda: yt_cookies['__Secure-1PAPISID'].value)
-
- # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
- # YouTube also falls back to __Secure-3PAPISID if SAPISID is missing.
- # See: https://github.com/yt-dlp/yt-dlp/issues/393
-
- return yt_sapisid or yt_3papisid, yt_1papisid, yt_3papisid
-
- def _get_sid_authorization_header(self, origin='https://www.youtube.com', user_session_id=None):
- """
- Generate API Session ID Authorization for Innertube requests. Assumes all requests are secure (https).
- @param origin: Origin URL
- @param user_session_id: Optional User Session ID
- @return: Authorization header value
- """
-
- authorizations = []
- additional_parts = {}
- if user_session_id:
- additional_parts['u'] = user_session_id
-
- yt_sapisid, yt_1psapisid, yt_3psapisid = self._get_sid_cookies()
-
- for scheme, sid in (('SAPISIDHASH', yt_sapisid),
- ('SAPISID1PHASH', yt_1psapisid),
- ('SAPISID3PHASH', yt_3psapisid)):
- if sid:
- authorizations.append(self._make_sid_authorization(scheme, sid, origin, additional_parts))
-
- if not authorizations:
- return None
-
- return ' '.join(authorizations)
-
- def _call_api(self, ep, query, video_id, fatal=True, headers=None,
- note='Downloading API JSON', errnote='Unable to download API page',
- context=None, api_key=None, api_hostname=None, default_client='web'):
-
- data = {'context': context} if context else {'context': self._extract_context(default_client=default_client)}
- data.update(query)
- real_headers = self.generate_api_headers(default_client=default_client)
- real_headers.update({'content-type': 'application/json'})
- if headers:
- real_headers.update(headers)
- return self._download_json(
- f'https://{self._select_api_hostname(api_hostname, default_client)}/youtubei/v1/{ep}',
- video_id=video_id, fatal=fatal, note=note, errnote=errnote,
- data=json.dumps(data).encode('utf8'), headers=real_headers,
- query=filter_dict({
- 'key': self._configuration_arg(
- 'innertube_key', [api_key], ie_key=YoutubeIE.ie_key(), casesense=True)[0],
- 'prettyPrint': 'false',
- }, cndn=lambda _, v: v))
-
- def extract_yt_initial_data(self, item_id, webpage, fatal=True):
- return self._search_json(self._YT_INITIAL_DATA_RE, webpage, 'yt initial data', item_id, fatal=fatal)
-
- @staticmethod
- def _extract_session_index(*data):
- """
- Index of current account in account list.
- See: https://github.com/yt-dlp/yt-dlp/pull/519
- """
- for ytcfg in data:
- session_index = int_or_none(try_get(ytcfg, lambda x: x['SESSION_INDEX']))
- if session_index is not None:
- return session_index
-
- @staticmethod
- def _parse_data_sync_id(data_sync_id):
- """
- Parse data_sync_id into delegated_session_id and user_session_id.
-
- data_sync_id is of the form "delegated_session_id||user_session_id" for secondary channel
- and just "user_session_id||" for primary channel.
-
- @param data_sync_id: data_sync_id string
- @return: Tuple of (delegated_session_id, user_session_id)
- """
- if not data_sync_id:
- return None, None
- first, _, second = data_sync_id.partition('||')
- if second:
- return first, second
- return None, first
-
- def _extract_delegated_session_id(self, *args):
- """
- Extract current delegated session ID required to download private playlists of secondary channels
- @params response and/or ytcfg
- @return: delegated session ID
- """
- # ytcfg includes channel_syncid if on secondary channel
- if delegated_sid := traverse_obj(args, (..., 'DELEGATED_SESSION_ID', {str}, any)):
- return delegated_sid
-
- data_sync_id = self._extract_data_sync_id(*args)
- return self._parse_data_sync_id(data_sync_id)[0]
-
- def _extract_user_session_id(self, *args):
- """
- Extract current user session ID
- @params response and/or ytcfg
- @return: user session ID
- """
- if user_sid := traverse_obj(args, (..., 'USER_SESSION_ID', {str}, any)):
- return user_sid
-
- data_sync_id = self._extract_data_sync_id(*args)
- return self._parse_data_sync_id(data_sync_id)[1]
-
- def _extract_data_sync_id(self, *args):
- """
- Extract current account dataSyncId.
- In the format DELEGATED_SESSION_ID||USER_SESSION_ID or USER_SESSION_ID||
- @params response and/or ytcfg
- """
- if data_sync_id := self._configuration_arg('data_sync_id', [None], ie_key=YoutubeIE, casesense=True)[0]:
- return data_sync_id
-
- return traverse_obj(
- args, (..., ('DATASYNC_ID', ('responseContext', 'mainAppWebResponseContext', 'datasyncId')), {str}, any))
-
- def _extract_visitor_data(self, *args):
- """
- Extracts visitorData from an API response or ytcfg
- Appears to be used to track session state
- """
- if visitor_data := self._configuration_arg('visitor_data', [None], ie_key=YoutubeIE, casesense=True)[0]:
- return visitor_data
- return get_first(
- args, [('VISITOR_DATA', ('INNERTUBE_CONTEXT', 'client', 'visitorData'), ('responseContext', 'visitorData'))],
- expected_type=str)
-
- @functools.cached_property
- def is_authenticated(self):
- return bool(self._get_sid_authorization_header())
-
- def extract_ytcfg(self, video_id, webpage):
- if not webpage:
- return {}
- return self._parse_json(
- self._search_regex(
- r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
- default='{}'), video_id, fatal=False) or {}
-
- def _generate_cookie_auth_headers(self, *, ytcfg=None, delegated_session_id=None, user_session_id=None, session_index=None, origin=None, **kwargs):
- headers = {}
- delegated_session_id = delegated_session_id or self._extract_delegated_session_id(ytcfg)
- if delegated_session_id:
- headers['X-Goog-PageId'] = delegated_session_id
- if session_index is None:
- session_index = self._extract_session_index(ytcfg)
- if delegated_session_id or session_index is not None:
- headers['X-Goog-AuthUser'] = session_index if session_index is not None else 0
-
- auth = self._get_sid_authorization_header(origin, user_session_id=user_session_id or self._extract_user_session_id(ytcfg))
- if auth is not None:
- headers['Authorization'] = auth
- headers['X-Origin'] = origin
-
- if traverse_obj(ytcfg, 'LOGGED_IN', expected_type=bool):
- headers['X-Youtube-Bootstrap-Logged-In'] = 'true'
-
- return headers
-
- def generate_api_headers(
- self, *, ytcfg=None, delegated_session_id=None, user_session_id=None, session_index=None,
- visitor_data=None, api_hostname=None, default_client='web', **kwargs):
-
- origin = 'https://' + (self._select_api_hostname(api_hostname, default_client))
- headers = {
- 'X-YouTube-Client-Name': str(
- self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT_CLIENT_NAME'], default_client=default_client)),
- 'X-YouTube-Client-Version': self._extract_client_version(ytcfg, default_client),
- 'Origin': origin,
- 'X-Goog-Visitor-Id': visitor_data or self._extract_visitor_data(ytcfg),
- 'User-Agent': self._ytcfg_get_safe(ytcfg, lambda x: x['INNERTUBE_CONTEXT']['client']['userAgent'], default_client=default_client),
- **self._generate_cookie_auth_headers(
- ytcfg=ytcfg,
- delegated_session_id=delegated_session_id,
- user_session_id=user_session_id,
- session_index=session_index,
- origin=origin),
- }
- return filter_dict(headers)
-
- def _download_webpage_with_retries(self, *args, retry_fatal=False, retry_on_status=None, **kwargs):
- for retry in self.RetryManager(fatal=retry_fatal):
- try:
- return self._download_webpage(*args, **kwargs)
- except ExtractorError as e:
- if isinstance(e.cause, network_exceptions):
- if not isinstance(e.cause, HTTPError) or e.cause.status not in (retry_on_status or (403, 429)):
- retry.error = e
- continue
- self._error_or_warning(e, fatal=retry_fatal)
- break
-
- def _download_ytcfg(self, client, video_id):
- url = {
- 'web': 'https://www.youtube.com',
- 'web_music': 'https://music.youtube.com',
- 'web_embedded': f'https://www.youtube.com/embed/{video_id}?html5=1',
- 'tv': 'https://www.youtube.com/tv',
- }.get(client)
- if not url:
- return {}
- webpage = self._download_webpage_with_retries(
- url, video_id, note=f'Downloading {client.replace("_", " ").strip()} client config',
- headers=traverse_obj(self._get_default_ytcfg(client), {
- 'User-Agent': ('INNERTUBE_CONTEXT', 'client', 'userAgent', {str}),
- }))
- return self.extract_ytcfg(video_id, webpage) or {}
-
- @staticmethod
- def _build_api_continuation_query(continuation, ctp=None):
- query = {
- 'continuation': continuation,
- }
- # TODO: Inconsistency with clickTrackingParams.
- # Currently we have a fixed ctp contained within context (from ytcfg)
- # and a ctp in root query for continuation.
- if ctp:
- query['clickTracking'] = {'clickTrackingParams': ctp}
- return query
-
- @classmethod
- def _extract_next_continuation_data(cls, renderer):
- next_continuation = try_get(
- renderer, (lambda x: x['continuations'][0]['nextContinuationData'],
- lambda x: x['continuation']['reloadContinuationData']), dict)
- if not next_continuation:
- return
- continuation = next_continuation.get('continuation')
- if not continuation:
- return
- ctp = next_continuation.get('clickTrackingParams')
- return cls._build_api_continuation_query(continuation, ctp)
-
- @classmethod
- def _extract_continuation_ep_data(cls, continuation_ep: dict):
- if isinstance(continuation_ep, dict):
- continuation = try_get(
- continuation_ep, lambda x: x['continuationCommand']['token'], str)
- if not continuation:
- return
- ctp = continuation_ep.get('clickTrackingParams')
- return cls._build_api_continuation_query(continuation, ctp)
-
- @classmethod
- def _extract_continuation(cls, renderer):
- next_continuation = cls._extract_next_continuation_data(renderer)
- if next_continuation:
- return next_continuation
-
- return traverse_obj(renderer, (
- ('contents', 'items', 'rows'), ..., 'continuationItemRenderer',
- ('continuationEndpoint', ('button', 'buttonRenderer', 'command')),
- ), get_all=False, expected_type=cls._extract_continuation_ep_data)
-
- @classmethod
- def _extract_alerts(cls, data):
- for alert_dict in try_get(data, lambda x: x['alerts'], list) or []:
- if not isinstance(alert_dict, dict):
- continue
- for alert in alert_dict.values():
- alert_type = alert.get('type')
- if not alert_type:
- continue
- message = cls._get_text(alert, 'text')
- if message:
- yield alert_type, message
-
- def _report_alerts(self, alerts, expected=True, fatal=True, only_once=False):
- errors, warnings = [], []
- for alert_type, alert_message in alerts:
- if alert_type.lower() == 'error' and fatal:
- errors.append([alert_type, alert_message])
- elif alert_message not in self._IGNORED_WARNINGS:
- warnings.append([alert_type, alert_message])
-
- for alert_type, alert_message in (warnings + errors[:-1]):
- self.report_warning(f'YouTube said: {alert_type} - {alert_message}', only_once=only_once)
- if errors:
- raise ExtractorError(f'YouTube said: {errors[-1][1]}', expected=expected)
-
- def _extract_and_report_alerts(self, data, *args, **kwargs):
- return self._report_alerts(self._extract_alerts(data), *args, **kwargs)
-
- def _extract_badges(self, badge_list: list):
- """
- Extract known BadgeType's from a list of badge renderers.
- @returns [{'type': BadgeType}]
- """
- icon_type_map = {
- 'PRIVACY_UNLISTED': BadgeType.AVAILABILITY_UNLISTED,
- 'PRIVACY_PRIVATE': BadgeType.AVAILABILITY_PRIVATE,
- 'PRIVACY_PUBLIC': BadgeType.AVAILABILITY_PUBLIC,
- 'CHECK_CIRCLE_THICK': BadgeType.VERIFIED,
- 'OFFICIAL_ARTIST_BADGE': BadgeType.VERIFIED,
- 'CHECK': BadgeType.VERIFIED,
- }
-
- badge_style_map = {
- 'BADGE_STYLE_TYPE_MEMBERS_ONLY': BadgeType.AVAILABILITY_SUBSCRIPTION,
- 'BADGE_STYLE_TYPE_PREMIUM': BadgeType.AVAILABILITY_PREMIUM,
- 'BADGE_STYLE_TYPE_LIVE_NOW': BadgeType.LIVE_NOW,
- 'BADGE_STYLE_TYPE_VERIFIED': BadgeType.VERIFIED,
- 'BADGE_STYLE_TYPE_VERIFIED_ARTIST': BadgeType.VERIFIED,
- }
-
- label_map = {
- 'unlisted': BadgeType.AVAILABILITY_UNLISTED,
- 'private': BadgeType.AVAILABILITY_PRIVATE,
- 'members only': BadgeType.AVAILABILITY_SUBSCRIPTION,
- 'live': BadgeType.LIVE_NOW,
- 'premium': BadgeType.AVAILABILITY_PREMIUM,
- 'verified': BadgeType.VERIFIED,
- 'official artist channel': BadgeType.VERIFIED,
- }
-
- badges = []
- for badge in traverse_obj(badge_list, (..., lambda key, _: re.search(r'[bB]adgeRenderer$', key))):
- badge_type = (
- icon_type_map.get(traverse_obj(badge, ('icon', 'iconType'), expected_type=str))
- or badge_style_map.get(traverse_obj(badge, 'style'))
- )
- if badge_type:
- badges.append({'type': badge_type})
- continue
-
- # fallback, won't work in some languages
- label = traverse_obj(
- badge, 'label', ('accessibilityData', 'label'), 'tooltip', 'iconTooltip', get_all=False, expected_type=str, default='')
- for match, label_badge_type in label_map.items():
- if match in label.lower():
- badges.append({'type': label_badge_type})
- break
-
- return badges
-
- @staticmethod
- def _has_badge(badges, badge_type):
- return bool(traverse_obj(badges, lambda _, v: v['type'] == badge_type))
-
- @staticmethod
- def _get_text(data, *path_list, max_runs=None):
- for path in path_list or [None]:
- if path is None:
- obj = [data]
- else:
- obj = traverse_obj(data, path, default=[])
- if not any(key is ... or isinstance(key, (list, tuple)) for key in variadic(path)):
- obj = [obj]
- for item in obj:
- text = try_get(item, lambda x: x['simpleText'], str)
- if text:
- return text
- runs = try_get(item, lambda x: x['runs'], list) or []
- if not runs and isinstance(item, list):
- runs = item
-
- runs = runs[:min(len(runs), max_runs or len(runs))]
- text = ''.join(traverse_obj(runs, (..., 'text'), expected_type=str))
- if text:
- return text
-
- def _get_count(self, data, *path_list):
- count_text = self._get_text(data, *path_list) or ''
- count = parse_count(count_text)
- if count is None:
- count = str_to_int(
- self._search_regex(r'^([\d,]+)', re.sub(r'\s', '', count_text), 'count', default=None))
- return count
-
- @staticmethod
- def _extract_thumbnails(data, *path_list, final_key='thumbnails'):
- """
- Extract thumbnails from thumbnails dict
- @param path_list: path list to level that contains 'thumbnails' key
- """
- thumbnails = []
- for path in path_list or [()]:
- for thumbnail in traverse_obj(data, (*variadic(path), final_key, ...)):
- thumbnail_url = url_or_none(thumbnail.get('url'))
- if not thumbnail_url:
- continue
- # Sometimes youtube gives a wrong thumbnail URL. See:
- # https://github.com/yt-dlp/yt-dlp/issues/233
- # https://github.com/ytdl-org/youtube-dl/issues/28023
- if 'maxresdefault' in thumbnail_url:
- thumbnail_url = thumbnail_url.split('?')[0]
- thumbnails.append({
- 'url': thumbnail_url,
- 'height': int_or_none(thumbnail.get('height')),
- 'width': int_or_none(thumbnail.get('width')),
- })
- return thumbnails
-
- @staticmethod
- def extract_relative_time(relative_time_text):
- """
- Extracts a relative time from string and converts to dt object
- e.g. 'streamed 6 days ago', '5 seconds ago (edited)', 'updated today', '8 yr ago'
- """
-
- # XXX: this could be moved to a general function in utils/_utils.py
- # The relative time text strings are roughly the same as what
- # Javascript's Intl.RelativeTimeFormat function generates.
- # See: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/RelativeTimeFormat
- mobj = re.search(
- r'(?P<start>today|yesterday|now)|(?P<time>\d+)\s*(?P<unit>sec(?:ond)?|s|min(?:ute)?|h(?:our|r)?|d(?:ay)?|w(?:eek|k)?|mo(?:nth)?|y(?:ear|r)?)s?\s*ago',
- relative_time_text)
- if mobj:
- start = mobj.group('start')
- if start:
- return datetime_from_str(start)
- try:
- return datetime_from_str('now-{}{}'.format(mobj.group('time'), mobj.group('unit')))
- except ValueError:
- return None
-
- def _parse_time_text(self, text):
- if not text:
- return
- dt_ = self.extract_relative_time(text)
- timestamp = None
- if isinstance(dt_, dt.datetime):
- timestamp = calendar.timegm(dt_.timetuple())
-
- if timestamp is None:
- timestamp = (
- unified_timestamp(text) or unified_timestamp(
- self._search_regex(
- (r'([a-z]+\s*\d{1,2},?\s*20\d{2})', r'(?:.+|^)(?:live|premieres|ed|ing)(?:\s*(?:on|for))?\s*(.+\d)'),
- text.lower(), 'time text', default=None)))
-
- if text and timestamp is None and self._preferred_lang in (None, 'en'):
- self.report_warning(
- f'Cannot parse localized time text "{text}"', only_once=True)
- return timestamp
-
- def _extract_response(self, item_id, query, note='Downloading API JSON', headers=None,
- ytcfg=None, check_get_keys=None, ep='browse', fatal=True, api_hostname=None,
- default_client='web'):
- raise_for_incomplete = bool(self._configuration_arg('raise_incomplete_data', ie_key=YoutubeIE))
- # Incomplete Data should be a warning by default when retries are exhausted, while other errors should be fatal.
- icd_retries = iter(self.RetryManager(fatal=raise_for_incomplete))
- icd_rm = next(icd_retries)
- main_retries = iter(self.RetryManager())
- main_rm = next(main_retries)
- # Manual retry loop for multiple RetryManagers
- # The proper RetryManager MUST be advanced after an error
- # and its result MUST be checked if the manager is non fatal
- while True:
- try:
- response = self._call_api(
- ep=ep, fatal=True, headers=headers,
- video_id=item_id, query=query, note=note,
- context=self._extract_context(ytcfg, default_client),
- api_hostname=api_hostname, default_client=default_client)
- except ExtractorError as e:
- if not isinstance(e.cause, network_exceptions):
- return self._error_or_warning(e, fatal=fatal)
- elif not isinstance(e.cause, HTTPError):
- main_rm.error = e
- next(main_retries)
- continue
-
- first_bytes = e.cause.response.read(512)
- if not is_html(first_bytes):
- yt_error = try_get(
- self._parse_json(
- self._webpage_read_content(e.cause.response, None, item_id, prefix=first_bytes) or '{}', item_id, fatal=False),
- lambda x: x['error']['message'], str)
- if yt_error:
- self._report_alerts([('ERROR', yt_error)], fatal=False)
- # Downloading page may result in intermittent 5xx HTTP error
- # Sometimes a 404 is also received. See: https://github.com/ytdl-org/youtube-dl/issues/28289
- # We also want to catch all other network exceptions since errors in later pages can be troublesome
- # See https://github.com/yt-dlp/yt-dlp/issues/507#issuecomment-880188210
- if e.cause.status not in (403, 429):
- main_rm.error = e
- next(main_retries)
- continue
- return self._error_or_warning(e, fatal=fatal)
-
- try:
- self._extract_and_report_alerts(response, only_once=True)
- except ExtractorError as e:
- # YouTube's servers may return errors we want to retry on in a 200 OK response
- # See: https://github.com/yt-dlp/yt-dlp/issues/839
- if 'unknown error' in e.msg.lower():
- main_rm.error = e
- next(main_retries)
- continue
- return self._error_or_warning(e, fatal=fatal)
- # Youtube sometimes sends incomplete data
- # See: https://github.com/ytdl-org/youtube-dl/issues/28194
- if not traverse_obj(response, *variadic(check_get_keys)):
- icd_rm.error = ExtractorError('Incomplete data received', expected=True)
- should_retry = next(icd_retries, None)
- if not should_retry:
- return None
- continue
-
- return response
-
- @staticmethod
- def is_music_url(url):
- return re.match(r'(https?://)?music\.youtube\.com/', url) is not None
-
- def _extract_video(self, renderer):
- video_id = renderer.get('videoId')
-
- reel_header_renderer = traverse_obj(renderer, (
- 'navigationEndpoint', 'reelWatchEndpoint', 'overlay', 'reelPlayerOverlayRenderer',
- 'reelPlayerHeaderSupportedRenderers', 'reelPlayerHeaderRenderer'))
-
- title = self._get_text(renderer, 'title', 'headline') or self._get_text(reel_header_renderer, 'reelTitleText')
- description = self._get_text(renderer, 'descriptionSnippet')
-
- duration = int_or_none(renderer.get('lengthSeconds'))
- if duration is None:
- duration = parse_duration(self._get_text(
- renderer, 'lengthText', ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'text')))
- if duration is None:
- # XXX: should write a parser to be more general to support more cases (e.g. shorts in shorts tab)
- duration = parse_duration(self._search_regex(
- r'(?i)(ago)(?!.*\1)\s+(?P<duration>[a-z0-9 ,]+?)(?:\s+[\d,]+\s+views)?(?:\s+-\s+play\s+short)?$',
- traverse_obj(renderer, ('title', 'accessibility', 'accessibilityData', 'label'), default='', expected_type=str),
- video_id, default=None, group='duration'))
-
- channel_id = traverse_obj(
- renderer, ('shortBylineText', 'runs', ..., 'navigationEndpoint', 'browseEndpoint', 'browseId'),
- expected_type=str, get_all=False)
- if not channel_id:
- channel_id = traverse_obj(reel_header_renderer, ('channelNavigationEndpoint', 'browseEndpoint', 'browseId'))
-
- channel_id = self.ucid_or_none(channel_id)
-
- overlay_style = traverse_obj(
- renderer, ('thumbnailOverlays', ..., 'thumbnailOverlayTimeStatusRenderer', 'style'),
- get_all=False, expected_type=str)
- badges = self._extract_badges(traverse_obj(renderer, 'badges'))
- owner_badges = self._extract_badges(traverse_obj(renderer, 'ownerBadges'))
- navigation_url = urljoin('https://www.youtube.com/', traverse_obj(
- renderer, ('navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
- expected_type=str)) or ''
- url = f'https://www.youtube.com/watch?v={video_id}'
- if overlay_style == 'SHORTS' or '/shorts/' in navigation_url:
- url = f'https://www.youtube.com/shorts/{video_id}'
-
- time_text = (self._get_text(renderer, 'publishedTimeText', 'videoInfo')
- or self._get_text(reel_header_renderer, 'timestampText') or '')
- scheduled_timestamp = str_to_int(traverse_obj(renderer, ('upcomingEventData', 'startTime'), get_all=False))
-
- live_status = (
- 'is_upcoming' if scheduled_timestamp is not None
- else 'was_live' if 'streamed' in time_text.lower()
- else 'is_live' if overlay_style == 'LIVE' or self._has_badge(badges, BadgeType.LIVE_NOW)
- else None)
-
- # videoInfo is a string like '50K views • 10 years ago'.
- view_count_text = self._get_text(renderer, 'viewCountText', 'shortViewCountText', 'videoInfo') or ''
- view_count = (0 if 'no views' in view_count_text.lower()
- else self._get_count({'simpleText': view_count_text}))
- view_count_field = 'concurrent_view_count' if live_status in ('is_live', 'is_upcoming') else 'view_count'
-
- channel = (self._get_text(renderer, 'ownerText', 'shortBylineText')
- or self._get_text(reel_header_renderer, 'channelTitleText'))
-
- channel_handle = traverse_obj(renderer, (
- 'shortBylineText', 'runs', ..., 'navigationEndpoint',
- (('commandMetadata', 'webCommandMetadata', 'url'), ('browseEndpoint', 'canonicalBaseUrl'))),
- expected_type=self.handle_from_url, get_all=False)
- return {
- '_type': 'url',
- 'ie_key': YoutubeIE.ie_key(),
- 'id': video_id,
- 'url': url,
- 'title': title,
- 'description': description,
- 'duration': duration,
- 'channel_id': channel_id,
- 'channel': channel,
- 'channel_url': f'https://www.youtube.com/channel/{channel_id}' if channel_id else None,
- 'uploader': channel,
- 'uploader_id': channel_handle,
- 'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
- 'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
- 'timestamp': (self._parse_time_text(time_text)
- if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
- else None),
- 'release_timestamp': scheduled_timestamp,
- 'availability':
- 'public' if self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
- else self._availability(
- is_private=self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE) or None,
- needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
- needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
- is_unlisted=self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED) or None),
- view_count_field: view_count,
- 'live_status': live_status,
- 'channel_is_verified': True if self._has_badge(owner_badges, BadgeType.VERIFIED) else None,
- }
-
-
class YoutubeIE(YoutubeBaseInfoExtractor):
IE_DESC = 'YouTube'
_VALID_URL = r'''(?x)^
@@ -2970,7 +1762,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
@classmethod
def suitable(cls, url):
- from ..utils import parse_qs
+ from yt_dlp.utils import parse_qs
qs = parse_qs(url)
if qs.get('list', [None])[0]:
@@ -5166,2847 +3958,3 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
self.mark_watched(video_id, player_responses)
return info
-
-
-class YoutubeTabBaseInfoExtractor(YoutubeBaseInfoExtractor):
- @staticmethod
- def passthrough_smuggled_data(func):
- def _smuggle(info, smuggled_data):
- if info.get('_type') not in ('url', 'url_transparent'):
- return info
- if smuggled_data.get('is_music_url'):
- parsed_url = urllib.parse.urlparse(info['url'])
- if parsed_url.netloc in ('www.youtube.com', 'music.youtube.com'):
- smuggled_data.pop('is_music_url')
- info['url'] = urllib.parse.urlunparse(parsed_url._replace(netloc='music.youtube.com'))
- if smuggled_data:
- info['url'] = smuggle_url(info['url'], smuggled_data)
- return info
-
- @functools.wraps(func)
- def wrapper(self, url):
- url, smuggled_data = unsmuggle_url(url, {})
- if self.is_music_url(url):
- smuggled_data['is_music_url'] = True
- info_dict = func(self, url, smuggled_data)
- if smuggled_data:
- _smuggle(info_dict, smuggled_data)
- if info_dict.get('entries'):
- info_dict['entries'] = (_smuggle(i, smuggled_data.copy()) for i in info_dict['entries'])
- return info_dict
- return wrapper
-
- @staticmethod
- def _extract_basic_item_renderer(item):
- # Modified from _extract_grid_item_renderer
- known_basic_renderers = (
- 'playlistRenderer', 'videoRenderer', 'channelRenderer', 'showRenderer', 'reelItemRenderer',
- )
- for key, renderer in item.items():
- if not isinstance(renderer, dict):
- continue
- elif key in known_basic_renderers:
- return renderer
- elif key.startswith('grid') and key.endswith('Renderer'):
- return renderer
-
- def _extract_channel_renderer(self, renderer):
- channel_id = self.ucid_or_none(renderer['channelId'])
- title = self._get_text(renderer, 'title')
- channel_url = format_field(channel_id, None, 'https://www.youtube.com/channel/%s', default=None)
- channel_handle = self.handle_from_url(
- traverse_obj(renderer, (
- 'navigationEndpoint', (('commandMetadata', 'webCommandMetadata', 'url'),
- ('browseEndpoint', 'canonicalBaseUrl')),
- {str}), get_all=False))
- if not channel_handle:
- # As of 2023-06-01, YouTube sets subscriberCountText to the handle in search
- channel_handle = self.handle_or_none(self._get_text(renderer, 'subscriberCountText'))
- return {
- '_type': 'url',
- 'url': channel_url,
- 'id': channel_id,
- 'ie_key': YoutubeTabIE.ie_key(),
- 'channel': title,
- 'uploader': title,
- 'channel_id': channel_id,
- 'channel_url': channel_url,
- 'title': title,
- 'uploader_id': channel_handle,
- 'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
- # See above. YouTube sets videoCountText to the subscriber text in search channel renderers.
- # However, in feed/channels this is set correctly to the subscriber count
- 'channel_follower_count': traverse_obj(
- renderer, 'subscriberCountText', 'videoCountText', expected_type=self._get_count),
- 'thumbnails': self._extract_thumbnails(renderer, 'thumbnail'),
- 'playlist_count': (
- # videoCountText may be the subscriber count
- self._get_count(renderer, 'videoCountText')
- if self._get_count(renderer, 'subscriberCountText') is not None else None),
- 'description': self._get_text(renderer, 'descriptionSnippet'),
- 'channel_is_verified': True if self._has_badge(
- self._extract_badges(traverse_obj(renderer, 'ownerBadges')), BadgeType.VERIFIED) else None,
- }
-
- def _grid_entries(self, grid_renderer):
- for item in grid_renderer['items']:
- if not isinstance(item, dict):
- continue
- if lockup_view_model := traverse_obj(item, ('lockupViewModel', {dict})):
- if entry := self._extract_lockup_view_model(lockup_view_model):
- yield entry
- continue
- renderer = self._extract_basic_item_renderer(item)
- if not isinstance(renderer, dict):
- continue
- title = self._get_text(renderer, 'title')
-
- # playlist
- playlist_id = renderer.get('playlistId')
- if playlist_id:
- yield self.url_result(
- f'https://www.youtube.com/playlist?list={playlist_id}',
- ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
- video_title=title)
- continue
- # video
- video_id = renderer.get('videoId')
- if video_id:
- yield self._extract_video(renderer)
- continue
- # channel
- channel_id = renderer.get('channelId')
- if channel_id:
- yield self._extract_channel_renderer(renderer)
- continue
- # generic endpoint URL support
- ep_url = urljoin('https://www.youtube.com/', try_get(
- renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
- str))
- if ep_url:
- for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
- if ie.suitable(ep_url):
- yield self.url_result(
- ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
- break
-
- def _music_reponsive_list_entry(self, renderer):
- video_id = traverse_obj(renderer, ('playlistItemData', 'videoId'))
- if video_id:
- title = traverse_obj(renderer, (
- 'flexColumns', 0, 'musicResponsiveListItemFlexColumnRenderer',
- 'text', 'runs', 0, 'text'))
- return self.url_result(f'https://music.youtube.com/watch?v={video_id}',
- ie=YoutubeIE.ie_key(), video_id=video_id, title=title)
- playlist_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'playlistId'))
- if playlist_id:
- video_id = traverse_obj(renderer, ('navigationEndpoint', 'watchEndpoint', 'videoId'))
- if video_id:
- return self.url_result(f'https://music.youtube.com/watch?v={video_id}&list={playlist_id}',
- ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
- return self.url_result(f'https://music.youtube.com/playlist?list={playlist_id}',
- ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
- browse_id = traverse_obj(renderer, ('navigationEndpoint', 'browseEndpoint', 'browseId'))
- if browse_id:
- return self.url_result(f'https://music.youtube.com/browse/{browse_id}',
- ie=YoutubeTabIE.ie_key(), video_id=browse_id)
-
- def _shelf_entries_from_content(self, shelf_renderer):
- content = shelf_renderer.get('content')
- if not isinstance(content, dict):
- return
- renderer = content.get('gridRenderer') or content.get('expandedShelfContentsRenderer')
- if renderer:
- # TODO: add support for nested playlists so each shelf is processed
- # as separate playlist
- # TODO: this includes only first N items
- yield from self._grid_entries(renderer)
- renderer = content.get('horizontalListRenderer')
- if renderer:
- # TODO: handle case
- pass
-
- def _shelf_entries(self, shelf_renderer, skip_channels=False):
- ep = try_get(
- shelf_renderer, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
- str)
- shelf_url = urljoin('https://www.youtube.com', ep)
- if shelf_url:
- # Skipping links to another channels, note that checking for
- # endpoint.commandMetadata.webCommandMetadata.webPageTypwebPageType == WEB_PAGE_TYPE_CHANNEL
- # will not work
- if skip_channels and '/channels?' in shelf_url:
- return
- title = self._get_text(shelf_renderer, 'title')
- yield self.url_result(shelf_url, video_title=title)
- # Shelf may not contain shelf URL, fallback to extraction from content
- yield from self._shelf_entries_from_content(shelf_renderer)
-
- def _playlist_entries(self, video_list_renderer):
- for content in video_list_renderer['contents']:
- if not isinstance(content, dict):
- continue
- renderer = content.get('playlistVideoRenderer') or content.get('playlistPanelVideoRenderer')
- if not isinstance(renderer, dict):
- continue
- video_id = renderer.get('videoId')
- if not video_id:
- continue
- yield self._extract_video(renderer)
-
- def _extract_lockup_view_model(self, view_model):
- content_id = view_model.get('contentId')
- if not content_id:
- return
- content_type = view_model.get('contentType')
- if content_type not in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'):
- self.report_warning(
- f'Unsupported lockup view model content type "{content_type}"{bug_reports_message()}', only_once=True)
- return
- return self.url_result(
- f'https://www.youtube.com/playlist?list={content_id}', ie=YoutubeTabIE, video_id=content_id,
- title=traverse_obj(view_model, (
- 'metadata', 'lockupMetadataViewModel', 'title', 'content', {str})),
- thumbnails=self._extract_thumbnails(view_model, (
- 'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail', 'thumbnailViewModel', 'image'), final_key='sources'))
-
- def _rich_entries(self, rich_grid_renderer):
- if lockup_view_model := traverse_obj(rich_grid_renderer, ('content', 'lockupViewModel', {dict})):
- if entry := self._extract_lockup_view_model(lockup_view_model):
- yield entry
- return
- renderer = traverse_obj(
- rich_grid_renderer,
- ('content', ('videoRenderer', 'reelItemRenderer', 'playlistRenderer', 'shortsLockupViewModel'), any)) or {}
- video_id = renderer.get('videoId')
- if video_id:
- yield self._extract_video(renderer)
- return
- playlist_id = renderer.get('playlistId')
- if playlist_id:
- yield self.url_result(
- f'https://www.youtube.com/playlist?list={playlist_id}',
- ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
- video_title=self._get_text(renderer, 'title'))
- return
- # shortsLockupViewModel extraction
- entity_id = renderer.get('entityId')
- if entity_id:
- video_id = traverse_obj(renderer, ('onTap', 'innertubeCommand', 'reelWatchEndpoint', 'videoId', {str}))
- if not video_id:
- return
- yield self.url_result(
- f'https://www.youtube.com/shorts/{video_id}',
- ie=YoutubeIE, video_id=video_id,
- **traverse_obj(renderer, {
- 'title': ((
- ('overlayMetadata', 'primaryText', 'content', {str}),
- ('accessibilityText', {lambda x: re.fullmatch(r'(.+), (?:[\d,.]+(?:[KM]| million)?|No) views? - play Short', x)}, 1)), any),
- 'view_count': ('overlayMetadata', 'secondaryText', 'content', {parse_count}),
- }),
- thumbnails=self._extract_thumbnails(renderer, 'thumbnail', final_key='sources'))
- return
-
- def _video_entry(self, video_renderer):
- video_id = video_renderer.get('videoId')
- if video_id:
- return self._extract_video(video_renderer)
-
- def _hashtag_tile_entry(self, hashtag_tile_renderer):
- url = urljoin('https://youtube.com', traverse_obj(
- hashtag_tile_renderer, ('onTapCommand', 'commandMetadata', 'webCommandMetadata', 'url')))
- if url:
- return self.url_result(
- url, ie=YoutubeTabIE.ie_key(), title=self._get_text(hashtag_tile_renderer, 'hashtag'))
-
- def _post_thread_entries(self, post_thread_renderer):
- post_renderer = try_get(
- post_thread_renderer, lambda x: x['post']['backstagePostRenderer'], dict)
- if not post_renderer:
- return
- # video attachment
- video_renderer = try_get(
- post_renderer, lambda x: x['backstageAttachment']['videoRenderer'], dict) or {}
- video_id = video_renderer.get('videoId')
- if video_id:
- entry = self._extract_video(video_renderer)
- if entry:
- yield entry
- # playlist attachment
- playlist_id = try_get(
- post_renderer, lambda x: x['backstageAttachment']['playlistRenderer']['playlistId'], str)
- if playlist_id:
- yield self.url_result(
- f'https://www.youtube.com/playlist?list={playlist_id}',
- ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
- # inline video links
- runs = try_get(post_renderer, lambda x: x['contentText']['runs'], list) or []
- for run in runs:
- if not isinstance(run, dict):
- continue
- ep_url = try_get(
- run, lambda x: x['navigationEndpoint']['urlEndpoint']['url'], str)
- if not ep_url:
- continue
- if not YoutubeIE.suitable(ep_url):
- continue
- ep_video_id = YoutubeIE._match_id(ep_url)
- if video_id == ep_video_id:
- continue
- yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=ep_video_id)
-
- def _post_thread_continuation_entries(self, post_thread_continuation):
- contents = post_thread_continuation.get('contents')
- if not isinstance(contents, list):
- return
- for content in contents:
- renderer = content.get('backstagePostThreadRenderer')
- if isinstance(renderer, dict):
- yield from self._post_thread_entries(renderer)
- continue
- renderer = content.get('videoRenderer')
- if isinstance(renderer, dict):
- yield self._video_entry(renderer)
-
- r''' # unused
- def _rich_grid_entries(self, contents):
- for content in contents:
- video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
- if video_renderer:
- entry = self._video_entry(video_renderer)
- if entry:
- yield entry
- '''
-
- def _report_history_entries(self, renderer):
- for url in traverse_obj(renderer, (
- 'rows', ..., 'reportHistoryTableRowRenderer', 'cells', ...,
- 'reportHistoryTableCellRenderer', 'cell', 'reportHistoryTableTextCellRenderer', 'text', 'runs', ...,
- 'navigationEndpoint', 'commandMetadata', 'webCommandMetadata', 'url')):
- yield self.url_result(urljoin('https://www.youtube.com', url), YoutubeIE)
-
- def _extract_entries(self, parent_renderer, continuation_list):
- # continuation_list is modified in-place with continuation_list = [continuation_token]
- continuation_list[:] = [None]
- contents = try_get(parent_renderer, lambda x: x['contents'], list) or []
- for content in contents:
- if not isinstance(content, dict):
- continue
- is_renderer = traverse_obj(
- content, 'itemSectionRenderer', 'musicShelfRenderer', 'musicShelfContinuation',
- expected_type=dict)
- if not is_renderer:
- if content.get('richItemRenderer'):
- for entry in self._rich_entries(content['richItemRenderer']):
- yield entry
- continuation_list[0] = self._extract_continuation(parent_renderer)
- elif content.get('reportHistorySectionRenderer'): # https://www.youtube.com/reporthistory
- table = traverse_obj(content, ('reportHistorySectionRenderer', 'table', 'tableRenderer'))
- yield from self._report_history_entries(table)
- continuation_list[0] = self._extract_continuation(table)
- continue
-
- isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
- for isr_content in isr_contents:
- if not isinstance(isr_content, dict):
- continue
-
- known_renderers = {
- 'playlistVideoListRenderer': self._playlist_entries,
- 'gridRenderer': self._grid_entries,
- 'reelShelfRenderer': self._grid_entries,
- 'shelfRenderer': self._shelf_entries,
- 'musicResponsiveListItemRenderer': lambda x: [self._music_reponsive_list_entry(x)],
- 'backstagePostThreadRenderer': self._post_thread_entries,
- 'videoRenderer': lambda x: [self._video_entry(x)],
- 'playlistRenderer': lambda x: self._grid_entries({'items': [{'playlistRenderer': x}]}),
- 'channelRenderer': lambda x: self._grid_entries({'items': [{'channelRenderer': x}]}),
- 'hashtagTileRenderer': lambda x: [self._hashtag_tile_entry(x)],
- 'richGridRenderer': lambda x: self._extract_entries(x, continuation_list),
- 'lockupViewModel': lambda x: [self._extract_lockup_view_model(x)],
- }
- for key, renderer in isr_content.items():
- if key not in known_renderers:
- continue
- for entry in known_renderers[key](renderer):
- if entry:
- yield entry
- continuation_list[0] = self._extract_continuation(renderer)
- break
-
- if not continuation_list[0]:
- continuation_list[0] = self._extract_continuation(is_renderer)
-
- if not continuation_list[0]:
- continuation_list[0] = self._extract_continuation(parent_renderer)
-
- def _entries(self, tab, item_id, ytcfg, delegated_session_id, visitor_data):
- continuation_list = [None]
- extract_entries = lambda x: self._extract_entries(x, continuation_list)
- tab_content = try_get(tab, lambda x: x['content'], dict)
- if not tab_content:
- return
- parent_renderer = (
- try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
- or try_get(tab_content, lambda x: x['richGridRenderer'], dict) or {})
- yield from extract_entries(parent_renderer)
- continuation = continuation_list[0]
- seen_continuations = set()
- for page_num in itertools.count(1):
- if not continuation:
- break
- continuation_token = continuation.get('continuation')
- if continuation_token is not None and continuation_token in seen_continuations:
- self.write_debug('Detected YouTube feed looping - assuming end of feed.')
- break
- seen_continuations.add(continuation_token)
- headers = self.generate_api_headers(
- ytcfg=ytcfg, delegated_session_id=delegated_session_id, visitor_data=visitor_data)
- response = self._extract_response(
- item_id=f'{item_id} page {page_num}',
- query=continuation, headers=headers, ytcfg=ytcfg,
- check_get_keys=('continuationContents', 'onResponseReceivedActions', 'onResponseReceivedEndpoints'))
-
- if not response:
- break
- # Extracting updated visitor data is required to prevent an infinite extraction loop in some cases
- # See: https://github.com/ytdl-org/youtube-dl/issues/28702
- visitor_data = self._extract_visitor_data(response) or visitor_data
-
- known_renderers = {
- 'videoRenderer': (self._grid_entries, 'items'), # for membership tab
- 'gridPlaylistRenderer': (self._grid_entries, 'items'),
- 'gridVideoRenderer': (self._grid_entries, 'items'),
- 'gridChannelRenderer': (self._grid_entries, 'items'),
- 'playlistVideoRenderer': (self._playlist_entries, 'contents'),
- 'itemSectionRenderer': (extract_entries, 'contents'), # for feeds
- 'richItemRenderer': (extract_entries, 'contents'), # for hashtag
- 'backstagePostThreadRenderer': (self._post_thread_continuation_entries, 'contents'),
- 'reportHistoryTableRowRenderer': (self._report_history_entries, 'rows'),
- 'playlistVideoListContinuation': (self._playlist_entries, None),
- 'gridContinuation': (self._grid_entries, None),
- 'itemSectionContinuation': (self._post_thread_continuation_entries, None),
- 'sectionListContinuation': (extract_entries, None), # for feeds
- }
-
- continuation_items = traverse_obj(response, (
- ('onResponseReceivedActions', 'onResponseReceivedEndpoints'), ...,
- 'appendContinuationItemsAction', 'continuationItems',
- ), 'continuationContents', get_all=False)
- continuation_item = traverse_obj(continuation_items, 0, None, expected_type=dict, default={})
-
- video_items_renderer = None
- for key in continuation_item:
- if key not in known_renderers:
- continue
- func, parent_key = known_renderers[key]
- video_items_renderer = {parent_key: continuation_items} if parent_key else continuation_items
- continuation_list = [None]
- yield from func(video_items_renderer)
- continuation = continuation_list[0] or self._extract_continuation(video_items_renderer)
-
- if not video_items_renderer:
- break
-
- @staticmethod
- def _extract_selected_tab(tabs, fatal=True):
- for tab_renderer in tabs:
- if tab_renderer.get('selected'):
- return tab_renderer
- if fatal:
- raise ExtractorError('Unable to find selected tab')
-
- @staticmethod
- def _extract_tab_renderers(response):
- return traverse_obj(
- response, ('contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., ('tabRenderer', 'expandableTabRenderer')), expected_type=dict)
-
- def _extract_from_tabs(self, item_id, ytcfg, data, tabs):
- metadata = self._extract_metadata_from_tabs(item_id, data)
-
- selected_tab = self._extract_selected_tab(tabs)
- metadata['title'] += format_field(selected_tab, 'title', ' - %s')
- metadata['title'] += format_field(selected_tab, 'expandedText', ' - %s')
-
- return self.playlist_result(
- self._entries(
- selected_tab, metadata['id'], ytcfg,
- self._extract_delegated_session_id(ytcfg, data),
- self._extract_visitor_data(data, ytcfg)),
- **metadata)
-
- def _extract_metadata_from_tabs(self, item_id, data):
- info = {'id': item_id}
-
- metadata_renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'), expected_type=dict)
- if metadata_renderer:
- channel_id = traverse_obj(metadata_renderer, ('externalId', {self.ucid_or_none}),
- ('channelUrl', {self.ucid_from_url}))
- info.update({
- 'channel': metadata_renderer.get('title'),
- 'channel_id': channel_id,
- })
- if info['channel_id']:
- info['id'] = info['channel_id']
- else:
- metadata_renderer = traverse_obj(data, ('metadata', 'playlistMetadataRenderer'), expected_type=dict)
-
- # pageHeaderViewModel slow rollout began April 2024
- page_header_view_model = traverse_obj(data, (
- 'header', 'pageHeaderRenderer', 'content', 'pageHeaderViewModel', {dict}))
-
- # We can get the uncropped banner/avatar by replacing the crop params with '=s0'
- # See: https://github.com/yt-dlp/yt-dlp/issues/2237#issuecomment-1013694714
- def _get_uncropped(url):
- return url_or_none((url or '').split('=')[0] + '=s0')
-
- avatar_thumbnails = self._extract_thumbnails(metadata_renderer, 'avatar')
- if avatar_thumbnails:
- uncropped_avatar = _get_uncropped(avatar_thumbnails[0]['url'])
- if uncropped_avatar:
- avatar_thumbnails.append({
- 'url': uncropped_avatar,
- 'id': 'avatar_uncropped',
- 'preference': 1,
- })
-
- channel_banners = (
- self._extract_thumbnails(data, ('header', ..., ('banner', 'mobileBanner', 'tvBanner')))
- or self._extract_thumbnails(
- page_header_view_model, ('banner', 'imageBannerViewModel', 'image'), final_key='sources'))
- for banner in channel_banners:
- banner['preference'] = -10
-
- if channel_banners:
- uncropped_banner = _get_uncropped(channel_banners[0]['url'])
- if uncropped_banner:
- channel_banners.append({
- 'url': uncropped_banner,
- 'id': 'banner_uncropped',
- 'preference': -5,
- })
-
- # Deprecated - remove primary_sidebar_renderer when layout discontinued
- primary_sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer')
- playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer'), expected_type=dict)
-
- primary_thumbnails = self._extract_thumbnails(
- primary_sidebar_renderer, ('thumbnailRenderer', ('playlistVideoThumbnailRenderer', 'playlistCustomThumbnailRenderer'), 'thumbnail'))
- playlist_thumbnails = self._extract_thumbnails(
- playlist_header_renderer, ('playlistHeaderBanner', 'heroPlaylistThumbnailRenderer', 'thumbnail'))
-
- info.update({
- 'title': (traverse_obj(metadata_renderer, 'title')
- or self._get_text(data, ('header', 'hashtagHeaderRenderer', 'hashtag'))
- or info['id']),
- 'availability': self._extract_availability(data),
- 'channel_follower_count': (
- self._get_count(data, ('header', ..., 'subscriberCountText'))
- or traverse_obj(page_header_view_model, (
- 'metadata', 'contentMetadataViewModel', 'metadataRows', ..., 'metadataParts',
- lambda _, v: 'subscribers' in v['text']['content'], 'text', 'content', {parse_count}, any))),
- 'description': try_get(metadata_renderer, lambda x: x.get('description', '')),
- 'tags': (traverse_obj(data, ('microformat', 'microformatDataRenderer', 'tags', ..., {str}))
- or traverse_obj(metadata_renderer, ('keywords', {lambda x: x and shlex.split(x)}, ...))),
- 'thumbnails': (primary_thumbnails or playlist_thumbnails) + avatar_thumbnails + channel_banners,
- })
-
- channel_handle = (
- traverse_obj(metadata_renderer, (('vanityChannelUrl', ('ownerUrls', ...)), {self.handle_from_url}), get_all=False)
- or traverse_obj(data, ('header', ..., 'channelHandleText', {self.handle_or_none}), get_all=False))
-
- if channel_handle:
- info.update({
- 'uploader_id': channel_handle,
- 'uploader_url': format_field(channel_handle, None, 'https://www.youtube.com/%s', default=None),
- })
-
- channel_badges = self._extract_badges(traverse_obj(data, ('header', ..., 'badges'), get_all=False))
- if self._has_badge(channel_badges, BadgeType.VERIFIED):
- info['channel_is_verified'] = True
- # Playlist stats is a text runs array containing [video count, view count, last updated].
- # last updated or (view count and last updated) may be missing.
- playlist_stats = get_first(
- (primary_sidebar_renderer, playlist_header_renderer), (('stats', 'briefStats', 'numVideosText'), ))
-
- last_updated_unix = self._parse_time_text(
- self._get_text(playlist_stats, 2) # deprecated, remove when old layout discontinued
- or self._get_text(playlist_header_renderer, ('byline', 1, 'playlistBylineRenderer', 'text')))
- info['modified_date'] = strftime_or_none(last_updated_unix)
-
- info['view_count'] = self._get_count(playlist_stats, 1)
- if info['view_count'] is None: # 0 is allowed
- info['view_count'] = self._get_count(playlist_header_renderer, 'viewCountText')
- if info['view_count'] is None:
- info['view_count'] = self._get_count(data, (
- 'contents', 'twoColumnBrowseResultsRenderer', 'tabs', ..., 'tabRenderer', 'content', 'sectionListRenderer',
- 'contents', ..., 'itemSectionRenderer', 'contents', ..., 'channelAboutFullMetadataRenderer', 'viewCountText'))
-
- info['playlist_count'] = self._get_count(playlist_stats, 0)
- if info['playlist_count'] is None: # 0 is allowed
- info['playlist_count'] = self._get_count(playlist_header_renderer, ('byline', 0, 'playlistBylineRenderer', 'text'))
-
- if not info.get('channel_id'):
- owner = traverse_obj(playlist_header_renderer, 'ownerText')
- if not owner: # Deprecated
- owner = traverse_obj(
- self._extract_sidebar_info_renderer(data, 'playlistSidebarSecondaryInfoRenderer'),
- ('videoOwner', 'videoOwnerRenderer', 'title'))
- owner_text = self._get_text(owner)
- browse_ep = traverse_obj(owner, ('runs', 0, 'navigationEndpoint', 'browseEndpoint')) or {}
- info.update({
- 'channel': self._search_regex(r'^by (.+) and \d+ others?$', owner_text, 'uploader', default=owner_text),
- 'channel_id': self.ucid_or_none(browse_ep.get('browseId')),
- 'uploader_id': self.handle_from_url(urljoin('https://www.youtube.com', browse_ep.get('canonicalBaseUrl'))),
- })
-
- info.update({
- 'uploader': info['channel'],
- 'channel_url': format_field(info.get('channel_id'), None, 'https://www.youtube.com/channel/%s', default=None),
- 'uploader_url': format_field(info.get('uploader_id'), None, 'https://www.youtube.com/%s', default=None),
- })
-
- return info
-
- def _extract_inline_playlist(self, playlist, playlist_id, data, ytcfg):
- first_id = last_id = response = None
- for page_num in itertools.count(1):
- videos = list(self._playlist_entries(playlist))
- if not videos:
- return
- start = next((i for i, v in enumerate(videos) if v['id'] == last_id), -1) + 1
- if start >= len(videos):
- return
- yield from videos[start:]
- first_id = first_id or videos[0]['id']
- last_id = videos[-1]['id']
- watch_endpoint = try_get(
- playlist, lambda x: x['contents'][-1]['playlistPanelVideoRenderer']['navigationEndpoint']['watchEndpoint'])
- headers = self.generate_api_headers(
- ytcfg=ytcfg, delegated_session_id=self._extract_delegated_session_id(ytcfg, data),
- visitor_data=self._extract_visitor_data(response, data, ytcfg))
- query = {
- 'playlistId': playlist_id,
- 'videoId': watch_endpoint.get('videoId') or last_id,
- 'index': watch_endpoint.get('index') or len(videos),
- 'params': watch_endpoint.get('params') or 'OAE%3D',
- }
- response = self._extract_response(
- item_id=f'{playlist_id} page {page_num}',
- query=query, ep='next', headers=headers, ytcfg=ytcfg,
- check_get_keys='contents',
- )
- playlist = try_get(
- response, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
-
- def _extract_from_playlist(self, item_id, url, data, playlist, ytcfg):
- title = playlist.get('title') or try_get(
- data, lambda x: x['titleText']['simpleText'], str)
- playlist_id = playlist.get('playlistId') or item_id
-
- # Delegating everything except mix playlists to regular tab-based playlist URL
- playlist_url = urljoin(url, try_get(
- playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
- str))
-
- # Some playlists are unviewable but YouTube still provides a link to the (broken) playlist page [1]
- # [1] MLCT, RLTDwFCb4jeqaKWnciAYM-ZVHg
- is_known_unviewable = re.fullmatch(r'MLCT|RLTD[\w-]{22}', playlist_id)
-
- if playlist_url and playlist_url != url and not is_known_unviewable:
- return self.url_result(
- playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
- video_title=title)
-
- return self.playlist_result(
- self._extract_inline_playlist(playlist, playlist_id, data, ytcfg),
- playlist_id=playlist_id, playlist_title=title)
-
- def _extract_availability(self, data):
- """
- Gets the availability of a given playlist/tab.
- Note: Unless YouTube tells us explicitly, we do not assume it is public
- @param data: response
- """
- sidebar_renderer = self._extract_sidebar_info_renderer(data, 'playlistSidebarPrimaryInfoRenderer') or {}
- playlist_header_renderer = traverse_obj(data, ('header', 'playlistHeaderRenderer')) or {}
- player_header_privacy = playlist_header_renderer.get('privacy')
-
- badges = self._extract_badges(traverse_obj(sidebar_renderer, 'badges'))
-
- # Personal playlists, when authenticated, have a dropdown visibility selector instead of a badge
- privacy_setting_icon = get_first(
- (playlist_header_renderer, sidebar_renderer),
- ('privacyForm', 'dropdownFormFieldRenderer', 'dropdown', 'dropdownRenderer', 'entries',
- lambda _, v: v['privacyDropdownItemRenderer']['isSelected'], 'privacyDropdownItemRenderer', 'icon', 'iconType'),
- expected_type=str)
-
- microformats_is_unlisted = traverse_obj(
- data, ('microformat', 'microformatDataRenderer', 'unlisted'), expected_type=bool)
-
- return (
- 'public' if (
- self._has_badge(badges, BadgeType.AVAILABILITY_PUBLIC)
- or player_header_privacy == 'PUBLIC'
- or privacy_setting_icon == 'PRIVACY_PUBLIC')
- else self._availability(
- is_private=(
- self._has_badge(badges, BadgeType.AVAILABILITY_PRIVATE)
- or player_header_privacy == 'PRIVATE' if player_header_privacy is not None
- else privacy_setting_icon == 'PRIVACY_PRIVATE' if privacy_setting_icon is not None else None),
- is_unlisted=(
- self._has_badge(badges, BadgeType.AVAILABILITY_UNLISTED)
- or player_header_privacy == 'UNLISTED' if player_header_privacy is not None
- else privacy_setting_icon == 'PRIVACY_UNLISTED' if privacy_setting_icon is not None
- else microformats_is_unlisted if microformats_is_unlisted is not None else None),
- needs_subscription=self._has_badge(badges, BadgeType.AVAILABILITY_SUBSCRIPTION) or None,
- needs_premium=self._has_badge(badges, BadgeType.AVAILABILITY_PREMIUM) or None,
- needs_auth=False))
-
- @staticmethod
- def _extract_sidebar_info_renderer(data, info_renderer, expected_type=dict):
- sidebar_renderer = try_get(
- data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list) or []
- for item in sidebar_renderer:
- renderer = try_get(item, lambda x: x[info_renderer], expected_type)
- if renderer:
- return renderer
-
- def _reload_with_unavailable_videos(self, item_id, data, ytcfg):
- """
- Reload playlists with unavailable videos (e.g. private videos, region blocked, etc.)
- """
- is_playlist = bool(traverse_obj(
- data, ('metadata', 'playlistMetadataRenderer'), ('header', 'playlistHeaderRenderer')))
- if not is_playlist:
- return
- headers = self.generate_api_headers(
- ytcfg=ytcfg, delegated_session_id=self._extract_delegated_session_id(ytcfg, data),
- visitor_data=self._extract_visitor_data(data, ytcfg))
- query = {
- 'params': 'wgYCCAA=',
- 'browseId': f'VL{item_id}',
- }
- return self._extract_response(
- item_id=item_id, headers=headers, query=query,
- check_get_keys='contents', fatal=False, ytcfg=ytcfg,
- note='Redownloading playlist API JSON with unavailable videos')
-
- @functools.cached_property
- def skip_webpage(self):
- return 'webpage' in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key())
-
- def _extract_webpage(self, url, item_id, fatal=True):
- webpage, data = None, None
- for retry in self.RetryManager(fatal=fatal):
- try:
- webpage = self._download_webpage(url, item_id, note='Downloading webpage')
- data = self.extract_yt_initial_data(item_id, webpage or '', fatal=fatal) or {}
- except ExtractorError as e:
- if isinstance(e.cause, network_exceptions):
- if not isinstance(e.cause, HTTPError) or e.cause.status not in (403, 429):
- retry.error = e
- continue
- self._error_or_warning(e, fatal=fatal)
- break
-
- try:
- self._extract_and_report_alerts(data)
- except ExtractorError as e:
- self._error_or_warning(e, fatal=fatal)
- break
-
- # Sometimes youtube returns a webpage with incomplete ytInitialData
- # See: https://github.com/yt-dlp/yt-dlp/issues/116
- if not traverse_obj(data, 'contents', 'currentVideoEndpoint', 'onResponseReceivedActions'):
- retry.error = ExtractorError('Incomplete yt initial data received')
- data = None
- continue
-
- return webpage, data
-
- def _report_playlist_authcheck(self, ytcfg, fatal=True):
- """Use if failed to extract ytcfg (and data) from initial webpage"""
- if not ytcfg and self.is_authenticated:
- msg = 'Playlists that require authentication may not extract correctly without a successful webpage download'
- if 'authcheck' not in self._configuration_arg('skip', ie_key=YoutubeTabIE.ie_key()) and fatal:
- raise ExtractorError(
- f'{msg}. If you are not downloading private content, or '
- 'your cookies are only for the first account and channel,'
- ' pass "--extractor-args youtubetab:skip=authcheck" to skip this check',
- expected=True)
- self.report_warning(msg, only_once=True)
-
- def _extract_data(self, url, item_id, ytcfg=None, fatal=True, webpage_fatal=False, default_client='web'):
- data = None
- if not self.skip_webpage:
- webpage, data = self._extract_webpage(url, item_id, fatal=webpage_fatal)
- ytcfg = ytcfg or self.extract_ytcfg(item_id, webpage)
- # Reject webpage data if redirected to home page without explicitly requesting
- selected_tab = self._extract_selected_tab(self._extract_tab_renderers(data), fatal=False) or {}
- if (url != 'https://www.youtube.com/feed/recommended'
- and selected_tab.get('tabIdentifier') == 'FEwhat_to_watch' # Home page
- and 'no-youtube-channel-redirect' not in self.get_param('compat_opts', [])):
- msg = 'The channel/playlist does not exist and the URL redirected to youtube.com home page'
- if fatal:
- raise ExtractorError(msg, expected=True)
- self.report_warning(msg, only_once=True)
- if not data:
- self._report_playlist_authcheck(ytcfg, fatal=fatal)
- data = self._extract_tab_endpoint(url, item_id, ytcfg, fatal=fatal, default_client=default_client)
- return data, ytcfg
-
- def _extract_tab_endpoint(self, url, item_id, ytcfg=None, fatal=True, default_client='web'):
- headers = self.generate_api_headers(ytcfg=ytcfg, default_client=default_client)
- resolve_response = self._extract_response(
- item_id=item_id, query={'url': url}, check_get_keys='endpoint', headers=headers, ytcfg=ytcfg, fatal=fatal,
- ep='navigation/resolve_url', note='Downloading API parameters API JSON', default_client=default_client)
- endpoints = {'browseEndpoint': 'browse', 'watchEndpoint': 'next'}
- for ep_key, ep in endpoints.items():
- params = try_get(resolve_response, lambda x: x['endpoint'][ep_key], dict)
- if params:
- return self._extract_response(
- item_id=item_id, query=params, ep=ep, headers=headers,
- ytcfg=ytcfg, fatal=fatal, default_client=default_client,
- check_get_keys=('contents', 'currentVideoEndpoint', 'onResponseReceivedActions'))
- err_note = 'Failed to resolve url (does the playlist exist?)'
- if fatal:
- raise ExtractorError(err_note, expected=True)
- self.report_warning(err_note, item_id)
-
- _SEARCH_PARAMS = None
-
- def _search_results(self, query, params=NO_DEFAULT, default_client='web'):
- data = {'query': query}
- if params is NO_DEFAULT:
- params = self._SEARCH_PARAMS
- if params:
- data['params'] = params
-
- content_keys = (
- ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', 'sectionListRenderer', 'contents'),
- ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', 'continuationItems'),
- # ytmusic search
- ('contents', 'tabbedSearchResultsRenderer', 'tabs', 0, 'tabRenderer', 'content', 'sectionListRenderer', 'contents'),
- ('continuationContents', ),
- )
- display_id = f'query "{query}"'
- check_get_keys = tuple({keys[0] for keys in content_keys})
- ytcfg = self._download_ytcfg(default_client, display_id) if not self.skip_webpage else {}
- self._report_playlist_authcheck(ytcfg, fatal=False)
-
- continuation_list = [None]
- search = None
- for page_num in itertools.count(1):
- data.update(continuation_list[0] or {})
- headers = self.generate_api_headers(
- ytcfg=ytcfg, visitor_data=self._extract_visitor_data(search), default_client=default_client)
- search = self._extract_response(
- item_id=f'{display_id} page {page_num}', ep='search', query=data,
- default_client=default_client, check_get_keys=check_get_keys, ytcfg=ytcfg, headers=headers)
- slr_contents = traverse_obj(search, *content_keys)
- yield from self._extract_entries({'contents': list(variadic(slr_contents))}, continuation_list)
- if not continuation_list[0]:
- break
-
-
-class YoutubeTabIE(YoutubeTabBaseInfoExtractor):
- IE_DESC = 'YouTube Tabs'
- _VALID_URL = r'''(?x:
- https?://
- (?!consent\.)(?:\w+\.)?
- (?:
- youtube(?:kids)?\.com|
- {invidious}
- )/
- (?:
- (?P<channel_type>channel|c|user|browse)/|
- (?P<not_channel>
- feed/|hashtag/|
- (?:playlist|watch)\?.*?\blist=
- )|
- (?!(?:{reserved_names})\b) # Direct URLs
- )
- (?P<id>[^/?\#&]+)
- )'''.format(
- reserved_names=YoutubeBaseInfoExtractor._RESERVED_NAMES,
- invidious='|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
- )
- IE_NAME = 'youtube:tab'
-
- _TESTS = [{
- 'note': 'playlists, multipage',
- 'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
- 'playlist_mincount': 94,
- 'info_dict': {
- 'id': 'UCqj7Cz7revf5maW9g5pgNcg',
- 'title': 'Igor Kleiner Ph.D. - Playlists',
- 'description': 'md5:15d7dd9e333cb987907fcb0d604b233a',
- 'uploader': 'Igor Kleiner Ph.D.',
- 'uploader_id': '@IgorDataScience',
- 'uploader_url': 'https://www.youtube.com/@IgorDataScience',
- 'channel': 'Igor Kleiner Ph.D.',
- 'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
- 'tags': ['критическое мышление', 'наука просто', 'математика', 'анализ данных'],
- 'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
- 'channel_follower_count': int,
- },
- }, {
- 'note': 'playlists, multipage, different order',
- 'url': 'https://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
- 'playlist_mincount': 94,
- 'info_dict': {
- 'id': 'UCqj7Cz7revf5maW9g5pgNcg',
- 'title': 'Igor Kleiner Ph.D. - Playlists',
- 'description': 'md5:15d7dd9e333cb987907fcb0d604b233a',
- 'uploader': 'Igor Kleiner Ph.D.',
- 'uploader_id': '@IgorDataScience',
- 'uploader_url': 'https://www.youtube.com/@IgorDataScience',
- 'tags': ['критическое мышление', 'наука просто', 'математика', 'анализ данных'],
- 'channel_id': 'UCqj7Cz7revf5maW9g5pgNcg',
- 'channel': 'Igor Kleiner Ph.D.',
- 'channel_url': 'https://www.youtube.com/channel/UCqj7Cz7revf5maW9g5pgNcg',
- 'channel_follower_count': int,
- },
- }, {
- 'note': 'playlists, series',
- 'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
- 'playlist_mincount': 5,
- 'info_dict': {
- 'id': 'UCYO_jab_esuFRV4b17AJtAw',
- 'title': '3Blue1Brown - Playlists',
- 'description': 'md5:602e3789e6a0cb7d9d352186b720e395',
- 'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
- 'channel': '3Blue1Brown',
- 'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
- 'uploader_id': '@3blue1brown',
- 'uploader_url': 'https://www.youtube.com/@3blue1brown',
- 'uploader': '3Blue1Brown',
- 'tags': ['Mathematics'],
- 'channel_follower_count': int,
- 'channel_is_verified': True,
- },
- }, {
- 'note': 'playlists, singlepage',
- 'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
- 'playlist_mincount': 4,
- 'info_dict': {
- 'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
- 'title': 'ThirstForScience - Playlists',
- 'description': 'md5:609399d937ea957b0f53cbffb747a14c',
- 'uploader': 'ThirstForScience',
- 'uploader_url': 'https://www.youtube.com/@ThirstForScience',
- 'uploader_id': '@ThirstForScience',
- 'channel_id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
- 'channel_url': 'https://www.youtube.com/channel/UCAEtajcuhQ6an9WEzY9LEMQ',
- 'tags': 'count:12',
- 'channel': 'ThirstForScience',
- 'channel_follower_count': int,
- },
- }, {
- 'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
- 'only_matching': True,
- }, {
- 'note': 'basic, single video playlist',
- 'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
- 'info_dict': {
- 'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
- 'title': 'youtube-dl public playlist',
- 'description': '',
- 'tags': [],
- 'view_count': int,
- 'modified_date': '20201130',
- 'channel': 'Sergey M.',
- 'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
- 'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
- 'availability': 'public',
- 'uploader': 'Sergey M.',
- 'uploader_url': 'https://www.youtube.com/@sergeym.6173',
- 'uploader_id': '@sergeym.6173',
- },
- 'playlist_count': 1,
- }, {
- 'note': 'empty playlist',
- 'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
- 'info_dict': {
- 'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
- 'title': 'youtube-dl empty playlist',
- 'tags': [],
- 'channel': 'Sergey M.',
- 'description': '',
- 'modified_date': '20230921',
- 'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
- 'channel_url': 'https://www.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
- 'availability': 'unlisted',
- 'uploader_url': 'https://www.youtube.com/@sergeym.6173',
- 'uploader_id': '@sergeym.6173',
- 'uploader': 'Sergey M.',
- },
- 'playlist_count': 0,
- }, {
- 'note': 'Home tab',
- 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/featured',
- 'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'title': 'lex will - Home',
- 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
- 'uploader': 'lex will',
- 'uploader_id': '@lexwill718',
- 'channel': 'lex will',
- 'tags': ['bible', 'history', 'prophesy'],
- 'uploader_url': 'https://www.youtube.com/@lexwill718',
- 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
- 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'channel_follower_count': int,
- },
- 'playlist_mincount': 2,
- }, {
- 'note': 'Videos tab',
- 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos',
- 'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'title': 'lex will - Videos',
- 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
- 'uploader': 'lex will',
- 'uploader_id': '@lexwill718',
- 'tags': ['bible', 'history', 'prophesy'],
- 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
- 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'uploader_url': 'https://www.youtube.com/@lexwill718',
- 'channel': 'lex will',
- 'channel_follower_count': int,
- },
- 'playlist_mincount': 975,
- }, {
- 'note': 'Videos tab, sorted by popular',
- 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/videos?view=0&sort=p&flow=grid',
- 'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'title': 'lex will - Videos',
- 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
- 'uploader': 'lex will',
- 'uploader_id': '@lexwill718',
- 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'uploader_url': 'https://www.youtube.com/@lexwill718',
- 'channel': 'lex will',
- 'tags': ['bible', 'history', 'prophesy'],
- 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
- 'channel_follower_count': int,
- },
- 'playlist_mincount': 199,
- }, {
- 'note': 'Playlists tab',
- 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/playlists',
- 'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'title': 'lex will - Playlists',
- 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
- 'uploader': 'lex will',
- 'uploader_id': '@lexwill718',
- 'uploader_url': 'https://www.youtube.com/@lexwill718',
- 'channel': 'lex will',
- 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
- 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'tags': ['bible', 'history', 'prophesy'],
- 'channel_follower_count': int,
- },
- 'playlist_mincount': 17,
- }, {
- 'note': 'Community tab',
- 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/community',
- 'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'title': 'lex will - Community',
- 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
- 'channel': 'lex will',
- 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
- 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'tags': ['bible', 'history', 'prophesy'],
- 'channel_follower_count': int,
- 'uploader_url': 'https://www.youtube.com/@lexwill718',
- 'uploader_id': '@lexwill718',
- 'uploader': 'lex will',
- },
- 'playlist_mincount': 18,
- }, {
- 'note': 'Channels tab',
- 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
- 'info_dict': {
- 'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'title': 'lex will - Channels',
- 'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
- 'channel': 'lex will',
- 'channel_url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
- 'channel_id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
- 'tags': ['bible', 'history', 'prophesy'],
- 'channel_follower_count': int,
- 'uploader_url': 'https://www.youtube.com/@lexwill718',
- 'uploader_id': '@lexwill718',
- 'uploader': 'lex will',
- },
- 'playlist_mincount': 12,
- }, {
- 'note': 'Search tab',
- 'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
- 'playlist_mincount': 40,
- 'info_dict': {
- 'id': 'UCYO_jab_esuFRV4b17AJtAw',
- 'title': '3Blue1Brown - Search - linear algebra',
- 'description': 'md5:4d1da95432004b7ba840ebc895b6b4c9',
- 'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
- 'tags': ['Mathematics'],
- 'channel': '3Blue1Brown',
- 'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
- 'channel_follower_count': int,
- 'uploader_url': 'https://www.youtube.com/@3blue1brown',
- 'uploader_id': '@3blue1brown',
- 'uploader': '3Blue1Brown',
- 'channel_is_verified': True,
- },
- }, {
- 'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtubekids.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
- 'only_matching': True,
- }, {
- 'url': 'https://music.youtube.com/channel/UCmlqkdCBesrv2Lak1mF_MxA',
- 'only_matching': True,
- }, {
- 'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
- 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
- 'info_dict': {
- 'title': '29C3: Not my department',
- 'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
- 'description': 'md5:a14dc1a8ef8307a9807fe136a0660268',
- 'tags': [],
- 'view_count': int,
- 'modified_date': '20150605',
- 'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
- 'channel_url': 'https://www.youtube.com/channel/UCEPzS1rYsrkqzSLNp76nrcg',
- 'channel': 'Christiaan008',
- 'availability': 'public',
- 'uploader_id': '@ChRiStIaAn008',
- 'uploader': 'Christiaan008',
- 'uploader_url': 'https://www.youtube.com/@ChRiStIaAn008',
- },
- 'playlist_count': 96,
- }, {
- 'note': 'Large playlist',
- 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
- 'info_dict': {
- 'title': 'Uploads from Cauchemar',
- 'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
- 'channel_url': 'https://www.youtube.com/channel/UCBABnxM4Ar9ten8Mdjj1j0Q',
- 'tags': [],
- 'modified_date': r're:\d{8}',
- 'channel': 'Cauchemar',
- 'view_count': int,
- 'description': '',
- 'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
- 'availability': 'public',
- 'uploader_id': '@Cauchemar89',
- 'uploader': 'Cauchemar',
- 'uploader_url': 'https://www.youtube.com/@Cauchemar89',
- },
- 'playlist_mincount': 1123,
- 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
- }, {
- 'note': 'even larger playlist, 8832 videos',
- 'url': 'http://www.youtube.com/user/NASAgovVideo/videos',
- 'only_matching': True,
- }, {
- 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
- 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
- 'info_dict': {
- 'title': 'Uploads from Interstellar Movie',
- 'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
- 'tags': [],
- 'view_count': int,
- 'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
- 'channel_url': 'https://www.youtube.com/channel/UCXw-G3eDE9trcvY2sBMM_aA',
- 'channel': 'Interstellar Movie',
- 'description': '',
- 'modified_date': r're:\d{8}',
- 'availability': 'public',
- 'uploader_id': '@InterstellarMovie',
- 'uploader': 'Interstellar Movie',
- 'uploader_url': 'https://www.youtube.com/@InterstellarMovie',
- },
- 'playlist_mincount': 21,
- }, {
- 'note': 'Playlist with "show unavailable videos" button',
- 'url': 'https://www.youtube.com/playlist?list=UUTYLiWFZy8xtPwxFwX9rV7Q',
- 'info_dict': {
- 'title': 'Uploads from Phim Siêu Nhân Nhật Bản',
- 'id': 'UUTYLiWFZy8xtPwxFwX9rV7Q',
- 'view_count': int,
- 'channel': 'Phim Siêu Nhân Nhật Bản',
- 'tags': [],
- 'description': '',
- 'channel_url': 'https://www.youtube.com/channel/UCTYLiWFZy8xtPwxFwX9rV7Q',
- 'channel_id': 'UCTYLiWFZy8xtPwxFwX9rV7Q',
- 'modified_date': r're:\d{8}',
- 'availability': 'public',
- 'uploader_url': 'https://www.youtube.com/@phimsieunhannhatban',
- 'uploader_id': '@phimsieunhannhatban',
- 'uploader': 'Phim Siêu Nhân Nhật Bản',
- },
- 'playlist_mincount': 200,
- 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
- }, {
- 'note': 'Playlist with unavailable videos in page 7',
- 'url': 'https://www.youtube.com/playlist?list=UU8l9frL61Yl5KFOl87nIm2w',
- 'info_dict': {
- 'title': 'Uploads from BlankTV',
- 'id': 'UU8l9frL61Yl5KFOl87nIm2w',
- 'channel': 'BlankTV',
- 'channel_url': 'https://www.youtube.com/channel/UC8l9frL61Yl5KFOl87nIm2w',
- 'channel_id': 'UC8l9frL61Yl5KFOl87nIm2w',
- 'view_count': int,
- 'tags': [],
- 'modified_date': r're:\d{8}',
- 'description': '',
- 'availability': 'public',
- 'uploader_id': '@blanktv',
- 'uploader': 'BlankTV',
- 'uploader_url': 'https://www.youtube.com/@blanktv',
- },
- 'playlist_mincount': 1000,
- 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
- }, {
- 'note': 'https://github.com/ytdl-org/youtube-dl/issues/21844',
- 'url': 'https://www.youtube.com/playlist?list=PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
- 'info_dict': {
- 'title': 'Data Analysis with Dr Mike Pound',
- 'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
- 'description': 'md5:7f567c574d13d3f8c0954d9ffee4e487',
- 'tags': [],
- 'view_count': int,
- 'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
- 'channel_url': 'https://www.youtube.com/channel/UC9-y-6csu5WGm29I7JiwpnA',
- 'channel': 'Computerphile',
- 'availability': 'public',
- 'modified_date': '20190712',
- 'uploader_id': '@Computerphile',
- 'uploader': 'Computerphile',
- 'uploader_url': 'https://www.youtube.com/@Computerphile',
- },
- 'playlist_mincount': 11,
- }, {
- 'url': 'https://invidio.us/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
- 'only_matching': True,
- }, {
- 'note': 'Playlist URL that does not actually serve a playlist',
- 'url': 'https://www.youtube.com/watch?v=FqZTN594JQw&list=PLMYEtVRpaqY00V9W81Cwmzp6N6vZqfUKD4',
- 'info_dict': {
- 'id': 'FqZTN594JQw',
- 'ext': 'webm',
- 'title': "Smiley's People 01 detective, Adventure Series, Action",
- 'upload_date': '20150526',
- 'license': 'Standard YouTube License',
- 'description': 'md5:507cdcb5a49ac0da37a920ece610be80',
- 'categories': ['People & Blogs'],
- 'tags': list,
- 'view_count': int,
- 'like_count': int,
- },
- 'params': {
- 'skip_download': True,
- },
- 'skip': 'This video is not available.',
- 'add_ie': [YoutubeIE.ie_key()],
- }, {
- 'url': 'https://www.youtubekids.com/watch?v=Agk7R8I8o5U&list=PUZ6jURNr1WQZCNHF0ao-c0g',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/watch?v=MuAGGZNfUkU&list=RDMM',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
- 'info_dict': {
- 'id': 'hGkQjiJLjWQ', # This will keep changing
- 'ext': 'mp4',
- 'title': str,
- 'upload_date': r're:\d{8}',
- 'description': str,
- 'categories': ['News & Politics'],
- 'tags': list,
- 'like_count': int,
- 'release_timestamp': int,
- 'channel': 'Sky News',
- 'channel_id': 'UCoMdktPbSTixAyNGwb-UYkQ',
- 'age_limit': 0,
- 'view_count': int,
- 'thumbnail': r're:https?://i\.ytimg\.com/vi/[^/]+/maxresdefault(?:_live)?\.jpg',
- 'playable_in_embed': True,
- 'release_date': r're:\d+',
- 'availability': 'public',
- 'live_status': 'is_live',
- 'channel_url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ',
- 'channel_follower_count': int,
- 'concurrent_view_count': int,
- 'uploader_url': 'https://www.youtube.com/@SkyNews',
- 'uploader_id': '@SkyNews',
- 'uploader': 'Sky News',
- 'channel_is_verified': True,
- },
- 'params': {
- 'skip_download': True,
- },
- 'expected_warnings': ['Ignoring subtitle tracks found in '],
- }, {
- 'url': 'https://www.youtube.com/user/TheYoungTurks/live',
- 'info_dict': {
- 'id': 'a48o2S1cPoo',
- 'ext': 'mp4',
- 'title': 'The Young Turks - Live Main Show',
- 'upload_date': '20150715',
- 'license': 'Standard YouTube License',
- 'description': 'md5:438179573adcdff3c97ebb1ee632b891',
- 'categories': ['News & Politics'],
- 'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
- 'like_count': int,
- },
- 'params': {
- 'skip_download': True,
- },
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/channel/UC1yBKRuGpC1tSM73A0ZjYjQ/live',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/c/CommanderVideoHq/live',
- 'only_matching': True,
- }, {
- 'note': 'A channel that is not live. Should raise error',
- 'url': 'https://www.youtube.com/user/numberphile/live',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/feed/trending',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/feed/library',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/feed/history',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/feed/subscriptions',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/feed/watch_later',
- 'only_matching': True,
- }, {
- 'note': 'Recommended - redirects to home page.',
- 'url': 'https://www.youtube.com/feed/recommended',
- 'only_matching': True,
- }, {
- 'note': 'inline playlist with not always working continuations',
- 'url': 'https://www.youtube.com/watch?v=UC6u0Tct-Fo&list=PL36D642111D65BE7C',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/course',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/zsecurity',
- 'only_matching': True,
- }, {
- 'url': 'http://www.youtube.com/NASAgovVideo/videos',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/TheYoungTurks/live',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/hashtag/cctv9',
- 'info_dict': {
- 'id': 'cctv9',
- 'title': 'cctv9 - All',
- 'tags': [],
- },
- 'playlist_mincount': 300, # not consistent but should be over 300
- }, {
- 'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
- 'only_matching': True,
- }, {
- 'note': 'Requires Premium: should request additional YTM-info webpage (and have format 141) for videos in playlist',
- 'url': 'https://music.youtube.com/playlist?list=PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
- 'only_matching': True,
- }, {
- 'note': '/browse/ should redirect to /channel/',
- 'url': 'https://music.youtube.com/browse/UC1a8OFewdjuLq6KlF8M_8Ng',
- 'only_matching': True,
- }, {
- 'note': 'VLPL, should redirect to playlist?list=PL...',
- 'url': 'https://music.youtube.com/browse/VLPLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
- 'info_dict': {
- 'id': 'PLRBp0Fe2GpgmgoscNFLxNyBVSFVdYmFkq',
- 'description': 'Providing you with copyright free / safe music for gaming, live streaming, studying and more!',
- 'title': 'NCS : All Releases 💿',
- 'channel_url': 'https://www.youtube.com/channel/UC_aEa8K-EOJ3D6gOs7HcyNg',
- 'modified_date': r're:\d{8}',
- 'view_count': int,
- 'channel_id': 'UC_aEa8K-EOJ3D6gOs7HcyNg',
- 'tags': [],
- 'channel': 'NoCopyrightSounds',
- 'availability': 'public',
- 'uploader_url': 'https://www.youtube.com/@NoCopyrightSounds',
- 'uploader': 'NoCopyrightSounds',
- 'uploader_id': '@NoCopyrightSounds',
- },
- 'playlist_mincount': 166,
- 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden', 'YouTube Music is not directly supported'],
- }, {
- # TODO: fix 'unviewable' issue with this playlist when reloading with unavailable videos
- 'note': 'Topic, should redirect to playlist?list=UU...',
- 'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
- 'info_dict': {
- 'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
- 'title': 'Uploads from Royalty Free Music - Topic',
- 'tags': [],
- 'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
- 'channel': 'Royalty Free Music - Topic',
- 'view_count': int,
- 'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
- 'modified_date': r're:\d{8}',
- 'description': '',
- 'availability': 'public',
- 'uploader': 'Royalty Free Music - Topic',
- },
- 'playlist_mincount': 101,
- 'expected_warnings': ['YouTube Music is not directly supported', r'[Uu]navailable videos (are|will be) hidden'],
- }, {
- # Destination channel with only a hidden self tab (tab id is UCtFRv9O2AHqOZjjynzrv-xg)
- # Treat as a general feed
- 'url': 'https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg',
- 'info_dict': {
- 'id': 'UCtFRv9O2AHqOZjjynzrv-xg',
- 'title': 'UCtFRv9O2AHqOZjjynzrv-xg',
- 'tags': [],
- },
- 'playlist_mincount': 9,
- }, {
- 'note': 'Youtube music Album',
- 'url': 'https://music.youtube.com/browse/MPREb_gTAcphH99wE',
- 'info_dict': {
- 'id': 'OLAK5uy_l1m0thk3g31NmIIz_vMIbWtyv7eZixlH0',
- 'title': 'Album - Royalty Free Music Library V2 (50 Songs)',
- 'tags': [],
- 'view_count': int,
- 'description': '',
- 'availability': 'unlisted',
- 'modified_date': r're:\d{8}',
- },
- 'playlist_count': 50,
- 'expected_warnings': ['YouTube Music is not directly supported'],
- }, {
- 'note': 'unlisted single video playlist',
- 'url': 'https://www.youtube.com/playlist?list=PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
- 'info_dict': {
- 'id': 'PLwL24UFy54GrB3s2KMMfjZscDi1x5Dajf',
- 'title': 'yt-dlp unlisted playlist test',
- 'availability': 'unlisted',
- 'tags': [],
- 'modified_date': '20220418',
- 'channel': 'colethedj',
- 'view_count': int,
- 'description': '',
- 'channel_id': 'UC9zHu_mHU96r19o-wV5Qs1Q',
- 'channel_url': 'https://www.youtube.com/channel/UC9zHu_mHU96r19o-wV5Qs1Q',
- 'uploader_url': 'https://www.youtube.com/@colethedj1894',
- 'uploader_id': '@colethedj1894',
- 'uploader': 'colethedj',
- },
- 'playlist': [{
- 'info_dict': {
- 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
- 'id': 'BaW_jenozKc',
- '_type': 'url',
- 'ie_key': 'Youtube',
- 'duration': 10,
- 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
- 'channel_url': 'https://www.youtube.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
- 'view_count': int,
- 'url': 'https://www.youtube.com/watch?v=BaW_jenozKc',
- 'channel': 'Philipp Hagemeister',
- 'uploader_id': '@PhilippHagemeister',
- 'uploader_url': 'https://www.youtube.com/@PhilippHagemeister',
- 'uploader': 'Philipp Hagemeister',
- },
- }],
- 'playlist_count': 1,
- 'params': {'extract_flat': True},
- }, {
- 'note': 'API Fallback: Recommended - redirects to home page. Requires visitorData',
- 'url': 'https://www.youtube.com/feed/recommended',
- 'info_dict': {
- 'id': 'recommended',
- 'title': 'recommended',
- 'tags': [],
- },
- 'playlist_mincount': 50,
- 'params': {
- 'skip_download': True,
- 'extractor_args': {'youtubetab': {'skip': ['webpage']}},
- },
- }, {
- 'note': 'API Fallback: /videos tab, sorted by oldest first',
- 'url': 'https://www.youtube.com/user/theCodyReeder/videos?view=0&sort=da&flow=grid',
- 'info_dict': {
- 'id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
- 'title': 'Cody\'sLab - Videos',
- 'description': 'md5:d083b7c2f0c67ee7a6c74c3e9b4243fa',
- 'channel': 'Cody\'sLab',
- 'channel_id': 'UCu6mSoMNzHQiBIOCkHUa2Aw',
- 'tags': [],
- 'channel_url': 'https://www.youtube.com/channel/UCu6mSoMNzHQiBIOCkHUa2Aw',
- 'channel_follower_count': int,
- },
- 'playlist_mincount': 650,
- 'params': {
- 'skip_download': True,
- 'extractor_args': {'youtubetab': {'skip': ['webpage']}},
- },
- 'skip': 'Query for sorting no longer works',
- }, {
- 'note': 'API Fallback: Topic, should redirect to playlist?list=UU...',
- 'url': 'https://music.youtube.com/browse/UC9ALqqC4aIeG5iDs7i90Bfw',
- 'info_dict': {
- 'id': 'UU9ALqqC4aIeG5iDs7i90Bfw',
- 'title': 'Uploads from Royalty Free Music - Topic',
- 'modified_date': r're:\d{8}',
- 'channel_id': 'UC9ALqqC4aIeG5iDs7i90Bfw',
- 'description': '',
- 'channel_url': 'https://www.youtube.com/channel/UC9ALqqC4aIeG5iDs7i90Bfw',
- 'tags': [],
- 'channel': 'Royalty Free Music - Topic',
- 'view_count': int,
- 'availability': 'public',
- 'uploader': 'Royalty Free Music - Topic',
- },
- 'playlist_mincount': 101,
- 'params': {
- 'skip_download': True,
- 'extractor_args': {'youtubetab': {'skip': ['webpage']}},
- },
- 'expected_warnings': ['YouTube Music is not directly supported', r'[Uu]navailable videos (are|will be) hidden'],
- }, {
- 'note': 'non-standard redirect to regional channel',
- 'url': 'https://www.youtube.com/channel/UCwVVpHQ2Cs9iGJfpdFngePQ',
- 'only_matching': True,
- }, {
- 'note': 'collaborative playlist (uploader name in the form "by <uploader> and x other(s)")',
- 'url': 'https://www.youtube.com/playlist?list=PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
- 'info_dict': {
- 'id': 'PLx-_-Kk4c89oOHEDQAojOXzEzemXxoqx6',
- 'modified_date': '20220407',
- 'channel_url': 'https://www.youtube.com/channel/UCKcqXmCcyqnhgpA5P0oHH_Q',
- 'tags': [],
- 'availability': 'unlisted',
- 'channel_id': 'UCKcqXmCcyqnhgpA5P0oHH_Q',
- 'channel': 'pukkandan',
- 'description': 'Test for collaborative playlist',
- 'title': 'yt-dlp test - collaborative playlist',
- 'view_count': int,
- 'uploader_url': 'https://www.youtube.com/@pukkandan',
- 'uploader_id': '@pukkandan',
- 'uploader': 'pukkandan',
- },
- 'playlist_mincount': 2,
- }, {
- 'note': 'translated tab name',
- 'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/playlists',
- 'info_dict': {
- 'id': 'UCiu-3thuViMebBjw_5nWYrA',
- 'tags': [],
- 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
- 'description': 'test description',
- 'title': 'cole-dlp-test-acc - 再生リスト',
- 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
- 'channel': 'cole-dlp-test-acc',
- 'uploader_url': 'https://www.youtube.com/@coletdjnz',
- 'uploader_id': '@coletdjnz',
- 'uploader': 'cole-dlp-test-acc',
- },
- 'playlist_mincount': 1,
- 'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
- 'expected_warnings': ['Preferring "ja"'],
- }, {
- # XXX: this should really check flat playlist entries, but the test suite doesn't support that
- 'note': 'preferred lang set with playlist with translated video titles',
- 'url': 'https://www.youtube.com/playlist?list=PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
- 'info_dict': {
- 'id': 'PLt5yu3-wZAlQAaPZ5Z-rJoTdbT-45Q7c0',
- 'tags': [],
- 'view_count': int,
- 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
- 'channel': 'cole-dlp-test-acc',
- 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
- 'description': 'test',
- 'title': 'dlp test playlist',
- 'availability': 'public',
- 'uploader_url': 'https://www.youtube.com/@coletdjnz',
- 'uploader_id': '@coletdjnz',
- 'uploader': 'cole-dlp-test-acc',
- },
- 'playlist_mincount': 1,
- 'params': {'extractor_args': {'youtube': {'lang': ['ja']}}},
- 'expected_warnings': ['Preferring "ja"'],
- }, {
- # shorts audio pivot for 2GtVksBMYFM.
- 'url': 'https://www.youtube.com/feed/sfv_audio_pivot?bp=8gUrCikSJwoLMkd0VmtzQk1ZRk0SCzJHdFZrc0JNWUZNGgsyR3RWa3NCTVlGTQ==',
- 'info_dict': {
- 'id': 'sfv_audio_pivot',
- 'title': 'sfv_audio_pivot',
- 'tags': [],
- },
- 'playlist_mincount': 50,
-
- }, {
- # Channel with a real live tab (not to be mistaken with streams tab)
- # Do not treat like it should redirect to live stream
- 'url': 'https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live',
- 'info_dict': {
- 'id': 'UCEH7P7kyJIkS_gJf93VYbmg',
- 'title': 'UCEH7P7kyJIkS_gJf93VYbmg - Live',
- 'tags': [],
- },
- 'playlist_mincount': 20,
- }, {
- # Tab name is not the same as tab id
- 'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/letsplay',
- 'info_dict': {
- 'id': 'UCQvWX73GQygcwXOTSf_VDVg',
- 'title': 'UCQvWX73GQygcwXOTSf_VDVg - Let\'s play',
- 'tags': [],
- },
- 'playlist_mincount': 8,
- }, {
- # Home tab id is literally home. Not to get mistaken with featured
- 'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/home',
- 'info_dict': {
- 'id': 'UCQvWX73GQygcwXOTSf_VDVg',
- 'title': 'UCQvWX73GQygcwXOTSf_VDVg - Home',
- 'tags': [],
- },
- 'playlist_mincount': 8,
- }, {
- # Should get three playlists for videos, shorts and streams tabs
- 'url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
- 'info_dict': {
- 'id': 'UCK9V2B22uJYu3N7eR_BT9QA',
- 'title': 'Polka Ch. 尾丸ポルカ',
- 'channel_follower_count': int,
- 'channel_id': 'UCK9V2B22uJYu3N7eR_BT9QA',
- 'channel_url': 'https://www.youtube.com/channel/UCK9V2B22uJYu3N7eR_BT9QA',
- 'description': 'md5:49809d8bf9da539bc48ed5d1f83c33f2',
- 'channel': 'Polka Ch. 尾丸ポルカ',
- 'tags': 'count:35',
- 'uploader_url': 'https://www.youtube.com/@OmaruPolka',
- 'uploader': 'Polka Ch. 尾丸ポルカ',
- 'uploader_id': '@OmaruPolka',
- 'channel_is_verified': True,
- },
- 'playlist_count': 3,
- }, {
- # Shorts tab with channel with handle
- # TODO: fix channel description
- 'url': 'https://www.youtube.com/@NotJustBikes/shorts',
- 'info_dict': {
- 'id': 'UC0intLFzLaudFG-xAvUEO-A',
- 'title': 'Not Just Bikes - Shorts',
- 'tags': 'count:10',
- 'channel_url': 'https://www.youtube.com/channel/UC0intLFzLaudFG-xAvUEO-A',
- 'description': 'md5:5e82545b3a041345927a92d0585df247',
- 'channel_follower_count': int,
- 'channel_id': 'UC0intLFzLaudFG-xAvUEO-A',
- 'channel': 'Not Just Bikes',
- 'uploader_url': 'https://www.youtube.com/@NotJustBikes',
- 'uploader': 'Not Just Bikes',
- 'uploader_id': '@NotJustBikes',
- 'channel_is_verified': True,
- },
- 'playlist_mincount': 10,
- }, {
- # Streams tab
- 'url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig/streams',
- 'info_dict': {
- 'id': 'UC3eYAvjCVwNHgkaGbXX3sig',
- 'title': '中村悠一 - Live',
- 'tags': 'count:7',
- 'channel_id': 'UC3eYAvjCVwNHgkaGbXX3sig',
- 'channel_url': 'https://www.youtube.com/channel/UC3eYAvjCVwNHgkaGbXX3sig',
- 'channel': '中村悠一',
- 'channel_follower_count': int,
- 'description': 'md5:e744f6c93dafa7a03c0c6deecb157300',
- 'uploader_url': 'https://www.youtube.com/@Yuichi-Nakamura',
- 'uploader_id': '@Yuichi-Nakamura',
- 'uploader': '中村悠一',
- },
- 'playlist_mincount': 60,
- }, {
- # Channel with no uploads and hence no videos, streams, shorts tabs or uploads playlist. This should fail.
- # See test_youtube_lists
- 'url': 'https://www.youtube.com/channel/UC2yXPzFejc422buOIzn_0CA',
- 'only_matching': True,
- }, {
- # No uploads and no UCID given. Should fail with no uploads error
- # See test_youtube_lists
- 'url': 'https://www.youtube.com/news',
- 'only_matching': True,
- }, {
- # No videos tab but has a shorts tab
- 'url': 'https://www.youtube.com/c/TKFShorts',
- 'info_dict': {
- 'id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
- 'title': 'Shorts Break - Shorts',
- 'tags': 'count:48',
- 'channel_id': 'UCgJ5_1F6yJhYLnyMszUdmUg',
- 'channel': 'Shorts Break',
- 'description': 'md5:6de33c5e7ba686e5f3efd4e19c7ef499',
- 'channel_follower_count': int,
- 'channel_url': 'https://www.youtube.com/channel/UCgJ5_1F6yJhYLnyMszUdmUg',
- 'uploader_url': 'https://www.youtube.com/@ShortsBreak_Official',
- 'uploader': 'Shorts Break',
- 'uploader_id': '@ShortsBreak_Official',
- },
- 'playlist_mincount': 30,
- }, {
- # Trending Now Tab. tab id is empty
- 'url': 'https://www.youtube.com/feed/trending',
- 'info_dict': {
- 'id': 'trending',
- 'title': 'trending - Now',
- 'tags': [],
- },
- 'playlist_mincount': 30,
- }, {
- # Trending Gaming Tab. tab id is empty
- 'url': 'https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D',
- 'info_dict': {
- 'id': 'trending',
- 'title': 'trending - Gaming',
- 'tags': [],
- },
- 'playlist_mincount': 30,
- }, {
- # Shorts url result in shorts tab
- # TODO: Fix channel id extraction
- 'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/shorts',
- 'info_dict': {
- 'id': 'UCiu-3thuViMebBjw_5nWYrA',
- 'title': 'cole-dlp-test-acc - Shorts',
- 'channel': 'cole-dlp-test-acc',
- 'description': 'test description',
- 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
- 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
- 'tags': [],
- 'uploader_url': 'https://www.youtube.com/@coletdjnz',
- 'uploader_id': '@coletdjnz',
- 'uploader': 'cole-dlp-test-acc',
- },
- 'playlist': [{
- 'info_dict': {
- # Channel data is not currently available for short renderers (as of 2023-03-01)
- '_type': 'url',
- 'ie_key': 'Youtube',
- 'url': 'https://www.youtube.com/shorts/sSM9J5YH_60',
- 'id': 'sSM9J5YH_60',
- 'title': 'SHORT short',
- 'view_count': int,
- 'thumbnails': list,
- },
- }],
- 'params': {'extract_flat': True},
- }, {
- # Live video status should be extracted
- 'url': 'https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/live',
- 'info_dict': {
- 'id': 'UCQvWX73GQygcwXOTSf_VDVg',
- 'title': 'UCQvWX73GQygcwXOTSf_VDVg - Live', # TODO: should be Minecraft - Live or Minecraft - Topic - Live
- 'tags': [],
- },
- 'playlist': [{
- 'info_dict': {
- '_type': 'url',
- 'ie_key': 'Youtube',
- 'url': 'startswith:https://www.youtube.com/watch?v=',
- 'id': str,
- 'title': str,
- 'live_status': 'is_live',
- 'channel_id': str,
- 'channel_url': str,
- 'concurrent_view_count': int,
- 'channel': str,
- 'uploader': str,
- 'uploader_url': str,
- 'uploader_id': str,
- 'channel_is_verified': bool, # this will keep changing
- },
- }],
- 'params': {'extract_flat': True, 'playlist_items': '1'},
- 'playlist_mincount': 1,
- }, {
- # Channel renderer metadata. Contains number of videos on the channel
- 'url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA/channels',
- 'info_dict': {
- 'id': 'UCiu-3thuViMebBjw_5nWYrA',
- 'title': 'cole-dlp-test-acc - Channels',
- 'channel': 'cole-dlp-test-acc',
- 'description': 'test description',
- 'channel_id': 'UCiu-3thuViMebBjw_5nWYrA',
- 'channel_url': 'https://www.youtube.com/channel/UCiu-3thuViMebBjw_5nWYrA',
- 'tags': [],
- 'uploader_url': 'https://www.youtube.com/@coletdjnz',
- 'uploader_id': '@coletdjnz',
- 'uploader': 'cole-dlp-test-acc',
- },
- 'playlist': [{
- 'info_dict': {
- '_type': 'url',
- 'ie_key': 'YoutubeTab',
- 'url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
- 'id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
- 'channel_id': 'UC-lHJZR3Gqxm24_Vd_AJ5Yw',
- 'title': 'PewDiePie',
- 'channel': 'PewDiePie',
- 'channel_url': 'https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw',
- 'thumbnails': list,
- 'channel_follower_count': int,
- 'playlist_count': int,
- 'uploader': 'PewDiePie',
- 'uploader_url': 'https://www.youtube.com/@PewDiePie',
- 'uploader_id': '@PewDiePie',
- 'channel_is_verified': True,
- },
- }],
- 'params': {'extract_flat': True},
- }, {
- 'url': 'https://www.youtube.com/@3blue1brown/about',
- 'info_dict': {
- 'id': '@3blue1brown',
- 'tags': ['Mathematics'],
- 'title': '3Blue1Brown',
- 'channel_follower_count': int,
- 'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
- 'channel': '3Blue1Brown',
- 'channel_url': 'https://www.youtube.com/channel/UCYO_jab_esuFRV4b17AJtAw',
- 'description': 'md5:4d1da95432004b7ba840ebc895b6b4c9',
- 'uploader_url': 'https://www.youtube.com/@3blue1brown',
- 'uploader_id': '@3blue1brown',
- 'uploader': '3Blue1Brown',
- 'channel_is_verified': True,
- },
- 'playlist_count': 0,
- }, {
- # Podcasts tab, with rich entry lockupViewModel
- 'url': 'https://www.youtube.com/@99percentinvisiblepodcast/podcasts',
- 'info_dict': {
- 'id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
- 'channel_id': 'UCVMF2HD4ZgC0QHpU9Yq5Xrw',
- 'uploader_url': 'https://www.youtube.com/@99percentinvisiblepodcast',
- 'description': 'md5:3a0ed38f1ad42a68ef0428c04a15695c',
- 'title': '99% Invisible - Podcasts',
- 'uploader': '99% Invisible',
- 'channel_follower_count': int,
- 'channel_url': 'https://www.youtube.com/channel/UCVMF2HD4ZgC0QHpU9Yq5Xrw',
- 'tags': [],
- 'channel': '99% Invisible',
- 'uploader_id': '@99percentinvisiblepodcast',
- },
- 'playlist_count': 5,
- }, {
- # Releases tab, with rich entry playlistRenderers (same as Podcasts tab)
- 'url': 'https://www.youtube.com/@AHimitsu/releases',
- 'info_dict': {
- 'id': 'UCgFwu-j5-xNJml2FtTrrB3A',
- 'channel': 'A Himitsu',
- 'uploader_url': 'https://www.youtube.com/@AHimitsu',
- 'title': 'A Himitsu - Releases',
- 'uploader_id': '@AHimitsu',
- 'uploader': 'A Himitsu',
- 'channel_id': 'UCgFwu-j5-xNJml2FtTrrB3A',
- 'tags': 'count:12',
- 'description': 'I make music',
- 'channel_url': 'https://www.youtube.com/channel/UCgFwu-j5-xNJml2FtTrrB3A',
- 'channel_follower_count': int,
- 'channel_is_verified': True,
- },
- 'playlist_mincount': 10,
- }, {
- # Playlist with only shorts, shown as reel renderers
- # FIXME: future: YouTube currently doesn't give continuation for this,
- # may do in future.
- 'url': 'https://www.youtube.com/playlist?list=UUxqPAgubo4coVn9Lx1FuKcg',
- 'info_dict': {
- 'id': 'UUxqPAgubo4coVn9Lx1FuKcg',
- 'channel_url': 'https://www.youtube.com/channel/UCxqPAgubo4coVn9Lx1FuKcg',
- 'view_count': int,
- 'uploader_id': '@BangyShorts',
- 'description': '',
- 'uploader_url': 'https://www.youtube.com/@BangyShorts',
- 'channel_id': 'UCxqPAgubo4coVn9Lx1FuKcg',
- 'channel': 'Bangy Shorts',
- 'uploader': 'Bangy Shorts',
- 'tags': [],
- 'availability': 'public',
- 'modified_date': r're:\d{8}',
- 'title': 'Uploads from Bangy Shorts',
- },
- 'playlist_mincount': 100,
- 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
- }, {
- 'note': 'Tags containing spaces',
- 'url': 'https://www.youtube.com/channel/UC7_YxT-KID8kRbqZo7MyscQ',
- 'playlist_count': 3,
- 'info_dict': {
- 'id': 'UC7_YxT-KID8kRbqZo7MyscQ',
- 'channel': 'Markiplier',
- 'channel_id': 'UC7_YxT-KID8kRbqZo7MyscQ',
- 'title': 'Markiplier',
- 'channel_follower_count': int,
- 'description': 'md5:0c010910558658824402809750dc5d97',
- 'uploader_id': '@markiplier',
- 'uploader_url': 'https://www.youtube.com/@markiplier',
- 'uploader': 'Markiplier',
- 'channel_url': 'https://www.youtube.com/channel/UC7_YxT-KID8kRbqZo7MyscQ',
- 'channel_is_verified': True,
- 'tags': ['markiplier', 'comedy', 'gaming', 'funny videos', 'funny moments',
- 'sketch comedy', 'laughing', 'lets play', 'challenge videos', 'hilarious',
- 'challenges', 'sketches', 'scary games', 'funny games', 'rage games',
- 'mark fischbach'],
- },
- }]
-
- @classmethod
- def suitable(cls, url):
- return False if YoutubeIE.suitable(url) else super().suitable(url)
-
- _URL_RE = re.compile(rf'(?P<pre>{_VALID_URL})(?(not_channel)|(?P<tab>/[^?#/]+))?(?P<post>.*)$')
-
- def _get_url_mobj(self, url):
- mobj = self._URL_RE.match(url).groupdict()
- mobj.update((k, '') for k, v in mobj.items() if v is None)
- return mobj
-
- def _extract_tab_id_and_name(self, tab, base_url='https://www.youtube.com'):
- tab_name = (tab.get('title') or '').lower()
- tab_url = urljoin(base_url, traverse_obj(
- tab, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url')))
-
- tab_id = ((tab_url and self._get_url_mobj(tab_url)['tab'][1:])
- or traverse_obj(tab, 'tabIdentifier', expected_type=str))
- if tab_id:
- return {
- 'TAB_ID_SPONSORSHIPS': 'membership',
- }.get(tab_id, tab_id), tab_name
-
- # Fallback to tab name if we cannot get the tab id.
- # XXX: should we strip non-ascii letters? e.g. in case of 'let's play' tab example on special gaming channel
- # Note that in the case of translated tab name this may result in an empty string, which we don't want.
- if tab_name:
- self.write_debug(f'Falling back to selected tab name: {tab_name}')
- return {
- 'home': 'featured',
- 'live': 'streams',
- }.get(tab_name, tab_name), tab_name
-
- def _has_tab(self, tabs, tab_id):
- return any(self._extract_tab_id_and_name(tab)[0] == tab_id for tab in tabs)
-
- def _empty_playlist(self, item_id, data):
- return self.playlist_result([], item_id, **self._extract_metadata_from_tabs(item_id, data))
-
- @YoutubeTabBaseInfoExtractor.passthrough_smuggled_data
- def _real_extract(self, url, smuggled_data):
- item_id = self._match_id(url)
- url = urllib.parse.urlunparse(
- urllib.parse.urlparse(url)._replace(netloc='www.youtube.com'))
- compat_opts = self.get_param('compat_opts', [])
-
- mobj = self._get_url_mobj(url)
- pre, tab, post, is_channel = mobj['pre'], mobj['tab'], mobj['post'], not mobj['not_channel']
- if is_channel and smuggled_data.get('is_music_url'):
- if item_id[:2] == 'VL': # Youtube music VL channels have an equivalent playlist
- return self.url_result(
- f'https://music.youtube.com/playlist?list={item_id[2:]}', YoutubeTabIE, item_id[2:])
- elif item_id[:2] == 'MP': # Resolve albums (/[channel/browse]/MP...) to their equivalent playlist
- mdata = self._extract_tab_endpoint(
- f'https://music.youtube.com/channel/{item_id}', item_id, default_client='web_music')
- murl = traverse_obj(mdata, ('microformat', 'microformatDataRenderer', 'urlCanonical'),
- get_all=False, expected_type=str)
- if not murl:
- raise ExtractorError('Failed to resolve album to playlist')
- return self.url_result(murl, YoutubeTabIE)
- elif mobj['channel_type'] == 'browse': # Youtube music /browse/ should be changed to /channel/
- return self.url_result(
- f'https://music.youtube.com/channel/{item_id}{tab}{post}', YoutubeTabIE, item_id)
-
- original_tab_id, display_id = tab[1:], f'{item_id}{tab}'
- if is_channel and not tab and 'no-youtube-channel-redirect' not in compat_opts:
- url = f'{pre}/videos{post}'
- if smuggled_data.get('is_music_url'):
- self.report_warning(f'YouTube Music is not directly supported. Redirecting to {url}')
-
- # Handle both video/playlist URLs
- qs = parse_qs(url)
- video_id, playlist_id = (traverse_obj(qs, (key, 0)) for key in ('v', 'list'))
- if not video_id and mobj['not_channel'].startswith('watch'):
- if not playlist_id:
- # If there is neither video or playlist ids, youtube redirects to home page, which is undesirable
- raise ExtractorError('A video URL was given without video ID', expected=True)
- # Common mistake: https://www.youtube.com/watch?list=playlist_id
- self.report_warning(f'A video URL was given without video ID. Trying to download playlist {playlist_id}')
- return self.url_result(
- f'https://www.youtube.com/playlist?list={playlist_id}', YoutubeTabIE, playlist_id)
-
- if not self._yes_playlist(playlist_id, video_id):
- return self.url_result(
- f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
-
- data, ytcfg = self._extract_data(url, display_id)
-
- # YouTube may provide a non-standard redirect to the regional channel
- # See: https://github.com/yt-dlp/yt-dlp/issues/2694
- # https://support.google.com/youtube/answer/2976814#zippy=,conditional-redirects
- redirect_url = traverse_obj(
- data, ('onResponseReceivedActions', ..., 'navigateAction', 'endpoint', 'commandMetadata', 'webCommandMetadata', 'url'), get_all=False)
- if redirect_url and 'no-youtube-channel-redirect' not in compat_opts:
- redirect_url = ''.join((urljoin('https://www.youtube.com', redirect_url), tab, post))
- self.to_screen(f'This playlist is likely not available in your region. Following conditional redirect to {redirect_url}')
- return self.url_result(redirect_url, YoutubeTabIE)
-
- tabs, extra_tabs = self._extract_tab_renderers(data), []
- if is_channel and tabs and 'no-youtube-channel-redirect' not in compat_opts:
- selected_tab = self._extract_selected_tab(tabs)
- selected_tab_id, selected_tab_name = self._extract_tab_id_and_name(selected_tab, url) # NB: Name may be translated
- self.write_debug(f'Selected tab: {selected_tab_id!r} ({selected_tab_name}), Requested tab: {original_tab_id!r}')
-
- # /about is no longer a tab
- if original_tab_id == 'about':
- return self._empty_playlist(item_id, data)
-
- if not original_tab_id and selected_tab_name:
- self.to_screen('Downloading all uploads of the channel. '
- 'To download only the videos in a specific tab, pass the tab\'s URL')
- if self._has_tab(tabs, 'streams'):
- extra_tabs.append(''.join((pre, '/streams', post)))
- if self._has_tab(tabs, 'shorts'):
- extra_tabs.append(''.join((pre, '/shorts', post)))
- # XXX: Members-only tab should also be extracted
-
- if not extra_tabs and selected_tab_id != 'videos':
- # Channel does not have streams, shorts or videos tabs
- if item_id[:2] != 'UC':
- return self._empty_playlist(item_id, data)
-
- # Topic channels don't have /videos. Use the equivalent playlist instead
- pl_id = f'UU{item_id[2:]}'
- pl_url = f'https://www.youtube.com/playlist?list={pl_id}'
- try:
- data, ytcfg = self._extract_data(pl_url, pl_id, ytcfg=ytcfg, fatal=True, webpage_fatal=True)
- except ExtractorError:
- return self._empty_playlist(item_id, data)
- else:
- item_id, url = pl_id, pl_url
- self.to_screen(
- f'The channel does not have a videos, shorts, or live tab. Redirecting to playlist {pl_id} instead')
-
- elif extra_tabs and selected_tab_id != 'videos':
- # When there are shorts/live tabs but not videos tab
- url, data = f'{pre}{post}', None
-
- elif (original_tab_id or 'videos') != selected_tab_id:
- if original_tab_id == 'live':
- # Live tab should have redirected to the video
- # Except in the case the channel has an actual live tab
- # Example: https://www.youtube.com/channel/UCEH7P7kyJIkS_gJf93VYbmg/live
- raise UserNotLive(video_id=item_id)
- elif selected_tab_name:
- raise ExtractorError(f'This channel does not have a {original_tab_id} tab', expected=True)
-
- # For channels such as https://www.youtube.com/channel/UCtFRv9O2AHqOZjjynzrv-xg
- url = f'{pre}{post}'
-
- # YouTube sometimes provides a button to reload playlist with unavailable videos.
- if 'no-youtube-unavailable-videos' not in compat_opts:
- data = self._reload_with_unavailable_videos(display_id, data, ytcfg) or data
- self._extract_and_report_alerts(data, only_once=True)
-
- tabs, entries = self._extract_tab_renderers(data), []
- if tabs:
- entries = [self._extract_from_tabs(item_id, ytcfg, data, tabs)]
- entries[0].update({
- 'extractor_key': YoutubeTabIE.ie_key(),
- 'extractor': YoutubeTabIE.IE_NAME,
- 'webpage_url': url,
- })
- if self.get_param('playlist_items') == '0':
- entries.extend(self.url_result(u, YoutubeTabIE) for u in extra_tabs)
- else: # Users expect to get all `video_id`s even with `--flat-playlist`. So don't return `url_result`
- entries.extend(map(self._real_extract, extra_tabs))
-
- if len(entries) == 1:
- return entries[0]
- elif entries:
- metadata = self._extract_metadata_from_tabs(item_id, data)
- uploads_url = 'the Uploads (UU) playlist URL'
- if try_get(metadata, lambda x: x['channel_id'].startswith('UC')):
- uploads_url = f'https://www.youtube.com/playlist?list=UU{metadata["channel_id"][2:]}'
- self.to_screen(
- 'Downloading as multiple playlists, separated by tabs. '
- f'To download as a single playlist instead, pass {uploads_url}')
- return self.playlist_result(entries, item_id, **metadata)
-
- # Inline playlist
- playlist = traverse_obj(
- data, ('contents', 'twoColumnWatchNextResults', 'playlist', 'playlist'), expected_type=dict)
- if playlist:
- return self._extract_from_playlist(item_id, url, data, playlist, ytcfg)
-
- video_id = traverse_obj(
- data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'), expected_type=str) or video_id
- if video_id:
- if tab != '/live': # live tab is expected to redirect to video
- self.report_warning(f'Unable to recognize playlist. Downloading just video {video_id}')
- return self.url_result(f'https://www.youtube.com/watch?v={video_id}', YoutubeIE, video_id)
-
- raise ExtractorError('Unable to recognize tab page')
-
-
-class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
- IE_DESC = 'YouTube playlists'
- _VALID_URL = r'''(?x)(?:
- (?:https?://)?
- (?:\w+\.)?
- (?:
- (?:
- youtube(?:kids)?\.com|
- {invidious}
- )
- /.*?\?.*?\blist=
- )?
- (?P<id>{playlist_id})
- )'''.format(
- playlist_id=YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
- invidious='|'.join(YoutubeBaseInfoExtractor._INVIDIOUS_SITES),
- )
- IE_NAME = 'youtube:playlist'
- _TESTS = [{
- 'note': 'issue #673',
- 'url': 'PLBB231211A4F62143',
- 'info_dict': {
- 'title': '[OLD]Team Fortress 2 (Class-based LP)',
- 'id': 'PLBB231211A4F62143',
- 'uploader': 'Wickman',
- 'uploader_id': '@WickmanVT',
- 'description': 'md5:8fa6f52abb47a9552002fa3ddfc57fc2',
- 'view_count': int,
- 'uploader_url': 'https://www.youtube.com/@WickmanVT',
- 'modified_date': r're:\d{8}',
- 'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
- 'channel': 'Wickman',
- 'tags': [],
- 'channel_url': 'https://www.youtube.com/channel/UCKSpbfbl5kRQpTdL7kMc-1Q',
- 'availability': 'public',
- },
- 'playlist_mincount': 29,
- }, {
- 'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
- 'info_dict': {
- 'title': 'YDL_safe_search',
- 'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
- },
- 'playlist_count': 2,
- 'skip': 'This playlist is private',
- }, {
- 'note': 'embedded',
- 'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
- 'playlist_count': 4,
- 'info_dict': {
- 'title': 'JODA15',
- 'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
- 'uploader': 'milan',
- 'uploader_id': '@milan5503',
- 'description': '',
- 'channel_url': 'https://www.youtube.com/channel/UCEI1-PVPcYXjB73Hfelbmaw',
- 'tags': [],
- 'modified_date': '20140919',
- 'view_count': int,
- 'channel': 'milan',
- 'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
- 'uploader_url': 'https://www.youtube.com/@milan5503',
- 'availability': 'public',
- },
- 'expected_warnings': [r'[Uu]navailable videos? (is|are|will be) hidden', 'Retrying', 'Giving up'],
- }, {
- 'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
- 'playlist_mincount': 455,
- 'info_dict': {
- 'title': '2018 Chinese New Singles (11/6 updated)',
- 'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
- 'uploader': 'LBK',
- 'uploader_id': '@music_king',
- 'description': 'md5:da521864744d60a198e3a88af4db0d9d',
- 'channel': 'LBK',
- 'view_count': int,
- 'channel_url': 'https://www.youtube.com/channel/UC21nz3_MesPLqtDqwdvnoxA',
- 'tags': [],
- 'uploader_url': 'https://www.youtube.com/@music_king',
- 'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
- 'modified_date': r're:\d{8}',
- 'availability': 'public',
- },
- 'expected_warnings': [r'[Uu]navailable videos (are|will be) hidden'],
- }, {
- 'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
- 'only_matching': True,
- }, {
- # music album playlist
- 'url': 'OLAK5uy_m4xAFdmMC5rX3Ji3g93pQe3hqLZw_9LhM',
- 'only_matching': True,
- }]
-
- @classmethod
- def suitable(cls, url):
- if YoutubeTabIE.suitable(url):
- return False
- from ..utils import parse_qs
- qs = parse_qs(url)
- if qs.get('v', [None])[0]:
- return False
- return super().suitable(url)
-
- def _real_extract(self, url):
- playlist_id = self._match_id(url)
- is_music_url = YoutubeBaseInfoExtractor.is_music_url(url)
- url = update_url_query(
- 'https://www.youtube.com/playlist',
- parse_qs(url) or {'list': playlist_id})
- if is_music_url:
- url = smuggle_url(url, {'is_music_url': True})
- return self.url_result(url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
-
-
-class YoutubeYtBeIE(YoutubeBaseInfoExtractor):
- IE_DESC = 'youtu.be'
- _VALID_URL = rf'https?://youtu\.be/(?P<id>[0-9A-Za-z_-]{{11}})/*?.*?\blist=(?P<playlist_id>{YoutubeBaseInfoExtractor._PLAYLIST_ID_RE})'
- _TESTS = [{
- 'url': 'https://youtu.be/yeWKywCrFtk?list=PL2qgrgXsNUG5ig9cat4ohreBjYLAPC0J5',
- 'info_dict': {
- 'id': 'yeWKywCrFtk',
- 'ext': 'mp4',
- 'title': 'Small Scale Baler and Braiding Rugs',
- 'uploader': 'Backus-Page House Museum',
- 'uploader_id': '@backuspagemuseum',
- 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@backuspagemuseum',
- 'upload_date': '20161008',
- 'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
- 'categories': ['Nonprofits & Activism'],
- 'tags': list,
- 'like_count': int,
- 'age_limit': 0,
- 'playable_in_embed': True,
- 'thumbnail': r're:^https?://.*\.webp',
- 'channel': 'Backus-Page House Museum',
- 'channel_id': 'UCEfMCQ9bs3tjvjy1s451zaw',
- 'live_status': 'not_live',
- 'view_count': int,
- 'channel_url': 'https://www.youtube.com/channel/UCEfMCQ9bs3tjvjy1s451zaw',
- 'availability': 'public',
- 'duration': 59,
- 'comment_count': int,
- 'channel_follower_count': int,
- },
- 'params': {
- 'noplaylist': True,
- 'skip_download': True,
- },
- }, {
- 'url': 'https://youtu.be/uWyaPkt-VOI?list=PL9D9FC436B881BA21',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- mobj = self._match_valid_url(url)
- video_id = mobj.group('id')
- playlist_id = mobj.group('playlist_id')
- return self.url_result(
- update_url_query('https://www.youtube.com/watch', {
- 'v': video_id,
- 'list': playlist_id,
- 'feature': 'youtu.be',
- }), ie=YoutubeTabIE.ie_key(), video_id=playlist_id)
-
-
-class YoutubeLivestreamEmbedIE(YoutubeBaseInfoExtractor):
- IE_DESC = 'YouTube livestream embeds'
- _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/embed/live_stream/?\?(?:[^#]+&)?channel=(?P<id>[^&#]+)'
- _TESTS = [{
- 'url': 'https://www.youtube.com/embed/live_stream?channel=UC2_KI6RB__jGdlnK6dvFEZA',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- channel_id = self._match_id(url)
- return self.url_result(
- f'https://www.youtube.com/channel/{channel_id}/live',
- ie=YoutubeTabIE.ie_key(), video_id=channel_id)
-
-
-class YoutubeYtUserIE(YoutubeBaseInfoExtractor):
- IE_DESC = 'YouTube user videos; "ytuser:" prefix'
- IE_NAME = 'youtube:user'
- _VALID_URL = r'ytuser:(?P<id>.+)'
- _TESTS = [{
- 'url': 'ytuser:phihag',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- user_id = self._match_id(url)
- return self.url_result(f'https://www.youtube.com/user/{user_id}', YoutubeTabIE, user_id)
-
-
-class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
- IE_NAME = 'youtube:favorites'
- IE_DESC = 'YouTube liked videos; ":ytfav" keyword (requires cookies)'
- _VALID_URL = r':ytfav(?:ou?rite)?s?'
- _LOGIN_REQUIRED = True
- _TESTS = [{
- 'url': ':ytfav',
- 'only_matching': True,
- }, {
- 'url': ':ytfavorites',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- return self.url_result(
- 'https://www.youtube.com/playlist?list=LL',
- ie=YoutubeTabIE.ie_key())
-
-
-class YoutubeNotificationsIE(YoutubeTabBaseInfoExtractor):
- IE_NAME = 'youtube:notif'
- IE_DESC = 'YouTube notifications; ":ytnotif" keyword (requires cookies)'
- _VALID_URL = r':ytnotif(?:ication)?s?'
- _LOGIN_REQUIRED = True
- _TESTS = [{
- 'url': ':ytnotif',
- 'only_matching': True,
- }, {
- 'url': ':ytnotifications',
- 'only_matching': True,
- }]
-
- def _extract_notification_menu(self, response, continuation_list):
- notification_list = traverse_obj(
- response,
- ('actions', 0, 'openPopupAction', 'popup', 'multiPageMenuRenderer', 'sections', 0, 'multiPageMenuNotificationSectionRenderer', 'items'),
- ('actions', 0, 'appendContinuationItemsAction', 'continuationItems'),
- expected_type=list) or []
- continuation_list[0] = None
- for item in notification_list:
- entry = self._extract_notification_renderer(item.get('notificationRenderer'))
- if entry:
- yield entry
- continuation = item.get('continuationItemRenderer')
- if continuation:
- continuation_list[0] = continuation
-
- def _extract_notification_renderer(self, notification):
- video_id = traverse_obj(
- notification, ('navigationEndpoint', 'watchEndpoint', 'videoId'), expected_type=str)
- url = f'https://www.youtube.com/watch?v={video_id}'
- channel_id = None
- if not video_id:
- browse_ep = traverse_obj(
- notification, ('navigationEndpoint', 'browseEndpoint'), expected_type=dict)
- channel_id = self.ucid_or_none(traverse_obj(browse_ep, 'browseId', expected_type=str))
- post_id = self._search_regex(
- r'/post/(.+)', traverse_obj(browse_ep, 'canonicalBaseUrl', expected_type=str),
- 'post id', default=None)
- if not channel_id or not post_id:
- return
- # The direct /post url redirects to this in the browser
- url = f'https://www.youtube.com/channel/{channel_id}/community?lb={post_id}'
-
- channel = traverse_obj(
- notification, ('contextualMenu', 'menuRenderer', 'items', 1, 'menuServiceItemRenderer', 'text', 'runs', 1, 'text'),
- expected_type=str)
- notification_title = self._get_text(notification, 'shortMessage')
- if notification_title:
- notification_title = notification_title.replace('\xad', '') # remove soft hyphens
- # TODO: handle recommended videos
- title = self._search_regex(
- rf'{re.escape(channel or "")}[^:]+: (.+)', notification_title,
- 'video title', default=None)
- timestamp = (self._parse_time_text(self._get_text(notification, 'sentTimeText'))
- if self._configuration_arg('approximate_date', ie_key=YoutubeTabIE)
- else None)
- return {
- '_type': 'url',
- 'url': url,
- 'ie_key': (YoutubeIE if video_id else YoutubeTabIE).ie_key(),
- 'video_id': video_id,
- 'title': title,
- 'channel_id': channel_id,
- 'channel': channel,
- 'uploader': channel,
- 'thumbnails': self._extract_thumbnails(notification, 'videoThumbnail'),
- 'timestamp': timestamp,
- }
-
- def _notification_menu_entries(self, ytcfg):
- continuation_list = [None]
- response = None
- for page in itertools.count(1):
- ctoken = traverse_obj(
- continuation_list, (0, 'continuationEndpoint', 'getNotificationMenuEndpoint', 'ctoken'), expected_type=str)
- response = self._extract_response(
- item_id=f'page {page}', query={'ctoken': ctoken} if ctoken else {}, ytcfg=ytcfg,
- ep='notification/get_notification_menu', check_get_keys='actions',
- headers=self.generate_api_headers(ytcfg=ytcfg, visitor_data=self._extract_visitor_data(response)))
- yield from self._extract_notification_menu(response, continuation_list)
- if not continuation_list[0]:
- break
-
- def _real_extract(self, url):
- display_id = 'notifications'
- ytcfg = self._download_ytcfg('web', display_id) if not self.skip_webpage else {}
- self._report_playlist_authcheck(ytcfg)
- return self.playlist_result(self._notification_menu_entries(ytcfg), display_id, display_id)
-
-
-class YoutubeSearchIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
- IE_DESC = 'YouTube search'
- IE_NAME = 'youtube:search'
- _SEARCH_KEY = 'ytsearch'
- _SEARCH_PARAMS = 'EgIQAfABAQ==' # Videos only
- _TESTS = [{
- 'url': 'ytsearch5:youtube-dl test video',
- 'playlist_count': 5,
- 'info_dict': {
- 'id': 'youtube-dl test video',
- 'title': 'youtube-dl test video',
- },
- }, {
- 'note': 'Suicide/self-harm search warning',
- 'url': 'ytsearch1:i hate myself and i wanna die',
- 'playlist_count': 1,
- 'info_dict': {
- 'id': 'i hate myself and i wanna die',
- 'title': 'i hate myself and i wanna die',
- },
- }]
-
-
-class YoutubeSearchDateIE(YoutubeTabBaseInfoExtractor, SearchInfoExtractor):
- IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
- _SEARCH_KEY = 'ytsearchdate'
- IE_DESC = 'YouTube search, newest videos first'
- _SEARCH_PARAMS = 'CAISAhAB8AEB' # Videos only, sorted by date
- _TESTS = [{
- 'url': 'ytsearchdate5:youtube-dl test video',
- 'playlist_count': 5,
- 'info_dict': {
- 'id': 'youtube-dl test video',
- 'title': 'youtube-dl test video',
- },
- }]
-
-
-class YoutubeSearchURLIE(YoutubeTabBaseInfoExtractor):
- IE_DESC = 'YouTube search URLs with sorting and filter support'
- IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
- _VALID_URL = r'https?://(?:www\.)?youtube\.com/(?:results|search)\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
- _TESTS = [{
- 'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
- 'playlist_mincount': 5,
- 'info_dict': {
- 'id': 'youtube-dl test video',
- 'title': 'youtube-dl test video',
- },
- }, {
- 'url': 'https://www.youtube.com/results?search_query=python&sp=EgIQAg%253D%253D',
- 'playlist_mincount': 5,
- 'info_dict': {
- 'id': 'python',
- 'title': 'python',
- },
- }, {
- 'url': 'https://www.youtube.com/results?search_query=%23cats',
- 'playlist_mincount': 1,
- 'info_dict': {
- 'id': '#cats',
- 'title': '#cats',
- # The test suite does not have support for nested playlists
- # 'entries': [{
- # 'url': r're:https://(www\.)?youtube\.com/hashtag/cats',
- # 'title': '#cats',
- # }],
- },
- }, {
- # Channel results
- 'url': 'https://www.youtube.com/results?search_query=kurzgesagt&sp=EgIQAg%253D%253D',
- 'info_dict': {
- 'id': 'kurzgesagt',
- 'title': 'kurzgesagt',
- },
- 'playlist': [{
- 'info_dict': {
- '_type': 'url',
- 'id': 'UCsXVk37bltHxD1rDPwtNM8Q',
- 'url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
- 'ie_key': 'YoutubeTab',
- 'channel': 'Kurzgesagt – In a Nutshell',
- 'description': 'md5:4ae48dfa9505ffc307dad26342d06bfc',
- 'title': 'Kurzgesagt – In a Nutshell',
- 'channel_id': 'UCsXVk37bltHxD1rDPwtNM8Q',
- # No longer available for search as it is set to the handle.
- # 'playlist_count': int,
- 'channel_url': 'https://www.youtube.com/channel/UCsXVk37bltHxD1rDPwtNM8Q',
- 'thumbnails': list,
- 'uploader_id': '@kurzgesagt',
- 'uploader_url': 'https://www.youtube.com/@kurzgesagt',
- 'uploader': 'Kurzgesagt – In a Nutshell',
- 'channel_is_verified': True,
- 'channel_follower_count': int,
- },
- }],
- 'params': {'extract_flat': True, 'playlist_items': '1'},
- 'playlist_mincount': 1,
- }, {
- 'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- qs = parse_qs(url)
- query = (qs.get('search_query') or qs.get('q'))[0]
- return self.playlist_result(self._search_results(query, qs.get('sp', (None,))[0]), query, query)
-
-
-class YoutubeMusicSearchURLIE(YoutubeTabBaseInfoExtractor):
- IE_DESC = 'YouTube music search URLs with selectable sections, e.g. #songs'
- IE_NAME = 'youtube:music:search_url'
- _VALID_URL = r'https?://music\.youtube\.com/search\?([^#]+&)?(?:search_query|q)=(?:[^&]+)(?:[&#]|$)'
- _TESTS = [{
- 'url': 'https://music.youtube.com/search?q=royalty+free+music',
- 'playlist_count': 16,
- 'info_dict': {
- 'id': 'royalty free music',
- 'title': 'royalty free music',
- },
- }, {
- 'url': 'https://music.youtube.com/search?q=royalty+free+music&sp=EgWKAQIIAWoKEAoQAxAEEAkQBQ%3D%3D',
- 'playlist_mincount': 30,
- 'info_dict': {
- 'id': 'royalty free music - songs',
- 'title': 'royalty free music - songs',
- },
- 'params': {'extract_flat': 'in_playlist'},
- }, {
- 'url': 'https://music.youtube.com/search?q=royalty+free+music#community+playlists',
- 'playlist_mincount': 30,
- 'info_dict': {
- 'id': 'royalty free music - community playlists',
- 'title': 'royalty free music - community playlists',
- },
- 'params': {'extract_flat': 'in_playlist'},
- }]
-
- _SECTIONS = {
- 'albums': 'EgWKAQIYAWoKEAoQAxAEEAkQBQ==',
- 'artists': 'EgWKAQIgAWoKEAoQAxAEEAkQBQ==',
- 'community playlists': 'EgeKAQQoAEABagoQChADEAQQCRAF',
- 'featured playlists': 'EgeKAQQoADgBagwQAxAJEAQQDhAKEAU==',
- 'songs': 'EgWKAQIIAWoKEAoQAxAEEAkQBQ==',
- 'videos': 'EgWKAQIQAWoKEAoQAxAEEAkQBQ==',
- }
-
- def _real_extract(self, url):
- qs = parse_qs(url)
- query = (qs.get('search_query') or qs.get('q'))[0]
- params = qs.get('sp', (None,))[0]
- if params:
- section = next((k for k, v in self._SECTIONS.items() if v == params), params)
- else:
- section = urllib.parse.unquote_plus(([*url.split('#'), ''])[1]).lower()
- params = self._SECTIONS.get(section)
- if not params:
- section = None
- title = join_nonempty(query, section, delim=' - ')
- return self.playlist_result(self._search_results(query, params, default_client='web_music'), title, title)
-
-
-class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
- """
- Base class for feed extractors
- Subclasses must re-define the _FEED_NAME property.
- """
- _LOGIN_REQUIRED = True
- _FEED_NAME = 'feeds'
-
- @classproperty
- def IE_NAME(cls):
- return f'youtube:{cls._FEED_NAME}'
-
- def _real_extract(self, url):
- return self.url_result(
- f'https://www.youtube.com/feed/{self._FEED_NAME}', ie=YoutubeTabIE.ie_key())
-
-
-class YoutubeWatchLaterIE(YoutubeBaseInfoExtractor):
- IE_NAME = 'youtube:watchlater'
- IE_DESC = 'Youtube watch later list; ":ytwatchlater" keyword (requires cookies)'
- _VALID_URL = r':ytwatchlater'
- _TESTS = [{
- 'url': ':ytwatchlater',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- return self.url_result(
- 'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key())
-
-
-class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
- IE_DESC = 'YouTube recommended videos; ":ytrec" keyword'
- _VALID_URL = r'https?://(?:www\.)?youtube\.com/?(?:[?#]|$)|:ytrec(?:ommended)?'
- _FEED_NAME = 'recommended'
- _LOGIN_REQUIRED = False
- _TESTS = [{
- 'url': ':ytrec',
- 'only_matching': True,
- }, {
- 'url': ':ytrecommended',
- 'only_matching': True,
- }, {
- 'url': 'https://youtube.com',
- 'only_matching': True,
- }]
-
-
-class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
- IE_DESC = 'YouTube subscriptions feed; ":ytsubs" keyword (requires cookies)'
- _VALID_URL = r':ytsub(?:scription)?s?'
- _FEED_NAME = 'subscriptions'
- _TESTS = [{
- 'url': ':ytsubs',
- 'only_matching': True,
- }, {
- 'url': ':ytsubscriptions',
- 'only_matching': True,
- }]
-
-
-class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
- IE_DESC = 'Youtube watch history; ":ythis" keyword (requires cookies)'
- _VALID_URL = r':ythis(?:tory)?'
- _FEED_NAME = 'history'
- _TESTS = [{
- 'url': ':ythistory',
- 'only_matching': True,
- }]
-
-
-class YoutubeShortsAudioPivotIE(YoutubeBaseInfoExtractor):
- IE_DESC = 'YouTube Shorts audio pivot (Shorts using audio of a given video)'
- IE_NAME = 'youtube:shorts:pivot:audio'
- _VALID_URL = r'https?://(?:www\.)?youtube\.com/source/(?P<id>[\w-]{11})/shorts'
- _TESTS = [{
- 'url': 'https://www.youtube.com/source/Lyj-MZSAA9o/shorts',
- 'only_matching': True,
- }]
-
- @staticmethod
- def _generate_audio_pivot_params(video_id):
- """
- Generates sfv_audio_pivot browse params for this video id
- """
- pb_params = b'\xf2\x05+\n)\x12\'\n\x0b%b\x12\x0b%b\x1a\x0b%b' % ((video_id.encode(),) * 3)
- return urllib.parse.quote(base64.b64encode(pb_params).decode())
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
- return self.url_result(
- f'https://www.youtube.com/feed/sfv_audio_pivot?bp={self._generate_audio_pivot_params(video_id)}',
- ie=YoutubeTabIE)
-
-
-class YoutubeTruncatedURLIE(YoutubeBaseInfoExtractor):
- IE_NAME = 'youtube:truncated_url'
- IE_DESC = False # Do not list
- _VALID_URL = r'''(?x)
- (?:https?://)?
- (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
- (?:watch\?(?:
- feature=[a-z_]+|
- annotation_id=annotation_[^&]+|
- x-yt-cl=[0-9]+|
- hl=[^&]*|
- t=[0-9]+
- )?
- |
- attribution_link\?a=[^&]+
- )
- $
- '''
-
- _TESTS = [{
- 'url': 'https://www.youtube.com/watch?annotation_id=annotation_3951667041',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/watch?',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/watch?feature=foo',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/watch?hl=en-GB',
- 'only_matching': True,
- }, {
- 'url': 'https://www.youtube.com/watch?t=2372',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- raise ExtractorError(
- 'Did you forget to quote the URL? Remember that & is a meta '
- 'character in most shells, so you want to put the URL in quotes, '
- 'like yt-dlp '
- '"https://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
- ' or simply yt-dlp BaW_jenozKc .',
- expected=True)
-
-
-class YoutubeClipIE(YoutubeTabBaseInfoExtractor):
- IE_NAME = 'youtube:clip'
- _VALID_URL = r'https?://(?:www\.)?youtube\.com/clip/(?P<id>[^/?#]+)'
- _TESTS = [{
- # FIXME: Other metadata should be extracted from the clip, not from the base video
- 'url': 'https://www.youtube.com/clip/UgytZKpehg-hEMBSn3F4AaABCQ',
- 'info_dict': {
- 'id': 'UgytZKpehg-hEMBSn3F4AaABCQ',
- 'ext': 'mp4',
- 'section_start': 29.0,
- 'section_end': 39.7,
- 'duration': 10.7,
- 'age_limit': 0,
- 'availability': 'public',
- 'categories': ['Gaming'],
- 'channel': 'Scott The Woz',
- 'channel_id': 'UC4rqhyiTs7XyuODcECvuiiQ',
- 'channel_url': 'https://www.youtube.com/channel/UC4rqhyiTs7XyuODcECvuiiQ',
- 'description': 'md5:7a4517a17ea9b4bd98996399d8bb36e7',
- 'like_count': int,
- 'playable_in_embed': True,
- 'tags': 'count:17',
- 'thumbnail': 'https://i.ytimg.com/vi_webp/ScPX26pdQik/maxresdefault.webp',
- 'title': 'Mobile Games on Console - Scott The Woz',
- 'upload_date': '20210920',
- 'uploader': 'Scott The Woz',
- 'uploader_id': '@ScottTheWoz',
- 'uploader_url': 'https://www.youtube.com/@ScottTheWoz',
- 'view_count': int,
- 'live_status': 'not_live',
- 'channel_follower_count': int,
- 'chapters': 'count:20',
- 'comment_count': int,
- 'heatmap': 'count:100',
- },
- }]
-
- def _real_extract(self, url):
- clip_id = self._match_id(url)
- _, data = self._extract_webpage(url, clip_id)
-
- video_id = traverse_obj(data, ('currentVideoEndpoint', 'watchEndpoint', 'videoId'))
- if not video_id:
- raise ExtractorError('Unable to find video ID')
-
- clip_data = traverse_obj(data, (
- 'engagementPanels', ..., 'engagementPanelSectionListRenderer', 'content', 'clipSectionRenderer',
- 'contents', ..., 'clipAttributionRenderer', 'onScrubExit', 'commandExecutorCommand', 'commands', ...,
- 'openPopupAction', 'popup', 'notificationActionRenderer', 'actionButton', 'buttonRenderer', 'command',
- 'commandExecutorCommand', 'commands', ..., 'loopCommand'), get_all=False)
-
- return {
- '_type': 'url_transparent',
- 'url': f'https://www.youtube.com/watch?v={video_id}',
- 'ie_key': YoutubeIE.ie_key(),
- 'id': clip_id,
- 'section_start': int(clip_data['startTimeMs']) / 1000,
- 'section_end': int(clip_data['endTimeMs']) / 1000,
- '_format_sort_fields': ( # https protocol is prioritized for ffmpeg compatibility
- 'proto:https', 'quality', 'res', 'fps', 'hdr:12', 'source', 'vcodec', 'channels', 'acodec', 'lang'),
- }
-
-
-class YoutubeConsentRedirectIE(YoutubeBaseInfoExtractor):
- IE_NAME = 'youtube:consent'
- IE_DESC = False # Do not list
- _VALID_URL = r'https?://consent\.youtube\.com/m\?'
- _TESTS = [{
- 'url': 'https://consent.youtube.com/m?continue=https%3A%2F%2Fwww.youtube.com%2Flive%2FqVv6vCqciTM%3Fcbrd%3D1&gl=NL&m=0&pc=yt&hl=en&src=1',
- 'info_dict': {
- 'id': 'qVv6vCqciTM',
- 'ext': 'mp4',
- 'age_limit': 0,
- 'uploader_id': '@sana_natori',
- 'comment_count': int,
- 'chapters': 'count:13',
- 'upload_date': '20221223',
- 'thumbnail': 'https://i.ytimg.com/vi/qVv6vCqciTM/maxresdefault.jpg',
- 'channel_url': 'https://www.youtube.com/channel/UCIdEIHpS0TdkqRkHL5OkLtA',
- 'uploader_url': 'https://www.youtube.com/@sana_natori',
- 'like_count': int,
- 'release_date': '20221223',
- 'tags': ['Vtuber', '月ノ美兎', '名取さな', 'にじさんじ', 'クリスマス', '3D配信'],
- 'title': '【 #インターネット女クリスマス 】3Dで歌ってはしゃぐインターネットの女たち【月ノ美兎/名取さな】',
- 'view_count': int,
- 'playable_in_embed': True,
- 'duration': 4438,
- 'availability': 'public',
- 'channel_follower_count': int,
- 'channel_id': 'UCIdEIHpS0TdkqRkHL5OkLtA',
- 'categories': ['Entertainment'],
- 'live_status': 'was_live',
- 'release_timestamp': 1671793345,
- 'channel': 'さなちゃんねる',
- 'description': 'md5:6aebf95cc4a1d731aebc01ad6cc9806d',
- 'uploader': 'さなちゃんねる',
- 'channel_is_verified': True,
- 'heatmap': 'count:100',
- },
- 'add_ie': ['Youtube'],
- 'params': {'skip_download': 'Youtube'},
- }]
-
- def _real_extract(self, url):
- redirect_url = url_or_none(parse_qs(url).get('continue', [None])[-1])
- if not redirect_url:
- raise ExtractorError('Invalid cookie consent redirect URL', expected=True)
- return self.url_result(redirect_url)
-
-
-class YoutubeTruncatedIDIE(YoutubeBaseInfoExtractor):
- IE_NAME = 'youtube:truncated_id'
- IE_DESC = False # Do not list
- _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
-
- _TESTS = [{
- 'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
- 'only_matching': True,
- }]
-
- def _real_extract(self, url):
- video_id = self._match_id(url)
- raise ExtractorError(
- f'Incomplete YouTube ID {video_id}. URL {url} looks truncated.',
- expected=True)