aboutsummaryrefslogtreecommitdiff
path: root/youtube_dl
diff options
context:
space:
mode:
authorFilippo Valsorda <filippo.valsorda@gmail.com>2012-12-11 16:45:46 +0100
committerFilippo Valsorda <filippo.valsorda@gmail.com>2012-12-11 16:58:36 +0100
commit9789a05c20d5f29cb93c371eaaa1c2a291297fb5 (patch)
treed01286187768aed5bc9ce2972ba6348b91922035 /youtube_dl
parentd050de77f94a5dc9071c0b459ba62cb09c6696c4 (diff)
fix playlist pagination and add YT playlist tests (closes #569)
Diffstat (limited to 'youtube_dl')
-rw-r--r--youtube_dl/InfoExtractors.py21
1 files changed, 15 insertions, 6 deletions
diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py
index 9cfff153b..5ebe64b6e 100644
--- a/youtube_dl/InfoExtractors.py
+++ b/youtube_dl/InfoExtractors.py
@@ -1674,7 +1674,7 @@ class YoutubePlaylistIE(InfoExtractor):
_VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*'
_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
_VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&amp;([^&"]+&amp;)*list=.*?%s'
- _MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
+ _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}"
IE_NAME = u'youtube:playlist'
def __init__(self, downloader=None):
@@ -1713,7 +1713,7 @@ class YoutubePlaylistIE(InfoExtractor):
url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
request = compat_urllib_request.Request(url)
try:
- page = compat_urllib_request.urlopen(request).read()
+ page = compat_urllib_request.urlopen(request).read().decode('utf8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return
@@ -1725,10 +1725,12 @@ class YoutubePlaylistIE(InfoExtractor):
ids_in_page.append(mobj.group(1))
video_ids.extend(ids_in_page)
- if re.search(self._MORE_PAGES_INDICATOR, page) is None:
+ if self._MORE_PAGES_INDICATOR not in page:
break
pagenum = pagenum + 1
+ total = len(video_ids)
+
playliststart = self._downloader.params.get('playliststart', 1) - 1
playlistend = self._downloader.params.get('playlistend', -1)
if playlistend == -1:
@@ -1736,6 +1738,11 @@ class YoutubePlaylistIE(InfoExtractor):
else:
video_ids = video_ids[playliststart:playlistend]
+ if len(video_ids) == total:
+ self._downloader.to_screen(u'[youtube] PL %s: Found %i videos' % (playlist_id, total))
+ else:
+ self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(video_ids)))
+
for id in video_ids:
self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
return
@@ -1746,7 +1753,7 @@ class YoutubeChannelIE(InfoExtractor):
_VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$"
_TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
- _MORE_PAGES_INDICATOR = r'yt-uix-button-content">Next' # TODO
+ _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}"
IE_NAME = u'youtube:channel'
def report_download_page(self, channel_id, pagenum):
@@ -1770,7 +1777,7 @@ class YoutubeChannelIE(InfoExtractor):
url = self._TEMPLATE_URL % (channel_id, pagenum)
request = compat_urllib_request.Request(url)
try:
- page = compat_urllib_request.urlopen(request).read()
+ page = compat_urllib_request.urlopen(request).read().decode('utf8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
return
@@ -1782,10 +1789,12 @@ class YoutubeChannelIE(InfoExtractor):
ids_in_page.append(mobj.group(1))
video_ids.extend(ids_in_page)
- if re.search(self._MORE_PAGES_INDICATOR, page) is None:
+ if self._MORE_PAGES_INDICATOR not in page:
break
pagenum = pagenum + 1
+ self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
+
for id in video_ids:
self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
return