From 597cc8a45536aa4207c5ffc3e421fcebf2e08fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 5 Mar 2013 11:58:01 +0100 Subject: Use extract_info in YoutubePlaylist and YoutubeSearch --- youtube_dl/InfoExtractors.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'youtube_dl/InfoExtractors.py') diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 7ce84fe79..8a7694a76 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1756,9 +1756,7 @@ class YoutubePlaylistIE(InfoExtractor): else: self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(videos))) - for video in videos: - self._downloader.download([video]) - return + return self._downloader.extract_info_iterable(videos) class YoutubeChannelIE(InfoExtractor): @@ -1892,8 +1890,8 @@ class YoutubeUserIE(InfoExtractor): self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" % (username, all_ids_count, len(video_ids))) - for video_id in video_ids: - self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id]) + urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids] + return self._downloader.extract_info_iterable(urls) class BlipTVUserIE(InfoExtractor): -- cgit v1.2.3 From f6e6da9525150487476d4990693eedf73acffab1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 5 Mar 2013 12:26:18 +0100 Subject: Use extract_info in BlipTV User and Youtube Channel --- youtube_dl/InfoExtractors.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'youtube_dl/InfoExtractors.py') diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 8a7694a76..d79f6068f 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1806,9 +1806,8 @@ class YoutubeChannelIE(InfoExtractor): self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids))) - for id in video_ids: - self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) - return + urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids] + return self._downloader.extract_info_iterable(urls) class YoutubeUserIE(InfoExtractor): @@ -1981,8 +1980,8 @@ class BlipTVUserIE(InfoExtractor): self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" % (self.IE_NAME, username, all_ids_count, len(video_ids))) - for video_id in video_ids: - self._downloader.download([u'http://blip.tv/'+video_id]) + urls = [u'http://blip.tv/%s' % video_id for video_id in video_ids] + return self._downloader.extract_info_iterable(urls) class DepositFilesIE(InfoExtractor): -- cgit v1.2.3 From 8a38a194fb08a253986cdbafa02cf699ef76c9a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 5 Mar 2013 20:55:48 +0100 Subject: Add auxiliary methods to InfoExtractor to set the '_type' key and use them for some playlist IEs --- youtube_dl/InfoExtractors.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) (limited to 'youtube_dl/InfoExtractors.py') diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index d79f6068f..895658f49 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -128,6 +128,24 @@ class InfoExtractor(object): urlh = self._request_webpage(url_or_request, video_id, note, errnote) webpage_bytes = urlh.read() return webpage_bytes.decode('utf-8', 'replace') + + #Methods for following #608 + #They set the correct value of the '_type' key + def video_result(self, video_info): + """Returns a video""" + video_info['_type'] = 'video' + return video_info + def url_result(self, url, ie=None): + """Returns a url that points to a page that should be processed""" + #TODO: ie should be the class used for getting the info + video_info = {'_type': 'url', + 'url': url} + return video_info + def playlist_result(self, entries): + """Returns a playlist""" + video_info = {'_type': 'playlist', + 'entries': entries} + return video_info class YoutubeIE(InfoExtractor): @@ -1756,7 +1774,8 @@ class YoutubePlaylistIE(InfoExtractor): else: self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(videos))) - return self._downloader.extract_info_iterable(videos) + url_results = [self.url_result(url) for url in videos] + return [self.playlist_result(url_results)] class YoutubeChannelIE(InfoExtractor): @@ -1807,7 +1826,8 @@ class YoutubeChannelIE(InfoExtractor): self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids))) urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids] - return self._downloader.extract_info_iterable(urls) + url_entries = [self.url_result(url) for url in urls] + return [self.playlist_result(url_entries)] class YoutubeUserIE(InfoExtractor): @@ -1890,7 +1910,8 @@ class YoutubeUserIE(InfoExtractor): (username, all_ids_count, len(video_ids))) urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids] - return self._downloader.extract_info_iterable(urls) + url_results = [self.url_result(url) for url in urls] + return [self.playlist_result(url_results)] class BlipTVUserIE(InfoExtractor): @@ -1981,7 +2002,8 @@ class BlipTVUserIE(InfoExtractor): (self.IE_NAME, username, all_ids_count, len(video_ids))) urls = [u'http://blip.tv/%s' % video_id for video_id in video_ids] - return self._downloader.extract_info_iterable(urls) + url_entries = [self.url_result(url) for url in urls] + return [self.playlist_result(url_entries)] class DepositFilesIE(InfoExtractor): -- cgit v1.2.3 From a0d6fe7b924697c089ed7ae37df0ca590ac38a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Tue, 5 Mar 2013 22:33:32 +0100 Subject: When a redirect is found return the new url using the new style --- youtube_dl/InfoExtractors.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'youtube_dl/InfoExtractors.py') diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 895658f49..e714fa6b0 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1311,7 +1311,7 @@ class GenericIE(InfoExtractor): self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url) def _test_redirect(self, url): - """Check if it is a redirect, like url shorteners, in case restart chain.""" + """Check if it is a redirect, like url shorteners, in case return the new url.""" class HeadRequest(compat_urllib_request.Request): def get_method(self): return "HEAD" @@ -1362,11 +1362,11 @@ class GenericIE(InfoExtractor): return False self.report_following_redirect(new_url) - self._downloader.download([new_url]) - return True + return new_url def _real_extract(self, url): - if self._test_redirect(url): return + new_url = self._test_redirect(url) + if new_url: return [self.url_result(new_url)] video_id = url.split('/')[-1] request = compat_urllib_request.Request(url) -- cgit v1.2.3 From d2c690828a8297c014d8053fbdee4e26fe11586a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Thu, 28 Mar 2013 13:39:00 +0100 Subject: Add title and id to playlist results Not all IE give both. They are not used yet. --- youtube_dl/InfoExtractors.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'youtube_dl/InfoExtractors.py') diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index dd4a776e4..6053d14ec 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -147,10 +147,14 @@ class InfoExtractor(object): video_info = {'_type': 'url', 'url': url} return video_info - def playlist_result(self, entries): + def playlist_result(self, entries, playlist_id=None, playlist_title=None): """Returns a playlist""" video_info = {'_type': 'playlist', 'entries': entries} + if playlist_id: + video_info['id'] = playlist_id + if playlist_title: + video_info['title'] = playlist_title return video_info @@ -1808,7 +1812,7 @@ class YoutubePlaylistIE(InfoExtractor): self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(videos))) url_results = [self.url_result(url) for url in videos] - return [self.playlist_result(url_results)] + return [self.playlist_result(url_results, playlist_id)] class YoutubeChannelIE(InfoExtractor): @@ -1860,7 +1864,7 @@ class YoutubeChannelIE(InfoExtractor): urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids] url_entries = [self.url_result(url) for url in urls] - return [self.playlist_result(url_entries)] + return [self.playlist_result(url_entries, channel_id)] class YoutubeUserIE(InfoExtractor): @@ -1944,7 +1948,7 @@ class YoutubeUserIE(InfoExtractor): urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids] url_results = [self.url_result(url) for url in urls] - return [self.playlist_result(url_results)] + return [self.playlist_result(url_results, playlist_title = username)] class BlipTVUserIE(InfoExtractor): @@ -2036,7 +2040,7 @@ class BlipTVUserIE(InfoExtractor): urls = [u'http://blip.tv/%s' % video_id for video_id in video_ids] url_entries = [self.url_result(url) for url in urls] - return [self.playlist_result(url_entries)] + return [self.playlist_result(url_entries, playlist_title = username)] class DepositFilesIE(InfoExtractor): -- cgit v1.2.3 From 532d797824a1ec48480f1d10075e66a90aa53449 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 10 Apr 2013 00:06:03 +0200 Subject: In MetacafeIE return a url if YoutubeIE should do the job --- youtube_dl/InfoExtractors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'youtube_dl/InfoExtractors.py') diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 81eaddc72..b7371365a 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -723,8 +723,7 @@ class MetacafeIE(InfoExtractor): # Check if video comes from YouTube mobj2 = re.match(r'^yt-(.*)$', video_id) if mobj2 is not None: - self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)]) - return + return [self.url_result('http://www.youtube.com/watch?v=%s' % mobj2.group(1))] # Retrieve video webpage to extract further information request = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id) -- cgit v1.2.3 From bce878a7c1678ac698ecd556b2c77a1e2f2306df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 10 Apr 2013 14:32:03 +0200 Subject: Implement the playlist/start options in FileDownloader It makes it available for all the InfoExtractors --- youtube_dl/InfoExtractors.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) (limited to 'youtube_dl/InfoExtractors.py') diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index b7371365a..a7fdf1607 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1806,19 +1806,6 @@ class YoutubePlaylistIE(InfoExtractor): page_num += 1 videos = [v[1] for v in sorted(videos)] - total = len(videos) - - playliststart = self._downloader.params.get('playliststart', 1) - 1 - playlistend = self._downloader.params.get('playlistend', -1) - if playlistend == -1: - videos = videos[playliststart:] - else: - videos = videos[playliststart:playlistend] - - if len(videos) == total: - self._downloader.to_screen(u'[youtube] PL %s: Found %i videos' % (playlist_id, total)) - else: - self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(videos))) url_results = [self.url_result(url) for url in videos] return [self.playlist_result(url_results, playlist_id)] @@ -1943,18 +1930,6 @@ class YoutubeUserIE(InfoExtractor): pagenum += 1 - all_ids_count = len(video_ids) - playliststart = self._downloader.params.get('playliststart', 1) - 1 - playlistend = self._downloader.params.get('playlistend', -1) - - if playlistend == -1: - video_ids = video_ids[playliststart:] - else: - video_ids = video_ids[playliststart:playlistend] - - self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" % - (username, all_ids_count, len(video_ids))) - urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids] url_results = [self.url_result(url) for url in urls] return [self.playlist_result(url_results, playlist_title = username)] @@ -2035,15 +2010,6 @@ class BlipTVUserIE(InfoExtractor): pagenum += 1 - all_ids_count = len(video_ids) - playliststart = self._downloader.params.get('playliststart', 1) - 1 - playlistend = self._downloader.params.get('playlistend', -1) - - if playlistend == -1: - video_ids = video_ids[playliststart:] - else: - video_ids = video_ids[playliststart:playlistend] - self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" % (self.IE_NAME, username, all_ids_count, len(video_ids))) -- cgit v1.2.3