diff options
Diffstat (limited to 'youtube_dl')
| -rw-r--r-- | youtube_dl/FileDownloader.py | 123 | ||||
| -rwxr-xr-x | youtube_dl/InfoExtractors.py | 51 |
2 files changed, 118 insertions, 56 deletions
diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py index 7c5a52be1..2237d355d 100644 --- a/youtube_dl/FileDownloader.py +++ b/youtube_dl/FileDownloader.py @@ -418,10 +418,85 @@ class FileDownloader(object): if re.search(rejecttitle, title, re.IGNORECASE): return u'"' + title + '" title matched reject pattern "' + rejecttitle + '"' return None + + def extract_info(self, url): + ''' + Returns a list with a dictionary for each video we find. + ''' + suitable_found = False + for ie in self._ies: + # Go to next InfoExtractor if not suitable + if not ie.suitable(url): + continue + + # Warn if the _WORKING attribute is False + if not ie.working(): + self.to_stderr(u'WARNING: the program functionality for this site has been marked as broken, ' + u'and will probably not work. If you want to go on, use the -i option.') + + # Suitable InfoExtractor found + suitable_found = True + + # Extract information from URL and process it + try: + ie_results = ie.extract(url) + results = self.process_ie_results(ie_results, ie) + return results + except ExtractorError as de: # An error we somewhat expected + self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback()) + break + except Exception as e: + if self.params.get('ignoreerrors', False): + self.trouble(u'ERROR: ' + compat_str(e), tb=compat_str(traceback.format_exc())) + break + else: + raise + if not suitable_found: + self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) + def extract_info_iterable(self, urls): + ''' + Return the videos founded for the urls + ''' + results = [] + for url in urls: + results.extend(self.extract_info(url)) + return results + + def process_ie_results(self, ie_results, ie): + """ + Take the results of the ie and return a list of videos. + For url elements it will seartch the suitable ie and get the videos + For playlist elements it will process each of the elements of the 'entries' key + """ + results = [] + for result in ie_results or []: + result_type = result.get('_type', 'video') #If not given we suppose it's a video, support the dafault old system + if result_type == 'video': + if not 'extractor' in result: + #The extractor has already been set somewhere else + result['extractor'] = ie.IE_NAME + results.append(result) + elif result_type == 'url': + #We get the videos pointed by the url + results.extend(self.extract_info(result['url'])) + elif result_type == 'playlist': + #We process each entry in the playlist + entries_result = self.process_ie_results(result['entries'], ie) + result['entries'] = entries_result + results.extend([result]) + return results def process_info(self, info_dict): """Process a single dictionary returned by an InfoExtractor.""" + if info_dict.get('_type','video') == 'playlist': + playlist = info_dict.get('title', None) or info_dict.get('id', None) + self.to_screen(u'[download] Downloading playlist: %s' % playlist) + for video in info_dict['entries']: + video['playlist'] = playlist + self.process_info(video) + return + # Keep for backwards compatibility info_dict['stitle'] = info_dict['title'] @@ -556,53 +631,17 @@ class FileDownloader(object): raise SameFileError(self.params['outtmpl']) for url in url_list: - suitable_found = False - for ie in self._ies: - # Go to next InfoExtractor if not suitable - if not ie.suitable(url): - continue + videos = self.extract_info(url) - # Warn if the _WORKING attribute is False - if not ie.working(): - self.report_warning(u'the program functionality for this site has been marked as broken, ' - u'and will probably not work. If you want to go on, use the -i option.') - - # Suitable InfoExtractor found - suitable_found = True - - # Extract information from URL and process it + for video in videos or []: try: - videos = ie.extract(url) - except ExtractorError as de: # An error we somewhat expected - self.trouble(u'ERROR: ' + compat_str(de), de.format_traceback()) - break + self.increment_downloads() + self.process_info(video) + except UnavailableVideoError: + self.trouble(u'\nERROR: unable to download video') except MaxDownloadsReached: self.to_screen(u'[info] Maximum number of downloaded files reached.') raise - except Exception as e: - if self.params.get('ignoreerrors', False): - self.report_error(u'' + compat_str(e), tb=compat_str(traceback.format_exc())) - break - else: - raise - - if len(videos or []) > 1 and self.fixed_template(): - raise SameFileError(self.params['outtmpl']) - - for video in videos or []: - video['extractor'] = ie.IE_NAME - try: - self.increment_downloads() - self.process_info(video) - except UnavailableVideoError: - self.to_stderr(u"\n") - self.report_error(u'unable to download video') - - # Suitable InfoExtractor had been found; go to next URL - break - - if not suitable_found: - self.report_error(u'no suitable InfoExtractor: %s' % url) return self._download_retcode diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index aa8074a9e..81eaddc72 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -134,6 +134,28 @@ class InfoExtractor(object): encoding = 'utf-8' webpage_bytes = urlh.read() return webpage_bytes.decode(encoding, 'replace') + + #Methods for following #608 + #They set the correct value of the '_type' key + def video_result(self, video_info): + """Returns a video""" + video_info['_type'] = 'video' + return video_info + def url_result(self, url, ie=None): + """Returns a url that points to a page that should be processed""" + #TODO: ie should be the class used for getting the info + video_info = {'_type': 'url', + 'url': url} + return video_info + def playlist_result(self, entries, playlist_id=None, playlist_title=None): + """Returns a playlist""" + video_info = {'_type': 'playlist', + 'entries': entries} + if playlist_id: + video_info['id'] = playlist_id + if playlist_title: + video_info['title'] = playlist_title + return video_info class YoutubeIE(InfoExtractor): @@ -1343,7 +1365,7 @@ class GenericIE(InfoExtractor): self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url) def _test_redirect(self, url): - """Check if it is a redirect, like url shorteners, in case restart chain.""" + """Check if it is a redirect, like url shorteners, in case return the new url.""" class HeadRequest(compat_urllib_request.Request): def get_method(self): return "HEAD" @@ -1394,11 +1416,11 @@ class GenericIE(InfoExtractor): return False self.report_following_redirect(new_url) - self._downloader.download([new_url]) - return True + return new_url def _real_extract(self, url): - if self._test_redirect(url): return + new_url = self._test_redirect(url) + if new_url: return [self.url_result(new_url)] video_id = url.split('/')[-1] try: @@ -1799,9 +1821,8 @@ class YoutubePlaylistIE(InfoExtractor): else: self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(videos))) - for video in videos: - self._downloader.download([video]) - return + url_results = [self.url_result(url) for url in videos] + return [self.playlist_result(url_results, playlist_id)] class YoutubeChannelIE(InfoExtractor): @@ -1851,9 +1872,9 @@ class YoutubeChannelIE(InfoExtractor): self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids))) - for id in video_ids: - self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) - return + urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids] + url_entries = [self.url_result(url) for url in urls] + return [self.playlist_result(url_entries, channel_id)] class YoutubeUserIE(InfoExtractor): @@ -1935,8 +1956,9 @@ class YoutubeUserIE(InfoExtractor): self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" % (username, all_ids_count, len(video_ids))) - for video_id in video_ids: - self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id]) + urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids] + url_results = [self.url_result(url) for url in urls] + return [self.playlist_result(url_results, playlist_title = username)] class BlipTVUserIE(InfoExtractor): @@ -2026,8 +2048,9 @@ class BlipTVUserIE(InfoExtractor): self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" % (self.IE_NAME, username, all_ids_count, len(video_ids))) - for video_id in video_ids: - self._downloader.download([u'http://blip.tv/'+video_id]) + urls = [u'http://blip.tv/%s' % video_id for video_id in video_ids] + url_entries = [self.url_result(url) for url in urls] + return [self.playlist_result(url_entries, playlist_title = username)] class DepositFilesIE(InfoExtractor): |