diff options
| -rw-r--r-- | Makefile | 8 | ||||
| -rw-r--r-- | test/helper.py | 5 | ||||
| -rw-r--r-- | test/test_download.py | 32 | ||||
| -rw-r--r-- | test/test_playlists.py | 11 | ||||
| -rwxr-xr-x | youtube_dl/YoutubeDL.py | 8 | ||||
| -rw-r--r-- | youtube_dl/__init__.py | 1 | ||||
| -rw-r--r-- | youtube_dl/extractor/__init__.py | 4 | ||||
| -rw-r--r-- | youtube_dl/extractor/ellentv.py | 79 |
8 files changed, 136 insertions, 12 deletions
@@ -6,10 +6,10 @@ clean: cleanall: clean rm -f youtube-dl youtube-dl.exe -PREFIX=/usr/local -BINDIR=$(PREFIX)/bin -MANDIR=$(PREFIX)/man -PYTHON=/usr/bin/env python +PREFIX ?= /usr/local +BINDIR ?= $(PREFIX)/bin +MANDIR ?= $(PREFIX)/man +PYTHON ?= /usr/bin/env python # set SYSCONFDIR to /etc if PREFIX=/usr or PREFIX=/usr/local ifeq ($(PREFIX),/usr) diff --git a/test/helper.py b/test/helper.py index b7299fb82..22d763860 100644 --- a/test/helper.py +++ b/test/helper.py @@ -117,8 +117,9 @@ def expect_info_dict(self, expected_dict, got_dict): u'invalid value for field %s, expected %r, got %r' % (info_field, expected, got)) # Check for the presence of mandatory fields - for key in ('id', 'url', 'title', 'ext'): - self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key) + if got_dict.get('_type') != 'playlist': + for key in ('id', 'url', 'title', 'ext'): + self.assertTrue(got_dict.get(key), 'Missing mandatory field %s' % key) # Check for mandatory fields that are automatically set by YoutubeDL for key in ['webpage_url', 'extractor', 'extractor_key']: self.assertTrue(got_dict.get(key), u'Missing field: %s' % key) diff --git a/test/test_download.py b/test/test_download.py index d6540588c..c8d4ec2c8 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -63,15 +63,21 @@ def generator(test_case): def test_template(self): ie = youtube_dl.extractor.get_info_extractor(test_case['name']) other_ies = [get_info_extractor(ie_key) for ie_key in test_case.get('add_ie', [])] + is_playlist = any(k.startswith('playlist') for k in test_case) + test_cases = test_case.get( + 'playlist', [] if is_playlist else [test_case]) + def print_skipping(reason): print('Skipping %s: %s' % (test_case['name'], reason)) if not ie.working(): print_skipping('IE marked as not _WORKING') return - if 'playlist' not in test_case: - info_dict = test_case.get('info_dict', {}) - if not test_case.get('file') and not (info_dict.get('id') and info_dict.get('ext')): + + for tc in test_cases: + info_dict = tc.get('info_dict', {}) + if not tc.get('file') and not (info_dict.get('id') and info_dict.get('ext')): raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?') + if 'skip' in test_case: print_skipping(test_case['skip']) return @@ -81,6 +87,9 @@ def generator(test_case): return params = get_params(test_case.get('params', {})) + if is_playlist and 'playlist' not in test_case: + params.setdefault('extract_flat', True) + params.setdefault('skip_download', True) ydl = YoutubeDL(params) ydl.add_default_info_extractors() @@ -93,7 +102,6 @@ def generator(test_case): def get_tc_filename(tc): return tc.get('file') or ydl.prepare_filename(tc.get('info_dict', {})) - test_cases = test_case.get('playlist', [test_case]) def try_rm_tcs_files(): for tc in test_cases: tc_filename = get_tc_filename(tc) @@ -105,7 +113,10 @@ def generator(test_case): try_num = 1 while True: try: - ydl.download([test_case['url']]) + # We're not using .download here sine that is just a shim + # for outside error handling, and returns the exit code + # instead of the result dict. + res_dict = ydl.extract_info(test_case['url']) except (DownloadError, ExtractorError) as err: # Check if the exception is not a network related one if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503): @@ -121,6 +132,17 @@ def generator(test_case): else: break + if is_playlist: + self.assertEqual(res_dict['_type'], 'playlist') + expect_info_dict(self, test_case.get('info_dict', {}), res_dict) + if 'playlist_mincount' in test_case: + self.assertGreaterEqual( + len(res_dict['entries']), + test_case['playlist_mincount'], + 'Expected at least %d in playlist %s, but got only %d' % ( + test_case['playlist_mincount'], test_case['url'], + len(res_dict['entries']))) + for tc in test_cases: tc_filename = get_tc_filename(tc) if not test_case.get('params', {}).get('skip_download', False): diff --git a/test/test_playlists.py b/test/test_playlists.py index 4f188345b..6448fea38 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -1,6 +1,17 @@ #!/usr/bin/env python # encoding: utf-8 +## DEPRECATED FILE! +# Add new tests to the extractors themselves, like this: +# _TEST = { +# 'url': 'http://example.com/playlist/42', +# 'playlist_mincount': 99, +# 'info_dict': { +# 'id': '42', +# 'title': 'Playlist number forty-two', +# } +# } + from __future__ import unicode_literals # Allow direct execution diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 14a1d06ab..e7194f3e3 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -162,6 +162,7 @@ class YoutubeDL(object): default_search: Prepend this string if an input url is not valid. 'auto' for elaborate guessing encoding: Use this encoding instead of the system-specified. + extract_flat: Do not resolve URLs, return the immediate result. The following parameters are not used by YoutubeDL itself, they are used by the FileDownloader: @@ -558,7 +559,12 @@ class YoutubeDL(object): Returns the resolved ie_result. """ - result_type = ie_result.get('_type', 'video') # If not given we suppose it's a video, support the default old system + result_type = ie_result.get('_type', 'video') + + if self.params.get('extract_flat', False): + if result_type in ('url', 'url_transparent'): + return ie_result + if result_type == 'video': self.add_extra_info(ie_result, extra_info) return self.process_video_result(ie_result, download=download) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 962aedbff..5d2137fe5 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -69,6 +69,7 @@ __authors__ = ( 'Dobrosław Żybort', 'David Fabijan', 'Sebastian Haas', + 'Alexander Kirk', ) __license__ = 'Public Domain' diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 632e832c7..be7616edc 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -77,6 +77,10 @@ from .ebaumsworld import EbaumsWorldIE from .ehow import EHowIE from .eighttracks import EightTracksIE from .eitb import EitbIE +from .ellentv import ( + EllenTVIE, + EllenTVClipsIE, +) from .elpais import ElPaisIE from .empflix import EmpflixIE from .engadget import EngadgetIE diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py new file mode 100644 index 000000000..3e7923648 --- /dev/null +++ b/youtube_dl/extractor/ellentv.py @@ -0,0 +1,79 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re +import json + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + parse_iso8601, +) + + +class EllenTVIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?ellentv\.com/videos/(?P<id>[a-z0-9_-]+)' + _TEST = { + 'url': 'http://www.ellentv.com/videos/0-7jqrsr18/', + 'md5': 'e4af06f3bf0d5f471921a18db5764642', + 'info_dict': { + 'id': '0-7jqrsr18', + 'ext': 'mp4', + 'title': 'What\'s Wrong with These Photos? A Whole Lot', + 'timestamp': 1406876400, + 'upload_date': '20140801', + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + timestamp = parse_iso8601(self._search_regex( + r'<span class="publish-date"><time datetime="([^"]+)">', + webpage, 'timestamp')) + + return { + 'id': video_id, + 'title': self._og_search_title(webpage), + 'url': self._html_search_meta('VideoURL', webpage, 'url'), + 'timestamp': timestamp, + } + + +class EllenTVClipsIE(InfoExtractor): + IE_NAME = 'EllenTV:clips' + _VALID_URL = r'https?://(?:www\.)?ellentv\.com/episodes/(?P<id>[a-z0-9_-]+)' + _TEST = { + 'url': 'http://www.ellentv.com/episodes/meryl-streep-vanessa-hudgens/', + 'info_dict': { + 'id': 'meryl-streep-vanessa-hudgens', + 'title': 'Meryl Streep, Vanessa Hudgens', + }, + 'playlist_mincount': 9, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + playlist_id = mobj.group('id') + + webpage = self._download_webpage(url, playlist_id) + playlist = self._extract_playlist(webpage) + + return { + '_type': 'playlist', + 'id': playlist_id, + 'title': self._og_search_title(webpage), + 'entries': self._extract_entries(playlist) + } + + def _extract_playlist(self, webpage): + json_string = self._search_regex(r'playerView.addClips\(\[\{(.*?)\}\]\);', webpage, 'json') + try: + return json.loads("[{" + json_string + "}]") + except ValueError as ve: + raise ExtractorError('Failed to download JSON', cause=ve) + + def _extract_entries(self, playlist): + return [self.url_result(item['url'], 'EllenTV') for item in playlist] |