diff options
| -rw-r--r-- | test/test_utils.py | 39 | ||||
| -rwxr-xr-x | youtube_dl/YoutubeDL.py | 20 | ||||
| -rw-r--r-- | youtube_dl/extractor/daum.py | 17 | ||||
| -rw-r--r-- | youtube_dl/extractor/facebook.py | 13 | ||||
| -rw-r--r-- | youtube_dl/utils.py | 28 | ||||
| -rw-r--r-- | youtube_dl/version.py | 2 |
6 files changed, 109 insertions, 10 deletions
diff --git a/test/test_utils.py b/test/test_utils.py index 8d8997977..3efbed29d 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -40,6 +40,9 @@ from youtube_dl.utils import ( parse_iso8601, strip_jsonp, uppercase_escape, + limit_length, + escape_rfc3986, + escape_url, ) @@ -286,5 +289,41 @@ class TestUtil(unittest.TestCase): self.assertEqual(uppercase_escape('aä'), 'aä') self.assertEqual(uppercase_escape('\\U0001d550'), '𝕐') + def test_limit_length(self): + self.assertEqual(limit_length(None, 12), None) + self.assertEqual(limit_length('foo', 12), 'foo') + self.assertTrue( + limit_length('foo bar baz asd', 12).startswith('foo bar')) + self.assertTrue('...' in limit_length('foo bar baz asd', 12)) + + def test_escape_rfc3986(self): + reserved = "!*'();:@&=+$,/?#[]" + unreserved = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.~' + self.assertEqual(escape_rfc3986(reserved), reserved) + self.assertEqual(escape_rfc3986(unreserved), unreserved) + self.assertEqual(escape_rfc3986('тест'), '%D1%82%D0%B5%D1%81%D1%82') + self.assertEqual(escape_rfc3986('%D1%82%D0%B5%D1%81%D1%82'), '%D1%82%D0%B5%D1%81%D1%82') + self.assertEqual(escape_rfc3986('foo bar'), 'foo%20bar') + self.assertEqual(escape_rfc3986('foo%20bar'), 'foo%20bar') + + def test_escape_url(self): + self.assertEqual( + escape_url('http://wowza.imust.org/srv/vod/telemb/new/UPLOAD/UPLOAD/20224_IncendieHavré_FD.mp4'), + 'http://wowza.imust.org/srv/vod/telemb/new/UPLOAD/UPLOAD/20224_IncendieHavre%CC%81_FD.mp4' + ) + self.assertEqual( + escape_url('http://www.ardmediathek.de/tv/Sturm-der-Liebe/Folge-2036-Zu-Mann-und-Frau-erklärt/Das-Erste/Video?documentId=22673108&bcastId=5290'), + 'http://www.ardmediathek.de/tv/Sturm-der-Liebe/Folge-2036-Zu-Mann-und-Frau-erkl%C3%A4rt/Das-Erste/Video?documentId=22673108&bcastId=5290' + ) + self.assertEqual( + escape_url('http://тест.рф/фрагмент'), + 'http://тест.рф/%D1%84%D1%80%D0%B0%D0%B3%D0%BC%D0%B5%D0%BD%D1%82' + ) + self.assertEqual( + escape_url('http://тест.рф/абв?абв=абв#абв'), + 'http://тест.рф/%D0%B0%D0%B1%D0%B2?%D0%B0%D0%B1%D0%B2=%D0%B0%D0%B1%D0%B2#%D0%B0%D0%B1%D0%B2' + ) + self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0') + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 553bf559b..9519594c9 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -28,6 +28,7 @@ from .utils import ( compat_str, compat_urllib_error, compat_urllib_request, + escape_url, ContentTooShortError, date_from_str, DateRange, @@ -1241,6 +1242,25 @@ class YoutubeDL(object): def urlopen(self, req): """ Start an HTTP download """ + + # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not + # always respected by websites, some tend to give out URLs with non percent-encoded + # non-ASCII characters (see telemb.py, ard.py [#3412]) + # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991) + # To work around aforementioned issue we will replace request's original URL with + # percent-encoded one + url = req if isinstance(req, compat_str) else req.get_full_url() + url_escaped = escape_url(url) + + # Substitute URL if any change after escaping + if url != url_escaped: + if isinstance(req, compat_str): + req = url_escaped + else: + req = compat_urllib_request.Request( + url_escaped, data=req.data, headers=req.headers, + origin_req_host=req.origin_req_host, unverifiable=req.unverifiable) + return self._opener.open(req, timeout=self._socket_timeout) def print_debug_header(self): diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py index 6033cd94a..45d66e2e6 100644 --- a/youtube_dl/extractor/daum.py +++ b/youtube_dl/extractor/daum.py @@ -11,10 +11,10 @@ from ..utils import ( class DaumIE(InfoExtractor): - _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/.*?clipid=(?P<id>\d+)' + _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/(?:v/|.*?clipid=)(?P<id>[^?#&]+)' IE_NAME = 'daum.net' - _TEST = { + _TESTS = [{ 'url': 'http://tvpot.daum.net/clip/ClipView.do?clipid=52554690', 'info_dict': { 'id': '52554690', @@ -24,11 +24,17 @@ class DaumIE(InfoExtractor): 'upload_date': '20130831', 'duration': 3868, }, - } + }, { + 'url': 'http://tvpot.daum.net/v/vab4dyeDBysyBssyukBUjBz', + 'only_matching': True, + }, { + 'url': 'http://tvpot.daum.net/v/07dXWRka62Y%24', + 'only_matching': True, + }] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) - video_id = mobj.group(1) + video_id = mobj.group('id') canonical_url = 'http://tvpot.daum.net/v/%s' % video_id webpage = self._download_webpage(canonical_url, video_id) full_id = self._search_regex( @@ -42,7 +48,6 @@ class DaumIE(InfoExtractor): 'http://videofarm.daum.net/controller/api/open/v1_2/MovieData.apixml?' + query, video_id, 'Downloading video formats info') - self.to_screen(u'%s: Getting video urls' % video_id) formats = [] for format_el in urls.findall('result/output_list/output_list'): profile = format_el.attrib['profile'] @@ -52,7 +57,7 @@ class DaumIE(InfoExtractor): }) url_doc = self._download_xml( 'http://videofarm.daum.net/controller/api/open/v1_2/MovieLocation.apixml?' + format_query, - video_id, note=False) + video_id, note='Downloading video data for %s format' % profile) format_url = url_doc.find('result/url').text formats.append({ 'url': format_url, diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py index d675a939d..60e68d98a 100644 --- a/youtube_dl/extractor/facebook.py +++ b/youtube_dl/extractor/facebook.py @@ -12,8 +12,8 @@ from ..utils import ( compat_urllib_parse, compat_urllib_request, urlencode_postdata, - ExtractorError, + limit_length, ) @@ -38,6 +38,14 @@ class FacebookIE(InfoExtractor): 'title': 'Did you know Kei Nishikori is the first Asian man to ever reach a Grand Slam fin...', } }, { + 'note': 'Video without discernible title', + 'url': 'https://www.facebook.com/video.php?v=274175099429670', + 'info_dict': { + 'id': '274175099429670', + 'ext': 'mp4', + 'title': 'Facebook video #274175099429670', + } + }, { 'url': 'https://www.facebook.com/video.php?v=10204634152394104', 'only_matching': True, }] @@ -131,8 +139,7 @@ class FacebookIE(InfoExtractor): video_title = self._html_search_regex( r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(.*?)</span>', webpage, 'alternative title', default=None) - if len(video_title) > 80 + 3: - video_title = video_title[:80] + '...' + video_title = limit_length(video_title, 80) if not video_title: video_title = 'Facebook video #%s' % video_id diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 247788078..b644f4e92 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1437,6 +1437,24 @@ def uppercase_escape(s): lambda m: unicode_escape(m.group(0))[0], s) + +def escape_rfc3986(s): + """Escape non-ASCII characters as suggested by RFC 3986""" + if sys.version_info < (3, 0) and isinstance(s, unicode): + s = s.encode('utf-8') + return compat_urllib_parse.quote(s, "%/;:@&=+$,!~*'()?#[]") + + +def escape_url(url): + """Escape URL as suggested by RFC 3986""" + url_parsed = compat_urllib_parse_urlparse(url) + return url_parsed._replace( + path=escape_rfc3986(url_parsed.path), + params=escape_rfc3986(url_parsed.params), + query=escape_rfc3986(url_parsed.query), + fragment=escape_rfc3986(url_parsed.fragment) + ).geturl() + try: struct.pack(u'!I', 0) except TypeError: @@ -1571,3 +1589,13 @@ except AttributeError: if ret: raise subprocess.CalledProcessError(ret, p.args, output=output) return output + + +def limit_length(s, length): + """ Add ellipses to overly long strings """ + if s is None: + return None + ELLIPSES = '...' + if len(s) > length: + return s[:length - len(ELLIPSES)] + ELLIPSES + return s diff --git a/youtube_dl/version.py b/youtube_dl/version.py index f32e792ad..cf0d862da 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,2 +1,2 @@ -__version__ = '2014.09.14.3' +__version__ = '2014.09.15.1' |