diff options
| -rw-r--r-- | test/test_http.py | 72 | ||||
| -rw-r--r-- | test/testcert.pem | 52 | ||||
| -rwxr-xr-x | youtube_dl/YoutubeDL.py | 2 | ||||
| -rw-r--r-- | youtube_dl/__init__.py | 4 | ||||
| -rw-r--r-- | youtube_dl/compat.py | 6 | ||||
| -rw-r--r-- | youtube_dl/extractor/__init__.py | 2 | ||||
| -rw-r--r-- | youtube_dl/extractor/ctsnews.py | 94 | ||||
| -rw-r--r-- | youtube_dl/extractor/srmediathek.py | 2 | ||||
| -rw-r--r-- | youtube_dl/extractor/viddler.py | 2 | ||||
| -rw-r--r-- | youtube_dl/extractor/xuite.py | 142 | ||||
| -rw-r--r-- | youtube_dl/utils.py | 7 |
11 files changed, 380 insertions, 5 deletions
diff --git a/test/test_http.py b/test/test_http.py new file mode 100644 index 000000000..bd4d46fef --- /dev/null +++ b/test/test_http.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +# Allow direct execution +import os +import sys +import unittest +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from youtube_dl import YoutubeDL +from youtube_dl.compat import compat_http_server +import ssl +import threading + +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass + + def do_GET(self): + if self.path == '/video.html': + self.send_response(200) + self.send_header('Content-Type', 'text/html; charset=utf-8') + self.end_headers() + self.wfile.write(b'<html><video src="/vid.mp4" /></html>') + elif self.path == '/vid.mp4': + self.send_response(200) + self.send_header('Content-Type', 'video/mp4') + self.end_headers() + self.wfile.write(b'\x00\x00\x00\x00\x20\x66\x74[video]') + else: + assert False + + +class FakeLogger(object): + def debug(self, msg): + pass + + def warning(self, msg): + pass + + def error(self, msg): + pass + + +class TestHTTP(unittest.TestCase): + def setUp(self): + certfn = os.path.join(TEST_DIR, 'testcert.pem') + self.httpd = compat_http_server.HTTPServer( + ('localhost', 0), HTTPTestRequestHandler) + self.httpd.socket = ssl.wrap_socket( + self.httpd.socket, certfile=certfn, server_side=True) + self.port = self.httpd.socket.getsockname()[1] + self.server_thread = threading.Thread(target=self.httpd.serve_forever) + self.server_thread.daemon = True + self.server_thread.start() + + def test_nocheckcertificate(self): + if sys.version_info >= (2, 7, 9): # No certificate checking anyways + ydl = YoutubeDL({'logger': FakeLogger()}) + self.assertRaises( + Exception, + ydl.extract_info, 'https://localhost:%d/video.html' % self.port) + + ydl = YoutubeDL({'logger': FakeLogger(), 'nocheckcertificate': True}) + r = ydl.extract_info('https://localhost:%d/video.html' % self.port) + self.assertEqual(r['url'], 'https://localhost:%d/vid.mp4' % self.port) + +if __name__ == '__main__': + unittest.main() diff --git a/test/testcert.pem b/test/testcert.pem new file mode 100644 index 000000000..b3e0f00c7 --- /dev/null +++ b/test/testcert.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDMF0bAzaHAdIyB +HRmnIp4vv40lGqEePmWqicCl0QZ0wsb5dNysSxSa7330M2QeQopGfdaUYF1uTcNp +Qx6ECgBSfg+RrOBI7r/u4F+sKX8MUXVaf/5QoBUrGNGSn/pp7HMGOuQqO6BVg4+h +A1ySSwUG8mZItLRry1ISyErmW8b9xlqfd97uLME/5tX+sMelRFjUbAx8A4CK58Ev +mMguHVTlXzx5RMdYcf1VScYcjlV/qA45uzP8zwI5aigfcmUD+tbGuQRhKxUhmw0J +aobtOR6+JSOAULW5gYa/egE4dWLwbyM6b6eFbdnjlQzEA1EW7ChMPAW/Mo83KyiP +tKMCSQulAgMBAAECggEALCfBDAexPjU5DNoh6bIorUXxIJzxTNzNHCdvgbCGiA54 +BBKPh8s6qwazpnjT6WQWDIg/O5zZufqjE4wM9x4+0Zoqfib742ucJO9wY4way6x4 +Clt0xzbLPabB+MoZ4H7ip+9n2+dImhe7pGdYyOHoNYeOL57BBi1YFW42Hj6u/8pd +63YCXisto3Rz1YvRQVjwsrS+cRKZlzAFQRviL30jav7Wh1aWEfcXxjj4zhm8pJdk +ITGtq6howz57M0NtX6hZnfe8ywzTnDFIGKIMA2cYHuYJcBh9bc4tCGubTvTKK9UE +8fM+f6UbfGqfpKCq1mcgs0XMoFDSzKS9+mSJn0+5JQKBgQD+OCKaeH3Yzw5zGnlw +XuQfMJGNcgNr+ImjmvzUAC2fAZUJLAcQueE5kzMv5Fmd+EFE2CEX1Vit3tg0SXvA +G+bq609doILHMA03JHnV1npO/YNIhG3AAtJlKYGxQNfWH9mflYj9mEui8ZFxG52o +zWhHYuifOjjZszUR+/eio6NPzwKBgQDNhUBTrT8LIX4SE/EFUiTlYmWIvOMgXYvN +8Cm3IRNQ/yyphZaXEU0eJzfX5uCDfSVOgd6YM/2pRah+t+1Hvey4H8e0GVTu5wMP +gkkqwKPGIR1YOmlw6ippqwvoJD7LuYrm6Q4D6e1PvkjwCq6lEndrOPmPrrXNd0JJ +XO60y3U2SwKBgQDLkyZarryQXxcCI6Q10Tc6pskYDMIit095PUbTeiUOXNT9GE28 +Hi32ziLCakk9kCysNasii81MxtQ54tJ/f5iGbNMMddnkKl2a19Hc5LjjAm4cJzg/ +98KGEhvyVqvAo5bBDZ06/rcrD+lZOzUglQS5jcIcqCIYa0LHWQ/wJLxFzwKBgFcZ +1SRhdSmDfUmuF+S4ZpistflYjC3IV5rk4NkS9HvMWaJS0nqdw4A3AMzItXgkjq4S +DkOVLTkTI5Do5HAWRv/VwC5M2hkR4NMu1VGAKSisGiKtRsirBWSZMEenLNHshbjN +Jrpz5rZ4H7NT46ZkCCZyFBpX4gb9NyOedjA7Via3AoGARF8RxbYjnEGGFuhnbrJB +FTPR0vaL4faY3lOgRZ8jOG9V2c9Hzi/y8a8TU4C11jnJSDqYCXBTd5XN28npYxtD +pjRsCwy6ze+yvYXPO7C978eMG3YRyj366NXUxnXN59ibwe/lxi2OD9z8J1LEdF6z +VJua1Wn8HKxnXMI61DhTCSo= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIEEzCCAvugAwIBAgIJAK1haYi6gmSKMA0GCSqGSIb3DQEBCwUAMIGeMQswCQYD +VQQGEwJERTEMMAoGA1UECAwDTlJXMRQwEgYDVQQHDAtEdWVzc2VsZG9yZjEbMBkG +A1UECgwSeW91dHViZS1kbCBwcm9qZWN0MRkwFwYDVQQLDBB5b3V0dWJlLWRsIHRl +c3RzMRIwEAYDVQQDDAlsb2NhbGhvc3QxHzAdBgkqhkiG9w0BCQEWEHBoaWhhZ0Bw +aGloYWcuZGUwIBcNMTUwMTMwMDExNTA4WhgPMjExNTAxMDYwMTE1MDhaMIGeMQsw +CQYDVQQGEwJERTEMMAoGA1UECAwDTlJXMRQwEgYDVQQHDAtEdWVzc2VsZG9yZjEb +MBkGA1UECgwSeW91dHViZS1kbCBwcm9qZWN0MRkwFwYDVQQLDBB5b3V0dWJlLWRs +IHRlc3RzMRIwEAYDVQQDDAlsb2NhbGhvc3QxHzAdBgkqhkiG9w0BCQEWEHBoaWhh +Z0BwaGloYWcuZGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDMF0bA +zaHAdIyBHRmnIp4vv40lGqEePmWqicCl0QZ0wsb5dNysSxSa7330M2QeQopGfdaU +YF1uTcNpQx6ECgBSfg+RrOBI7r/u4F+sKX8MUXVaf/5QoBUrGNGSn/pp7HMGOuQq +O6BVg4+hA1ySSwUG8mZItLRry1ISyErmW8b9xlqfd97uLME/5tX+sMelRFjUbAx8 +A4CK58EvmMguHVTlXzx5RMdYcf1VScYcjlV/qA45uzP8zwI5aigfcmUD+tbGuQRh +KxUhmw0JaobtOR6+JSOAULW5gYa/egE4dWLwbyM6b6eFbdnjlQzEA1EW7ChMPAW/ +Mo83KyiPtKMCSQulAgMBAAGjUDBOMB0GA1UdDgQWBBTBUZoqhQkzHQ6xNgZfFxOd +ZEVt8TAfBgNVHSMEGDAWgBTBUZoqhQkzHQ6xNgZfFxOdZEVt8TAMBgNVHRMEBTAD +AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQCUOCl3T/J9B08Z+ijfOJAtkbUaEHuVZb4x +5EpZSy2ZbkLvtsftMFieHVNXn9dDswQc5qjYStCC4o60LKw4M6Y63FRsAZ/DNaqb +PY3jyCyuugZ8/sNf50vHYkAcF7SQYqOQFQX4TQsNUk2xMJIt7H0ErQFmkf/u3dg6 +cy89zkT462IwxzSG7NNhIlRkL9o5qg+Y1mF9eZA1B0rcL6hO24PPTHOd90HDChBu +SZ6XMi/LzYQSTf0Vg2R+uMIVlzSlkdcZ6sqVnnqeLL8dFyIa4e9sj/D4ZCYP8Mqe +Z73H5/NNhmwCHRqVUTgm307xblQaWGhwAiDkaRvRW2aJQ0qGEdZK +-----END CERTIFICATE----- diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 4c238c555..14e92ddcf 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -958,7 +958,7 @@ class YoutubeDL(object): if thumbnails is None: thumbnail = info_dict.get('thumbnail') if thumbnail: - thumbnails = [{'url': thumbnail}] + info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}] if thumbnails: thumbnails.sort(key=lambda t: ( t.get('preference'), t.get('width'), t.get('height'), diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 71d2c6f35..e90679ff9 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -361,7 +361,9 @@ def _real_main(argv=None): sys.exit() ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv) - parser.error('you must provide at least one URL') + parser.error( + 'You must provide at least one URL.\n' + 'Type youtube-dl --help to see a list of all options.') try: if opts.load_info_filename is not None: diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 4453b34fc..497ca52de 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -72,6 +72,11 @@ except ImportError: compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w') try: + import http.server as compat_http_server +except ImportError: + import BaseHTTPServer as compat_http_server + +try: from urllib.parse import unquote as compat_urllib_parse_unquote except ImportError: def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'): @@ -365,6 +370,7 @@ __all__ = [ 'compat_html_entities', 'compat_html_parser', 'compat_http_client', + 'compat_http_server', 'compat_kwargs', 'compat_ord', 'compat_parse_qs', diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 444af8b6e..ad4ed103b 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -82,6 +82,7 @@ from .crunchyroll import ( CrunchyrollShowPlaylistIE ) from .cspan import CSpanIE +from .ctsnews import CtsNewsIE from .dailymotion import ( DailymotionIE, DailymotionPlaylistIE, @@ -552,6 +553,7 @@ from .xminus import XMinusIE from .xnxx import XNXXIE from .xvideos import XVideosIE from .xtube import XTubeUserIE, XTubeIE +from .xuite import XuiteIE from .xxxymovies import XXXYMoviesIE from .yahoo import ( YahooIE, diff --git a/youtube_dl/extractor/ctsnews.py b/youtube_dl/extractor/ctsnews.py new file mode 100644 index 000000000..f769dc031 --- /dev/null +++ b/youtube_dl/extractor/ctsnews.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from .common import InfoExtractor +from ..compat import compat_urllib_request +from ..utils import parse_iso8601, ExtractorError + + +class CtsNewsIE(InfoExtractor): + # https connection failed (Connection reset) + _VALID_URL = r'http://news\.cts\.com\.tw/[a-z]+/[a-z]+/\d+/(?P<id>\d+)\.html' + _TESTS = [{ + 'url': 'http://news.cts.com.tw/cts/international/201501/201501291578109.html', + 'md5': 'a9875cb790252b08431186d741beaabe', + 'info_dict': { + 'id': '201501291578109', + 'ext': 'mp4', + 'title': '以色列.真主黨交火 3人死亡', + 'description': 'md5:95e9b295c898b7ff294f09d450178d7d', + 'timestamp': 1422528540, + 'upload_date': '20150129', + } + }, { + # News count not appear on page but still available in database + 'url': 'http://news.cts.com.tw/cts/international/201309/201309031304098.html', + 'md5': '3aee7e0df7cdff94e43581f54c22619e', + 'info_dict': { + 'id': '201309031304098', + 'ext': 'mp4', + 'title': '韓國31歲童顏男 貌如十多歲小孩', + 'description': 'md5:f183feeba3752b683827aab71adad584', + 'thumbnail': 're:^https?://.*\.jpg$', + 'timestamp': 1378205880, + 'upload_date': '20130903', + } + }, { + # With Youtube embedded video + 'url': 'http://news.cts.com.tw/cts/money/201501/201501291578003.html', + 'md5': '1d842c771dc94c8c3bca5af2cc1db9c5', + 'add_ie': ['Youtube'], + 'info_dict': { + 'id': 'OVbfO7d0_hQ', + 'ext': 'mp4', + 'title': 'iPhone6熱銷 蘋果財報亮眼', + 'description': 'md5:f395d4f485487bb0f992ed2c4b07aa7d', + 'thumbnail': 're:^https?://.*\.jpg$', + 'upload_date': '20150128', + 'uploader_id': 'TBSCTS', + 'uploader': '中華電視公司', + } + }] + + def _real_extract(self, url): + news_id = self._match_id(url) + page = self._download_webpage(url, news_id) + + if self._search_regex(r'(CTSPlayer2)', page, 'CTSPlayer2 identifier', default=None): + feed_url = self._html_search_regex( + r'(http://news\.cts\.com\.tw/action/mp4feed\.php\?news_id=\d+)', + page, 'feed url') + video_url = self._download_webpage( + feed_url, news_id, note='Fetching feed') + else: + self.to_screen('Not CTSPlayer video, trying Youtube...') + youtube_url = self._search_regex( + r'src="(//www\.youtube\.com/embed/[^"]+)"', page, 'youtube url', + default=None) + if not youtube_url: + raise ExtractorError('The news includes no videos!', expected=True) + + return { + '_type': 'url', + 'url': youtube_url, + 'ie_key': 'Youtube', + } + + description = self._html_search_meta('description', page) + title = self._html_search_meta('title', page) + thumbnail = self._html_search_meta('image', page) + + datetime_str = self._html_search_regex( + r'(\d{4}/\d{2}/\d{2} \d{2}:\d{2})', page, 'date and time') + # Transform into ISO 8601 format with timezone info + datetime_str = datetime_str.replace('/', '-') + ':00+0800' + timestamp = parse_iso8601(datetime_str, delimiter=' ') + + return { + 'id': news_id, + 'url': video_url, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'timestamp': timestamp, + } diff --git a/youtube_dl/extractor/srmediathek.py b/youtube_dl/extractor/srmediathek.py index 666a7dcc8..5d583c720 100644 --- a/youtube_dl/extractor/srmediathek.py +++ b/youtube_dl/extractor/srmediathek.py @@ -8,7 +8,7 @@ from ..utils import js_to_json class SRMediathekIE(InfoExtractor): - IE_DESC = 'Süddeutscher Rundfunk' + IE_DESC = 'Saarländischer Rundfunk' _VALID_URL = r'https?://sr-mediathek\.sr-online\.de/index\.php\?.*?&id=(?P<id>[0-9]+)' _TEST = { diff --git a/youtube_dl/extractor/viddler.py b/youtube_dl/extractor/viddler.py index ef104dc29..8516a2940 100644 --- a/youtube_dl/extractor/viddler.py +++ b/youtube_dl/extractor/viddler.py @@ -6,7 +6,7 @@ from ..utils import ( int_or_none, ) from ..compat import ( - compat_urllib_request + compat_urllib_request ) diff --git a/youtube_dl/extractor/xuite.py b/youtube_dl/extractor/xuite.py new file mode 100644 index 000000000..4971965f9 --- /dev/null +++ b/youtube_dl/extractor/xuite.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import base64 + +from .common import InfoExtractor +from ..compat import compat_urllib_parse_unquote +from ..utils import ( + ExtractorError, + parse_iso8601, + parse_duration, +) + + +class XuiteIE(InfoExtractor): + _REGEX_BASE64 = r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?' + _VALID_URL = r'https?://vlog\.xuite\.net/(?:play|embed)/(?P<id>%s)' % _REGEX_BASE64 + _TESTS = [{ + # Audio + 'url': 'http://vlog.xuite.net/play/RGkzc1ZULTM4NjA5MTQuZmx2', + 'md5': '63a42c705772aa53fd4c1a0027f86adf', + 'info_dict': { + 'id': '3860914', + 'ext': 'mp3', + 'title': '孤單南半球-歐德陽', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 247.246, + 'timestamp': 1314932940, + 'upload_date': '20110902', + 'uploader': '阿能', + 'uploader_id': '15973816', + 'categories': ['個人短片'], + }, + }, { + # Video with only one format + 'url': 'http://vlog.xuite.net/play/TkRZNjhULTM0NDE2MjkuZmx2', + 'md5': 'c45737fc8ac5dc8ac2f92ecbcecf505e', + 'info_dict': { + 'id': '3441629', + 'ext': 'mp4', + 'title': '孫燕姿 - 眼淚成詩', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 217.399, + 'timestamp': 1299383640, + 'upload_date': '20110306', + 'uploader': 'Valen', + 'uploader_id': '10400126', + 'categories': ['影視娛樂'], + }, + }, { + # Video with two formats + 'url': 'http://vlog.xuite.net/play/bWo1N1pLLTIxMzAxMTcwLmZsdg==', + 'md5': '1166e0f461efe55b62e26a2d2a68e6de', + 'info_dict': { + 'id': '21301170', + 'ext': 'mp4', + 'title': '暗殺教室 02', + 'description': '字幕:【極影字幕社】', + 'thumbnail': 're:^https?://.*\.jpg$', + 'duration': 1384.907, + 'timestamp': 1421481240, + 'upload_date': '20150117', + 'uploader': '我只是想認真點', + 'uploader_id': '242127761', + 'categories': ['電玩動漫'], + }, + }, { + 'url': 'http://vlog.xuite.net/play/S1dDUjdyLTMyOTc3NjcuZmx2/%E5%AD%AB%E7%87%95%E5%A7%BF-%E7%9C%BC%E6%B7%9A%E6%88%90%E8%A9%A9', + 'only_matching': True, + }] + + def _extract_flv_config(self, media_id): + base64_media_id = base64.b64encode(media_id.encode('utf-8')).decode('utf-8') + flv_config = self._download_xml( + 'http://vlog.xuite.net/flash/player?media=%s' % base64_media_id, + 'flv config') + prop_dict = {} + for prop in flv_config.findall('./property'): + prop_id = base64.b64decode(prop.attrib['id']).decode('utf-8') + # CDATA may be empty in flv config + if not prop.text: + continue + encoded_content = base64.b64decode(prop.text).decode('utf-8') + prop_dict[prop_id] = compat_urllib_parse_unquote(encoded_content) + return prop_dict + + def _real_extract(self, url): + video_id = self._match_id(url) + + webpage = self._download_webpage(url, video_id) + + error_msg = self._search_regex( + r'<div id="error-message-content">([^<]+)', + webpage, 'error message', default=None) + if error_msg: + raise ExtractorError( + '%s returned error: %s' % (self.IE_NAME, error_msg), + expected=True) + + video_id = self._html_search_regex( + r'data-mediaid="(\d+)"', webpage, 'media id') + flv_config = self._extract_flv_config(video_id) + + FORMATS = { + 'audio': 'mp3', + 'video': 'mp4', + } + + formats = [] + for format_tag in ('src', 'hq_src'): + video_url = flv_config.get(format_tag) + if not video_url: + continue + format_id = self._search_regex( + r'\bq=(.+?)\b', video_url, 'format id', default=format_tag) + formats.append({ + 'url': video_url, + 'ext': FORMATS.get(flv_config['type'], 'mp4'), + 'format_id': format_id, + 'height': int(format_id) if format_id.isnumeric() else None, + }) + self._sort_formats(formats) + + timestamp = flv_config.get('publish_datetime') + if timestamp: + timestamp = parse_iso8601(timestamp + ' +0800', ' ') + + category = flv_config.get('category') + categories = [category] if category else [] + + return { + 'id': video_id, + 'title': flv_config['title'], + 'description': flv_config.get('description'), + 'thumbnail': flv_config.get('thumb'), + 'timestamp': timestamp, + 'uploader': flv_config.get('author_name'), + 'uploader_id': flv_config.get('author_id'), + 'duration': parse_duration(flv_config.get('duration')), + 'categories': categories, + 'formats': formats, + } diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index b8c52af74..a4c9813ec 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -654,9 +654,14 @@ class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler): self._params = params def https_open(self, req): + kwargs = {} + if hasattr(self, '_context'): # python > 2.6 + kwargs['context'] = self._context + if hasattr(self, '_check_hostname'): # python 3.x + kwargs['check_hostname'] = self._check_hostname return self.do_open(functools.partial( _create_http_connection, self, self._https_conn_class, True), - req) + req, **kwargs) def parse_iso8601(date_str, delimiter='T'): |