aboutsummaryrefslogtreecommitdiff
path: root/youtube_dl/extractor
diff options
context:
space:
mode:
Diffstat (limited to 'youtube_dl/extractor')
-rw-r--r--youtube_dl/extractor/__init__.py5
-rw-r--r--youtube_dl/extractor/canalc2.py2
-rw-r--r--youtube_dl/extractor/canalplus.py22
-rw-r--r--youtube_dl/extractor/dailymotion.py28
-rw-r--r--youtube_dl/extractor/francetv.py77
-rw-r--r--youtube_dl/extractor/funnyordie.py2
-rw-r--r--youtube_dl/extractor/gamespot.py2
-rw-r--r--youtube_dl/extractor/howcast.py3
-rw-r--r--youtube_dl/extractor/kickstarter.py32
-rw-r--r--youtube_dl/extractor/slideshare.py47
-rw-r--r--youtube_dl/extractor/sohu.py16
-rw-r--r--youtube_dl/extractor/subtitles.py92
-rw-r--r--youtube_dl/extractor/youtube.py156
13 files changed, 344 insertions, 140 deletions
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5711b6bba..d093d5c28 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -29,6 +29,10 @@ from .escapist import EscapistIE
from .exfm import ExfmIE
from .facebook import FacebookIE
from .flickr import FlickrIE
+from .francetv import (
+ PluzzIE,
+ FranceTvInfoIE,
+)
from .freesound import FreesoundIE
from .funnyordie import FunnyOrDieIE
from .gamespot import GameSpotIE
@@ -76,6 +80,7 @@ from .roxwel import RoxwelIE
from .rtlnow import RTLnowIE
from .sina import SinaIE
from .slashdot import SlashdotIE
+from .slideshare import SlideshareIE
from .sohu import SohuIE
from .soundcloud import SoundcloudIE, SoundcloudSetIE
from .spiegel import SpiegelIE
diff --git a/youtube_dl/extractor/canalc2.py b/youtube_dl/extractor/canalc2.py
index 50832217a..e7f4fa9fd 100644
--- a/youtube_dl/extractor/canalc2.py
+++ b/youtube_dl/extractor/canalc2.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
class Canalc2IE(InfoExtractor):
- _IE_NAME = 'canalc2.tv'
+ IE_NAME = 'canalc2.tv'
_VALID_URL = r'http://.*?\.canalc2\.tv/video\.asp\?idVideo=(\d+)&voir=oui'
_TEST = {
diff --git a/youtube_dl/extractor/canalplus.py b/youtube_dl/extractor/canalplus.py
index 1f02519a0..1db9b24cf 100644
--- a/youtube_dl/extractor/canalplus.py
+++ b/youtube_dl/extractor/canalplus.py
@@ -1,3 +1,4 @@
+# encoding: utf-8
import re
import xml.etree.ElementTree
@@ -5,24 +6,29 @@ from .common import InfoExtractor
from ..utils import unified_strdate
class CanalplusIE(InfoExtractor):
- _VALID_URL = r'https?://(www\.canalplus\.fr/.*?\?vid=|player\.canalplus\.fr/#/)(?P<id>\d+)'
+ _VALID_URL = r'https?://(www\.canalplus\.fr/.*?/(?P<path>.*)|player\.canalplus\.fr/#/(?P<id>\d+))'
_VIDEO_INFO_TEMPLATE = 'http://service.canal-plus.com/video/rest/getVideosLiees/cplus/%s'
IE_NAME = u'canalplus.fr'
_TEST = {
- u'url': u'http://www.canalplus.fr/c-divertissement/pid3351-c-le-petit-journal.html?vid=889861',
- u'file': u'889861.flv',
- u'md5': u'590a888158b5f0d6832f84001fbf3e99',
+ u'url': u'http://www.canalplus.fr/c-infos-documentaires/pid1830-c-zapping.html?vid=922470',
+ u'file': u'922470.flv',
u'info_dict': {
- u'title': u'Le Petit Journal 20/06/13 - La guerre des drone',
- u'upload_date': u'20130620',
+ u'title': u'Zapping - 26/08/13',
+ u'description': u'Le meilleur de toutes les chaînes, tous les jours.\nEmission du 26 août 2013',
+ u'upload_date': u'20130826',
+ },
+ u'params': {
+ u'skip_download': True,
},
- u'skip': u'Requires rtmpdump'
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
+ if video_id is None:
+ webpage = self._download_webpage(url, mobj.group('path'))
+ video_id = self._search_regex(r'videoId = "(\d+)";', webpage, u'video id')
info_url = self._VIDEO_INFO_TEMPLATE % video_id
info_page = self._download_webpage(info_url,video_id,
u'Downloading video info')
@@ -43,4 +49,6 @@ class CanalplusIE(InfoExtractor):
'ext': 'flv',
'upload_date': unified_strdate(infos.find('PUBLICATION/DATE').text),
'thumbnail': media.find('IMAGES/GRAND').text,
+ 'description': infos.find('DESCRIPTION').text,
+ 'view_count': int(infos.find('NB_VUES').text),
}
diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py
index 3c616e089..360113f9c 100644
--- a/youtube_dl/extractor/dailymotion.py
+++ b/youtube_dl/extractor/dailymotion.py
@@ -3,15 +3,19 @@ import json
import itertools
from .common import InfoExtractor
+from .subtitles import SubtitlesInfoExtractor
+
from ..utils import (
compat_urllib_request,
+ compat_str,
get_element_by_attribute,
get_element_by_id,
ExtractorError,
)
-class DailymotionIE(InfoExtractor):
+
+class DailymotionIE(SubtitlesInfoExtractor):
"""Information Extractor for Dailymotion"""
_VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/(?:embed/)?video/([^/]+)'
@@ -73,6 +77,12 @@ class DailymotionIE(InfoExtractor):
raise ExtractorError(u'Unable to extract video URL')
video_url = info[max_quality]
+ # subtitles
+ video_subtitles = self.extract_subtitles(video_id)
+ if self._downloader.params.get('listsubtitles', False):
+ self._list_available_subtitles(video_id)
+ return
+
return [{
'id': video_id,
'url': video_url,
@@ -80,9 +90,25 @@ class DailymotionIE(InfoExtractor):
'upload_date': video_upload_date,
'title': self._og_search_title(webpage),
'ext': video_extension,
+ 'subtitles': video_subtitles,
'thumbnail': info['thumbnail_url']
}]
+ def _get_available_subtitles(self, video_id):
+ try:
+ sub_list = self._download_webpage(
+ 'https://api.dailymotion.com/video/%s/subtitles?fields=id,language,url' % video_id,
+ video_id, note=False)
+ except ExtractorError as err:
+ self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
+ return {}
+ info = json.loads(sub_list)
+ if (info['total'] > 0):
+ sub_lang_list = dict((l['language'], l['url']) for l in info['list'])
+ return sub_lang_list
+ self._downloader.report_warning(u'video doesn\'t have subtitles')
+ return {}
+
class DailymotionPlaylistIE(InfoExtractor):
_VALID_URL = r'(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/playlist/(?P<id>.+?)/'
diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
new file mode 100644
index 000000000..f2b12c884
--- /dev/null
+++ b/youtube_dl/extractor/francetv.py
@@ -0,0 +1,77 @@
+# encoding: utf-8
+import re
+import xml.etree.ElementTree
+
+from .common import InfoExtractor
+from ..utils import (
+ compat_urlparse,
+)
+
+
+class FranceTVBaseInfoExtractor(InfoExtractor):
+ def _extract_video(self, video_id):
+ xml_desc = self._download_webpage(
+ 'http://www.francetvinfo.fr/appftv/webservices/video/'
+ 'getInfosOeuvre.php?id-diffusion='
+ + video_id, video_id, 'Downloading XML config')
+ info = xml.etree.ElementTree.fromstring(xml_desc.encode('utf-8'))
+
+ manifest_url = info.find('videos/video/url').text
+ video_url = manifest_url.replace('manifest.f4m', 'index_2_av.m3u8')
+ video_url = video_url.replace('/z/', '/i/')
+ thumbnail_path = info.find('image').text
+
+ return {'id': video_id,
+ 'ext': 'mp4',
+ 'url': video_url,
+ 'title': info.find('titre').text,
+ 'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', thumbnail_path),
+ 'description': info.find('synopsis').text,
+ }
+
+
+class PluzzIE(FranceTVBaseInfoExtractor):
+ IE_NAME = u'pluzz.francetv.fr'
+ _VALID_URL = r'https?://pluzz\.francetv\.fr/videos/(.*?)\.html'
+
+ _TEST = {
+ u'url': u'http://pluzz.francetv.fr/videos/allo_rufo_saison5_,88439064.html',
+ u'file': u'88439064.mp4',
+ u'info_dict': {
+ u'title': u'Allô Rufo',
+ u'description': u'md5:d909f1ebdf963814b65772aea250400e',
+ },
+ u'params': {
+ u'skip_download': True,
+ },
+ }
+
+ def _real_extract(self, url):
+ title = re.match(self._VALID_URL, url).group(1)
+ webpage = self._download_webpage(url, title)
+ video_id = self._search_regex(
+ r'data-diffusion="(\d+)"', webpage, 'ID')
+ return self._extract_video(video_id)
+
+
+class FranceTvInfoIE(FranceTVBaseInfoExtractor):
+ IE_NAME = u'francetvinfo.fr'
+ _VALID_URL = r'https?://www\.francetvinfo\.fr/replay.*/(?P<title>.+).html'
+
+ _TEST = {
+ u'url': u'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html',
+ u'file': u'84981923.mp4',
+ u'info_dict': {
+ u'title': u'Soir 3',
+ },
+ u'params': {
+ u'skip_download': True,
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ page_title = mobj.group('title')
+ webpage = self._download_webpage(url, page_title)
+ video_id = self._search_regex(r'id-video=(\d+?)"', webpage, u'video id')
+ return self._extract_video(video_id)
diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py
index 4508f0dfa..f3d86a711 100644
--- a/youtube_dl/extractor/funnyordie.py
+++ b/youtube_dl/extractor/funnyordie.py
@@ -21,7 +21,7 @@ class FunnyOrDieIE(InfoExtractor):
video_id = mobj.group('id')
webpage = self._download_webpage(url, video_id)
- video_url = self._search_regex(r'type: "video/mp4", src: "(.*?)"',
+ video_url = self._search_regex(r'type="video/mp4" src="(.*?)"',
webpage, u'video URL', flags=re.DOTALL)
info = {
diff --git a/youtube_dl/extractor/gamespot.py b/youtube_dl/extractor/gamespot.py
index 7585b7061..cd3bbe65f 100644
--- a/youtube_dl/extractor/gamespot.py
+++ b/youtube_dl/extractor/gamespot.py
@@ -14,7 +14,7 @@ class GameSpotIE(InfoExtractor):
u"file": u"6410818.mp4",
u"md5": u"b2a30deaa8654fcccd43713a6b6a4825",
u"info_dict": {
- u"title": u"Arma III - Community Guide: SITREP I",
+ u"title": u"Arma 3 - Community Guide: SITREP I",
u"upload_date": u"20130627",
}
}
diff --git a/youtube_dl/extractor/howcast.py b/youtube_dl/extractor/howcast.py
index 6104c4b5e..46954337f 100644
--- a/youtube_dl/extractor/howcast.py
+++ b/youtube_dl/extractor/howcast.py
@@ -19,8 +19,7 @@ class HowcastIE(InfoExtractor):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
- webpage_url = 'http://www.howcast.com/videos/' + video_id
- webpage = self._download_webpage(webpage_url, video_id)
+ webpage = self._download_webpage(url, video_id)
self.report_extraction(video_id)
diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py
index 7f6f2b064..50bc883ef 100644
--- a/youtube_dl/extractor/kickstarter.py
+++ b/youtube_dl/extractor/kickstarter.py
@@ -4,40 +4,34 @@ from .common import InfoExtractor
class KickStarterIE(InfoExtractor):
- _VALID_URL = r'https?://www\.kickstarter\.com/projects/(?P<id>.*)/.*\?'
+ _VALID_URL = r'https?://www\.kickstarter\.com/projects/(?P<id>\d*)/.*'
_TEST = {
- "url": "https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant?ref=home_location",
- "file": "1404461844.mp4",
- "md5": "c81addca81327ffa66c642b5d8b08cab",
- "info_dict": {
- "title": u"Intersection: The Story of Josh Grant by Kyle Cowling \u2014 Kickstarter"
- }
+ u"url": u"https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant?ref=home_location",
+ u"file": u"1404461844.mp4",
+ u"md5": u"c81addca81327ffa66c642b5d8b08cab",
+ u"info_dict": {
+ u"title": u"Intersection: The Story of Josh Grant by Kyle Cowling",
+ },
}
-
def _real_extract(self, url):
m = re.match(self._VALID_URL, url)
video_id = m.group('id')
-
webpage_src = self._download_webpage(url, video_id)
video_url = self._search_regex(r'data-video="(.*?)">',
webpage_src, u'video URL')
-
if 'mp4' in video_url:
ext = 'mp4'
else:
ext = 'flv'
-
- video_title = self._html_search_regex(r"<title>(.*)</title>?",
- webpage_src, u'title')
-
+ video_title = self._html_search_regex(r"<title>(.*?)</title>",
+ webpage_src, u'title').rpartition(u'\u2014 Kickstarter')[0].strip()
results = [{
'id': video_id,
- 'url' : video_url,
- 'title' : video_title,
- 'ext' : ext,
+ 'url': video_url,
+ 'title': video_title,
+ 'ext': ext,
}]
-
- return results \ No newline at end of file
+ return results
diff --git a/youtube_dl/extractor/slideshare.py b/youtube_dl/extractor/slideshare.py
new file mode 100644
index 000000000..afc3001b5
--- /dev/null
+++ b/youtube_dl/extractor/slideshare.py
@@ -0,0 +1,47 @@
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+ compat_urlparse,
+ ExtractorError,
+)
+
+
+class SlideshareIE(InfoExtractor):
+ _VALID_URL = r'https?://www\.slideshare\.net/[^/]+?/(?P<title>.+?)($|\?)'
+
+ _TEST = {
+ u'url': u'http://www.slideshare.net/Dataversity/keynote-presentation-managing-scale-and-complexity',
+ u'file': u'25665706.mp4',
+ u'info_dict': {
+ u'title': u'Managing Scale and Complexity',
+ u'description': u'This was a keynote presentation at the NoSQL Now! 2013 Conference & Expo (http://www.nosqlnow.com). This presentation was given by Adrian Cockcroft from Netflix',
+ },
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ page_title = mobj.group('title')
+ webpage = self._download_webpage(url, page_title)
+ slideshare_obj = self._search_regex(
+ r'var slideshare_object = ({.*?}); var user_info =',
+ webpage, u'slideshare object')
+ info = json.loads(slideshare_obj)
+ if info['slideshow']['type'] != u'video':
+ raise ExtractorError(u'Webpage type is "%s": only video extraction is supported for Slideshare' % info['slideshow']['type'], expected=True)
+
+ doc = info['doc']
+ bucket = info['jsplayer']['video_bucket']
+ ext = info['jsplayer']['video_extension']
+ video_url = compat_urlparse.urljoin(bucket, doc + '-SD.' + ext)
+
+ return {
+ '_type': 'video',
+ 'id': info['slideshow']['id'],
+ 'title': info['slideshow']['title'],
+ 'ext': ext,
+ 'url': video_url,
+ 'thumbnail': info['slideshow']['pin_image_url'],
+ 'description': self._og_search_description(webpage),
+ }
diff --git a/youtube_dl/extractor/sohu.py b/youtube_dl/extractor/sohu.py
index 77bb0a8dc..2b9bf0cb7 100644
--- a/youtube_dl/extractor/sohu.py
+++ b/youtube_dl/extractor/sohu.py
@@ -8,7 +8,7 @@ from ..utils import ExtractorError
class SohuIE(InfoExtractor):
- _VALID_URL = r'https?://tv\.sohu\.com/\d+?/n(?P<id>\d+)\.shtml.*?'
+ _VALID_URL = r'https?://(?P<mytv>my\.)?tv\.sohu\.com/.+?/(?(mytv)|n)(?P<id>\d+)\.shtml.*?'
_TEST = {
u'url': u'http://tv.sohu.com/20130724/n382479172.shtml#super',
@@ -21,8 +21,11 @@ class SohuIE(InfoExtractor):
def _real_extract(self, url):
- def _fetch_data(vid_id):
- base_data_url = u'http://hot.vrs.sohu.com/vrs_flash.action?vid='
+ def _fetch_data(vid_id, mytv=False):
+ if mytv:
+ base_data_url = 'http://my.tv.sohu.com/play/videonew.do?vid='
+ else:
+ base_data_url = u'http://hot.vrs.sohu.com/vrs_flash.action?vid='
data_url = base_data_url + str(vid_id)
data_json = self._download_webpage(
data_url, video_id,
@@ -31,15 +34,16 @@ class SohuIE(InfoExtractor):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('id')
+ mytv = mobj.group('mytv') is not None
webpage = self._download_webpage(url, video_id)
raw_title = self._html_search_regex(r'(?s)<title>(.+?)</title>',
webpage, u'video title')
title = raw_title.partition('-')[0].strip()
- vid = self._html_search_regex(r'var vid="(\d+)"', webpage,
+ vid = self._html_search_regex(r'var vid ?= ?["\'](\d+)["\']', webpage,
u'video path')
- data = _fetch_data(vid)
+ data = _fetch_data(vid, mytv)
QUALITIES = ('ori', 'super', 'high', 'nor')
vid_ids = [data['data'][q + 'Vid']
@@ -51,7 +55,7 @@ class SohuIE(InfoExtractor):
# For now, we just pick the highest available quality
vid_id = vid_ids[-1]
- format_data = data if vid == vid_id else _fetch_data(vid_id)
+ format_data = data if vid == vid_id else _fetch_data(vid_id, mytv)
part_count = format_data['data']['totalBlocks']
allot = format_data['allot']
prot = format_data['prot']
diff --git a/youtube_dl/extractor/subtitles.py b/youtube_dl/extractor/subtitles.py
new file mode 100644
index 000000000..97215f289
--- /dev/null
+++ b/youtube_dl/extractor/subtitles.py
@@ -0,0 +1,92 @@
+from .common import InfoExtractor
+
+from ..utils import (
+ compat_str,
+ ExtractorError,
+)
+
+
+class SubtitlesInfoExtractor(InfoExtractor):
+ @property
+ def _have_to_download_any_subtitles(self):
+ return any([self._downloader.params.get('writesubtitles', False),
+ self._downloader.params.get('writeautomaticsub'),
+ self._downloader.params.get('allsubtitles', False)])
+
+ def _list_available_subtitles(self, video_id, webpage=None):
+ """ outputs the available subtitles for the video """
+ sub_lang_list = self._get_available_subtitles(video_id)
+ auto_captions_list = self._get_available_automatic_caption(video_id, webpage)
+ sub_lang = ",".join(list(sub_lang_list.keys()))
+ self.to_screen(u'%s: Available subtitles for video: %s' %
+ (video_id, sub_lang))
+ auto_lang = ",".join(auto_captions_list.keys())
+ self.to_screen(u'%s: Available automatic captions for video: %s' %
+ (video_id, auto_lang))
+
+ def extract_subtitles(self, video_id, video_webpage=None):
+ """
+ returns {sub_lang: sub} ,{} if subtitles not found or None if the
+ subtitles aren't requested.
+ """
+ if not self._have_to_download_any_subtitles:
+ return None
+ available_subs_list = {}
+ if self._downloader.params.get('writeautomaticsub', False):
+ available_subs_list.update(self._get_available_automatic_caption(video_id, video_webpage))
+ if self._downloader.params.get('writesubtitles', False) or self._downloader.params.get('allsubtitles', False):
+ available_subs_list.update(self._get_available_subtitles(video_id))
+
+ if not available_subs_list: # error, it didn't get the available subtitles
+ return {}
+ if self._downloader.params.get('allsubtitles', False):
+ sub_lang_list = available_subs_list
+ else:
+ if self._downloader.params.get('subtitleslangs', False):
+ requested_langs = self._downloader.params.get('subtitleslangs')
+ elif 'en' in available_subs_list:
+ requested_langs = ['en']
+ else:
+ requested_langs = [list(available_subs_list.keys())[0]]
+
+ sub_lang_list = {}
+ for sub_lang in requested_langs:
+ if not sub_lang in available_subs_list:
+ self._downloader.report_warning(u'no closed captions found in the specified language "%s"' % sub_lang)
+ continue
+ sub_lang_list[sub_lang] = available_subs_list[sub_lang]
+
+ subtitles = {}
+ for sub_lang, url in sub_lang_list.items():
+ subtitle = self._request_subtitle_url(sub_lang, url)
+ if subtitle:
+ subtitles[sub_lang] = subtitle
+ return subtitles
+
+ def _request_subtitle_url(self, sub_lang, url):
+ """ makes the http request for the subtitle """
+ try:
+ sub = self._download_webpage(url, None, note=False)
+ except ExtractorError as err:
+ self._downloader.report_warning(u'unable to download video subtitles for %s: %s' % (sub_lang, compat_str(err)))
+ return
+ if not sub:
+ self._downloader.report_warning(u'Did not fetch video subtitles')
+ return
+ return sub
+
+ def _get_available_subtitles(self, video_id):
+ """
+ returns {sub_lang: url} or {} if not available
+ Must be redefined by the subclasses
+ """
+ pass
+
+ def _get_available_automatic_caption(self, video_id, webpage):
+ """
+ returns {sub_lang: url} or {} if not available
+ Must be redefined by the subclasses that support automatic captions,
+ otherwise it will return {}
+ """
+ self._downloader.report_warning(u'Automatic Captions not supported by this server')
+ return {}
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 423a5e973..f49665925 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -5,8 +5,10 @@ import netrc
import re
import socket
import itertools
+import xml.etree.ElementTree
from .common import InfoExtractor, SearchInfoExtractor
+from .subtitles import SubtitlesInfoExtractor
from ..utils import (
compat_http_client,
compat_parse_qs,
@@ -130,7 +132,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
return
self._confirm_age()
-class YoutubeIE(YoutubeBaseInfoExtractor):
+
+class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
IE_DESC = u'YouTube.com'
_VALID_URL = r"""^
(
@@ -150,7 +153,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
|youtu\.be/ # just youtu.be/xxxx
)
)? # all until now is optional -> you can pass the naked ID
- ([0-9A-Za-z_-]+) # here is it! the YouTube video ID
+ ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
(?(1).+)? # if we found the ID, everything can follow
$"""
_NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
@@ -397,19 +400,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
"""Report attempt to download video info webpage."""
self.to_screen(u'%s: Downloading video info webpage' % video_id)
- def report_video_subtitles_download(self, video_id):
- """Report attempt to download video info webpage."""
- self.to_screen(u'%s: Checking available subtitles' % video_id)
-
- def report_video_subtitles_request(self, video_id, sub_lang, format):
- """Report attempt to download video info webpage."""
- self.to_screen(u'%s: Downloading video subtitles for %s.%s' % (video_id, sub_lang, format))
-
- def report_video_subtitles_available(self, video_id, sub_lang_list):
- """Report available subtitles."""
- sub_lang = ",".join(list(sub_lang_list.keys()))
- self.to_screen(u'%s: Available subtitles for video: %s' % (video_id, sub_lang))
-
def report_information_extraction(self, video_id):
"""Report attempt to extract video information."""
self.to_screen(u'%s: Extracting video information' % video_id)
@@ -438,13 +428,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
elif len(s) == 86:
return s[5:34] + s[0] + s[35:38] + s[3] + s[39:45] + s[38] + s[46:53] + s[73] + s[54:73] + s[85] + s[74:85] + s[53]
elif len(s) == 85:
- return s[83:34:-1] + s[0] + s[33:27:-1] + s[3] + s[26:19:-1] + s[34] + s[18:3:-1] + s[27]
+ return s[40] + s[82:43:-1] + s[22] + s[42:40:-1] + s[83] + s[39:22:-1] + s[0] + s[21:2:-1]
elif len(s) == 84:
return s[81:36:-1] + s[0] + s[35:2:-1]
elif len(s) == 83:
return s[81:64:-1] + s[82] + s[63:52:-1] + s[45] + s[51:45:-1] + s[1] + s[44:1:-1] + s[0]
elif len(s) == 82:
- return s[1:19] + s[0] + s[20:68] + s[19] + s[69:82]
+ return s[80:73:-1] + s[81] + s[72:54:-1] + s[2] + s[53:43:-1] + s[0] + s[42:2:-1] + s[43] + s[1] + s[54]
elif len(s) == 81:
return s[56] + s[79:56:-1] + s[41] + s[55:41:-1] + s[80] + s[40:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
elif len(s) == 80:
@@ -464,56 +454,38 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
# Fallback to the other algortihms
return self._decrypt_signature(s)
-
def _get_available_subtitles(self, video_id):
- self.report_video_subtitles_download(video_id)
- request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
try:
- sub_list = compat_urllib_request.urlopen(request).read().decode('utf-8')
- except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+ sub_list = self._download_webpage(
+ 'http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
+ video_id, note=False)
+ except ExtractorError as err:
self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
return {}
- sub_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
- sub_lang_list = dict((l[1], l[0]) for l in sub_lang_list)
+ lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
+
+ sub_lang_list = {}
+ for l in lang_list:
+ lang = l[1]
+ params = compat_urllib_parse.urlencode({
+ 'lang': lang,
+ 'v': video_id,
+ 'fmt': self._downloader.params.get('subtitlesformat'),
+ })
+ url = u'http://www.youtube.com/api/timedtext?' + params
+ sub_lang_list[lang] = url
if not sub_lang_list:
self._downloader.report_warning(u'video doesn\'t have subtitles')
return {}
return sub_lang_list
- def _list_available_subtitles(self, video_id):
- sub_lang_list = self._get_available_subtitles(video_id)
- self.report_video_subtitles_available(video_id, sub_lang_list)
-
- def _request_subtitle(self, sub_lang, sub_name, video_id, format):
- """
- Return the subtitle as a string or None if they are not found
- """
- self.report_video_subtitles_request(video_id, sub_lang, format)
- params = compat_urllib_parse.urlencode({
- 'lang': sub_lang,
- 'name': sub_name,
- 'v': video_id,
- 'fmt': format,
- })
- url = 'http://www.youtube.com/api/timedtext?' + params
- try:
- sub = compat_urllib_request.urlopen(url).read().decode('utf-8')
- except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
- self._downloader.report_warning(u'unable to download video subtitles for %s: %s' % (sub_lang, compat_str(err)))
- return
- if not sub:
- self._downloader.report_warning(u'Did not fetch video subtitles')
- return
- return sub
-
- def _request_automatic_caption(self, video_id, webpage):
+ def _get_available_automatic_caption(self, video_id, webpage):
"""We need the webpage for getting the captions url, pass it as an
argument to speed up the process."""
- sub_lang = (self._downloader.params.get('subtitleslangs') or ['en'])[0]
sub_format = self._downloader.params.get('subtitlesformat')
self.to_screen(u'%s: Looking for automatic captions' % video_id)
mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
- err_msg = u'Couldn\'t find automatic captions for "%s"' % sub_lang
+ err_msg = u'Couldn\'t find automatic captions for %s' % video_id
if mobj is None:
self._downloader.report_warning(err_msg)
return {}
@@ -522,53 +494,38 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
args = player_config[u'args']
caption_url = args[u'ttsurl']
timestamp = args[u'timestamp']
- params = compat_urllib_parse.urlencode({
- 'lang': 'en',
- 'tlang': sub_lang,
- 'fmt': sub_format,
- 'ts': timestamp,
- 'kind': 'asr',
+ # We get the available subtitles
+ list_params = compat_urllib_parse.urlencode({
+ 'type': 'list',
+ 'tlangs': 1,
+ 'asrs': 1,
})
- subtitles_url = caption_url + '&' + params
- sub = self._download_webpage(subtitles_url, video_id, u'Downloading automatic captions')
- return {sub_lang: sub}
+ list_url = caption_url + '&' + list_params
+ list_page = self._download_webpage(list_url, video_id)
+ caption_list = xml.etree.ElementTree.fromstring(list_page.encode('utf-8'))
+ original_lang_node = caption_list.find('track')
+ if original_lang_node.attrib.get('kind') != 'asr' :
+ self._downloader.report_warning(u'Video doesn\'t have automatic captions')
+ return {}
+ original_lang = original_lang_node.attrib['lang_code']
+
+ sub_lang_list = {}
+ for lang_node in caption_list.findall('target'):
+ sub_lang = lang_node.attrib['lang_code']
+ params = compat_urllib_parse.urlencode({
+ 'lang': original_lang,
+ 'tlang': sub_lang,
+ 'fmt': sub_format,
+ 'ts': timestamp,
+ 'kind': 'asr',
+ })
+ sub_lang_list[sub_lang] = caption_url + '&' + params
+ return sub_lang_list
# An extractor error can be raise by the download process if there are
# no automatic captions but there are subtitles
except (KeyError, ExtractorError):
self._downloader.report_warning(err_msg)
return {}
-
- def _extract_subtitles(self, video_id):
- """
- Return a dictionary: {language: subtitles} or {} if the subtitles
- couldn't be found
- """
- available_subs_list = self._get_available_subtitles(video_id)
- sub_format = self._downloader.params.get('subtitlesformat')
- if not available_subs_list: #There was some error, it didn't get the available subtitles
- return {}
- if self._downloader.params.get('allsubtitles', False):
- sub_lang_list = available_subs_list
- else:
- if self._downloader.params.get('subtitleslangs', False):
- reqested_langs = self._downloader.params.get('subtitleslangs')
- elif 'en' in available_subs_list:
- reqested_langs = ['en']
- else:
- reqested_langs = [list(available_subs_list.keys())[0]]
-
- sub_lang_list = {}
- for sub_lang in reqested_langs:
- if not sub_lang in available_subs_list:
- self._downloader.report_warning(u'no closed captions found in the specified language "%s"' % sub_lang)
- continue
- sub_lang_list[sub_lang] = available_subs_list[sub_lang]
- subtitles = {}
- for sub_lang in sub_lang_list:
- subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format)
- if subtitle:
- subtitles[sub_lang] = subtitle
- return subtitles
def _print_formats(self, formats):
print('Available formats:')
@@ -643,7 +600,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
manifest = self._download_webpage(manifest_url, video_id, u'Downloading formats manifest')
formats_urls = _get_urls(manifest)
for format_url in formats_urls:
- itag = self._search_regex(r'itag%3D(\d+?)/', format_url, 'itag')
+ itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
url_map[itag] = format_url
return url_map
@@ -768,15 +725,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
video_description = u''
# subtitles
- video_subtitles = None
-
- if self._downloader.params.get('writesubtitles', False) or self._downloader.params.get('allsubtitles', False):
- video_subtitles = self._extract_subtitles(video_id)
- elif self._downloader.params.get('writeautomaticsub', False):
- video_subtitles = self._request_automatic_caption(video_id, video_webpage)
+ video_subtitles = self.extract_subtitles(video_id, video_webpage)
if self._downloader.params.get('listsubtitles', False):
- self._list_available_subtitles(video_id)
+ self._list_available_subtitles(video_id, video_webpage)
return
if 'length_seconds' not in video_info:
generated by cgit v1.2.3 (git 2.26.2) at 2024-11-03 11:07:05 +0000