aboutsummaryrefslogtreecommitdiff
path: root/youtube_dl/extractor
diff options
context:
space:
mode:
Diffstat (limited to 'youtube_dl/extractor')
-rw-r--r--youtube_dl/extractor/__init__.py6
-rw-r--r--youtube_dl/extractor/collegehumor.py106
-rw-r--r--youtube_dl/extractor/comedycentral.py4
-rw-r--r--youtube_dl/extractor/common.py5
-rw-r--r--youtube_dl/extractor/dreisat.py2
-rw-r--r--youtube_dl/extractor/jpopsukitv.py73
-rw-r--r--youtube_dl/extractor/lynda.py102
-rw-r--r--youtube_dl/extractor/macgamestore.py43
-rw-r--r--youtube_dl/extractor/mixcloud.py2
-rw-r--r--youtube_dl/extractor/pornhd.py2
-rw-r--r--youtube_dl/extractor/soundcloud.py4
-rw-r--r--youtube_dl/extractor/wistia.py1
12 files changed, 282 insertions, 68 deletions
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index 5319a9292..21d564dba 100644
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -95,12 +95,18 @@ from .ivi import (
from .jeuxvideo import JeuxVideoIE
from .jukebox import JukeboxIE
from .justintv import JustinTVIE
+from .jpopsukitv import JpopsukiIE
from .kankan import KankanIE
from .keezmovies import KeezMoviesIE
from .kickstarter import KickStarterIE
from .keek import KeekIE
from .liveleak import LiveLeakIE
from .livestream import LivestreamIE, LivestreamOriginalIE
+from .lynda import (
+ LyndaIE,
+ LyndaCourseIE
+)
+from .macgamestore import MacGameStoreIE
from .mdr import MDRIE
from .metacafe import MetacafeIE
from .metacritic import MetacriticIE
diff --git a/youtube_dl/extractor/collegehumor.py b/youtube_dl/extractor/collegehumor.py
index b27c1dfc5..d10b7bd0c 100644
--- a/youtube_dl/extractor/collegehumor.py
+++ b/youtube_dl/extractor/collegehumor.py
@@ -1,82 +1,68 @@
+from __future__ import unicode_literals
+
+import json
import re
from .common import InfoExtractor
-from ..utils import (
- compat_urllib_parse_urlparse,
- determine_ext,
-
- ExtractorError,
-)
class CollegeHumorIE(InfoExtractor):
_VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/(video|embed|e)/(?P<videoid>[0-9]+)/?(?P<shorttitle>.*)$'
_TESTS = [{
- u'url': u'http://www.collegehumor.com/video/6902724/comic-con-cosplay-catastrophe',
- u'file': u'6902724.mp4',
- u'md5': u'1264c12ad95dca142a9f0bf7968105a0',
- u'info_dict': {
- u'title': u'Comic-Con Cosplay Catastrophe',
- u'description': u'Fans get creative this year at San Diego. Too creative. And yes, that\'s really Joss Whedon.',
+ 'url': 'http://www.collegehumor.com/video/6902724/comic-con-cosplay-catastrophe',
+ 'file': '6902724.mp4',
+ 'md5': 'dcc0f5c1c8be98dc33889a191f4c26bd',
+ 'info_dict': {
+ 'title': 'Comic-Con Cosplay Catastrophe',
+ 'description': 'Fans get creative this year at San Diego. Too',
+ 'age_limit': 13,
},
},
{
- u'url': u'http://www.collegehumor.com/video/3505939/font-conference',
- u'file': u'3505939.mp4',
- u'md5': u'c51ca16b82bb456a4397987791a835f5',
- u'info_dict': {
- u'title': u'Font Conference',
- u'description': u'This video wasn\'t long enough, so we made it double-spaced.',
+ 'url': 'http://www.collegehumor.com/video/3505939/font-conference',
+ 'file': '3505939.mp4',
+ 'md5': '72fa701d8ef38664a4dbb9e2ab721816',
+ 'info_dict': {
+ 'title': 'Font Conference',
+ 'description': 'This video wasn\'t long enough, so we made it double-spaced.',
+ 'age_limit': 10,
},
}]
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
- if mobj is None:
- raise ExtractorError(u'Invalid URL: %s' % url)
video_id = mobj.group('videoid')
- info = {
- 'id': video_id,
- 'uploader': None,
- 'upload_date': None,
- }
-
- self.report_extraction(video_id)
- xmlUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id
- mdoc = self._download_xml(xmlUrl, video_id,
- u'Downloading info XML',
- u'Unable to download video info XML')
+ jsonUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id + '.json'
+ data = json.loads(self._download_webpage(
+ jsonUrl, video_id, 'Downloading info JSON'))
+ vdata = data['video']
- try:
- videoNode = mdoc.findall('./video')[0]
- youtubeIdNode = videoNode.find('./youtubeID')
- if youtubeIdNode is not None:
- return self.url_result(youtubeIdNode.text, 'Youtube')
- info['description'] = videoNode.findall('./description')[0].text
- info['title'] = videoNode.findall('./caption')[0].text
- info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
- next_url = videoNode.findall('./file')[0].text
- except IndexError:
- raise ExtractorError(u'Invalid metadata XML file')
-
- if next_url.endswith(u'manifest.f4m'):
- manifest_url = next_url + '?hdcore=2.10.3'
- adoc = self._download_xml(manifest_url, video_id,
- u'Downloading XML manifest',
- u'Unable to download video info XML')
-
- try:
- video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text
- except IndexError:
- raise ExtractorError(u'Invalid manifest file')
- url_pr = compat_urllib_parse_urlparse(info['thumbnail'])
- info['url'] = url_pr.scheme + '://' + url_pr.netloc + video_id[:-2].replace('.csmil','').replace(',','')
- info['ext'] = 'mp4'
+ AGE_LIMITS = {'nc17': 18, 'r': 18, 'pg13': 13, 'pg': 10, 'g': 0}
+ rating = vdata.get('rating')
+ if rating:
+ age_limit = AGE_LIMITS.get(rating.lower())
else:
- # Old-style direct links
- info['url'] = next_url
- info['ext'] = determine_ext(info['url'])
+ age_limit = None # None = No idea
+
+ PREFS = {'high_quality': 2, 'low_quality': 0}
+ formats = []
+ for format_key in ('mp4', 'webm'):
+ for qname, qurl in vdata[format_key].items():
+ formats.append({
+ 'format_id': format_key + '_' + qname,
+ 'url': qurl,
+ 'format': format_key,
+ 'preference': PREFS.get(qname),
+ })
+ self._sort_formats(formats)
- return info
+ return {
+ 'id': video_id,
+ 'title': vdata['title'],
+ 'description': vdata.get('description'),
+ 'thumbnail': vdata.get('thumbnail'),
+ 'formats': formats,
+ 'age_limit': age_limit,
+ }
diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index a54ce3ee7..27bd8256e 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -12,7 +12,9 @@ from ..utils import (
class ComedyCentralIE(MTVServicesInfoExtractor):
- _VALID_URL = r'https?://(?:www.)?comedycentral.com/(video-clips|episodes|cc-studios)/(?P<title>.*)'
+ _VALID_URL = r'''(?x)https?://(?:www.)?comedycentral.com/
+ (video-clips|episodes|cc-studios|video-collections)
+ /(?P<title>.*)'''
_FEED_URL = u'http://comedycentral.com/feeds/mrss/'
_TEST = {
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index f34d36cb0..f498bcf6f 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -69,7 +69,8 @@ class InfoExtractor(object):
download, lower-case.
"http", "https", "rtsp", "rtmp" or so.
* preference Order number of this format. If this field is
- present, the formats get sorted by this field.
+ present and not None, the formats get sorted
+ by this field.
-1 for default (order by other properties),
-2 or smaller for less than default.
url: Final video URL.
@@ -377,7 +378,7 @@ class InfoExtractor(object):
@staticmethod
def _og_regexes(prop):
content_re = r'content=(?:"([^>]+?)"|\'(.+?)\')'
- property_re = r'property=[\'"]og:%s[\'"]' % re.escape(prop)
+ property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
template = r'<meta[^>]+?%s[^>]+?%s'
return [
template % (property_re, content_re),
diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index b0e4300bd..0b11d1f10 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -10,7 +10,7 @@ from ..utils import (
class DreiSatIE(InfoExtractor):
IE_NAME = '3sat'
- _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/index\.php\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
+ _VALID_URL = r'(?:http://)?(?:www\.)?3sat\.de/mediathek/(?:index\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)$'
_TEST = {
u"url": u"http://www.3sat.de/mediathek/index.php?obj=36983",
u'file': u'36983.mp4',
diff --git a/youtube_dl/extractor/jpopsukitv.py b/youtube_dl/extractor/jpopsukitv.py
new file mode 100644
index 000000000..aad782578
--- /dev/null
+++ b/youtube_dl/extractor/jpopsukitv.py
@@ -0,0 +1,73 @@
+# coding=utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+ int_or_none,
+ unified_strdate,
+)
+
+
+class JpopsukiIE(InfoExtractor):
+ IE_NAME = 'jpopsuki.tv'
+ _VALID_URL = r'https?://(?:www\.)?jpopsuki\.tv/video/(.*?)/(?P<id>\S+)'
+
+ _TEST = {
+ 'url': 'http://www.jpopsuki.tv/video/ayumi-hamasaki---evolution/00be659d23b0b40508169cdee4545771',
+ 'md5': '88018c0c1a9b1387940e90ec9e7e198e',
+ 'file': '00be659d23b0b40508169cdee4545771.mp4',
+ 'info_dict': {
+ 'id': '00be659d23b0b40508169cdee4545771',
+ 'title': 'ayumi hamasaki - evolution',
+ 'description': 'Release date: 2001.01.31\r\n浜崎あゆみ - evolution',
+ 'thumbnail': 'http://www.jpopsuki.tv/cache/89722c74d2a2ebe58bcac65321c115b2.jpg',
+ 'uploader': 'plama_chan',
+ 'uploader_id': '404',
+ 'upload_date': '20121101'
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id)
+
+ video_url = 'http://www.jpopsuki.tv' + self._html_search_regex(
+ r'<source src="(.*?)" type', webpage, 'video url')
+
+ video_title = self._og_search_title(webpage)
+ description = self._og_search_description(webpage)
+ thumbnail = self._og_search_thumbnail(webpage)
+ uploader = self._html_search_regex(
+ r'<li>from: <a href="/user/view/user/(.*?)/uid/',
+ webpage, 'video uploader', fatal=False)
+ uploader_id = self._html_search_regex(
+ r'<li>from: <a href="/user/view/user/\S*?/uid/(\d*)',
+ webpage, 'video uploader_id', fatal=False)
+ upload_date = self._html_search_regex(
+ r'<li>uploaded: (.*?)</li>', webpage, 'video upload_date',
+ fatal=False)
+ if upload_date is not None:
+ upload_date = unified_strdate(upload_date)
+ view_count_str = self._html_search_regex(
+ r'<li>Hits: ([0-9]+?)</li>', webpage, 'video view_count',
+ fatal=False)
+ comment_count_str = self._html_search_regex(
+ r'<h2>([0-9]+?) comments</h2>', webpage, 'video comment_count',
+ fatal=False)
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': video_title,
+ 'description': description,
+ 'thumbnail': thumbnail,
+ 'uploader': uploader,
+ 'uploader_id': uploader_id,
+ 'upload_date': upload_date,
+ 'view_count': int_or_none(view_count_str),
+ 'comment_count': int_or_none(comment_count_str),
+ }
diff --git a/youtube_dl/extractor/lynda.py b/youtube_dl/extractor/lynda.py
new file mode 100644
index 000000000..592ed747a
--- /dev/null
+++ b/youtube_dl/extractor/lynda.py
@@ -0,0 +1,102 @@
+from __future__ import unicode_literals
+
+import re
+import json
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class LyndaIE(InfoExtractor):
+ IE_NAME = 'lynda'
+ IE_DESC = 'lynda.com videos'
+ _VALID_URL = r'https?://www\.lynda\.com/[^/]+/[^/]+/\d+/(\d+)-\d\.html'
+
+ _TEST = {
+ 'url': 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html',
+ 'file': '114408.mp4',
+ 'md5': 'ecfc6862da89489161fb9cd5f5a6fac1',
+ u"info_dict": {
+ 'title': 'Using the exercise files',
+ 'duration': 68
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group(1)
+
+ page = self._download_webpage('http://www.lynda.com/ajax/player?videoId=%s&type=video' % video_id,
+ video_id, 'Downloading video JSON')
+ video_json = json.loads(page)
+
+ if 'Status' in video_json and video_json['Status'] == 'NotFound':
+ raise ExtractorError('Video %s does not exist' % video_id, expected=True)
+
+ if video_json['HasAccess'] is False:
+ raise ExtractorError('Video %s is only available for members' % video_id, expected=True)
+
+ video_id = video_json['ID']
+ duration = video_json['DurationInSeconds']
+ title = video_json['Title']
+
+ formats = [{'url': fmt['Url'],
+ 'ext': fmt['Extension'],
+ 'width': fmt['Width'],
+ 'height': fmt['Height'],
+ 'filesize': fmt['FileSize'],
+ 'format_id': fmt['Resolution']
+ } for fmt in video_json['Formats']]
+
+ self._sort_formats(formats)
+
+ return {
+ 'id': video_id,
+ 'title': title,
+ 'duration': duration,
+ 'formats': formats
+ }
+
+
+class LyndaCourseIE(InfoExtractor):
+ IE_NAME = 'lynda:course'
+ IE_DESC = 'lynda.com online courses'
+
+ # Course link equals to welcome/introduction video link of same course
+ # We will recognize it as course link
+ _VALID_URL = r'https?://(?:www|m)\.lynda\.com/(?P<coursepath>[^/]+/[^/]+/(?P<courseid>\d+))-\d\.html'
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ course_path = mobj.group('coursepath')
+ course_id = mobj.group('courseid')
+
+ page = self._download_webpage('http://www.lynda.com/ajax/player?courseId=%s&type=course' % course_id,
+ course_id, 'Downloading course JSON')
+ course_json = json.loads(page)
+
+ if 'Status' in course_json and course_json['Status'] == 'NotFound':
+ raise ExtractorError('Course %s does not exist' % course_id, expected=True)
+
+ unaccessible_videos = 0
+ videos = []
+
+ for chapter in course_json['Chapters']:
+ for video in chapter['Videos']:
+ if video['HasAccess'] is not True:
+ unaccessible_videos += 1
+ continue
+ videos.append(video['ID'])
+
+ if unaccessible_videos > 0:
+ self._downloader.report_warning('%s videos are only available for members and will not be downloaded' % unaccessible_videos)
+
+ entries = [
+ self.url_result('http://www.lynda.com/%s/%s-4.html' %
+ (course_path, video_id),
+ 'Lynda')
+ for video_id in videos]
+
+ course_title = course_json['Title']
+
+ return self.playlist_result(entries, course_id, course_title)
diff --git a/youtube_dl/extractor/macgamestore.py b/youtube_dl/extractor/macgamestore.py
new file mode 100644
index 000000000..b818cf50c
--- /dev/null
+++ b/youtube_dl/extractor/macgamestore.py
@@ -0,0 +1,43 @@
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import ExtractorError
+
+
+class MacGameStoreIE(InfoExtractor):
+ IE_NAME = 'macgamestore'
+ IE_DESC = 'MacGameStore trailers'
+ _VALID_URL = r'https?://www\.macgamestore\.com/mediaviewer\.php\?trailer=(?P<id>\d+)'
+
+ _TEST = {
+ 'url': 'http://www.macgamestore.com/mediaviewer.php?trailer=2450',
+ 'file': '2450.m4v',
+ 'md5': '8649b8ea684b6666b4c5be736ecddc61',
+ 'info_dict': {
+ 'title': 'Crow',
+ }
+ }
+
+ def _real_extract(self, url):
+ mobj = re.match(self._VALID_URL, url)
+ video_id = mobj.group('id')
+
+ webpage = self._download_webpage(url, video_id, 'Downloading trailer page')
+
+ if re.search(r'>Missing Media<', webpage) is not None:
+ raise ExtractorError('Trailer %s does not exist' % video_id, expected=True)
+
+ video_title = self._html_search_regex(
+ r'<title>MacGameStore: (.*?) Trailer</title>', webpage, 'title')
+
+ video_url = self._html_search_regex(
+ r'(?s)<div\s+id="video-player".*?href="([^"]+)"\s*>',
+ webpage, 'video URL')
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': video_title
+ }
diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 125d81551..7c54ea0f4 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -53,7 +53,7 @@ class MixcloudIE(InfoExtractor):
info = json.loads(json_data)
preview_url = self._search_regex(r'data-preview-url="(.+?)"', webpage, u'preview url')
- song_url = preview_url.replace('/previews/', '/cloudcasts/originals/')
+ song_url = preview_url.replace('/previews/', '/c/originals/')
template_url = re.sub(r'(stream\d*)', 'stream%d', song_url)
final_song_url = self._get_url(template_url)
if final_song_url is None:
diff --git a/youtube_dl/extractor/pornhd.py b/youtube_dl/extractor/pornhd.py
index 71abd5013..e9ff8d1af 100644
--- a/youtube_dl/extractor/pornhd.py
+++ b/youtube_dl/extractor/pornhd.py
@@ -5,7 +5,7 @@ from ..utils import compat_urllib_parse
class PornHdIE(InfoExtractor):
- _VALID_URL = r'(?:http://)?(?:www\.)?pornhd\.com/videos/(?P<video_id>[0-9]+)/(?P<video_title>.+)'
+ _VALID_URL = r'(?:http://)?(?:www\.)?pornhd\.com/(?:[a-z]{2,4}/)?videos/(?P<video_id>[0-9]+)/(?P<video_title>.+)'
_TEST = {
u'url': u'http://www.pornhd.com/videos/1962/sierra-day-gets-his-cum-all-over-herself-hd-porn-video',
u'file': u'1962.flv',
diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py
index e22ff9c38..951e977bd 100644
--- a/youtube_dl/extractor/soundcloud.py
+++ b/youtube_dl/extractor/soundcloud.py
@@ -29,7 +29,7 @@ class SoundcloudIE(InfoExtractor):
(?!sets/)(?P<title>[\w\d-]+)/?
(?P<token>[^?]+?)?(?:[?].*)?$)
|(?:api\.soundcloud\.com/tracks/(?P<track_id>\d+))
- |(?P<widget>w\.soundcloud\.com/player/?.*?url=.*)
+ |(?P<player>(?:w|player|p.)\.soundcloud\.com/player/?.*?url=.*)
)
'''
IE_NAME = u'soundcloud'
@@ -193,7 +193,7 @@ class SoundcloudIE(InfoExtractor):
if track_id is not None:
info_json_url = 'http://api.soundcloud.com/tracks/' + track_id + '.json?client_id=' + self._CLIENT_ID
full_title = track_id
- elif mobj.group('widget'):
+ elif mobj.group('player'):
query = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
return self.url_result(query['url'][0], ie='Soundcloud')
else:
diff --git a/youtube_dl/extractor/wistia.py b/youtube_dl/extractor/wistia.py
index 584550455..bc31c2e64 100644
--- a/youtube_dl/extractor/wistia.py
+++ b/youtube_dl/extractor/wistia.py
@@ -44,6 +44,7 @@ class WistiaIE(InfoExtractor):
'height': a['height'],
'filesize': a['size'],
'ext': a['ext'],
+ 'preference': 1 if atype == 'original' else None,
})
self._sort_formats(formats)
generated by cgit v1.2.3 (git 2.26.2) at 2024-09-15 23:39:24 +0000