diff options
Diffstat (limited to 'youtube_dl/extractor')
98 files changed, 3951 insertions, 1032 deletions
diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 17b695a56..de6e8ee30 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -1,5 +1,7 @@ +from .abc import ABCIE from .academicearth import AcademicEarthCourseIE from .addanime import AddAnimeIE +from .adultswim import AdultSwimIE from .aftonbladet import AftonbladetIE from .anitube import AnitubeIE from .aol import AolIE @@ -52,6 +54,7 @@ from .cnn import ( from .collegehumor import CollegeHumorIE from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE from .condenast import CondeNastIE +from .cracked import CrackedIE from .criterion import CriterionIE from .crunchyroll import CrunchyrollIE from .cspan import CSpanIE @@ -62,9 +65,11 @@ from .dailymotion import ( DailymotionUserIE, ) from .daum import DaumIE +from .dfb import DFBIE from .dotsub import DotsubIE from .dreisat import DreiSatIE from .drtv import DRTVIE +from .dump import DumpIE from .defense import DefenseGouvFrIE from .discovery import DiscoveryIE from .divxstage import DivxStageIE @@ -73,12 +78,17 @@ from .ebaumsworld import EbaumsWorldIE from .ehow import EHowIE from .eighttracks import EightTracksIE from .eitb import EitbIE +from .ellentv import ( + EllenTVIE, + EllenTVClipsIE, +) from .elpais import ElPaisIE from .empflix import EmpflixIE from .engadget import EngadgetIE from .escapist import EscapistIE from .everyonesmixtape import EveryonesMixtapeIE from .exfm import ExfmIE +from .expotv import ExpoTVIE from .extremetube import ExtremeTubeIE from .facebook import FacebookIE from .faz import FazIE @@ -111,18 +121,22 @@ from .gameone import ( GameOnePlaylistIE, ) from .gamespot import GameSpotIE +from .gamestar import GameStarIE from .gametrailers import GametrailersIE from .gdcvault import GDCVaultIE from .generic import GenericIE +from .godtube import GodTubeIE from .googleplus import GooglePlusIE from .googlesearch import GoogleSearchIE from .gorillavid import GorillaVidIE from .goshgay import GoshgayIE +from .grooveshark import GroovesharkIE from .hark import HarkIE from .helsinki import HelsinkiIE from .hentaistigma import HentaiStigmaIE from .hotnewhiphop import HotNewHipHopIE from .howcast import HowcastIE +from .howstuffworks import HowStuffWorksIE from .huffpost import HuffPostIE from .hypem import HypemIE from .iconosquare import IconosquareIE @@ -140,8 +154,10 @@ from .ivi import ( IviIE, IviCompilationIE ) +from .izlesene import IzleseneIE from .jadorecettepub import JadoreCettePubIE from .jeuxvideo import JeuxVideoIE +from .jove import JoveIE from .jukebox import JukeboxIE from .justintv import JustinTVIE from .jpopsukitv import JpopsukiIE @@ -151,6 +167,7 @@ from .khanacademy import KhanAcademyIE from .kickstarter import KickStarterIE from .keek import KeekIE from .kontrtube import KontrTubeIE +from .krasview import KrasViewIE from .ku6 import Ku6IE from .la7 import LA7IE from .lifenews import LifeNewsIE @@ -171,14 +188,19 @@ from .malemotion import MalemotionIE from .mdr import MDRIE from .metacafe import MetacafeIE from .metacritic import MetacriticIE +from .ministrygrid import MinistryGridIE from .mit import TechTVMITIE, MITIE, OCWMITIE +from .mitele import MiTeleIE from .mixcloud import MixcloudIE +from .mlb import MLBIE from .mpora import MporaIE from .mofosex import MofosexIE +from .mojvideo import MojvideoIE from .mooshare import MooshareIE from .morningstar import MorningstarIE from .motherless import MotherlessIE from .motorsport import MotorsportIE +from .movieclips import MovieClipsIE from .moviezine import MoviezineIE from .movshare import MovShareIE from .mtv import ( @@ -218,12 +240,17 @@ from .nrk import ( from .ntv import NTVIE from .nytimes import NYTimesIE from .nuvid import NuvidIE -from .oe1 import OE1IE from .ooyala import OoyalaIE -from .orf import ORFIE +from .orf import ( + ORFTVthekIE, + ORFOE1IE, + ORFFM4IE, +) from .parliamentliveuk import ParliamentLiveUKIE +from .patreon import PatreonIE from .pbs import PBSIE from .photobucket import PhotobucketIE +from .playfm import PlayFMIE from .playvid import PlayvidIE from .podomatic import PodomaticIE from .pornhd import PornHdIE @@ -241,9 +268,10 @@ from .ro220 import Ro220IE from .rottentomatoes import RottenTomatoesIE from .roxwel import RoxwelIE from .rtbf import RTBFIE +from .rtlnl import RtlXlIE from .rtlnow import RTLnowIE from .rts import RTSIE -from .rtve import RTVEALaCartaIE +from .rtve import RTVEALaCartaIE, RTVELiveIE from .ruhd import RUHDIE from .rutube import ( RutubeIE, @@ -252,10 +280,13 @@ from .rutube import ( RutubePersonIE, ) from .rutv import RUTVIE +from .sapo import SapoIE from .savefrom import SaveFromIE +from .sbs import SBSIE from .scivee import SciVeeIE from .screencast import ScreencastIE from .servingsys import ServingSysIE +from .shared import SharedIE from .sina import SinaIE from .slideshare import SlideshareIE from .slutload import SlutloadIE @@ -265,6 +296,8 @@ from .smotri import ( SmotriUserIE, SmotriBroadcastIE, ) +from .snotr import SnotrIE +from .sockshare import SockshareIE from .sohu import SohuIE from .soundcloud import ( SoundcloudIE, @@ -273,8 +306,8 @@ from .soundcloud import ( SoundcloudPlaylistIE ) from .soundgasm import SoundgasmIE -from .southparkstudios import ( - SouthParkStudiosIE, +from .southpark import ( + SouthParkIE, SouthparkDeIE, ) from .space import SpaceIE @@ -316,6 +349,8 @@ from .tumblr import TumblrIE from .tutv import TutvIE from .tvigle import TvigleIE from .tvp import TvpIE +from .tvplay import TVPlayIE +from .ubu import UbuIE from .udemy import ( UdemyIE, UdemyCourseIE @@ -337,6 +372,7 @@ from .videofyme import VideofyMeIE from .videopremium import VideoPremiumIE from .videott import VideoTtIE from .videoweed import VideoWeedIE +from .vidme import VidmeIE from .vimeo import ( VimeoIE, VimeoChannelIE, @@ -359,6 +395,7 @@ from .vuclip import VuClipIE from .vulture import VultureIE from .washingtonpost import WashingtonPostIE from .wat import WatIE +from .wayofthemaster import WayOfTheMasterIE from .wdr import ( WDRIE, WDRMobileIE, @@ -370,6 +407,7 @@ from .wistia import WistiaIE from .worldstarhiphop import WorldStarHipHopIE from .wrzuta import WrzutaIE from .xbef import XBefIE +from .xboxclips import XboxClipsIE from .xhamster import XHamsterIE from .xnxx import XNXXIE from .xvideos import XVideosIE @@ -399,6 +437,7 @@ from .youtube import ( YoutubeUserIE, YoutubeWatchLaterIE, ) + from .zdf import ZDFIE diff --git a/youtube_dl/extractor/abc.py b/youtube_dl/extractor/abc.py new file mode 100644 index 000000000..7d89f44ee --- /dev/null +++ b/youtube_dl/extractor/abc.py @@ -0,0 +1,48 @@ +from __future__ import unicode_literals + +import re +import json + +from .common import InfoExtractor + + +class ABCIE(InfoExtractor): + IE_NAME = 'abc.net.au' + _VALID_URL = r'http://www\.abc\.net\.au/news/[^/]+/[^/]+/(?P<id>\d+)' + + _TEST = { + 'url': 'http://www.abc.net.au/news/2014-07-25/bringing-asylum-seekers-to-australia-would-give/5624716', + 'md5': 'dad6f8ad011a70d9ddf887ce6d5d0742', + 'info_dict': { + 'id': '5624716', + 'ext': 'mp4', + 'title': 'Bringing asylum seekers to Australia would give them right to asylum claims: professor', + 'description': 'md5:ba36fa5e27e5c9251fd929d339aea4af', + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + webpage = self._download_webpage(url, video_id) + + urls_info_json = self._search_regex( + r'inlineVideoData\.push\((.*?)\);', webpage, 'video urls', + flags=re.DOTALL) + urls_info = json.loads(urls_info_json.replace('\'', '"')) + formats = [{ + 'url': url_info['url'], + 'width': int(url_info['width']), + 'height': int(url_info['height']), + 'tbr': int(url_info['bitrate']), + 'filesize': int(url_info['filesize']), + } for url_info in urls_info] + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': self._og_search_title(webpage), + 'formats': formats, + 'description': self._og_search_description(webpage), + 'thumbnail': self._og_search_thumbnail(webpage), + } diff --git a/youtube_dl/extractor/adultswim.py b/youtube_dl/extractor/adultswim.py new file mode 100644 index 000000000..a00bfcb35 --- /dev/null +++ b/youtube_dl/extractor/adultswim.py @@ -0,0 +1,139 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor + +class AdultSwimIE(InfoExtractor): + _VALID_URL = r'https?://video\.adultswim\.com/(?P<path>.+?)(?:\.html)?(?:\?.*)?(?:#.*)?$' + _TEST = { + 'url': 'http://video.adultswim.com/rick-and-morty/close-rick-counters-of-the-rick-kind.html?x=y#title', + 'playlist': [ + { + 'md5': '4da359ec73b58df4575cd01a610ba5dc', + 'info_dict': { + 'id': '8a250ba1450996e901453d7f02ca02f5', + 'ext': 'flv', + 'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 1', + 'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?', + 'uploader': 'Rick and Morty', + 'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg' + } + }, + { + 'md5': 'ffbdf55af9331c509d95350bd0cc1819', + 'info_dict': { + 'id': '8a250ba1450996e901453d7f4bd102f6', + 'ext': 'flv', + 'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 2', + 'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?', + 'uploader': 'Rick and Morty', + 'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg' + } + }, + { + 'md5': 'b92409635540304280b4b6c36bd14a0a', + 'info_dict': { + 'id': '8a250ba1450996e901453d7fa73c02f7', + 'ext': 'flv', + 'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 3', + 'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?', + 'uploader': 'Rick and Morty', + 'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg' + } + }, + { + 'md5': 'e8818891d60e47b29cd89d7b0278156d', + 'info_dict': { + 'id': '8a250ba1450996e901453d7fc8ba02f8', + 'ext': 'flv', + 'title': 'Rick and Morty Close Rick-Counters of the Rick Kind part 4', + 'description': 'Rick has a run in with some old associates, resulting in a fallout with Morty. You got any chips, broh?', + 'uploader': 'Rick and Morty', + 'thumbnail': 'http://i.cdn.turner.com/asfix/repository/8a250ba13f865824013fc9db8b6b0400/thumbnail_267549017116827057.jpg' + } + } + ] + } + + _video_extensions = { + '3500': 'flv', + '640': 'mp4', + '150': 'mp4', + 'ipad': 'm3u8', + 'iphone': 'm3u8' + } + _video_dimensions = { + '3500': (1280, 720), + '640': (480, 270), + '150': (320, 180) + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_path = mobj.group('path') + + webpage = self._download_webpage(url, video_path) + episode_id = self._html_search_regex(r'<link rel="video_src" href="http://i\.adultswim\.com/adultswim/adultswimtv/tools/swf/viralplayer.swf\?id=([0-9a-f]+?)"\s*/?\s*>', webpage, 'episode_id') + title = self._og_search_title(webpage) + + index_url = 'http://asfix.adultswim.com/asfix-svc/episodeSearch/getEpisodesByIDs?networkName=AS&ids=%s' % episode_id + idoc = self._download_xml(index_url, title, 'Downloading episode index', 'Unable to download episode index') + + episode_el = idoc.find('.//episode') + show_title = episode_el.attrib.get('collectionTitle') + episode_title = episode_el.attrib.get('title') + thumbnail = episode_el.attrib.get('thumbnailUrl') + description = episode_el.find('./description').text.strip() + + entries = [] + segment_els = episode_el.findall('./segments/segment') + + for part_num, segment_el in enumerate(segment_els): + segment_id = segment_el.attrib.get('id') + segment_title = '%s %s part %d' % (show_title, episode_title, part_num + 1) + thumbnail = segment_el.attrib.get('thumbnailUrl') + duration = segment_el.attrib.get('duration') + + segment_url = 'http://asfix.adultswim.com/asfix-svc/episodeservices/getCvpPlaylist?networkName=AS&id=%s' % segment_id + idoc = self._download_xml(segment_url, segment_title, 'Downloading segment information', 'Unable to download segment information') + + formats = [] + file_els = idoc.findall('.//files/file') + + for file_el in file_els: + bitrate = file_el.attrib.get('bitrate') + type = file_el.attrib.get('type') + width, height = self._video_dimensions.get(bitrate, (None, None)) + formats.append({ + 'format_id': '%s-%s' % (bitrate, type), + 'url': file_el.text, + 'ext': self._video_extensions.get(bitrate, 'mp4'), + # The bitrate may not be a number (for example: 'iphone') + 'tbr': int(bitrate) if bitrate.isdigit() else None, + 'height': height, + 'width': width + }) + + self._sort_formats(formats) + + entries.append({ + 'id': segment_id, + 'title': segment_title, + 'formats': formats, + 'uploader': show_title, + 'thumbnail': thumbnail, + 'duration': duration, + 'description': description + }) + + return { + '_type': 'playlist', + 'id': episode_id, + 'display_id': video_path, + 'entries': entries, + 'title': '%s %s' % (show_title, episode_title), + 'description': description, + 'thumbnail': thumbnail + } diff --git a/youtube_dl/extractor/allocine.py b/youtube_dl/extractor/allocine.py index 34f0cd49b..7bd797884 100644 --- a/youtube_dl/extractor/allocine.py +++ b/youtube_dl/extractor/allocine.py @@ -32,7 +32,7 @@ class AllocineIE(InfoExtractor): 'id': '19540403', 'ext': 'mp4', 'title': 'Planes 2 Bande-annonce VF', - 'description': 'md5:c4b1f7bd682a91de6491ada267ec0f4d', + 'description': 'md5:eeaffe7c2d634525e21159b93acf3b1e', 'thumbnail': 're:http://.*\.jpg', }, }, { @@ -42,7 +42,7 @@ class AllocineIE(InfoExtractor): 'id': '19544709', 'ext': 'mp4', 'title': 'Dragons 2 - Bande annonce finale VF', - 'description': 'md5:e74a4dc750894bac300ece46c7036490', + 'description': 'md5:71742e3a74b0d692c7fce0dd2017a4ac', 'thumbnail': 're:http://.*\.jpg', }, }] diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py index 7e93bc4df..748608826 100644 --- a/youtube_dl/extractor/aparat.py +++ b/youtube_dl/extractor/aparat.py @@ -1,5 +1,7 @@ #coding: utf-8 +from __future__ import unicode_literals + import re from .common import InfoExtractor @@ -13,13 +15,14 @@ class AparatIE(InfoExtractor): _VALID_URL = r'^https?://(?:www\.)?aparat\.com/(?:v/|video/video/embed/videohash/)(?P<id>[a-zA-Z0-9]+)' _TEST = { - u'url': u'http://www.aparat.com/v/wP8On', - u'file': u'wP8On.mp4', - u'md5': u'6714e0af7e0d875c5a39c4dc4ab46ad1', - u'info_dict': { - u"title": u"تیم گلکسی 11 - زومیت", + 'url': 'http://www.aparat.com/v/wP8On', + 'md5': '6714e0af7e0d875c5a39c4dc4ab46ad1', + 'info_dict': { + 'id': 'wP8On', + 'ext': 'mp4', + 'title': 'تیم گلکسی 11 - زومیت', }, - #u'skip': u'Extremely unreliable', + # 'skip': 'Extremely unreliable', } def _real_extract(self, url): @@ -29,8 +32,8 @@ class AparatIE(InfoExtractor): # Note: There is an easier-to-parse configuration at # http://www.aparat.com/video/video/config/videohash/%video_id # but the URL in there does not work - embed_url = (u'http://www.aparat.com/video/video/embed/videohash/' + - video_id + u'/vt/frame') + embed_url = ('http://www.aparat.com/video/video/embed/videohash/' + + video_id + '/vt/frame') webpage = self._download_webpage(embed_url, video_id) video_urls = re.findall(r'fileList\[[0-9]+\]\s*=\s*"([^"]+)"', webpage) diff --git a/youtube_dl/extractor/appletrailers.py b/youtube_dl/extractor/appletrailers.py index dc8657b67..4359b88d1 100644 --- a/youtube_dl/extractor/appletrailers.py +++ b/youtube_dl/extractor/appletrailers.py @@ -6,6 +6,7 @@ import json from .common import InfoExtractor from ..utils import ( compat_urlparse, + int_or_none, ) @@ -110,8 +111,8 @@ class AppleTrailersIE(InfoExtractor): formats.append({ 'url': format_url, 'format': format['type'], - 'width': format['width'], - 'height': int(format['height']), + 'width': int_or_none(format['width']), + 'height': int_or_none(format['height']), }) self._sort_formats(formats) diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py index b36a4d46a..7f0da8ab6 100644 --- a/youtube_dl/extractor/ard.py +++ b/youtube_dl/extractor/ard.py @@ -7,23 +7,34 @@ from .common import InfoExtractor from ..utils import ( determine_ext, ExtractorError, + qualities, + compat_urllib_parse_urlparse, + compat_urllib_parse, ) class ARDIE(InfoExtractor): - _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P<video_id>[^/\?]+)(?:\?.*)?' + _VALID_URL = r'^https?://(?:(?:www\.)?ardmediathek\.de|mediathek\.daserste\.de)/(?:.*/)(?P<video_id>[0-9]+|[^0-9][^/\?]+)[^/\?]*(?:\?.*)?' - _TEST = { - 'url': 'http://www.ardmediathek.de/das-erste/guenther-jauch/edward-snowden-im-interview-held-oder-verraeter?documentId=19288786', - 'file': '19288786.mp4', - 'md5': '515bf47ce209fb3f5a61b7aad364634c', + _TESTS = [{ + 'url': 'http://mediathek.daserste.de/sendungen_a-z/328454_anne-will/22429276_vertrauen-ist-gut-spionieren-ist-besser-geht', + 'file': '22429276.mp4', + 'md5': '469751912f1de0816a9fc9df8336476c', 'info_dict': { - 'title': 'Edward Snowden im Interview - Held oder Verräter?', - 'description': 'Edward Snowden hat alles aufs Spiel gesetzt, um die weltweite \xdcberwachung durch die Geheimdienste zu enttarnen. Nun stellt sich der ehemalige NSA-Mitarbeiter erstmals weltweit in einem TV-Interview den Fragen eines NDR-Journalisten. Die Sendung vom Sonntagabend.', - 'thumbnail': 'http://www.ardmediathek.de/ard/servlet/contentblob/19/28/87/90/19288790/bild/2250037', + 'title': 'Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?', + 'description': 'Das Erste Mediathek [ARD]: Vertrauen ist gut, Spionieren ist besser - Geht so deutsch-amerikanische Freundschaft?, Anne Will, Über die Spionage-Affäre diskutieren Clemens Binninger, Katrin Göring-Eckardt, Georg Mascolo, Andrew B. Denison und Constanze Kurz.. Das Video zur Sendung Anne Will am Mittwoch, 16.07.2014', }, 'skip': 'Blocked outside of Germany', - } + }, { + 'url': 'http://www.ardmediathek.de/tv/Tatort/Das-Wunder-von-Wolbeck-Video-tgl-ab-20/Das-Erste/Video?documentId=22490580&bcastId=602916', + 'info_dict': { + 'id': '22490580', + 'ext': 'mp4', + 'title': 'Das Wunder von Wolbeck (Video tgl. ab 20 Uhr)', + 'description': 'Auf einem restaurierten Hof bei Wolbeck wird der Heilpraktiker Raffael Lembeck eines morgens von seiner Frau Stella tot aufgefunden. Das Opfer war offensichtlich in seiner Praxis zu Fall gekommen und ist dann verblutet, erklärt Prof. Boerne am Tatort.', + }, + 'skip': 'Blocked outside of Germany', + }] def _real_extract(self, url): # determine video id from url @@ -35,48 +46,78 @@ class ARDIE(InfoExtractor): else: video_id = m.group('video_id') + urlp = compat_urllib_parse_urlparse(url) + url = urlp._replace(path=compat_urllib_parse.quote(urlp.path.encode('utf-8'))).geturl() + webpage = self._download_webpage(url, video_id) + if '>Der gewünschte Beitrag ist nicht mehr verfügbar.<' in webpage: + raise ExtractorError('Video %s is no longer available' % video_id, expected=True) + title = self._html_search_regex( [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>', r'<meta name="dcterms.title" content="(.*?)"/>', r'<h4 class="headline">(.*?)</h4>'], webpage, 'title') description = self._html_search_meta( - 'dcterms.abstract', webpage, 'description') - thumbnail = self._og_search_thumbnail(webpage) - - - media_info = self._download_json( - 'http://www.ardmediathek.de/play/media/%s' % video_id, video_id) - # The second element of the _mediaArray contains the standard http urls - streams = media_info['_mediaArray'][1]['_mediaStreamArray'] - if not streams: - if '"fsk"' in webpage: - raise ExtractorError('This video is only available after 20:00') - - formats = [] - - for s in streams: - if type(s['_stream']) == list: - for index, url in enumerate(s['_stream'][::-1]): - quality = s['_quality'] + index - formats.append({ - 'quality': quality, - 'url': url, - 'format_id': '%s-%s' % (determine_ext(url), quality) + 'dcterms.abstract', webpage, 'description', default=None) + if description is None: + description = self._html_search_meta( + 'description', webpage, 'meta description') + + # Thumbnail is sometimes not present. + # It is in the mobile version, but that seems to use a different URL + # structure altogether. + thumbnail = self._og_search_thumbnail(webpage, default=None) + + media_streams = re.findall(r'''(?x) + mediaCollection\.addMediaStream\([0-9]+,\s*[0-9]+,\s*"[^"]*",\s* + "([^"]+)"''', webpage) + + if media_streams: + QUALITIES = qualities(['lo', 'hi', 'hq']) + formats = [] + for furl in set(media_streams): + if furl.endswith('.f4m'): + fid = 'f4m' + else: + fid_m = re.match(r'.*\.([^.]+)\.[^.]+$', furl) + fid = fid_m.group(1) if fid_m else None + formats.append({ + 'quality': QUALITIES(fid), + 'format_id': fid, + 'url': furl, + }) + else: # request JSON file + media_info = self._download_json( + 'http://www.ardmediathek.de/play/media/%s' % video_id, video_id) + # The second element of the _mediaArray contains the standard http urls + streams = media_info['_mediaArray'][1]['_mediaStreamArray'] + if not streams: + if '"fsk"' in webpage: + raise ExtractorError('This video is only available after 20:00') + + formats = [] + for s in streams: + if type(s['_stream']) == list: + for index, url in enumerate(s['_stream'][::-1]): + quality = s['_quality'] + index + formats.append({ + 'quality': quality, + 'url': url, + 'format_id': '%s-%s' % (determine_ext(url), quality) }) - continue + continue - format = { - 'quality': s['_quality'], - 'url': s['_stream'], - } + format = { + 'quality': s['_quality'], + 'url': s['_stream'], + } - format['format_id'] = '%s-%s' % ( - determine_ext(format['url']), format['quality']) + format['format_id'] = '%s-%s' % ( + determine_ext(format['url']), format['quality']) - formats.append(format) + formats.append(format) self._sort_formats(formats) diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 9591bad8a..1c72b2ff6 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -109,15 +109,19 @@ class ArteTVPlus7IE(InfoExtractor): regexes = [r'VO?%s' % l, r'VO?.-ST%s' % l] return any(re.match(r, f['versionCode']) for r in regexes) # Some formats may not be in the same language as the url + # TODO: Might want not to drop videos that does not match requested language + # but to process those formats with lower precedence formats = filter(_match_lang, all_formats) - formats = list(formats) # in python3 filter returns an iterator + formats = list(formats) # in python3 filter returns an iterator if not formats: # Some videos are only available in the 'Originalversion' # they aren't tagged as being in French or German - if all(f['versionCode'] == 'VO' or f['versionCode'] == 'VA' for f in all_formats): - formats = all_formats - else: - raise ExtractorError(u'The formats list is empty') + # Sometimes there are neither videos of requested lang code + # nor original version videos available + # For such cases we just take all_formats as is + formats = all_formats + if not formats: + raise ExtractorError('The formats list is empty') if re.match(r'[A-Z]Q', formats[0]['quality']) is not None: def sort_key(f): @@ -173,16 +177,26 @@ class ArteTVPlus7IE(InfoExtractor): # It also uses the arte_vp_url url from the webpage to extract the information class ArteTVCreativeIE(ArteTVPlus7IE): IE_NAME = 'arte.tv:creative' - _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de)/magazine?/(?P<id>.+)' + _VALID_URL = r'https?://creative\.arte\.tv/(?P<lang>fr|de)/(?:magazine?/)?(?P<id>[^?#]+)' - _TEST = { + _TESTS = [{ 'url': 'http://creative.arte.tv/de/magazin/agentur-amateur-corporate-design', 'info_dict': { - 'id': '050489-002', + 'id': '72176', 'ext': 'mp4', - 'title': 'Agentur Amateur / Agence Amateur #2 : Corporate Design', + 'title': 'Folge 2 - Corporate Design', + 'upload_date': '20131004', }, - } + }, { + 'url': 'http://creative.arte.tv/fr/Monty-Python-Reunion', + 'info_dict': { + 'id': '160676', + 'ext': 'mp4', + 'title': 'Monty Python live (mostly)', + 'description': 'Événement ! Quarante-cinq ans après leurs premiers succès, les légendaires Monty Python remontent sur scène.\n', + 'upload_date': '20140805', + } + }] class ArteTVFutureIE(ArteTVPlus7IE): diff --git a/youtube_dl/extractor/blinkx.py b/youtube_dl/extractor/blinkx.py index 7d558e262..3e461e715 100644 --- a/youtube_dl/extractor/blinkx.py +++ b/youtube_dl/extractor/blinkx.py @@ -52,7 +52,7 @@ class BlinkxIE(InfoExtractor): 'height': int(m['h']), }) elif m['type'] == 'original': - duration = m['d'] + duration = float(m['d']) elif m['type'] == 'youtube': yt_id = m['link'] self.to_screen('Youtube video detected: %s' % yt_id) diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py index acfc4ad73..261ead98f 100644 --- a/youtube_dl/extractor/bliptv.py +++ b/youtube_dl/extractor/bliptv.py @@ -15,7 +15,7 @@ from ..utils import ( class BlipTVIE(SubtitlesInfoExtractor): - _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P<id>\d+)|((?:play/|api\.swf#)(?P<lookup_id>[\da-zA-Z+]+)))' + _VALID_URL = r'https?://(?:\w+\.)?blip\.tv/(?:(?:.+-|rss/flash/)(?P<id>\d+)|((?:play/|api\.swf#)(?P<lookup_id>[\da-zA-Z+_TESTS]+)))' _TESTS = [ { @@ -49,6 +49,21 @@ class BlipTVIE(SubtitlesInfoExtractor): 'uploader_id': '792887', 'duration': 279, } + }, + { + # https://bugzilla.redhat.com/show_bug.cgi?id=967465 + 'url': 'http://a.blip.tv/api.swf#h6Uag5KbVwI', + 'md5': '314e87b1ebe7a48fcbfdd51b791ce5a6', + 'info_dict': { + 'id': '6573122', + 'ext': 'mov', + 'upload_date': '20130520', + 'description': 'Two hapless space marines argue over what to do when they realize they have an astronomically huge problem on their hands.', + 'title': 'Red vs. Blue Season 11 Trailer', + 'timestamp': 1369029609, + 'uploader': 'redvsblue', + 'uploader_id': '792887', + } } ] @@ -150,7 +165,7 @@ class BlipTVIE(SubtitlesInfoExtractor): class BlipTVUserIE(InfoExtractor): - _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$' + _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)(?!api\.swf)([^/]+)/*$' _PAGE_SIZE = 12 IE_NAME = 'blip.tv:user' diff --git a/youtube_dl/extractor/bloomberg.py b/youtube_dl/extractor/bloomberg.py index 25fb79e14..c51a97ce4 100644 --- a/youtube_dl/extractor/bloomberg.py +++ b/youtube_dl/extractor/bloomberg.py @@ -10,7 +10,7 @@ class BloombergIE(InfoExtractor): _TEST = { 'url': 'http://www.bloomberg.com/video/shah-s-presentation-on-foreign-exchange-strategies-qurhIVlJSB6hzkVi229d8g.html', - 'md5': '7bf08858ff7c203c870e8a6190e221e5', + # The md5 checksum changes 'info_dict': { 'id': 'qurhIVlJSB6hzkVi229d8g', 'ext': 'flv', @@ -31,8 +31,7 @@ class BloombergIE(InfoExtractor): return { 'id': name.split('-')[-1], 'title': title, - 'url': f4m_url, - 'ext': 'flv', + 'formats': self._extract_f4m_formats(f4m_url, name), 'description': self._og_search_description(webpage), 'thumbnail': self._og_search_thumbnail(webpage), } diff --git a/youtube_dl/extractor/br.py b/youtube_dl/extractor/br.py index 993360714..86f0c2861 100644 --- a/youtube_dl/extractor/br.py +++ b/youtube_dl/extractor/br.py @@ -7,12 +7,13 @@ from .common import InfoExtractor from ..utils import ( ExtractorError, int_or_none, + parse_duration, ) class BRIE(InfoExtractor): IE_DESC = 'Bayerischer Rundfunk Mediathek' - _VALID_URL = r'https?://(?:www\.)?br\.de/(?:[a-z0-9\-]+/)+(?P<id>[a-z0-9\-]+)\.html' + _VALID_URL = r'https?://(?:www\.)?br\.de/(?:[a-z0-9\-_]+/)+(?P<id>[a-z0-9\-_]+)\.html' _BASE_URL = 'http://www.br.de' _TESTS = [ @@ -22,8 +23,9 @@ class BRIE(InfoExtractor): 'info_dict': { 'id': '25e279aa-1ffd-40fd-9955-5325bd48a53a', 'ext': 'mp4', - 'title': 'Am 1. und 2. August in Oberammergau', - 'description': 'md5:dfd224e5aa6819bc1fcbb7826a932021', + 'title': 'Wenn das Traditions-Theater wackelt', + 'description': 'Heimatsound-Festival 2014: Wenn das Traditions-Theater wackelt', + 'duration': 34, } }, { @@ -34,6 +36,7 @@ class BRIE(InfoExtractor): 'ext': 'mp4', 'title': 'Über den Pass', 'description': 'Die Eroberung der Alpen: Über den Pass', + 'duration': 2588, } }, { @@ -44,6 +47,7 @@ class BRIE(InfoExtractor): 'ext': 'aac', 'title': '"Keine neuen Schulden im nächsten Jahr"', 'description': 'Haushaltsentwurf: "Keine neuen Schulden im nächsten Jahr"', + 'duration': 64, } }, { @@ -54,6 +58,7 @@ class BRIE(InfoExtractor): 'ext': 'mp4', 'title': 'Umweltbewusster Häuslebauer', 'description': 'Uwe Erdelt: Umweltbewusster Häuslebauer', + 'duration': 116, } }, { @@ -64,6 +69,7 @@ class BRIE(InfoExtractor): 'ext': 'mp4', 'title': 'Folge 1 - Metaphysik', 'description': 'Kant für Anfänger: Folge 1 - Metaphysik', + 'duration': 893, 'uploader': 'Eva Maria Steimle', 'upload_date': '20140117', } @@ -84,6 +90,7 @@ class BRIE(InfoExtractor): media = { 'id': xml_media.get('externalId'), 'title': xml_media.find('title').text, + 'duration': parse_duration(xml_media.find('duration').text), 'formats': self._extract_formats(xml_media.find('assets')), 'thumbnails': self._extract_thumbnails(xml_media.find('teaserImage/variants')), 'description': ' '.join(xml_media.find('shareTitle').text.splitlines()), diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index 419951b62..294670386 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -154,12 +154,14 @@ class BrightcoveIE(InfoExtractor): def _extract_brightcove_urls(cls, webpage): """Return a list of all Brightcove URLs from the webpage """ - url_m = re.search(r'<meta\s+property="og:video"\s+content="(http://c.brightcove.com/[^"]+)"', webpage) + url_m = re.search( + r'<meta\s+property="og:video"\s+content="(https?://(?:secure|c)\.brightcove.com/[^"]+)"', + webpage) if url_m: url = unescapeHTML(url_m.group(1)) # Some sites don't add it, we can't download with this url, for example: # http://www.ktvu.com/videos/news/raw-video-caltrain-releases-video-of-man-almost/vCTZdY/ - if 'playerKey' in url: + if 'playerKey' in url or 'videoId' in url: return [url] matches = re.findall( @@ -188,9 +190,13 @@ class BrightcoveIE(InfoExtractor): referer = smuggled_data.get('Referer', url) return self._get_video_info( videoPlayer[0], query_str, query, referer=referer) - else: + elif 'playerKey' in query: player_key = query['playerKey'] return self._get_playlist_info(player_key[0]) + else: + raise ExtractorError( + 'Cannot find playerKey= variable. Did you forget quotes in a shell invocation?', + expected=True) def _get_video_info(self, video_id, query_str, query, referer=None): request_url = self._FEDERATED_URL_TEMPLATE % query_str @@ -202,6 +208,13 @@ class BrightcoveIE(InfoExtractor): req.add_header('Referer', referer) webpage = self._download_webpage(req, video_id) + error_msg = self._html_search_regex( + r"<h1>We're sorry.</h1>\s*<p>(.*?)</p>", webpage, + 'error message', default=None) + if error_msg is not None: + raise ExtractorError( + 'brightcove said: %s' % error_msg, expected=True) + self.report_extraction(video_id) info = self._search_regex(r'var experienceJSON = ({.*});', webpage, 'json') info = json.loads(info)['data'] diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py index ac0315853..822f9a7be 100644 --- a/youtube_dl/extractor/cbs.py +++ b/youtube_dl/extractor/cbs.py @@ -1,24 +1,42 @@ +from __future__ import unicode_literals + import re from .common import InfoExtractor class CBSIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?cbs\.com/shows/[^/]+/video/(?P<id>[^/]+)/.*' + _VALID_URL = r'https?://(?:www\.)?cbs\.com/shows/[^/]+/(?:video|artist)/(?P<id>[^/]+)/.*' - _TEST = { - u'url': u'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/', - u'file': u'4JUVEwq3wUT7.flv', - u'info_dict': { - u'title': u'Connect Chat feat. Garth Brooks', - u'description': u'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!', - u'duration': 1495, + _TESTS = [{ + 'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/', + 'info_dict': { + 'id': '4JUVEwq3wUT7', + 'ext': 'flv', + 'title': 'Connect Chat feat. Garth Brooks', + 'description': 'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!', + 'duration': 1495, + }, + 'params': { + # rtmp download + 'skip_download': True, + }, + '_skip': 'Blocked outside the US', + }, { + 'url': 'http://www.cbs.com/shows/liveonletterman/artist/221752/st-vincent/', + 'info_dict': { + 'id': 'P9gjWjelt6iP', + 'ext': 'flv', + 'title': 'Live on Letterman - St. Vincent', + 'description': 'Live On Letterman: St. Vincent in concert from New York\'s Ed Sullivan Theater on Tuesday, July 16, 2014.', + 'duration': 3221, }, - u'params': { + 'params': { # rtmp download - u'skip_download': True, + 'skip_download': True, }, - } + '_skip': 'Blocked outside the US', + }] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) @@ -26,5 +44,5 @@ class CBSIE(InfoExtractor): webpage = self._download_webpage(url, video_id) real_id = self._search_regex( r"video\.settings\.pid\s*=\s*'([^']+)';", - webpage, u'real video ID') + webpage, 'real video ID') return self.url_result(u'theplatform:%s' % real_id) diff --git a/youtube_dl/extractor/chilloutzone.py b/youtube_dl/extractor/chilloutzone.py index 02d5ba527..a62395d4b 100644 --- a/youtube_dl/extractor/chilloutzone.py +++ b/youtube_dl/extractor/chilloutzone.py @@ -42,7 +42,7 @@ class ChilloutzoneIE(InfoExtractor): 'id': '85523671', 'ext': 'mp4', 'title': 'The Sunday Times - Icons', - 'description': 'md5:3e1c0dc6047498d6728dcdaad0891762', + 'description': 'md5:a5f7ff82e2f7a9ed77473fe666954e84', 'uploader': 'Us', 'uploader_id': 'usfilms', 'upload_date': '20140131' diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py index a94f42571..710d5009b 100644 --- a/youtube_dl/extractor/cnet.py +++ b/youtube_dl/extractor/cnet.py @@ -43,7 +43,11 @@ class CNETIE(InfoExtractor): raise ExtractorError('Cannot find video data') video_id = vdata['id'] - title = vdata['headline'] + title = vdata.get('headline') + if title is None: + title = vdata.get('title') + if title is None: + raise ExtractorError('Cannot find title!') description = vdata.get('dek') thumbnail = vdata.get('image', {}).get('path') author = vdata.get('author') diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py index 8af0abade..c81ce5a96 100644 --- a/youtube_dl/extractor/comedycentral.py +++ b/youtube_dl/extractor/comedycentral.py @@ -14,13 +14,13 @@ from ..utils import ( class ComedyCentralIE(MTVServicesInfoExtractor): - _VALID_URL = r'''(?x)https?://(?:www\.)?(comedycentral|cc)\.com/ - (video-clips|episodes|cc-studios|video-collections) + _VALID_URL = r'''(?x)https?://(?:www\.)?cc\.com/ + (video-clips|episodes|cc-studios|video-collections|full-episodes) /(?P<title>.*)''' _FEED_URL = 'http://comedycentral.com/feeds/mrss/' _TEST = { - 'url': 'http://www.comedycentral.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother', + 'url': 'http://www.cc.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother', 'md5': 'c4f48e9eda1b16dd10add0744344b6d8', 'info_dict': { 'id': 'cef0cbb3-e776-4bc9-b62e-8016deccb354', diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index e68657314..9d85a538c 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -18,6 +18,7 @@ from ..utils import ( clean_html, compiled_regex_type, ExtractorError, + int_or_none, RegexNotFoundError, sanitize_filename, unescapeHTML, @@ -69,6 +70,7 @@ class InfoExtractor(object): * vcodec Name of the video codec in use * container Name of the container format * filesize The number of bytes, if known in advance + * filesize_approx An estimate for the number of bytes * player_url SWF Player URL (used for rtmpdump). * protocol The protocol that will be used for the actual download, lower-case. @@ -82,6 +84,12 @@ class InfoExtractor(object): format, irrespective of the file format. -1 for default (order by other properties), -2 or smaller for less than default. + * http_referer HTTP Referer header value to set. + * http_method HTTP method to use for the download. + * http_headers A dictionary of additional HTTP headers + to add to the request. + * http_post_data Additional data to send with a POST + request. url: Final video URL. ext: Video filename extension. format: The video format, defaults to ext (used for --get-format) @@ -300,8 +308,12 @@ class InfoExtractor(object): def _download_json(self, url_or_request, video_id, note=u'Downloading JSON metadata', errnote=u'Unable to download JSON metadata', - transform_source=None): - json_string = self._download_webpage(url_or_request, video_id, note, errnote) + transform_source=None, + fatal=True): + json_string = self._download_webpage( + url_or_request, video_id, note, errnote, fatal=fatal) + if (not fatal) and json_string is False: + return None if transform_source: json_string = transform_source(json_string) try: @@ -368,7 +380,8 @@ class InfoExtractor(object): else: for p in pattern: mobj = re.search(p, string, flags) - if mobj: break + if mobj: + break if os.name != 'nt' and sys.stderr.isatty(): _name = u'\033[0;34m%s\033[0m' % name @@ -456,8 +469,9 @@ class InfoExtractor(object): return self._og_search_property('title', html, **kargs) def _og_search_video_url(self, html, name='video url', secure=True, **kargs): - regexes = self._og_regexes('video') - if secure: regexes = self._og_regexes('video:secure_url') + regexes + regexes = self._og_regexes('video') + self._og_regexes('video:url') + if secure: + regexes = self._og_regexes('video:secure_url') + regexes return self._html_search_regex(regexes, html, name, **kargs) def _og_search_url(self, html, **kargs): @@ -468,7 +482,7 @@ class InfoExtractor(object): display_name = name return self._html_search_regex( r'''(?ix)<meta - (?=[^>]+(?:itemprop|name|property)=["\']%s["\']) + (?=[^>]+(?:itemprop|name|property)=["\']?%s["\']?) [^>]+content=["\']([^"\']+)["\']''' % re.escape(name), html, display_name, fatal=fatal, **kwargs) @@ -555,6 +569,7 @@ class InfoExtractor(object): f.get('abr') if f.get('abr') is not None else -1, audio_ext_preference, f.get('filesize') if f.get('filesize') is not None else -1, + f.get('filesize_approx') if f.get('filesize_approx') is not None else -1, f.get('format_id'), ) formats.sort(key=_formats_key) @@ -583,6 +598,24 @@ class InfoExtractor(object): self.to_screen(msg) time.sleep(timeout) + def _extract_f4m_formats(self, manifest_url, video_id): + manifest = self._download_xml( + manifest_url, video_id, 'Downloading f4m manifest', + 'Unable to download f4m manifest') + + formats = [] + for media_el in manifest.findall('{http://ns.adobe.com/f4m/1.0}media'): + formats.append({ + 'url': manifest_url, + 'ext': 'flv', + 'tbr': int_or_none(media_el.attrib.get('bitrate')), + 'width': int_or_none(media_el.attrib.get('width')), + 'height': int_or_none(media_el.attrib.get('height')), + }) + self._sort_formats(formats) + + return formats + class SearchInfoExtractor(InfoExtractor): """ diff --git a/youtube_dl/extractor/cracked.py b/youtube_dl/extractor/cracked.py new file mode 100644 index 000000000..74b880ffc --- /dev/null +++ b/youtube_dl/extractor/cracked.py @@ -0,0 +1,65 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + parse_iso8601, + str_to_int, +) + + +class CrackedIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?cracked\.com/video_(?P<id>\d+)_[\da-z-]+\.html' + _TEST = { + 'url': 'http://www.cracked.com/video_19006_4-plot-holes-you-didnt-notice-in-your-favorite-movies.html', + 'md5': '4b29a5eeec292cd5eca6388c7558db9e', + 'info_dict': { + 'id': '19006', + 'ext': 'mp4', + 'title': '4 Plot Holes You Didn\'t Notice in Your Favorite Movies', + 'description': 'md5:3b909e752661db86007d10e5ec2df769', + 'timestamp': 1405659600, + 'upload_date': '20140718', + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + + video_url = self._html_search_regex( + [r'var\s+CK_vidSrc\s*=\s*"([^"]+)"', r'<video\s+src="([^"]+)"'], webpage, 'video URL') + + title = self._og_search_title(webpage) + description = self._og_search_description(webpage) + + timestamp = self._html_search_regex(r'<time datetime="([^"]+)"', webpage, 'upload date', fatal=False) + if timestamp: + timestamp = parse_iso8601(timestamp[:-6]) + + view_count = str_to_int(self._html_search_regex( + r'<span class="views" id="viewCounts">([\d,\.]+) Views</span>', webpage, 'view count', fatal=False)) + comment_count = str_to_int(self._html_search_regex( + r'<span id="commentCounts">([\d,\.]+)</span>', webpage, 'comment count', fatal=False)) + + m = re.search(r'_(?P<width>\d+)X(?P<height>\d+)\.mp4$', video_url) + if m: + width = int(m.group('width')) + height = int(m.group('height')) + else: + width = height = None + + return { + 'id': video_id, + 'url':video_url, + 'title': title, + 'description': description, + 'timestamp': timestamp, + 'view_count': view_count, + 'comment_count': comment_count, + 'height': height, + 'width': width, + }
\ No newline at end of file diff --git a/youtube_dl/extractor/dfb.py b/youtube_dl/extractor/dfb.py new file mode 100644 index 000000000..8049779b0 --- /dev/null +++ b/youtube_dl/extractor/dfb.py @@ -0,0 +1,44 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor + + +class DFBIE(InfoExtractor): + IE_NAME = 'tv.dfb.de' + _VALID_URL = r'https?://tv\.dfb\.de/video/[^/]+/(?P<id>\d+)' + + _TEST = { + 'url': 'http://tv.dfb.de/video/highlights-des-empfangs-in-berlin/9070/', + # The md5 is different each time + 'info_dict': { + 'id': '9070', + 'ext': 'flv', + 'title': 'Highlights des Empfangs in Berlin', + 'upload_date': '20140716', + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + player_info = self._download_xml( + 'http://tv.dfb.de/server/hd_video.php?play=%s' % video_id, + video_id) + video_info = player_info.find('video') + + f4m_info = self._download_xml(self._proto_relative_url(video_info.find('url').text.strip()), video_id) + token_el = f4m_info.find('token') + manifest_url = token_el.attrib['url'] + '?' + 'hdnea=' + token_el.attrib['auth'] + '&hdcore=3.2.0' + + return { + 'id': video_id, + 'title': video_info.find('title').text, + 'url': manifest_url, + 'ext': 'flv', + 'thumbnail': self._og_search_thumbnail(webpage), + 'upload_date': ''.join(video_info.find('time_date').text.split('.')[::-1]), + } diff --git a/youtube_dl/extractor/dropbox.py b/youtube_dl/extractor/dropbox.py index 41208c976..9f569aa93 100644 --- a/youtube_dl/extractor/dropbox.py +++ b/youtube_dl/extractor/dropbox.py @@ -5,24 +5,26 @@ import os.path import re from .common import InfoExtractor +from ..utils import compat_urllib_parse_unquote class DropboxIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?dropbox[.]com/s/(?P<id>[a-zA-Z0-9]{15})/(?P<title>[^?#]*)' _TEST = { - 'url': 'https://www.dropbox.com/s/0qr9sai2veej4f8/THE_DOCTOR_GAMES.mp4', - 'md5': '8ae17c51172fb7f93bdd6a214cc8c896', + 'url': 'https://www.dropbox.com/s/nelirfsxnmcfbfh/youtube-dl%20test%20video%20%27%C3%A4%22BaW_jenozKc.mp4', + 'md5': '8a3d905427a6951ccb9eb292f154530b', 'info_dict': { - 'id': '0qr9sai2veej4f8', + 'id': 'nelirfsxnmcfbfh', 'ext': 'mp4', - 'title': 'THE_DOCTOR_GAMES' + 'title': 'youtube-dl test video \'ä"BaW_jenozKc' } } def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') - title = os.path.splitext(mobj.group('title'))[0] + fn = compat_urllib_parse_unquote(mobj.group('title')) + title = os.path.splitext(fn)[0] video_url = url + '?dl=1' return { diff --git a/youtube_dl/extractor/dump.py b/youtube_dl/extractor/dump.py new file mode 100644 index 000000000..6b651778a --- /dev/null +++ b/youtube_dl/extractor/dump.py @@ -0,0 +1,39 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor + + +class DumpIE(InfoExtractor): + _VALID_URL = r'^https?://(?:www\.)?dump\.com/(?P<id>[a-zA-Z0-9]+)/' + + _TEST = { + 'url': 'http://www.dump.com/oneus/', + 'md5': 'ad71704d1e67dfd9e81e3e8b42d69d99', + 'info_dict': { + 'id': 'oneus', + 'ext': 'flv', + 'title': "He's one of us.", + 'thumbnail': 're:^https?://.*\.jpg$', + }, + } + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + video_id = m.group('id') + + webpage = self._download_webpage(url, video_id) + video_url = self._search_regex( + r's1.addVariable\("file",\s*"([^"]+)"', webpage, 'video URL') + + thumb = self._og_search_thumbnail(webpage) + title = self._search_regex(r'<b>([^"]+)</b>', webpage, 'title') + + return { + 'id': video_id, + 'title': title, + 'url': video_url, + 'thumbnail': thumb, + } diff --git a/youtube_dl/extractor/ebaumsworld.py b/youtube_dl/extractor/ebaumsworld.py index 877113d63..63c2549d3 100644 --- a/youtube_dl/extractor/ebaumsworld.py +++ b/youtube_dl/extractor/ebaumsworld.py @@ -1,19 +1,21 @@ +from __future__ import unicode_literals + import re from .common import InfoExtractor -from ..utils import determine_ext class EbaumsWorldIE(InfoExtractor): _VALID_URL = r'https?://www\.ebaumsworld\.com/video/watch/(?P<id>\d+)' _TEST = { - u'url': u'http://www.ebaumsworld.com/video/watch/83367677/', - u'file': u'83367677.mp4', - u'info_dict': { - u'title': u'A Giant Python Opens The Door', - u'description': u'This is how nightmares start...', - u'uploader': u'jihadpizza', + 'url': 'http://www.ebaumsworld.com/video/watch/83367677/', + 'info_dict': { + 'id': '83367677', + 'ext': 'mp4', + 'title': 'A Giant Python Opens The Door', + 'description': 'This is how nightmares start...', + 'uploader': 'jihadpizza', }, } @@ -28,7 +30,6 @@ class EbaumsWorldIE(InfoExtractor): 'id': video_id, 'title': config.find('title').text, 'url': video_url, - 'ext': determine_ext(video_url), 'description': config.find('description').text, 'thumbnail': config.find('image').text, 'uploader': config.find('username').text, diff --git a/youtube_dl/extractor/ellentv.py b/youtube_dl/extractor/ellentv.py new file mode 100644 index 000000000..3e7923648 --- /dev/null +++ b/youtube_dl/extractor/ellentv.py @@ -0,0 +1,79 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re +import json + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + parse_iso8601, +) + + +class EllenTVIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?ellentv\.com/videos/(?P<id>[a-z0-9_-]+)' + _TEST = { + 'url': 'http://www.ellentv.com/videos/0-7jqrsr18/', + 'md5': 'e4af06f3bf0d5f471921a18db5764642', + 'info_dict': { + 'id': '0-7jqrsr18', + 'ext': 'mp4', + 'title': 'What\'s Wrong with These Photos? A Whole Lot', + 'timestamp': 1406876400, + 'upload_date': '20140801', + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + timestamp = parse_iso8601(self._search_regex( + r'<span class="publish-date"><time datetime="([^"]+)">', + webpage, 'timestamp')) + + return { + 'id': video_id, + 'title': self._og_search_title(webpage), + 'url': self._html_search_meta('VideoURL', webpage, 'url'), + 'timestamp': timestamp, + } + + +class EllenTVClipsIE(InfoExtractor): + IE_NAME = 'EllenTV:clips' + _VALID_URL = r'https?://(?:www\.)?ellentv\.com/episodes/(?P<id>[a-z0-9_-]+)' + _TEST = { + 'url': 'http://www.ellentv.com/episodes/meryl-streep-vanessa-hudgens/', + 'info_dict': { + 'id': 'meryl-streep-vanessa-hudgens', + 'title': 'Meryl Streep, Vanessa Hudgens', + }, + 'playlist_mincount': 9, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + playlist_id = mobj.group('id') + + webpage = self._download_webpage(url, playlist_id) + playlist = self._extract_playlist(webpage) + + return { + '_type': 'playlist', + 'id': playlist_id, + 'title': self._og_search_title(webpage), + 'entries': self._extract_entries(playlist) + } + + def _extract_playlist(self, webpage): + json_string = self._search_regex(r'playerView.addClips\(\[\{(.*?)\}\]\);', webpage, 'json') + try: + return json.loads("[{" + json_string + "}]") + except ValueError as ve: + raise ExtractorError('Failed to download JSON', cause=ve) + + def _extract_entries(self, playlist): + return [self.url_result(item['url'], 'EllenTV') for item in playlist] diff --git a/youtube_dl/extractor/escapist.py b/youtube_dl/extractor/escapist.py index 272dfe1f6..476fc22b9 100644 --- a/youtube_dl/extractor/escapist.py +++ b/youtube_dl/extractor/escapist.py @@ -36,7 +36,7 @@ class EscapistIE(InfoExtractor): r'<meta name="description" content="([^"]*)"', webpage, 'description', fatal=False) - playerUrl = self._og_search_video_url(webpage, name=u'player URL') + playerUrl = self._og_search_video_url(webpage, name='player URL') title = self._html_search_regex( r'<meta name="title" content="([^"]*)"', diff --git a/youtube_dl/extractor/expotv.py b/youtube_dl/extractor/expotv.py new file mode 100644 index 000000000..a38b773e8 --- /dev/null +++ b/youtube_dl/extractor/expotv.py @@ -0,0 +1,73 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + int_or_none, + unified_strdate, +) + + +class ExpoTVIE(InfoExtractor): + _VALID_URL = r'https?://www\.expotv\.com/videos/[^?#]*/(?P<id>[0-9]+)($|[?#])' + _TEST = { + 'url': 'http://www.expotv.com/videos/reviews/1/24/LinneCardscom/17561', + 'md5': '2985e6d7a392b2f7a05e0ca350fe41d0', + 'info_dict': { + 'id': '17561', + 'ext': 'mp4', + 'upload_date': '20060212', + 'title': 'My Favorite Online Scrapbook Store', + 'view_count': int, + 'description': 'You\'ll find most everything you need at this virtual store front.', + 'uploader': 'Anna T.', + 'thumbnail': 're:^https?://.*\.jpg$', + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + player_key = self._search_regex( + r'<param name="playerKey" value="([^"]+)"', webpage, 'player key') + config_url = 'http://client.expotv.com/video/config/%s/%s' % ( + video_id, player_key) + config = self._download_json( + config_url, video_id, + note='Downloading video configuration') + + formats = [{ + 'url': fcfg['file'], + 'height': int_or_none(fcfg.get('height')), + 'format_note': fcfg.get('label'), + 'ext': self._search_regex( + r'filename=.*\.([a-z0-9_A-Z]+)&', fcfg['file'], + 'file extension', default=None), + } for fcfg in config['sources']] + self._sort_formats(formats) + + title = self._og_search_title(webpage) + description = self._og_search_description(webpage) + thumbnail = config.get('image') + view_count = int_or_none(self._search_regex( + r'<h5>Plays: ([0-9]+)</h5>', webpage, 'view counts')) + uploader = self._search_regex( + r'<div class="reviewer">\s*<img alt="([^"]+)"', webpage, 'uploader', + fatal=False) + upload_date = unified_strdate(self._search_regex( + r'<h5>Reviewed on ([0-9/.]+)</h5>', webpage, 'upload date', + fatal=False)) + + return { + 'id': video_id, + 'formats': formats, + 'title': title, + 'description': description, + 'view_count': view_count, + 'thumbnail': thumbnail, + 'uploader': uploader, + 'upload_date': upload_date, + } diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py index f0cd8f156..f7cf700b5 100644 --- a/youtube_dl/extractor/facebook.py +++ b/youtube_dl/extractor/facebook.py @@ -20,7 +20,7 @@ from ..utils import ( class FacebookIE(InfoExtractor): _VALID_URL = r'''(?x) https?://(?:\w+\.)?facebook\.com/ - (?:[^#?]*\#!/)? + (?:[^#]*?\#!/)? (?:video/video\.php|photo\.php|video/embed)\?(?:.*?) (?:v|video_id)=(?P<id>[0-9]+) (?:.*)''' diff --git a/youtube_dl/extractor/firedrive.py b/youtube_dl/extractor/firedrive.py index d26145db1..af439ccfe 100644 --- a/youtube_dl/extractor/firedrive.py +++ b/youtube_dl/extractor/firedrive.py @@ -8,7 +8,6 @@ from ..utils import ( ExtractorError, compat_urllib_parse, compat_urllib_request, - determine_ext, ) @@ -43,7 +42,6 @@ class FiredriveIE(InfoExtractor): fields = dict(re.findall(r'''(?x)<input\s+ type="hidden"\s+ name="([^"]+)"\s+ - (?:id="[^"]+"\s+)? value="([^"]*)" ''', webpage)) @@ -67,7 +65,7 @@ class FiredriveIE(InfoExtractor): ext = self._search_regex(r'type:\s?\'([^\']+)\',', webpage, 'extension', fatal=False) video_url = self._search_regex( - r'file:\s?\'(http[^\']+)\',', webpage, 'file url') + r'file:\s?loadURL\(\'(http[^\']+)\'\),', webpage, 'file url') formats = [{ 'format_id': 'sd', diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py index f3e0f38b7..1b0e8e5d5 100644 --- a/youtube_dl/extractor/francetv.py +++ b/youtube_dl/extractor/francetv.py @@ -19,17 +19,35 @@ class FranceTVBaseInfoExtractor(InfoExtractor): + video_id, video_id, 'Downloading XML config') manifest_url = info.find('videos/video/url').text - video_url = manifest_url.replace('manifest.f4m', 'index_2_av.m3u8') - video_url = video_url.replace('/z/', '/i/') + manifest_url = manifest_url.replace('/z/', '/i/') + + if manifest_url.startswith('rtmp'): + formats = [{'url': manifest_url, 'ext': 'flv'}] + else: + formats = [] + available_formats = self._search_regex(r'/[^,]*,(.*?),k\.mp4', manifest_url, 'available formats') + for index, format_descr in enumerate(available_formats.split(',')): + format_info = { + 'url': manifest_url.replace('manifest.f4m', 'index_%d_av.m3u8' % index), + 'ext': 'mp4', + } + m_resolution = re.search(r'(?P<width>\d+)x(?P<height>\d+)', format_descr) + if m_resolution is not None: + format_info.update({ + 'width': int(m_resolution.group('width')), + 'height': int(m_resolution.group('height')), + }) + formats.append(format_info) + thumbnail_path = info.find('image').text - return {'id': video_id, - 'ext': 'flv' if video_url.startswith('rtmp') else 'mp4', - 'url': video_url, - 'title': info.find('titre').text, - 'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', thumbnail_path), - 'description': info.find('synopsis').text, - } + return { + 'id': video_id, + 'title': info.find('titre').text, + 'formats': formats, + 'thumbnail': compat_urlparse.urljoin('http://pluzz.francetv.fr', thumbnail_path), + 'description': info.find('synopsis').text, + } class PluzzIE(FranceTVBaseInfoExtractor): @@ -48,7 +66,7 @@ class PluzzIE(FranceTVBaseInfoExtractor): class FranceTvInfoIE(FranceTVBaseInfoExtractor): IE_NAME = 'francetvinfo.fr' - _VALID_URL = r'https?://www\.francetvinfo\.fr/.*/(?P<title>.+)\.html' + _VALID_URL = r'https?://(?:www|mobile)\.francetvinfo\.fr/.*/(?P<title>.+)\.html' _TESTS = [{ 'url': 'http://www.francetvinfo.fr/replay-jt/france-3/soir-3/jt-grand-soir-3-lundi-26-aout-2013_393427.html', @@ -211,7 +229,7 @@ class GenerationQuoiIE(InfoExtractor): class CultureboxIE(FranceTVBaseInfoExtractor): IE_NAME = 'culturebox.francetvinfo.fr' - _VALID_URL = r'https?://culturebox\.francetvinfo\.fr/(?P<name>.*?)(\?|$)' + _VALID_URL = r'https?://(?:m\.)?culturebox\.francetvinfo\.fr/(?P<name>.*?)(\?|$)' _TEST = { 'url': 'http://culturebox.francetvinfo.fr/einstein-on-the-beach-au-theatre-du-chatelet-146813', diff --git a/youtube_dl/extractor/funnyordie.py b/youtube_dl/extractor/funnyordie.py index 6e6b66660..721e5fce0 100644 --- a/youtube_dl/extractor/funnyordie.py +++ b/youtube_dl/extractor/funnyordie.py @@ -26,7 +26,7 @@ class FunnyOrDieIE(InfoExtractor): 'id': 'e402820827', 'ext': 'mp4', 'title': 'Please Use This Song (Jon Lajoie)', - 'description': 'md5:2ed27d364f5a805a6dba199faaf6681d', + 'description': 'Please use this to sell something. www.jonlajoie.com', 'thumbnail': 're:^http:.*\.jpg$', }, }] diff --git a/youtube_dl/extractor/gamestar.py b/youtube_dl/extractor/gamestar.py new file mode 100644 index 000000000..50f8fc7e7 --- /dev/null +++ b/youtube_dl/extractor/gamestar.py @@ -0,0 +1,74 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + int_or_none, + parse_duration, + str_to_int, + unified_strdate, +) + + +class GameStarIE(InfoExtractor): + _VALID_URL = r'http://www\.gamestar\.de/videos/.*,(?P<id>[0-9]+)\.html' + _TEST = { + 'url': 'http://www.gamestar.de/videos/trailer,3/hobbit-3-die-schlacht-der-fuenf-heere,76110.html', + 'md5': '96974ecbb7fd8d0d20fca5a00810cea7', + 'info_dict': { + 'id': '76110', + 'ext': 'mp4', + 'title': 'Hobbit 3: Die Schlacht der Fünf Heere - Teaser-Trailer zum dritten Teil', + 'description': 'Der Teaser-Trailer zu Hobbit 3: Die Schlacht der Fünf Heere zeigt einige Szenen aus dem dritten Teil der Saga und kündigt den vollständigen Trailer an.', + 'thumbnail': 'http://images.gamestar.de/images/idgwpgsgp/bdb/2494525/600x.jpg', + 'upload_date': '20140728', + 'duration': 17 + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + + og_title = self._og_search_title(webpage) + title = og_title.replace(' - Video bei GameStar.de', '').strip() + + url = 'http://gamestar.de/_misc/videos/portal/getVideoUrl.cfm?premium=0&videoId=' + video_id + + description = self._og_search_description(webpage).strip() + + thumbnail = self._proto_relative_url( + self._og_search_thumbnail(webpage), scheme='http:') + + upload_date = unified_strdate(self._html_search_regex( + r'<span style="float:left;font-size:11px;">Datum: ([0-9]+\.[0-9]+\.[0-9]+) ', + webpage, 'upload_date', fatal=False)) + + duration = parse_duration(self._html_search_regex( + r' Länge: ([0-9]+:[0-9]+)</span>', webpage, 'duration', + fatal=False)) + + view_count = str_to_int(self._html_search_regex( + r' Zuschauer: ([0-9\.]+) ', webpage, + 'view_count', fatal=False)) + + comment_count = int_or_none(self._html_search_regex( + r'>Kommentieren \(([0-9]+)\)</a>', webpage, 'comment_count', + fatal=False)) + + return { + 'id': video_id, + 'title': title, + 'url': url, + 'ext': 'mp4', + 'thumbnail': thumbnail, + 'description': description, + 'upload_date': upload_date, + 'duration': duration, + 'view_count': view_count, + 'comment_count': comment_count + } diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py index 89d5994ee..de14ae1fb 100644 --- a/youtube_dl/extractor/gdcvault.py +++ b/youtube_dl/extractor/gdcvault.py @@ -8,6 +8,7 @@ from ..utils import ( compat_urllib_request, ) + class GDCVaultIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?gdcvault\.com/play/(?P<id>\d+)/(?P<name>(\w|-)+)' _TESTS = [ @@ -31,6 +32,15 @@ class GDCVaultIE(InfoExtractor): 'skip_download': True, # Requires rtmpdump } }, + { + 'url': 'http://www.gdcvault.com/play/1015301/Thexder-Meets-Windows-95-or', + 'md5': 'a5eb77996ef82118afbbe8e48731b98e', + 'info_dict': { + 'id': '1015301', + 'ext': 'flv', + 'title': 'Thexder Meets Windows 95, or Writing Great Games in the Windows 95 Environment', + } + } ] def _parse_mp4(self, xml_description): @@ -103,18 +113,40 @@ class GDCVaultIE(InfoExtractor): webpage_url = 'http://www.gdcvault.com/play/' + video_id start_page = self._download_webpage(webpage_url, video_id) - xml_root = self._html_search_regex(r'<iframe src="(?P<xml_root>.*?)player.html.*?".*?</iframe>', start_page, 'xml root', None, False) + direct_url = self._search_regex( + r's1\.addVariable\("file",\s*encodeURIComponent\("(/[^"]+)"\)\);', + start_page, 'url', default=None) + if direct_url: + video_url = 'http://www.gdcvault.com/' + direct_url + title = self._html_search_regex( + r'<td><strong>Session Name</strong></td>\s*<td>(.*?)</td>', + start_page, 'title') + + return { + 'id': video_id, + 'url': video_url, + 'ext': 'flv', + 'title': title, + } + xml_root = self._html_search_regex( + r'<iframe src="(?P<xml_root>.*?)player.html.*?".*?</iframe>', + start_page, 'xml root', default=None) if xml_root is None: # Probably need to authenticate - start_page = self._login(webpage_url, video_id) - if start_page is None: + login_res = self._login(webpage_url, video_id) + if login_res is None: self.report_warning('Could not login.') else: + start_page = login_res # Grab the url from the authenticated page - xml_root = self._html_search_regex(r'<iframe src="(?P<xml_root>.*?)player.html.*?".*?</iframe>', start_page, 'xml root') + xml_root = self._html_search_regex( + r'<iframe src="(.*?)player.html.*?".*?</iframe>', + start_page, 'xml root') - xml_name = self._html_search_regex(r'<iframe src=".*?\?xml=(?P<xml_file>.+?\.xml).*?".*?</iframe>', start_page, 'xml filename', None, False) + xml_name = self._html_search_regex( + r'<iframe src=".*?\?xml=(.+?\.xml).*?".*?</iframe>', + start_page, 'xml filename', default=None) if xml_name is None: # Fallback to the older format xml_name = self._html_search_regex(r'<iframe src=".*?\?xmlURL=xml/(?P<xml_file>.+?\.xml).*?".*?</iframe>', start_page, 'xml filename') diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index f97b59845..8b11f7f7a 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -15,11 +15,14 @@ from ..utils import ( compat_xml_parse_error, ExtractorError, + float_or_none, HEADRequest, + orderedSet, parse_xml, smuggle_url, unescapeHTML, unified_strdate, + unsmuggle_url, url_basename, ) from .brightcove import BrightcoveIE @@ -289,6 +292,58 @@ class GenericIE(InfoExtractor): 'description': 'Mario\'s life in the fast lane has never looked so good.', }, }, + # YouTube embed via <data-embed-url=""> + { + 'url': 'https://play.google.com/store/apps/details?id=com.gameloft.android.ANMP.GloftA8HM', + 'info_dict': { + 'id': 'jpSGZsgga_I', + 'ext': 'mp4', + 'title': 'Asphalt 8: Airborne - Launch Trailer', + 'uploader': 'Gameloft', + 'uploader_id': 'gameloft', + 'upload_date': '20130821', + 'description': 'md5:87bd95f13d8be3e7da87a5f2c443106a', + }, + 'params': { + 'skip_download': True, + } + }, + # Camtasia studio + { + 'url': 'http://www.ll.mit.edu/workshops/education/videocourses/antennas/lecture1/video/', + 'playlist': [{ + 'md5': '0c5e352edabf715d762b0ad4e6d9ee67', + 'info_dict': { + 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final', + 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - video1', + 'ext': 'flv', + 'duration': 2235.90, + } + }, { + 'md5': '10e4bb3aaca9fd630e273ff92d9f3c63', + 'info_dict': { + 'id': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final_PIP', + 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final - pip', + 'ext': 'flv', + 'duration': 2235.93, + } + }], + 'info_dict': { + 'title': 'Fenn-AA_PA_Radar_Course_Lecture_1c_Final', + } + }, + # Flowplayer + { + 'url': 'http://www.handjobhub.com/video/busty-blonde-siri-tit-fuck-while-wank-6313.html', + 'md5': '9d65602bf31c6e20014319c7d07fba27', + 'info_dict': { + 'id': '5123ea6d5e5a7', + 'ext': 'mp4', + 'age_limit': 18, + 'uploader': 'www.handjobhub.com', + 'title': 'Busty Blonde Siri Tit Fuck While Wank at Handjob Hub', + } + } ] def report_download_webpage(self, video_id): @@ -301,58 +356,6 @@ class GenericIE(InfoExtractor): """Report information extraction.""" self._downloader.to_screen('[redirect] Following redirect to %s' % new_url) - def _send_head(self, url): - """Check if it is a redirect, like url shorteners, in case return the new url.""" - - class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler): - """ - Subclass the HTTPRedirectHandler to make it use our - HEADRequest also on the redirected URL - """ - def redirect_request(self, req, fp, code, msg, headers, newurl): - if code in (301, 302, 303, 307): - newurl = newurl.replace(' ', '%20') - newheaders = dict((k,v) for k,v in req.headers.items() - if k.lower() not in ("content-length", "content-type")) - try: - # This function was deprecated in python 3.3 and removed in 3.4 - origin_req_host = req.get_origin_req_host() - except AttributeError: - origin_req_host = req.origin_req_host - return HEADRequest(newurl, - headers=newheaders, - origin_req_host=origin_req_host, - unverifiable=True) - else: - raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp) - - class HTTPMethodFallback(compat_urllib_request.BaseHandler): - """ - Fallback to GET if HEAD is not allowed (405 HTTP error) - """ - def http_error_405(self, req, fp, code, msg, headers): - fp.read() - fp.close() - - newheaders = dict((k,v) for k,v in req.headers.items() - if k.lower() not in ("content-length", "content-type")) - return self.parent.open(compat_urllib_request.Request(req.get_full_url(), - headers=newheaders, - origin_req_host=req.get_origin_req_host(), - unverifiable=True)) - - # Build our opener - opener = compat_urllib_request.OpenerDirector() - for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler, - HTTPMethodFallback, HEADRedirectHandler, - compat_urllib_request.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]: - opener.add_handler(handler()) - - response = opener.open(HEADRequest(url)) - if response is None: - raise ExtractorError('Invalid URL protocol') - return response - def _extract_rss(self, url, video_id, doc): playlist_title = doc.find('./channel/title').text playlist_desc_el = doc.find('./channel/description') @@ -372,6 +375,43 @@ class GenericIE(InfoExtractor): 'entries': entries, } + def _extract_camtasia(self, url, video_id, webpage): + """ Returns None if no camtasia video can be found. """ + + camtasia_cfg = self._search_regex( + r'fo\.addVariable\(\s*"csConfigFile",\s*"([^"]+)"\s*\);', + webpage, 'camtasia configuration file', default=None) + if camtasia_cfg is None: + return None + + title = self._html_search_meta('DC.title', webpage, fatal=True) + + camtasia_url = compat_urlparse.urljoin(url, camtasia_cfg) + camtasia_cfg = self._download_xml( + camtasia_url, video_id, + note='Downloading camtasia configuration', + errnote='Failed to download camtasia configuration') + fileset_node = camtasia_cfg.find('./playlist/array/fileset') + + entries = [] + for n in fileset_node.getchildren(): + url_n = n.find('./uri') + if url_n is None: + continue + + entries.append({ + 'id': os.path.splitext(url_n.text.rpartition('/')[2])[0], + 'title': '%s - %s' % (title, n.tag), + 'url': compat_urlparse.urljoin(url, url_n.text), + 'duration': float_or_none(n.find('./duration').text), + }) + + return { + '_type': 'playlist', + 'entries': entries, + 'title': title, + } + def _real_extract(self, url): if url.startswith('//'): return { @@ -383,13 +423,13 @@ class GenericIE(InfoExtractor): if not parsed_url.scheme: default_search = self._downloader.params.get('default_search') if default_search is None: - default_search = 'error' + default_search = 'fixup_error' - if default_search in ('auto', 'auto_warning'): + if default_search in ('auto', 'auto_warning', 'fixup_error'): if '/' in url: self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http') return self.url_result('http://' + url) - else: + elif default_search != 'fixup_error': if default_search == 'auto_warning': if re.match(r'^(?:url|URL)$', url): raise ExtractorError( @@ -399,25 +439,40 @@ class GenericIE(InfoExtractor): self._downloader.report_warning( 'Falling back to youtube search for %s . Set --default-search "auto" to suppress this warning.' % url) return self.url_result('ytsearch:' + url) - elif default_search == 'error': + + if default_search in ('error', 'fixup_error'): raise ExtractorError( ('%r is not a valid URL. ' - 'Set --default-search "ytseach" (or run youtube-dl "ytsearch:%s" ) to search YouTube' + 'Set --default-search "ytsearch" (or run youtube-dl "ytsearch:%s" ) to search YouTube' ) % (url, url), expected=True) else: assert ':' in default_search return self.url_result(default_search + url) - video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0] + + url, smuggled_data = unsmuggle_url(url) + force_videoid = None + if smuggled_data and 'force_videoid' in smuggled_data: + force_videoid = smuggled_data['force_videoid'] + video_id = force_videoid + else: + video_id = os.path.splitext(url.rstrip('/').split('/')[-1])[0] self.to_screen('%s: Requesting header' % video_id) - try: - response = self._send_head(url) + head_req = HEADRequest(url) + response = self._request_webpage( + head_req, video_id, + note=False, errnote='Could not send HEAD request to %s' % url, + fatal=False) + if response is not False: # Check for redirect new_url = response.geturl() if url != new_url: self.report_following_redirect(new_url) + if force_videoid: + new_url = smuggle_url( + new_url, {'force_videoid': force_videoid}) return self.url_result(new_url) # Check for direct link to a video @@ -438,10 +493,6 @@ class GenericIE(InfoExtractor): 'upload_date': upload_date, } - except compat_urllib_error.HTTPError: - # This may be a stupid server that doesn't like HEAD, our UA, or so - pass - try: webpage = self._download_webpage(url, video_id) except ValueError: @@ -459,6 +510,11 @@ class GenericIE(InfoExtractor): except compat_xml_parse_error: pass + # Is it a Camtasia project? + camtasia_res = self._extract_camtasia(url, video_id, webpage) + if camtasia_res is not None: + return camtasia_res + # Sometimes embedded video player is hidden behind percent encoding # (e.g. https://github.com/rg3/youtube-dl/issues/2448) # Unescaping the whole page allows to handle those cases in a generic way @@ -474,10 +530,26 @@ class GenericIE(InfoExtractor): r'(?s)<title>(.*?)</title>', webpage, 'video title', default='video') + # Try to detect age limit automatically + age_limit = self._rta_search(webpage) + # And then there are the jokers who advertise that they use RTA, + # but actually don't. + AGE_LIMIT_MARKERS = [ + r'Proudly Labeled <a href="http://www.rtalabel.org/" title="Restricted to Adults">RTA</a>', + ] + if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS): + age_limit = 18 + # video uploader is domain name video_uploader = self._search_regex( r'^(?:https?://)?([^/]*)/.*', url, 'video uploader') + # Helper method + def _playlist_from_matches(matches, getter, ie=None): + urlrs = orderedSet(self.url_result(getter(m), ie) for m in matches) + return self.playlist_result( + urlrs, playlist_id=video_id, playlist_title=video_title) + # Look for BrightCove: bc_urls = BrightcoveIE._extract_brightcove_urls(webpage) if bc_urls: @@ -513,6 +585,7 @@ class GenericIE(InfoExtractor): matches = re.findall(r'''(?x) (?: <iframe[^>]+?src=| + data-video-url=| <embed[^>]+?src=| embedSWF\(?:\s* ) @@ -521,19 +594,15 @@ class GenericIE(InfoExtractor): (?:embed|v)/.+?) \1''', webpage) if matches: - urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube') - for tuppl in matches] - return self.playlist_result( - urlrs, playlist_id=video_id, playlist_title=video_title) + return _playlist_from_matches( + matches, lambda m: unescapeHTML(m[1]), ie='Youtube') # Look for embedded Dailymotion player matches = re.findall( r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//(?:www\.)?dailymotion\.com/embed/video/.+?)\1', webpage) if matches: - urlrs = [self.url_result(unescapeHTML(tuppl[1])) - for tuppl in matches] - return self.playlist_result( - urlrs, playlist_id=video_id, playlist_title=video_title) + return _playlist_from_matches( + matches, lambda m: unescapeHTML(m[1])) # Look for embedded Wistia player match = re.search( @@ -552,7 +621,7 @@ class GenericIE(InfoExtractor): mobj = re.search(r'<meta\s[^>]*https?://api\.blip\.tv/\w+/redirect/\w+/(\d+)', webpage) if mobj: return self.url_result('http://blip.tv/a/a-'+mobj.group(1), 'BlipTV') - mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9]+)', webpage) + mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*(https?://(?:\w+\.)?blip\.tv/(?:play/|api\.swf#)[a-zA-Z0-9_]+)', webpage) if mobj: return self.url_result(mobj.group(1), 'BlipTV') @@ -647,10 +716,8 @@ class GenericIE(InfoExtractor): # Look for funnyordie embed matches = re.findall(r'<iframe[^>]+?src="(https?://(?:www\.)?funnyordie\.com/embed/[^"]+)"', webpage) if matches: - urlrs = [self.url_result(unescapeHTML(eurl), 'FunnyOrDie') - for eurl in matches] - return self.playlist_result( - urlrs, playlist_id=video_id, playlist_title=video_title) + return _playlist_from_matches( + matches, getter=unescapeHTML, ie='FunnyOrDie') # Look for embedded RUTV player rutv_url = RUTVIE._extract_url(webpage) @@ -705,6 +772,20 @@ class GenericIE(InfoExtractor): url = unescapeHTML(mobj.group('url')) return self.url_result(url, ie='MTVServicesEmbedded') + # Look for embedded yahoo player + mobj = re.search( + r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:screen|movies)\.yahoo\.com/.+?\.html\?format=embed)\1', + webpage) + if mobj is not None: + return self.url_result(mobj.group('url'), 'Yahoo') + + # Look for embedded sbs.com.au player + mobj = re.search( + r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)sbs\.com\.au/ondemand/video/single/.+?)\1', + webpage) + if mobj is not None: + return self.url_result(mobj.group('url'), 'SBS') + # Start with something easy: JW Player in SWFObject found = re.findall(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage) if not found: @@ -723,6 +804,15 @@ class GenericIE(InfoExtractor): # Broaden the findall a little bit: JWPlayer JS loader found = re.findall(r'[^A-Za-z0-9]?file["\']?:\s*["\'](http(?![^\'"]+\.[0-9]+[\'"])[^\'"]+)["\']', webpage) if not found: + # Flow player + found = re.findall(r'''(?xs) + flowplayer\("[^"]+",\s* + \{[^}]+?\}\s*, + \s*{[^}]+? ["']?clip["']?\s*:\s*\{\s* + ["']?url["']?\s*:\s*["']([^"']+)["'] + ''', webpage) + assert found + if not found: # Try to find twitter cards info found = re.findall(r'<meta (?:property|name)="twitter:player:stream" (?:content|value)="(.+?)"', webpage) if not found: @@ -731,7 +821,12 @@ class GenericIE(InfoExtractor): m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage) # We only look in og:video if the MIME type is a video, don't try if it's a Flash player: if m_video_type is not None: - found = re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage) + def check_video(vurl): + vpath = compat_urlparse.urlparse(vurl).path + return '.' in vpath and not vpath.endswith('.swf') + found = list(filter( + check_video, + re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))) if not found: # HTML5 video found = re.findall(r'(?s)<video[^<]*(?:>.*?<source.*?)? src="([^"]+)"', webpage) @@ -768,6 +863,7 @@ class GenericIE(InfoExtractor): 'url': video_url, 'uploader': video_uploader, 'title': video_title, + 'age_limit': age_limit, }) if len(entries) == 1: diff --git a/youtube_dl/extractor/godtube.py b/youtube_dl/extractor/godtube.py new file mode 100644 index 000000000..73bd6d890 --- /dev/null +++ b/youtube_dl/extractor/godtube.py @@ -0,0 +1,58 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + parse_duration, + parse_iso8601, +) + + +class GodTubeIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?godtube\.com/watch/\?v=(?P<id>[\da-zA-Z]+)' + _TESTS = [ + { + 'url': 'https://www.godtube.com/watch/?v=0C0CNNNU', + 'md5': '77108c1e4ab58f48031101a1a2119789', + 'info_dict': { + 'id': '0C0CNNNU', + 'ext': 'mp4', + 'title': 'Woman at the well.', + 'duration': 159, + 'timestamp': 1205712000, + 'uploader': 'beverlybmusic', + 'upload_date': '20080317', + 'thumbnail': 're:^https?://.*\.jpg$', + }, + }, + ] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + config = self._download_xml( + 'http://www.godtube.com/resource/mediaplayer/%s.xml' % video_id.lower(), + video_id, 'Downloading player config XML') + + video_url = config.find('.//file').text + uploader = config.find('.//author').text + timestamp = parse_iso8601(config.find('.//date').text) + duration = parse_duration(config.find('.//duration').text) + thumbnail = config.find('.//image').text + + media = self._download_xml( + 'http://www.godtube.com/media/xml/?v=%s' % video_id, video_id, 'Downloading media XML') + + title = media.find('.//title').text + + return { + 'id': video_id, + 'url': video_url, + 'title': title, + 'thumbnail': thumbnail, + 'timestamp': timestamp, + 'uploader': uploader, + 'duration': duration, + } diff --git a/youtube_dl/extractor/grooveshark.py b/youtube_dl/extractor/grooveshark.py new file mode 100644 index 000000000..726adff77 --- /dev/null +++ b/youtube_dl/extractor/grooveshark.py @@ -0,0 +1,190 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import time +import math +import os.path +import re + + +from .common import InfoExtractor +from ..utils import ExtractorError, compat_urllib_request, compat_html_parser + +from ..utils import ( + compat_urllib_parse, + compat_urlparse, +) + + +class GroovesharkHtmlParser(compat_html_parser.HTMLParser): + def __init__(self): + self._current_object = None + self.objects = [] + compat_html_parser.HTMLParser.__init__(self) + + def handle_starttag(self, tag, attrs): + attrs = dict((k, v) for k, v in attrs) + if tag == 'object': + self._current_object = {'attrs': attrs, 'params': []} + elif tag == 'param': + self._current_object['params'].append(attrs) + + def handle_endtag(self, tag): + if tag == 'object': + self.objects.append(self._current_object) + self._current_object = None + + @classmethod + def extract_object_tags(cls, html): + p = cls() + p.feed(html) + p.close() + return p.objects + + +class GroovesharkIE(InfoExtractor): + _VALID_URL = r'https?://(www\.)?grooveshark\.com/#!/s/([^/]+)/([^/]+)' + _TEST = { + 'url': 'http://grooveshark.com/#!/s/Jolene+Tenth+Key+Remix+Ft+Will+Sessions/6SS1DW?src=5', + 'md5': '7ecf8aefa59d6b2098517e1baa530023', + 'info_dict': { + 'id': '6SS1DW', + 'title': 'Jolene (Tenth Key Remix ft. Will Sessions)', + 'ext': 'mp3', + 'duration': 227, + } + } + + do_playerpage_request = True + do_bootstrap_request = True + + def _parse_target(self, target): + uri = compat_urlparse.urlparse(target) + hash = uri.fragment[1:].split('?')[0] + token = os.path.basename(hash.rstrip('/')) + return (uri, hash, token) + + def _build_bootstrap_url(self, target): + (uri, hash, token) = self._parse_target(target) + query = 'getCommunicationToken=1&hash=%s&%d' % (compat_urllib_parse.quote(hash, safe=''), self.ts) + return (compat_urlparse.urlunparse((uri.scheme, uri.netloc, '/preload.php', None, query, None)), token) + + def _build_meta_url(self, target): + (uri, hash, token) = self._parse_target(target) + query = 'hash=%s&%d' % (compat_urllib_parse.quote(hash, safe=''), self.ts) + return (compat_urlparse.urlunparse((uri.scheme, uri.netloc, '/preload.php', None, query, None)), token) + + def _build_stream_url(self, meta): + return compat_urlparse.urlunparse(('http', meta['streamKey']['ip'], '/stream.php', None, None, None)) + + def _build_swf_referer(self, target, obj): + (uri, _, _) = self._parse_target(target) + return compat_urlparse.urlunparse((uri.scheme, uri.netloc, obj['attrs']['data'], None, None, None)) + + def _transform_bootstrap(self, js): + return re.split('(?m)^\s*try\s*{', js)[0] \ + .split(' = ', 1)[1].strip().rstrip(';') + + def _transform_meta(self, js): + return js.split('\n')[0].split('=')[1].rstrip(';') + + def _get_meta(self, target): + (meta_url, token) = self._build_meta_url(target) + self.to_screen('Metadata URL: %s' % meta_url) + + headers = {'Referer': compat_urlparse.urldefrag(target)[0]} + req = compat_urllib_request.Request(meta_url, headers=headers) + res = self._download_json(req, token, + transform_source=self._transform_meta) + + if 'getStreamKeyWithSong' not in res: + raise ExtractorError( + 'Metadata not found. URL may be malformed, or Grooveshark API may have changed.') + + if res['getStreamKeyWithSong'] is None: + raise ExtractorError( + 'Metadata download failed, probably due to Grooveshark anti-abuse throttling. Wait at least an hour before retrying from this IP.', + expected=True) + + return res['getStreamKeyWithSong'] + + def _get_bootstrap(self, target): + (bootstrap_url, token) = self._build_bootstrap_url(target) + + headers = {'Referer': compat_urlparse.urldefrag(target)[0]} + req = compat_urllib_request.Request(bootstrap_url, headers=headers) + res = self._download_json(req, token, fatal=False, + note='Downloading player bootstrap data', + errnote='Unable to download player bootstrap data', + transform_source=self._transform_bootstrap) + return res + + def _get_playerpage(self, target): + (_, _, token) = self._parse_target(target) + + webpage = self._download_webpage( + target, token, + note='Downloading player page', + errnote='Unable to download player page', + fatal=False) + + if webpage is not None: + # Search (for example German) error message + error_msg = self._html_search_regex( + r'<div id="content">\s*<h2>(.*?)</h2>', webpage, + 'error message', default=None) + if error_msg is not None: + error_msg = error_msg.replace('\n', ' ') + raise ExtractorError('Grooveshark said: %s' % error_msg) + + if webpage is not None: + o = GroovesharkHtmlParser.extract_object_tags(webpage) + return (webpage, [x for x in o if x['attrs']['id'] == 'jsPlayerEmbed']) + + return (webpage, None) + + def _real_initialize(self): + self.ts = int(time.time() * 1000) # timestamp in millis + + def _real_extract(self, url): + (target_uri, _, token) = self._parse_target(url) + + # 1. Fill cookiejar by making a request to the player page + swf_referer = None + if self.do_playerpage_request: + (_, player_objs) = self._get_playerpage(url) + if player_objs is not None: + swf_referer = self._build_swf_referer(url, player_objs[0]) + self.to_screen('SWF Referer: %s' % swf_referer) + + # 2. Ask preload.php for swf bootstrap data to better mimic webapp + if self.do_bootstrap_request: + bootstrap = self._get_bootstrap(url) + self.to_screen('CommunicationToken: %s' % bootstrap['getCommunicationToken']) + + # 3. Ask preload.php for track metadata. + meta = self._get_meta(url) + + # 4. Construct stream request for track. + stream_url = self._build_stream_url(meta) + duration = int(math.ceil(float(meta['streamKey']['uSecs']) / 1000000)) + post_dict = {'streamKey': meta['streamKey']['streamKey']} + post_data = compat_urllib_parse.urlencode(post_dict).encode('utf-8') + headers = { + 'Content-Length': len(post_data), + 'Content-Type': 'application/x-www-form-urlencoded' + } + if swf_referer is not None: + headers['Referer'] = swf_referer + + return { + 'id': token, + 'title': meta['song']['Name'], + 'http_method': 'POST', + 'url': stream_url, + 'ext': 'mp3', + 'format': 'mp3 audio', + 'duration': duration, + 'http_post_data': post_data, + 'http_headers': headers, + } diff --git a/youtube_dl/extractor/howstuffworks.py b/youtube_dl/extractor/howstuffworks.py new file mode 100644 index 000000000..68684b997 --- /dev/null +++ b/youtube_dl/extractor/howstuffworks.py @@ -0,0 +1,134 @@ +from __future__ import unicode_literals + +import re +import json +import random +import string + +from .common import InfoExtractor +from ..utils import find_xpath_attr + + +class HowStuffWorksIE(InfoExtractor): + _VALID_URL = r'https?://[\da-z-]+\.howstuffworks\.com/(?:[^/]+/)*\d+-(?P<id>.+?)-video\.htm' + _TESTS = [ + { + 'url': 'http://adventure.howstuffworks.com/5266-cool-jobs-iditarod-musher-video.htm', + 'info_dict': { + 'id': '450221', + 'display_id': 'cool-jobs-iditarod-musher', + 'ext': 'flv', + 'title': 'Cool Jobs - Iditarod Musher', + 'description': 'md5:82bb58438a88027b8186a1fccb365f90', + 'thumbnail': 're:^https?://.*\.jpg$', + }, + 'params': { + # md5 is not consistent + 'skip_download': True + } + }, + { + 'url': 'http://adventure.howstuffworks.com/39516-deadliest-catch-jakes-farewell-pots-video.htm', + 'info_dict': { + 'id': '553470', + 'display_id': 'deadliest-catch-jakes-farewell-pots', + 'ext': 'mp4', + 'title': 'Deadliest Catch: Jake\'s Farewell Pots', + 'description': 'md5:9632c346d5e43ee238028c9cefd8dbbc', + 'thumbnail': 're:^https?://.*\.jpg$', + }, + 'params': { + # md5 is not consistent + 'skip_download': True + } + }, + { + 'url': 'http://entertainment.howstuffworks.com/arts/2706-sword-swallowing-1-by-dan-meyer-video.htm', + 'info_dict': { + 'id': '440011', + 'display_id': 'sword-swallowing-1-by-dan-meyer', + 'ext': 'flv', + 'title': 'Sword Swallowing #1 by Dan Meyer', + 'description': 'md5:b2409e88172913e2e7d3d1159b0ef735', + 'thumbnail': 're:^https?://.*\.jpg$', + }, + 'params': { + # md5 is not consistent + 'skip_download': True + } + }, + ] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + display_id = mobj.group('id') + webpage = self._download_webpage(url, display_id) + + content_id = self._search_regex(r'var siteSectionId="(\d+)";', webpage, 'content id') + + mp4 = self._search_regex( + r'''(?xs)var\s+clip\s*=\s*{\s* + .+?\s* + content_id\s*:\s*%s\s*,\s* + .+?\s* + mp4\s*:\s*\[(.*?),?\]\s* + };\s* + videoData\.push\(clip\);''' % content_id, + webpage, 'mp4', fatal=False, default=None) + + smil = self._download_xml( + 'http://services.media.howstuffworks.com/videos/%s/smil-service.smil' % content_id, + content_id, 'Downloading video SMIL') + + http_base = find_xpath_attr( + smil, + './{0}head/{0}meta'.format('{http://www.w3.org/2001/SMIL20/Language}'), + 'name', + 'httpBase').get('content') + + def random_string(str_len=0): + return ''.join([random.choice(string.ascii_uppercase) for _ in range(str_len)]) + + URL_SUFFIX = '?v=2.11.3&fp=LNX 11,2,202,356&r=%s&g=%s' % (random_string(5), random_string(12)) + + formats = [] + + if mp4: + for video in json.loads('[%s]' % mp4): + bitrate = video['bitrate'] + fmt = { + 'url': video['src'].replace('http://pmd.video.howstuffworks.com', http_base) + URL_SUFFIX, + 'format_id': bitrate, + } + m = re.search(r'(?P<vbr>\d+)[Kk]', bitrate) + if m: + fmt['vbr'] = int(m.group('vbr')) + formats.append(fmt) + else: + for video in smil.findall( + './/{0}body/{0}switch/{0}video'.format('{http://www.w3.org/2001/SMIL20/Language}')): + vbr = int(video.attrib['system-bitrate']) / 1000 + formats.append({ + 'url': '%s/%s%s' % (http_base, video.attrib['src'], URL_SUFFIX), + 'format_id': '%dk' % vbr, + 'vbr': vbr, + }) + + self._sort_formats(formats) + + title = self._og_search_title(webpage) + TITLE_SUFFIX = ' : HowStuffWorks' + if title.endswith(TITLE_SUFFIX): + title = title[:-len(TITLE_SUFFIX)] + + description = self._og_search_description(webpage) + thumbnail = self._og_search_thumbnail(webpage) + + return { + 'id': content_id, + 'display_id': display_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'formats': formats, + } diff --git a/youtube_dl/extractor/izlesene.py b/youtube_dl/extractor/izlesene.py new file mode 100644 index 000000000..79e8430b5 --- /dev/null +++ b/youtube_dl/extractor/izlesene.py @@ -0,0 +1,97 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + get_element_by_id, + parse_iso8601, + determine_ext, + int_or_none, + str_to_int, +) + + +class IzleseneIE(InfoExtractor): + _VALID_URL = r'https?://(?:(?:www|m)\.)?izlesene\.com/(?:video|embedplayer)/(?:[^/]+/)?(?P<id>[0-9]+)' + _STREAM_URL = 'http://panel.izlesene.com/api/streamurl/{id:}/{format:}' + _TEST = { + 'url': 'http://www.izlesene.com/video/sevincten-cildirtan-dogum-gunu-hediyesi/7599694', + 'md5': '4384f9f0ea65086734b881085ee05ac2', + 'info_dict': { + 'id': '7599694', + 'ext': 'mp4', + 'title': 'Sevinçten Çıldırtan Doğum Günü Hediyesi', + 'description': 'Annesi oğluna doğum günü hediyesi olarak minecraft cd si alıyor, ve çocuk hunharca seviniyor', + 'thumbnail': 're:^http://.*\.jpg', + 'uploader_id': 'pelikzzle', + 'timestamp': 1404298698, + 'upload_date': '20140702', + 'duration': 95.395, + 'age_limit': 0, + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + url = 'http://www.izlesene.com/video/%s' % video_id + + webpage = self._download_webpage(url, video_id) + + title = self._og_search_title(webpage) + description = self._og_search_description(webpage) + thumbnail = self._og_search_thumbnail(webpage) + + uploader = self._html_search_regex( + r"adduserUsername\s*=\s*'([^']+)';", webpage, 'uploader', fatal=False, default='') + timestamp = parse_iso8601(self._html_search_meta( + 'uploadDate', webpage, 'upload date', fatal=False)) + + duration = int_or_none(self._html_search_regex( + r'"videoduration"\s*:\s*"([^"]+)"', webpage, 'duration', fatal=False)) + if duration: + duration /= 1000.0 + + view_count = str_to_int(get_element_by_id('videoViewCount', webpage)) + comment_count = self._html_search_regex( + r'comment_count\s*=\s*\'([^\']+)\';', webpage, 'uploader', fatal=False) + + family_friendly = self._html_search_meta( + 'isFamilyFriendly', webpage, 'age limit', fatal=False) + + content_url = self._html_search_meta( + 'contentURL', webpage, 'content URL', fatal=False) + ext = determine_ext(content_url, 'mp4') + + # Might be empty for some videos. + qualities = self._html_search_regex( + r'"quality"\s*:\s*"([^"]+)"', webpage, 'qualities', fatal=False, default='') + + formats = [] + for quality in qualities.split('|'): + json = self._download_json( + self._STREAM_URL.format(id=video_id, format=quality), video_id, + note='Getting video URL for "%s" quality' % quality, + errnote='Failed to get video URL for "%s" quality' % quality + ) + formats.append({ + 'url': json.get('streamurl'), + 'ext': ext, + 'format_id': '%sp' % quality if quality else 'sd', + }) + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'uploader_id': uploader, + 'timestamp': timestamp, + 'duration': duration, + 'view_count': int_or_none(view_count), + 'comment_count': int_or_none(comment_count), + 'age_limit': 18 if family_friendly == 'False' else 0, + 'formats': formats, + } diff --git a/youtube_dl/extractor/jove.py b/youtube_dl/extractor/jove.py new file mode 100644 index 000000000..cf73cd753 --- /dev/null +++ b/youtube_dl/extractor/jove.py @@ -0,0 +1,80 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + unified_strdate +) + + +class JoveIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?jove\.com/video/(?P<id>[0-9]+)' + _CHAPTERS_URL = 'http://www.jove.com/video-chapters?videoid={video_id:}' + _TESTS = [ + { + 'url': 'http://www.jove.com/video/2744/electrode-positioning-montage-transcranial-direct-current', + 'md5': '93723888d82dbd6ba8b3d7d0cd65dd2b', + 'info_dict': { + 'id': '2744', + 'ext': 'mp4', + 'title': 'Electrode Positioning and Montage in Transcranial Direct Current Stimulation', + 'description': 'md5:015dd4509649c0908bc27f049e0262c6', + 'thumbnail': 're:^https?://.*\.png$', + 'upload_date': '20110523', + } + }, + { + 'url': 'http://www.jove.com/video/51796/culturing-caenorhabditis-elegans-axenic-liquid-media-creation', + 'md5': '914aeb356f416811d911996434811beb', + 'info_dict': { + 'id': '51796', + 'ext': 'mp4', + 'title': 'Culturing Caenorhabditis elegans in Axenic Liquid Media and Creation of Transgenic Worms by Microparticle Bombardment', + 'description': 'md5:35ff029261900583970c4023b70f1dc9', + 'thumbnail': 're:^https?://.*\.png$', + 'upload_date': '20140802', + } + }, + + ] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + + chapters_id = self._html_search_regex( + r'/video-chapters\?videoid=([0-9]+)', webpage, 'chapters id') + + chapters_xml = self._download_xml( + self._CHAPTERS_URL.format(video_id=chapters_id), + video_id, note='Downloading chapters XML', + errnote='Failed to download chapters XML') + + video_url = chapters_xml.attrib.get('video') + if not video_url: + raise ExtractorError('Failed to get the video URL') + + title = self._html_search_meta('citation_title', webpage, 'title') + thumbnail = self._og_search_thumbnail(webpage) + description = self._html_search_regex( + r'<div id="section_body_summary"><p class="jove_content">(.+?)</p>', + webpage, 'description', fatal=False) + publish_date = unified_strdate(self._html_search_meta( + 'citation_publication_date', webpage, 'publish date', fatal=False)) + comment_count = self._html_search_regex( + r'<meta name="num_comments" content="(\d+) Comments?"', + webpage, 'comment count', fatal=False) + + return { + 'id': video_id, + 'title': title, + 'url': video_url, + 'thumbnail': thumbnail, + 'description': description, + 'upload_date': publish_date, + 'comment_count': comment_count, + } diff --git a/youtube_dl/extractor/justintv.py b/youtube_dl/extractor/justintv.py index 7083db12e..27017e89f 100644 --- a/youtube_dl/extractor/justintv.py +++ b/youtube_dl/extractor/justintv.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import itertools import json import os import re @@ -43,10 +44,11 @@ class JustinTVIE(InfoExtractor): } # Return count of items, list of *valid* items - def _parse_page(self, url, video_id): - info_json = self._download_webpage(url, video_id, - 'Downloading video info JSON', - 'unable to download video info JSON') + def _parse_page(self, url, video_id, counter): + info_json = self._download_webpage( + url, video_id, + 'Downloading video info JSON on page %d' % counter, + 'Unable to download video info JSON %d' % counter) response = json.loads(info_json) if type(response) != list: @@ -138,11 +140,10 @@ class JustinTVIE(InfoExtractor): entries = [] offset = 0 limit = self._JUSTIN_PAGE_LIMIT - while True: - if paged: - self.report_download_page(video_id, offset) + for counter in itertools.count(1): page_url = api + ('?offset=%d&limit=%d' % (offset, limit)) - page_count, page_info = self._parse_page(page_url, video_id) + page_count, page_info = self._parse_page( + page_url, video_id, counter) entries.extend(page_info) if not paged or page_count != limit: break diff --git a/youtube_dl/extractor/kickstarter.py b/youtube_dl/extractor/kickstarter.py index 961dd1aa6..56a76380c 100644 --- a/youtube_dl/extractor/kickstarter.py +++ b/youtube_dl/extractor/kickstarter.py @@ -8,7 +8,7 @@ from .common import InfoExtractor class KickStarterIE(InfoExtractor): _VALID_URL = r'https?://www\.kickstarter\.com/projects/(?P<id>[^/]*)/.*' - _TEST = { + _TESTS = [{ 'url': 'https://www.kickstarter.com/projects/1404461844/intersection-the-story-of-josh-grant?ref=home_location', 'md5': 'c81addca81327ffa66c642b5d8b08cab', 'info_dict': { @@ -18,22 +18,45 @@ class KickStarterIE(InfoExtractor): 'description': 'A unique motocross documentary that examines the ' 'life and mind of one of sports most elite athletes: Josh Grant.', }, - } + }, { + 'note': 'Embedded video (not using the native kickstarter video service)', + 'url': 'https://www.kickstarter.com/projects/597507018/pebble-e-paper-watch-for-iphone-and-android/posts/659178', + 'playlist': [ + { + 'info_dict': { + 'id': '78704821', + 'ext': 'mp4', + 'uploader_id': 'pebble', + 'uploader': 'Pebble Technology', + 'title': 'Pebble iOS Notifications', + } + } + ], + }] def _real_extract(self, url): m = re.match(self._VALID_URL, url) video_id = m.group('id') webpage = self._download_webpage(url, video_id) - video_url = self._search_regex(r'data-video-url="(.*?)"', - webpage, 'video URL') - video_title = self._html_search_regex(r'<title>(.*?)</title>', - webpage, 'title').rpartition('— Kickstarter')[0].strip() + title = self._html_search_regex( + r'<title>\s*(.*?)(?:\s*— Kickstarter)?\s*</title>', + webpage, 'title') + video_url = self._search_regex( + r'data-video-url="(.*?)"', + webpage, 'video URL', default=None) + if video_url is None: # No native kickstarter, look for embedded videos + return { + '_type': 'url_transparent', + 'ie_key': 'Generic', + 'url': url, + 'title': title, + } return { 'id': video_id, 'url': video_url, - 'title': video_title, + 'title': title, 'description': self._og_search_description(webpage), 'thumbnail': self._og_search_thumbnail(webpage), } diff --git a/youtube_dl/extractor/krasview.py b/youtube_dl/extractor/krasview.py new file mode 100644 index 000000000..6f3d2345b --- /dev/null +++ b/youtube_dl/extractor/krasview.py @@ -0,0 +1,59 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +import json +import re + +from .common import InfoExtractor +from ..utils import ( + int_or_none, + unescapeHTML, +) + + +class KrasViewIE(InfoExtractor): + IE_DESC = 'Красвью' + _VALID_URL = r'https?://krasview\.ru/video/(?P<id>\d+)' + + _TEST = { + 'url': 'http://krasview.ru/video/512228', + 'md5': '3b91003cf85fc5db277870c8ebd98eae', + 'info_dict': { + 'id': '512228', + 'ext': 'mp4', + 'title': 'Снег, лёд, заносы', + 'description': 'Снято в городе Нягань, в Ханты-Мансийском автономном округе.', + 'duration': 27, + 'thumbnail': 're:^https?://.*\.jpg', + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + + flashvars = json.loads(self._search_regex( + r'flashvars\s*:\s*({.+?})\s*}\);', webpage, 'flashvars')) + + video_url = flashvars['url'] + title = unescapeHTML(flashvars['title']) + description = unescapeHTML(flashvars.get('subtitle') or self._og_search_description(webpage, default=None)) + thumbnail = flashvars['image'] + duration = int(flashvars['duration']) + filesize = int(flashvars['size']) + width = int_or_none(self._og_search_property('video:width', webpage, 'video width')) + height = int_or_none(self._og_search_property('video:height', webpage, 'video height')) + + return { + 'id': video_id, + 'url': video_url, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'duration': duration, + 'filesize': filesize, + 'width': width, + 'height': height, + } diff --git a/youtube_dl/extractor/livestream.py b/youtube_dl/extractor/livestream.py index 2c100d424..281a0ce40 100644 --- a/youtube_dl/extractor/livestream.py +++ b/youtube_dl/extractor/livestream.py @@ -5,11 +5,14 @@ import json from .common import InfoExtractor from ..utils import ( + compat_str, compat_urllib_parse_urlparse, compat_urlparse, - xpath_with_ns, - compat_str, + ExtractorError, + find_xpath_attr, + int_or_none, orderedSet, + xpath_with_ns, ) @@ -24,18 +27,82 @@ class LivestreamIE(InfoExtractor): 'ext': 'mp4', 'title': 'Live from Webster Hall NYC', 'upload_date': '20121012', + 'like_count': int, + 'view_count': int, + 'thumbnail': 're:^http://.*\.jpg$' } } + def _parse_smil(self, video_id, smil_url): + formats = [] + _SWITCH_XPATH = ( + './/{http://www.w3.org/2001/SMIL20/Language}body/' + '{http://www.w3.org/2001/SMIL20/Language}switch') + smil_doc = self._download_xml( + smil_url, video_id, + note='Downloading SMIL information', + errnote='Unable to download SMIL information', + fatal=False) + if smil_doc is False: # Download failed + return formats + title_node = find_xpath_attr( + smil_doc, './/{http://www.w3.org/2001/SMIL20/Language}meta', + 'name', 'title') + if title_node is None: + self.report_warning('Cannot find SMIL id') + switch_node = smil_doc.find(_SWITCH_XPATH) + else: + title_id = title_node.attrib['content'] + switch_node = find_xpath_attr( + smil_doc, _SWITCH_XPATH, 'id', title_id) + if switch_node is None: + raise ExtractorError('Cannot find switch node') + video_nodes = switch_node.findall( + '{http://www.w3.org/2001/SMIL20/Language}video') + + for vn in video_nodes: + tbr = int_or_none(vn.attrib.get('system-bitrate')) + furl = ( + 'http://livestream-f.akamaihd.net/%s?v=3.0.3&fp=WIN%%2014,0,0,145' % + (vn.attrib['src'])) + if 'clipBegin' in vn.attrib: + furl += '&ssek=' + vn.attrib['clipBegin'] + formats.append({ + 'url': furl, + 'format_id': 'smil_%d' % tbr, + 'ext': 'flv', + 'tbr': tbr, + 'preference': -1000, + }) + return formats + def _extract_video_info(self, video_data): - video_url = video_data.get('progressive_url_hd') or video_data.get('progressive_url') + video_id = compat_str(video_data['id']) + + FORMAT_KEYS = ( + ('sd', 'progressive_url'), + ('hd', 'progressive_url_hd'), + ) + formats = [{ + 'format_id': format_id, + 'url': video_data[key], + 'quality': i + 1, + } for i, (format_id, key) in enumerate(FORMAT_KEYS) + if video_data.get(key)] + + smil_url = video_data.get('smil_url') + if smil_url: + formats.extend(self._parse_smil(video_id, smil_url)) + self._sort_formats(formats) + return { - 'id': compat_str(video_data['id']), - 'url': video_url, - 'ext': 'mp4', + 'id': video_id, + 'formats': formats, 'title': video_data['caption'], - 'thumbnail': video_data['thumbnail_url'], + 'thumbnail': video_data.get('thumbnail_url'), 'upload_date': video_data['updated_at'].replace('-', '')[:8], + 'like_count': video_data.get('likes', {}).get('total'), + 'view_count': video_data.get('views'), } def _real_extract(self, url): @@ -44,16 +111,28 @@ class LivestreamIE(InfoExtractor): event_name = mobj.group('event_name') webpage = self._download_webpage(url, video_id or event_name) - if video_id is None: - # This is an event page: + og_video = self._og_search_video_url(webpage, 'player url', fatal=False, default=None) + if og_video is None: config_json = self._search_regex( r'window.config = ({.*?});', webpage, 'window config') info = json.loads(config_json)['event'] + + def is_relevant(vdata, vid): + result = vdata['type'] == 'video' + if video_id is not None: + result = result and compat_str(vdata['data']['id']) == vid + return result + videos = [self._extract_video_info(video_data['data']) - for video_data in info['feed']['data'] if video_data['type'] == 'video'] - return self.playlist_result(videos, info['id'], info['full_name']) + for video_data in info['feed']['data'] + if is_relevant(video_data, video_id)] + if video_id is None: + # This is an event page: + return self.playlist_result(videos, info['id'], info['full_name']) + else: + if videos: + return videos[0] else: - og_video = self._og_search_video_url(webpage, 'player url') query_str = compat_urllib_parse_urlparse(og_video).query query = compat_urlparse.parse_qs(query_str) api_url = query['play_url'][0].replace('.smil', '') diff --git a/youtube_dl/extractor/metacafe.py b/youtube_dl/extractor/metacafe.py index 6436c05a3..1a896b536 100644 --- a/youtube_dl/extractor/metacafe.py +++ b/youtube_dl/extractor/metacafe.py @@ -9,6 +9,7 @@ from ..utils import ( compat_urllib_request, determine_ext, ExtractorError, + int_or_none, ) @@ -83,6 +84,21 @@ class MetacafeIE(InfoExtractor): 'skip_download': True, }, }, + # Movieclips.com video + { + 'url': 'http://www.metacafe.com/watch/mv-Wy7ZU/my_week_with_marilyn_do_you_love_me/', + 'info_dict': { + 'id': 'mv-Wy7ZU', + 'ext': 'mp4', + 'title': 'My Week with Marilyn - Do You Love Me?', + 'description': 'From the movie My Week with Marilyn - Colin (Eddie Redmayne) professes his love to Marilyn (Michelle Williams) and gets her to promise to return to set and finish the movie.', + 'uploader': 'movie_trailers', + 'duration': 176, + }, + 'params': { + 'skip_download': 'requires rtmpdump', + } + } ] def report_disclaimer(self): @@ -134,6 +150,7 @@ class MetacafeIE(InfoExtractor): # Extract URL, uploader and title from webpage self.report_extraction(video_id) + video_url = None mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage) if mobj is not None: mediaURL = compat_urllib_parse.unquote(mobj.group(1)) @@ -146,16 +163,17 @@ class MetacafeIE(InfoExtractor): else: gdaKey = mobj.group(1) video_url = '%s?__gda__=%s' % (mediaURL, gdaKey) - else: + if video_url is None: mobj = re.search(r'<video src="([^"]+)"', webpage) if mobj: video_url = mobj.group(1) video_ext = 'mp4' - else: - mobj = re.search(r' name="flashvars" value="(.*?)"', webpage) - if mobj is None: - raise ExtractorError('Unable to extract media URL') - vardict = compat_parse_qs(mobj.group(1)) + if video_url is None: + flashvars = self._search_regex( + r' name="flashvars" value="(.*?)"', webpage, 'flashvars', + default=None) + if flashvars: + vardict = compat_parse_qs(flashvars) if 'mediaData' not in vardict: raise ExtractorError('Unable to extract media URL') mobj = re.search( @@ -165,26 +183,68 @@ class MetacafeIE(InfoExtractor): mediaURL = mobj.group('mediaURL').replace('\\/', '/') video_url = '%s?__gda__=%s' % (mediaURL, mobj.group('key')) video_ext = determine_ext(video_url) - - video_title = self._html_search_regex(r'(?im)<title>(.*) - Video</title>', webpage, 'title') + if video_url is None: + player_url = self._search_regex( + r"swfobject\.embedSWF\('([^']+)'", + webpage, 'config URL', default=None) + if player_url: + config_url = self._search_regex( + r'config=(.+)$', player_url, 'config URL') + config_doc = self._download_xml( + config_url, video_id, + note='Downloading video config') + smil_url = config_doc.find('.//properties').attrib['smil_file'] + smil_doc = self._download_xml( + smil_url, video_id, + note='Downloading SMIL document') + base_url = smil_doc.find('./head/meta').attrib['base'] + video_url = [] + for vn in smil_doc.findall('.//video'): + br = int(vn.attrib['system-bitrate']) + play_path = vn.attrib['src'] + video_url.append({ + 'format_id': 'smil-%d' % br, + 'url': base_url, + 'play_path': play_path, + 'page_url': url, + 'player_url': player_url, + 'ext': play_path.partition(':')[0], + }) + + if video_url is None: + raise ExtractorError('Unsupported video type') + + video_title = self._html_search_regex( + r'(?im)<title>(.*) - Video</title>', webpage, 'title') description = self._og_search_description(webpage) thumbnail = self._og_search_thumbnail(webpage) video_uploader = self._html_search_regex( r'submitter=(.*?);|googletag\.pubads\(\)\.setTargeting\("(?:channel|submiter)","([^"]+)"\);', webpage, 'uploader nickname', fatal=False) + duration = int_or_none( + self._html_search_meta('video:duration', webpage)) + + age_limit = ( + 18 + if re.search(r'"contentRating":"restricted"', webpage) + else 0) - if re.search(r'"contentRating":"restricted"', webpage) is not None: - age_limit = 18 + if isinstance(video_url, list): + formats = video_url else: - age_limit = 0 + formats = [{ + 'url': video_url, + 'ext': video_ext, + }] + self._sort_formats(formats) return { 'id': video_id, - 'url': video_url, 'description': description, 'uploader': video_uploader, 'title': video_title, - 'thumbnail':thumbnail, - 'ext': video_ext, + 'thumbnail': thumbnail, 'age_limit': age_limit, + 'formats': formats, + 'duration': duration, } diff --git a/youtube_dl/extractor/ministrygrid.py b/youtube_dl/extractor/ministrygrid.py new file mode 100644 index 000000000..949ad11db --- /dev/null +++ b/youtube_dl/extractor/ministrygrid.py @@ -0,0 +1,57 @@ +from __future__ import unicode_literals + +import json +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + smuggle_url, +) + + +class MinistryGridIE(InfoExtractor): + _VALID_URL = r'https?://www\.ministrygrid.com/([^/?#]*/)*(?P<id>[^/#?]+)/?(?:$|[?#])' + + _TEST = { + 'url': 'http://www.ministrygrid.com/training-viewer/-/training/t4g-2014-conference/the-gospel-by-numbers-4/the-gospel-by-numbers', + 'md5': '844be0d2a1340422759c2a9101bab017', + 'info_dict': { + 'id': '3453494717001', + 'ext': 'mp4', + 'title': 'The Gospel by Numbers', + 'description': 'Coming soon from T4G 2014!', + 'uploader': 'LifeWay Christian Resources (MG)', + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + portlets_json = self._search_regex( + r'Liferay\.Portlet\.list=(\[.+?\])', webpage, 'portlet list') + portlets = json.loads(portlets_json) + pl_id = self._search_regex( + r'<!--\s*p_l_id - ([0-9]+)<br>', webpage, 'p_l_id') + + for i, portlet in enumerate(portlets): + portlet_url = 'http://www.ministrygrid.com/c/portal/render_portlet?p_l_id=%s&p_p_id=%s' % (pl_id, portlet) + portlet_code = self._download_webpage( + portlet_url, video_id, + note='Looking in portlet %s (%d/%d)' % (portlet, i + 1, len(portlets)), + fatal=False) + video_iframe_url = self._search_regex( + r'<iframe.*?src="([^"]+)"', portlet_code, 'video iframe', + default=None) + if video_iframe_url: + surl = smuggle_url( + video_iframe_url, {'force_videoid': video_id}) + return { + '_type': 'url', + 'id': video_id, + 'url': surl, + } + + raise ExtractorError('Could not find video iframe in any portlets') diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py new file mode 100644 index 000000000..979f3d692 --- /dev/null +++ b/youtube_dl/extractor/mitele.py @@ -0,0 +1,60 @@ +from __future__ import unicode_literals + +import re +import json + +from .common import InfoExtractor +from ..utils import ( + compat_urllib_parse, + get_element_by_attribute, + parse_duration, + strip_jsonp, +) + + +class MiTeleIE(InfoExtractor): + IE_NAME = 'mitele.es' + _VALID_URL = r'http://www\.mitele\.es/[^/]+/[^/]+/[^/]+/(?P<episode>[^/]+)/' + + _TEST = { + 'url': 'http://www.mitele.es/programas-tv/diario-de/la-redaccion/programa-144/', + 'md5': '6a75fe9d0d3275bead0cb683c616fddb', + 'info_dict': { + 'id': '0fce117d', + 'ext': 'mp4', + 'title': 'Programa 144 - Tor, la web invisible', + 'description': 'md5:3b6fce7eaa41b2d97358726378d9369f', + 'display_id': 'programa-144', + 'duration': 2913, + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + episode = mobj.group('episode') + webpage = self._download_webpage(url, episode) + embed_data_json = self._search_regex( + r'MSV\.embedData\[.*?\]\s*=\s*({.*?});', webpage, 'embed data', + flags=re.DOTALL + ).replace('\'', '"') + embed_data = json.loads(embed_data_json) + + info_url = embed_data['flashvars']['host'] + info_el = self._download_xml(info_url, episode).find('./video/info') + + video_link = info_el.find('videoUrl/link').text + token_query = compat_urllib_parse.urlencode({'id': video_link}) + token_info = self._download_json( + 'http://token.mitele.es/?' + token_query, episode, + transform_source=strip_jsonp + ) + + return { + 'id': embed_data['videoId'], + 'display_id': episode, + 'title': info_el.find('title').text, + 'url': token_info['tokenizedUrl'], + 'description': get_element_by_attribute('class', 'text', webpage), + 'thumbnail': info_el.find('thumb').text, + 'duration': parse_duration(info_el.find('duration').text), + } diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py new file mode 100644 index 000000000..37c72bc53 --- /dev/null +++ b/youtube_dl/extractor/mlb.py @@ -0,0 +1,116 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + parse_duration, + parse_iso8601, + find_xpath_attr, +) + + +class MLBIE(InfoExtractor): + _VALID_URL = r'https?://m\.mlb\.com/(?:.*?/)?video/(?:topic/[\da-z_-]+/)?v(?P<id>n?\d+)' + _TESTS = [ + { + 'url': 'http://m.mlb.com/sea/video/topic/51231442/v34698933/nymsea-ackley-robs-a-home-run-with-an-amazing-catch/?c_id=sea', + 'md5': 'ff56a598c2cf411a9a38a69709e97079', + 'info_dict': { + 'id': '34698933', + 'ext': 'mp4', + 'title': "Ackley's spectacular catch", + 'description': 'md5:7f5a981eb4f3cbc8daf2aeffa2215bf0', + 'duration': 66, + 'timestamp': 1405980600, + 'upload_date': '20140721', + 'thumbnail': 're:^https?://.*\.jpg$', + }, + }, + { + 'url': 'http://m.mlb.com/video/topic/81536970/v34496663/mianym-stanton-practices-for-the-home-run-derby', + 'md5': 'd9c022c10d21f849f49c05ae12a8a7e9', + 'info_dict': { + 'id': '34496663', + 'ext': 'mp4', + 'title': 'Stanton prepares for Derby', + 'description': 'md5:d00ce1e5fd9c9069e9c13ab4faedfa57', + 'duration': 46, + 'timestamp': 1405105800, + 'upload_date': '20140711', + 'thumbnail': 're:^https?://.*\.jpg$', + }, + }, + { + 'url': 'http://m.mlb.com/video/topic/vtp_hrd_sponsor/v34578115/hrd-cespedes-wins-2014-gillette-home-run-derby', + 'md5': '0e6e73d509321e142409b695eadd541f', + 'info_dict': { + 'id': '34578115', + 'ext': 'mp4', + 'title': 'Cespedes repeats as Derby champ', + 'description': 'md5:08df253ce265d4cf6fb09f581fafad07', + 'duration': 488, + 'timestamp': 1405399936, + 'upload_date': '20140715', + 'thumbnail': 're:^https?://.*\.jpg$', + }, + }, + { + 'url': 'http://m.mlb.com/video/v34577915/bautista-on-derby-captaining-duties-his-performance', + 'md5': 'b8fd237347b844365d74ea61d4245967', + 'info_dict': { + 'id': '34577915', + 'ext': 'mp4', + 'title': 'Bautista on Home Run Derby', + 'description': 'md5:b80b34031143d0986dddc64a8839f0fb', + 'duration': 52, + 'timestamp': 1405390722, + 'upload_date': '20140715', + 'thumbnail': 're:^https?://.*\.jpg$', + }, + }, + ] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + detail = self._download_xml( + 'http://m.mlb.com/gen/multimedia/detail/%s/%s/%s/%s.xml' + % (video_id[-3], video_id[-2], video_id[-1], video_id), video_id) + + title = detail.find('./headline').text + description = detail.find('./big-blurb').text + duration = parse_duration(detail.find('./duration').text) + timestamp = parse_iso8601(detail.attrib['date'][:-5]) + + thumbnail = find_xpath_attr( + detail, './thumbnailScenarios/thumbnailScenario', 'type', '45').text + + formats = [] + for media_url in detail.findall('./url'): + playback_scenario = media_url.attrib['playback_scenario'] + fmt = { + 'url': media_url.text, + 'format_id': playback_scenario, + } + m = re.search(r'(?P<vbr>\d+)K_(?P<width>\d+)X(?P<height>\d+)', playback_scenario) + if m: + fmt.update({ + 'vbr': int(m.group('vbr')) * 1000, + 'width': int(m.group('width')), + 'height': int(m.group('height')), + }) + formats.append(fmt) + + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'duration': duration, + 'timestamp': timestamp, + 'formats': formats, + 'thumbnail': thumbnail, + } diff --git a/youtube_dl/extractor/mojvideo.py b/youtube_dl/extractor/mojvideo.py new file mode 100644 index 000000000..90b460d65 --- /dev/null +++ b/youtube_dl/extractor/mojvideo.py @@ -0,0 +1,58 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + parse_duration, +) + + +class MojvideoIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?mojvideo\.com/video-(?P<display_id>[^/]+)/(?P<id>[a-f0-9]+)' + _TEST = { + 'url': 'http://www.mojvideo.com/video-v-avtu-pred-mano-rdecelaska-alfi-nipic/3d1ed4497707730b2906', + 'md5': 'f7fd662cc8ce2be107b0d4f2c0483ae7', + 'info_dict': { + 'id': '3d1ed4497707730b2906', + 'display_id': 'v-avtu-pred-mano-rdecelaska-alfi-nipic', + 'ext': 'mp4', + 'title': 'V avtu pred mano rdečelaska - Alfi Nipič', + 'thumbnail': 're:^http://.*\.jpg$', + 'duration': 242, + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + display_id = mobj.group('display_id') + + # XML is malformed + playerapi = self._download_webpage( + 'http://www.mojvideo.com/playerapi.php?v=%s&t=1' % video_id, display_id) + + if '<error>true</error>' in playerapi: + error_desc = self._html_search_regex( + r'<errordesc>([^<]*)</errordesc>', playerapi, 'error description', fatal=False) + raise ExtractorError('%s said: %s' % (self.IE_NAME, error_desc), expected=True) + + title = self._html_search_regex( + r'<title>([^<]+)</title>', playerapi, 'title') + video_url = self._html_search_regex( + r'<file>([^<]+)</file>', playerapi, 'video URL') + thumbnail = self._html_search_regex( + r'<preview>([^<]+)</preview>', playerapi, 'thumbnail', fatal=False) + duration = parse_duration(self._html_search_regex( + r'<duration>([^<]+)</duration>', playerapi, 'duration', fatal=False)) + + return { + 'id': video_id, + 'display_id': display_id, + 'url': video_url, + 'title': title, + 'thumbnail': thumbnail, + 'duration': duration, + }
\ No newline at end of file diff --git a/youtube_dl/extractor/movieclips.py b/youtube_dl/extractor/movieclips.py new file mode 100644 index 000000000..456807dd1 --- /dev/null +++ b/youtube_dl/extractor/movieclips.py @@ -0,0 +1,78 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + compat_str, + clean_html, +) + + +class MovieClipsIE(InfoExtractor): + _VALID_URL = r'https?://movieclips\.com/(?P<id>[\da-zA-Z]+)(?:-(?P<display_id>[\da-z-]+))?' + _TEST = { + 'url': 'http://movieclips.com/Wy7ZU-my-week-with-marilyn-movie-do-you-love-me/', + 'info_dict': { + 'id': 'Wy7ZU', + 'display_id': 'my-week-with-marilyn-movie-do-you-love-me', + 'ext': 'mp4', + 'title': 'My Week with Marilyn - Do You Love Me?', + 'description': 'md5:e86795bd332fe3cff461e7c8dc542acb', + 'thumbnail': 're:^https?://.*\.jpg$', + }, + 'params': { + # rtmp download + 'skip_download': True, + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + display_id = mobj.group('display_id') + show_id = display_id or video_id + + config = self._download_xml( + 'http://config.movieclips.com/player/config/%s' % video_id, + show_id, 'Downloading player config') + + if config.find('./country-region').text == 'false': + raise ExtractorError( + '%s said: %s' % (self.IE_NAME, config.find('./region_alert').text), expected=True) + + properties = config.find('./video/properties') + smil_file = properties.attrib['smil_file'] + + smil = self._download_xml(smil_file, show_id, 'Downloading SMIL') + base_url = smil.find('./head/meta').attrib['base'] + + formats = [] + for video in smil.findall('./body/switch/video'): + vbr = int(video.attrib['system-bitrate']) / 1000 + src = video.attrib['src'] + formats.append({ + 'url': base_url, + 'play_path': src, + 'ext': src.split(':')[0], + 'vbr': vbr, + 'format_id': '%dk' % vbr, + }) + + self._sort_formats(formats) + + title = '%s - %s' % (properties.attrib['clip_movie_title'], properties.attrib['clip_title']) + description = clean_html(compat_str(properties.attrib['clip_description'])) + thumbnail = properties.attrib['image'] + categories = properties.attrib['clip_categories'].split(',') + + return { + 'id': video_id, + 'display_id': display_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'categories': categories, + 'formats': formats, + } diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py index af9490ccc..228b42d2b 100644 --- a/youtube_dl/extractor/mtv.py +++ b/youtube_dl/extractor/mtv.py @@ -158,6 +158,9 @@ class MTVServicesInfoExtractor(InfoExtractor): if mgid.endswith('.swf'): mgid = mgid[:-4] except RegexNotFoundError: + mgid = None + + if mgid is None or ':' not in mgid: mgid = self._search_regex( [r'data-mgid="(.*?)"', r'swfobject.embedSWF\(".*?(mgid:.*?)"'], webpage, u'mgid') diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py index aa34665d1..d2e4acbad 100644 --- a/youtube_dl/extractor/nbc.py +++ b/youtube_dl/extractor/nbc.py @@ -4,7 +4,11 @@ import re import json from .common import InfoExtractor -from ..utils import find_xpath_attr, compat_str +from ..utils import ( + compat_str, + ExtractorError, + find_xpath_attr, +) class NBCIE(InfoExtractor): @@ -85,11 +89,25 @@ class NBCNewsIE(InfoExtractor): flags=re.MULTILINE) bootstrap = json.loads(bootstrap_json) info = bootstrap['results'][0]['video'] - playlist_url = info['fallbackPlaylistUrl'] + '?form=MPXNBCNewsAPI' mpxid = info['mpxId'] - all_videos = self._download_json(playlist_url, title)['videos'] - # The response contains additional videos - info = next(v for v in all_videos if v['mpxId'] == mpxid) + + base_urls = [ + info['fallbackPlaylistUrl'], + info['associatedPlaylistUrl'], + ] + + for base_url in base_urls: + playlist_url = base_url + '?form=MPXNBCNewsAPI' + all_videos = self._download_json(playlist_url, title)['videos'] + + try: + info = next(v for v in all_videos if v['mpxId'] == mpxid) + break + except StopIteration: + continue + + if info is None: + raise ExtractorError('Could not find video in playlists') return { '_type': 'url', diff --git a/youtube_dl/extractor/nowness.py b/youtube_dl/extractor/nowness.py index 1c5e9401f..6b2f3f55a 100644 --- a/youtube_dl/extractor/nowness.py +++ b/youtube_dl/extractor/nowness.py @@ -1,3 +1,4 @@ +# encoding: utf-8 from __future__ import unicode_literals import re @@ -8,19 +9,34 @@ from ..utils import ExtractorError class NownessIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?nowness\.com/[^?#]*?/(?P<id>[0-9]+)/(?P<slug>[^/]+?)(?:$|[?#])' - - _TEST = { - 'url': 'http://www.nowness.com/day/2013/6/27/3131/candor--the-art-of-gesticulation', - 'md5': '068bc0202558c2e391924cb8cc470676', - 'info_dict': { - 'id': '2520295746001', - 'ext': 'mp4', - 'description': 'Candor: The Art of Gesticulation', - 'uploader': 'Nowness', - 'title': 'Candor: The Art of Gesticulation', - } - } + _VALID_URL = r'https?://(?:(?:www|cn)\.)?nowness\.com/[^?#]*?/(?P<id>[0-9]+)/(?P<slug>[^/]+?)(?:$|[?#])' + + _TESTS = [ + { + 'url': 'http://www.nowness.com/day/2013/6/27/3131/candor--the-art-of-gesticulation', + 'md5': '068bc0202558c2e391924cb8cc470676', + 'info_dict': { + 'id': '2520295746001', + 'ext': 'mp4', + 'title': 'Candor: The Art of Gesticulation', + 'description': 'Candor: The Art of Gesticulation', + 'thumbnail': 're:^https?://.*\.jpg', + 'uploader': 'Nowness', + } + }, + { + 'url': 'http://cn.nowness.com/day/2014/8/7/4069/kasper-bj-rke-ft-jaakko-eino-kalevi--tnr', + 'md5': 'e79cf125e387216f86b2e0a5b5c63aa3', + 'info_dict': { + 'id': '3716354522001', + 'ext': 'mp4', + 'title': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR', + 'description': 'Kasper Bjørke ft. Jaakko Eino Kalevi: TNR', + 'thumbnail': 're:^https?://.*\.jpg', + 'uploader': 'Nowness', + } + }, + ] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) diff --git a/youtube_dl/extractor/npo.py b/youtube_dl/extractor/npo.py index fbcbe1f40..12e85a716 100644 --- a/youtube_dl/extractor/npo.py +++ b/youtube_dl/extractor/npo.py @@ -32,7 +32,7 @@ class NPOIE(InfoExtractor): 'http://e.omroep.nl/metadata/aflevering/%s' % video_id, video_id, # We have to remove the javascript callback - transform_source=lambda j: re.sub(r'parseMetadata\((.*?)\);\n//epc', r'\1', j) + transform_source=lambda j: re.sub(r'parseMetadata\((.*?)\);\n//.*$', r'\1', j) ) token_page = self._download_webpage( 'http://ida.omroep.nl/npoplayer/i.js', diff --git a/youtube_dl/extractor/nuvid.py b/youtube_dl/extractor/nuvid.py index 280328b78..58ec81f91 100644 --- a/youtube_dl/extractor/nuvid.py +++ b/youtube_dl/extractor/nuvid.py @@ -38,7 +38,7 @@ class NuvidIE(InfoExtractor): webpage = self._download_webpage( request, video_id, 'Downloading %s page' % format_id) video_url = self._html_search_regex( - r'<a href="([^"]+)"\s*>Continue to watch video', webpage, '%s video URL' % format_id, fatal=False) + r'<a\s+href="([^"]+)"\s+class="b_link">', webpage, '%s video URL' % format_id, fatal=False) if not video_url: continue formats.append({ @@ -49,19 +49,24 @@ class NuvidIE(InfoExtractor): webpage = self._download_webpage( 'http://m.nuvid.com/video/%s' % video_id, video_id, 'Downloading video page') title = self._html_search_regex( - r'<div class="title">\s+<h2[^>]*>([^<]+)</h2>', webpage, 'title').strip() - thumbnail = self._html_search_regex( - r'href="(/thumbs/[^"]+)"[^>]*data-link_type="thumbs"', - webpage, 'thumbnail URL', fatal=False) + [r'<span title="([^"]+)">', + r'<div class="thumb-holder video">\s*<h5[^>]*>([^<]+)</h5>'], webpage, 'title').strip() + thumbnails = [ + { + 'url': thumb_url, + } for thumb_url in re.findall(r'<img src="([^"]+)" alt="" />', webpage) + ] + thumbnail = thumbnails[0]['url'] if thumbnails else None duration = parse_duration(self._html_search_regex( - r'Length:\s*<span>(\d{2}:\d{2})</span>',webpage, 'duration', fatal=False)) + r'<i class="fa fa-clock-o"></i>\s*(\d{2}:\d{2})', webpage, 'duration', fatal=False)) upload_date = unified_strdate(self._html_search_regex( - r'Added:\s*<span>(\d{4}-\d{2}-\d{2})</span>', webpage, 'upload date', fatal=False)) + r'<i class="fa fa-user"></i>\s*(\d{4}-\d{2}-\d{2})', webpage, 'upload date', fatal=False)) return { 'id': video_id, 'title': title, - 'thumbnail': 'http://m.nuvid.com%s' % thumbnail, + 'thumbnails': thumbnails, + 'thumbnail': thumbnail, 'duration': duration, 'upload_date': upload_date, 'age_limit': 18, diff --git a/youtube_dl/extractor/oe1.py b/youtube_dl/extractor/oe1.py deleted file mode 100644 index 38971ab4d..000000000 --- a/youtube_dl/extractor/oe1.py +++ /dev/null @@ -1,40 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -import calendar -import datetime -import re - -from .common import InfoExtractor - -# audios on oe1.orf.at are only available for 7 days, so we can't -# add tests. - - -class OE1IE(InfoExtractor): - IE_DESC = 'oe1.orf.at' - _VALID_URL = r'http://oe1\.orf\.at/programm/(?P<id>[0-9]+)' - - def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - show_id = mobj.group('id') - - data = self._download_json( - 'http://oe1.orf.at/programm/%s/konsole' % show_id, - show_id - ) - - timestamp = datetime.datetime.strptime('%s %s' % ( - data['item']['day_label'], - data['item']['time'] - ), '%d.%m.%Y %H:%M') - unix_timestamp = calendar.timegm(timestamp.utctimetuple()) - - return { - 'id': show_id, - 'title': data['item']['title'], - 'url': data['item']['url_stream'], - 'ext': 'mp3', - 'description': data['item'].get('info'), - 'timestamp': unix_timestamp - } diff --git a/youtube_dl/extractor/ooyala.py b/youtube_dl/extractor/ooyala.py index 13f12824c..2044e107e 100644 --- a/youtube_dl/extractor/ooyala.py +++ b/youtube_dl/extractor/ooyala.py @@ -3,23 +3,38 @@ import re import json from .common import InfoExtractor -from ..utils import unescapeHTML +from ..utils import ( + unescapeHTML, + ExtractorError, +) class OoyalaIE(InfoExtractor): _VALID_URL = r'(?:ooyala:|https?://.+?\.ooyala\.com/.*?(?:embedCode|ec)=)(?P<id>.+?)(&|$)' - _TEST = { - # From http://it.slashdot.org/story/13/04/25/178216/recovering-data-from-broken-hard-drives-and-ssds-video - 'url': 'http://player.ooyala.com/player.js?embedCode=pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8', - 'md5': '3f5cceb3a7bf461d6c29dc466cf8033c', - 'info_dict': { - 'id': 'pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8', - 'ext': 'mp4', - 'title': 'Explaining Data Recovery from Hard Drives and SSDs', - 'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.', + _TESTS = [ + { + # From http://it.slashdot.org/story/13/04/25/178216/recovering-data-from-broken-hard-drives-and-ssds-video + 'url': 'http://player.ooyala.com/player.js?embedCode=pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8', + 'md5': '3f5cceb3a7bf461d6c29dc466cf8033c', + 'info_dict': { + 'id': 'pxczE2YjpfHfn1f3M-ykG_AmJRRn0PD8', + 'ext': 'mp4', + 'title': 'Explaining Data Recovery from Hard Drives and SSDs', + 'description': 'How badly damaged does a drive have to be to defeat Russell and his crew? Apparently, smashed to bits.', + }, + }, { + # Only available for ipad + 'url': 'http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0', + 'md5': '4b9754921fddb68106e48c142e2a01e6', + 'info_dict': { + 'id': 'x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0', + 'ext': 'mp4', + 'title': 'Simulation Overview - Levels of Simulation', + 'description': '', + }, }, - } + ] @staticmethod def _url_for_embed_code(embed_code): @@ -47,13 +62,30 @@ class OoyalaIE(InfoExtractor): player = self._download_webpage(player_url, embedCode) mobile_url = self._search_regex(r'mobile_player_url="(.+?)&device="', player, 'mobile player url') - mobile_player = self._download_webpage(mobile_url, embedCode) - videos_info = self._search_regex( - r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);', - mobile_player, 'info').replace('\\"','"') - videos_more_info = self._search_regex(r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"','"') + # Looks like some videos are only available for particular devices + # (e.g. http://player.ooyala.com/player.js?embedCode=x1b3lqZDq9y_7kMyC2Op5qo-p077tXD0 + # is only available for ipad) + # Working around with fetching URLs for all the devices found starting with 'unknown' + # until we succeed or eventually fail for each device. + devices = re.findall(r'device\s*=\s*"([^"]+)";', player) + devices.remove('unknown') + devices.insert(0, 'unknown') + for device in devices: + mobile_player = self._download_webpage( + '%s&device=%s' % (mobile_url, device), embedCode, + 'Downloading mobile player JS for %s device' % device) + videos_info = self._search_regex( + r'var streams=window.oo_testEnv\?\[\]:eval\("\((\[{.*?}\])\)"\);', + mobile_player, 'info', fatal=False, default=None) + if videos_info: + break + if not videos_info: + raise ExtractorError('Unable to extract info') + videos_info = videos_info.replace('\\"', '"') + videos_more_info = self._search_regex( + r'eval\("\(({.*?\\"promo\\".*?})\)"', mobile_player, 'more info').replace('\\"', '"') videos_info = json.loads(videos_info) - videos_more_info =json.loads(videos_more_info) + videos_more_info = json.loads(videos_more_info) if videos_more_info.get('lineup'): videos = [self._extract_result(info, more_info) for (info, more_info) in zip(videos_info, videos_more_info['lineup'])] diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py index 03421d1d5..011e6be13 100644 --- a/youtube_dl/extractor/orf.py +++ b/youtube_dl/extractor/orf.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals import json import re +import calendar +import datetime from .common import InfoExtractor from ..utils import ( @@ -12,7 +14,9 @@ from ..utils import ( ) -class ORFIE(InfoExtractor): +class ORFTVthekIE(InfoExtractor): + IE_NAME = 'orf:tvthek' + IE_DESC = 'ORF TVthek' _VALID_URL = r'https?://tvthek\.orf\.at/(?:programs/.+?/episodes|topics/.+?|program/[^/]+)/(?P<id>\d+)' _TEST = { @@ -105,3 +109,73 @@ class ORFIE(InfoExtractor): 'entries': entries, 'id': playlist_id, } + + +# Audios on ORF radio are only available for 7 days, so we can't add tests. + + +class ORFOE1IE(InfoExtractor): + IE_NAME = 'orf:oe1' + IE_DESC = 'Radio Österreich 1' + _VALID_URL = r'http://oe1\.orf\.at/programm/(?P<id>[0-9]+)' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + show_id = mobj.group('id') + + data = self._download_json( + 'http://oe1.orf.at/programm/%s/konsole' % show_id, + show_id + ) + + timestamp = datetime.datetime.strptime('%s %s' % ( + data['item']['day_label'], + data['item']['time'] + ), '%d.%m.%Y %H:%M') + unix_timestamp = calendar.timegm(timestamp.utctimetuple()) + + return { + 'id': show_id, + 'title': data['item']['title'], + 'url': data['item']['url_stream'], + 'ext': 'mp3', + 'description': data['item'].get('info'), + 'timestamp': unix_timestamp + } + + +class ORFFM4IE(InfoExtractor): + IE_DESC = 'orf:fm4' + IE_DESC = 'radio FM4' + _VALID_URL = r'http://fm4\.orf\.at/7tage/?#(?P<date>[0-9]+)/(?P<show>\w+)' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + show_date = mobj.group('date') + show_id = mobj.group('show') + + data = self._download_json( + 'http://audioapi.orf.at/fm4/json/2.0/broadcasts/%s/4%s' % (show_date, show_id), + show_id + ) + + def extract_entry_dict(info, title, subtitle): + return { + 'id': info['loopStreamId'].replace('.mp3', ''), + 'url': 'http://loopstream01.apa.at/?channel=fm4&id=%s' % info['loopStreamId'], + 'title': title, + 'description': subtitle, + 'duration': (info['end'] - info['start']) / 1000, + 'timestamp': info['start'] / 1000, + 'ext': 'mp3' + } + + entries = [extract_entry_dict(t, data['title'], data['subtitle']) for t in data['streams']] + + return { + '_type': 'playlist', + 'id': show_id, + 'title': data['title'], + 'description': data['subtitle'], + 'entries': entries + }
\ No newline at end of file diff --git a/youtube_dl/extractor/patreon.py b/youtube_dl/extractor/patreon.py new file mode 100644 index 000000000..5429592a7 --- /dev/null +++ b/youtube_dl/extractor/patreon.py @@ -0,0 +1,100 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +import json +import re + +from .common import InfoExtractor +from ..utils import ( + js_to_json, +) + + +class PatreonIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?patreon\.com/creation\?hid=(.+)' + _TESTS = [ + { + 'url': 'http://www.patreon.com/creation?hid=743933', + 'md5': 'e25505eec1053a6e6813b8ed369875cc', + 'info_dict': { + 'id': '743933', + 'ext': 'mp3', + 'title': 'Episode 166: David Smalley of Dogma Debate', + 'uploader': 'Cognitive Dissonance Podcast', + 'thumbnail': 're:^https?://.*$', + }, + }, + { + 'url': 'http://www.patreon.com/creation?hid=754133', + 'md5': '3eb09345bf44bf60451b8b0b81759d0a', + 'info_dict': { + 'id': '754133', + 'ext': 'mp3', + 'title': 'CD 167 Extra', + 'uploader': 'Cognitive Dissonance Podcast', + 'thumbnail': 're:^https?://.*$', + }, + }, + ] + + # Currently Patreon exposes download URL via hidden CSS, so login is not + # needed. Keeping this commented for when this inevitably changes. + ''' + def _login(self): + (username, password) = self._get_login_info() + if username is None: + return + + login_form = { + 'redirectUrl': 'http://www.patreon.com/', + 'email': username, + 'password': password, + } + + request = compat_urllib_request.Request( + 'https://www.patreon.com/processLogin', + compat_urllib_parse.urlencode(login_form).encode('utf-8') + ) + login_page = self._download_webpage(request, None, note='Logging in as %s' % username) + + if re.search(r'onLoginFailed', login_page): + raise ExtractorError('Unable to login, incorrect username and/or password', expected=True) + + def _real_initialize(self): + self._login() + ''' + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group(1) + + webpage = self._download_webpage(url, video_id) + title = self._og_search_title(webpage).strip() + + attach_fn = self._html_search_regex( + r'<div class="attach"><a target="_blank" href="([^"]+)">', + webpage, 'attachment URL', default=None) + if attach_fn is not None: + video_url = 'http://www.patreon.com' + attach_fn + thumbnail = self._og_search_thumbnail(webpage) + uploader = self._html_search_regex( + r'<strong>(.*?)</strong> is creating', webpage, 'uploader') + else: + playlist_js = self._search_regex( + r'(?s)new\s+jPlayerPlaylist\(\s*\{\s*[^}]*},\s*(\[.*?,?\s*\])', + webpage, 'playlist JSON') + playlist_json = js_to_json(playlist_js) + playlist = json.loads(playlist_json) + data = playlist[0] + video_url = self._proto_relative_url(data['mp3']) + thumbnail = self._proto_relative_url(data.get('cover')) + uploader = data.get('artist') + + return { + 'id': video_id, + 'url': video_url, + 'ext': 'mp3', + 'title': title, + 'uploader': uploader, + 'thumbnail': thumbnail, + } diff --git a/youtube_dl/extractor/pbs.py b/youtube_dl/extractor/pbs.py index 64cded707..2adfde909 100644 --- a/youtube_dl/extractor/pbs.py +++ b/youtube_dl/extractor/pbs.py @@ -20,27 +20,74 @@ class PBSIE(InfoExtractor): ) ''' - _TEST = { - 'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/', - 'md5': 'ce1888486f0908d555a8093cac9a7362', - 'info_dict': { - 'id': '2365006249', - 'ext': 'mp4', - 'title': 'A More Perfect Union', - 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a', - 'duration': 3190, + _TESTS = [ + { + 'url': 'http://www.pbs.org/tpt/constitution-usa-peter-sagal/watch/a-more-perfect-union/', + 'md5': 'ce1888486f0908d555a8093cac9a7362', + 'info_dict': { + 'id': '2365006249', + 'ext': 'mp4', + 'title': 'A More Perfect Union', + 'description': 'md5:ba0c207295339c8d6eced00b7c363c6a', + 'duration': 3190, + }, + }, + { + 'url': 'http://www.pbs.org/wgbh/pages/frontline/losing-iraq/', + 'md5': '143c98aa54a346738a3d78f54c925321', + 'info_dict': { + 'id': '2365297690', + 'ext': 'mp4', + 'title': 'Losing Iraq', + 'description': 'md5:f5bfbefadf421e8bb8647602011caf8e', + 'duration': 5050, + }, + }, + { + 'url': 'http://www.pbs.org/newshour/bb/education-jan-june12-cyberschools_02-23/', + 'md5': 'b19856d7f5351b17a5ab1dc6a64be633', + 'info_dict': { + 'id': '2201174722', + 'ext': 'mp4', + 'title': 'Cyber Schools Gain Popularity, but Quality Questions Persist', + 'description': 'md5:5871c15cba347c1b3d28ac47a73c7c28', + 'duration': 801, + }, }, - } + { + 'url': 'http://www.pbs.org/wnet/gperf/dudamel-conducts-verdi-requiem-hollywood-bowl-full-episode/3374/', + 'md5': 'c62859342be2a0358d6c9eb306595978', + 'info_dict': { + 'id': '2365297708', + 'ext': 'mp4', + 'description': 'md5:68d87ef760660eb564455eb30ca464fe', + 'title': 'Dudamel Conducts Verdi Requiem at the Hollywood Bowl - Full', + 'duration': 6559, + 'thumbnail': 're:^https?://.*\.jpg$', + } + } + ] - def _real_extract(self, url): + def _extract_ids(self, url): mobj = re.match(self._VALID_URL, url) presumptive_id = mobj.group('presumptive_id') display_id = presumptive_id if presumptive_id: webpage = self._download_webpage(url, display_id) + + MEDIA_ID_REGEXES = [ + r"div\s*:\s*'videoembed'\s*,\s*mediaid\s*:\s*'(\d+)'", # frontline video embed + r'class="coveplayerid">([^<]+)<', # coveplayer + ] + + media_id = self._search_regex( + MEDIA_ID_REGEXES, webpage, 'media ID', fatal=False, default=None) + if media_id: + return media_id, presumptive_id + url = self._search_regex( - r'<iframe\s+id=["\']partnerPlayer["\'].*?\s+src=["\'](.*?)["\']>', + r'<iframe\s+(?:class|id)=["\']partnerPlayer["\'].*?\s+src=["\'](.*?)["\']>', webpage, 'player URL') mobj = re.match(self._VALID_URL, url) @@ -57,6 +104,11 @@ class PBSIE(InfoExtractor): video_id = mobj.group('id') display_id = video_id + return video_id, display_id + + def _real_extract(self, url): + video_id, display_id = self._extract_ids(url) + info_url = 'http://video.pbs.org/videoInfo/%s?format=json' % video_id info = self._download_json(info_url, display_id) diff --git a/youtube_dl/extractor/playfm.py b/youtube_dl/extractor/playfm.py new file mode 100644 index 000000000..72df4d842 --- /dev/null +++ b/youtube_dl/extractor/playfm.py @@ -0,0 +1,82 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + compat_urllib_parse, + compat_urllib_request, + ExtractorError, + float_or_none, + int_or_none, +) + + +class PlayFMIE(InfoExtractor): + IE_NAME = 'play.fm' + _VALID_URL = r'https?://(?:www\.)?play\.fm/[^?#]*(?P<upload_date>[0-9]{8})(?P<id>[0-9]{6})(?:$|[?#])' + + _TEST = { + 'url': 'http://www.play.fm/recording/leipzigelectronicmusicbatofarparis_fr20140712137220', + 'md5': 'c505f8307825a245d0c7ad1850001f22', + 'info_dict': { + 'id': '137220', + 'ext': 'mp3', + 'title': 'LEIPZIG ELECTRONIC MUSIC @ Batofar (Paris,FR) - 2014-07-12', + 'uploader': 'Sven Tasnadi', + 'uploader_id': 'sventasnadi', + 'duration': 5627.428, + 'upload_date': '20140712', + 'view_count': int, + 'thumbnail': 're:^https?://.*\.jpg$', + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + upload_date = mobj.group('upload_date') + + rec_data = compat_urllib_parse.urlencode({'rec_id': video_id}) + req = compat_urllib_request.Request( + 'http://www.play.fm/flexRead/recording', data=rec_data) + req.add_header('Content-Type', 'application/x-www-form-urlencoded') + rec_doc = self._download_xml(req, video_id) + + error_node = rec_doc.find('./error') + if error_node is not None: + raise ExtractorError('An error occured: %s (code %s)' % ( + error_node.text, rec_doc.find('./status').text)) + + recording = rec_doc.find('./recording') + title = recording.find('./title').text + view_count = int_or_none(recording.find('./stats/playcount').text) + duration = float_or_none(recording.find('./duration').text, scale=1000) + thumbnail = recording.find('./image').text + + artist = recording.find('./artists/artist') + uploader = artist.find('./name').text + uploader_id = artist.find('./slug').text + + video_url = '%s//%s/%s/%s/offset/0/sh/%s/rec/%s/jingle/%s/loc/%s' % ( + 'http:', recording.find('./url').text, + recording.find('./_class').text, recording.find('./file_id').text, + rec_doc.find('./uuid').text, video_id, + rec_doc.find('./jingle/file_id').text, + 'http%3A%2F%2Fwww.play.fm%2Fplayer', + ) + + return { + 'id': video_id, + 'url': video_url, + 'ext': 'mp3', + 'filesize': int_or_none(recording.find('./size').text), + 'title': title, + 'upload_date': upload_date, + 'view_count': view_count, + 'duration': duration, + 'thumbnail': thumbnail, + 'uploader': uploader, + 'uploader_id': uploader_id, + } diff --git a/youtube_dl/extractor/pornotube.py b/youtube_dl/extractor/pornotube.py index 35dc5a9ff..04bd3d979 100644 --- a/youtube_dl/extractor/pornotube.py +++ b/youtube_dl/extractor/pornotube.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import re from .common import InfoExtractor @@ -9,15 +11,16 @@ from ..utils import ( class PornotubeIE(InfoExtractor): - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$' + _VALID_URL = r'https?://(?:\w+\.)?pornotube\.com(/c/(?P<channel>[0-9]+))?(/m/(?P<videoid>[0-9]+))(/(?P<title>.+))$' _TEST = { - u'url': u'http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing', - u'file': u'1689755.flv', - u'md5': u'374dd6dcedd24234453b295209aa69b6', - u'info_dict': { - u"upload_date": u"20090708", - u"title": u"Marilyn-Monroe-Bathing", - u"age_limit": 18 + 'url': 'http://pornotube.com/c/173/m/1689755/Marilyn-Monroe-Bathing', + 'md5': '374dd6dcedd24234453b295209aa69b6', + 'info_dict': { + 'id': '1689755', + 'ext': 'flv', + 'upload_date': '20090708', + 'title': 'Marilyn-Monroe-Bathing', + 'age_limit': 18 } } @@ -32,22 +35,22 @@ class PornotubeIE(InfoExtractor): # Get the video URL VIDEO_URL_RE = r'url: "(?P<url>http://video[0-9].pornotube.com/.+\.flv)",' - video_url = self._search_regex(VIDEO_URL_RE, webpage, u'video url') + video_url = self._search_regex(VIDEO_URL_RE, webpage, 'video url') video_url = compat_urllib_parse.unquote(video_url) #Get the uploaded date VIDEO_UPLOADED_RE = r'<div class="video_added_by">Added (?P<date>[0-9\/]+) by' - upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, u'upload date', fatal=False) - if upload_date: upload_date = unified_strdate(upload_date) + upload_date = self._html_search_regex(VIDEO_UPLOADED_RE, webpage, 'upload date', fatal=False) + if upload_date: + upload_date = unified_strdate(upload_date) age_limit = self._rta_search(webpage) - info = {'id': video_id, - 'url': video_url, - 'uploader': None, - 'upload_date': upload_date, - 'title': video_title, - 'ext': 'flv', - 'format': 'flv', - 'age_limit': age_limit} - - return [info] + return { + 'id': video_id, + 'url': video_url, + 'upload_date': upload_date, + 'title': video_title, + 'ext': 'flv', + 'format': 'flv', + 'age_limit': age_limit, + } diff --git a/youtube_dl/extractor/pyvideo.py b/youtube_dl/extractor/pyvideo.py index 0bc0859b4..6d5732d45 100644 --- a/youtube_dl/extractor/pyvideo.py +++ b/youtube_dl/extractor/pyvideo.py @@ -46,7 +46,7 @@ class PyvideoIE(InfoExtractor): return self.url_result(m_youtube.group(1), 'Youtube') title = self._html_search_regex( - r'<div class="section">.*?<h3(?:\s+class="[^"]*")?>([^>]+?)</h3>', + r'<div class="section">\s*<h3(?:\s+class="[^"]*"[^>]*)?>([^>]+?)</h3>', webpage, 'title', flags=re.DOTALL) video_url = self._search_regex( [r'<source src="(.*?)"', r'<dt>Download</dt>.*?<a href="(.+?)"'], diff --git a/youtube_dl/extractor/redtube.py b/youtube_dl/extractor/redtube.py index 4295cf93a..d1e12dd8d 100644 --- a/youtube_dl/extractor/redtube.py +++ b/youtube_dl/extractor/redtube.py @@ -35,9 +35,7 @@ class RedTubeIE(InfoExtractor): r'<h1 class="videoTitle[^"]*">(.+?)</h1>', webpage, u'title') - video_thumbnail = self._html_search_regex( - r'playerInnerHTML.+?<img\s+src="(.+?)"', - webpage, u'thumbnail', fatal=False) + video_thumbnail = self._og_search_thumbnail(webpage) # No self-labeling, but they describe themselves as # "Home of Videos Porno" diff --git a/youtube_dl/extractor/reverbnation.py b/youtube_dl/extractor/reverbnation.py index 49cf427a1..ec7e7df7b 100644 --- a/youtube_dl/extractor/reverbnation.py +++ b/youtube_dl/extractor/reverbnation.py @@ -1,23 +1,23 @@ from __future__ import unicode_literals import re -import time from .common import InfoExtractor -from ..utils import strip_jsonp +from ..utils import str_or_none class ReverbNationIE(InfoExtractor): _VALID_URL = r'^https?://(?:www\.)?reverbnation\.com/.*?/song/(?P<id>\d+).*?$' _TESTS = [{ 'url': 'http://www.reverbnation.com/alkilados/song/16965047-mona-lisa', - 'file': '16965047.mp3', 'md5': '3da12ebca28c67c111a7f8b262d3f7a7', 'info_dict': { + "id": "16965047", + "ext": "mp3", "title": "MONA LISA", "uploader": "ALKILADOS", - "uploader_id": 216429, - "thumbnail": "//gp1.wac.edgecastcdn.net/802892/production_public/Photo/13761700/image/1366002176_AVATAR_MONA_LISA.jpg" + "uploader_id": "216429", + "thumbnail": "re:^https://gp1\.wac\.edgecastcdn\.net/.*?\.jpg$" }, }] @@ -26,10 +26,8 @@ class ReverbNationIE(InfoExtractor): song_id = mobj.group('id') api_res = self._download_json( - 'https://api.reverbnation.com/song/%s?callback=api_response_5&_=%d' - % (song_id, int(time.time() * 1000)), + 'https://api.reverbnation.com/song/%s' % song_id, song_id, - transform_source=strip_jsonp, note='Downloading information of song %s' % song_id ) @@ -38,8 +36,9 @@ class ReverbNationIE(InfoExtractor): 'title': api_res.get('name'), 'url': api_res.get('url'), 'uploader': api_res.get('artist', {}).get('name'), - 'uploader_id': api_res.get('artist', {}).get('id'), - 'thumbnail': api_res.get('image', api_res.get('thumbnail')), + 'uploader_id': str_or_none(api_res.get('artist', {}).get('id')), + 'thumbnail': self._proto_relative_url( + api_res.get('image', api_res.get('thumbnail'))), 'ext': 'mp3', 'vcodec': 'none', } diff --git a/youtube_dl/extractor/rtbf.py b/youtube_dl/extractor/rtbf.py index 205f8a167..dce64e151 100644 --- a/youtube_dl/extractor/rtbf.py +++ b/youtube_dl/extractor/rtbf.py @@ -30,7 +30,7 @@ class RTBFIE(InfoExtractor): page = self._download_webpage('https://www.rtbf.be/video/embed?id=%s' % video_id, video_id) data = json.loads(self._html_search_regex( - r'<div class="js-player-embed" data-video="([^"]+)"', page, 'data video'))['data'] + r'<div class="js-player-embed(?: player-embed)?" data-video="([^"]+)"', page, 'data video'))['data'] video_url = data.get('downloadUrl') or data.get('url') diff --git a/youtube_dl/extractor/rtlnl.py b/youtube_dl/extractor/rtlnl.py new file mode 100644 index 000000000..190c8f226 --- /dev/null +++ b/youtube_dl/extractor/rtlnl.py @@ -0,0 +1,51 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor + + +class RtlXlIE(InfoExtractor): + IE_NAME = 'rtlxl.nl' + _VALID_URL = r'https?://www\.rtlxl\.nl/#!/[^/]+/(?P<uuid>[^/?]+)' + + _TEST = { + 'url': 'http://www.rtlxl.nl/#!/rtl-nieuws-132237/6e4203a6-0a5e-3596-8424-c599a59e0677', + 'info_dict': { + 'id': '6e4203a6-0a5e-3596-8424-c599a59e0677', + 'ext': 'flv', + 'title': 'RTL Nieuws - Laat', + 'description': 'Dagelijks het laatste nieuws uit binnen- en ' + 'buitenland. Voor nog meer nieuws kunt u ook gebruikmaken van ' + 'onze mobiele apps.', + 'timestamp': 1408051800, + 'upload_date': '20140814', + }, + 'params': { + # We download the first bytes of the first fragment, it can't be + # processed by the f4m downloader beacuse it isn't complete + 'skip_download': True, + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + uuid = mobj.group('uuid') + + info = self._download_json( + 'http://www.rtl.nl/system/s4m/vfd/version=2/uuid=%s/fmt=flash/' % uuid, + uuid) + material = info['material'][0] + episode_info = info['episodes'][0] + + f4m_url = 'http://manifest.us.rtl.nl' + material['videopath'] + progname = info['abstracts'][0]['name'] + subtitle = material['title'] or info['episodes'][0]['name'] + + return { + 'id': uuid, + 'title': '%s - %s' % (progname, subtitle), + 'formats': self._extract_f4m_formats(f4m_url, uuid), + 'timestamp': material['original_date'], + 'description': episode_info['synopsis'], + } diff --git a/youtube_dl/extractor/rtlnow.py b/youtube_dl/extractor/rtlnow.py index 4835ec5ec..a45884b25 100644 --- a/youtube_dl/extractor/rtlnow.py +++ b/youtube_dl/extractor/rtlnow.py @@ -92,16 +92,7 @@ class RTLnowIE(InfoExtractor): }, { 'url': 'http://www.n-tvnow.de/deluxe-alles-was-spass-macht/thema-ua-luxushotel-fuer-vierbeiner.php?container_id=153819&player=1&season=0', - 'info_dict': { - 'id': '153819', - 'ext': 'flv', - 'title': 'Deluxe - Alles was Spaß macht - Thema u.a.: Luxushotel für Vierbeiner', - 'description': 'md5:c3705e1bb32e1a5b2bcd634fc065c631', - 'thumbnail': 'http://autoimg.static-fra.de/ntvnow/383157/1500x1500/image2.jpg', - 'upload_date': '20140221', - 'duration': 2429, - }, - 'skip': 'Only works from Germany', + 'only_matching': True, }, ] diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py index 77fd08dde..4dd35a47b 100644 --- a/youtube_dl/extractor/rtve.py +++ b/youtube_dl/extractor/rtve.py @@ -1,70 +1,82 @@ # encoding: utf-8 from __future__ import unicode_literals -import re import base64 +import re +import time from .common import InfoExtractor from ..utils import ( struct_unpack, + remove_end, ) +def _decrypt_url(png): + encrypted_data = base64.b64decode(png) + text_index = encrypted_data.find(b'tEXt') + text_chunk = encrypted_data[text_index - 4:] + length = struct_unpack('!I', text_chunk[:4])[0] + # Use bytearray to get integers when iterating in both python 2.x and 3.x + data = bytearray(text_chunk[8:8 + length]) + data = [chr(b) for b in data if b != 0] + hash_index = data.index('#') + alphabet_data = data[:hash_index] + url_data = data[hash_index + 1:] + + alphabet = [] + e = 0 + d = 0 + for l in alphabet_data: + if d == 0: + alphabet.append(l) + d = e = (e + 1) % 4 + else: + d -= 1 + url = '' + f = 0 + e = 3 + b = 1 + for letter in url_data: + if f == 0: + l = int(letter) * 10 + f = 1 + else: + if e == 0: + l += int(letter) + url += alphabet[l] + e = (b + 3) % 4 + f = 0 + b += 1 + else: + e -= 1 + + return url + + + class RTVEALaCartaIE(InfoExtractor): IE_NAME = 'rtve.es:alacarta' IE_DESC = 'RTVE a la carta' _VALID_URL = r'http://www\.rtve\.es/alacarta/videos/[^/]+/[^/]+/(?P<id>\d+)' - _TEST = { + _TESTS = [{ 'url': 'http://www.rtve.es/alacarta/videos/balonmano/o-swiss-cup-masculina-final-espana-suecia/2491869/', - 'md5': '18fcd45965bdd076efdb12cd7f6d7b9e', + 'md5': '1d49b7e1ca7a7502c56a4bf1b60f1b43', 'info_dict': { 'id': '2491869', 'ext': 'mp4', 'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia', }, - } - - def _decrypt_url(self, png): - encrypted_data = base64.b64decode(png) - text_index = encrypted_data.find(b'tEXt') - text_chunk = encrypted_data[text_index-4:] - length = struct_unpack('!I', text_chunk[:4])[0] - # Use bytearray to get integers when iterating in both python 2.x and 3.x - data = bytearray(text_chunk[8:8+length]) - data = [chr(b) for b in data if b != 0] - hash_index = data.index('#') - alphabet_data = data[:hash_index] - url_data = data[hash_index+1:] - - alphabet = [] - e = 0 - d = 0 - for l in alphabet_data: - if d == 0: - alphabet.append(l) - d = e = (e + 1) % 4 - else: - d -= 1 - url = '' - f = 0 - e = 3 - b = 1 - for letter in url_data: - if f == 0: - l = int(letter)*10 - f = 1 - else: - if e == 0: - l += int(letter) - url += alphabet[l] - e = (b + 3) % 4 - f = 0 - b += 1 - else: - e -= 1 - - return url + }, { + 'note': 'Live stream', + 'url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/', + 'info_dict': { + 'id': '1694255', + 'ext': 'flv', + 'title': 'TODO', + } + }] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) @@ -74,11 +86,57 @@ class RTVEALaCartaIE(InfoExtractor): video_id)['page']['items'][0] png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % video_id png = self._download_webpage(png_url, video_id, 'Downloading url information') - video_url = self._decrypt_url(png) + video_url = _decrypt_url(png) return { 'id': video_id, 'title': info['title'], 'url': video_url, - 'thumbnail': info['image'], + 'thumbnail': info.get('image'), + 'page_url': url, + } + + +class RTVELiveIE(InfoExtractor): + IE_NAME = 'rtve.es:live' + IE_DESC = 'RTVE.es live streams' + _VALID_URL = r'http://www\.rtve\.es/(?:deportes/directo|noticias|television)/(?P<id>[a-zA-Z0-9-]+)' + + _TESTS = [{ + 'url': 'http://www.rtve.es/noticias/directo-la-1/', + 'info_dict': { + 'id': 'directo-la-1', + 'ext': 'flv', + 'title': 're:^La 1 de TVE [0-9]{4}-[0-9]{2}-[0-9]{2}Z[0-9]{6}$', + }, + 'params': { + 'skip_download': 'live stream', + } + }] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + start_time = time.gmtime() + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + player_url = self._search_regex( + r'<param name="movie" value="([^"]+)"/>', webpage, 'player URL') + title = remove_end(self._og_search_title(webpage), ' en directo') + title += ' ' + time.strftime('%Y-%m-%dZ%H%M%S', start_time) + + vidplayer_id = self._search_regex( + r' id="vidplayer([0-9]+)"', webpage, 'internal video ID') + png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id + png = self._download_webpage(png_url, video_id, 'Downloading url information') + video_url = _decrypt_url(png) + + return { + 'id': video_id, + 'ext': 'flv', + 'title': title, + 'url': video_url, + 'app': 'rtve-live-live?ovpfv=2.1.2', + 'player_url': player_url, + 'rtmp_live': True, } diff --git a/youtube_dl/extractor/sapo.py b/youtube_dl/extractor/sapo.py new file mode 100644 index 000000000..172cc1275 --- /dev/null +++ b/youtube_dl/extractor/sapo.py @@ -0,0 +1,119 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + parse_duration, + unified_strdate, +) + + +class SapoIE(InfoExtractor): + IE_DESC = 'SAPO Vídeos' + _VALID_URL = r'https?://(?:(?:v2|www)\.)?videos\.sapo\.(?:pt|cv|ao|mz|tl)/(?P<id>[\da-zA-Z]{20})' + + _TESTS = [ + { + 'url': 'http://videos.sapo.pt/UBz95kOtiWYUMTA5Ghfi', + 'md5': '79ee523f6ecb9233ac25075dee0eda83', + 'note': 'SD video', + 'info_dict': { + 'id': 'UBz95kOtiWYUMTA5Ghfi', + 'ext': 'mp4', + 'title': 'Benfica - Marcas na Hitória', + 'description': 'md5:c9082000a128c3fd57bf0299e1367f22', + 'duration': 264, + 'uploader': 'tiago_1988', + 'upload_date': '20080229', + 'categories': ['benfica', 'cabral', 'desporto', 'futebol', 'geovanni', 'hooijdonk', 'joao', 'karel', 'lisboa', 'miccoli'], + }, + }, + { + 'url': 'http://videos.sapo.pt/IyusNAZ791ZdoCY5H5IF', + 'md5': '90a2f283cfb49193fe06e861613a72aa', + 'note': 'HD video', + 'info_dict': { + 'id': 'IyusNAZ791ZdoCY5H5IF', + 'ext': 'mp4', + 'title': 'Codebits VII - Report', + 'description': 'md5:6448d6fd81ce86feac05321f354dbdc8', + 'duration': 144, + 'uploader': 'codebits', + 'upload_date': '20140427', + 'categories': ['codebits', 'codebits2014'], + }, + }, + { + 'url': 'http://v2.videos.sapo.pt/yLqjzPtbTimsn2wWBKHz', + 'md5': 'e5aa7cc0bdc6db9b33df1a48e49a15ac', + 'note': 'v2 video', + 'info_dict': { + 'id': 'yLqjzPtbTimsn2wWBKHz', + 'ext': 'mp4', + 'title': 'Hipnose Condicionativa 4', + 'description': 'md5:ef0481abf8fb4ae6f525088a6dadbc40', + 'duration': 692, + 'uploader': 'sapozen', + 'upload_date': '20090609', + 'categories': ['condicionativa', 'heloisa', 'hipnose', 'miranda', 'sapo', 'zen'], + }, + }, + ] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + item = self._download_xml( + 'http://rd3.videos.sapo.pt/%s/rss2' % video_id, video_id).find('./channel/item') + + title = item.find('./title').text + description = item.find('./{http://videos.sapo.pt/mrss/}synopse').text + thumbnail = item.find('./{http://search.yahoo.com/mrss/}content').get('url') + duration = parse_duration(item.find('./{http://videos.sapo.pt/mrss/}time').text) + uploader = item.find('./{http://videos.sapo.pt/mrss/}author').text + upload_date = unified_strdate(item.find('./pubDate').text) + view_count = int(item.find('./{http://videos.sapo.pt/mrss/}views').text) + comment_count = int(item.find('./{http://videos.sapo.pt/mrss/}comment_count').text) + tags = item.find('./{http://videos.sapo.pt/mrss/}tags').text + categories = tags.split() if tags else [] + age_limit = 18 if item.find('./{http://videos.sapo.pt/mrss/}m18').text == 'true' else 0 + + video_url = item.find('./{http://videos.sapo.pt/mrss/}videoFile').text + video_size = item.find('./{http://videos.sapo.pt/mrss/}videoSize').text.split('x') + + formats = [{ + 'url': video_url, + 'ext': 'mp4', + 'format_id': 'sd', + 'width': int(video_size[0]), + 'height': int(video_size[1]), + }] + + if item.find('./{http://videos.sapo.pt/mrss/}HD').text == 'true': + formats.append({ + 'url': re.sub(r'/mov/1$', '/mov/39', video_url), + 'ext': 'mp4', + 'format_id': 'hd', + 'width': 1280, + 'height': 720, + }) + + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'duration': duration, + 'uploader': uploader, + 'upload_date': upload_date, + 'view_count': view_count, + 'comment_count': comment_count, + 'categories': categories, + 'age_limit': age_limit, + 'formats': formats, + } diff --git a/youtube_dl/extractor/savefrom.py b/youtube_dl/extractor/savefrom.py index 198a08c1c..5b7367b94 100644 --- a/youtube_dl/extractor/savefrom.py +++ b/youtube_dl/extractor/savefrom.py @@ -20,7 +20,7 @@ class SaveFromIE(InfoExtractor): 'upload_date': '20120816', 'uploader': 'Howcast', 'uploader_id': 'Howcast', - 'description': 'md5:4f0aac94361a12e1ce57d74f85265175', + 'description': 're:(?s).* Hi, my name is Rene Dreifuss\. And I\'m here to show you some MMA.*', }, 'params': { 'skip_download': True diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py new file mode 100644 index 000000000..34058fd4b --- /dev/null +++ b/youtube_dl/extractor/sbs.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import json +import re +from .common import InfoExtractor +from ..utils import ( + js_to_json, + remove_end, +) + + +class SBSIE(InfoExtractor): + IE_DESC = 'sbs.com.au' + _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/ondemand/video/single/(?P<id>[0-9]+)/' + + _TESTS = [{ + # Original URL is handled by the generic IE which finds the iframe: + # http://www.sbs.com.au/thefeed/blog/2014/08/21/dingo-conservation + 'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed', + 'md5': '3150cf278965eeabb5b4cea1c963fe0a', + 'info_dict': { + 'id': '320403011771', + 'ext': 'flv', + 'title': 'Dingo Conservation', + 'description': 'Dingoes are on the brink of extinction; most of the animals we think are dingoes are in fact crossbred with wild dogs. This family run a dingo conservation park to prevent their extinction', + 'thumbnail': 're:http://.*\.jpg', + }, + 'add_ies': ['generic'], + }] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + webpage = self._download_webpage(url, video_id) + + release_urls_json = js_to_json(self._search_regex( + r'(?s)playerParams\.releaseUrls\s*=\s*(\{.*?\n\});\n', + webpage, '')) + release_urls = json.loads(release_urls_json) + theplatform_url = ( + release_urls.get('progressive') or release_urls.get('standard')) + + title = remove_end(self._og_search_title(webpage), ' (The Feed)') + description = self._html_search_meta('description', webpage) + thumbnail = self._og_search_thumbnail(webpage) + + return { + '_type': 'url_transparent', + 'id': video_id, + 'url': theplatform_url, + + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + } diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py new file mode 100644 index 000000000..badba2ac6 --- /dev/null +++ b/youtube_dl/extractor/shared.py @@ -0,0 +1,57 @@ +from __future__ import unicode_literals + +import re +import base64 + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + compat_urllib_request, + compat_urllib_parse, + int_or_none, +) + + +class SharedIE(InfoExtractor): + _VALID_URL = r'http://shared\.sx/(?P<id>[\da-z]{10})' + + _TEST = { + 'url': 'http://shared.sx/0060718775', + 'md5': '106fefed92a8a2adb8c98e6a0652f49b', + 'info_dict': { + 'id': '0060718775', + 'ext': 'mp4', + 'title': 'Bmp4', + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + page = self._download_webpage(url, video_id) + + if re.search(r'>File does not exist<', page) is not None: + raise ExtractorError('Video %s does not exist' % video_id, expected=True) + + download_form = dict(re.findall(r'<input type="hidden" name="([^"]+)" value="([^"]*)"', page)) + + request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(download_form)) + request.add_header('Content-Type', 'application/x-www-form-urlencoded') + + video_page = self._download_webpage(request, video_id, 'Downloading video page') + + video_url = self._html_search_regex(r'data-url="([^"]+)"', video_page, 'video URL') + title = base64.b64decode(self._html_search_meta('full:title', page, 'title')).decode('utf-8') + filesize = int_or_none(self._html_search_meta('full:size', page, 'file size', fatal=False)) + thumbnail = self._html_search_regex( + r'data-poster="([^"]+)"', video_page, 'thumbnail', fatal=False, default=None) + + return { + 'id': video_id, + 'url': video_url, + 'ext': 'mp4', + 'filesize': filesize, + 'title': title, + 'thumbnail': thumbnail, + }
\ No newline at end of file diff --git a/youtube_dl/extractor/snotr.py b/youtube_dl/extractor/snotr.py new file mode 100644 index 000000000..da3b05a8d --- /dev/null +++ b/youtube_dl/extractor/snotr.py @@ -0,0 +1,68 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + float_or_none, + str_to_int, + parse_duration, +) + + +class SnotrIE(InfoExtractor): + _VALID_URL = r'http?://(?:www\.)?snotr\.com/video/(?P<id>\d+)/([\w]+)' + _TESTS = [{ + 'url': 'http://www.snotr.com/video/13708/Drone_flying_through_fireworks', + 'info_dict': { + 'id': '13708', + 'ext': 'flv', + 'title': 'Drone flying through fireworks!', + 'duration': 247, + 'filesize_approx': 98566144, + 'description': 'A drone flying through Fourth of July Fireworks', + } + }, { + 'url': 'http://www.snotr.com/video/530/David_Letteman_-_George_W_Bush_Top_10', + 'info_dict': { + 'id': '530', + 'ext': 'flv', + 'title': 'David Letteman - George W. Bush Top 10', + 'duration': 126, + 'filesize_approx': 8912896, + 'description': 'The top 10 George W. Bush moments, brought to you by David Letterman!', + } + }] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + title = self._og_search_title(webpage) + + description = self._og_search_description(webpage) + video_url = "http://cdn.videos.snotr.com/%s.flv" % video_id + + view_count = str_to_int(self._html_search_regex( + r'<p>\n<strong>Views:</strong>\n([\d,\.]+)</p>', + webpage, 'view count', fatal=False)) + + duration = parse_duration(self._html_search_regex( + r'<p>\n<strong>Length:</strong>\n\s*([0-9:]+).*?</p>', + webpage, 'duration', fatal=False)) + + filesize_approx = float_or_none(self._html_search_regex( + r'<p>\n<strong>Filesize:</strong>\n\s*([0-9.]+)\s*megabyte</p>', + webpage, 'filesize', fatal=False), invscale=1024 * 1024) + + return { + 'id': video_id, + 'description': description, + 'title': title, + 'url': video_url, + 'view_count': view_count, + 'duration': duration, + 'filesize_approx': filesize_approx, + } diff --git a/youtube_dl/extractor/sockshare.py b/youtube_dl/extractor/sockshare.py new file mode 100644 index 000000000..dc9f80550 --- /dev/null +++ b/youtube_dl/extractor/sockshare.py @@ -0,0 +1,80 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from ..utils import ( + ExtractorError, + compat_urllib_parse, + compat_urllib_request, + determine_ext, +) +import re + +from .common import InfoExtractor + + +class SockshareIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?sockshare\.com/file/(?P<id>[0-9A-Za-z]+)' + _FILE_DELETED_REGEX = r'This file doesn\'t exist, or has been removed\.</div>' + _TEST = { + 'url': 'http://www.sockshare.com/file/437BE28B89D799D7', + 'md5': '9d0bf1cfb6dbeaa8d562f6c97506c5bd', + 'info_dict': { + 'id': '437BE28B89D799D7', + 'title': 'big_buck_bunny_720p_surround.avi', + 'ext': 'avi', + 'thumbnail': 're:^http://.*\.jpg$', + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + url = 'http://sockshare.com/file/%s' % video_id + webpage = self._download_webpage(url, video_id) + + if re.search(self._FILE_DELETED_REGEX, webpage) is not None: + raise ExtractorError('Video %s does not exist' % video_id, + expected=True) + + confirm_hash = self._html_search_regex(r'''(?x)<input\s+ + type="hidden"\s+ + value="([^"]*)"\s+ + name="hash" + ''', webpage, 'hash') + + fields = { + "hash": confirm_hash, + "confirm": "Continue as Free User" + } + + post = compat_urllib_parse.urlencode(fields) + req = compat_urllib_request.Request(url, post) + # Apparently, this header is required for confirmation to work. + req.add_header('Host', 'www.sockshare.com') + req.add_header('Content-type', 'application/x-www-form-urlencoded') + + webpage = self._download_webpage( + req, video_id, 'Downloading video page') + + video_url = self._html_search_regex( + r'<a href="([^"]*)".+class="download_file_link"', + webpage, 'file url') + video_url = "http://www.sockshare.com" + video_url + title = self._html_search_regex(r'<h1>(.+)<strong>', webpage, 'title') + thumbnail = self._html_search_regex( + r'<img\s+src="([^"]*)".+?name="bg"', + webpage, 'thumbnail') + + formats = [{ + 'format_id': 'sd', + 'url': video_url, + 'ext': determine_ext(title), + }] + + return { + 'id': video_id, + 'title': title, + 'thumbnail': thumbnail, + 'formats': formats, + } diff --git a/youtube_dl/extractor/soundcloud.py b/youtube_dl/extractor/soundcloud.py index 14ec9452d..097d0e418 100644 --- a/youtube_dl/extractor/soundcloud.py +++ b/youtube_dl/extractor/soundcloud.py @@ -81,16 +81,16 @@ class SoundcloudIE(InfoExtractor): }, # downloadable song { - 'url': 'https://soundcloud.com/simgretina/just-your-problem-baby-1', - 'md5': '56a8b69568acaa967b4c49f9d1d52d19', + 'url': 'https://soundcloud.com/oddsamples/bus-brakes', + 'md5': '7624f2351f8a3b2e7cd51522496e7631', 'info_dict': { - 'id': '105614606', - 'ext': 'wav', - 'title': 'Just Your Problem Baby (Acapella)', - 'description': 'Vocals', - 'uploader': 'Sim Gretina', - 'upload_date': '20130815', - #'duration': 42, + 'id': '128590877', + 'ext': 'mp3', + 'title': 'Bus Brakes', + 'description': 'md5:0170be75dd395c96025d210d261c784e', + 'uploader': 'oddsamples', + 'upload_date': '20140109', + 'duration': 17, }, }, ] diff --git a/youtube_dl/extractor/southparkstudios.py b/youtube_dl/extractor/southpark.py index aea8e6439..c20397b3d 100644 --- a/youtube_dl/extractor/southparkstudios.py +++ b/youtube_dl/extractor/southpark.py @@ -3,24 +3,24 @@ from __future__ import unicode_literals from .mtv import MTVServicesInfoExtractor -class SouthParkStudiosIE(MTVServicesInfoExtractor): - IE_NAME = 'southparkstudios.com' - _VALID_URL = r'https?://(www\.)?(?P<url>southparkstudios\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$))' +class SouthParkIE(MTVServicesInfoExtractor): + IE_NAME = 'southpark.cc.com' + _VALID_URL = r'https?://(www\.)?(?P<url>southpark\.cc\.com/(clips|full-episodes)/(?P<id>.+?)(\?|#|$))' _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss' _TESTS = [{ - 'url': 'http://www.southparkstudios.com/clips/104437/bat-daded#tab=featured', + 'url': 'http://southpark.cc.com/clips/104437/bat-daded#tab=featured', 'info_dict': { 'id': 'a7bff6c2-ed00-11e0-aca6-0026b9414f30', 'ext': 'mp4', - 'title': 'Bat Daded', + 'title': 'South Park|Bat Daded', 'description': 'Randy disqualifies South Park by getting into a fight with Bat Dad.', }, }] -class SouthparkDeIE(SouthParkStudiosIE): +class SouthparkDeIE(SouthParkIE): IE_NAME = 'southpark.de' _VALID_URL = r'https?://(www\.)?(?P<url>southpark\.de/(clips|alle-episoden)/(?P<id>.+?)(\?|#|$))' _FEED_URL = 'http://www.southpark.de/feeds/video-player/mrss/' diff --git a/youtube_dl/extractor/steam.py b/youtube_dl/extractor/steam.py index af689e2c2..183dcb03c 100644 --- a/youtube_dl/extractor/steam.py +++ b/youtube_dl/extractor/steam.py @@ -53,7 +53,7 @@ class SteamIE(InfoExtractor): 'ext': 'mp4', 'upload_date': '20140329', 'title': 'FRONTIERS - Final Greenlight Trailer', - 'description': 'md5:6df4fe8dd494ae811869672b0767e025', + 'description': 'md5:dc96a773669d0ca1b36c13c1f30250d9', 'uploader': 'AAD Productions', 'uploader_id': 'AtomicAgeDogGames', } diff --git a/youtube_dl/extractor/streamcloud.py b/youtube_dl/extractor/streamcloud.py index 9faf3a5e3..172def221 100644 --- a/youtube_dl/extractor/streamcloud.py +++ b/youtube_dl/extractor/streamcloud.py @@ -1,4 +1,6 @@ # coding: utf-8 +from __future__ import unicode_literals + import re import time @@ -10,18 +12,18 @@ from ..utils import ( class StreamcloudIE(InfoExtractor): - IE_NAME = u'streamcloud.eu' + IE_NAME = 'streamcloud.eu' _VALID_URL = r'https?://streamcloud\.eu/(?P<id>[a-zA-Z0-9_-]+)/(?P<fname>[^#?]*)\.html' _TEST = { - u'url': u'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html', - u'file': u'skp9j99s4bpz.mp4', - u'md5': u'6bea4c7fa5daaacc2a946b7146286686', - u'info_dict': { - u'title': u'youtube-dl test video \'/\\ ä ↭', - u'duration': 9, + 'url': 'http://streamcloud.eu/skp9j99s4bpz/youtube-dl_test_video_____________-BaW_jenozKc.mp4.html', + 'md5': '6bea4c7fa5daaacc2a946b7146286686', + 'info_dict': { + 'id': 'skp9j99s4bpz', + 'ext': 'mp4', + 'title': 'youtube-dl test video \'/\\ ä ↭', }, - u'skip': u'Only available from the EU' + 'skip': 'Only available from the EU' } def _real_extract(self, url): @@ -46,21 +48,17 @@ class StreamcloudIE(InfoExtractor): req = compat_urllib_request.Request(url, post, headers) webpage = self._download_webpage( - req, video_id, note=u'Downloading video page ...') + req, video_id, note='Downloading video page ...') title = self._html_search_regex( - r'<h1[^>]*>([^<]+)<', webpage, u'title') + r'<h1[^>]*>([^<]+)<', webpage, 'title') video_url = self._search_regex( - r'file:\s*"([^"]+)"', webpage, u'video URL') - duration_str = self._search_regex( - r'duration:\s*"?([0-9]+)"?', webpage, u'duration', fatal=False) - duration = None if duration_str is None else int(duration_str) + r'file:\s*"([^"]+)"', webpage, 'video URL') thumbnail = self._search_regex( - r'image:\s*"([^"]+)"', webpage, u'thumbnail URL', fatal=False) + r'image:\s*"([^"]+)"', webpage, 'thumbnail URL', fatal=False) return { 'id': video_id, 'title': title, 'url': video_url, - 'duration': duration, 'thumbnail': thumbnail, } diff --git a/youtube_dl/extractor/swrmediathek.py b/youtube_dl/extractor/swrmediathek.py index 6c688c520..5d9d70367 100644 --- a/youtube_dl/extractor/swrmediathek.py +++ b/youtube_dl/extractor/swrmediathek.py @@ -8,7 +8,7 @@ from ..utils import parse_duration class SWRMediathekIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?swrmediathek\.de/player\.htm\?show=(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})' + _VALID_URL = r'https?://(?:www\.)?swrmediathek\.de/(?:content/)?player\.htm\?show=(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})' _TESTS = [{ 'url': 'http://swrmediathek.de/player.htm?show=849790d0-dab8-11e3-a953-0026b975f2e6', @@ -52,6 +52,20 @@ class SWRMediathekIE(InfoExtractor): 'uploader': 'SWR 2', 'uploader_id': '284670', } + }, { + 'url': 'http://swrmediathek.de/content/player.htm?show=52dc7e00-15c5-11e4-84bc-0026b975f2e6', + 'md5': '881531487d0633080a8cc88d31ef896f', + 'info_dict': { + 'id': '52dc7e00-15c5-11e4-84bc-0026b975f2e6', + 'ext': 'mp4', + 'title': 'Familienspaß am Bodensee', + 'description': 'md5:0b591225a32cfde7be1629ed49fe4315', + 'thumbnail': 're:http://.*\.jpg', + 'duration': 1784, + 'upload_date': '20140727', + 'uploader': 'SWR Fernsehen BW', + 'uploader_id': '281130', + } }] def _real_extract(self, url): diff --git a/youtube_dl/extractor/tagesschau.py b/youtube_dl/extractor/tagesschau.py index 25b9864ad..b87047451 100644 --- a/youtube_dl/extractor/tagesschau.py +++ b/youtube_dl/extractor/tagesschau.py @@ -19,16 +19,6 @@ class TagesschauIE(InfoExtractor): 'description': 'md5:69da3c61275b426426d711bde96463ab', 'thumbnail': 're:^http:.*\.jpg$', }, - }, { - 'url': 'http://www.tagesschau.de/multimedia/video/video-5964.html', - 'md5': '66652566900963a3f962333579eeffcf', - 'info_dict': { - 'id': '5964', - 'ext': 'mp4', - 'title': 'Nahost-Konflikt: Israel bombadiert Ziele im Gazastreifen und Westjordanland', - 'description': 'md5:07bfc78c48eec3145ed4805299a1900a', - 'thumbnail': 're:http://.*\.jpg', - }, }] _FORMATS = { diff --git a/youtube_dl/extractor/teachertube.py b/youtube_dl/extractor/teachertube.py index 2c2113b14..46d727d1d 100644 --- a/youtube_dl/extractor/teachertube.py +++ b/youtube_dl/extractor/teachertube.py @@ -62,7 +62,7 @@ class TeacherTubeIE(InfoExtractor): webpage = self._download_webpage(url, video_id) - title = self._html_search_meta('title', webpage, 'title') + title = self._html_search_meta('title', webpage, 'title', fatal=True) TITLE_SUFFIX = ' - TeacherTube' if title.endswith(TITLE_SUFFIX): title = title[:-len(TITLE_SUFFIX)].strip() @@ -101,7 +101,11 @@ class TeacherTubeUserIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?teachertube\.com/(user/profile|collection)/(?P<user>[0-9a-zA-Z]+)/?' - _MEDIA_RE = r'(?s)"sidebar_thumb_time">[0-9:]+</div>.+?<a href="(https?://(?:www\.)?teachertube\.com/(?:video|audio)/[^"]+)">' + _MEDIA_RE = r'''(?sx) + class="?sidebar_thumb_time"?>[0-9:]+</div> + \s* + <a\s+href="(https?://(?:www\.)?teachertube\.com/(?:video|audio)/[^"]+)" + ''' def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) @@ -111,14 +115,12 @@ class TeacherTubeUserIE(InfoExtractor): webpage = self._download_webpage(url, user_id) urls.extend(re.findall(self._MEDIA_RE, webpage)) - pages = re.findall(r'/ajax-user/user-videos/%s\?page=([0-9]+)' % user_id, webpage)[1:-1] + pages = re.findall(r'/ajax-user/user-videos/%s\?page=([0-9]+)' % user_id, webpage)[:-1] for p in pages: more = 'http://www.teachertube.com/ajax-user/user-videos/%s?page=%s' % (user_id, p) - webpage = self._download_webpage(more, user_id, 'Downloading page %s/%s' % (p, len(pages) + 1)) - urls.extend(re.findall(self._MEDIA_RE, webpage)) - - entries = [] - for url in urls: - entries.append(self.url_result(url, 'TeacherTube')) + webpage = self._download_webpage(more, user_id, 'Downloading page %s/%s' % (p, len(pages))) + video_urls = re.findall(self._MEDIA_RE, webpage) + urls.extend(video_urls) + entries = [self.url_result(vurl, 'TeacherTube') for vurl in urls] return self.playlist_result(entries, user_id) diff --git a/youtube_dl/extractor/teamcoco.py b/youtube_dl/extractor/teamcoco.py index f8dd7e955..fa796ce72 100644 --- a/youtube_dl/extractor/teamcoco.py +++ b/youtube_dl/extractor/teamcoco.py @@ -37,7 +37,7 @@ class TeamcocoIE(InfoExtractor): video_id = mobj.group("video_id") if not video_id: video_id = self._html_search_regex( - r'<article class="video" data-id="(\d+?)"', + r'data-node-id="(\d+?)"', webpage, 'video id') data_url = 'http://teamcoco.com/cvp/2.0/%s.xml' % video_id diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py index 8477840fc..81ba169fb 100644 --- a/youtube_dl/extractor/tenplay.py +++ b/youtube_dl/extractor/tenplay.py @@ -1,8 +1,6 @@ # coding: utf-8 from __future__ import unicode_literals -import re - from .common import InfoExtractor diff --git a/youtube_dl/extractor/tlc.py b/youtube_dl/extractor/tlc.py index ad175b83e..d848ee186 100644 --- a/youtube_dl/extractor/tlc.py +++ b/youtube_dl/extractor/tlc.py @@ -5,6 +5,7 @@ import re from .common import InfoExtractor from .brightcove import BrightcoveIE from .discovery import DiscoveryIE +from ..utils import compat_urlparse class TlcIE(DiscoveryIE): @@ -51,6 +52,10 @@ class TlcDeIE(InfoExtractor): # Otherwise we don't get the correct 'BrightcoveExperience' element, # example: http://www.tlc.de/sendungen/cake-boss/videos/cake-boss-cannoli-drama/ iframe_url = iframe_url.replace('.htm?', '.php?') + url_fragment = compat_urlparse.urlparse(url).fragment + if url_fragment: + # Since the fragment is not send to the server, we always get the same iframe + iframe_url = re.sub(r'playlist=(\d+)', 'playlist=%s' % url_fragment, iframe_url) iframe = self._download_webpage(iframe_url, title) return { diff --git a/youtube_dl/extractor/tutv.py b/youtube_dl/extractor/tutv.py index c980153ec..d516b6427 100644 --- a/youtube_dl/extractor/tutv.py +++ b/youtube_dl/extractor/tutv.py @@ -1,21 +1,21 @@ from __future__ import unicode_literals + import base64 import re from .common import InfoExtractor -from ..utils import ( - compat_parse_qs, -) +from ..utils import compat_parse_qs class TutvIE(InfoExtractor): _VALID_URL = r'https?://(?:www\.)?tu\.tv/videos/(?P<id>[^/?]+)' _TEST = { - 'url': 'http://tu.tv/videos/noah-en-pabellon-cuahutemoc', - 'file': '2742556.flv', - 'md5': '5eb766671f69b82e528dc1e7769c5cb2', + 'url': 'http://tu.tv/videos/robots-futbolistas', + 'md5': '627c7c124ac2a9b5ab6addb94e0e65f7', 'info_dict': { - 'title': 'Noah en pabellon cuahutemoc', + 'id': '2973058', + 'ext': 'flv', + 'title': 'Robots futbolistas', }, } @@ -26,10 +26,9 @@ class TutvIE(InfoExtractor): webpage = self._download_webpage(url, video_id) internal_id = self._search_regex(r'codVideo=([0-9]+)', webpage, 'internal video ID') - data_url = 'http://tu.tv/flvurl.php?codVideo=' + str(internal_id) - data_content = self._download_webpage(data_url, video_id, note='Downloading video info') - data = compat_parse_qs(data_content) - video_url = base64.b64decode(data['kpt'][0]).decode('utf-8') + data_content = self._download_webpage( + 'http://tu.tv/flvurl.php?codVideo=%s' % internal_id, video_id, 'Downloading video info') + video_url = base64.b64decode(compat_parse_qs(data_content)['kpt'][0]).decode('utf-8') return { 'id': internal_id, diff --git a/youtube_dl/extractor/tvplay.py b/youtube_dl/extractor/tvplay.py new file mode 100644 index 000000000..a56a7ab5f --- /dev/null +++ b/youtube_dl/extractor/tvplay.py @@ -0,0 +1,85 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + parse_iso8601, + qualities, +) + + +class TVPlayIE(InfoExtractor): + _VALID_URL = r'http://(?:www\.)?tvplay\.lv/parraides/[^/]+/(?P<id>\d+)' + _TESTS = [ + { + 'url': 'http://www.tvplay.lv/parraides/vinas-melo-labak/418113?autostart=true', + 'info_dict': { + 'id': '418113', + 'ext': 'flv', + 'title': 'Kādi ir īri? - Viņas melo labāk', + 'description': 'Baiba apsmej īrus, kādi tie ir un ko viņi dara.', + 'duration': 25, + 'timestamp': 1406097056, + 'upload_date': '20140723', + }, + 'params': { + # rtmp download + 'skip_download': True, + }, + }, + ] + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + video = self._download_json( + 'http://playapi.mtgx.tv/v1/videos/%s' % video_id, video_id, 'Downloading video JSON') + + if video['is_geo_blocked']: + raise ExtractorError( + 'This content is not available in your country due to copyright reasons', expected=True) + + streams = self._download_json( + 'http://playapi.mtgx.tv/v1/videos/stream/%s' % video_id, video_id, 'Downloading streams JSON') + + quality = qualities(['hls', 'medium', 'high']) + formats = [] + for format_id, video_url in streams['streams'].items(): + if not video_url: + continue + fmt = { + 'format_id': format_id, + 'preference': quality(format_id), + } + if video_url.startswith('rtmp'): + m = re.search(r'^(?P<url>rtmp://[^/]+/(?P<app>[^/]+))/(?P<playpath>.+)$', video_url) + if not m: + continue + fmt.update({ + 'ext': 'flv', + 'url': m.group('url'), + 'app': m.group('app'), + 'play_path': m.group('playpath'), + }) + else: + fmt.update({ + 'url': video_url, + }) + formats.append(fmt) + + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': video['title'], + 'description': video['description'], + 'duration': video['duration'], + 'timestamp': parse_iso8601(video['created_at']), + 'view_count': video['views']['total'], + 'age_limit': video.get('age_limit', 0), + 'formats': formats, + } diff --git a/youtube_dl/extractor/ubu.py b/youtube_dl/extractor/ubu.py new file mode 100644 index 000000000..0182d67ec --- /dev/null +++ b/youtube_dl/extractor/ubu.py @@ -0,0 +1,56 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import int_or_none + + +class UbuIE(InfoExtractor): + _VALID_URL = r'http://(?:www\.)?ubu\.com/film/(?P<id>[\da-z_-]+)\.html' + _TEST = { + 'url': 'http://ubu.com/film/her_noise.html', + 'md5': '8edd46ee8aa6b265fb5ed6cf05c36bc9', + 'info_dict': { + 'id': 'her_noise', + 'ext': 'mp4', + 'title': 'Her Noise - The Making Of (2007)', + 'duration': 3600, + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + + title = self._html_search_regex( + r'<title>.+?Film & Video: ([^<]+)</title>', webpage, 'title') + + duration = int_or_none(self._html_search_regex( + r'Duration: (\d+) minutes', webpage, 'duration', fatal=False, default=None)) + if duration: + duration *= 60 + + formats = [] + + FORMAT_REGEXES = [ + ['sq', r"'flashvars'\s*,\s*'file=([^']+)'"], + ['hq', r'href="(http://ubumexico\.centro\.org\.mx/video/[^"]+)"'] + ] + + for format_id, format_regex in FORMAT_REGEXES: + m = re.search(format_regex, webpage) + if m: + formats.append({ + 'url': m.group(1), + 'format_id': format_id, + }) + + return { + 'id': video_id, + 'title': title, + 'duration': duration, + 'formats': formats, + } diff --git a/youtube_dl/extractor/vevo.py b/youtube_dl/extractor/vevo.py index eada13ce9..d2ffd1b6b 100644 --- a/youtube_dl/extractor/vevo.py +++ b/youtube_dl/extractor/vevo.py @@ -177,6 +177,7 @@ class VevoIE(InfoExtractor): self._downloader.report_warning( 'Cannot download SMIL information, falling back to JSON ..') + self._sort_formats(formats) timestamp_ms = int(self._search_regex( r'/Date\((\d+)\)/', video_info['launchDate'], 'launch date')) diff --git a/youtube_dl/extractor/vidme.py b/youtube_dl/extractor/vidme.py new file mode 100644 index 000000000..5c89824c1 --- /dev/null +++ b/youtube_dl/extractor/vidme.py @@ -0,0 +1,68 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + int_or_none, + float_or_none, + str_to_int, +) + + +class VidmeIE(InfoExtractor): + _VALID_URL = r'https?://vid\.me/(?:e/)?(?P<id>[\da-zA-Z]+)' + _TEST = { + 'url': 'https://vid.me/QNB', + 'md5': 'f42d05e7149aeaec5c037b17e5d3dc82', + 'info_dict': { + 'id': 'QNB', + 'ext': 'mp4', + 'title': 'Fishing for piranha - the easy way', + 'description': 'source: https://www.facebook.com/photo.php?v=312276045600871', + 'duration': 119.92, + 'timestamp': 1406313244, + 'upload_date': '20140725', + 'thumbnail': 're:^https?://.*\.jpg', + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + + video_url = self._html_search_regex(r'<source src="([^"]+)"', webpage, 'video URL') + + title = self._og_search_title(webpage) + description = self._og_search_description(webpage, default='') + thumbnail = self._og_search_thumbnail(webpage) + timestamp = int_or_none(self._og_search_property('updated_time', webpage, fatal=False)) + width = int_or_none(self._og_search_property('video:width', webpage, fatal=False)) + height = int_or_none(self._og_search_property('video:height', webpage, fatal=False)) + duration = float_or_none(self._html_search_regex( + r'data-duration="([^"]+)"', webpage, 'duration', fatal=False)) + view_count = str_to_int(self._html_search_regex( + r'<span class="video_views">\s*([\d,\.]+)\s*plays?', webpage, 'view count', fatal=False)) + like_count = str_to_int(self._html_search_regex( + r'class="score js-video-vote-score"[^>]+data-score="([\d,\.\s]+)">', + webpage, 'like count', fatal=False)) + comment_count = str_to_int(self._html_search_regex( + r'class="js-comment-count"[^>]+data-count="([\d,\.\s]+)">', + webpage, 'comment count', fatal=False)) + + return { + 'id': video_id, + 'url': video_url, + 'title': title, + 'description': description, + 'thumbnail': thumbnail, + 'timestamp': timestamp, + 'width': width, + 'height': height, + 'duration': duration, + 'view_count': view_count, + 'like_count': like_count, + 'comment_count': comment_count, + } diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py index 255855558..11c7d7e81 100644 --- a/youtube_dl/extractor/vimeo.py +++ b/youtube_dl/extractor/vimeo.py @@ -98,7 +98,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor): 'info_dict': { 'id': '54469442', 'ext': 'mp4', - 'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software', + 'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software 2012', 'uploader': 'The BLN & Business of Software', 'uploader_id': 'theblnbusinessofsoftware', 'duration': 3610, @@ -122,6 +122,21 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor): }, }, { + 'url': 'http://vimeo.com/channels/keypeele/75629013', + 'md5': '2f86a05afe9d7abc0b9126d229bbe15d', + 'note': 'Video is freely available via original URL ' + 'and protected with password when accessed via http://vimeo.com/75629013', + 'info_dict': { + 'id': '75629013', + 'ext': 'mp4', + 'title': 'Key & Peele: Terrorist Interrogation', + 'description': 'md5:8678b246399b070816b12313e8b4eb5c', + 'uploader_id': 'atencio', + 'uploader': 'Peter Atencio', + 'duration': 187, + }, + }, + { 'url': 'http://vimeo.com/76979871', 'md5': '3363dd6ffebe3784d56f4132317fd446', 'note': 'Video with subtitles', @@ -196,8 +211,6 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor): video_id = mobj.group('id') if mobj.group('pro') or mobj.group('player'): url = 'http://player.vimeo.com/video/' + video_id - else: - url = 'https://vimeo.com/' + video_id # Retrieve video webpage to extract further information request = compat_urllib_request.Request(url, None, headers) @@ -263,7 +276,7 @@ class VimeoIE(VimeoBaseInfoExtractor, SubtitlesInfoExtractor): if video_thumbnail is None: video_thumbs = config["video"].get("thumbs") if video_thumbs and isinstance(video_thumbs, dict): - _, video_thumbnail = sorted((int(width), t_url) for (width, t_url) in video_thumbs.items())[-1] + _, video_thumbnail = sorted((int(width if width.isdigit() else 0), t_url) for (width, t_url) in video_thumbs.items())[-1] # Extract video description video_description = None diff --git a/youtube_dl/extractor/vodlocker.py b/youtube_dl/extractor/vodlocker.py index 68c59364b..affef6507 100644 --- a/youtube_dl/extractor/vodlocker.py +++ b/youtube_dl/extractor/vodlocker.py @@ -10,7 +10,7 @@ from ..utils import ( class VodlockerIE(InfoExtractor): - _VALID_URL = r'https?://(?:www\.)?vodlocker.com/(?P<id>[0-9a-zA-Z]+)(?:\..*?)?' + _VALID_URL = r'https?://(?:www\.)?vodlocker\.com/(?P<id>[0-9a-zA-Z]+)(?:\..*?)?' _TESTS = [{ 'url': 'http://vodlocker.com/e8wvyzz4sl42', @@ -44,7 +44,7 @@ class VodlockerIE(InfoExtractor): req, video_id, 'Downloading video page') title = self._search_regex( - r'id="file_title".*?>\s*(.*?)\s*<span', webpage, 'title') + r'id="file_title".*?>\s*(.*?)\s*<(?:br|span)', webpage, 'title') thumbnail = self._search_regex( r'image:\s*"(http[^\"]+)",', webpage, 'thumbnail') url = self._search_regex( diff --git a/youtube_dl/extractor/vube.py b/youtube_dl/extractor/vube.py index 7b77865cb..2544c24bd 100644 --- a/youtube_dl/extractor/vube.py +++ b/youtube_dl/extractor/vube.py @@ -3,7 +3,10 @@ from __future__ import unicode_literals import re from .common import InfoExtractor -from ..utils import int_or_none +from ..utils import ( + int_or_none, + compat_str, +) class VubeIE(InfoExtractor): @@ -20,12 +23,15 @@ class VubeIE(InfoExtractor): 'ext': 'mp4', 'title': 'Chiara Grispo - Price Tag by Jessie J', 'description': 'md5:8ea652a1f36818352428cb5134933313', - 'thumbnail': 'http://frame.thestaticvube.com/snap/228x128/102e7e63057-5ebc-4f5c-4065-6ce4ebde131f.jpg', + 'thumbnail': 're:^http://frame\.thestaticvube\.com/snap/[0-9x]+/102e7e63057-5ebc-4f5c-4065-6ce4ebde131f\.jpg$', 'uploader': 'Chiara.Grispo', - 'uploader_id': '1u3hX0znhP', 'timestamp': 1388743358, 'upload_date': '20140103', - 'duration': 170.56 + 'duration': 170.56, + 'like_count': int, + 'dislike_count': int, + 'comment_count': int, + 'categories': ['pop', 'music', 'cover', 'singing', 'jessie j', 'price tag', 'chiara grispo'], } }, { @@ -36,12 +42,33 @@ class VubeIE(InfoExtractor): 'ext': 'mp4', 'title': 'My 7 year old Sister and I singing "Alive" by Krewella', 'description': 'md5:40bcacb97796339f1690642c21d56f4a', - 'thumbnail': 'http://frame.thestaticvube.com/snap/228x128/102265d5a9f-0f17-4f6b-5753-adf08484ee1e.jpg', + 'thumbnail': 're:^http://frame\.thestaticvube\.com/snap/[0-9x]+/102265d5a9f-0f17-4f6b-5753-adf08484ee1e\.jpg$', 'uploader': 'Seraina', - 'uploader_id': 'XU9VE2BQ2q', 'timestamp': 1396492438, 'upload_date': '20140403', - 'duration': 240.107 + 'duration': 240.107, + 'like_count': int, + 'dislike_count': int, + 'comment_count': int, + 'categories': ['seraina', 'jessica', 'krewella', 'alive'], + } + }, { + 'url': 'http://vube.com/vote/Siren+Gene/0nmsMY5vEq?n=2&t=s', + 'md5': '0584fc13b50f887127d9d1007589d27f', + 'info_dict': { + 'id': '0nmsMY5vEq', + 'ext': 'mp4', + 'title': 'Frozen - Let It Go Cover by Siren Gene', + 'description': 'My rendition of "Let It Go" originally sung by Idina Menzel.', + 'thumbnail': 're:^http://frame\.thestaticvube\.com/snap/[0-9x]+/10283ab622a-86c9-4681-51f2-30d1f65774af\.jpg$', + 'uploader': 'Siren', + 'timestamp': 1395448018, + 'upload_date': '20140322', + 'duration': 221.788, + 'like_count': int, + 'dislike_count': int, + 'comment_count': int, + 'categories': ['let it go', 'cover', 'idina menzel', 'frozen', 'singing', 'disney', 'siren gene'], } } ] @@ -51,39 +78,52 @@ class VubeIE(InfoExtractor): video_id = mobj.group('id') video = self._download_json( - 'http://vube.com/api/v2/video/%s' % video_id, video_id, 'Downloading video JSON') + 'http://vube.com/t-api/v1/video/%s' % video_id, video_id, 'Downloading video JSON') public_id = video['public_id'] - formats = [ - { - 'url': 'http://video.thestaticvube.com/video/%s/%s.mp4' % (fmt['media_resolution_id'], public_id), - 'height': int(fmt['height']), - 'abr': int(fmt['audio_bitrate']), - 'vbr': int(fmt['video_bitrate']), - 'format_id': fmt['media_resolution_id'] - } for fmt in video['mtm'] if fmt['transcoding_status'] == 'processed' - ] + formats = [] + + for media in video['media'].get('video', []) + video['media'].get('audio', []): + if media['transcoding_status'] != 'processed': + continue + fmt = { + 'url': 'http://video.thestaticvube.com/video/%s/%s.mp4' % (media['media_resolution_id'], public_id), + 'abr': int(media['audio_bitrate']), + 'format_id': compat_str(media['media_resolution_id']), + } + vbr = int(media['video_bitrate']) + if vbr: + fmt.update({ + 'vbr': vbr, + 'height': int(media['height']), + }) + formats.append(fmt) self._sort_formats(formats) title = video['title'] description = video.get('description') - thumbnail = video['thumbnail_src'] - if thumbnail.startswith('//'): - thumbnail = 'http:' + thumbnail - uploader = video['user_alias'] - uploader_id = video['user_url_id'] - timestamp = int(video['upload_time']) + thumbnail = self._proto_relative_url(video.get('thumbnail_src'), scheme='http:') + uploader = video.get('user_alias') or video.get('channel') + timestamp = int_or_none(video.get('upload_time')) duration = video['duration'] view_count = video.get('raw_view_count') like_count = video.get('total_likes') - dislike_count= video.get('total_hates') + dislike_count = video.get('total_hates') - comment = self._download_json( - 'http://vube.com/api/video/%s/comment' % video_id, video_id, 'Downloading video comment JSON') + comments = video.get('comments') + comment_count = None + if comments is None: + comment_data = self._download_json( + 'http://vube.com/api/video/%s/comment' % video_id, + video_id, 'Downloading video comment JSON', fatal=False) + if comment_data is not None: + comment_count = int_or_none(comment_data.get('total')) + else: + comment_count = len(comments) - comment_count = int_or_none(comment.get('total')) + categories = [tag['text'] for tag in video['tags']] return { 'id': video_id, @@ -92,11 +132,11 @@ class VubeIE(InfoExtractor): 'description': description, 'thumbnail': thumbnail, 'uploader': uploader, - 'uploader_id': uploader_id, 'timestamp': timestamp, 'duration': duration, 'view_count': view_count, 'like_count': like_count, 'dislike_count': dislike_count, 'comment_count': comment_count, + 'categories': categories, } diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py index a584e0896..1f330378a 100644 --- a/youtube_dl/extractor/wat.py +++ b/youtube_dl/extractor/wat.py @@ -2,27 +2,30 @@ from __future__ import unicode_literals import re +import time +import hashlib from .common import InfoExtractor from ..utils import ( + ExtractorError, unified_strdate, ) class WatIE(InfoExtractor): - _VALID_URL = r'http://www\.wat\.tv/.*-(?P<shortID>.*?)_.*?\.html' + _VALID_URL = r'http://www\.wat\.tv/video/(?P<display_id>.*)-(?P<short_id>.*?)_.*?\.html' IE_NAME = 'wat.tv' _TEST = { - 'url': 'http://www.wat.tv/video/world-war-philadelphia-vost-6bv55_2fjr7_.html', + 'url': 'http://www.wat.tv/video/soupe-figues-l-orange-aux-epices-6z1uz_2hvf7_.html', + 'md5': 'ce70e9223945ed26a8056d413ca55dc9', 'info_dict': { - 'id': '10631273', + 'id': '11713067', + 'display_id': 'soupe-figues-l-orange-aux-epices', 'ext': 'mp4', - 'title': 'World War Z - Philadelphia VOST', - 'description': 'La menace est partout. Que se passe-t-il à Philadelphia ?\r\nWORLD WAR Z, avec Brad Pitt, au cinéma le 3 juillet.\r\nhttp://www.worldwarz.fr', - }, - 'params': { - # Sometimes wat serves the whole file with the --test option - 'skip_download': True, + 'title': 'Soupe de figues à l\'orange et aux épices', + 'description': 'Retrouvez l\'émission "Petits plats en équilibre", diffusée le 18 août 2014.', + 'upload_date': '20140819', + 'duration': 120, }, } @@ -36,13 +39,20 @@ class WatIE(InfoExtractor): def real_id_for_chapter(chapter): return chapter['tc_start'].split('-')[0] mobj = re.match(self._VALID_URL, url) - short_id = mobj.group('shortID') - webpage = self._download_webpage(url, short_id) + short_id = mobj.group('short_id') + display_id = mobj.group('display_id') + webpage = self._download_webpage(url, display_id or short_id) real_id = self._search_regex(r'xtpage = ".*-(.*?)";', webpage, 'real id') video_info = self.download_video_info(real_id) + + if video_info.get('geolock'): + raise ExtractorError('This content is not available in your area', expected=True) + chapters = video_info['chapters'] first_chapter = chapters[0] + files = video_info['files'] + first_file = files[0] if real_id_for_chapter(first_chapter) != real_id: self.to_screen('Multipart video detected') @@ -61,12 +71,45 @@ class WatIE(InfoExtractor): upload_date = unified_strdate(first_chapter['date_diffusion']) # Otherwise we can continue and extract just one part, we have to use # the short id for getting the video url + + formats = [{ + 'url': 'http://wat.tv/get/android5/%s.mp4' % real_id, + 'format_id': 'Mobile', + }] + + fmts = [('SD', 'web')] + if first_file.get('hasHD'): + fmts.append(('HD', 'webhd')) + + def compute_token(param): + timestamp = '%08x' % int(time.time()) + magic = '9b673b13fa4682ed14c3cfa5af5310274b514c4133e9b3a81e6e3aba009l2564' + return '%s/%s' % (hashlib.md5((magic + param + timestamp).encode('ascii')).hexdigest(), timestamp) + + for fmt in fmts: + webid = '/%s/%s' % (fmt[1], real_id) + video_url = self._download_webpage( + 'http://www.wat.tv/get%s?token=%s&getURL=1' % (webid, compute_token(webid)), + real_id, + 'Downloding %s video URL' % fmt[0], + 'Failed to download %s video URL' % fmt[0], + False) + if not video_url: + continue + formats.append({ + 'url': video_url, + 'ext': 'mp4', + 'format_id': fmt[0], + }) + return { 'id': real_id, - 'url': 'http://wat.tv/get/android5/%s.mp4' % real_id, + 'display_id': display_id, 'title': first_chapter['title'], 'thumbnail': first_chapter['preview'], 'description': first_chapter['description'], 'view_count': video_info['views'], 'upload_date': upload_date, + 'duration': first_file['duration'], + 'formats': formats, } diff --git a/youtube_dl/extractor/wayofthemaster.py b/youtube_dl/extractor/wayofthemaster.py new file mode 100644 index 000000000..af7bb8b49 --- /dev/null +++ b/youtube_dl/extractor/wayofthemaster.py @@ -0,0 +1,52 @@ +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor + + +class WayOfTheMasterIE(InfoExtractor): + _VALID_URL = r'https?://www\.wayofthemaster\.com/([^/?#]*/)*(?P<id>[^/?#]+)\.s?html(?:$|[?#])' + + _TEST = { + 'url': 'http://www.wayofthemaster.com/hbks.shtml', + 'md5': '5316b57487ada8480606a93cb3d18d24', + 'info_dict': { + 'id': 'hbks', + 'ext': 'mp4', + 'title': 'Intelligent Design vs. Evolution', + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + + title = self._search_regex( + r'<img src="images/title_[^"]+".*?alt="([^"]+)"', + webpage, 'title', default=None) + if title is None: + title = self._html_search_regex( + r'<title>(.*?)</title>', webpage, 'page title') + + url_base = self._search_regex( + r'<param\s+name="?movie"?\s+value=".*?/wotm_videoplayer_highlow[0-9]*\.swf\?vid=([^"]+)"', + webpage, 'URL base') + formats = [{ + 'format_id': 'low', + 'quality': 1, + 'url': url_base + '_low.mp4', + }, { + 'format_id': 'high', + 'quality': 2, + 'url': url_base + '_high.mp4', + }] + self._sort_formats(formats) + + return { + 'id': video_id, + 'title': title, + 'formats': formats, + } diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py index f741ba540..54d37da61 100644 --- a/youtube_dl/extractor/wdr.py +++ b/youtube_dl/extractor/wdr.py @@ -55,14 +55,14 @@ class WDRIE(InfoExtractor): }, }, { - 'url': 'http://www.funkhauseuropa.de/av/audiosuepersongsoulbossanova100-audioplayer.html', - 'md5': '24e83813e832badb0a8d7d1ef9ef0691', + 'url': 'http://www.funkhauseuropa.de/av/audioflaviacoelhoamaramar100-audioplayer.html', + 'md5': '99a1443ff29af19f6c52cf6f4dc1f4aa', 'info_dict': { - 'id': 'mdb-463528', + 'id': 'mdb-478135', 'ext': 'mp3', - 'title': 'Süpersong: Soul Bossa Nova', + 'title': 'Flavia Coelho: Amar é Amar', 'description': 'md5:7b29e97e10dfb6e265238b32fa35b23a', - 'upload_date': '20140630', + 'upload_date': '20140717', }, }, ] @@ -81,7 +81,7 @@ class WDRIE(InfoExtractor): ] return self.playlist_result(entries, page_id) - flashvars = compat_urlparse.parse_qs( + flashvars = compat_parse_qs( self._html_search_regex(r'<param name="flashvars" value="([^"]+)"', webpage, 'flashvars')) page_id = flashvars['trackerClipId'][0] diff --git a/youtube_dl/extractor/xboxclips.py b/youtube_dl/extractor/xboxclips.py new file mode 100644 index 000000000..a9aa72e73 --- /dev/null +++ b/youtube_dl/extractor/xboxclips.py @@ -0,0 +1,57 @@ +# encoding: utf-8 +from __future__ import unicode_literals + +import re + +from .common import InfoExtractor +from ..utils import ( + parse_iso8601, + float_or_none, + int_or_none, +) + + +class XboxClipsIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?xboxclips\.com/video\.php\?.*vid=(?P<id>[\w-]{36})' + _TEST = { + 'url': 'https://xboxclips.com/video.php?uid=2533274823424419&gamertag=Iabdulelah&vid=074a69a9-5faf-46aa-b93b-9909c1720325', + 'md5': 'fbe1ec805e920aeb8eced3c3e657df5d', + 'info_dict': { + 'id': '074a69a9-5faf-46aa-b93b-9909c1720325', + 'ext': 'mp4', + 'title': 'Iabdulelah playing Upload Studio', + 'filesize_approx': 28101836.8, + 'timestamp': 1407388500, + 'upload_date': '20140807', + 'duration': 56, + } + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + + webpage = self._download_webpage(url, video_id) + + video_url = self._html_search_regex( + r'>Link: <a href="([^"]+)">', webpage, 'video URL') + title = self._html_search_regex( + r'<title>XboxClips \| ([^<]+)</title>', webpage, 'title') + timestamp = parse_iso8601(self._html_search_regex( + r'>Recorded: ([^<]+)<', webpage, 'upload date', fatal=False)) + filesize = float_or_none(self._html_search_regex( + r'>Size: ([\d\.]+)MB<', webpage, 'file size', fatal=False), invscale=1024 * 1024) + duration = int_or_none(self._html_search_regex( + r'>Duration: (\d+) Seconds<', webpage, 'duration', fatal=False)) + view_count = int_or_none(self._html_search_regex( + r'>Views: (\d+)<', webpage, 'view count', fatal=False)) + + return { + 'id': video_id, + 'url': video_url, + 'title': title, + 'timestamp': timestamp, + 'filesize_approx': filesize, + 'duration': duration, + 'view_count': view_count, + } diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py index 5374495f9..00b6d1eba 100644 --- a/youtube_dl/extractor/xhamster.py +++ b/youtube_dl/extractor/xhamster.py @@ -14,7 +14,7 @@ from ..utils import ( class XHamsterIE(InfoExtractor): """Information Extractor for xHamster""" - _VALID_URL = r'http://(?:www\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?' + _VALID_URL = r'http://(?:.+?\.)?xhamster\.com/movies/(?P<id>[0-9]+)/(?P<seo>.+?)\.html(?:\?.*)?' _TESTS = [ { 'url': 'http://xhamster.com/movies/1509445/femaleagent_shy_beauty_takes_the_bait.html', diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py index d84be2562..0e3b33b16 100644 --- a/youtube_dl/extractor/yahoo.py +++ b/youtube_dl/extractor/yahoo.py @@ -15,7 +15,7 @@ from ..utils import ( class YahooIE(InfoExtractor): IE_DESC = 'Yahoo screen and movies' - _VALID_URL = r'https?://(?:screen|movies)\.yahoo\.com/.*?-(?P<id>[0-9]+)(?:-[a-z]+)?\.html' + _VALID_URL = r'(?P<url>https?://(?:screen|movies)\.yahoo\.com/.*?-(?P<id>[0-9]+)(?:-[a-z]+)?\.html)' _TESTS = [ { 'url': 'http://screen.yahoo.com/julian-smith-travis-legg-watch-214727115.html', @@ -46,12 +46,23 @@ class YahooIE(InfoExtractor): 'title': 'The World Loves Spider-Man', 'description': '''People all over the world are celebrating the release of \"The Amazing Spider-Man 2.\" We're taking a look at the enthusiastic response Spider-Man has received from viewers all over the world.''', } - } + }, + { + 'url': 'https://screen.yahoo.com/community/community-sizzle-reel-203225340.html?format=embed', + 'md5': '60e8ac193d8fb71997caa8fce54c6460', + 'info_dict': { + 'id': '4fe78544-8d48-39d8-97cd-13f205d9fcdb', + 'ext': 'mp4', + 'title': "Yahoo Saves 'Community'", + 'description': 'md5:4d4145af2fd3de00cbb6c1d664105053', + } + }, ] def _real_extract(self, url): mobj = re.match(self._VALID_URL, url) video_id = mobj.group('id') + url = mobj.group('url') webpage = self._download_webpage(url, video_id) items_json = self._search_regex( diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 6123e1256..225e2b7f4 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1,19 +1,17 @@ # coding: utf-8 -import collections import errno import io import itertools import json import os.path import re -import struct import traceback -import zlib from .common import InfoExtractor, SearchInfoExtractor from .subtitles import SubtitlesInfoExtractor from ..jsinterp import JSInterpreter +from ..swfinterp import SWFInterpreter from ..utils import ( compat_chr, compat_parse_qs, @@ -227,7 +225,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40}, # Dash webm audio - '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 48, 'preference': -50}, + '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50}, '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50}, # RTMP (unnamed) @@ -346,14 +344,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): """Indicate the download will use the RTMP protocol.""" self.to_screen(u'RTMP download detected') - def _extract_signature_function(self, video_id, player_url, slen): - id_m = re.match(r'.*-(?P<id>[a-zA-Z0-9_-]+)\.(?P<ext>[a-z]+)$', - player_url) + def _signature_cache_id(self, example_sig): + """ Return a string representation of a signature """ + return u'.'.join(compat_str(len(part)) for part in example_sig.split('.')) + + def _extract_signature_function(self, video_id, player_url, example_sig): + id_m = re.match( + r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.(?P<ext>[a-z]+)$', + player_url) + if not id_m: + raise ExtractorError('Cannot identify player %r' % player_url) player_type = id_m.group('ext') player_id = id_m.group('id') # Read from filesystem cache - func_id = '%s_%s_%d' % (player_type, player_id, slen) + func_id = '%s_%s_%s' % ( + player_type, player_id, self._signature_cache_id(example_sig)) assert os.path.basename(func_id) == func_id cache_dir = get_cachedir(self._downloader.params) @@ -368,6 +374,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): return lambda s: u''.join(s[i] for i in cache_spec) except IOError: pass # No cache available + except ValueError: + try: + file_size = os.path.getsize(cache_fn) + except (OSError, IOError) as oe: + file_size = str(oe) + self._downloader.report_warning( + u'Cache %s failed (%s)' % (cache_fn, file_size)) if player_type == 'js': code = self._download_webpage( @@ -387,7 +400,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): if cache_enabled: try: - test_string = u''.join(map(compat_chr, range(slen))) + test_string = u''.join(map(compat_chr, range(len(example_sig)))) cache_res = res(test_string) cache_spec = [ord(c) for c in cache_res] try: @@ -403,7 +416,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): return res - def _print_sig_code(self, func, slen): + def _print_sig_code(self, func, example_sig): def gen_sig_code(idxs): def _genslice(start, end, step): starts = u'' if start == 0 else str(start) @@ -432,11 +445,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): else: yield _genslice(start, i, step) - test_string = u''.join(map(compat_chr, range(slen))) + test_string = u''.join(map(compat_chr, range(len(example_sig)))) cache_res = func(test_string) cache_spec = [ord(c) for c in cache_res] expr_code = u' + '.join(gen_sig_code(cache_spec)) - code = u'if len(s) == %d:\n return %s\n' % (slen, expr_code) + signature_id_tuple = '(%s)' % ( + ', '.join(compat_str(len(p)) for p in example_sig.split('.'))) + code = (u'if tuple(len(p) for p in s.split(\'.\')) == %s:\n' + u' return %s\n') % (signature_id_tuple, expr_code) self.to_screen(u'Extracted signature function:\n' + code) def _parse_sig_js(self, jscode): @@ -449,417 +465,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): return lambda s: initial_function([s]) def _parse_sig_swf(self, file_contents): - if file_contents[1:3] != b'WS': - raise ExtractorError( - u'Not an SWF file; header is %r' % file_contents[:3]) - if file_contents[:1] == b'C': - content = zlib.decompress(file_contents[8:]) - else: - raise NotImplementedError(u'Unsupported compression format %r' % - file_contents[:1]) - - def extract_tags(content): - pos = 0 - while pos < len(content): - header16 = struct.unpack('<H', content[pos:pos+2])[0] - pos += 2 - tag_code = header16 >> 6 - tag_len = header16 & 0x3f - if tag_len == 0x3f: - tag_len = struct.unpack('<I', content[pos:pos+4])[0] - pos += 4 - assert pos+tag_len <= len(content) - yield (tag_code, content[pos:pos+tag_len]) - pos += tag_len - - code_tag = next(tag - for tag_code, tag in extract_tags(content) - if tag_code == 82) - p = code_tag.index(b'\0', 4) + 1 - code_reader = io.BytesIO(code_tag[p:]) - - # Parse ABC (AVM2 ByteCode) - def read_int(reader=None): - if reader is None: - reader = code_reader - res = 0 - shift = 0 - for _ in range(5): - buf = reader.read(1) - assert len(buf) == 1 - b = struct.unpack('<B', buf)[0] - res = res | ((b & 0x7f) << shift) - if b & 0x80 == 0: - break - shift += 7 - return res - - def u30(reader=None): - res = read_int(reader) - assert res & 0xf0000000 == 0 - return res - u32 = read_int - - def s32(reader=None): - v = read_int(reader) - if v & 0x80000000 != 0: - v = - ((v ^ 0xffffffff) + 1) - return v - - def read_string(reader=None): - if reader is None: - reader = code_reader - slen = u30(reader) - resb = reader.read(slen) - assert len(resb) == slen - return resb.decode('utf-8') - - def read_bytes(count, reader=None): - if reader is None: - reader = code_reader - resb = reader.read(count) - assert len(resb) == count - return resb - - def read_byte(reader=None): - resb = read_bytes(1, reader=reader) - res = struct.unpack('<B', resb)[0] - return res - - # minor_version + major_version - read_bytes(2 + 2) - - # Constant pool - int_count = u30() - for _c in range(1, int_count): - s32() - uint_count = u30() - for _c in range(1, uint_count): - u32() - double_count = u30() - read_bytes((double_count-1) * 8) - string_count = u30() - constant_strings = [u''] - for _c in range(1, string_count): - s = read_string() - constant_strings.append(s) - namespace_count = u30() - for _c in range(1, namespace_count): - read_bytes(1) # kind - u30() # name - ns_set_count = u30() - for _c in range(1, ns_set_count): - count = u30() - for _c2 in range(count): - u30() - multiname_count = u30() - MULTINAME_SIZES = { - 0x07: 2, # QName - 0x0d: 2, # QNameA - 0x0f: 1, # RTQName - 0x10: 1, # RTQNameA - 0x11: 0, # RTQNameL - 0x12: 0, # RTQNameLA - 0x09: 2, # Multiname - 0x0e: 2, # MultinameA - 0x1b: 1, # MultinameL - 0x1c: 1, # MultinameLA - } - multinames = [u''] - for _c in range(1, multiname_count): - kind = u30() - assert kind in MULTINAME_SIZES, u'Invalid multiname kind %r' % kind - if kind == 0x07: - u30() # namespace_idx - name_idx = u30() - multinames.append(constant_strings[name_idx]) - else: - multinames.append('[MULTINAME kind: %d]' % kind) - for _c2 in range(MULTINAME_SIZES[kind]): - u30() - - # Methods - method_count = u30() - MethodInfo = collections.namedtuple( - 'MethodInfo', - ['NEED_ARGUMENTS', 'NEED_REST']) - method_infos = [] - for method_id in range(method_count): - param_count = u30() - u30() # return type - for _ in range(param_count): - u30() # param type - u30() # name index (always 0 for youtube) - flags = read_byte() - if flags & 0x08 != 0: - # Options present - option_count = u30() - for c in range(option_count): - u30() # val - read_bytes(1) # kind - if flags & 0x80 != 0: - # Param names present - for _ in range(param_count): - u30() # param name - mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0) - method_infos.append(mi) - - # Metadata - metadata_count = u30() - for _c in range(metadata_count): - u30() # name - item_count = u30() - for _c2 in range(item_count): - u30() # key - u30() # value - - def parse_traits_info(): - trait_name_idx = u30() - kind_full = read_byte() - kind = kind_full & 0x0f - attrs = kind_full >> 4 - methods = {} - if kind in [0x00, 0x06]: # Slot or Const - u30() # Slot id - u30() # type_name_idx - vindex = u30() - if vindex != 0: - read_byte() # vkind - elif kind in [0x01, 0x02, 0x03]: # Method / Getter / Setter - u30() # disp_id - method_idx = u30() - methods[multinames[trait_name_idx]] = method_idx - elif kind == 0x04: # Class - u30() # slot_id - u30() # classi - elif kind == 0x05: # Function - u30() # slot_id - function_idx = u30() - methods[function_idx] = multinames[trait_name_idx] - else: - raise ExtractorError(u'Unsupported trait kind %d' % kind) - - if attrs & 0x4 != 0: # Metadata present - metadata_count = u30() - for _c3 in range(metadata_count): - u30() # metadata index - - return methods - - # Classes + swfi = SWFInterpreter(file_contents) TARGET_CLASSNAME = u'SignatureDecipher' - searched_idx = multinames.index(TARGET_CLASSNAME) - searched_class_id = None - class_count = u30() - for class_id in range(class_count): - name_idx = u30() - if name_idx == searched_idx: - # We found the class we're looking for! - searched_class_id = class_id - u30() # super_name idx - flags = read_byte() - if flags & 0x08 != 0: # Protected namespace is present - u30() # protected_ns_idx - intrf_count = u30() - for _c2 in range(intrf_count): - u30() - u30() # iinit - trait_count = u30() - for _c2 in range(trait_count): - parse_traits_info() - - if searched_class_id is None: - raise ExtractorError(u'Target class %r not found' % - TARGET_CLASSNAME) - - method_names = {} - method_idxs = {} - for class_id in range(class_count): - u30() # cinit - trait_count = u30() - for _c2 in range(trait_count): - trait_methods = parse_traits_info() - if class_id == searched_class_id: - method_names.update(trait_methods.items()) - method_idxs.update(dict( - (idx, name) - for name, idx in trait_methods.items())) - - # Scripts - script_count = u30() - for _c in range(script_count): - u30() # init - trait_count = u30() - for _c2 in range(trait_count): - parse_traits_info() - - # Method bodies - method_body_count = u30() - Method = collections.namedtuple('Method', ['code', 'local_count']) - methods = {} - for _c in range(method_body_count): - method_idx = u30() - u30() # max_stack - local_count = u30() - u30() # init_scope_depth - u30() # max_scope_depth - code_length = u30() - code = read_bytes(code_length) - if method_idx in method_idxs: - m = Method(code, local_count) - methods[method_idxs[method_idx]] = m - exception_count = u30() - for _c2 in range(exception_count): - u30() # from - u30() # to - u30() # target - u30() # exc_type - u30() # var_name - trait_count = u30() - for _c2 in range(trait_count): - parse_traits_info() - - assert p + code_reader.tell() == len(code_tag) - assert len(methods) == len(method_idxs) - - method_pyfunctions = {} - - def extract_function(func_name): - if func_name in method_pyfunctions: - return method_pyfunctions[func_name] - if func_name not in methods: - raise ExtractorError(u'Cannot find function %r' % func_name) - m = methods[func_name] - - def resfunc(args): - registers = ['(this)'] + list(args) + [None] * m.local_count - stack = [] - coder = io.BytesIO(m.code) - while True: - opcode = struct.unpack('!B', coder.read(1))[0] - if opcode == 36: # pushbyte - v = struct.unpack('!B', coder.read(1))[0] - stack.append(v) - elif opcode == 44: # pushstring - idx = u30(coder) - stack.append(constant_strings[idx]) - elif opcode == 48: # pushscope - # We don't implement the scope register, so we'll just - # ignore the popped value - stack.pop() - elif opcode == 70: # callproperty - index = u30(coder) - mname = multinames[index] - arg_count = u30(coder) - args = list(reversed( - [stack.pop() for _ in range(arg_count)])) - obj = stack.pop() - if mname == u'split': - assert len(args) == 1 - assert isinstance(args[0], compat_str) - assert isinstance(obj, compat_str) - if args[0] == u'': - res = list(obj) - else: - res = obj.split(args[0]) - stack.append(res) - elif mname == u'slice': - assert len(args) == 1 - assert isinstance(args[0], int) - assert isinstance(obj, list) - res = obj[args[0]:] - stack.append(res) - elif mname == u'join': - assert len(args) == 1 - assert isinstance(args[0], compat_str) - assert isinstance(obj, list) - res = args[0].join(obj) - stack.append(res) - elif mname in method_pyfunctions: - stack.append(method_pyfunctions[mname](args)) - else: - raise NotImplementedError( - u'Unsupported property %r on %r' - % (mname, obj)) - elif opcode == 72: # returnvalue - res = stack.pop() - return res - elif opcode == 79: # callpropvoid - index = u30(coder) - mname = multinames[index] - arg_count = u30(coder) - args = list(reversed( - [stack.pop() for _ in range(arg_count)])) - obj = stack.pop() - if mname == u'reverse': - assert isinstance(obj, list) - obj.reverse() - else: - raise NotImplementedError( - u'Unsupported (void) property %r on %r' - % (mname, obj)) - elif opcode == 93: # findpropstrict - index = u30(coder) - mname = multinames[index] - res = extract_function(mname) - stack.append(res) - elif opcode == 97: # setproperty - index = u30(coder) - value = stack.pop() - idx = stack.pop() - obj = stack.pop() - assert isinstance(obj, list) - assert isinstance(idx, int) - obj[idx] = value - elif opcode == 98: # getlocal - index = u30(coder) - stack.append(registers[index]) - elif opcode == 99: # setlocal - index = u30(coder) - value = stack.pop() - registers[index] = value - elif opcode == 102: # getproperty - index = u30(coder) - pname = multinames[index] - if pname == u'length': - obj = stack.pop() - assert isinstance(obj, list) - stack.append(len(obj)) - else: # Assume attribute access - idx = stack.pop() - assert isinstance(idx, int) - obj = stack.pop() - assert isinstance(obj, list) - stack.append(obj[idx]) - elif opcode == 128: # coerce - u30(coder) - elif opcode == 133: # coerce_s - assert isinstance(stack[-1], (type(None), compat_str)) - elif opcode == 164: # modulo - value2 = stack.pop() - value1 = stack.pop() - res = value1 % value2 - stack.append(res) - elif opcode == 208: # getlocal_0 - stack.append(registers[0]) - elif opcode == 209: # getlocal_1 - stack.append(registers[1]) - elif opcode == 210: # getlocal_2 - stack.append(registers[2]) - elif opcode == 211: # getlocal_3 - stack.append(registers[3]) - elif opcode == 214: # setlocal_2 - registers[2] = stack.pop() - elif opcode == 215: # setlocal_3 - registers[3] = stack.pop() - else: - raise NotImplementedError( - u'Unsupported opcode %d' % opcode) - - method_pyfunctions[func_name] = resfunc - return resfunc - - initial_function = extract_function(u'decipher') + searched_class = swfi.extract_class(TARGET_CLASSNAME) + initial_function = swfi.extract_function(searched_class, u'decipher') return lambda s: initial_function([s]) def _decrypt_signature(self, s, video_id, player_url, age_gate=False): @@ -871,20 +480,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): if player_url.startswith(u'//'): player_url = u'https:' + player_url try: - player_id = (player_url, len(s)) + player_id = (player_url, self._signature_cache_id(s)) if player_id not in self._player_cache: func = self._extract_signature_function( - video_id, player_url, len(s) + video_id, player_url, s ) self._player_cache[player_id] = func func = self._player_cache[player_id] if self._downloader.params.get('youtube_print_sig_code'): - self._print_sig_code(func, len(s)) + self._print_sig_code(func, s) return func(s) except Exception as e: tb = traceback.format_exc() raise ExtractorError( - u'Automatic signature extraction failed: ' + tb, cause=e) + u'Signature extraction failed: ' + tb, cause=e) def _get_available_subtitles(self, video_id, webpage): try: @@ -899,6 +508,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): sub_lang_list = {} for l in lang_list: lang = l[1] + if lang in sub_lang_list: + continue params = compat_urllib_parse.urlencode({ 'lang': lang, 'v': video_id, @@ -1014,14 +625,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): age_gate = True # We simulate the access to the video from www.youtube.com/v/{video_id} # this can be viewed without login into Youtube - data = compat_urllib_parse.urlencode({'video_id': video_id, - 'el': 'player_embedded', - 'gl': 'US', - 'hl': 'en', - 'eurl': 'https://youtube.googleapis.com/v/' + video_id, - 'asv': 3, - 'sts':'1588', - }) + data = compat_urllib_parse.urlencode({ + 'video_id': video_id, + 'eurl': 'https://youtube.googleapis.com/v/' + video_id, + 'sts': self._search_regex( + r'"sts"\s*:\s*(\d+)', video_webpage, 'sts'), + }) video_info_url = proto + '://www.youtube.com/get_video_info?' + data video_info_webpage = self._download_webpage(video_info_url, video_id, note=False, @@ -1214,43 +823,54 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor): url_map = {} for url_data_str in encoded_url_map.split(','): url_data = compat_parse_qs(url_data_str) - if 'itag' in url_data and 'url' in url_data: - url = url_data['url'][0] - if 'sig' in url_data: - url += '&signature=' + url_data['sig'][0] - elif 's' in url_data: - encrypted_sig = url_data['s'][0] - if self._downloader.params.get('verbose'): - if age_gate: - if player_url is None: - player_version = 'unknown' - else: - player_version = self._search_regex( - r'-(.+)\.swf$', player_url, - u'flash player', fatal=False) + if 'itag' not in url_data or 'url' not in url_data: + continue + format_id = url_data['itag'][0] + url = url_data['url'][0] + + if 'sig' in url_data: + url += '&signature=' + url_data['sig'][0] + elif 's' in url_data: + encrypted_sig = url_data['s'][0] + + if not age_gate: + jsplayer_url_json = self._search_regex( + r'"assets":.+?"js":\s*("[^"]+")', + video_webpage, u'JS player URL') + player_url = json.loads(jsplayer_url_json) + if player_url is None: + player_url_json = self._search_regex( + r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")', + video_webpage, u'age gate player URL') + player_url = json.loads(player_url_json) + + if self._downloader.params.get('verbose'): + if player_url is None: + player_version = 'unknown' + player_desc = 'unknown' + else: + if player_url.endswith('swf'): + player_version = self._search_regex( + r'-(.+?)(?:/watch_as3)?\.swf$', player_url, + u'flash player', fatal=False) player_desc = 'flash player %s' % player_version else: player_version = self._search_regex( - r'html5player-(.+?)\.js', video_webpage, + r'html5player-([^/]+?)(?:/html5player)?\.js', + player_url, 'html5 player', fatal=False) player_desc = u'html5 player %s' % player_version - parts_sizes = u'.'.join(compat_str(len(part)) for part in encrypted_sig.split('.')) - self.to_screen(u'encrypted signature length %d (%s), itag %s, %s' % - (len(encrypted_sig), parts_sizes, url_data['itag'][0], player_desc)) - - if not age_gate: - jsplayer_url_json = self._search_regex( - r'"assets":.+?"js":\s*("[^"]+")', - video_webpage, u'JS player URL') - player_url = json.loads(jsplayer_url_json) - - signature = self._decrypt_signature( - encrypted_sig, video_id, player_url, age_gate) - url += '&signature=' + signature - if 'ratebypass' not in url: - url += '&ratebypass=yes' - url_map[url_data['itag'][0]] = url + parts_sizes = self._signature_cache_id(encrypted_sig) + self.to_screen(u'{%s} signature length %s, %s' % + (format_id, parts_sizes, player_desc)) + + signature = self._decrypt_signature( + encrypted_sig, video_id, player_url, age_gate) + url += '&signature=' + signature + if 'ratebypass' not in url: + url += '&ratebypass=yes' + url_map[format_id] = url formats = _map_to_format_list(url_map) elif video_info.get('hlsvp'): manifest_url = video_info['hlsvp'][0] |