From 5c2266df4b9aeb7881ed8c026a038e2a25e43734 Mon Sep 17 00:00:00 2001 From: Sergey M? Date: Sat, 21 Nov 2015 22:18:17 +0600 Subject: Switch codebase to use sanitized_Request instead of compat_urllib_request.Request [downloader/dash] Use sanitized_Request [downloader/http] Use sanitized_Request [atresplayer] Use sanitized_Request [bambuser] Use sanitized_Request [bliptv] Use sanitized_Request [brightcove] Use sanitized_Request [cbs] Use sanitized_Request [ceskatelevize] Use sanitized_Request [collegerama] Use sanitized_Request [extractor/common] Use sanitized_Request [crunchyroll] Use sanitized_Request [dailymotion] Use sanitized_Request [dcn] Use sanitized_Request [dramafever] Use sanitized_Request [dumpert] Use sanitized_Request [eitb] Use sanitized_Request [escapist] Use sanitized_Request [everyonesmixtape] Use sanitized_Request [extremetube] Use sanitized_Request [facebook] Use sanitized_Request [fc2] Use sanitized_Request [flickr] Use sanitized_Request [4tube] Use sanitized_Request [gdcvault] Use sanitized_Request [extractor/generic] Use sanitized_Request [hearthisat] Use sanitized_Request [hotnewhiphop] Use sanitized_Request [hypem] Use sanitized_Request [iprima] Use sanitized_Request [ivi] Use sanitized_Request [keezmovies] Use sanitized_Request [letv] Use sanitized_Request [lynda] Use sanitized_Request [metacafe] Use sanitized_Request [minhateca] Use sanitized_Request [miomio] Use sanitized_Request [meovideo] Use sanitized_Request [mofosex] Use sanitized_Request [moniker] Use sanitized_Request [mooshare] Use sanitized_Request [movieclips] Use sanitized_Request [mtv] Use sanitized_Request [myvideo] Use sanitized_Request [neteasemusic] Use sanitized_Request [nfb] Use sanitized_Request [niconico] Use sanitized_Request [noco] Use sanitized_Request [nosvideo] Use sanitized_Request [novamov] Use sanitized_Request [nowness] Use sanitized_Request [nuvid] Use sanitized_Request [played] Use sanitized_Request [pluralsight] Use sanitized_Request [pornhub] Use sanitized_Request [pornotube] Use sanitized_Request [primesharetv] Use sanitized_Request [promptfile] Use sanitized_Request [qqmusic] Use sanitized_Request [rtve] Use sanitized_Request [safari] Use sanitized_Request [sandia] Use sanitized_Request [shared] Use sanitized_Request [sharesix] Use sanitized_Request [sina] Use sanitized_Request [smotri] Use sanitized_Request [sohu] Use sanitized_Request [spankwire] Use sanitized_Request [sportdeutschland] Use sanitized_Request [streamcloud] Use sanitized_Request [streamcz] Use sanitized_Request [tapely] Use sanitized_Request [tube8] Use sanitized_Request [tubitv] Use sanitized_Request [twitch] Use sanitized_Request [twitter] Use sanitized_Request [udemy] Use sanitized_Request [vbox7] Use sanitized_Request [veoh] Use sanitized_Request [vessel] Use sanitized_Request [vevo] Use sanitized_Request [viddler] Use sanitized_Request [videomega] Use sanitized_Request [viewvster] Use sanitized_Request [viki] Use sanitized_Request [vk] Use sanitized_Request [vodlocker] Use sanitized_Request [voicerepublic] Use sanitized_Request [wistia] Use sanitized_Request [xfileshare] Use sanitized_Request [xtube] Use sanitized_Request [xvideos] Use sanitized_Request [yandexmusic] Use sanitized_Request [youku] Use sanitized_Request [youporn] Use sanitized_Request [youtube] Use sanitized_Request [patreon] Use sanitized_Request [extractor/common] Remove unused import [nfb] PEP 8 --- youtube_dl/extractor/udemy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'youtube_dl/extractor/udemy.py') diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py index 365d8b4bf..825172806 100644 --- a/youtube_dl/extractor/udemy.py +++ b/youtube_dl/extractor/udemy.py @@ -9,6 +9,7 @@ from ..compat import ( ) from ..utils import ( ExtractorError, + sanitized_Request, ) @@ -58,7 +59,7 @@ class UdemyIE(InfoExtractor): for header, value in headers.items(): url_or_request.add_header(header, value) else: - url_or_request = compat_urllib_request.Request(url_or_request, headers=headers) + url_or_request = sanitized_Request(url_or_request, headers=headers) response = super(UdemyIE, self)._download_json(url_or_request, video_id, note) self._handle_error(response) @@ -89,7 +90,7 @@ class UdemyIE(InfoExtractor): 'password': password.encode('utf-8'), }) - request = compat_urllib_request.Request( + request = sanitized_Request( self._LOGIN_URL, compat_urllib_parse.urlencode(login_form).encode('utf-8')) request.add_header('Referer', self._ORIGIN_URL) request.add_header('Origin', self._ORIGIN_URL) -- cgit v1.2.3 From 3b35c3425e2c4d9836a3efe5cf73e35a60f674d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 1 Dec 2015 20:35:46 +0600 Subject: [udemy] Extract formats from data.outputs (#7704) --- youtube_dl/extractor/udemy.py | 46 ++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 12 deletions(-) (limited to 'youtube_dl/extractor/udemy.py') diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py index 825172806..9d31a3f5b 100644 --- a/youtube_dl/extractor/udemy.py +++ b/youtube_dl/extractor/udemy.py @@ -9,6 +9,7 @@ from ..compat import ( ) from ..utils import ( ExtractorError, + int_or_none, sanitized_Request, ) @@ -127,26 +128,47 @@ class UdemyIE(InfoExtractor): video_id = asset['id'] thumbnail = asset.get('thumbnailUrl') or asset.get('thumbnail_url') - duration = asset['data']['duration'] + duration = int_or_none(asset.get('data', {}).get('duration')) download_url = asset.get('downloadUrl') or asset.get('download_url') video = download_url.get('Video') or download_url.get('video') video_480p = download_url.get('Video480p') or download_url.get('video_480p') - formats = [ - { - 'url': video_480p[0], - 'format_id': '360p', - }, - { - 'url': video[0], - 'format_id': '720p', - }, - ] + formats = [{ + 'url': video_480p[0], + 'format_id': 'download-360p', + }, { + 'url': video[0], + 'format_id': 'download-720p', + }] + + # Some videos also contain formats in asset['data']['outputs'] (e.g. + # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208) + outputs = asset.get('data', {}).get('outputs') + if isinstance(outputs, dict): + for format_id, f in outputs.items(): + video_url = f.get('url') + if video_url: + formats.append({ + 'url': video_url, + 'format_id': '%sp' % (f.get('labe1l') or format_id), + 'width': int_or_none(f.get('width')), + 'height': int_or_none(f.get('height')), + 'vbr': int_or_none(f.get('video_bitrate_in_kbps')), + 'vcodec': f.get('video_codec'), + 'fps': int_or_none(f.get('frame_rate')), + 'abr': int_or_none(f.get('audio_bitrate_in_kbps')), + 'acodec': f.get('audio_codec'), + 'asr': int_or_none(f.get('audio_sample_rate')), + 'tbr': int_or_none(f.get('total_bitrate_in_kbps')), + 'filesize': int_or_none(f.get('file_size_in_bytes')), + }) + + self._sort_formats(formats) title = lecture['title'] - description = lecture['description'] + description = lecture.get('description') return { 'id': video_id, -- cgit v1.2.3 From 78717fc32802cb1e0f240cfbdf02207c6356ff2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Tue, 1 Dec 2015 22:10:10 +0600 Subject: [udemy] Allow authentication via cookies --- youtube_dl/extractor/udemy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube_dl/extractor/udemy.py') diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py index 9d31a3f5b..e743c0262 100644 --- a/youtube_dl/extractor/udemy.py +++ b/youtube_dl/extractor/udemy.py @@ -72,7 +72,7 @@ class UdemyIE(InfoExtractor): def _login(self): (username, password) = self._get_login_info() if username is None: - self.raise_login_required('Udemy account is required') + return login_popup = self._download_webpage( self._LOGIN_URL, None, 'Downloading login popup') -- cgit v1.2.3 From 328f82d59a0d3b743280c3ab532949c9220383ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 2 Dec 2015 00:48:27 +0600 Subject: [udemy] Semi-switch to api 2.0 (Closes #7704) * Use api 2.0 to get lectures since it provides more formats * Fix authorization for api 2.0 * Autotry enrolling in the course for single lectures * Extract additional metadata rom asset['data']['outputs'] --- youtube_dl/extractor/udemy.py | 162 +++++++++++++++++++++++++----------------- 1 file changed, 96 insertions(+), 66 deletions(-) (limited to 'youtube_dl/extractor/udemy.py') diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py index e743c0262..26ae8cc36 100644 --- a/youtube_dl/extractor/udemy.py +++ b/youtube_dl/extractor/udemy.py @@ -1,14 +1,15 @@ from __future__ import unicode_literals -import re - from .common import InfoExtractor from ..compat import ( + compat_HTTPError, compat_urllib_parse, compat_urllib_request, + compat_urlparse, ) from ..utils import ( ExtractorError, + float_or_none, int_or_none, sanitized_Request, ) @@ -19,6 +20,8 @@ class UdemyIE(InfoExtractor): _VALID_URL = r'https?://www\.udemy\.com/(?:[^#]+#/lecture/|lecture/view/?\?lectureId=)(?P\d+)' _LOGIN_URL = 'https://www.udemy.com/join/login-popup/?displayType=ajax&showSkipButton=1' _ORIGIN_URL = 'https://www.udemy.com' + _SUCCESSFULLY_ENROLLED = '>You have enrolled in this course!<' + _ALREADY_ENROLLED = '>You are already taking this course.<' _NETRC_MACHINE = 'udemy' _TESTS = [{ @@ -34,6 +37,29 @@ class UdemyIE(InfoExtractor): 'skip': 'Requires udemy account credentials', }] + def _enroll_course(self, webpage, course_id): + enroll_url = self._search_regex( + r'href=(["\'])(?Phttps?://(?:www\.)?udemy\.com/course/subscribe/.+?)\1', + webpage, 'enroll url', group='url', + default='https://www.udemy.com/course/subscribe/?courseId=%s' % course_id) + webpage = self._download_webpage(enroll_url, course_id, 'Enrolling in the course') + if self._SUCCESSFULLY_ENROLLED in webpage: + self.to_screen('%s: Successfully enrolled in' % course_id) + elif self._ALREADY_ENROLLED in webpage: + self.to_screen('%s: Already enrolled in' % course_id) + + def _download_lecture(self, course_id, lecture_id): + return self._download_json( + 'https://www.udemy.com/api-2.0/users/me/subscribed-courses/%s/lectures/%s?%s' % ( + course_id, lecture_id, compat_urllib_parse.urlencode({ + 'video_only': '', + 'auto_play': '', + 'fields[lecture]': 'title,description,asset', + 'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data', + 'instructorPreviewMode': 'False', + })), + lecture_id, 'Downloading lecture JSON', fatal=False) + def _handle_error(self, response): if not isinstance(response, dict): return @@ -45,7 +71,7 @@ class UdemyIE(InfoExtractor): error_str += ' - %s' % error_data.get('formErrors') raise ExtractorError(error_str, expected=True) - def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'): + def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', *args, **kwargs): headers = { 'X-Udemy-Snail-Case': 'true', 'X-Requested-With': 'XMLHttpRequest', @@ -55,6 +81,7 @@ class UdemyIE(InfoExtractor): headers['X-Udemy-Client-Id'] = cookie.value elif cookie.name == 'access_token': headers['X-Udemy-Bearer-Token'] = cookie.value + headers['X-Udemy-Authorization'] = 'Bearer %s' % cookie.value if isinstance(url_or_request, compat_urllib_request.Request): for header, value in headers.items(): @@ -62,7 +89,7 @@ class UdemyIE(InfoExtractor): else: url_or_request = sanitized_Request(url_or_request, headers=headers) - response = super(UdemyIE, self)._download_json(url_or_request, video_id, note) + response = super(UdemyIE, self)._download_json(url_or_request, video_id, note, *args, **kwargs) self._handle_error(response) return response @@ -110,66 +137,77 @@ class UdemyIE(InfoExtractor): def _real_extract(self, url): lecture_id = self._match_id(url) - lecture = self._download_json( - 'https://www.udemy.com/api-1.1/lectures/%s' % lecture_id, - lecture_id, 'Downloading lecture JSON') + webpage = self._download_webpage(url, lecture_id) + + course_id = self._search_regex( + r'data-course-id=["\'](\d+)', webpage, 'course id') - asset_type = lecture.get('assetType') or lecture.get('asset_type') + try: + lecture = self._download_lecture(course_id, lecture_id) + except ExtractorError as e: + # Error could possibly mean we are not enrolled in the course + if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403: + self._enroll_course(webpage, course_id) + lecture_id = self._download_lecture(course_id, lecture_id) + else: + raise + + title = lecture['title'] + description = lecture.get('description') + + asset = lecture['asset'] + + asset_type = asset.get('assetType') or asset.get('asset_type') if asset_type != 'Video': raise ExtractorError( 'Lecture %s is not a video' % lecture_id, expected=True) - asset = lecture['asset'] - stream_url = asset.get('streamUrl') or asset.get('stream_url') - mobj = re.search(r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url) - if mobj: - return self.url_result(mobj.group(1), 'Youtube') + if stream_url: + youtube_url = self._search_regex( + r'(https?://www\.youtube\.com/watch\?v=.*)', stream_url, 'youtube URL', default=None) + if youtube_url: + return self.url_result(youtube_url, 'Youtube') video_id = asset['id'] thumbnail = asset.get('thumbnailUrl') or asset.get('thumbnail_url') - duration = int_or_none(asset.get('data', {}).get('duration')) - - download_url = asset.get('downloadUrl') or asset.get('download_url') - - video = download_url.get('Video') or download_url.get('video') - video_480p = download_url.get('Video480p') or download_url.get('video_480p') - - formats = [{ - 'url': video_480p[0], - 'format_id': 'download-360p', - }, { - 'url': video[0], - 'format_id': 'download-720p', - }] - - # Some videos also contain formats in asset['data']['outputs'] (e.g. - # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208) - outputs = asset.get('data', {}).get('outputs') - if isinstance(outputs, dict): - for format_id, f in outputs.items(): - video_url = f.get('url') - if video_url: - formats.append({ - 'url': video_url, - 'format_id': '%sp' % (f.get('labe1l') or format_id), - 'width': int_or_none(f.get('width')), - 'height': int_or_none(f.get('height')), - 'vbr': int_or_none(f.get('video_bitrate_in_kbps')), - 'vcodec': f.get('video_codec'), - 'fps': int_or_none(f.get('frame_rate')), - 'abr': int_or_none(f.get('audio_bitrate_in_kbps')), - 'acodec': f.get('audio_codec'), - 'asr': int_or_none(f.get('audio_sample_rate')), - 'tbr': int_or_none(f.get('total_bitrate_in_kbps')), - 'filesize': int_or_none(f.get('file_size_in_bytes')), + duration = float_or_none(asset.get('data', {}).get('duration')) + outputs = asset.get('data', {}).get('outputs', {}) + + formats = [] + for format_ in asset.get('download_urls', {}).get('Video', []): + video_url = format_.get('file') + if not video_url: + continue + format_id = format_.get('label') + f = { + 'url': format_['file'], + 'height': int_or_none(format_id), + } + if format_id: + # Some videos contain additional metadata (e.g. + # https://www.udemy.com/ios9-swift/learn/#/lecture/3383208) + output = outputs.get(format_id) + if isinstance(output, dict): + f.update({ + 'format_id': '%sp' % (output.get('label') or format_id), + 'width': int_or_none(output.get('width')), + 'height': int_or_none(output.get('height')), + 'vbr': int_or_none(output.get('video_bitrate_in_kbps')), + 'vcodec': output.get('video_codec'), + 'fps': int_or_none(output.get('frame_rate')), + 'abr': int_or_none(output.get('audio_bitrate_in_kbps')), + 'acodec': output.get('audio_codec'), + 'asr': int_or_none(output.get('audio_sample_rate')), + 'tbr': int_or_none(output.get('total_bitrate_in_kbps')), + 'filesize': int_or_none(output.get('file_size_in_bytes')), }) + else: + f['format_id'] = '%sp' % format_id + formats.append(f) self._sort_formats(formats) - title = lecture['title'] - description = lecture.get('description') - return { 'id': video_id, 'title': title, @@ -182,9 +220,7 @@ class UdemyIE(InfoExtractor): class UdemyCourseIE(UdemyIE): IE_NAME = 'udemy:course' - _VALID_URL = r'https?://www\.udemy\.com/(?P[\da-z-]+)' - _SUCCESSFULLY_ENROLLED = '>You have enrolled in this course!<' - _ALREADY_ENROLLED = '>You are already taking this course.<' + _VALID_URL = r'https?://www\.udemy\.com/(?P[\da-z-]+)' _TESTS = [] @classmethod @@ -192,24 +228,18 @@ class UdemyCourseIE(UdemyIE): return False if UdemyIE.suitable(url) else super(UdemyCourseIE, cls).suitable(url) def _real_extract(self, url): - mobj = re.match(self._VALID_URL, url) - course_path = mobj.group('coursepath') + course_path = self._match_id(url) + + webpage = self._download_webpage(url, course_path) response = self._download_json( 'https://www.udemy.com/api-1.1/courses/%s' % course_path, course_path, 'Downloading course JSON') - course_id = int(response['id']) - course_title = response['title'] + course_id = response['id'] + course_title = response.get('title') - webpage = self._download_webpage( - 'https://www.udemy.com/course/subscribe/?courseId=%s' % course_id, - course_id, 'Enrolling in the course') - - if self._SUCCESSFULLY_ENROLLED in webpage: - self.to_screen('%s: Successfully enrolled in' % course_id) - elif self._ALREADY_ENROLLED in webpage: - self.to_screen('%s: Already enrolled in' % course_id) + self._enroll_course(webpage, course_id) response = self._download_json( 'https://www.udemy.com/api-1.1/courses/%s/curriculum' % course_id, -- cgit v1.2.3 From 9fc87fa767425ee7d56b2ce4650c9b06b480e005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 2 Dec 2015 00:51:47 +0600 Subject: [udemy] Remove unused import --- youtube_dl/extractor/udemy.py | 1 - 1 file changed, 1 deletion(-) (limited to 'youtube_dl/extractor/udemy.py') diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py index 26ae8cc36..2a54f3764 100644 --- a/youtube_dl/extractor/udemy.py +++ b/youtube_dl/extractor/udemy.py @@ -5,7 +5,6 @@ from ..compat import ( compat_HTTPError, compat_urllib_parse, compat_urllib_request, - compat_urlparse, ) from ..utils import ( ExtractorError, -- cgit v1.2.3 From 24121bc703152312cfbb70f01ebd39e2fe1197e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergey=20M=E2=80=A4?= Date: Wed, 2 Dec 2015 00:53:03 +0600 Subject: [udemy] Make lecture downloading fatal --- youtube_dl/extractor/udemy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'youtube_dl/extractor/udemy.py') diff --git a/youtube_dl/extractor/udemy.py b/youtube_dl/extractor/udemy.py index 2a54f3764..59832b1ec 100644 --- a/youtube_dl/extractor/udemy.py +++ b/youtube_dl/extractor/udemy.py @@ -57,7 +57,7 @@ class UdemyIE(InfoExtractor): 'fields[asset]': 'asset_type,stream_url,thumbnail_url,download_urls,data', 'instructorPreviewMode': 'False', })), - lecture_id, 'Downloading lecture JSON', fatal=False) + lecture_id, 'Downloading lecture JSON') def _handle_error(self, response): if not isinstance(response, dict): @@ -70,7 +70,7 @@ class UdemyIE(InfoExtractor): error_str += ' - %s' % error_data.get('formErrors') raise ExtractorError(error_str, expected=True) - def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata', *args, **kwargs): + def _download_json(self, url_or_request, video_id, note='Downloading JSON metadata'): headers = { 'X-Udemy-Snail-Case': 'true', 'X-Requested-With': 'XMLHttpRequest', @@ -88,7 +88,7 @@ class UdemyIE(InfoExtractor): else: url_or_request = sanitized_Request(url_or_request, headers=headers) - response = super(UdemyIE, self)._download_json(url_or_request, video_id, note, *args, **kwargs) + response = super(UdemyIE, self)._download_json(url_or_request, video_id, note) self._handle_error(response) return response -- cgit v1.2.3