diff options
author | remitamine <remitamine@gmail.com> | 2015-12-20 12:43:00 +0100 |
---|---|---|
committer | remitamine <remitamine@gmail.com> | 2015-12-20 12:43:00 +0100 |
commit | c6ed6fadc248bd4957fd7cccc1eeb0f109ac3ffa (patch) | |
tree | 5ed84454704732b7281d2d2c1627f692865e0c01 /youtube_dl/extractor/cnet.py | |
parent | a641b2459263228fb1dd86dfe05d6047cedbf345 (diff) | |
download | youtube-dl-c6ed6fadc248bd4957fd7cccc1eeb0f109ac3ffa.tar.xz |
[cnet] improve extraction
- relex data json regex
- extract the platform metadata once
- extract hds formats
- extract duration
- extract thumbnail
Diffstat (limited to 'youtube_dl/extractor/cnet.py')
-rw-r--r-- | youtube_dl/extractor/cnet.py | 41 |
1 files changed, 23 insertions, 18 deletions
diff --git a/youtube_dl/extractor/cnet.py b/youtube_dl/extractor/cnet.py index 3ecf0efd4..5c3908f72 100644 --- a/youtube_dl/extractor/cnet.py +++ b/youtube_dl/extractor/cnet.py @@ -1,33 +1,33 @@ # coding: utf-8 from __future__ import unicode_literals -import json - -from .common import InfoExtractor from .theplatform import ThePlatformIE +from ..utils import int_or_none -class CNETIE(InfoExtractor): +class CNETIE(ThePlatformIE): _VALID_URL = r'https?://(?:www\.)?cnet\.com/videos/(?P<id>[^/]+)/' _TESTS = [{ 'url': 'http://www.cnet.com/videos/hands-on-with-microsofts-windows-8-1-update/', 'info_dict': { 'id': '56f4ea68-bd21-4852-b08c-4de5b8354c60', - 'ext': 'mp4', + 'ext': 'flv', 'title': 'Hands-on with Microsoft Windows 8.1 Update', 'description': 'The new update to the Windows 8 OS brings improved performance for mouse and keyboard users.', 'uploader_id': '6085384d-619e-11e3-b231-14feb5ca9861', 'uploader': 'Sarah Mitroff', + 'duration': 70, }, }, { 'url': 'http://www.cnet.com/videos/whiny-pothole-tweets-at-local-government-when-hit-by-cars-tomorrow-daily-187/', 'info_dict': { 'id': '56527b93-d25d-44e3-b738-f989ce2e49ba', - 'ext': 'mp4', + 'ext': 'flv', + 'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)', 'description': 'Khail and Ashley wonder what other civic woes can be solved by self-tweeting objects, investigate a new kind of VR camera and watch an origami robot self-assemble, walk, climb, dig and dissolve. #TDPothole', 'uploader_id': 'b163284d-6b73-44fc-b3e6-3da66c392d40', 'uploader': 'Ashley Esqueda', - 'title': 'Whiny potholes tweet at local government when hit by cars (Tomorrow Daily 187)', + 'duration': 1482, }, }] @@ -36,10 +36,10 @@ class CNETIE(InfoExtractor): webpage = self._download_webpage(url, display_id) data_json = self._html_search_regex( - r"<div class=\"videoPlayer\"\s+.*?data-cnet-video-uvp-options='([^']+)'", + r"data-cnet-video(?:-uvp)?-options='([^']+)'", webpage, 'data json') - data = json.loads(data_json) - vdata = data['videos'][0] + data = self._parse_json(data_json, display_id) + vdata = data.get('video') or data['videos'][0] video_id = vdata['id'] title = vdata['title'] @@ -52,19 +52,22 @@ class CNETIE(InfoExtractor): uploader_id = None mpx_account = data['config']['uvpConfig']['default']['mpx_account'] - tp = ThePlatformIE(self._downloader) + + metadata = self.get_metadata('%s/%s' % (mpx_account, list(vdata['files'].values())[0]), video_id) + description = vdata.get('description') or metadata.get('description') + duration = int_or_none(vdata.get('duration')) or metadata.get('duration') + formats = [] subtitles = {} - description = vdata.get('description') - for (fkey, vid) in vdata['files'].items(): if fkey == 'hls_phone' and 'hls_tablet' in vdata['files']: continue - result = tp.extract(('http://link.theplatform.com/s/%s/%s' % (mpx_account, vid))) - formats.extend(result['formats']) - subtitles = self._merge_subtitles(subtitles, result['subtitles']) - description = description or result.get('description') - + release_url = 'http://link.theplatform.com/s/%s/%s?format=SMIL&mbr=true' % (mpx_account, vid) + if fkey == 'hds': + release_url += '&manifest=f4m' + tp_formats, tp_subtitles = self._extract_theplatform_smil(release_url, video_id, 'Downloading %s SMIL data' % fkey) + formats.extend(tp_formats) + subtitles = self._merge_subtitles(subtitles, tp_subtitles) self._sort_formats(formats) return { @@ -72,6 +75,8 @@ class CNETIE(InfoExtractor): 'display_id': display_id, 'title': title, 'description': description, + 'thumbnail': metadata.get('thumbnail'), + 'duration': duration, 'uploader': uploader, 'uploader_id': uploader_id, 'subtitles': subtitles, |