diff options
| -rw-r--r-- | youtube_dl/extractor/imgur.py | 348 |
1 files changed, 279 insertions, 69 deletions
diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py index a5ba03efa..59f129d6a 100644 --- a/youtube_dl/extractor/imgur.py +++ b/youtube_dl/extractor/imgur.py @@ -1,101 +1,267 @@ +# coding: utf-8 from __future__ import unicode_literals import re from .common import InfoExtractor from ..utils import ( + determine_ext, + ExtractorError, + float_or_none, int_or_none, js_to_json, + merge_dicts, mimetype2ext, - ExtractorError, + parse_iso8601, + T, + traverse_obj, + txt_or_none, + url_or_none, ) -class ImgurIE(InfoExtractor): - _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|(?:t(?:opic)?|r)/[^/]+)/)(?P<id>[a-zA-Z0-9]+)' +class ImgurBaseIE(InfoExtractor): + # hard-coded value, as also used by ArchiveTeam + _CLIENT_ID = '546c25a59c58ad7' + + @classmethod + def _imgur_result(cls, item_id): + return cls.url_result('imgur:%s' % item_id, ImgurIE.ie_key(), item_id) + + def _call_api(self, endpoint, video_id, **kwargs): + return self._download_json( + 'https://api.imgur.com/post/v1/%s/%s?client_id=%s&include=media,account' % (endpoint, video_id, self._CLIENT_ID), + video_id, **kwargs) + + @staticmethod + def get_description(s): + if 'Discover the magic of the internet at Imgur' in s: + return None + return txt_or_none(s) + + +class ImgurIE(ImgurBaseIE): + _VALID_URL = r'''(?x) + (?: + https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|t|topic|r)/)| + imgur: + )(?P<id>[a-zA-Z0-9]+) + ''' _TESTS = [{ - 'url': 'https://i.imgur.com/A61SaA1.gifv', + 'url': 'https://imgur.com/A61SaA1', 'info_dict': { 'id': 'A61SaA1', 'ext': 'mp4', 'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$', + 'timestamp': 1416446068, + 'upload_date': '20141120', }, }, { - 'url': 'https://imgur.com/A61SaA1', + 'url': 'https://i.imgur.com/A61SaA1.gifv', 'only_matching': True, }, { 'url': 'https://i.imgur.com/crGpqCV.mp4', 'only_matching': True, }, { - # no title + # previously, no title 'url': 'https://i.imgur.com/jxBXAMC.gifv', - 'only_matching': True, + 'info_dict': { + 'id': 'jxBXAMC', + 'ext': 'mp4', + 'title': 'Fahaka puffer feeding', + 'timestamp': 1533835503, + 'upload_date': '20180809', + }, }] + def _extract_twitter_formats(self, html, tw_id='twitter', **kwargs): + fatal = kwargs.pop('fatal', False) + tw_stream = self._html_search_meta('twitter:player:stream', html, fatal=fatal, **kwargs) + if not tw_stream: + return [] + ext = mimetype2ext(self._html_search_meta( + 'twitter:player:stream:content_type', html, default=None)) + width, height = (int_or_none(self._html_search_meta('twitter:player:' + v, html, default=None)) + for v in ('width', 'height')) + return [{ + 'format_id': tw_id, + 'url': tw_stream, + 'ext': ext or determine_ext(tw_stream), + 'width': width, + 'height': height, + }] + def _real_extract(self, url): video_id = self._match_id(url) + data = self._call_api('media', video_id, fatal=False, expected_status=404) webpage = self._download_webpage( - 'https://i.imgur.com/{id}.gifv'.format(id=video_id), video_id) + 'https://i.imgur.com/{id}.gifv'.format(id=video_id), video_id, fatal=not data) or '' + + if not traverse_obj(data, ('media', 0, ( + ('type', T(lambda t: t == 'video' or None)), + ('metadata', 'is_animated'))), get_all=False): + raise ExtractorError( + '%s is not a video or animated image' % video_id, + expected=True) + + media_fmt = traverse_obj(data, ('media', 0, { + 'url': ('url', T(url_or_none)), + 'ext': 'ext', + 'width': ('width', T(int_or_none)), + 'height': ('height', T(int_or_none)), + 'filesize': ('size', T(int_or_none)), + 'acodec': ('metadata', 'has_sound', T(lambda b: None if b else 'none')), + })) - width = int_or_none(self._og_search_property( - 'video:width', webpage, default=None)) - height = int_or_none(self._og_search_property( - 'video:height', webpage, default=None)) + media_url = traverse_obj(media_fmt, 'url') + if media_url: + if not media_fmt.get('ext'): + media_fmt['ext'] = mimetype2ext(traverse_obj( + data, ('media', 0, 'mime_type'))) or determine_ext(media_url) + if traverse_obj(data, ('media', 0, 'type')) == 'image': + media_fmt['acodec'] = 'none' + media_fmt.setdefault('preference', -10) + + tw_formats = self._extract_twitter_formats(webpage) + if traverse_obj(tw_formats, (0, 'url')) == media_url: + tw_formats = [] + else: + # maybe this isn't an animated image/video? + self._check_formats(tw_formats, video_id) video_elements = self._search_regex( r'(?s)<div class="video-elements">(.*?)</div>', webpage, 'video elements', default=None) - if not video_elements: + if not (video_elements or tw_formats or media_url): raise ExtractorError( - 'No sources found for video %s. Maybe an image?' % video_id, + 'No sources found for video %s. Maybe a plain image?' % video_id, expected=True) - formats = [] - for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements): - formats.append({ - 'format_id': m.group('type').partition('/')[2], - 'url': self._proto_relative_url(m.group('src')), - 'ext': mimetype2ext(m.group('type')), - 'width': width, - 'height': height, + def mung_format(fmt, *extra): + fmt.update({ 'http_headers': { 'User-Agent': 'youtube-dl (like wget)', }, }) + for d in extra: + fmt.update(d) + return fmt - gif_json = self._search_regex( - r'(?s)var\s+videoItem\s*=\s*(\{.*?\})', - webpage, 'GIF code', fatal=False) - if gif_json: - gifd = self._parse_json( - gif_json, video_id, transform_source=js_to_json) - formats.append({ - 'format_id': 'gif', - 'preference': -10, - 'width': width, - 'height': height, - 'ext': 'gif', - 'acodec': 'none', - 'vcodec': 'gif', - 'container': 'gif', - 'url': self._proto_relative_url(gifd['gifUrl']), - 'filesize': gifd.get('size'), - 'http_headers': { - 'User-Agent': 'youtube-dl (like wget)', - }, - }) + if video_elements: + def og_get_size(media_type): + return dict((p, int_or_none(self._og_search_property( + ':'.join((media_type, p)), webpage, default=None))) + for p in ('width', 'height')) + + size = og_get_size('video') + if all(v is None for v in size.values()): + size = og_get_size('image') + + formats = traverse_obj( + re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements), + (Ellipsis, { + 'format_id': ('type', T(lambda s: s.partition('/')[2])), + 'url': ('src', T(self._proto_relative_url)), + 'ext': ('type', T(mimetype2ext)), + }, T(lambda f: mung_format(f, size)))) + + gif_json = self._search_regex( + r'(?s)var\s+videoItem\s*=\s*(\{.*?\})', + webpage, 'GIF code', fatal=False) + MUST_BRANCH = (None, T(lambda _: None)) + formats.extend(traverse_obj(gif_json, ( + T(lambda j: self._parse_json( + j, video_id, transform_source=js_to_json, fatal=False)), { + 'url': ('gifUrl', T(self._proto_relative_url)), + 'filesize': ('size', T(int_or_none)), + }, T(lambda f: mung_format(f, size, { + 'format_id': 'gif', + 'preference': -10, # gifs are worse than videos + 'ext': 'gif', + 'acodec': 'none', + 'vcodec': 'gif', + 'container': 'gif', + })), MUST_BRANCH))) + else: + formats = [] + + # maybe add formats from JSON or page Twitter metadata + if not any((u == media_url) for u in traverse_obj(formats, (Ellipsis, 'url'))): + formats.append(mung_format(media_fmt)) + tw_url = traverse_obj(tw_formats, (0, 'url')) + if not any((u == tw_url) for u in traverse_obj(formats, (Ellipsis, 'url'))): + formats.extend(mung_format(f) for f in tw_formats) self._sort_formats(formats) - return { + return merge_dicts(traverse_obj(data, { + 'uploader_id': ('account_id', T(txt_or_none), + T(lambda a: a if int_or_none(a) != 0 else None)), + 'uploader': ('account', 'username', T(txt_or_none)), + 'uploader_url': ('account', 'avatar_url', T(url_or_none)), + 'like_count': ('upvote_count', T(int_or_none)), + 'dislike_count': ('downvote_count', T(int_or_none)), + 'comment_count': ('comment_count', T(int_or_none)), + 'age_limit': ('is_mature', T(lambda x: 18 if x else None)), + 'timestamp': (('updated_at', 'created_at'), T(parse_iso8601)), + 'release_timestamp': ('created_at', T(parse_iso8601)), + }, get_all=False), traverse_obj(data, ('media', 0, 'metadata', { + 'title': ('title', T(txt_or_none)), + 'description': ('description', T(self.get_description)), + 'duration': ('duration', T(float_or_none)), + 'timestamp': (('updated_at', 'created_at'), T(parse_iso8601)), + 'release_timestamp': ('created_at', T(parse_iso8601)), + })), { 'id': video_id, 'formats': formats, - 'title': self._og_search_title(webpage, default=video_id), - } + 'title': self._og_search_title(webpage, default='Imgur video ' + video_id), + 'description': self.get_description(self._og_search_description(webpage)), + 'thumbnail': url_or_none(self._html_search_meta('thumbnailUrl', webpage, default=None)), + }) + + +class ImgurGalleryBaseIE(ImgurBaseIE): + _GALLERY = True + + def _real_extract(self, url): + gallery_id = self._match_id(url) + data = self._call_api('albums', gallery_id, fatal=False, expected_status=404) -class ImgurGalleryIE(InfoExtractor): + info = traverse_obj(data, { + 'title': ('title', T(txt_or_none)), + 'description': ('description', T(self.get_description)), + }) + + if traverse_obj(data, 'is_album'): + + def yield_media_ids(): + for m_id in traverse_obj(data, ( + 'media', lambda _, v: v.get('type') == 'video' or v['metadata']['is_animated'], + 'id', T(txt_or_none))): + yield m_id + + # if a gallery with exactly one video, apply album metadata to video + media_id = ( + self._GALLERY + and traverse_obj(data, ('image_count', T(lambda c: c == 1))) + and next(yield_media_ids(), None)) + + if not media_id: + result = self.playlist_result( + map(self._imgur_result, yield_media_ids()), gallery_id) + result.update(info) + return result + gallery_id = media_id + + result = self._imgur_result(gallery_id) + info['_type'] = 'url_transparent' + result.update(info) + return result + + +class ImgurGalleryIE(ImgurGalleryBaseIE): IE_NAME = 'imgur:gallery' _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:gallery|(?:t(?:opic)?|r)/[^/]+)/(?P<id>[a-zA-Z0-9]+)' @@ -106,49 +272,93 @@ class ImgurGalleryIE(InfoExtractor): 'title': 'Adding faces make every GIF better', }, 'playlist_count': 25, + 'skip': 'Zoinks! You\'ve taken a wrong turn.', }, { + # TODO: static images - replace with animated/video gallery 'url': 'http://imgur.com/topic/Aww/ll5Vk', 'only_matching': True, }, { 'url': 'https://imgur.com/gallery/YcAQlkx', + 'add_ies': ['Imgur'], 'info_dict': { 'id': 'YcAQlkx', 'ext': 'mp4', 'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....', - } + 'timestamp': 1358554297, + 'upload_date': '20130119', + 'uploader_id': '1648642', + 'uploader': 'wittyusernamehere', + }, }, { + # TODO: static image - replace with animated/video gallery 'url': 'http://imgur.com/topic/Funny/N8rOudd', 'only_matching': True, }, { 'url': 'http://imgur.com/r/aww/VQcQPhM', - 'only_matching': True, + 'add_ies': ['Imgur'], + 'info_dict': { + 'id': 'VQcQPhM', + 'ext': 'mp4', + 'title': 'The boss is here', + 'timestamp': 1476494751, + 'upload_date': '20161015', + 'uploader_id': '19138530', + 'uploader': 'thematrixcam', + }, + }, + # from PR #16674 + { + 'url': 'https://imgur.com/t/unmuted/6lAn9VQ', + 'info_dict': { + 'id': '6lAn9VQ', + 'title': 'Penguins !', + }, + 'playlist_count': 3, + }, { + 'url': 'https://imgur.com/t/unmuted/kx2uD3C', + 'add_ies': ['Imgur'], + 'info_dict': { + 'id': 'ZVMv45i', + 'ext': 'mp4', + 'title': 'Intruder', + 'timestamp': 1528129683, + 'upload_date': '20180604', + }, + }, { + 'url': 'https://imgur.com/t/unmuted/wXSK0YH', + 'add_ies': ['Imgur'], + 'info_dict': { + 'id': 'JCAP4io', + 'ext': 'mp4', + 'title': 're:I got the blues$', + 'description': 'Luka’s vocal stylings.\n\nFP edit: don’t encourage me. I’ll never stop posting Luka and friends.', + 'timestamp': 1527809525, + 'upload_date': '20180531', + }, }] - def _real_extract(self, url): - gallery_id = self._match_id(url) - - data = self._download_json( - 'https://imgur.com/gallery/%s.json' % gallery_id, - gallery_id)['data']['image'] - if data.get('is_album'): - entries = [ - self.url_result('http://imgur.com/%s' % image['hash'], ImgurIE.ie_key(), image['hash']) - for image in data['album_images']['images'] if image.get('hash')] - return self.playlist_result(entries, gallery_id, data.get('title'), data.get('description')) - - return self.url_result('http://imgur.com/%s' % gallery_id, ImgurIE.ie_key(), gallery_id) - - -class ImgurAlbumIE(ImgurGalleryIE): +class ImgurAlbumIE(ImgurGalleryBaseIE): IE_NAME = 'imgur:album' _VALID_URL = r'https?://(?:i\.)?imgur\.com/a/(?P<id>[a-zA-Z0-9]+)' - + _GALLERY = False _TESTS = [{ + # TODO: only static images - replace with animated/video gallery 'url': 'http://imgur.com/a/j6Orj', + 'only_matching': True, + }, + # from PR #21693 + { + 'url': 'https://imgur.com/a/iX265HX', + 'info_dict': { + 'id': 'iX265HX', + 'title': 'enen-no-shouboutai' + }, + 'playlist_count': 2, + }, { + 'url': 'https://imgur.com/a/8pih2Ed', 'info_dict': { - 'id': 'j6Orj', - 'title': 'A Literary Analysis of "Star Wars: The Force Awakens"', + 'id': '8pih2Ed' }, - 'playlist_count': 12, + 'playlist_mincount': 1, }] |