aboutsummaryrefslogtreecommitdiff
path: root/youtube_dl/extractor/kinja.py
blob: 79e3026d2f4a00e47299f6a1c2ac838cd5759f8d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
# coding: utf-8
from __future__ import unicode_literals

import re

from .common import InfoExtractor
from ..compat import (
    compat_str,
    compat_urllib_parse_unquote,
)
from ..utils import (
    int_or_none,
    parse_iso8601,
    strip_or_none,
    try_get,
    unescapeHTML,
    urljoin,
)


class KinjaEmbedIE(InfoExtractor):
    IENAME = 'kinja:embed'
    _DOMAIN_REGEX = r'''(?:[^.]+\.)?
        (?:
            avclub|
            clickhole|
            deadspin|
            gizmodo|
            jalopnik|
            jezebel|
            kinja|
            kotaku|
            lifehacker|
            splinternews|
            the(?:inventory|onion|root|takeout)
        )\.com'''
    _COMMON_REGEX = r'''/
        (?:
            ajax/inset|
            embed/video
        )/iframe\?.*?\bid='''
    _VALID_URL = r'''(?x)https?://%s%s
        (?P<type>
            fb|
            imgur|
            instagram|
            jwp(?:layer)?-video|
            kinjavideo|
            mcp|
            megaphone|
            ooyala|
            soundcloud(?:-playlist)?|
            tumblr-post|
            twitch-stream|
            twitter|
            ustream-channel|
            vimeo|
            vine|
            youtube-(?:list|video)
        )-(?P<id>[^&]+)''' % (_DOMAIN_REGEX, _COMMON_REGEX)
    _TESTS = [{
        'url': 'https://kinja.com/ajax/inset/iframe?id=fb-10103303356633621',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=kinjavideo-100313',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=megaphone-PPY1300931075',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=ooyala-xzMXhleDpopuT0u1ijt_qZj3Va-34pEX%2FZTIxYmJjZDM2NWYzZDViZGRiOWJjYzc5',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=soundcloud-128574047',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=soundcloud-playlist-317413750',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=tumblr-post-160130699814-daydreams-at-midnight',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=twitch-stream-libratus_extra',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=twitter-1068875942473404422',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=ustream-channel-10414700',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=vimeo-120153502',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=vine-5BlvV5qqPrD',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=youtube-list-BCQ3KyrPjgA/PLE6509247C270A72E',
        'only_matching': True,
    }, {
        'url': 'https://kinja.com/ajax/inset/iframe?id=youtube-video-00QyL0AgPAE',
        'only_matching': True,
    }]
    _JWPLATFORM_PROVIDER = ('cdn.jwplayer.com/v2/media/', 'JWPlatform')
    _PROVIDER_MAP = {
        'fb': ('facebook.com/video.php?v=', 'Facebook'),
        'imgur': ('imgur.com/', 'Imgur'),
        'instagram': ('instagram.com/p/', 'Instagram'),
        'jwplayer-video': _JWPLATFORM_PROVIDER,
        'jwp-video': _JWPLATFORM_PROVIDER,
        'megaphone': ('player.megaphone.fm/', 'Generic'),
        'ooyala': ('player.ooyala.com/player.js?embedCode=', 'Ooyala'),
        'soundcloud': ('api.soundcloud.com/tracks/', 'Soundcloud'),
        'soundcloud-playlist': ('api.soundcloud.com/playlists/', 'SoundcloudPlaylist'),
        'tumblr-post': ('%s.tumblr.com/post/%s', 'Tumblr'),
        'twitch-stream': ('twitch.tv/', 'TwitchStream'),
        'twitter': ('twitter.com/i/cards/tfw/v1/', 'TwitterCard'),
        'ustream-channel': ('ustream.tv/embed/', 'Ustream'),
        'vimeo': ('vimeo.com/', 'Vimeo'),
        'vine': ('vine.co/v/', 'Vine'),
        'youtube-list': ('youtube.com/embed/%s?list=%s', 'YoutubePlaylist'),
        'youtube-video': ('youtube.com/embed/', 'Youtube'),
    }

    @staticmethod
    def _extract_urls(webpage, url):
        return [urljoin(url, unescapeHTML(mobj.group('url'))) for mobj in re.finditer(
            r'(?x)<iframe[^>]+?src=(?P<q>["\'])(?P<url>(?:(?:https?:)?//%s)?%s(?:(?!\1).)+)\1' % (KinjaEmbedIE._DOMAIN_REGEX, KinjaEmbedIE._COMMON_REGEX),
            webpage)]

    def _real_extract(self, url):
        video_type, video_id = re.match(self._VALID_URL, url).groups()

        provider = self._PROVIDER_MAP.get(video_type)
        if provider:
            video_id = compat_urllib_parse_unquote(video_id)
            if video_type == 'tumblr-post':
                video_id, blog = video_id.split('-', 1)
                result_url = provider[0] % (blog, video_id)
            elif video_type == 'youtube-list':
                video_id, playlist_id = video_id.split('/')
                result_url = provider[0] % (video_id, playlist_id)
            else:
                if video_type == 'ooyala':
                    video_id = video_id.split('/')[0]
                result_url = provider[0] + video_id
            return self.url_result('http://' + result_url, provider[1])

        if video_type == 'kinjavideo':
            data = self._download_json(
                'https://kinja.com/api/core/video/views/videoById',
                video_id, query={'videoId': video_id})['data']
            title = data['title']

            formats = []
            for k in ('signedPlaylist', 'streaming'):
                m3u8_url = data.get(k + 'Url')
                if m3u8_url:
                    formats.extend(self._extract_m3u8_formats(
                        m3u8_url, video_id, 'mp4', 'm3u8_native',
                        m3u8_id='hls', fatal=False))
            self._sort_formats(formats)

            thumbnail = None
            poster = data.get('poster') or {}
            poster_id = poster.get('id')
            if poster_id:
                thumbnail = 'https://i.kinja-img.com/gawker-media/image/upload/%s.%s' % (poster_id, poster.get('format') or 'jpg')

            return {
                'id': video_id,
                'title': title,
                'description': strip_or_none(data.get('description')),
                'formats': formats,
                'tags': data.get('tags'),
                'timestamp': int_or_none(try_get(
                    data, lambda x: x['postInfo']['publishTimeMillis']), 1000),
                'thumbnail': thumbnail,
                'uploader': data.get('network'),
            }
        else:
            video_data = self._download_json(
                'https://api.vmh.univision.com/metadata/v1/content/' + video_id,
                video_id)['videoMetadata']
            iptc = video_data['photoVideoMetadataIPTC']
            title = iptc['title']['en']
            fmg = video_data.get('photoVideoMetadata_fmg') or {}
            tvss_domain = fmg.get('tvssDomain') or 'https://auth.univision.com'
            data = self._download_json(
                tvss_domain + '/api/v3/video-auth/url-signature-tokens',
                video_id, query={'mcpids': video_id})['data'][0]
            formats = []

            rendition_url = data.get('renditionUrl')
            if rendition_url:
                formats = self._extract_m3u8_formats(
                    rendition_url, video_id, 'mp4',
                    'm3u8_native', m3u8_id='hls', fatal=False)

            fallback_rendition_url = data.get('fallbackRenditionUrl')
            if fallback_rendition_url:
                formats.append({
                    'format_id': 'fallback',
                    'tbr': int_or_none(self._search_regex(
                        r'_(\d+)\.mp4', fallback_rendition_url,
                        'bitrate', default=None)),
                    'url': fallback_rendition_url,
                })

            self._sort_formats(formats)

            return {
                'id': video_id,
                'title': title,
                'thumbnail': try_get(iptc, lambda x: x['cloudinaryLink']['link'], compat_str),
                'uploader': fmg.get('network'),
                'duration': int_or_none(iptc.get('fileDuration')),
                'formats': formats,
                'description': try_get(iptc, lambda x: x['description']['en'], compat_str),
                'timestamp': parse_iso8601(iptc.get('dateReleased')),
            }