aboutsummaryrefslogtreecommitdiff
path: root/youtube_dl/downloader/external.py
blob: 5bf24ccbb75fde93514a294f8cb1afd3aff9e943 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
from __future__ import unicode_literals

import os.path
import subprocess
import sys

from .common import FileDownloader
from ..utils import (
    encodeFilename,
    std_headers,
)


class ExternalFD(FileDownloader):
    def real_download(self, filename, info_dict):
        self.report_destination(filename)
        tmpfilename = self.temp_name(filename)

        retval = self._call_downloader(tmpfilename, info_dict)
        if retval == 0:
            fsize = os.path.getsize(encodeFilename(tmpfilename))
            self.to_screen('\r[%s] Downloaded %s bytes' % (self.get_basename(), fsize))
            self.try_rename(tmpfilename, filename)
            self._hook_progress({
                'downloaded_bytes': fsize,
                'total_bytes': fsize,
                'filename': filename,
                'status': 'finished',
            })
            return True
        else:
            self.to_stderr('\n')
            self.report_error('%s exited with code %d' % (
                self.get_basename(), retval))
            return False

    @classmethod
    def get_basename(cls):
        return cls.__name__[:-2].lower()

    @property
    def exe(self):
        return self.params.get('external_downloader')

    @classmethod
    def supports(cls, info_dict):
        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps')

    def _calc_headers(self, info_dict):
        res = std_headers.copy()

        add_headers = info_dict.get('http_headers')
        if add_headers:
            res.update(add_headers)

        cookies = self._calc_cookies(info_dict)
        if cookies:
            res['Cookie'] = cookies

        return res

    def _calc_cookies(self, info_dict):
        class _PseudoRequest(object):
            def __init__(self, url):
                self.url = url
                self.headers = {}
                self.unverifiable = False

            def add_unredirected_header(self, k, v):
                self.headers[k] = v

            def get_full_url(self):
                return self.url

            def is_unverifiable(self):
                return self.unverifiable

            def has_header(self, h):
                return h in self.headers

        pr = _PseudoRequest(info_dict['url'])
        self.ydl.cookiejar.add_cookie_header(pr)
        return pr.headers.get('Cookie')

    def _call_downloader(self, tmpfilename, info_dict):
        """ Either overwrite this or implement _make_cmd """
        cmd = self._make_cmd(tmpfilename, info_dict)

        if sys.platform == 'win32' and sys.version_info < (3, 0):
            # Windows subprocess module does not actually support Unicode
            # on Python 2.x
            # See http://stackoverflow.com/a/9951851/35070
            subprocess_encoding = sys.getfilesystemencoding()
            cmd = [a.encode(subprocess_encoding, 'ignore') for a in cmd]
        else:
            subprocess_encoding = None
        self._debug_cmd(cmd, subprocess_encoding)

        p = subprocess.Popen(
            cmd, stderr=subprocess.PIPE)
        _, stderr = p.communicate()
        if p.returncode != 0:
            self.to_stderr(stderr)
        return p.returncode


class CurlFD(ExternalFD):
    def _make_cmd(self, tmpfilename, info_dict):
        cmd = [self.exe, '-o', tmpfilename]
        for key, val in self._calc_headers(info_dict).items():
            cmd += ['--header', '%s: %s' % (key, val)]
        cmd += ['--', info_dict['url']]
        return cmd


class WgetFD(ExternalFD):
    def _make_cmd(self, tmpfilename, info_dict):
        cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
        for key, val in self._calc_headers(info_dict).items():
            cmd += ['--header', '%s: %s' % (key, val)]
        cmd += ['--', info_dict['url']]
        return cmd


class Aria2cFD(ExternalFD):
    def _make_cmd(self, tmpfilename, info_dict):
        cmd = [
            self.exe, '-c',
            '--min-split-size', '1M', '--max-connection-per-server', '4']
        dn = os.path.dirname(tmpfilename)
        if dn:
            cmd += ['--dir', dn]
        cmd += ['--out', os.path.basename(tmpfilename)]
        for key, val in self._calc_headers(info_dict).items():
            cmd += ['--header', '%s: %s' % (key, val)]
        cmd += ['--', info_dict['url']]
        return cmd

_BY_NAME = dict(
    (klass.get_basename(), klass)
    for name, klass in globals().items()
    if name.endswith('FD') and name != 'ExternalFD'
)


def list_external_downloaders():
    return sorted(_BY_NAME.keys())


def get_external_downloader(external_downloader):
    """ Given the name of the executable, see whether we support the given
        downloader . """
    bn = os.path.basename(external_downloader)
    return _BY_NAME[bn]