diff options
Diffstat (limited to 'youtube_dl/downloader')
| -rw-r--r-- | youtube_dl/downloader/common.py | 32 | ||||
| -rw-r--r-- | youtube_dl/downloader/external.py | 43 | ||||
| -rw-r--r-- | youtube_dl/downloader/http.py | 12 |
3 files changed, 28 insertions, 59 deletions
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index c35c42c1d..7bb3a948d 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -25,21 +25,23 @@ class FileDownloader(object): Available options: - verbose: Print additional info to stdout. - quiet: Do not print messages to stdout. - ratelimit: Download speed limit, in bytes/sec. - retries: Number of times to retry for HTTP error 5xx - buffersize: Size of download buffer in bytes. - noresizebuffer: Do not automatically resize the download buffer. - continuedl: Try to continue downloads if possible. - noprogress: Do not print the progress bar. - logtostderr: Log messages to stderr instead of stdout. - consoletitle: Display progress in console window's titlebar. - nopart: Do not use temporary .part files. - updatetime: Use the Last-modified header to set output file timestamps. - test: Download only first bytes to test the downloader. - min_filesize: Skip files smaller than this size - max_filesize: Skip files larger than this size + verbose: Print additional info to stdout. + quiet: Do not print messages to stdout. + ratelimit: Download speed limit, in bytes/sec. + retries: Number of times to retry for HTTP error 5xx + buffersize: Size of download buffer in bytes. + noresizebuffer: Do not automatically resize the download buffer. + continuedl: Try to continue downloads if possible. + noprogress: Do not print the progress bar. + logtostderr: Log messages to stderr instead of stdout. + consoletitle: Display progress in console window's titlebar. + nopart: Do not use temporary .part files. + updatetime: Use the Last-modified header to set output file timestamps. + test: Download only first bytes to test the downloader. + min_filesize: Skip files smaller than this size + max_filesize: Skip files larger than this size + xattr_set_filesize: Set ytdl.filesize user xattribute with expected size. + (experimenatal) Subclasses of this one must re-define the real_download method. """ diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py index 7ebe40096..af9fdba75 100644 --- a/youtube_dl/downloader/external.py +++ b/youtube_dl/downloader/external.py @@ -7,7 +7,6 @@ import sys from .common import FileDownloader from ..utils import ( encodeFilename, - std_headers, ) @@ -46,42 +45,6 @@ class ExternalFD(FileDownloader): def supports(cls, info_dict): return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps') - def _calc_headers(self, info_dict): - res = std_headers.copy() - - ua = info_dict.get('user_agent') - if ua is not None: - res['User-Agent'] = ua - - cookies = self._calc_cookies(info_dict) - if cookies: - res['Cookie'] = cookies - - return res - - def _calc_cookies(self, info_dict): - class _PseudoRequest(object): - def __init__(self, url): - self.url = url - self.headers = {} - self.unverifiable = False - - def add_unredirected_header(self, k, v): - self.headers[k] = v - - def get_full_url(self): - return self.url - - def is_unverifiable(self): - return self.unverifiable - - def has_header(self, h): - return h in self.headers - - pr = _PseudoRequest(info_dict['url']) - self.ydl.cookiejar.add_cookie_header(pr) - return pr.headers.get('Cookie') - def _call_downloader(self, tmpfilename, info_dict): """ Either overwrite this or implement _make_cmd """ cmd = self._make_cmd(tmpfilename, info_dict) @@ -107,7 +70,7 @@ class ExternalFD(FileDownloader): class CurlFD(ExternalFD): def _make_cmd(self, tmpfilename, info_dict): cmd = [self.exe, '-o', tmpfilename] - for key, val in self._calc_headers(info_dict).items(): + for key, val in info_dict['http_headers'].items(): cmd += ['--header', '%s: %s' % (key, val)] cmd += ['--', info_dict['url']] return cmd @@ -116,7 +79,7 @@ class CurlFD(ExternalFD): class WgetFD(ExternalFD): def _make_cmd(self, tmpfilename, info_dict): cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies'] - for key, val in self._calc_headers(info_dict).items(): + for key, val in info_dict['http_headers'].items(): cmd += ['--header', '%s: %s' % (key, val)] cmd += ['--', info_dict['url']] return cmd @@ -131,7 +94,7 @@ class Aria2cFD(ExternalFD): if dn: cmd += ['--dir', dn] cmd += ['--out', os.path.basename(tmpfilename)] - for key, val in self._calc_headers(info_dict).items(): + for key, val in info_dict['http_headers'].items(): cmd += ['--header', '%s: %s' % (key, val)] cmd += ['--', info_dict['url']] return cmd diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py index e68f20c9f..8a1d578d5 100644 --- a/youtube_dl/downloader/http.py +++ b/youtube_dl/downloader/http.py @@ -24,10 +24,6 @@ class HttpFD(FileDownloader): # Do not include the Accept-Encoding header headers = {'Youtubedl-no-compression': 'True'} - if 'user_agent' in info_dict: - headers['Youtubedl-user-agent'] = info_dict['user_agent'] - if 'http_referer' in info_dict: - headers['Referer'] = info_dict['http_referer'] add_headers = info_dict.get('http_headers') if add_headers: headers.update(add_headers) @@ -161,6 +157,14 @@ class HttpFD(FileDownloader): except (OSError, IOError) as err: self.report_error('unable to open for writing: %s' % str(err)) return False + + if self.params.get('xattr_set_filesize', False) and data_len is not None: + try: + import xattr + xattr.setxattr(tmpfilename, 'user.ytdl.filesize', str(data_len)) + except(OSError, IOError, ImportError) as err: + self.report_error('unable to set filesize xattr: %s' % str(err)) + try: stream.write(data_block) except (IOError, OSError) as err: |