diff options
Diffstat (limited to 'devscripts')
26 files changed, 779 insertions, 156 deletions
diff --git a/devscripts/__init__.py b/devscripts/__init__.py new file mode 100644 index 000000000..750dbdca7 --- /dev/null +++ b/devscripts/__init__.py @@ -0,0 +1 @@ +# Empty file needed to make devscripts.utils properly importable from outside diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index ce68f26f9..7db396a77 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -5,8 +5,12 @@ import os from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) + import youtube_dl +from youtube_dl.compat import compat_open as open + +from utils import read_file BASH_COMPLETION_FILE = "youtube-dl.bash-completion" BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in" @@ -18,12 +22,12 @@ def build_completion(opt_parser): for option in group.option_list: # for every long flag opts_flag.append(option.get_opt_string()) - with open(BASH_COMPLETION_TEMPLATE) as f: - template = f.read() - with open(BASH_COMPLETION_FILE, "w") as f: + template = read_file(BASH_COMPLETION_TEMPLATE) + with open(BASH_COMPLETION_FILE, "w", encoding='utf-8') as f: # just using the special char filled_template = template.replace("{{flags}}", " ".join(opts_flag)) f.write(filled_template) + parser = youtube_dl.parseOpts()[0] build_completion(parser) diff --git a/devscripts/buildserver.py b/devscripts/buildserver.py index 7c2f49f8b..4a4295ba9 100644 --- a/devscripts/buildserver.py +++ b/devscripts/buildserver.py @@ -1,17 +1,38 @@ #!/usr/bin/python3 -from http.server import HTTPServer, BaseHTTPRequestHandler -from socketserver import ThreadingMixIn import argparse import ctypes import functools +import shutil +import subprocess import sys +import tempfile import threading import traceback import os.path +sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__))))) +from youtube_dl.compat import ( + compat_input, + compat_http_server, + compat_str, + compat_urlparse, +) + +# These are not used outside of buildserver.py thus not in compat.py + +try: + import winreg as compat_winreg +except ImportError: # Python 2 + import _winreg as compat_winreg -class BuildHTTPServer(ThreadingMixIn, HTTPServer): +try: + import socketserver as compat_socketserver +except ImportError: # Python 2 + import SocketServer as compat_socketserver + + +class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer): allow_reuse_address = True @@ -191,7 +212,7 @@ def main(args=None): action='store_const', dest='action', const='service', help='Run as a Windows service') parser.add_argument('-b', '--bind', metavar='<host:port>', - action='store', default='localhost:8142', + action='store', default='0.0.0.0:8142', help='Bind to host:port (default %default)') options = parser.parse_args(args=args) @@ -216,7 +237,7 @@ def main(args=None): srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler) thr = threading.Thread(target=srv.serve_forever) thr.start() - input('Press ENTER to shut down') + compat_input('Press ENTER to shut down') srv.shutdown() thr.join() @@ -231,8 +252,6 @@ def rmtree(path): os.remove(fname) os.rmdir(path) -#============================================================================== - class BuildError(Exception): def __init__(self, output, code=500): @@ -249,15 +268,25 @@ class HTTPError(BuildError): class PythonBuilder(object): def __init__(self, **kwargs): - pythonVersion = kwargs.pop('python', '2.7') - try: - key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r'SOFTWARE\Python\PythonCore\%s\InstallPath' % pythonVersion) + python_version = kwargs.pop('python', '3.4') + python_path = None + for node in ('Wow6432Node\\', ''): try: - self.pythonPath, _ = _winreg.QueryValueEx(key, '') - finally: - _winreg.CloseKey(key) - except Exception: - raise BuildError('No such Python version: %s' % pythonVersion) + key = compat_winreg.OpenKey( + compat_winreg.HKEY_LOCAL_MACHINE, + r'SOFTWARE\%sPython\PythonCore\%s\InstallPath' % (node, python_version)) + try: + python_path, _ = compat_winreg.QueryValueEx(key, '') + finally: + compat_winreg.CloseKey(key) + break + except Exception: + pass + + if not python_path: + raise BuildError('No such Python version: %s' % python_version) + + self.pythonPath = python_path super(PythonBuilder, self).__init__(**kwargs) @@ -293,7 +322,7 @@ class GITBuilder(GITInfoBuilder): class YoutubeDLBuilder(object): - authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile'] + authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile', 'ytdl-org'] def __init__(self, **kwargs): if self.repoName != 'youtube-dl': @@ -305,8 +334,10 @@ class YoutubeDLBuilder(object): def build(self): try: - subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], - cwd=self.buildPath) + proc = subprocess.Popen([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], stdin=subprocess.PIPE, cwd=self.buildPath) + proc.wait() + #subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], + # cwd=self.buildPath) except subprocess.CalledProcessError as e: raise BuildError(e.output) @@ -369,12 +400,12 @@ class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, Clea pass -class BuildHTTPRequestHandler(BaseHTTPRequestHandler): +class BuildHTTPRequestHandler(compat_http_server.BaseHTTPRequestHandler): actionDict = {'build': Builder, 'download': Builder} # They're the same, no more caching. def do_GET(self): - path = urlparse.urlparse(self.path) - paramDict = dict([(key, value[0]) for key, value in urlparse.parse_qs(path.query).items()]) + path = compat_urlparse.urlparse(self.path) + paramDict = dict([(key, value[0]) for key, value in compat_urlparse.parse_qs(path.query).items()]) action, _, path = path.path.strip('/').partition('/') if path: path = path.split('/') @@ -388,19 +419,15 @@ class BuildHTTPRequestHandler(BaseHTTPRequestHandler): builder.close() except BuildError as e: self.send_response(e.code) - msg = unicode(e).encode('UTF-8') + msg = compat_str(e).encode('UTF-8') self.send_header('Content-Type', 'text/plain; charset=UTF-8') self.send_header('Content-Length', len(msg)) self.end_headers() self.wfile.write(msg) - except HTTPError as e: - self.send_response(e.code, str(e)) else: self.send_response(500, 'Unknown build method "%s"' % action) else: self.send_response(500, 'Malformed URL') -#============================================================================== - if __name__ == '__main__': main() diff --git a/devscripts/check-porn.py b/devscripts/check-porn.py index 7a219ebe9..740f04de0 100644 --- a/devscripts/check-porn.py +++ b/devscripts/check-porn.py @@ -14,7 +14,7 @@ import os import sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import get_testcases +from test.helper import gettestcases from youtube_dl.utils import compat_urllib_parse_urlparse from youtube_dl.utils import compat_urllib_request @@ -24,7 +24,7 @@ if len(sys.argv) > 1: else: METHOD = 'EURISTIC' -for test in get_testcases(): +for test in gettestcases(): if METHOD == 'EURISTIC': try: webpage = compat_urllib_request.urlopen(test['url'], timeout=10).read() @@ -45,12 +45,12 @@ for test in get_testcases(): RESULT = ('.' + domain + '\n' in LIST or '\n' + domain + '\n' in LIST) - if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict'] or - test['info_dict']['age_limit'] != 18): + if RESULT and ('info_dict' not in test or 'age_limit' not in test['info_dict'] + or test['info_dict']['age_limit'] != 18): print('\nPotential missing age_limit check: {0}'.format(test['name'])) - elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict'] and - test['info_dict']['age_limit'] == 18): + elif not RESULT and ('info_dict' in test and 'age_limit' in test['info_dict'] + and test['info_dict']['age_limit'] == 18): print('\nPotential false negative: {0}'.format(test['name'])) else: diff --git a/devscripts/cli_to_api.py b/devscripts/cli_to_api.py new file mode 100755 index 000000000..9fb1d2ba8 --- /dev/null +++ b/devscripts/cli_to_api.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# coding: utf-8 + +from __future__ import unicode_literals + +""" +This script displays the API parameters corresponding to a yt-dl command line + +Example: +$ ./cli_to_api.py -f best +{u'format': 'best'} +$ +""" + +# Allow direct execution +import os +import sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import youtube_dl +from types import MethodType + + +def cli_to_api(*opts): + YDL = youtube_dl.YoutubeDL + + # to extract the parsed options, break out of YoutubeDL instantiation + + # return options via this Exception + class ParseYTDLResult(Exception): + def __init__(self, result): + super(ParseYTDLResult, self).__init__('result') + self.opts = result + + # replacement constructor that raises ParseYTDLResult + def ytdl_init(ydl, ydl_opts): + super(YDL, ydl).__init__(ydl_opts) + raise ParseYTDLResult(ydl_opts) + + # patch in the constructor + YDL.__init__ = MethodType(ytdl_init, YDL) + + # core parser + def parsed_options(argv): + try: + youtube_dl._real_main(list(argv)) + except ParseYTDLResult as result: + return result.opts + + # from https://github.com/yt-dlp/yt-dlp/issues/5859#issuecomment-1363938900 + default = parsed_options([]) + + def neq_opt(a, b): + if a == b: + return False + if a is None and repr(type(object)).endswith(".utils.DateRange'>"): + return '0001-01-01 - 9999-12-31' != '{0}'.format(b) + return a != b + + diff = dict((k, v) for k, v in parsed_options(opts).items() if neq_opt(default[k], v)) + if 'postprocessors' in diff: + diff['postprocessors'] = [pp for pp in diff['postprocessors'] if pp not in default['postprocessors']] + return diff + + +def main(): + from pprint import PrettyPrinter + + pprint = PrettyPrinter() + super_format = pprint.format + + def format(object, context, maxlevels, level): + if repr(type(object)).endswith(".utils.DateRange'>"): + return '{0}: {1}>'.format(repr(object)[:-2], object), True, False + return super_format(object, context, maxlevels, level) + + pprint.format = format + + pprint.pprint(cli_to_api(*sys.argv)) + + +if __name__ == '__main__': + main() diff --git a/devscripts/create-github-release.py b/devscripts/create-github-release.py new file mode 100644 index 000000000..320bcfc27 --- /dev/null +++ b/devscripts/create-github-release.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +import json +import mimetypes +import netrc +import optparse +import os +import re +import sys + +dirn = os.path.dirname + +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) + +from youtube_dl.compat import ( + compat_basestring, + compat_getpass, + compat_print, + compat_urllib_request, +) +from youtube_dl.utils import ( + make_HTTPS_handler, + sanitized_Request, +) +from utils import read_file + + +class GitHubReleaser(object): + _API_URL = 'https://api.github.com/repos/ytdl-org/youtube-dl/releases' + _UPLOADS_URL = 'https://uploads.github.com/repos/ytdl-org/youtube-dl/releases/%s/assets?name=%s' + _NETRC_MACHINE = 'github.com' + + def __init__(self, debuglevel=0): + self._init_github_account() + https_handler = make_HTTPS_handler({}, debuglevel=debuglevel) + self._opener = compat_urllib_request.build_opener(https_handler) + + def _init_github_account(self): + try: + info = netrc.netrc().authenticators(self._NETRC_MACHINE) + if info is not None: + self._token = info[2] + compat_print('Using GitHub credentials found in .netrc...') + return + else: + compat_print('No GitHub credentials found in .netrc') + except (IOError, netrc.NetrcParseError): + compat_print('Unable to parse .netrc') + self._token = compat_getpass( + 'Type your GitHub PAT (personal access token) and press [Return]: ') + + def _call(self, req): + if isinstance(req, compat_basestring): + req = sanitized_Request(req) + req.add_header('Authorization', 'token %s' % self._token) + response = self._opener.open(req).read().decode('utf-8') + return json.loads(response) + + def list_releases(self): + return self._call(self._API_URL) + + def create_release(self, tag_name, name=None, body='', draft=False, prerelease=False): + data = { + 'tag_name': tag_name, + 'target_commitish': 'master', + 'name': name, + 'body': body, + 'draft': draft, + 'prerelease': prerelease, + } + req = sanitized_Request(self._API_URL, json.dumps(data).encode('utf-8')) + return self._call(req) + + def create_asset(self, release_id, asset): + asset_name = os.path.basename(asset) + url = self._UPLOADS_URL % (release_id, asset_name) + # Our files are small enough to be loaded directly into memory. + data = open(asset, 'rb').read() + req = sanitized_Request(url, data) + mime_type, _ = mimetypes.guess_type(asset_name) + req.add_header('Content-Type', mime_type or 'application/octet-stream') + return self._call(req) + + +def main(): + parser = optparse.OptionParser(usage='%prog CHANGELOG VERSION BUILDPATH') + options, args = parser.parse_args() + if len(args) != 3: + parser.error('Expected a version and a build directory') + + changelog_file, version, build_path = args + + changelog = read_file(changelog_file) + + mobj = re.search(r'(?s)version %s\n{2}(.+?)\n{3}' % version, changelog) + body = mobj.group(1) if mobj else '' + + releaser = GitHubReleaser() + + new_release = releaser.create_release( + version, name='youtube-dl %s' % version, body=body) + release_id = new_release['id'] + + for asset in os.listdir(build_path): + compat_print('Uploading %s...' % asset) + releaser.create_asset(release_id, os.path.join(build_path, asset)) + + +if __name__ == '__main__': + main() diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index 41629d87d..ef8a39e0b 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -6,10 +6,13 @@ import os from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) + import youtube_dl from youtube_dl.utils import shell_quote +from utils import read_file, write_file + FISH_COMPLETION_FILE = 'youtube-dl.fish' FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in' @@ -38,11 +41,10 @@ def build_completion(opt_parser): complete_cmd.extend(EXTRA_ARGS.get(long_option, [])) commands.append(shell_quote(complete_cmd)) - with open(FISH_COMPLETION_TEMPLATE) as f: - template = f.read() + template = read_file(FISH_COMPLETION_TEMPLATE) filled_template = template.replace('{{commands}}', '\n'.join(commands)) - with open(FISH_COMPLETION_FILE, 'w') as f: - f.write(filled_template) + write_file(FISH_COMPLETION_FILE, filled_template) + parser = youtube_dl.parseOpts()[0] build_completion(parser) diff --git a/devscripts/generate_aes_testdata.py b/devscripts/generate_aes_testdata.py index 2e389fc8e..e3df42cc2 100644 --- a/devscripts/generate_aes_testdata.py +++ b/devscripts/generate_aes_testdata.py @@ -23,6 +23,7 @@ def openssl_encode(algo, key, iv): out, _ = prog.communicate(secret_msg) return out + iv = key = [0x20, 0x15] + 14 * [0] r = openssl_encode('aes-128-cbc', key, iv) diff --git a/devscripts/gh-pages/add-version.py b/devscripts/gh-pages/add-version.py index 867ea0048..b84908f85 100755 --- a/devscripts/gh-pages/add-version.py +++ b/devscripts/gh-pages/add-version.py @@ -6,16 +6,21 @@ import sys import hashlib import os.path +dirn = os.path.dirname + +sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__))))) + +from devscripts.utils import read_file, write_file +from youtube_dl.compat import compat_open as open if len(sys.argv) <= 1: print('Specify the version number as parameter') sys.exit() version = sys.argv[1] -with open('update/LATEST_VERSION', 'w') as f: - f.write(version) +write_file('update/LATEST_VERSION', version) -versions_info = json.load(open('update/versions.json')) +versions_info = json.loads(read_file('update/versions.json')) if 'signature' in versions_info: del versions_info['signature'] @@ -39,5 +44,5 @@ for key, filename in filenames.items(): versions_info['versions'][version] = new_version versions_info['latest'] = version -with open('update/versions.json', 'w') as jsonf: - json.dump(versions_info, jsonf, indent=4, sort_keys=True) +with open('update/versions.json', 'w', encoding='utf-8') as jsonf: + json.dumps(versions_info, jsonf, indent=4, sort_keys=True) diff --git a/devscripts/gh-pages/generate-download.py b/devscripts/gh-pages/generate-download.py index 392e3ba21..3e38e9299 100755 --- a/devscripts/gh-pages/generate-download.py +++ b/devscripts/gh-pages/generate-download.py @@ -1,31 +1,29 @@ #!/usr/bin/env python3 from __future__ import unicode_literals -import hashlib -import urllib.request import json +import os.path +import sys -versions_info = json.load(open('update/versions.json')) -version = versions_info['latest'] -URL = versions_info['versions'][version]['bin'][0] +dirn = os.path.dirname + +sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) -data = urllib.request.urlopen(URL).read() +from utils import read_file, write_file + +versions_info = json.loads(read_file('update/versions.json')) +version = versions_info['latest'] +version_dict = versions_info['versions'][version] # Read template page -with open('download.html.in', 'r', encoding='utf-8') as tmplf: - template = tmplf.read() +template = read_file('download.html.in') -md5sum = hashlib.md5(data).hexdigest() -sha1sum = hashlib.sha1(data).hexdigest() -sha256sum = hashlib.sha256(data).hexdigest() template = template.replace('@PROGRAM_VERSION@', version) -template = template.replace('@PROGRAM_URL@', URL) -template = template.replace('@PROGRAM_MD5SUM@', md5sum) -template = template.replace('@PROGRAM_SHA1SUM@', sha1sum) -template = template.replace('@PROGRAM_SHA256SUM@', sha256sum) -template = template.replace('@EXE_URL@', versions_info['versions'][version]['exe'][0]) -template = template.replace('@EXE_SHA256SUM@', versions_info['versions'][version]['exe'][1]) -template = template.replace('@TAR_URL@', versions_info['versions'][version]['tar'][0]) -template = template.replace('@TAR_SHA256SUM@', versions_info['versions'][version]['tar'][1]) -with open('download.html', 'w', encoding='utf-8') as dlf: - dlf.write(template) +template = template.replace('@PROGRAM_URL@', version_dict['bin'][0]) +template = template.replace('@PROGRAM_SHA256SUM@', version_dict['bin'][1]) +template = template.replace('@EXE_URL@', version_dict['exe'][0]) +template = template.replace('@EXE_SHA256SUM@', version_dict['exe'][1]) +template = template.replace('@TAR_URL@', version_dict['tar'][0]) +template = template.replace('@TAR_SHA256SUM@', version_dict['tar'][1]) + +write_file('download.html', template) diff --git a/devscripts/gh-pages/update-copyright.py b/devscripts/gh-pages/update-copyright.py index 3663c8afe..444595c48 100755 --- a/devscripts/gh-pages/update-copyright.py +++ b/devscripts/gh-pages/update-copyright.py @@ -5,17 +5,22 @@ from __future__ import with_statement, unicode_literals import datetime import glob -import io # For Python 2 compatibilty import os import re +import sys -year = str(datetime.datetime.now().year) +dirn = os.path.dirname + +sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__))))) + +from devscripts.utils import read_file, write_file +from youtube_dl import compat_str + +year = compat_str(datetime.datetime.now().year) for fn in glob.glob('*.html*'): - with io.open(fn, encoding='utf-8') as f: - content = f.read() - newc = re.sub(r'(?P<copyright>Copyright © 2006-)(?P<year>[0-9]{4})', 'Copyright © 2006-' + year, content) + content = read_file(fn) + newc = re.sub(r'(?P<copyright>Copyright © 2011-)(?P<year>[0-9]{4})', 'Copyright © 2011-' + year, content) if content != newc: tmpFn = fn + '.part' - with io.open(tmpFn, 'wt', encoding='utf-8') as outf: - outf.write(newc) + write_file(tmpFn, newc) os.rename(tmpFn, fn) diff --git a/devscripts/gh-pages/update-feed.py b/devscripts/gh-pages/update-feed.py index e93eb60fb..13a367d34 100755 --- a/devscripts/gh-pages/update-feed.py +++ b/devscripts/gh-pages/update-feed.py @@ -2,15 +2,21 @@ from __future__ import unicode_literals import datetime -import io import json +import os.path import textwrap +import sys +dirn = os.path.dirname + +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) + +from utils import write_file atom_template = textwrap.dedent("""\ <?xml version="1.0" encoding="utf-8"?> <feed xmlns="http://www.w3.org/2005/Atom"> - <link rel="self" href="http://rg3.github.io/youtube-dl/update/releases.atom" /> + <link rel="self" href="http://ytdl-org.github.io/youtube-dl/update/releases.atom" /> <title>youtube-dl releases</title> <id>https://yt-dl.org/feed/youtube-dl-updates-feed</id> <updated>@TIMESTAMP@</updated> @@ -21,7 +27,7 @@ entry_template = textwrap.dedent(""" <entry> <id>https://yt-dl.org/feed/youtube-dl-updates-feed/youtube-dl-@VERSION@</id> <title>New version @VERSION@</title> - <link href="http://rg3.github.io/youtube-dl" /> + <link href="http://ytdl-org.github.io/youtube-dl" /> <content type="xhtml"> <div xmlns="http://www.w3.org/1999/xhtml"> Downloads available at <a href="https://yt-dl.org/downloads/@VERSION@/">https://yt-dl.org/downloads/@VERSION@/</a> @@ -72,5 +78,4 @@ for v in versions: entries_str = textwrap.indent(''.join(entries), '\t') atom_template = atom_template.replace('@ENTRIES@', entries_str) -with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file: - atom_file.write(atom_template) +write_file('update/releases.atom', atom_template) diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py index 503c1372f..06a8a474c 100755 --- a/devscripts/gh-pages/update-sites.py +++ b/devscripts/gh-pages/update-sites.py @@ -5,15 +5,17 @@ import sys import os import textwrap +dirn = os.path.dirname + # We must be able to import youtube_dl -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__))))) import youtube_dl +from devscripts.utils import read_file, write_file def main(): - with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf: - template = tmplf.read() + template = read_file('supportedsites.html.in') ie_htmls = [] for ie in youtube_dl.list_extractors(age_limit=None): @@ -29,8 +31,8 @@ def main(): template = template.replace('@SITES@', textwrap.indent('\n'.join(ie_htmls), '\t')) - with open('supportedsites.html', 'w', encoding='utf-8') as sitesf: - sitesf.write(template) + write_file('supportedsites.html', template) + if __name__ == '__main__': main() diff --git a/devscripts/lazy_load_template.py b/devscripts/lazy_load_template.py new file mode 100644 index 000000000..c4e5fc1f4 --- /dev/null +++ b/devscripts/lazy_load_template.py @@ -0,0 +1,19 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re + + +class LazyLoadExtractor(object): + _module = None + + @classmethod + def ie_key(cls): + return cls.__name__[:-2] + + def __new__(cls, *args, **kwargs): + mod = __import__(cls._module, fromlist=(cls.__name__,)) + real_cls = getattr(mod, cls.__name__) + instance = real_cls.__new__(real_cls) + instance.__init__(*args, **kwargs) + return instance diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py index 5e454a429..5a9eb194f 100755 --- a/devscripts/make_contributing.py +++ b/devscripts/make_contributing.py @@ -1,10 +1,11 @@ #!/usr/bin/env python from __future__ import unicode_literals -import io import optparse import re +from utils import read_file, write_file + def main(): parser = optparse.OptionParser(usage='%prog INFILE OUTFILE') @@ -14,8 +15,7 @@ def main(): infile, outfile = args - with io.open(infile, encoding='utf-8') as inf: - readme = inf.read() + readme = read_file(infile) bug_text = re.search( r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1) @@ -25,8 +25,8 @@ def main(): out = bug_text + dev_text - with io.open(outfile, 'w', encoding='utf-8') as outf: - outf.write(out) + write_file(outfile, out) + if __name__ == '__main__': main() diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py new file mode 100644 index 000000000..65fa8169f --- /dev/null +++ b/devscripts/make_issue_template.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +import optparse +import os.path +import sys + +from utils import read_file, read_version, write_file + + +def main(): + parser = optparse.OptionParser(usage='%prog INFILE OUTFILE') + options, args = parser.parse_args() + if len(args) != 2: + parser.error('Expected an input and an output filename') + + infile, outfile = args + + issue_template_tmpl = read_file(infile) + + out = issue_template_tmpl % {'version': read_version()} + + write_file(outfile, out) + +if __name__ == '__main__': + main() diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py new file mode 100644 index 000000000..5b8b123a4 --- /dev/null +++ b/devscripts/make_lazy_extractors.py @@ -0,0 +1,131 @@ +from __future__ import unicode_literals, print_function + +from inspect import getsource +import os +from os.path import dirname as dirn +import re +import sys + +print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr) + +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) + +lazy_extractors_filename = sys.argv[1] +if os.path.exists(lazy_extractors_filename): + os.remove(lazy_extractors_filename) +# Py2: may be confused by leftover lazy_extractors.pyc +if sys.version_info[0] < 3: + for c in ('c', 'o'): + try: + os.remove(lazy_extractors_filename + 'c') + except OSError: + pass + +from devscripts.utils import read_file, write_file +from youtube_dl.compat import compat_register_utf8 + +compat_register_utf8() + +from youtube_dl.extractor import _ALL_CLASSES +from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor + +module_template = read_file('devscripts/lazy_load_template.py') + + +def get_source(m): + return re.sub(r'(?m)^\s*#.*\n', '', getsource(m)) + + +module_contents = [ + module_template, + get_source(InfoExtractor.suitable), + get_source(InfoExtractor._match_valid_url) + '\n', + 'class LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n', + # needed for suitable() methods of Youtube extractor (see #28780) + 'from youtube_dl.utils import parse_qs, variadic\n', +] + +ie_template = ''' +class {name}({bases}): + _VALID_URL = {valid_url!r} + _module = '{module}' +''' + +make_valid_template = ''' + @classmethod + def _make_valid_url(cls): + return {valid_url!r} +''' + + +def get_base_name(base): + if base is InfoExtractor: + return 'LazyLoadExtractor' + elif base is SearchInfoExtractor: + return 'LazyLoadSearchExtractor' + else: + return base.__name__ + + +def build_lazy_ie(ie, name): + valid_url = getattr(ie, '_VALID_URL', None) + s = ie_template.format( + name=name, + bases=', '.join(map(get_base_name, ie.__bases__)), + valid_url=valid_url, + module=ie.__module__) + if ie.suitable.__func__ is not InfoExtractor.suitable.__func__: + s += '\n' + get_source(ie.suitable) + if hasattr(ie, '_make_valid_url'): + # search extractors + s += make_valid_template.format(valid_url=ie._make_valid_url()) + return s + + +# find the correct sorting and add the required base classes so that subclasses +# can be correctly created +classes = _ALL_CLASSES[:-1] +ordered_cls = [] +while classes: + for c in classes[:]: + bases = set(c.__bases__) - set((object, InfoExtractor, SearchInfoExtractor)) + stop = False + for b in bases: + if b not in classes and b not in ordered_cls: + if b.__name__ == 'GenericIE': + exit() + classes.insert(0, b) + stop = True + if stop: + break + if all(b in ordered_cls for b in bases): + ordered_cls.append(c) + classes.remove(c) + break +ordered_cls.append(_ALL_CLASSES[-1]) + +names = [] +for ie in ordered_cls: + name = ie.__name__ + src = build_lazy_ie(ie, name) + module_contents.append(src) + if ie in _ALL_CLASSES: + names.append(name) + +module_contents.append( + '_ALL_CLASSES = [{0}]'.format(', '.join(names))) + +module_src = '\n'.join(module_contents) + +write_file(lazy_extractors_filename, module_src + '\n') + +# work around JVM byte code module limit in Jython +if sys.platform.startswith('java') and sys.version_info[:2] == (2, 7): + import subprocess + from youtube_dl.compat import compat_subprocess_get_DEVNULL + # if Python 2.7 is available, use it to compile the module for Jython + try: + # if Python 2.7 is available, use it to compile the module for Jython + subprocess.check_call(['python2.7', '-m', 'py_compile', lazy_extractors_filename], stdout=compat_subprocess_get_DEVNULL()) + except Exception: + pass diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 8fbce0796..7a5b04dcc 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -1,8 +1,14 @@ from __future__ import unicode_literals -import io -import sys +import os.path import re +import sys +dirn = os.path.dirname + +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) + +from utils import read_file +from youtube_dl.compat import compat_open as open README_FILE = 'README.md' helptext = sys.stdin.read() @@ -10,8 +16,7 @@ helptext = sys.stdin.read() if isinstance(helptext, bytes): helptext = helptext.decode('utf-8') -with io.open(README_FILE, encoding='utf-8') as f: - oldreadme = f.read() +oldreadme = read_file(README_FILE) header = oldreadme[:oldreadme.index('# OPTIONS')] footer = oldreadme[oldreadme.index('# CONFIGURATION'):] @@ -20,7 +25,7 @@ options = helptext[helptext.index(' General Options:') + 19:] options = re.sub(r'(?m)^ (\w.+)$', r'## \1', options) options = '# OPTIONS\n' + options + '\n' -with io.open(README_FILE, 'w', encoding='utf-8') as f: +with open(README_FILE, 'w', encoding='utf-8') as f: f.write(header) f.write(options) f.write(footer) diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index 8cb4a4638..c424d18d7 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -1,17 +1,19 @@ #!/usr/bin/env python from __future__ import unicode_literals -import io import optparse -import os +import os.path import sys - # Import youtube_dl -ROOT_DIR = os.path.join(os.path.dirname(__file__), '..') -sys.path.insert(0, ROOT_DIR) +dirn = os.path.dirname + +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) + import youtube_dl +from utils import write_file + def main(): parser = optparse.OptionParser(usage='%prog OUTFILE.md') @@ -38,8 +40,8 @@ def main(): ' - ' + md + '\n' for md in gen_ies_md(ies)) - with io.open(outfile, 'w', encoding='utf-8') as outf: - outf.write(out) + write_file(outfile, out) + if __name__ == '__main__': main() diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py index 776e6556e..0090ada3e 100644 --- a/devscripts/prepare_manpage.py +++ b/devscripts/prepare_manpage.py @@ -1,12 +1,43 @@ from __future__ import unicode_literals -import io +import optparse import os.path -import sys import re +from utils import read_file, write_file + ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) README_FILE = os.path.join(ROOT_DIR, 'README.md') +PREFIX = r'''%YOUTUBE-DL(1) + +# NAME + +youtube\-dl \- download videos from youtube.com or other video platforms + +# SYNOPSIS + +**youtube-dl** \[OPTIONS\] URL [URL...] + +''' + + +def main(): + parser = optparse.OptionParser(usage='%prog OUTFILE.md') + options, args = parser.parse_args() + if len(args) != 1: + parser.error('Expected an output filename') + + outfile, = args + + readme = read_file(README_FILE) + + readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme) + readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme) + readme = PREFIX + readme + + readme = filter_options(readme) + + write_file(outfile, readme) def filter_options(readme): @@ -21,43 +52,26 @@ def filter_options(readme): if in_options: if line.lstrip().startswith('-'): - option, description = re.split(r'\s{2,}', line.lstrip()) - split_option = option.split(' ') - - if not split_option[-1].startswith('-'): # metavar - option = ' '.join(split_option[:-1] + ['*%s*' % split_option[-1]]) - - # Pandoc's definition_lists. See http://pandoc.org/README.html - # for more information. - ret += '\n%s\n: %s\n' % (option, description) - else: - ret += line.lstrip() + '\n' + split = re.split(r'\s{2,}', line.lstrip()) + # Description string may start with `-` as well. If there is + # only one piece then it's a description bit not an option. + if len(split) > 1: + option, description = split + split_option = option.split(' ') + + if not split_option[-1].startswith('-'): # metavar + option = ' '.join(split_option[:-1] + ['*%s*' % split_option[-1]]) + + # Pandoc's definition_lists. See http://pandoc.org/README.html + # for more information. + ret += '\n%s\n: %s\n' % (option, description) + continue + ret += line.lstrip() + '\n' else: ret += line + '\n' return ret -with io.open(README_FILE, encoding='utf-8') as f: - readme = f.read() - -PREFIX = '''%YOUTUBE-DL(1) - -# NAME - -youtube\-dl \- download videos from youtube.com or other video platforms - -# SYNOPSIS - -**youtube-dl** \[OPTIONS\] URL [URL...] - -''' -readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme) -readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme) -readme = PREFIX + readme - -readme = filter_options(readme) -if sys.version_info < (3, 0): - print(readme.encode('utf-8')) -else: - print(readme) +if __name__ == '__main__': + main() diff --git a/devscripts/release.sh b/devscripts/release.sh index 61806961c..f2411c927 100755 --- a/devscripts/release.sh +++ b/devscripts/release.sh @@ -6,7 +6,7 @@ # * the git config user.signingkey is properly set # You will need -# pip install coverage nose rsa +# pip install coverage nose rsa wheel # TODO # release notes @@ -15,10 +15,33 @@ set -e skip_tests=true -if [ "$1" = '--run-tests' ]; then - skip_tests=false - shift -fi +gpg_sign_commits="" +buildserver='localhost:8142' + +while true +do +case "$1" in + --run-tests) + skip_tests=false + shift + ;; + --gpg-sign-commits|-S) + gpg_sign_commits="-S" + shift + ;; + --buildserver) + buildserver="$2" + shift 2 + ;; + --*) + echo "ERROR: unknown option $1" + exit 1 + ;; + *) + break + ;; +esac +done if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi version="$1" @@ -33,6 +56,12 @@ if [ ! -z "`git status --porcelain | grep -v CHANGELOG`" ]; then echo 'ERROR: th useless_files=$(find youtube_dl -type f -not -name '*.py') if [ ! -z "$useless_files" ]; then echo "ERROR: Non-.py files in youtube_dl: $useless_files"; exit 1; fi if [ ! -f "updates_key.pem" ]; then echo 'ERROR: updates_key.pem missing'; exit 1; fi +if ! type pandoc >/dev/null 2>/dev/null; then echo 'ERROR: pandoc is missing'; exit 1; fi +if ! python3 -c 'import rsa' 2>/dev/null; then echo 'ERROR: python3-rsa is missing'; exit 1; fi +if ! python3 -c 'import wheel' 2>/dev/null; then echo 'ERROR: wheel is missing'; exit 1; fi + +read -p "Is ChangeLog up to date? (y/n) " -n 1 +if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi /bin/echo -e "\n### First of all, testing..." make clean @@ -45,10 +74,13 @@ fi /bin/echo -e "\n### Changing version in version.py..." sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/version.py -/bin/echo -e "\n### Committing documentation and youtube_dl/version.py..." -make README.md CONTRIBUTING.md supportedsites -git add README.md CONTRIBUTING.md docs/supportedsites.md youtube_dl/version.py -git commit -m "release $version" +/bin/echo -e "\n### Changing version in ChangeLog..." +sed -i "s/<unreleased>/$version/" ChangeLog + +/bin/echo -e "\n### Committing documentation, templates and youtube_dl/version.py..." +make README.md CONTRIBUTING.md issuetemplates supportedsites +git add README.md CONTRIBUTING.md .github/ISSUE_TEMPLATE/1_broken_site.md .github/ISSUE_TEMPLATE/2_site_support_request.md .github/ISSUE_TEMPLATE/3_site_feature_request.md .github/ISSUE_TEMPLATE/4_bug_report.md .github/ISSUE_TEMPLATE/5_feature_request.md .github/ISSUE_TEMPLATE/6_question.md docs/supportedsites.md youtube_dl/version.py ChangeLog +git commit $gpg_sign_commits -m "release $version" /bin/echo -e "\n### Now tagging, signing and pushing..." git tag -s -m "Release $version" "$version" @@ -64,7 +96,7 @@ git push origin "$version" REV=$(git rev-parse HEAD) make youtube-dl youtube-dl.tar.gz read -p "VM running? (y/n) " -n 1 -wget "http://localhost:8142/build/rg3/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe +wget "http://$buildserver/build/ytdl-org/youtube-dl/youtube-dl.exe?rev=$REV" -O youtube-dl.exe mkdir -p "build/$version" mv youtube-dl youtube-dl.exe "build/$version" mv youtube-dl.tar.gz "build/$version/youtube-dl-$version.tar.gz" @@ -74,15 +106,16 @@ RELEASE_FILES="youtube-dl youtube-dl.exe youtube-dl-$version.tar.gz" (cd build/$version/ && sha256sum $RELEASE_FILES > SHA2-256SUMS) (cd build/$version/ && sha512sum $RELEASE_FILES > SHA2-512SUMS) -/bin/echo -e "\n### Signing and uploading the new binaries to yt-dl.org ..." +/bin/echo -e "\n### Signing and uploading the new binaries to GitHub..." for f in $RELEASE_FILES; do gpg --passphrase-repeat 5 --detach-sig "build/$version/$f"; done -scp -r "build/$version" ytdl@yt-dl.org:html/tmp/ -ssh ytdl@yt-dl.org "mv html/tmp/$version html/downloads/" + +ROOT=$(pwd) +python devscripts/create-github-release.py ChangeLog $version "$ROOT/build/$version" + ssh ytdl@yt-dl.org "sh html/update_latest.sh $version" /bin/echo -e "\n### Now switching to gh-pages..." git clone --branch gh-pages --single-branch . build/gh-pages -ROOT=$(pwd) ( set -e ORIGIN_URL=$(git config --get remote.origin.url) @@ -94,7 +127,7 @@ ROOT=$(pwd) "$ROOT/devscripts/gh-pages/update-copyright.py" "$ROOT/devscripts/gh-pages/update-sites.py" git add *.html *.html.in update - git commit -m "release $version" + git commit $gpg_sign_commits -m "release $version" git push "$ROOT" gh-pages git push "$ORIGIN_URL" gh-pages ) diff --git a/devscripts/run_tests.bat b/devscripts/run_tests.bat new file mode 100644 index 000000000..79359b5a7 --- /dev/null +++ b/devscripts/run_tests.bat @@ -0,0 +1,17 @@ +@echo off + +rem Keep this list in sync with the `offlinetest` target in Makefile +set DOWNLOAD_TESTS="age_restriction^|download^|iqiyi_sdk_interpreter^|socks^|subtitles^|write_annotations^|youtube_lists^|youtube_signature" + +if "%YTDL_TEST_SET%" == "core" ( + set test_set="-I test_("%DOWNLOAD_TESTS%")\.py" + set multiprocess_args="" +) else if "%YTDL_TEST_SET%" == "download" ( + set test_set="-I test_(?!"%DOWNLOAD_TESTS%").+\.py" + set multiprocess_args="--processes=4 --process-timeout=540" +) else ( + echo YTDL_TEST_SET is not set or invalid + exit /b 1 +) + +nosetests test --verbose %test_set:"=% %multiprocess_args:"=% diff --git a/devscripts/run_tests.sh b/devscripts/run_tests.sh new file mode 100755 index 000000000..dd37a80f5 --- /dev/null +++ b/devscripts/run_tests.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Keep this list in sync with the `offlinetest` target in Makefile +DOWNLOAD_TESTS="age_restriction|download|iqiyi_sdk_interpreter|socks|subtitles|write_annotations|youtube_lists|youtube_signature" + +test_set="" +multiprocess_args="" + +case "$YTDL_TEST_SET" in + core) + test_set="-I test_($DOWNLOAD_TESTS)\.py" + ;; + download) + test_set="-I test_(?!$DOWNLOAD_TESTS).+\.py" + multiprocess_args="--processes=4 --process-timeout=540" + ;; + *) + break + ;; +esac + +nosetests test --verbose $test_set $multiprocess_args diff --git a/devscripts/show-downloads-statistics.py b/devscripts/show-downloads-statistics.py new file mode 100644 index 000000000..6c8d1cc2d --- /dev/null +++ b/devscripts/show-downloads-statistics.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +import itertools +import json +import os +import re +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from youtube_dl.compat import ( + compat_print, + compat_urllib_request, +) +from youtube_dl.utils import format_bytes + + +def format_size(bytes): + return '%s (%d bytes)' % (format_bytes(bytes), bytes) + + +total_bytes = 0 + +for page in itertools.count(1): + releases = json.loads(compat_urllib_request.urlopen( + 'https://api.github.com/repos/ytdl-org/youtube-dl/releases?page=%s' % page + ).read().decode('utf-8')) + + if not releases: + break + + for release in releases: + compat_print(release['name']) + for asset in release['assets']: + asset_name = asset['name'] + total_bytes += asset['download_count'] * asset['size'] + if all(not re.match(p, asset_name) for p in ( + r'^youtube-dl$', + r'^youtube-dl-\d{4}\.\d{2}\.\d{2}(?:\.\d+)?\.tar\.gz$', + r'^youtube-dl\.exe$')): + continue + compat_print( + ' %s size: %s downloads: %d' + % (asset_name, format_size(asset['size']), asset['download_count'])) + +compat_print('total downloads traffic: %s' % format_size(total_bytes)) diff --git a/devscripts/utils.py b/devscripts/utils.py new file mode 100644 index 000000000..2d072d2e0 --- /dev/null +++ b/devscripts/utils.py @@ -0,0 +1,62 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import argparse +import functools +import os.path +import subprocess +import sys + +dirn = os.path.dirname + +sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) + +from youtube_dl.compat import ( + compat_kwargs, + compat_open as open, +) + + +def read_file(fname): + with open(fname, encoding='utf-8') as f: + return f.read() + + +def write_file(fname, content, mode='w'): + with open(fname, mode, encoding='utf-8') as f: + return f.write(content) + + +def read_version(fname='youtube_dl/version.py'): + """Get the version without importing the package""" + exec(compile(read_file(fname), fname, 'exec')) + return locals()['__version__'] + + +def get_filename_args(has_infile=False, default_outfile=None): + parser = argparse.ArgumentParser() + if has_infile: + parser.add_argument('infile', help='Input file') + kwargs = {'nargs': '?', 'default': default_outfile} if default_outfile else {} + kwargs['help'] = 'Output file' + parser.add_argument('outfile', **compat_kwargs(kwargs)) + + opts = parser.parse_args() + if has_infile: + return opts.infile, opts.outfile + return opts.outfile + + +def compose_functions(*functions): + return lambda x: functools.reduce(lambda y, f: f(y), functions, x) + + +def run_process(*args, **kwargs): + kwargs.setdefault('text', True) + kwargs.setdefault('check', True) + kwargs.setdefault('capture_output', True) + if kwargs['text']: + kwargs.setdefault('encoding', 'utf-8') + kwargs.setdefault('errors', 'replace') + kwargs = compat_kwargs(kwargs) + return subprocess.run(args, **kwargs) diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index 04728e8e2..ebd552fcb 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -7,6 +7,8 @@ import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) import youtube_dl +from utils import read_file, write_file + ZSH_COMPLETION_FILE = "youtube-dl.zsh" ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in" @@ -34,15 +36,14 @@ def build_completion(opt_parser): flags = [opt.get_opt_string() for opt in opts] - with open(ZSH_COMPLETION_TEMPLATE) as f: - template = f.read() + template = read_file(ZSH_COMPLETION_TEMPLATE) template = template.replace("{{fileopts}}", "|".join(fileopts)) template = template.replace("{{diropts}}", "|".join(diropts)) template = template.replace("{{flags}}", " ".join(flags)) - with open(ZSH_COMPLETION_FILE, "w") as f: - f.write(template) + write_file(ZSH_COMPLETION_FILE, template) + parser = youtube_dl.parseOpts()[0] build_completion(parser) |