From 779822d945dc7ebba7062ac9a5e760d21a7f362a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Wed, 10 Feb 2016 14:01:31 +0100 Subject: Add experimental support for lazy loading the info extractors 'make lazy-extractors' creates the youtube_dl/extractor/lazy_extractors.py (imported by youtube_dl/extractor/__init__.py), which contains simplified classes that only have the 'suitable' class method and that load the appropiate class with the '__new__' method when a instance is created. --- devscripts/make_lazy_extractors.py | 63 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 devscripts/make_lazy_extractors.py (limited to 'devscripts/make_lazy_extractors.py') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py new file mode 100644 index 000000000..8627d0b1c --- /dev/null +++ b/devscripts/make_lazy_extractors.py @@ -0,0 +1,63 @@ +from __future__ import unicode_literals, print_function + +from inspect import getsource +import os +from os.path import dirname as dirn +import sys + +print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr) + +sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) + +lazy_extractors_filename = sys.argv[1] +if os.path.exists(lazy_extractors_filename): + os.remove(lazy_extractors_filename) + +from youtube_dl.extractor import _ALL_CLASSES +from youtube_dl.extractor.common import InfoExtractor + +with open('devscripts/lazy_load_template.py', 'rt') as f: + module_template = f.read() + +module_contents = [module_template + '\n' + getsource(InfoExtractor.suitable)] + +ie_template = ''' +class {name}(LazyLoadExtractor): + _VALID_URL = {valid_url!r} + _module = '{module}' +''' + +make_valid_template = ''' + @classmethod + def _make_valid_url(cls): + return {!r} +''' + + +def build_lazy_ie(ie, name): + valid_url = getattr(ie, '_VALID_URL', None) + s = ie_template.format( + name=name, + valid_url=valid_url, + module=ie.__module__) + if ie.suitable.__func__ is not InfoExtractor.suitable.__func__: + s += getsource(ie.suitable) + if hasattr(ie, '_make_valid_url'): + # search extractors + s += make_valid_template.format(ie._make_valid_url()) + return s + +names = [] +for ie in _ALL_CLASSES: + name = ie.ie_key() + 'IE' + src = build_lazy_ie(ie, name) + module_contents.append(src) + names.append(name) + +module_contents.append( + '_ALL_CLASSES = [{}]'.format(', '.join(names))) + +module_src = '\n'.join(module_contents) + +with open(lazy_extractors_filename, 'wt') as f: + f.write(module_src) -- cgit v1.2.3 From c1ce6acdd73da7744f4bbe27698e96275467e14d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sun, 21 Feb 2016 11:53:48 +0100 Subject: lazy extractors: Fix building with python2.6 --- devscripts/make_lazy_extractors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'devscripts/make_lazy_extractors.py') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 8627d0b1c..5d0ddb401 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -30,7 +30,7 @@ class {name}(LazyLoadExtractor): make_valid_template = ''' @classmethod def _make_valid_url(cls): - return {!r} + return {valid_url!r} ''' @@ -44,7 +44,7 @@ def build_lazy_ie(ie, name): s += getsource(ie.suitable) if hasattr(ie, '_make_valid_url'): # search extractors - s += make_valid_template.format(ie._make_valid_url()) + s += make_valid_template.format(valid_url=ie._make_valid_url()) return s names = [] @@ -55,7 +55,7 @@ for ie in _ALL_CLASSES: names.append(name) module_contents.append( - '_ALL_CLASSES = [{}]'.format(', '.join(names))) + '_ALL_CLASSES = [{0}]'.format(', '.join(names))) module_src = '\n'.join(module_contents) -- cgit v1.2.3 From 6b97ca96fc242c1d7639d080e2c8e3ee9f9d0bed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaime=20Marqui=CC=81nez=20Ferra=CC=81ndiz?= Date: Sun, 21 Feb 2016 12:22:12 +0100 Subject: lazy extractors: Style fixes * Sort extractors alphabetically * Add newlines when needed (youtube_dl/extractors/lazy_extractors.py pass the flake8 test now) --- devscripts/make_lazy_extractors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'devscripts/make_lazy_extractors.py') diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 5d0ddb401..b5a8b9190 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -41,14 +41,14 @@ def build_lazy_ie(ie, name): valid_url=valid_url, module=ie.__module__) if ie.suitable.__func__ is not InfoExtractor.suitable.__func__: - s += getsource(ie.suitable) + s += '\n' + getsource(ie.suitable) if hasattr(ie, '_make_valid_url'): # search extractors s += make_valid_template.format(valid_url=ie._make_valid_url()) return s names = [] -for ie in _ALL_CLASSES: +for ie in list(sorted(_ALL_CLASSES[:-1], key=lambda cls: cls.ie_key())) + _ALL_CLASSES[-1:]: name = ie.ie_key() + 'IE' src = build_lazy_ie(ie, name) module_contents.append(src) @@ -57,7 +57,7 @@ for ie in _ALL_CLASSES: module_contents.append( '_ALL_CLASSES = [{0}]'.format(', '.join(names))) -module_src = '\n'.join(module_contents) +module_src = '\n'.join(module_contents) + '\n' with open(lazy_extractors_filename, 'wt') as f: f.write(module_src) -- cgit v1.2.3