From 0704d2224b328caeafbce6a029904472628d12bd Mon Sep 17 00:00:00 2001 From: Unknown Date: Thu, 29 Oct 2020 01:56:55 +0100 Subject: [core] be able to hand over id and title using url_result --- youtube_dlc/YoutubeDL.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'youtube_dlc/YoutubeDL.py') diff --git a/youtube_dlc/YoutubeDL.py b/youtube_dlc/YoutubeDL.py index fc351db0d..f959a4e47 100644 --- a/youtube_dlc/YoutubeDL.py +++ b/youtube_dlc/YoutubeDL.py @@ -801,7 +801,7 @@ class YoutubeDL(object): for key, value in extra_info.items(): info_dict.setdefault(key, value) - def extract_info(self, url, download=True, ie_key=None, extra_info={}, + def extract_info(self, url, download=True, ie_key=None, info_dict=None, extra_info={}, process=True, force_generic_extractor=False): ''' Returns a list with a dictionary for each video we find. @@ -836,6 +836,11 @@ class YoutubeDL(object): '_type': 'compat_list', 'entries': ie_result, } + if info_dict: + if info_dict.get('id'): + ie_result['id'] = info_dict['id'] + if info_dict.get('title'): + ie_result['title'] = info_dict['title'] self.add_default_extra_info(ie_result, ie, url) if process: return self.process_ie_result(ie_result, download, extra_info) @@ -898,7 +903,7 @@ class YoutubeDL(object): # We have to add extra_info to the results because it may be # contained in a playlist return self.extract_info(ie_result['url'], - download, + download, info_dict=ie_result, ie_key=ie_result.get('ie_key'), extra_info=extra_info) elif result_type == 'url_transparent': -- cgit v1.2.3 From 9f448fcb269d52e757999cba6dab3ff7046d2c19 Mon Sep 17 00:00:00 2001 From: Unknown Date: Sat, 31 Oct 2020 05:46:51 +0100 Subject: [core/yt_live_chat] live_chat is back. dl() new parameter --- youtube_dlc/YoutubeDL.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'youtube_dlc/YoutubeDL.py') diff --git a/youtube_dlc/YoutubeDL.py b/youtube_dlc/YoutubeDL.py index f959a4e47..360595918 100644 --- a/youtube_dlc/YoutubeDL.py +++ b/youtube_dlc/YoutubeDL.py @@ -1857,13 +1857,13 @@ class YoutubeDL(object): self.report_error('Cannot write annotations file: ' + annofn) return - def dl(name, info): + def dl(name, info, subtitle=False): fd = get_suitable_downloader(info, self.params)(self, self.params) for ph in self._progress_hooks: fd.add_progress_hook(ph) if self.params.get('verbose'): self.to_stdout('[debug] Invoking downloader on %r' % info.get('url')) - return fd.download(name, info) + return fd.download(name, info, subtitle) subtitles_are_requested = any([self.params.get('writesubtitles', False), self.params.get('writeautomaticsub')]) @@ -1891,6 +1891,8 @@ class YoutubeDL(object): return else: try: + dl(sub_filename, sub_info, subtitle=True) + ''' if self.params.get('sleep_interval_subtitles', False): dl(sub_filename, sub_info) else: @@ -1898,6 +1900,7 @@ class YoutubeDL(object): sub_info['url'], info_dict['id'], note=False).read() with io.open(encodeFilename(sub_filename), 'wb') as subfile: subfile.write(sub_data) + ''' except (ExtractorError, IOError, OSError, ValueError, compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_warning('Unable to download subtitle for "%s": %s' % (sub_lang, error_to_compat_str(err))) -- cgit v1.2.3 From fa57af1ef333b11630ba6ae4353a94ea118883d4 Mon Sep 17 00:00:00 2001 From: Unknown Date: Sat, 31 Oct 2020 08:57:55 +0100 Subject: flake8. added sha256 check to updater. --- youtube_dlc/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube_dlc/YoutubeDL.py') diff --git a/youtube_dlc/YoutubeDL.py b/youtube_dlc/YoutubeDL.py index 360595918..dd55ba0f2 100644 --- a/youtube_dlc/YoutubeDL.py +++ b/youtube_dlc/YoutubeDL.py @@ -1872,7 +1872,7 @@ class YoutubeDL(object): # subtitles download errors are already managed as troubles in relevant IE # that way it will silently go on when used with unsupporting IE subtitles = info_dict['requested_subtitles'] - ie = self.get_info_extractor(info_dict['extractor_key']) + # ie = self.get_info_extractor(info_dict['extractor_key']) for sub_lang, sub_info in subtitles.items(): sub_format = sub_info['ext'] sub_filename = subtitles_filename(filename, sub_lang, sub_format, info_dict.get('ext')) -- cgit v1.2.3 From 9a68de12179c92b578fd00e16ff3ca63aab94c94 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 13 Nov 2020 02:40:51 +0530 Subject: Pre-check video IDs in the archive before downloading --- youtube_dlc/YoutubeDL.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'youtube_dlc/YoutubeDL.py') diff --git a/youtube_dlc/YoutubeDL.py b/youtube_dlc/YoutubeDL.py index dd55ba0f2..373e83715 100644 --- a/youtube_dlc/YoutubeDL.py +++ b/youtube_dlc/YoutubeDL.py @@ -821,12 +821,22 @@ class YoutubeDL(object): if not ie.suitable(url): continue - ie = self.get_info_extractor(ie.ie_key()) + ie_key = ie.ie_key() + ie = self.get_info_extractor(ie_key) if not ie.working(): self.report_warning('The program functionality for this site has been marked as broken, ' 'and will probably not work.') try: + try: + temp_id = ie.extract_id(url) if callable(getattr(ie, 'extract_id', None)) else ie._match_id(url) + except AssertionError: + temp_id = None + if temp_id is not None and self.in_download_archive({'id': temp_id, 'ie_key': ie_key}): + self.to_screen("[download] [%s] %s has already been recorded in archive" % ( + ie_key, temp_id)) + break + ie_result = ie.extract(url) if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here) break -- cgit v1.2.3 From fe5caa2a7c0bb6f17c6833b540691b4df4cbde90 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Fri, 13 Nov 2020 03:05:29 +0530 Subject: Handle IndexError --- youtube_dlc/YoutubeDL.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'youtube_dlc/YoutubeDL.py') diff --git a/youtube_dlc/YoutubeDL.py b/youtube_dlc/YoutubeDL.py index 373e83715..c85cbd88f 100644 --- a/youtube_dlc/YoutubeDL.py +++ b/youtube_dlc/YoutubeDL.py @@ -830,10 +830,10 @@ class YoutubeDL(object): try: try: temp_id = ie.extract_id(url) if callable(getattr(ie, 'extract_id', None)) else ie._match_id(url) - except AssertionError: + except (AssertionError, IndexError): temp_id = None if temp_id is not None and self.in_download_archive({'id': temp_id, 'ie_key': ie_key}): - self.to_screen("[download] [%s] %s has already been recorded in archive" % ( + self.to_screen("[%s] %s: has already been recorded in archive" % ( ie_key, temp_id)) break -- cgit v1.2.3 From ea6e0c2b0d3e83f2f8a7766e07a57a2b5495afcc Mon Sep 17 00:00:00 2001 From: pukkandan Date: Mon, 21 Sep 2020 20:56:18 +0530 Subject: Add --break-on-existing by @gergesh Authored-by: Yoav Shai --- youtube_dlc/YoutubeDL.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'youtube_dlc/YoutubeDL.py') diff --git a/youtube_dlc/YoutubeDL.py b/youtube_dlc/YoutubeDL.py index dd55ba0f2..1cb1b421a 100644 --- a/youtube_dlc/YoutubeDL.py +++ b/youtube_dlc/YoutubeDL.py @@ -210,6 +210,8 @@ class YoutubeDL(object): download_archive: File name of a file where all downloads are recorded. Videos already present in the file are not downloaded again. + break_on_existing: Stop the download process after attempting to download a file that's + in the archive. cookiefile: File name where cookies should be read from and dumped to. nocheckcertificate:Do not verify SSL certificates prefer_insecure: Use HTTP instead of HTTPS to retrieve information. @@ -1038,8 +1040,12 @@ class YoutubeDL(object): reason = self._match_entry(entry, incomplete=True) if reason is not None: - self.to_screen('[download] ' + reason) - continue + if reason.endswith('has already been recorded in the archive') and self.params.get('break_on_existing'): + print('[download] tried downloading a file that\'s already in the archive, stopping since --break-on-existing is set.') + break + else: + self.to_screen('[download] ' + reason) + continue entry_result = self.process_ie_result(entry, download=download, -- cgit v1.2.3 From 3462ffa8929d2a40588669578ca912d57a0da1bb Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sun, 22 Nov 2020 03:51:09 +0530 Subject: Implemented all Youtube Feeds (ytfav, ytwatchlater, ytsubs, ythistory, ytrec) and SearchURL --- youtube_dlc/YoutubeDL.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'youtube_dlc/YoutubeDL.py') diff --git a/youtube_dlc/YoutubeDL.py b/youtube_dlc/YoutubeDL.py index bf02192eb..3c2970d9f 100644 --- a/youtube_dlc/YoutubeDL.py +++ b/youtube_dlc/YoutubeDL.py @@ -832,7 +832,7 @@ class YoutubeDL(object): try: try: temp_id = ie.extract_id(url) if callable(getattr(ie, 'extract_id', None)) else ie._match_id(url) - except (AssertionError, IndexError): + except (AssertionError, IndexError, AttributeError): temp_id = None if temp_id is not None and self.in_download_archive({'id': temp_id, 'ie_key': ie_key}): self.to_screen("[%s] %s: has already been recorded in archive" % ( -- cgit v1.2.3 From a0566bbf5c3dea282447efb2926d71bafe1b7720 Mon Sep 17 00:00:00 2001 From: pukkandan Date: Sat, 21 Nov 2020 20:20:42 +0530 Subject: Updated to release 2020.11.21.1 --- youtube_dlc/YoutubeDL.py | 79 +++++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 34 deletions(-) (limited to 'youtube_dlc/YoutubeDL.py') diff --git a/youtube_dlc/YoutubeDL.py b/youtube_dlc/YoutubeDL.py index 3c2970d9f..ef6fe0a78 100644 --- a/youtube_dlc/YoutubeDL.py +++ b/youtube_dlc/YoutubeDL.py @@ -830,34 +830,23 @@ class YoutubeDL(object): 'and will probably not work.') try: - try: - temp_id = ie.extract_id(url) if callable(getattr(ie, 'extract_id', None)) else ie._match_id(url) - except (AssertionError, IndexError, AttributeError): - temp_id = None - if temp_id is not None and self.in_download_archive({'id': temp_id, 'ie_key': ie_key}): - self.to_screen("[%s] %s: has already been recorded in archive" % ( - ie_key, temp_id)) - break + temp_id = ie.extract_id(url) if callable(getattr(ie, 'extract_id', None)) else ie._match_id(url) + except (AssertionError, IndexError, AttributeError): + temp_id = None + if temp_id is not None and self.in_download_archive({'id': temp_id, 'ie_key': ie_key}): + self.to_screen("[%s] %s: has already been recorded in archive" % ( + ie_key, temp_id)) + break - ie_result = ie.extract(url) - if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here) - break - if isinstance(ie_result, list): - # Backwards compatibility: old IE result format - ie_result = { - '_type': 'compat_list', - 'entries': ie_result, - } - if info_dict: - if info_dict.get('id'): - ie_result['id'] = info_dict['id'] - if info_dict.get('title'): - ie_result['title'] = info_dict['title'] - self.add_default_extra_info(ie_result, ie, url) - if process: - return self.process_ie_result(ie_result, download, extra_info) - else: - return ie_result + return self.__extract_info(url, ie, download, extra_info, process, info_dict) + + else: + self.report_error('no suitable InfoExtractor for URL %s' % url) + + def __handle_extraction_exceptions(func): + def wrapper(self, *args, **kwargs): + try: + return func(self, *args, **kwargs) except GeoRestrictedError as e: msg = e.msg if e.countries: @@ -865,20 +854,38 @@ class YoutubeDL(object): map(ISO3166Utils.short2full, e.countries)) msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.' self.report_error(msg) - break except ExtractorError as e: # An error we somewhat expected self.report_error(compat_str(e), e.format_traceback()) - break except MaxDownloadsReached: raise except Exception as e: if self.params.get('ignoreerrors', False): self.report_error(error_to_compat_str(e), tb=encode_compat_str(traceback.format_exc())) - break else: raise + return wrapper + + @__handle_extraction_exceptions + def __extract_info(self, url, ie, download, extra_info, process, info_dict): + ie_result = ie.extract(url) + if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here) + return + if isinstance(ie_result, list): + # Backwards compatibility: old IE result format + ie_result = { + '_type': 'compat_list', + 'entries': ie_result, + } + if info_dict: + if info_dict.get('id'): + ie_result['id'] = info_dict['id'] + if info_dict.get('title'): + ie_result['title'] = info_dict['title'] + self.add_default_extra_info(ie_result, ie, url) + if process: + return self.process_ie_result(ie_result, download, extra_info) else: - self.report_error('no suitable InfoExtractor for URL %s' % url) + return ie_result def add_default_extra_info(self, ie_result, ie, url): self.add_extra_info(ie_result, { @@ -1057,9 +1064,8 @@ class YoutubeDL(object): self.to_screen('[download] ' + reason) continue - entry_result = self.process_ie_result(entry, - download=download, - extra_info=extra) + entry_result = self.__process_iterable_entry(entry, download, extra) + # TODO: skip failed (empty) entries? playlist_results.append(entry_result) ie_result['entries'] = playlist_results self.to_screen('[download] Finished downloading playlist: %s' % playlist) @@ -1088,6 +1094,11 @@ class YoutubeDL(object): else: raise Exception('Invalid result type: %s' % result_type) + @__handle_extraction_exceptions + def __process_iterable_entry(self, entry, download, extra_info): + return self.process_ie_result( + entry, download=download, extra_info=extra_info) + def _build_format_filter(self, filter_spec): " Returns a function to filter the formats according to the filter_spec " -- cgit v1.2.3