diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | devscripts/transition_helper_exe/youtube-dl.py | 63 | ||||
| -rwxr-xr-x | youtube-dl | 57 | ||||
| -rw-r--r-- | youtube-dl.exe | bin | 3790446 -> 3803016 bytes | |||
| -rw-r--r-- | youtube_dl/__init__.py | 133 | ||||
| -rw-r--r-- | youtube_dl/utils.py | 28 |
6 files changed, 224 insertions, 58 deletions
diff --git a/.gitignore b/.gitignore index f07fc5d60..564bde1d1 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ youtube-dl.exe youtube-dl.tar.gz .coverage cover/ +updates_key.pem diff --git a/devscripts/transition_helper_exe/youtube-dl.py b/devscripts/transition_helper_exe/youtube-dl.py index 409f980bc..dbb4c99e1 100644 --- a/devscripts/transition_helper_exe/youtube-dl.py +++ b/devscripts/transition_helper_exe/youtube-dl.py @@ -2,17 +2,48 @@ import sys, os import urllib2 +import json, hashlib + +def rsa_verify(message, signature, key): + from struct import pack + from hashlib import sha256 + from sys import version_info + def b(x): + if version_info[0] == 2: return x + else: return x.encode('latin1') + assert(type(message) == type(b(''))) + block_size = 0 + n = key[0] + while n: + block_size += 1 + n >>= 8 + signature = pow(int(signature, 16), key[1], key[0]) + raw_bytes = [] + while signature: + raw_bytes.insert(0, pack("B", signature & 0xFF)) + signature >>= 8 + signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes) + if signature[0:2] != b('\x00\x01'): return False + signature = signature[2:] + if not b('\x00') in signature: return False + signature = signature[signature.index(b('\x00'))+1:] + if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False + signature = signature[19:] + if signature != sha256(message).digest(): return False + return True sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n') sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n') -sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n') +sys.stderr.write(u'From now on, get the binaries from http://rg3.github.com/youtube-dl/download.html, not from the git repository.\n\n') raw_input() filename = sys.argv[0] -API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads" -EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe" +UPDATE_URL = "http://rg3.github.com/youtube-dl/update/" +VERSION_URL = UPDATE_URL + 'LATEST_VERSION' +JSON_URL = UPDATE_URL + 'versions.json' +UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537) if not os.access(filename, os.W_OK): sys.exit('ERROR: no write permissions on %s' % filename) @@ -23,13 +54,35 @@ if not os.access(directory, os.W_OK): sys.exit('ERROR: no write permissions on %s' % directory) try: - urlh = urllib2.urlopen(EXE_URL) + versions_info = urllib2.urlopen(JSON_URL).read().decode('utf-8') + versions_info = json.loads(versions_info) +except: + sys.exit(u'ERROR: can\'t obtain versions info. Please try again later.') +if not 'signature' in versions_info: + sys.exit(u'ERROR: the versions file is not signed or corrupted. Aborting.') +signature = versions_info['signature'] +del versions_info['signature'] +if not rsa_verify(json.dumps(versions_info, sort_keys=True), signature, UPDATES_RSA_KEY): + sys.exit(u'ERROR: the versions file signature is invalid. Aborting.') + +version = versions_info['versions'][versions_info['latest']] + +try: + urlh = urllib2.urlopen(version['exe'][0]) newcontent = urlh.read() urlh.close() +except (IOError, OSError) as err: + sys.exit('ERROR: unable to download latest version') + +newcontent_hash = hashlib.sha256(newcontent).hexdigest() +if newcontent_hash != version['exe'][1]: + sys.exit(u'ERROR: the downloaded file hash does not match. Aborting.') + +try: with open(exe + '.new', 'wb') as outf: outf.write(newcontent) except (IOError, OSError) as err: - sys.exit('ERROR: unable to download latest version') + sys.exit(u'ERROR: unable to write the new version') try: bat = os.path.join(directory, 'youtube-dl-updater.bat') diff --git a/youtube-dl b/youtube-dl index d5ca2d4ba..9766d905e 100755 --- a/youtube-dl +++ b/youtube-dl @@ -1,15 +1,44 @@ #!/usr/bin/env python import sys, os +import json, hashlib try: import urllib.request as compat_urllib_request except ImportError: # Python 2 import urllib2 as compat_urllib_request +def rsa_verify(message, signature, key): + from struct import pack + from hashlib import sha256 + from sys import version_info + def b(x): + if version_info[0] == 2: return x + else: return x.encode('latin1') + assert(type(message) == type(b(''))) + block_size = 0 + n = key[0] + while n: + block_size += 1 + n >>= 8 + signature = pow(int(signature, 16), key[1], key[0]) + raw_bytes = [] + while signature: + raw_bytes.insert(0, pack("B", signature & 0xFF)) + signature >>= 8 + signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes) + if signature[0:2] != b('\x00\x01'): return False + signature = signature[2:] + if not b('\x00') in signature: return False + signature = signature[signature.index(b('\x00'))+1:] + if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False + signature = signature[19:] + if signature != sha256(message).digest(): return False + return True + sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n') sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n') -sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n') +sys.stderr.write(u'From now on, get the binaries from http://rg3.github.com/youtube-dl/download.html, not from the git repository.\n\n') try: raw_input() @@ -18,19 +47,39 @@ except NameError: # Python 3 filename = sys.argv[0] -API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads" -BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl" +UPDATE_URL = "http://rg3.github.com/youtube-dl/update/" +VERSION_URL = UPDATE_URL + 'LATEST_VERSION' +JSON_URL = UPDATE_URL + 'versions.json' +UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537) if not os.access(filename, os.W_OK): sys.exit('ERROR: no write permissions on %s' % filename) try: - urlh = compat_urllib_request.urlopen(BIN_URL) + versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8') + versions_info = json.loads(versions_info) +except: + sys.exit(u'ERROR: can\'t obtain versions info. Please try again later.') +if not 'signature' in versions_info: + sys.exit(u'ERROR: the versions file is not signed or corrupted. Aborting.') +signature = versions_info['signature'] +del versions_info['signature'] +if not rsa_verify(json.dumps(versions_info, sort_keys=True), signature, UPDATES_RSA_KEY): + sys.exit(u'ERROR: the versions file signature is invalid. Aborting.') + +version = versions_info['versions'][versions_info['latest']] + +try: + urlh = compat_urllib_request.urlopen(version['bin'][0]) newcontent = urlh.read() urlh.close() except (IOError, OSError) as err: sys.exit('ERROR: unable to download latest version') +newcontent_hash = hashlib.sha256(newcontent).hexdigest() +if newcontent_hash != version['bin'][1]: + sys.exit(u'ERROR: the downloaded file hash does not match. Aborting.') + try: with open(filename, 'wb') as outf: outf.write(newcontent) diff --git a/youtube-dl.exe b/youtube-dl.exe Binary files differindex a11878986..45eee04bb 100644 --- a/youtube-dl.exe +++ b/youtube-dl.exe diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index d12ece21e..cc2f555b9 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -41,47 +41,90 @@ from .FileDownloader import * from .InfoExtractors import * from .PostProcessor import * -def updateSelf(downloader, filename): +def update_self(to_screen, verbose, filename): """Update the program file with the latest version from the repository""" - # TODO: at least, check https certificates - from zipimport import zipimporter + import json, traceback, hashlib - API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads" - BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl" - EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe" + UPDATE_URL = "http://rg3.github.com/youtube-dl/update/" + VERSION_URL = UPDATE_URL + 'LATEST_VERSION' + JSON_URL = UPDATE_URL + 'versions.json' + UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537) - if hasattr(sys, "frozen"): # PY2EXE - if not os.access(filename, os.W_OK): - sys.exit('ERROR: no write permissions on %s' % filename) - downloader.to_screen(u'Updating to latest version...') + if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"): + to_screen(u'It looks like you installed youtube-dl with pip, setup.py or a tarball. Please use that to update.') + return - urla = compat_urllib_request.urlopen(API_URL) - download = filter(lambda x: x["name"] == "youtube-dl.exe", json.loads(urla.read())) - if not download: - downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.') - return - newversion = download[0]["description"].strip() - if newversion == __version__: - downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') - return - urla.close() + # Check if there is a new version + try: + newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip() + except: + if verbose: to_screen(traceback.format_exc().decode()) + to_screen(u'ERROR: can\'t find the current version. Please try again later.') + return + if newversion == __version__: + to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') + return + # Download and check versions info + try: + versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8') + versions_info = json.loads(versions_info) + except: + if verbose: to_screen(traceback.format_exc().decode()) + to_screen(u'ERROR: can\'t obtain versions info. Please try again later.') + return + if not 'signature' in versions_info: + to_screen(u'ERROR: the versions file is not signed or corrupted. Aborting.') + return + signature = versions_info['signature'] + del versions_info['signature'] + if not rsa_verify(json.dumps(versions_info, sort_keys=True), signature, UPDATES_RSA_KEY): + to_screen(u'ERROR: the versions file signature is invalid. Aborting.') + return + + to_screen(u'Updating to version ' + versions_info['latest'] + '...') + version = versions_info['versions'][versions_info['latest']] + if version.get('notes'): + to_screen(u'PLEASE NOTE:') + for note in version['notes']: + to_screen(note) + + if not os.access(filename, os.W_OK): + to_screen(u'ERROR: no write permissions on %s' % filename) + return + + # Py2EXE + if hasattr(sys, "frozen"): exe = os.path.abspath(filename) directory = os.path.dirname(exe) if not os.access(directory, os.W_OK): - sys.exit('ERROR: no write permissions on %s' % directory) + to_screen(u'ERROR: no write permissions on %s' % directory) + return try: - urlh = compat_urllib_request.urlopen(EXE_URL) + urlh = compat_urllib_request.urlopen(version['exe'][0]) newcontent = urlh.read() urlh.close() + except (IOError, OSError) as err: + if verbose: to_screen(traceback.format_exc().decode()) + to_screen(u'ERROR: unable to download latest version') + return + + newcontent_hash = hashlib.sha256(newcontent).hexdigest() + if newcontent_hash != version['exe'][1]: + to_screen(u'ERROR: the downloaded file hash does not match. Aborting.') + return + + try: with open(exe + '.new', 'wb') as outf: outf.write(newcontent) except (IOError, OSError) as err: - sys.exit('ERROR: unable to download latest version') + if verbose: to_screen(traceback.format_exc().decode()) + to_screen(u'ERROR: unable to write the new version') + return try: bat = os.path.join(directory, 'youtube-dl-updater.bat') @@ -96,43 +139,35 @@ del "%s" os.startfile(bat) except (IOError, OSError) as err: - sys.exit('ERROR: unable to overwrite current version') - - elif isinstance(globals().get('__loader__'), zipimporter): # UNIX ZIP - if not os.access(filename, os.W_OK): - sys.exit('ERROR: no write permissions on %s' % filename) - - downloader.to_screen(u'Updating to latest version...') - - urla = compat_urllib_request.urlopen(API_URL) - download = [x for x in json.loads(urla.read().decode('utf8')) if x["name"] == "youtube-dl"] - if not download: - downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.') + if verbose: to_screen(traceback.format_exc().decode()) + to_screen(u'ERROR: unable to overwrite current version') return - newversion = download[0]["description"].strip() - if newversion == __version__: - downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') - return - urla.close() + # Zip unix package + elif isinstance(globals().get('__loader__'), zipimporter): try: - urlh = compat_urllib_request.urlopen(BIN_URL) + urlh = compat_urllib_request.urlopen(version['bin'][0]) newcontent = urlh.read() urlh.close() except (IOError, OSError) as err: - sys.exit('ERROR: unable to download latest version') + if verbose: to_screen(traceback.format_exc().decode()) + to_screen(u'ERROR: unable to download latest version') + return + + newcontent_hash = hashlib.sha256(newcontent).hexdigest() + if newcontent_hash != version['bin'][1]: + to_screen(u'ERROR: the downloaded file hash does not match. Aborting.') + return try: with open(filename, 'wb') as outf: outf.write(newcontent) except (IOError, OSError) as err: - sys.exit('ERROR: unable to overwrite current version') - - else: - downloader.to_screen(u'It looks like you installed youtube-dl with pip or setup.py. Please use that to update.') - return + if verbose: to_screen(traceback.format_exc().decode()) + to_screen(u'ERROR: unable to overwrite current version') + return - downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.') + to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.') def parseOpts(): def _readOptions(filename_bytes): @@ -578,7 +613,7 @@ def _real_main(): # Update version if opts.update_self: - updateSelf(fd, sys.argv[0]) + update_self(fd.to_screen, opts.verbose, sys.argv[0]) # Maybe do nothing if len(all_urls) < 1: diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 463804e18..7d6041929 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -410,6 +410,34 @@ def encodeFilename(s): else: return s.encode(sys.getfilesystemencoding(), 'ignore') +def rsa_verify(message, signature, key): + from struct import pack + from hashlib import sha256 + from sys import version_info + def b(x): + if version_info[0] == 2: return x + else: return x.encode('latin1') + assert(type(message) == type(b(''))) + block_size = 0 + n = key[0] + while n: + block_size += 1 + n >>= 8 + signature = pow(int(signature, 16), key[1], key[0]) + raw_bytes = [] + while signature: + raw_bytes.insert(0, pack("B", signature & 0xFF)) + signature >>= 8 + signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes) + if signature[0:2] != b('\x00\x01'): return False + signature = signature[2:] + if not b('\x00') in signature: return False + signature = signature[signature.index(b('\x00'))+1:] + if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False + signature = signature[19:] + if signature != sha256(message).digest(): return False + return True + class DownloadError(Exception): """Download Error exception. |