aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--README.md7
-rwxr-xr-xtest/gentests.py18
-rw-r--r--test/test_download.py42
-rw-r--r--test/test_execution.py26
-rw-r--r--test/test_import.py13
-rw-r--r--test/test_utils.py7
-rw-r--r--test/test_youtube_playlist_ids.py22
-rw-r--r--test/tests.json14
-rw-r--r--youtube_dl/FileDownloader.py7
-rw-r--r--youtube_dl/InfoExtractors.py76
-rw-r--r--youtube_dl/__init__.py2
-rw-r--r--youtube_dl/utils.py23
12 files changed, 144 insertions, 113 deletions
diff --git a/README.md b/README.md
index b218fc3ec..2ca021125 100644
--- a/README.md
+++ b/README.md
@@ -51,7 +51,10 @@ which means you can modify it, redistribute it or use it however you like.
%(upload_date)s for the upload date (YYYYMMDD),
%(extractor)s for the provider (youtube, metacafe,
etc), %(id)s for the video id and %% for a literal
- percent. Use - to output to stdout.
+ percent. Use - to output to stdout. Can also be
+ used to download to a different directory, for
+ example with -o '/my/downloads/%(uploader)s/%(title
+ )s-%(id)s.%(ext)s' .
--restrict-filenames Restrict filenames to only ASCII characters, and
avoid "&" and spaces in filenames
-a, --batch-file FILE file containing URLs to download ('-' for stdin)
@@ -194,3 +197,5 @@ Please include:
* The output of `youtube-dl --version`
* The output of `python --version`
* The name and version of your Operating System ("Ubuntu 11.04 x64" or "Windows 7 x64" is usually enough).
+
+For discussions, join us in the irc channel #youtube-dl on freenode.
diff --git a/test/gentests.py b/test/gentests.py
index c3cca5156..9352d6d94 100755
--- a/test/gentests.py
+++ b/test/gentests.py
@@ -34,31 +34,19 @@ import youtube_dl.InfoExtractors
def _file_md5(fn):
with open(fn, 'rb') as f:
return hashlib.md5(f.read()).hexdigest()
-
-def md5_for_file(filename, block_size=2**20):
- with open(filename) as f:
- md5 = hashlib.md5()
- while True:
- data = f.read(block_size)
- if not data:
- break
- md5.update(data)
- return md5.hexdigest()
-_file_md5 = md5_for_file
-
-
try:
_skip_unless = unittest.skipUnless
except AttributeError: # Python 2.6
def _skip_unless(cond, reason='No reason given'):
def resfunc(f):
- def wfunc(*args, **kwargs):
+ # Start the function name with test to appease nosetests-2.6
+ def test_wfunc(*args, **kwargs):
if cond:
return f(*args, **kwargs)
else:
print('Skipped test')
return
- return wfunc
+ return test_wfunc
return resfunc
_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
diff --git a/test/test_download.py b/test/test_download.py
index b7cb6cebb..6926dd04e 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -20,31 +20,19 @@ import youtube_dl.InfoExtractors
def _file_md5(fn):
with open(fn, 'rb') as f:
return hashlib.md5(f.read()).hexdigest()
-
-def md5_for_file(filename, block_size=2**20):
- with open(filename) as f:
- md5 = hashlib.md5()
- while True:
- data = f.read(block_size)
- if not data:
- break
- md5.update(data)
- return md5.hexdigest()
-_file_md5 = md5_for_file
-
-
try:
_skip_unless = unittest.skipUnless
except AttributeError: # Python 2.6
def _skip_unless(cond, reason='No reason given'):
def resfunc(f):
- def wfunc(*args, **kwargs):
+ # Start the function name with test to appease nosetests-2.6
+ def test_wfunc(*args, **kwargs):
if cond:
return f(*args, **kwargs)
else:
print('Skipped test')
return
- return wfunc
+ return test_wfunc
return resfunc
_skip = lambda *args, **kwargs: _skip_unless(False, *args, **kwargs)
@@ -79,7 +67,7 @@ class DownloadTest(unittest.TestCase):
@_skip_unless(youtube_dl.InfoExtractors.MetacafeIE._WORKING, "IE marked as not _WORKING")
def test_Metacafe(self):
- filename = 'aUehQsCQtM.flv'
+ filename = '_aUehQsCQtM.flv'
fd = FileDownloader(self.parameters)
fd.add_info_extractor(youtube_dl.InfoExtractors.MetacafeIE())
fd.add_info_extractor(youtube_dl.InfoExtractors.YoutubeIE())
@@ -120,7 +108,7 @@ class DownloadTest(unittest.TestCase):
@_skip_unless(youtube_dl.InfoExtractors.SoundcloudIE._WORKING, "IE marked as not _WORKING")
def test_Soundcloud(self):
- filename = 'n6FLbx6ZzMiu.mp3'
+ filename = '62986583.mp3'
fd = FileDownloader(self.parameters)
fd.add_info_extractor(youtube_dl.InfoExtractors.SoundcloudIE())
fd.download(['http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy'])
@@ -159,26 +147,38 @@ class DownloadTest(unittest.TestCase):
md5_for_file = _file_md5(filename)
self.assertEqual(md5_for_file, 'c5c67df477eb0d9b058200351448ba4c')
+ @_skip_unless(youtube_dl.InfoExtractors.YoukuIE._WORKING, "IE marked as not _WORKING")
+ def test_Youku(self):
+ filename = 'XNDgyMDQ2NTQw_part00.flv'
+ fd = FileDownloader(self.parameters)
+ fd.add_info_extractor(youtube_dl.InfoExtractors.YoukuIE())
+ fd.download(['http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html'])
+ self.assertTrue(os.path.exists(filename))
+ md5_for_file = _file_md5(filename)
+ self.assertEqual(md5_for_file, 'ffe3f2e435663dc2d1eea34faeff5b5b')
+
def tearDown(self):
if os.path.exists('BaW_jenozKc.mp4'):
os.remove('BaW_jenozKc.mp4')
if os.path.exists('x33vw9.mp4'):
os.remove('x33vw9.mp4')
- if os.path.exists('aUehQsCQtM.flv'):
- os.remove('aUehQsCQtM.flv')
+ if os.path.exists('_aUehQsCQtM.flv'):
+ os.remove('_aUehQsCQtM.flv')
if os.path.exists('5779306.m4v'):
os.remove('5779306.m4v')
if os.path.exists('939581.flv'):
os.remove('939581.flv')
# No file specified for Vimeo
- if os.path.exists('n6FLbx6ZzMiu.mp3'):
- os.remove('n6FLbx6ZzMiu.mp3')
+ if os.path.exists('62986583.mp3'):
+ os.remove('62986583.mp3')
if os.path.exists('PracticalUnix_intro-environment.mp4'):
os.remove('PracticalUnix_intro-environment.mp4')
# No file specified for CollegeHumor
if os.path.exists('1135332.flv'):
os.remove('1135332.flv')
+ if os.path.exists('XNDgyMDQ2NTQw_part00.flv'):
+ os.remove('XNDgyMDQ2NTQw_part00.flv')
diff --git a/test/test_execution.py b/test/test_execution.py
new file mode 100644
index 000000000..2b115fb31
--- /dev/null
+++ b/test/test_execution.py
@@ -0,0 +1,26 @@
+import unittest
+
+import sys
+import os
+import subprocess
+
+rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+try:
+ _DEV_NULL = subprocess.DEVNULL
+except AttributeError:
+ _DEV_NULL = open(os.devnull, 'wb')
+
+class TestExecution(unittest.TestCase):
+ def test_import(self):
+ subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
+
+ def test_module_exec(self):
+ if sys.version_info >= (2,7): # Python 2.6 doesn't support package execution
+ subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+ def test_main_exec(self):
+ subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/test_import.py b/test/test_import.py
deleted file mode 100644
index acf95444d..000000000
--- a/test/test_import.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import unittest
-
-import sys
-import os.path
-import subprocess
-
-class TestImport(unittest.TestCase):
- def test_import(self):
- rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/test/test_utils.py b/test/test_utils.py
index fd8190f51..796ed4ab2 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
# Various small unit tests
import sys
@@ -79,6 +81,11 @@ class TestUtil(unittest.TestCase):
self.assertTrue(sanitize_filename('-', restricted=True) != '')
self.assertTrue(sanitize_filename(':', restricted=True) != '')
+ def test_sanitize_ids(self):
+ self.assertEquals(sanitize_filename('_n_cd26wFpw', is_id=True), '_n_cd26wFpw')
+ self.assertEquals(sanitize_filename('_BD_eEpuzXw', is_id=True), '_BD_eEpuzXw')
+ self.assertEquals(sanitize_filename('N0Y__7-UOdI', is_id=True), 'N0Y__7-UOdI')
+
def test_ordered_set(self):
self.assertEqual(orderedSet([1, 1, 2, 3, 4, 4, 5, 6, 7, 3, 5]), [1, 2, 3, 4, 5, 6, 7])
self.assertEqual(orderedSet([]), [])
diff --git a/test/test_youtube_playlist_ids.py b/test/test_youtube_playlist_ids.py
new file mode 100644
index 000000000..b4dcedb45
--- /dev/null
+++ b/test/test_youtube_playlist_ids.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+import sys
+import unittest
+
+# Allow direct execution
+import os
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from youtube_dl.InfoExtractors import YoutubeIE, YoutubePlaylistIE
+
+class TestYoutubePlaylistMatching(unittest.TestCase):
+ def test_playlist_matching(self):
+ assert YoutubePlaylistIE().suitable(u'ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
+ assert YoutubePlaylistIE().suitable(u'PL63F0C78739B09958')
+ assert not YoutubePlaylistIE().suitable(u'PLtS2H6bU1M')
+
+ def test_youtube_matching(self):
+ assert YoutubeIE().suitable(u'PLtS2H6bU1M')
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/test/tests.json b/test/tests.json
index 731c91718..cb29f2053 100644
--- a/test/tests.json
+++ b/test/tests.json
@@ -16,7 +16,7 @@
"size": 5754305,
"addIEs": ["Youtube"],
"url": "http://www.metacafe.com/watch/yt-_aUehQsCQtM/the_electric_company_short_i_pbs_kids_go/",
- "file": "aUehQsCQtM.flv"
+ "file": "_aUehQsCQtM.flv"
},
{
"name": "BlipTV",
@@ -40,7 +40,7 @@
"name": "Soundcloud",
"md5": "c1b9b9ea8bfd620b96b2628664576e1c",
"url": "http://soundcloud.com/ethmusic/lostin-powers-she-so-heavy",
- "file": "n6FLbx6ZzMiu.mp3"
+ "file": "62986583.mp3"
},
{
"name": "StanfordOpenClassroom",
@@ -54,10 +54,16 @@
"url": "http://www.collegehumor.com/video/6830834/mitt-romney-style-gangnam-style-parody",
"file": ""
},
- {
+ {
"name": "XNXX",
"md5": "c5c67df477eb0d9b058200351448ba4c",
"url": "http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_",
"file": "1135332.flv"
+ },
+ {
+ "name": "Youku",
+ "url": "http://v.youku.com/v_show/id_XNDgyMDQ2NTQw.html",
+ "file": "XNDgyMDQ2NTQw_part00.flv",
+ "md5": "ffe3f2e435663dc2d1eea34faeff5b5b"
}
-] \ No newline at end of file
+]
diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index 48c8eb126..c471cc160 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -334,8 +334,11 @@ class FileDownloader(object):
template_dict['epoch'] = int(time.time())
template_dict['autonumber'] = u'%05d' % self._num_downloads
- template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items())
- template_dict = dict((k, sanitize_filename(compat_str(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
+ sanitize = lambda k,v: sanitize_filename(
+ u'NA' if v is None else compat_str(v),
+ restricted=self.params.get('restrictfilenames'),
+ is_id=(k==u'id'))
+ template_dict = dict((k, sanitize(k, v)) for k,v in template_dict.items())
filename = self.params['outtmpl'] % template_dict
return filename
diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py
index 925dabc96..49976c25b 100644
--- a/youtube_dl/InfoExtractors.py
+++ b/youtube_dl/InfoExtractors.py
@@ -1671,7 +1671,7 @@ class YahooSearchIE(InfoExtractor):
class YoutubePlaylistIE(InfoExtractor):
"""Information Extractor for YouTube playlists."""
- _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
+ _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*'
_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
_VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s'
_MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
@@ -2803,13 +2803,13 @@ class SoundcloudIE(InfoExtractor):
def __init__(self, downloader=None):
InfoExtractor.__init__(self, downloader)
- def report_webpage(self, video_id):
+ def report_resolve(self, video_id):
"""Report information extraction."""
- self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
+ self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id))
def report_extraction(self, video_id):
"""Report information extraction."""
- self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
+ self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id))
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
@@ -2818,65 +2818,47 @@ class SoundcloudIE(InfoExtractor):
return
# extract uploader (which is in the url)
- uploader = mobj.group(1).decode('utf-8')
+ uploader = mobj.group(1)
# extract simple title (uploader + slug of song title)
- slug_title = mobj.group(2).decode('utf-8')
+ slug_title = mobj.group(2)
simple_title = uploader + u'-' + slug_title
- self.report_webpage('%s/%s' % (uploader, slug_title))
+ self.report_resolve('%s/%s' % (uploader, slug_title))
- request = compat_urllib_request.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
+ url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title)
+ resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
+ request = compat_urllib_request.Request(resolv_url)
try:
- webpage = compat_urllib_request.urlopen(request).read()
+ info_json_bytes = compat_urllib_request.urlopen(request).read()
+ info_json = info_json_bytes.decode('utf-8')
except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
return
+ info = json.loads(info_json)
+ video_id = info['id']
self.report_extraction('%s/%s' % (uploader, slug_title))
- # extract uid and stream token that soundcloud hands out for access
- mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
- if mobj:
- video_id = mobj.group(1)
- stream_token = mobj.group(2)
-
- # extract unsimplified title
- mobj = re.search('"title":"(.*?)",', webpage)
- if mobj:
- title = mobj.group(1).decode('utf-8')
- else:
- title = simple_title
-
- # construct media url (with uid/token)
- mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
- mediaURL = mediaURL % (video_id, stream_token)
-
- # description
- description = u'No description available'
- mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
- if mobj:
- description = mobj.group(1)
-
- # upload date
- upload_date = None
- mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
- if mobj:
- try:
- upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
- except Exception as err:
- self._downloader.to_stderr(compat_str(err))
+ streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
+ request = compat_urllib_request.Request(streams_url)
+ try:
+ stream_json_bytes = compat_urllib_request.urlopen(request).read()
+ stream_json = stream_json_bytes.decode('utf-8')
+ except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+ return
- # for soundcloud, a request to a cross domain is required for cookies
- request = compat_urllib_request.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
+ streams = json.loads(stream_json)
+ mediaURL = streams['http_mp3_128_url']
return [{
- 'id': video_id.decode('utf-8'),
+ 'id': info['id'],
'url': mediaURL,
- 'uploader': uploader.decode('utf-8'),
- 'upload_date': upload_date,
- 'title': title,
+ 'uploader': info['user']['username'],
+ 'upload_date': info['created_at'],
+ 'title': info['title'],
'ext': u'mp3',
- 'description': description.decode('utf-8')
+ 'description': info['description'],
}]
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index f953f1fb9..8e46dfea9 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -305,7 +305,7 @@ def parseOpts():
action='store_true', dest='autonumber',
help='number downloaded files starting from 00000', default=False)
filesystem.add_option('-o', '--output',
- dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout.')
+ dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(title)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, %(upload_date)s for the upload date (YYYYMMDD), %(extractor)s for the provider (youtube, metacafe, etc), %(id)s for the video id and %% for a literal percent. Use - to output to stdout. Can also be used to download to a different directory, for example with -o \'/my/downloads/%(uploader)s/%(title)s-%(id)s.%(ext)s\' .')
filesystem.add_option('--restrict-filenames',
action='store_true', dest='restrictfilenames',
help='Restrict filenames to only ASCII characters, and avoid "&" and spaces in filenames', default=False)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 7f73b8476..44f939053 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -317,9 +317,10 @@ def timeconvert(timestr):
timestamp = email.utils.mktime_tz(timetuple)
return timestamp
-def sanitize_filename(s, restricted=False):
+def sanitize_filename(s, restricted=False, is_id=False):
"""Sanitizes a string so it could be used as part of a filename.
If restricted is set, use a stricter subset of allowed characters.
+ Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
"""
def replace_insane(char):
if char == '?' or ord(char) < 32 or ord(char) == 127:
@@ -337,14 +338,15 @@ def sanitize_filename(s, restricted=False):
return char
result = u''.join(map(replace_insane, s))
- while '__' in result:
- result = result.replace('__', '_')
- result = result.strip('_')
- # Common case of "Foreign band name - English song title"
- if restricted and result.startswith('-_'):
- result = result[2:]
- if not result:
- result = '_'
+ if not is_id:
+ while '__' in result:
+ result = result.replace('__', '_')
+ result = result.strip('_')
+ # Common case of "Foreign band name - English song title"
+ if restricted and result.startswith('-_'):
+ result = result[2:]
+ if not result:
+ result = '_'
return result
def orderedSet(iterable):
@@ -504,3 +506,6 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
resp.msg = old_resp.msg
return resp
+
+ https_request = http_request
+ https_response = http_response