Merge remote-tracking branch 'origin/master' into IE_cleanup

Conflicts:
	youtube_dl/FileDownloader.py

4 files changed, 255 insertions, 97 deletions

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index a7997c4f2..870c82272 100644
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -62,6 +62,8 @@ class FileDownloader(object):
 	ratelimit:         Download speed limit, in bytes/sec.
 	nooverwrites:      Prevent overwriting files.
 	retries:           Number of times to retry for HTTP error 5xx
+	buffersize:        Size of download buffer in bytes.
+	noresizebuffer:    Do not automatically resize the download buffer.
 	continuedl:        Try to continue downloads if possible.
 	noprogress:        Do not print the progress bar.
 	playliststart:     Playlist item to start at.
@@ -106,7 +108,7 @@ class FileDownloader(object):
 		if bytes == 0.0:
 			exponent = 0
 		else:
-			exponent = long(math.log(bytes, 1024.0))
+			exponent = int(math.log(bytes, 1024.0))
 		suffix = 'bkMGTPEZY'[exponent]
 		converted = float(bytes) / float(1024 ** exponent)
 		return '%.2f%s' % (converted, suffix)
@@ -125,7 +127,7 @@ class FileDownloader(object):
 		if current == 0 or dif < 0.001: # One millisecond
 			return '--:--'
 		rate = float(current) / dif
-		eta = long((float(total) - float(current)) / rate)
+		eta = int((float(total) - float(current)) / rate)
 		(eta_mins, eta_secs) = divmod(eta, 60)
 		if eta_mins > 99:
 			return '--:--'
@@ -177,7 +179,7 @@ class FileDownloader(object):
 		if not self.params.get('quiet', False):
 			terminator = [u'\n', u''][skip_eol]
 			output = message + terminator
-			if 'b' not in self._screen_file.mode or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
+			if 'b' in getattr(self._screen_file, 'mode', '') or sys.version_info[0] < 3: # Python 2 lies about the mode of sys.stdout/sys.stderr
 				output = output.encode(preferredencoding(), 'ignore')
 			self._screen_file.write(output)
 			self._screen_file.flush()
@@ -325,9 +327,13 @@ class FileDownloader(object):
 		"""Generate the output filename."""
 		try:
 			template_dict = dict(info_dict)
-			template_dict['epoch'] = unicode(int(time.time()))
-			template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
+
+			template_dict['epoch'] = int(time.time())
+			template_dict['autonumber'] = u'%05d' % self._num_downloads
+
 			template_dict = dict((key, u'NA' if val is None else val) for key, val in template_dict.items())
+			template_dict = dict((k, sanitize_filename(u(v), self.params.get('restrictfilenames'))) for k,v in template_dict.items())
+
 			filename = self.params['outtmpl'] % template_dict
 			return filename
 		except (ValueError, KeyError), err:
@@ -370,7 +376,6 @@ class FileDownloader(object):
 				raise MaxDownloadsReached()
 
 		filename = self.prepare_filename(info_dict)
-		filename = sanitize_filename(filename, self.params.get('restrictfilenames'))
 
 		# Forced printings
 		if self.params.get('forcetitle', False):
@@ -398,7 +403,7 @@ class FileDownloader(object):
 			if dn != '' and not os.path.exists(dn): # dn is already encoded
 				os.makedirs(dn)
 		except (OSError, IOError), err:
-			self.trouble(u'ERROR: unable to create directory ' + unicode(err))
+			self.trouble(u'ERROR: unable to create directory ' + u(err))
 			return
 
 		if self.params.get('writedescription', False):
@@ -623,7 +628,7 @@ class FileDownloader(object):
 					else:
 						# Examine the reported length
 						if (content_length is not None and
-								(resume_len - 100 < long(content_length) < resume_len + 100)):
+								(resume_len - 100 < int(content_length) < resume_len + 100)):
 							# The file had already been fully downloaded.
 							# Explanation to the above condition: in issue #175 it was revealed that
 							# YouTube sometimes adds or removes a few bytes from the end of the file,
@@ -650,10 +655,10 @@ class FileDownloader(object):
 
 		data_len = data.info().get('Content-length', None)
 		if data_len is not None:
-			data_len = long(data_len) + resume_len
+			data_len = int(data_len) + resume_len
 		data_len_str = self.format_bytes(data_len)
 		byte_counter = 0 + resume_len
-		block_size = 1024
+		block_size = self.params.get('buffersize', 1024)
 		start = time.time()
 		while True:
 			# Download and write
@@ -679,7 +684,8 @@ class FileDownloader(object):
 			except (IOError, OSError), err:
 				self.trouble(u'\nERROR: unable to write data: %s' % str(err))
 				return False
-			block_size = self.best_block_size(after - before, len(data_block))
+			if not self.params.get('noresizebuffer', False):
+				block_size = self.best_block_size(after - before, len(data_block))
 
 			# Progress message
 			speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
@@ -699,7 +705,7 @@ class FileDownloader(object):
 		stream.close()
 		self.report_finish()
 		if data_len is not None and byte_counter != data_len:
-			raise ContentTooShortError(byte_counter, long(data_len))
+			raise ContentTooShortError(byte_counter, int(data_len))
 		self.try_rename(tmpfilename, filename)
 
 		# Update file modification time
diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py
index b730b5773..3b5be1d42 100644
--- a/youtube_dl/InfoExtractors.py
+++ b/youtube_dl/InfoExtractors.py
@@ -253,7 +253,7 @@ class YoutubeIE(InfoExtractor):
 				else:
 					raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 			except (IOError, netrc.NetrcParseError), err:
-				self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
+				self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % u(err))
 				return
 
 		# Set language
@@ -262,7 +262,7 @@ class YoutubeIE(InfoExtractor):
 			self.report_lang()
 			urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err))
+			self._downloader.to_stderr(u'WARNING: unable to set language: %s' % u(err))
 			return
 
 		# No authentication to be performed
@@ -285,7 +285,7 @@ class YoutubeIE(InfoExtractor):
 				self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 				return
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
+			self._downloader.to_stderr(u'WARNING: unable to log in: %s' % u(err))
 			return
 
 		# Confirm age
@@ -298,7 +298,7 @@ class YoutubeIE(InfoExtractor):
 			self.report_age_confirmation()
 			age_results = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to confirm age: %s' % u(err))
 			return
 
 	def _real_extract(self, url):
@@ -320,7 +320,7 @@ class YoutubeIE(InfoExtractor):
 		try:
 			video_webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
 			return
 
 		# Attempt to extract SWF player URL
@@ -342,7 +342,7 @@ class YoutubeIE(InfoExtractor):
 				if 'token' in video_info:
 					break
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
+				self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % u(err))
 				return
 		if 'token' not in video_info:
 			if 'reason' in video_info:
@@ -405,7 +405,7 @@ class YoutubeIE(InfoExtractor):
 				try:
 					srt_list = urllib2.urlopen(request).read()
 				except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-					raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
+					raise Trouble(u'WARNING: unable to download video subtitles: %s' % u(err))
 				srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list)
 				srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list)
 				if not srt_lang_list:
@@ -422,7 +422,7 @@ class YoutubeIE(InfoExtractor):
 				try:
 					srt_xml = urllib2.urlopen(request).read()
 				except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-					raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
+					raise Trouble(u'WARNING: unable to download video subtitles: %s' % u(err))
 				if not srt_xml:
 					raise Trouble(u'WARNING: unable to download video subtitles')
 				video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
@@ -544,7 +544,7 @@ class MetacafeIE(InfoExtractor):
 			self.report_disclaimer()
 			disclaimer = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % u(err))
 			return
 
 		# Confirm age
@@ -557,7 +557,7 @@ class MetacafeIE(InfoExtractor):
 			self.report_age_confirmation()
 			disclaimer = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to confirm age: %s' % u(err))
 			return
 
 	def _real_extract(self, url):
@@ -581,7 +581,7 @@ class MetacafeIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % u(err))
 			return
 
 		# Extract URL, uploader and title from webpage
@@ -672,7 +672,7 @@ class DailymotionIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % u(err))
 			return
 
 		# Extract URL, uploader and title from webpage
@@ -768,7 +768,7 @@ class GoogleIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
 			return
 
 		# Extract URL, uploader, and title from webpage
@@ -807,7 +807,7 @@ class GoogleIE(InfoExtractor):
 			try:
 				webpage = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+				self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
 				return
 			mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
 			if mobj is None:
@@ -861,7 +861,7 @@ class PhotobucketIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
 			return
 
 		# Extract URL, uploader, and title from webpage
@@ -929,7 +929,7 @@ class YahooIE(InfoExtractor):
 			try:
 				webpage = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+				self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
 				return
 
 			mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
@@ -953,7 +953,7 @@ class YahooIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
 			return
 
 		# Extract uploader and title from webpage
@@ -1011,7 +1011,7 @@ class YahooIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
 			return
 
 		# Extract media URL from playlist XML
@@ -1067,7 +1067,7 @@ class VimeoIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
 			return
 
 		# Now we begin extracting as much information as we can from what we
@@ -1147,6 +1147,143 @@ class VimeoIE(InfoExtractor):
 		}]
 
 
+class ArteTvIE(InfoExtractor):
+	"""arte.tv information extractor."""
+
+	_VALID_URL = r'(?:http://)?videos\.arte\.tv/(?:fr|de)/videos/.*'
+	_LIVE_URL = r'index-[0-9]+\.html$'
+
+	IE_NAME = u'arte.tv'
+
+	def __init__(self, downloader=None):
+		InfoExtractor.__init__(self, downloader)
+
+	def report_download_webpage(self, video_id):
+		"""Report webpage download."""
+		self._downloader.to_screen(u'[arte.tv] %s: Downloading webpage' % video_id)
+
+	def report_extraction(self, video_id):
+		"""Report information extraction."""
+		self._downloader.to_screen(u'[arte.tv] %s: Extracting information' % video_id)
+
+	def fetch_webpage(self, url):
+		self._downloader.increment_downloads()
+		request = urllib2.Request(url)
+		try:
+			self.report_download_webpage(url)
+			webpage = urllib2.urlopen(request).read()
+		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
+			return
+		except ValueError, err:
+			self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
+			return
+		return webpage
+
+	def grep_webpage(self, url, regex, regexFlags, matchTuples):
+		page = self.fetch_webpage(url)
+		mobj = re.search(regex, page, regexFlags)
+		info = {}
+
+		if mobj is None:
+			self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
+			return
+
+		for (i, key, err) in matchTuples:
+			if mobj.group(i) is None:
+				self._downloader.trouble(err)
+				return
+			else:
+				info[key] = mobj.group(i)
+
+		return info
+
+	def extractLiveStream(self, url):
+		video_lang = url.split('/')[-4]
+		info = self.grep_webpage(
+			url,
+			r'src="(.*?/videothek_js.*?\.js)',
+			0,
+			[
+				(1, 'url', u'ERROR: Invalid URL: %s' % url)
+			]
+		)
+		http_host = url.split('/')[2]
+		next_url = 'http://%s%s' % (http_host, urllib.unquote(info.get('url')))
+		info = self.grep_webpage(
+			next_url,
+			r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' +
+				'(http://.*?\.swf).*?' +
+				'(rtmp://.*?)\'',
+			re.DOTALL,
+			[
+				(1, 'path',   u'ERROR: could not extract video path: %s' % url),
+				(2, 'player', u'ERROR: could not extract video player: %s' % url),
+				(3, 'url',    u'ERROR: could not extract video url: %s' % url)
+			]
+		)
+		video_url = u'%s/%s' % (info.get('url'), info.get('path'))
+
+	def extractPlus7Stream(self, url):
+		video_lang = url.split('/')[-3]
+		info = self.grep_webpage(
+			url,
+			r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)',
+			0,
+			[
+				(1, 'url', u'ERROR: Invalid URL: %s' % url)
+			]
+		)
+		next_url = urllib.unquote(info.get('url'))
+		info = self.grep_webpage(
+			next_url,
+			r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang,
+			0,
+			[
+				(1, 'url', u'ERROR: Could not find <video> tag: %s' % url)
+			]
+		)
+		next_url = urllib.unquote(info.get('url'))
+
+		info = self.grep_webpage(
+			next_url,
+			r'<video id="(.*?)".*?>.*?' +
+				'<name>(.*?)</name>.*?' +
+				'<dateVideo>(.*?)</dateVideo>.*?' +
+				'<url quality="hd">(.*?)</url>',
+			re.DOTALL,
+			[
+				(1, 'id',    u'ERROR: could not extract video id: %s' % url),
+				(2, 'title', u'ERROR: could not extract video title: %s' % url),
+				(3, 'date',  u'ERROR: could not extract video date: %s' % url),
+				(4, 'url',   u'ERROR: could not extract video url: %s' % url)
+			]
+		)
+
+		return {
+			'id':           info.get('id'),
+			'url':          urllib.unquote(info.get('url')),
+			'uploader':     u'arte.tv',
+			'upload_date':  info.get('date'),
+			'title':        info.get('title'),
+			'ext':          u'mp4',
+			'format':       u'NA',
+			'player_url':   None,
+		}
+
+	def _real_extract(self, url):
+		video_id = url.split('/')[-1]
+		self.report_extraction(video_id)
+
+		if re.search(self._LIVE_URL, video_id) is not None:
+			self.extractLiveStream(url)
+			return
+		else:
+			info = self.extractPlus7Stream(url)
+
+		return [info]
+
+
 class GenericIE(InfoExtractor):
 	"""Generic last-resort information extractor."""
 
@@ -1232,7 +1369,7 @@ class GenericIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
 			return
 		except ValueError, err:
 			# since this is the last-resort InfoExtractor, if
@@ -1324,7 +1461,7 @@ class YoutubeSearchIE(InfoExtractor):
 			return
 		else:
 			try:
-				n = long(prefix)
+				n = int(prefix)
 				if n <= 0:
 					self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
 					return
@@ -1351,7 +1488,7 @@ class YoutubeSearchIE(InfoExtractor):
 			try:
 				data = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err))
+				self._downloader.trouble(u'ERROR: unable to download API page: %s' % u(err))
 				return
 			api_response = json.loads(data)['data']
 
@@ -1402,7 +1539,7 @@ class GoogleSearchIE(InfoExtractor):
 			return
 		else:
 			try:
-				n = long(prefix)
+				n = int(prefix)
 				if n <= 0:
 					self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
 					return
@@ -1428,7 +1565,7 @@ class GoogleSearchIE(InfoExtractor):
 			try:
 				page = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
 				return
 
 			# Extract video identifiers
@@ -1484,7 +1621,7 @@ class YahooSearchIE(InfoExtractor):
 			return
 		else:
 			try:
-				n = long(prefix)
+				n = int(prefix)
 				if n <= 0:
 					self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
 					return
@@ -1511,7 +1648,7 @@ class YahooSearchIE(InfoExtractor):
 			try:
 				page = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
 				return
 
 			# Extract video identifiers
@@ -1581,7 +1718,7 @@ class YoutubePlaylistIE(InfoExtractor):
 			try:
 				page = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
 				return
 
 			# Extract video identifiers
@@ -1638,7 +1775,7 @@ class YoutubeChannelIE(InfoExtractor):
 			try:
 				page = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
 				return
 
 			# Extract video identifiers
@@ -1701,7 +1838,7 @@ class YoutubeUserIE(InfoExtractor):
 			try:
 				page = urllib2.urlopen(request).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
 				return
 
 			# Extract video identifiers
@@ -1773,7 +1910,7 @@ class BlipTVUserIE(InfoExtractor):
 			mobj = re.search(r'data-users-id="([^"]+)"', page)
 			page_base = page_base % mobj.group(1)
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
 			return
 
 
@@ -1861,7 +1998,7 @@ class DepositFilesIE(InfoExtractor):
 			self.report_download_webpage(file_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % u(err))
 			return
 
 		# Search for the real file URL
@@ -1977,7 +2114,7 @@ class FacebookIE(InfoExtractor):
 				else:
 					raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 			except (IOError, netrc.NetrcParseError), err:
-				self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
+				self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % u(err))
 				return
 
 		if useremail is None:
@@ -1997,7 +2134,7 @@ class FacebookIE(InfoExtractor):
 				self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
 				return
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
+			self._downloader.to_stderr(u'WARNING: unable to log in: %s' % u(err))
 			return
 
 	def _real_extract(self, url):
@@ -2014,7 +2151,7 @@ class FacebookIE(InfoExtractor):
 			page = urllib2.urlopen(request)
 			video_webpage = page.read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
 			return
 
 		# Start extracting information
@@ -2149,13 +2286,13 @@ class BlipTVIE(InfoExtractor):
 					'urlhandle': urlh
 				}
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % u(err))
 			return
 		if info is None: # Regular URL
 			try:
 				json_code = urlh.read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err))
+				self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % u(err))
 				return
 
 			try:
@@ -2223,7 +2360,7 @@ class MyVideoIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
 			return
 
 		self.report_extraction(video_id)
@@ -2320,7 +2457,7 @@ class ComedyCentralIE(InfoExtractor):
 			htmlHandle = urllib2.urlopen(req)
 			html = htmlHandle.read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
+			self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
 			return
 		if dlNewest:
 			url = htmlHandle.geturl()
@@ -2353,7 +2490,7 @@ class ComedyCentralIE(InfoExtractor):
 			urlHandle = urllib2.urlopen(playerUrl_raw)
 			playerUrl = urlHandle.geturl()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to find out player URL: ' + unicode(err))
+			self._downloader.trouble(u'ERROR: unable to find out player URL: ' + u(err))
 			return
 
 		uri = mMovieParams[0][1]
@@ -2362,7 +2499,7 @@ class ComedyCentralIE(InfoExtractor):
 		try:
 			indexXml = urllib2.urlopen(indexUrl).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download episode index: ' + unicode(err))
+			self._downloader.trouble(u'ERROR: unable to download episode index: ' + u(err))
 			return
 
 		results = []
@@ -2383,7 +2520,7 @@ class ComedyCentralIE(InfoExtractor):
 			try:
 				configXml = urllib2.urlopen(configReq).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
+				self._downloader.trouble(u'ERROR: unable to download webpage: %s' % u(err))
 				return
 
 			cdoc = xml.etree.ElementTree.fromstring(configXml)
@@ -2466,7 +2603,7 @@ class EscapistIE(InfoExtractor):
 			m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
 			webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download webpage: ' + unicode(err))
+			self._downloader.trouble(u'ERROR: unable to download webpage: ' + u(err))
 			return
 
 		descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
@@ -2482,7 +2619,7 @@ class EscapistIE(InfoExtractor):
 		try:
 			configJSON = urllib2.urlopen(configUrl).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download configuration: ' + unicode(err))
+			self._downloader.trouble(u'ERROR: unable to download configuration: ' + u(err))
 			return
 
 		# Technically, it's JavaScript, not JSON
@@ -2491,7 +2628,7 @@ class EscapistIE(InfoExtractor):
 		try:
 			config = json.loads(configJSON)
 		except (ValueError,), err:
-			self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + unicode(err))
+			self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + u(err))
 			return
 
 		playlist = config['playlist']
@@ -2538,7 +2675,7 @@ class CollegeHumorIE(InfoExtractor):
 		try:
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
 			return
 
 		m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
@@ -2559,7 +2696,7 @@ class CollegeHumorIE(InfoExtractor):
 		try:
 			metaXml = urllib2.urlopen(xmlUrl).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % u(err))
 			return
 
 		mdoc = xml.etree.ElementTree.fromstring(metaXml)
@@ -2604,7 +2741,7 @@ class XVideosIE(InfoExtractor):
 		try:
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
 			return
 
 		self.report_extraction(video_id)
@@ -2688,7 +2825,7 @@ class SoundcloudIE(InfoExtractor):
 		try:
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
 			return
 
 		self.report_extraction('%s/%s' % (uploader, slug_title))
@@ -2723,7 +2860,7 @@ class SoundcloudIE(InfoExtractor):
 			try:
 				upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
 			except Exception, e:
-				self._downloader.to_stderr(compat_str(e))
+				self._downloader.to_stderr(u(e))
 
 		# for soundcloud, a request to a cross domain is required for cookies
 		request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
@@ -2765,7 +2902,7 @@ class InfoQIE(InfoExtractor):
 		try:
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
 			return
 
 		self.report_extraction(url)
@@ -2877,7 +3014,7 @@ class MixcloudIE(InfoExtractor):
 			self.report_download_json(file_url)
 			jsonData = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % u(err))
 			return
 
 		# parse JSON
@@ -2956,7 +3093,7 @@ class StanfordOpenClassroomIE(InfoExtractor):
 			try:
 				metaXml = urllib2.urlopen(xmlUrl).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % unicode(err))
+				self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % u(err))
 				return
 			mdoc = xml.etree.ElementTree.fromstring(metaXml)
 			try:
@@ -2980,7 +3117,7 @@ class StanfordOpenClassroomIE(InfoExtractor):
 			try:
 				coursepage = urllib2.urlopen(url).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
+				self._downloader.trouble(u'ERROR: unable to download course info page: ' + u(err))
 				return
 
 			m = re.search('<h1>([^<]+)</h1>', coursepage)
@@ -3019,7 +3156,7 @@ class StanfordOpenClassroomIE(InfoExtractor):
 			try:
 				rootpage = urllib2.urlopen(rootURL).read()
 			except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-				self._downloader.trouble(u'ERROR: unable to download course info page: ' + unicode(err))
+				self._downloader.trouble(u'ERROR: unable to download course info page: ' + u(err))
 				return
 
 			info['title'] = info['id']
@@ -3066,7 +3203,7 @@ class MTVIE(InfoExtractor):
 		try:
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % u(err))
 			return
 
 		mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
@@ -3099,7 +3236,7 @@ class MTVIE(InfoExtractor):
 		try:
 			metadataXml = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % u(err))
 			return
 
 		mdoc = xml.etree.ElementTree.fromstring(metadataXml)
@@ -3187,7 +3324,7 @@ class YoukuIE(InfoExtractor):
 			self.report_download_webpage(video_id)
 			jsondata = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error) as err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
 			return
 
 		self.report_extraction(video_id)
@@ -3361,7 +3498,7 @@ class GooglePlusIE(InfoExtractor):
 		try:
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % u(err))
 			return
 
 		# Extract update date
@@ -3403,7 +3540,7 @@ class GooglePlusIE(InfoExtractor):
 		try:
 			webpage = urllib2.urlopen(request).read()
 		except (urllib2.URLError, httplib.HTTPException, socket.error), err:
-			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
+			self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % u(err))
 			return
 		self.report_extract_vid_page(video_page)
 
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 5fc39184a..54c7d7f72 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -18,10 +18,11 @@ __authors__  = (
 	'Ori Avtalion',
 	'shizeeg',
 	'Filippo Valsorda',
+	'Christian Albrecht',
 	)
 
 __license__ = 'Public Domain'
-__version__ = '2012.11.28'
+__version__ = '2012.11.29'
 
 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
 UPDATE_URL_VERSION = 'https://raw.github.com/rg3/youtube-dl/master/LATEST_VERSION'
@@ -126,9 +127,12 @@ def parseOpts():
 
 		opts = []
 
-		if option._short_opts: opts.append(option._short_opts[0])
-		if option._long_opts: opts.append(option._long_opts[0])
-		if len(opts) > 1: opts.insert(1, ', ')
+		if option._short_opts:
+			opts.append(option._short_opts[0])
+		if option._long_opts:
+			opts.append(option._long_opts[0])
+		if len(opts) > 1:
+			opts.insert(1, ', ')
 
 		if option.takes_value(): opts.append(' %s' % option.metavar)
 
@@ -187,6 +191,11 @@ def parseOpts():
 			dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
 	general.add_option('-R', '--retries',
 			dest='retries', metavar='RETRIES', help='number of retries (default is %default)', default=10)
+	general.add_option('--buffer-size',
+			dest='buffersize', metavar='SIZE', help='size of download buffer (e.g. 1024 or 16k) (default is %default)', default="1024")
+	general.add_option('--no-resize-buffer',
+			action='store_true', dest='noresizebuffer',
+			help='do not automatically adjust the buffer size. By default, the buffer size is automatically resized from an initial value of SIZE.', default=False)
 	general.add_option('--dump-user-agent',
 			action='store_true', dest='dump_user_agent',
 			help='display the current browser identification', default=False)
@@ -362,7 +371,7 @@ def gen_extractors():
 		YoukuIE(),
 		XNXXIE(),
 		GooglePlusIE(),
-
+		ArteTvIE(),
 		GenericIE()
 	]
 
@@ -440,9 +449,14 @@ def _real_main():
 		opts.ratelimit = numeric_limit
 	if opts.retries is not None:
 		try:
-			opts.retries = long(opts.retries)
+			opts.retries = int(opts.retries)
 		except (TypeError, ValueError), err:
 			parser.error(u'invalid retry count specified')
+	if opts.buffersize is not None:
+		numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
+		if numeric_buffersize is None:
+			parser.error(u'invalid buffer size specified')
+		opts.buffersize = numeric_buffersize
 	try:
 		opts.playliststart = int(opts.playliststart)
 		if opts.playliststart <= 0:
@@ -493,6 +507,8 @@ def _real_main():
 		'ratelimit': opts.ratelimit,
 		'nooverwrites': opts.nooverwrites,
 		'retries': opts.retries,
+		'buffersize': opts.buffersize,
+		'noresizebuffer': opts.noresizebuffer,
 		'continuedl': opts.continue_dl,
 		'noprogress': opts.noprogress,
 		'playliststart': opts.playliststart,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4ace22c2f..bde446bcb 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -27,9 +27,9 @@ std_headers = {
 }
 
 try:
-    compat_str = unicode # Python 2
+	u = unicode # Python 2
 except NameError:
-    compat_str = str
+	u = str
 
 def preferredencoding():
 	"""Get preferred encoding.
@@ -37,19 +37,17 @@ def preferredencoding():
 	Returns the best encoding scheme for the system, based on
 	locale.getpreferredencoding() and some further tweaks.
 	"""
-	def yield_preferredencoding():
-		try:
-			pref = locale.getpreferredencoding()
-			u'TEST'.encode(pref)
-		except:
-			pref = 'UTF-8'
-		while True:
-			yield pref
-	return yield_preferredencoding().next()
+	try:
+		pref = locale.getpreferredencoding()
+		u'TEST'.encode(pref)
+	except:
+		pref = 'UTF-8'
+
+	return pref
 
 
 def htmlentity_transform(matchobj):
-	"""Transforms an HTML entity to a Unicode character.
+	"""Transforms an HTML entity to a character.
 
 	This function receives a match object and is intended to be used with
 	the re.sub() function.
@@ -60,7 +58,6 @@ def htmlentity_transform(matchobj):
 	if entity in htmlentitydefs.name2codepoint:
 		return unichr(htmlentitydefs.name2codepoint[entity])
 
-	# Unicode character
 	mobj = re.match(ur'(?u)#(x?\d+)', entity)
 	if mobj is not None:
 		numstr = mobj.group(1)
@@ -69,7 +66,7 @@ def htmlentity_transform(matchobj):
 			numstr = u'0%s' % numstr
 		else:
 			base = 10
-		return unichr(long(numstr, base))
+		return unichr(int(numstr, base))
 
 	# Unknown entity in name, return its literal representation
 	return (u'&%s;' % entity)
@@ -128,8 +125,10 @@ class IDParser(HTMLParser.HTMLParser):
 	handle_decl = handle_pi = unknown_decl = find_startpos
 
 	def get_result(self):
-		if self.result == None: return None
-		if len(self.result) != 3: return None
+		if self.result is None:
+			return None
+		if len(self.result) != 3:
+			return None
 		lines = self.html.split('\n')
 		lines = lines[self.result[1][0]-1:self.result[2][0]]
 		lines[0] = lines[0][self.result[1][1]:]
@@ -208,7 +207,7 @@ def sanitize_filename(s, restricted=False):
 			return '_-' if restricted else ' -'
 		elif char in '\\/|*<>':
 			return '_'
-		if restricted and (char in '&\'' or char.isspace()):
+		if restricted and (char in '!&\'' or char.isspace()):
 			return '_'
 		if restricted and ord(char) > 127:
 			return '_'
@@ -235,7 +234,7 @@ def orderedSet(iterable):
 
 def unescapeHTML(s):
 	"""
-	@param s a string (of type unicode)
+	@param s a string
 	"""
 	assert type(s) == type(u'')
 
@@ -244,7 +243,7 @@ def unescapeHTML(s):
 
 def encodeFilename(s):
 	"""
-	@param s The name of the file (of type unicode)
+	@param s The name of the file
 	"""
 
 	assert type(s) == type(u'')
@@ -316,7 +315,7 @@ class ContentTooShortError(Exception):
 
 class Trouble(Exception):
 	"""Trouble helper exception
-	
+
 	This is an exception to be handled with
 	FileDownloader.trouble
 	"""