Merge remote-tracking branch 'FiloSottille/automation'

10 files changed, 272 insertions, 36 deletions

diff --git a/Makefile b/Makefile
index e2da02e96..625948211 100644
--- a/Makefile
+++ b/Makefile
@@ -1,26 +1,48 @@
-default: update
+all: youtube-dl README.md youtube-dl.1 youtube-dl.bash-completion LATEST_VERSION
 
-update: compile update-readme update-latest
+# TODO: re-add youtube-dl.exe, and make sure it's 1. safe and 2. doesn't need sudo
 
-update-latest:
-	./youtube-dl.dev --version > LATEST_VERSION
+clean:
+	rm -f youtube-dl youtube-dl.exe youtube-dl.1 LATEST_VERSION
 
-update-readme:
-	@options=$$(COLUMNS=80 ./youtube-dl.dev --help | sed -e '1,/.*General Options.*/ d' -e 's/^\W\{2\}\(\w\)/### \1/') && \
-		header=$$(sed -e '/.*## OPTIONS/,$$ d' README.md) && \
-		footer=$$(sed -e '1,/.*## FAQ/ d' README.md) && \
+PREFIX=/usr/local
+install: youtube-dl youtube-dl.1 youtube-dl.bash-completion
+	install -m 755 --owner root --group root youtube-dl $(PREFIX)/bin/
+	install -m 644 --owner root --group root youtube-dl.1 $(PREFIX)/man/man1
+	install -m 644 --owner root --group root youtube-dl.bash-completion /etc/bash_completion.d/youtube-dl
+
+.PHONY: all clean install README.md youtube-dl.bash-completion
+# TODO un-phone README.md and youtube-dl.bash_completion by reading from .in files and generating from them
+
+youtube-dl: youtube_dl/*.py
+	zip --quiet --junk-paths youtube-dl youtube_dl/*.py
+	echo '#!/usr/bin/env python' > youtube-dl
+	cat youtube-dl.zip >> youtube-dl
+	rm youtube-dl.zip
+	chmod a+x youtube-dl
+
+youtube-dl.exe: youtube_dl/*.py
+	bash devscripts/wine-py2exe.sh build_exe.py
+
+README.md: youtube-dl
+	@options=$$(COLUMNS=80 ./youtube-dl --help | sed -e '1,/.*General Options.*/ d' -e 's/^\W\{2\}\(\w\)/## \1/') && \
+		header=$$(sed -e '/.*# OPTIONS/,$$ d' README.md) && \
+		footer=$$(sed -e '1,/.*# FAQ/ d' README.md) && \
 		echo "$${header}" > README.md && \
 		echo >> README.md && \
-		echo '## OPTIONS' >> README.md && \
+		echo '# OPTIONS' >> README.md && \
 		echo "$${options}" >> README.md&& \
 		echo >> README.md && \
-		echo '## FAQ' >> README.md && \
+		echo '# FAQ' >> README.md && \
 		echo "$${footer}" >> README.md
 
-compile:
-	zip --quiet --junk-paths youtube-dl youtube_dl/*.py
-	echo '#!/usr/bin/env python' > youtube-dl
-	cat youtube-dl.zip >> youtube-dl
-	rm youtube-dl.zip
+youtube-dl.1: README.md
+	pandoc -s -w man README.md -o youtube-dl.1
+
+youtube-dl.bash-completion: README.md
+	@options=`egrep -o '(--[a-z-]+) ' README.md | sort -u | xargs echo` && \
+		content=`sed "s/opts=\"[^\"]*\"/opts=\"$${options}\"/g" youtube-dl.bash-completion` && \
+		echo "$${content}" > youtube-dl.bash-completion
 
-.PHONY: default compile update update-latest update-readme
+LATEST_VERSION: youtube-dl
+	./youtube-dl --version > LATEST_VERSION
diff --git a/README.md b/README.md
index f04b96128..05f437db9 100644
--- a/README.md
+++ b/README.md
@@ -1,16 +1,19 @@
-# youtube-dl
+% youtube-dl(1)
 
-## USAGE
-youtube-dl [options] url [url...]
+# NAME
+youtube-dl
 
-## DESCRIPTION
+# SYNOPSIS
+**youtube-dl** [OPTIONS] URL [URL...]
+
+# DESCRIPTION
 **youtube-dl** is a small command-line program to download videos from
 YouTube.com and a few more sites. It requires the Python interpreter, version
 2.x (x being at least 6), and it is not platform specific. It should work in
 your Unix box, in Windows or in Mac OS X. It is released to the public domain,
 which means you can modify it, redistribute it or use it however you like.
 
-## OPTIONS
+# OPTIONS
     -h, --help               print this help text and exit
     --version                print program version and exit
     -U, --update             update this program to latest version
@@ -21,7 +24,7 @@ which means you can modify it, redistribute it or use it however you like.
     --list-extractors        List all supported extractors and the URLs they
                              would handle
 
-### Video Selection:
+## Video Selection:
     --playlist-start NUMBER  playlist video to start at (default is 1)
     --playlist-end NUMBER    playlist video to end at (default is last)
     --match-title REGEX      download only matching titles (regex or caseless
@@ -30,7 +33,7 @@ which means you can modify it, redistribute it or use it however you like.
                              caseless sub-string)
     --max-downloads NUMBER   Abort after downloading NUMBER files
 
-### Filesystem Options:
+## Filesystem Options:
     -t, --title              use title in file name
     -l, --literal            use literal title in file name
     -A, --auto-number        number downloaded files starting from 00000
@@ -53,7 +56,7 @@ which means you can modify it, redistribute it or use it however you like.
     --write-description      write video description to a .description file
     --write-info-json        write video metadata to a .info.json file
 
-### Verbosity / Simulation Options:
+## Verbosity / Simulation Options:
     -q, --quiet              activates quiet mode
     -s, --simulate           do not download the video and do not write anything
                              to disk
@@ -68,7 +71,7 @@ which means you can modify it, redistribute it or use it however you like.
     --console-title          display progress in console titlebar
     -v, --verbose            print various debugging information
 
-### Video Format Options:
+## Video Format Options:
     -f, --format FORMAT      video format code
     --all-formats            download all available video formats
     --prefer-free-formats    prefer free video formats unless a specific one is
@@ -80,12 +83,12 @@ which means you can modify it, redistribute it or use it however you like.
     --srt-lang LANG          language of the closed captions to download
                              (optional) use IETF language tags like 'en'
 
-### Authentication Options:
+## Authentication Options:
     -u, --username USERNAME  account username
     -p, --password PASSWORD  account password
     -n, --netrc              use .netrc authentication data
 
-### Post-processing Options:
+## Post-processing Options:
     --extract-audio          convert video files to audio-only files (requires
                              ffmpeg or avconv and ffprobe or avprobe)
     --audio-format FORMAT    "best", "aac", "vorbis", "mp3", "m4a", or "wav";
@@ -95,7 +98,7 @@ which means you can modify it, redistribute it or use it however you like.
     -k, --keep-video         keeps the video file on disk after the post-
                              processing; the video is erased by default
 
-## FAQ
+# FAQ
 
 ### Can you please put the -b option back?
 
@@ -117,13 +120,13 @@ The URLs youtube-dl outputs require the downloader to have the correct cookies.
 
 youtube has switched to a new video info format in July 2011 which is not supported by old versions of youtube-dl. You can update youtube-dl with `sudo youtube-dl --update`.
 
-## COPYRIGHT
+# COPYRIGHT
 
 youtube-dl is released into the public domain by the copyright holders.
 
 This README file was originally written by Daniel Bolton (<https://github.com/dbbolton>) and is likewise released into the public domain.
 
-## BUGS
+# BUGS
 
 Bugs and suggestions should be reported at: <https://github.com/rg3/youtube-dl/issues>
 
diff --git a/devscripts/posix-locale.sh b/devscripts/posix-locale.sh
index 0aa7a592d..0aa7a592d 100644..100755
--- a/devscripts/posix-locale.sh
+++ b/devscripts/posix-locale.sh
diff --git a/devscripts/release.sh b/devscripts/release.sh
new file mode 100755
index 000000000..75f8ec8eb
--- /dev/null
+++ b/devscripts/release.sh
@@ -0,0 +1,11 @@
+#! /bin/bash
+
+if [ -z "$1" ]; then echo "ERROR: specify version number like this: $0 1994.09.06"; exit 1; fi
+version="$1"
+if [ ! -z "`git tag | grep "$version"`" ]; then echo 'ERROR: version already present'; exit 1; fi
+if [ ! -z "`git status --porcelain`" ]; then echo 'ERROR: the working directory is not clean; commit or stash changes'; exit 1; fi
+sed -i "s/__version__ = '.*'/__version__ = '$version'/" youtube_dl/__init__.py
+make all
+git add -A
+git commit -m "release $version"
+git tag -m "Release $version" "$version"
\ No newline at end of file
diff --git a/devscripts/wine-py2exe.sh b/devscripts/wine-py2exe.sh
index dc2d6501a..dc2d6501a 100644..100755
--- a/devscripts/wine-py2exe.sh
+++ b/devscripts/wine-py2exe.sh
diff --git a/youtube-dl b/youtube-dl
index 21e604a25..9c78766ff 100755
--- a/youtube-dl
+++ b/youtube-dl
diff --git a/youtube-dl.1 b/youtube-dl.1
new file mode 100644
index 000000000..09bb843f4
--- /dev/null
+++ b/youtube-dl.1
@@ -0,0 +1,192 @@
+.TH youtube-dl 1 "" 
+.SH NAME
+.PP
+youtube-dl
+.SH SYNOPSIS
+.PP
+\f[B]youtube-dl\f[] [OPTIONS] URL [URL...]
+.SH DESCRIPTION
+.PP
+\f[B]youtube-dl\f[] is a small command-line program to download videos
+from YouTube.com and a few more sites.
+It requires the Python interpreter, version 2.x (x being at least 6),
+and it is not platform specific.
+It should work in your Unix box, in Windows or in Mac OS X.
+It is released to the public domain, which means you can modify it,
+redistribute it or use it however you like.
+.SH OPTIONS
+.IP
+.nf
+\f[C]
+-h,\ --help\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ this\ help\ text\ and\ exit
+--version\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ print\ program\ version\ and\ exit
+-U,\ --update\ \ \ \ \ \ \ \ \ \ \ \ \ update\ this\ program\ to\ latest\ version
+-i,\ --ignore-errors\ \ \ \ \ \ continue\ on\ download\ errors
+-r,\ --rate-limit\ LIMIT\ \ \ download\ rate\ limit\ (e.g.\ 50k\ or\ 44.6m)
+-R,\ --retries\ RETRIES\ \ \ \ number\ of\ retries\ (default\ is\ 10)
+--dump-user-agent\ \ \ \ \ \ \ \ display\ the\ current\ browser\ identification
+--list-extractors\ \ \ \ \ \ \ \ List\ all\ supported\ extractors\ and\ the\ URLs\ they
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ would\ handle
+\f[]
+.fi
+.SS Video Selection:
+.IP
+.nf
+\f[C]
+--playlist-start\ NUMBER\ \ playlist\ video\ to\ start\ at\ (default\ is\ 1)
+--playlist-end\ NUMBER\ \ \ \ playlist\ video\ to\ end\ at\ (default\ is\ last)
+--match-title\ REGEX\ \ \ \ \ \ download\ only\ matching\ titles\ (regex\ or\ caseless
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ sub-string)
+--reject-title\ REGEX\ \ \ \ \ skip\ download\ for\ matching\ titles\ (regex\ or
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ caseless\ sub-string)
+--max-downloads\ NUMBER\ \ \ Abort\ after\ downloading\ NUMBER\ files
+\f[]
+.fi
+.SS Filesystem Options:
+.IP
+.nf
+\f[C]
+-t,\ --title\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ title\ in\ file\ name
+-l,\ --literal\ \ \ \ \ \ \ \ \ \ \ \ use\ literal\ title\ in\ file\ name
+-A,\ --auto-number\ \ \ \ \ \ \ \ number\ downloaded\ files\ starting\ from\ 00000
+-o,\ --output\ TEMPLATE\ \ \ \ output\ filename\ template.\ Use\ %(stitle)s\ to\ get\ the
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ title,\ %(uploader)s\ for\ the\ uploader\ name,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(autonumber)s\ to\ get\ an\ automatically\ incremented
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ number,\ %(ext)s\ for\ the\ filename\ extension,
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %(upload_date)s\ for\ the\ upload\ date\ (YYYYMMDD),\ and
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ %%\ for\ a\ literal\ percent.\ Use\ -\ to\ output\ to
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ stdout.
+-a,\ --batch-file\ FILE\ \ \ \ file\ containing\ URLs\ to\ download\ (\[aq]-\[aq]\ for\ stdin)
+-w,\ --no-overwrites\ \ \ \ \ \ do\ not\ overwrite\ files
+-c,\ --continue\ \ \ \ \ \ \ \ \ \ \ resume\ partially\ downloaded\ files
+--no-continue\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ resume\ partially\ downloaded\ files\ (restart
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ from\ beginning)
+--cookies\ FILE\ \ \ \ \ \ \ \ \ \ \ file\ to\ read\ cookies\ from\ and\ dump\ cookie\ jar\ in
+--no-part\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ .part\ files
+--no-mtime\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ do\ not\ use\ the\ Last-modified\ header\ to\ set\ the\ file
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ modification\ time
+--write-description\ \ \ \ \ \ write\ video\ description\ to\ a\ .description\ file
+--write-info-json\ \ \ \ \ \ \ \ write\ video\ metadata\ to\ a\ .info.json\ file
+\f[]
+.fi
+.SS Verbosity / Simulation Options:
+.IP
+.nf
+\f[C]
+-q,\ --quiet\ \ \ \ \ \ \ \ \ \ \ \ \ \ activates\ quiet\ mode
+-s,\ --simulate\ \ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video\ and\ do\ not\ write\ anything
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ to\ disk
+--skip-download\ \ \ \ \ \ \ \ \ \ do\ not\ download\ the\ video
+-g,\ --get-url\ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ URL
+-e,\ --get-title\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ title
+--get-thumbnail\ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ thumbnail\ URL
+--get-description\ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ video\ description
+--get-filename\ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ filename
+--get-format\ \ \ \ \ \ \ \ \ \ \ \ \ simulate,\ quiet\ but\ print\ output\ format
+--no-progress\ \ \ \ \ \ \ \ \ \ \ \ do\ not\ print\ progress\ bar
+--console-title\ \ \ \ \ \ \ \ \ \ display\ progress\ in\ console\ titlebar
+-v,\ --verbose\ \ \ \ \ \ \ \ \ \ \ \ print\ various\ debugging\ information
+\f[]
+.fi
+.SS Video Format Options:
+.IP
+.nf
+\f[C]
+-f,\ --format\ FORMAT\ \ \ \ \ \ video\ format\ code
+--all-formats\ \ \ \ \ \ \ \ \ \ \ \ download\ all\ available\ video\ formats
+--prefer-free-formats\ \ \ \ prefer\ free\ video\ formats\ unless\ a\ specific\ one\ is
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ requested
+--max-quality\ FORMAT\ \ \ \ \ highest\ quality\ format\ to\ download
+-F,\ --list-formats\ \ \ \ \ \ \ list\ all\ available\ formats\ (currently\ youtube\ only)
+--write-srt\ \ \ \ \ \ \ \ \ \ \ \ \ \ write\ video\ closed\ captions\ to\ a\ .srt\ file
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (currently\ youtube\ only)
+--srt-lang\ LANG\ \ \ \ \ \ \ \ \ \ language\ of\ the\ closed\ captions\ to\ download
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ (optional)\ use\ IETF\ language\ tags\ like\ \[aq]en\[aq]
+\f[]
+.fi
+.SS Authentication Options:
+.IP
+.nf
+\f[C]
+-u,\ --username\ USERNAME\ \ account\ username
+-p,\ --password\ PASSWORD\ \ account\ password
+-n,\ --netrc\ \ \ \ \ \ \ \ \ \ \ \ \ \ use\ .netrc\ authentication\ data
+\f[]
+.fi
+.SS Post-processing Options:
+.IP
+.nf
+\f[C]
+--extract-audio\ \ \ \ \ \ \ \ \ \ convert\ video\ files\ to\ audio-only\ files\ (requires
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ ffmpeg\ or\ avconv\ and\ ffprobe\ or\ avprobe)
+--audio-format\ FORMAT\ \ \ \ "best",\ "aac",\ "vorbis",\ "mp3",\ "m4a",\ or\ "wav";
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ best\ by\ default
+--audio-quality\ QUALITY\ \ ffmpeg/avconv\ audio\ bitrate\ specification,\ 128k\ by
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ default
+-k,\ --keep-video\ \ \ \ \ \ \ \ \ keeps\ the\ video\ file\ on\ disk\ after\ the\ post-
+\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ processing;\ the\ video\ is\ erased\ by\ default
+\f[]
+.fi
+.SH FAQ
+.SS Can you please put the -b option back?
+.PP
+Most people asking this question are not aware that youtube-dl now
+defaults to downloading the highest available quality as reported by
+YouTube, which will be 1080p or 720p in some cases, so you no longer
+need the -b option.
+For some specific videos, maybe YouTube does not report them to be
+available in a specific high quality format you\[aq]\[aq]re interested
+in.
+In that case, simply request it with the -f option and youtube-dl will
+try to download it.
+.SS I get HTTP error 402 when trying to download a video. What\[aq]s
+this?
+.PP
+Apparently YouTube requires you to pass a CAPTCHA test if you download
+too much.
+We\[aq]\[aq]re considering to provide a way to let you solve the
+CAPTCHA (https://github.com/rg3/youtube-dl/issues/154), but at the
+moment, your best course of action is pointing a webbrowser to the
+youtube URL, solving the CAPTCHA, and restart youtube-dl.
+.SS I have downloaded a video but how can I play it?
+.PP
+Once the video is fully downloaded, use any video player, such as
+vlc (http://www.videolan.org) or mplayer (http://www.mplayerhq.hu/).
+.SS The links provided by youtube-dl -g are not working anymore
+.PP
+The URLs youtube-dl outputs require the downloader to have the correct
+cookies.
+Use the \f[C]--cookies\f[] option to write the required cookies into a
+file, and advise your downloader to read cookies from that file.
+Some sites also require a common user agent to be used, use
+\f[C]--dump-user-agent\f[] to see the one in use by youtube-dl.
+.SS ERROR: no fmt_url_map or conn information found in video info
+.PP
+youtube has switched to a new video info format in July 2011 which is
+not supported by old versions of youtube-dl.
+You can update youtube-dl with \f[C]sudo\ youtube-dl\ --update\f[].
+.SH COPYRIGHT
+.PP
+youtube-dl is released into the public domain by the copyright holders.
+.PP
+This README file was originally written by Daniel Bolton
+(<https://github.com/dbbolton>) and is likewise released into the public
+domain.
+.SH BUGS
+.PP
+Bugs and suggestions should be reported at:
+<https://github.com/rg3/youtube-dl/issues>
+.PP
+Please include:
+.IP \[bu] 2
+Your exact command line, like
+\f[C]youtube-dl\ -t\ "http://www.youtube.com/watch?v=uHlDtZ6Oc3s&feature=channel_video_title"\f[].
+A common mistake is not to escape the \f[C]&\f[].
+Putting URLs in quotes should solve this problem.
+.IP \[bu] 2
+The output of \f[C]youtube-dl\ --version\f[]
+.IP \[bu] 2
+The output of \f[C]python\ --version\f[]
+.IP \[bu] 2
+The name and version of your Operating System ("Ubuntu 11.04 x64" or
+"Windows 7 x64" is usually enough).
diff --git a/youtube-dl.bash-completion b/youtube-dl.bash-completion
new file mode 100644
index 000000000..8704d0fd3
--- /dev/null
+++ b/youtube-dl.bash-completion
@@ -0,0 +1,14 @@
+__youtube-dl()
+{
+    local cur prev opts
+    COMPREPLY=()
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    opts="--all-formats --audio-format --audio-quality --auto-number --batch-file --console-title --continue --cookies --dump-user-agent --extract-audio --format --get-description --get-filename --get-format --get-thumbnail --get-title --get-url --help --ignore-errors --keep-video --list-extractors --list-formats --literal --match-title --max-downloads --max-quality --netrc --no-continue --no-mtime --no-overwrites --no-part --no-progress --output --password --playlist-end --playlist-start --prefer-free-formats --quiet --rate-limit --reject-title --retries --simulate --skip-download --srt-lang --title --update --username --verbose --version --write-description --write-info-json --write-srt"
+
+    if [[ ${cur} == * ]] ; then
+        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+        return 0
+    fi
+}
+
+complete -F __youtube-dl youtube-dl
diff --git a/youtube-dl.dev b/youtube-dl.dev
deleted file mode 100755
index 473b7f42d..000000000
--- a/youtube-dl.dev
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-import youtube_dl
-
-youtube_dl.main()
diff --git a/youtube-dl.exe b/youtube-dl.exe
deleted file mode 100755
index fbed2841a..000000000
--- a/youtube-dl.exe
+++ /dev/null