aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--ci/retry/README.md123
-rwxr-xr-xci/retry/retry163
2 files changed, 286 insertions, 0 deletions
diff --git a/ci/retry/README.md b/ci/retry/README.md
new file mode 100644
index 0000000000..ead2271f64
--- /dev/null
+++ b/ci/retry/README.md
@@ -0,0 +1,123 @@
+retry - The command line retry tool
+------------------------------------------
+
+Retry any shell command with exponential backoff or constant delay.
+
+### Instructions
+
+Install:
+
+retry is a shell script, so drop it somewhere and make sure it's added to your $PATH. Or you can use the following one-liner:
+
+```sh
+sudo sh -c "curl https://raw.githubusercontent.com/kadwanev/retry/master/retry -o /usr/local/bin/retry && chmod +x /usr/local/bin/retry"
+```
+
+If you're on OS X, retry is also on Homebrew:
+
+```
+brew pull 27283
+brew install retry
+```
+Not popular enough for homebrew-core. Please star this project to help.
+
+### Usage
+
+Help:
+
+`retry -?`
+
+ Usage: retry [options] -- execute command
+ -h, -?, --help
+ -v, --verbose Verbose output
+ -t, --tries=# Set max retries: Default 10
+ -s, --sleep=secs Constant sleep amount (seconds)
+ -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3
+ -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60
+ -f, --fail="script +cmds" Fail Script: run in case of final failure
+
+### Examples
+
+No problem:
+
+`retry echo u work good`
+
+ u work good
+
+Test functionality:
+
+`retry 'echo "y u no work"; false'`
+
+ y u no work
+ Before retry #1: sleeping 0.3 seconds
+ y u no work
+ Before retry #2: sleeping 0.6 seconds
+ y u no work
+ Before retry #3: sleeping 1.2 seconds
+ y u no work
+ Before retry #4: sleeping 2.4 seconds
+ y u no work
+ Before retry #5: sleeping 4.8 seconds
+ y u no work
+ Before retry #6: sleeping 9.6 seconds
+ y u no work
+ Before retry #7: sleeping 19.2 seconds
+ y u no work
+ Before retry #8: sleeping 38.4 seconds
+ y u no work
+ Before retry #9: sleeping 60.0 seconds
+ y u no work
+ Before retry #10: sleeping 60.0 seconds
+ y u no work
+ etc..
+
+Limit retries:
+
+`retry -t 4 'echo "y u no work"; false'`
+
+ y u no work
+ Before retry #1: sleeping 0.3 seconds
+ y u no work
+ Before retry #2: sleeping 0.6 seconds
+ y u no work
+ Before retry #3: sleeping 1.2 seconds
+ y u no work
+ Before retry #4: sleeping 2.4 seconds
+ y u no work
+ Retries exhausted
+
+Bad command:
+
+`retry poop`
+
+ bash: poop: command not found
+
+Fail command:
+
+`retry -t 3 -f 'echo "oh poopsickles"' 'echo "y u no work"; false'`
+
+ y u no work
+ Before retry #1: sleeping 0.3 seconds
+ y u no work
+ Before retry #2: sleeping 0.6 seconds
+ y u no work
+ Before retry #3: sleeping 1.2 seconds
+ y u no work
+ Retries exhausted, running fail script
+ oh poopsickles
+
+Last attempt passed:
+
+`retry -t 3 -- 'if [ $RETRY_ATTEMPT -eq 3 ]; then echo Passed at attempt $RETRY_ATTEMPT; true; else echo Failed at attempt $RETRY_ATTEMPT; false; fi;'`
+
+ Failed at attempt 0
+ Before retry #1: sleeping 0.3 seconds
+ Failed at attempt 1
+ Before retry #2: sleeping 0.6 seconds
+ Failed at attempt 2
+ Before retry #3: sleeping 1.2 seconds
+ Passed at attempt 3
+
+### License
+
+Apache 2.0 - go nuts
diff --git a/ci/retry/retry b/ci/retry/retry
new file mode 100755
index 0000000000..03dc180ef2
--- /dev/null
+++ b/ci/retry/retry
@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+
+GETOPT_BIN=$IN_GETOPT_BIN
+GETOPT_BIN=${GETOPT_BIN:-getopt}
+
+__sleep_amount() {
+ if [ -n "$constant_sleep" ]; then
+ sleep_time=$constant_sleep
+ else
+ #TODO: check for awk
+ #TODO: check if user would rather use one of the other possible dependencies: python, ruby, bc, dc
+ sleep_time=`awk "BEGIN {t = $min_sleep * $(( (1<<($attempts -1)) )); print (t > $max_sleep ? $max_sleep : t)}"`
+ fi
+}
+
+__log_out() {
+ echo "$1" 1>&2
+}
+
+# Paramters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND
+retry()
+{
+ local max_tries="$1"; shift
+ local min_sleep="$1"; shift
+ local max_sleep="$1"; shift
+ local constant_sleep="$1"; shift
+ local fail_script="$1"; shift
+ if [ -n "$VERBOSE" ]; then
+ __log_out "Retry Parameters: max_tries=$max_tries min_sleep=$min_sleep max_sleep=$max_sleep constant_sleep=$constant_sleep"
+ if [ -n "$fail_script" ]; then __log_out "Fail script: $fail_script"; fi
+ __log_out ""
+ __log_out "Execution Command: $*"
+ __log_out ""
+ fi
+
+ local attempts=0
+ local return_code=1
+
+
+ while [[ $return_code -ne 0 && $attempts -le $max_tries ]]; do
+ if [ $attempts -gt 0 ]; then
+ __sleep_amount
+ __log_out "Before retry #$attempts: sleeping $sleep_time seconds"
+ sleep $sleep_time
+ fi
+
+ P="$1"
+ for param in "${@:2}"; do P="$P '$param'"; done
+ #TODO: replace single quotes in each arg with '"'"' ?
+ export RETRY_ATTEMPT=$attempts
+ bash -c "$P"
+ return_code=$?
+ #__log_out "Process returned $return_code on attempt $attempts"
+ if [ $return_code -eq 127 ]; then
+ # command not found
+ exit $return_code
+ elif [ $return_code -ne 0 ]; then
+ attempts=$[$attempts +1]
+ fi
+ done
+
+ if [ $attempts -gt $max_tries ]; then
+ if [ -n "$fail_script" ]; then
+ __log_out "Retries exhausted, running fail script"
+ eval $fail_script
+ else
+ __log_out "Retries exhausted"
+ fi
+ fi
+
+ exit $return_code
+}
+
+# If we're being sourced, don't worry about such things
+if [ "$BASH_SOURCE" == "$0" ]; then
+ # Prints the help text
+ help()
+ {
+ local retry=$(basename $0)
+ cat <<EOF
+Usage: $retry [options] -- execute command
+ -h, -?, --help
+ -v, --verbose Verbose output
+ -t, --tries=# Set max retries: Default 10
+ -s, --sleep=secs Constant sleep amount (seconds)
+ -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3
+ -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60
+ -f, --fail="script +cmds" Fail Script: run in case of final failure
+EOF
+ }
+
+ # show help for no arguments if stdin is a terminal
+ if { [ -z "$1" ] && [ -t 0 ] ; } || [ "$1" == '-h' ] || [ "$1" == '-?' ] || [ "$1" == '--help' ]
+ then
+ help
+ exit 0
+ fi
+
+ $GETOPT_BIN --test > /dev/null
+ if [[ $? -ne 4 ]]; then
+ echo "I’m sorry, 'getopt --test' failed in this environment. Please load GNU getopt."
+ exit 1
+ fi
+
+ OPTIONS=vt:s:m:x:f:
+ LONGOPTIONS=verbose,tries:,sleep:,min:,max:,fail:
+
+ PARSED=$($GETOPT_BIN --options="$OPTIONS" --longoptions="$LONGOPTIONS" --name "$0" -- "$@")
+ if [[ $? -ne 0 ]]; then
+ # e.g. $? == 1
+ # then getopt has complained about wrong arguments to stdout
+ exit 2
+ fi
+ # read getopt’s output this way to handle the quoting right:
+ eval set -- "$PARSED"
+
+ max_tries=10
+ min_sleep=0.3
+ max_sleep=60.0
+ constant_sleep=
+ fail_script=
+
+ # now enjoy the options in order and nicely split until we see --
+ while true; do
+ case "$1" in
+ -v|--verbose)
+ VERBOSE=true
+ shift
+ ;;
+ -t|--tries)
+ max_tries="$2"
+ shift 2
+ ;;
+ -s|--sleep)
+ constant_sleep="$2"
+ shift 2
+ ;;
+ -m|--min)
+ min_sleep="$2"
+ shift 2
+ ;;
+ -x|--max)
+ max_sleep="$2"
+ shift 2
+ ;;
+ -f|--fail)
+ fail_script="$2"
+ shift 2
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ echo "Programming error"
+ exit 3
+ ;;
+ esac
+ done
+
+ retry "$max_tries" "$min_sleep" "$max_sleep" "$constant_sleep" "$fail_script" "$@"
+
+fi