diff options
| -rw-r--r-- | ci/retry/README.md | 123 | ||||
| -rwxr-xr-x | ci/retry/retry | 163 |
2 files changed, 286 insertions, 0 deletions
diff --git a/ci/retry/README.md b/ci/retry/README.md new file mode 100644 index 0000000000..ead2271f64 --- /dev/null +++ b/ci/retry/README.md @@ -0,0 +1,123 @@ +retry - The command line retry tool +------------------------------------------ + +Retry any shell command with exponential backoff or constant delay. + +### Instructions + +Install: + +retry is a shell script, so drop it somewhere and make sure it's added to your $PATH. Or you can use the following one-liner: + +```sh +sudo sh -c "curl https://raw.githubusercontent.com/kadwanev/retry/master/retry -o /usr/local/bin/retry && chmod +x /usr/local/bin/retry" +``` + +If you're on OS X, retry is also on Homebrew: + +``` +brew pull 27283 +brew install retry +``` +Not popular enough for homebrew-core. Please star this project to help. + +### Usage + +Help: + +`retry -?` + + Usage: retry [options] -- execute command + -h, -?, --help + -v, --verbose Verbose output + -t, --tries=# Set max retries: Default 10 + -s, --sleep=secs Constant sleep amount (seconds) + -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3 + -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60 + -f, --fail="script +cmds" Fail Script: run in case of final failure + +### Examples + +No problem: + +`retry echo u work good` + + u work good + +Test functionality: + +`retry 'echo "y u no work"; false'` + + y u no work + Before retry #1: sleeping 0.3 seconds + y u no work + Before retry #2: sleeping 0.6 seconds + y u no work + Before retry #3: sleeping 1.2 seconds + y u no work + Before retry #4: sleeping 2.4 seconds + y u no work + Before retry #5: sleeping 4.8 seconds + y u no work + Before retry #6: sleeping 9.6 seconds + y u no work + Before retry #7: sleeping 19.2 seconds + y u no work + Before retry #8: sleeping 38.4 seconds + y u no work + Before retry #9: sleeping 60.0 seconds + y u no work + Before retry #10: sleeping 60.0 seconds + y u no work + etc.. + +Limit retries: + +`retry -t 4 'echo "y u no work"; false'` + + y u no work + Before retry #1: sleeping 0.3 seconds + y u no work + Before retry #2: sleeping 0.6 seconds + y u no work + Before retry #3: sleeping 1.2 seconds + y u no work + Before retry #4: sleeping 2.4 seconds + y u no work + Retries exhausted + +Bad command: + +`retry poop` + + bash: poop: command not found + +Fail command: + +`retry -t 3 -f 'echo "oh poopsickles"' 'echo "y u no work"; false'` + + y u no work + Before retry #1: sleeping 0.3 seconds + y u no work + Before retry #2: sleeping 0.6 seconds + y u no work + Before retry #3: sleeping 1.2 seconds + y u no work + Retries exhausted, running fail script + oh poopsickles + +Last attempt passed: + +`retry -t 3 -- 'if [ $RETRY_ATTEMPT -eq 3 ]; then echo Passed at attempt $RETRY_ATTEMPT; true; else echo Failed at attempt $RETRY_ATTEMPT; false; fi;'` + + Failed at attempt 0 + Before retry #1: sleeping 0.3 seconds + Failed at attempt 1 + Before retry #2: sleeping 0.6 seconds + Failed at attempt 2 + Before retry #3: sleeping 1.2 seconds + Passed at attempt 3 + +### License + +Apache 2.0 - go nuts diff --git a/ci/retry/retry b/ci/retry/retry new file mode 100755 index 0000000000..03dc180ef2 --- /dev/null +++ b/ci/retry/retry @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +GETOPT_BIN=$IN_GETOPT_BIN +GETOPT_BIN=${GETOPT_BIN:-getopt} + +__sleep_amount() { + if [ -n "$constant_sleep" ]; then + sleep_time=$constant_sleep + else + #TODO: check for awk + #TODO: check if user would rather use one of the other possible dependencies: python, ruby, bc, dc + sleep_time=`awk "BEGIN {t = $min_sleep * $(( (1<<($attempts -1)) )); print (t > $max_sleep ? $max_sleep : t)}"` + fi +} + +__log_out() { + echo "$1" 1>&2 +} + +# Paramters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND +retry() +{ + local max_tries="$1"; shift + local min_sleep="$1"; shift + local max_sleep="$1"; shift + local constant_sleep="$1"; shift + local fail_script="$1"; shift + if [ -n "$VERBOSE" ]; then + __log_out "Retry Parameters: max_tries=$max_tries min_sleep=$min_sleep max_sleep=$max_sleep constant_sleep=$constant_sleep" + if [ -n "$fail_script" ]; then __log_out "Fail script: $fail_script"; fi + __log_out "" + __log_out "Execution Command: $*" + __log_out "" + fi + + local attempts=0 + local return_code=1 + + + while [[ $return_code -ne 0 && $attempts -le $max_tries ]]; do + if [ $attempts -gt 0 ]; then + __sleep_amount + __log_out "Before retry #$attempts: sleeping $sleep_time seconds" + sleep $sleep_time + fi + + P="$1" + for param in "${@:2}"; do P="$P '$param'"; done + #TODO: replace single quotes in each arg with '"'"' ? + export RETRY_ATTEMPT=$attempts + bash -c "$P" + return_code=$? + #__log_out "Process returned $return_code on attempt $attempts" + if [ $return_code -eq 127 ]; then + # command not found + exit $return_code + elif [ $return_code -ne 0 ]; then + attempts=$[$attempts +1] + fi + done + + if [ $attempts -gt $max_tries ]; then + if [ -n "$fail_script" ]; then + __log_out "Retries exhausted, running fail script" + eval $fail_script + else + __log_out "Retries exhausted" + fi + fi + + exit $return_code +} + +# If we're being sourced, don't worry about such things +if [ "$BASH_SOURCE" == "$0" ]; then + # Prints the help text + help() + { + local retry=$(basename $0) + cat <<EOF +Usage: $retry [options] -- execute command + -h, -?, --help + -v, --verbose Verbose output + -t, --tries=# Set max retries: Default 10 + -s, --sleep=secs Constant sleep amount (seconds) + -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3 + -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60 + -f, --fail="script +cmds" Fail Script: run in case of final failure +EOF + } + + # show help for no arguments if stdin is a terminal + if { [ -z "$1" ] && [ -t 0 ] ; } || [ "$1" == '-h' ] || [ "$1" == '-?' ] || [ "$1" == '--help' ] + then + help + exit 0 + fi + + $GETOPT_BIN --test > /dev/null + if [[ $? -ne 4 ]]; then + echo "I’m sorry, 'getopt --test' failed in this environment. Please load GNU getopt." + exit 1 + fi + + OPTIONS=vt:s:m:x:f: + LONGOPTIONS=verbose,tries:,sleep:,min:,max:,fail: + + PARSED=$($GETOPT_BIN --options="$OPTIONS" --longoptions="$LONGOPTIONS" --name "$0" -- "$@") + if [[ $? -ne 0 ]]; then + # e.g. $? == 1 + # then getopt has complained about wrong arguments to stdout + exit 2 + fi + # read getopt’s output this way to handle the quoting right: + eval set -- "$PARSED" + + max_tries=10 + min_sleep=0.3 + max_sleep=60.0 + constant_sleep= + fail_script= + + # now enjoy the options in order and nicely split until we see -- + while true; do + case "$1" in + -v|--verbose) + VERBOSE=true + shift + ;; + -t|--tries) + max_tries="$2" + shift 2 + ;; + -s|--sleep) + constant_sleep="$2" + shift 2 + ;; + -m|--min) + min_sleep="$2" + shift 2 + ;; + -x|--max) + max_sleep="$2" + shift 2 + ;; + -f|--fail) + fail_script="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "Programming error" + exit 3 + ;; + esac + done + + retry "$max_tries" "$min_sleep" "$max_sleep" "$constant_sleep" "$fail_script" "$@" + +fi |