ci: Add retry

From: https://github.com/kadwanev/retry/tree/a1b1826bdb0a78189d5c70c858dc676f5133b1d7

2 files changed, 286 insertions, 0 deletions

diff --git a/ci/retry/README.md b/ci/retry/README.md
new file mode 100644
index 0000000000..ead2271f64
--- /dev/null
+++ b/ci/retry/README.md
@@ -0,0 +1,123 @@
+retry - The command line retry tool
+------------------------------------------
+
+Retry any shell command with exponential backoff or constant delay.
+
+### Instructions
+
+Install:
+
+retry is a shell script, so drop it somewhere and make sure it's added to your $PATH. Or you can use the following one-liner:
+
+```sh
+sudo sh -c "curl https://raw.githubusercontent.com/kadwanev/retry/master/retry -o /usr/local/bin/retry && chmod +x /usr/local/bin/retry"
+```
+
+If you're on OS X, retry is also on Homebrew:
+
+```
+brew pull 27283
+brew install retry
+```
+Not popular enough for homebrew-core. Please star this project to help.
+
+### Usage
+
+Help:
+
+`retry -?`
+
+    Usage: retry [options] -- execute command
+        -h, -?, --help
+        -v, --verbose                    Verbose output
+        -t, --tries=#                    Set max retries: Default 10
+        -s, --sleep=secs                 Constant sleep amount (seconds)
+        -m, --min=secs                   Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3
+        -x, --max=secs                   Exponenetial Backoff: maximum sleep amount (seconds): Default 60
+        -f, --fail="script +cmds"        Fail Script: run in case of final failure
+    
+### Examples
+
+No problem:
+
+`retry echo u work good`
+
+    u work good
+
+Test functionality:
+
+`retry 'echo "y u no work"; false'`
+
+    y u no work
+    Before retry #1: sleeping 0.3 seconds
+    y u no work
+    Before retry #2: sleeping 0.6 seconds
+    y u no work
+    Before retry #3: sleeping 1.2 seconds
+    y u no work
+    Before retry #4: sleeping 2.4 seconds
+    y u no work
+    Before retry #5: sleeping 4.8 seconds
+    y u no work
+    Before retry #6: sleeping 9.6 seconds
+    y u no work
+    Before retry #7: sleeping 19.2 seconds
+    y u no work
+    Before retry #8: sleeping 38.4 seconds
+    y u no work
+    Before retry #9: sleeping 60.0 seconds
+    y u no work
+    Before retry #10: sleeping 60.0 seconds
+    y u no work
+    etc..
+
+Limit retries:
+
+`retry -t 4 'echo "y u no work"; false'`
+
+    y u no work
+    Before retry #1: sleeping 0.3 seconds
+    y u no work
+    Before retry #2: sleeping 0.6 seconds
+    y u no work
+    Before retry #3: sleeping 1.2 seconds
+    y u no work
+    Before retry #4: sleeping 2.4 seconds
+    y u no work
+    Retries exhausted
+
+Bad command:
+
+`retry poop`
+
+    bash: poop: command not found
+
+Fail command:
+
+`retry -t 3 -f 'echo "oh poopsickles"' 'echo "y u no work"; false'`
+
+    y u no work
+    Before retry #1: sleeping 0.3 seconds
+    y u no work
+    Before retry #2: sleeping 0.6 seconds
+    y u no work
+    Before retry #3: sleeping 1.2 seconds
+    y u no work
+    Retries exhausted, running fail script
+    oh poopsickles
+
+Last attempt passed:
+
+`retry -t 3 -- 'if [ $RETRY_ATTEMPT -eq 3 ]; then echo Passed at attempt $RETRY_ATTEMPT; true; else echo Failed at attempt $RETRY_ATTEMPT; false; fi;'`
+
+    Failed at attempt 0
+    Before retry #1: sleeping 0.3 seconds
+    Failed at attempt 1
+    Before retry #2: sleeping 0.6 seconds
+    Failed at attempt 2
+    Before retry #3: sleeping 1.2 seconds
+    Passed at attempt 3
+
+### License
+
+Apache 2.0 - go nuts
diff --git a/ci/retry/retry b/ci/retry/retry
new file mode 100755
index 0000000000..03dc180ef2
--- /dev/null
+++ b/ci/retry/retry
@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+
+GETOPT_BIN=$IN_GETOPT_BIN
+GETOPT_BIN=${GETOPT_BIN:-getopt}
+
+__sleep_amount() {
+  if [ -n "$constant_sleep" ]; then 
+    sleep_time=$constant_sleep
+  else
+    #TODO: check for awk
+    #TODO: check if user would rather use one of the other possible dependencies: python, ruby, bc, dc
+    sleep_time=`awk "BEGIN {t = $min_sleep * $(( (1<<($attempts -1)) )); print (t > $max_sleep ? $max_sleep : t)}"`
+  fi
+}
+
+__log_out() {
+  echo "$1" 1>&2
+}
+
+# Paramters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND
+retry()
+{
+  local max_tries="$1"; shift
+  local min_sleep="$1"; shift
+  local max_sleep="$1"; shift
+  local constant_sleep="$1"; shift
+  local fail_script="$1"; shift
+  if [ -n "$VERBOSE" ]; then
+    __log_out "Retry Parameters: max_tries=$max_tries min_sleep=$min_sleep max_sleep=$max_sleep constant_sleep=$constant_sleep"
+    if [ -n "$fail_script" ]; then __log_out "Fail script: $fail_script"; fi
+    __log_out ""
+    __log_out "Execution Command: $*"
+    __log_out ""
+  fi
+
+  local attempts=0
+  local return_code=1
+
+
+  while [[ $return_code -ne 0 && $attempts -le $max_tries ]]; do
+    if [ $attempts -gt 0 ]; then
+      __sleep_amount
+      __log_out "Before retry #$attempts: sleeping $sleep_time seconds"
+      sleep $sleep_time
+    fi
+
+    P="$1"
+    for param in "${@:2}"; do P="$P '$param'"; done
+    #TODO: replace single quotes in each arg with '"'"' ?
+    export RETRY_ATTEMPT=$attempts
+    bash -c "$P"
+    return_code=$?
+    #__log_out "Process returned $return_code on attempt $attempts"
+    if [ $return_code -eq 127 ]; then
+      # command not found
+      exit $return_code
+    elif [ $return_code -ne 0 ]; then
+      attempts=$[$attempts +1]
+    fi
+  done
+
+  if [ $attempts -gt $max_tries ]; then
+    if [ -n "$fail_script" ]; then
+      __log_out "Retries exhausted, running fail script"
+      eval $fail_script
+    else
+      __log_out "Retries exhausted"
+    fi
+  fi
+
+  exit $return_code
+}
+
+# If we're being sourced, don't worry about such things
+if [ "$BASH_SOURCE" == "$0" ]; then
+  # Prints the help text
+  help()
+  {
+    local retry=$(basename $0)
+    cat <<EOF
+Usage: $retry [options] -- execute command
+    -h, -?, --help
+    -v, --verbose                    Verbose output
+    -t, --tries=#                    Set max retries: Default 10
+    -s, --sleep=secs                 Constant sleep amount (seconds)
+    -m, --min=secs                   Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3
+    -x, --max=secs                   Exponenetial Backoff: maximum sleep amount (seconds): Default 60
+    -f, --fail="script +cmds"        Fail Script: run in case of final failure
+EOF
+  }
+
+  # show help for no arguments if stdin is a terminal
+  if { [ -z "$1" ] && [ -t 0 ] ; } || [ "$1" == '-h' ] || [ "$1" == '-?' ] || [ "$1" == '--help' ]
+  then
+    help
+    exit 0
+  fi
+
+  $GETOPT_BIN --test > /dev/null
+  if [[ $? -ne 4 ]]; then
+      echo "I’m sorry, 'getopt --test' failed in this environment. Please load GNU getopt."
+      exit 1
+  fi
+
+  OPTIONS=vt:s:m:x:f:
+  LONGOPTIONS=verbose,tries:,sleep:,min:,max:,fail:
+
+  PARSED=$($GETOPT_BIN --options="$OPTIONS" --longoptions="$LONGOPTIONS" --name "$0" -- "$@")
+  if [[ $? -ne 0 ]]; then
+    # e.g. $? == 1
+    #  then getopt has complained about wrong arguments to stdout
+    exit 2
+  fi
+  # read getopt’s output this way to handle the quoting right:
+  eval set -- "$PARSED"
+
+  max_tries=10
+  min_sleep=0.3
+  max_sleep=60.0
+  constant_sleep=
+  fail_script=
+
+  # now enjoy the options in order and nicely split until we see --
+  while true; do
+      case "$1" in
+          -v|--verbose)
+              VERBOSE=true
+              shift
+              ;;
+          -t|--tries)
+              max_tries="$2"
+              shift 2
+              ;;
+          -s|--sleep)
+              constant_sleep="$2"
+              shift 2
+              ;;
+          -m|--min)
+              min_sleep="$2"
+              shift 2
+              ;;
+          -x|--max)
+              max_sleep="$2"
+              shift 2
+              ;;
+          -f|--fail)
+              fail_script="$2"
+              shift 2
+              ;;
+          --)
+              shift
+              break
+              ;;
+          *)
+              echo "Programming error"
+              exit 3
+              ;;
+      esac
+  done
+
+  retry "$max_tries" "$min_sleep" "$max_sleep" "$constant_sleep" "$fail_script" "$@"
+  
+fi