1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
|
#!/usr/bin/env bash
GETOPT_BIN=$IN_GETOPT_BIN
GETOPT_BIN=${GETOPT_BIN:-getopt}
__sleep_amount() {
if [ -n "$constant_sleep" ]; then
sleep_time=$constant_sleep
else
#TODO: check for awk
#TODO: check if user would rather use one of the other possible dependencies: python, ruby, bc, dc
sleep_time=`awk "BEGIN {t = $min_sleep * $(( (1<<($attempts -1)) )); print (t > $max_sleep ? $max_sleep : t)}"`
fi
}
__log_out() {
echo "$1" 1>&2
}
# Parameters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND
retry()
{
local max_tries="$1"; shift
local min_sleep="$1"; shift
local max_sleep="$1"; shift
local constant_sleep="$1"; shift
local fail_script="$1"; shift
if [ -n "$VERBOSE" ]; then
__log_out "Retry Parameters: max_tries=$max_tries min_sleep=$min_sleep max_sleep=$max_sleep constant_sleep=$constant_sleep"
if [ -n "$fail_script" ]; then __log_out "Fail script: $fail_script"; fi
__log_out ""
__log_out "Execution Command: $*"
__log_out ""
fi
local attempts=0
local return_code=1
while [[ $return_code -ne 0 && $attempts -le $max_tries ]]; do
if [ $attempts -gt 0 ]; then
__sleep_amount
__log_out "Before retry #$attempts: sleeping $sleep_time seconds"
sleep $sleep_time
fi
P="$1"
for param in "${@:2}"; do P="$P '$param'"; done
#TODO: replace single quotes in each arg with '"'"' ?
export RETRY_ATTEMPT=$attempts
bash -c "$P"
return_code=$?
#__log_out "Process returned $return_code on attempt $attempts"
if [ $return_code -eq 127 ]; then
# command not found
exit $return_code
elif [ $return_code -ne 0 ]; then
attempts=$[$attempts +1]
fi
done
if [ $attempts -gt $max_tries ]; then
if [ -n "$fail_script" ]; then
__log_out "Retries exhausted, running fail script"
eval $fail_script
else
__log_out "Retries exhausted"
fi
fi
exit $return_code
}
# If we're being sourced, don't worry about such things
if [ "$BASH_SOURCE" == "$0" ]; then
# Prints the help text
help()
{
local retry=$(basename $0)
cat <<EOF
Usage: $retry [options] -- execute command
-h, -?, --help
-v, --verbose Verbose output
-t, --tries=# Set max retries: Default 10
-s, --sleep=secs Constant sleep amount (seconds)
-m, --min=secs Exponential Backoff: minimum sleep amount (seconds): Default 0.3
-x, --max=secs Exponential Backoff: maximum sleep amount (seconds): Default 60
-f, --fail="script +cmds" Fail Script: run in case of final failure
EOF
}
# show help for no arguments if stdin is a terminal
if { [ -z "$1" ] && [ -t 0 ] ; } || [ "$1" == '-h' ] || [ "$1" == '-?' ] || [ "$1" == '--help' ]
then
help
exit 0
fi
$GETOPT_BIN --test > /dev/null
if [[ $? -ne 4 ]]; then
echo "I’m sorry, 'getopt --test' failed in this environment. Please load GNU getopt."
exit 1
fi
OPTIONS=vt:s:m:x:f:
LONGOPTIONS=verbose,tries:,sleep:,min:,max:,fail:
PARSED=$($GETOPT_BIN --options="$OPTIONS" --longoptions="$LONGOPTIONS" --name "$0" -- "$@")
if [[ $? -ne 0 ]]; then
# e.g. $? == 1
# then getopt has complained about wrong arguments to stdout
exit 2
fi
# read getopt’s output this way to handle the quoting right:
eval set -- "$PARSED"
max_tries=10
min_sleep=0.3
max_sleep=60.0
constant_sleep=
fail_script=
# now enjoy the options in order and nicely split until we see --
while true; do
case "$1" in
-v|--verbose)
VERBOSE=true
shift
;;
-t|--tries)
max_tries="$2"
shift 2
;;
-s|--sleep)
constant_sleep="$2"
shift 2
;;
-m|--min)
min_sleep="$2"
shift 2
;;
-x|--max)
max_sleep="$2"
shift 2
;;
-f|--fail)
fail_script="$2"
shift 2
;;
--)
shift
break
;;
*)
echo "Programming error"
exit 3
;;
esac
done
retry "$max_tries" "$min_sleep" "$max_sleep" "$constant_sleep" "$fail_script" "$@"
fi
|
