1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
|
# ***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
# ***** END LICENSE BLOCK *****
import time
from functools import wraps
from contextlib import contextmanager
import logging
import random
log = logging.getLogger(__name__)
def retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=1.5, jitter=1):
"""
A generator function that sleeps between retries, handles exponential
backoff and jitter. The action you are retrying is meant to run after
retrier yields.
At each iteration, we sleep for sleeptime + random.randint(-jitter, jitter).
Afterwards sleeptime is multiplied by sleepscale for the next iteration.
Args:
attempts (int): maximum number of times to try; defaults to 5
sleeptime (float): how many seconds to sleep between tries; defaults to
60s (one minute)
max_sleeptime (float): the longest we'll sleep, in seconds; defaults to
300s (five minutes)
sleepscale (float): how much to multiply the sleep time by each
iteration; defaults to 1.5
jitter (int): random jitter to introduce to sleep time each iteration.
the amount is chosen at random between [-jitter, +jitter]
defaults to 1
Yields:
None, a maximum of `attempts` number of times
Example:
>>> n = 0
>>> for _ in retrier(sleeptime=0, jitter=0):
... if n == 3:
... # We did the thing!
... break
... n += 1
>>> n
3
>>> n = 0
>>> for _ in retrier(sleeptime=0, jitter=0):
... if n == 6:
... # We did the thing!
... break
... n += 1
... else:
... print("max tries hit")
max tries hit
"""
jitter = jitter or 0 # py35 barfs on the next line if jitter is None
if jitter > sleeptime:
# To prevent negative sleep times
raise Exception('jitter ({}) must be less than sleep time ({})'.format(jitter, sleeptime))
sleeptime_real = sleeptime
for _ in range(attempts):
log.debug("attempt %i/%i", _ + 1, attempts)
yield sleeptime_real
if jitter:
sleeptime_real = sleeptime + random.randint(-jitter, jitter)
# our jitter should scale along with the sleeptime
jitter = int(jitter * sleepscale)
else:
sleeptime_real = sleeptime
sleeptime *= sleepscale
if sleeptime_real > max_sleeptime:
sleeptime_real = max_sleeptime
# Don't need to sleep the last time
if _ < attempts - 1:
log.debug("sleeping for %.2fs (attempt %i/%i)", sleeptime_real, _ + 1, attempts)
time.sleep(sleeptime_real)
def retry(action, attempts=5, sleeptime=60, max_sleeptime=5 * 60,
sleepscale=1.5, jitter=1, retry_exceptions=(Exception,),
cleanup=None, args=(), kwargs={}):
"""
Calls an action function until it succeeds, or we give up.
Args:
action (callable): the function to retry
attempts (int): maximum number of times to try; defaults to 5
sleeptime (float): how many seconds to sleep between tries; defaults to
60s (one minute)
max_sleeptime (float): the longest we'll sleep, in seconds; defaults to
300s (five minutes)
sleepscale (float): how much to multiply the sleep time by each
iteration; defaults to 1.5
jitter (int): random jitter to introduce to sleep time each iteration.
the amount is chosen at random between [-jitter, +jitter]
defaults to 1
retry_exceptions (tuple): tuple of exceptions to be caught. If other
exceptions are raised by action(), then these
are immediately re-raised to the caller.
cleanup (callable): optional; called if one of `retry_exceptions` is
caught. No arguments are passed to the cleanup
function; if your cleanup requires arguments,
consider using functools.partial or a lambda
function.
args (tuple): positional arguments to call `action` with
kwargs (dict): keyword arguments to call `action` with
Returns:
Whatever action(*args, **kwargs) returns
Raises:
Whatever action(*args, **kwargs) raises. `retry_exceptions` are caught
up until the last attempt, in which case they are re-raised.
Example:
>>> count = 0
>>> def foo():
... global count
... count += 1
... print(count)
... if count < 3:
... raise ValueError("count is too small!")
... return "success!"
>>> retry(foo, sleeptime=0, jitter=0)
1
2
3
'success!'
"""
assert callable(action)
assert not cleanup or callable(cleanup)
action_name = getattr(action, '__name__', action)
if args or kwargs:
log_attempt_format = ("retry: calling %s with args: %s,"
" kwargs: %s, attempt #%%d"
% (action_name, args, kwargs))
else:
log_attempt_format = ("retry: calling %s, attempt #%%d"
% action_name)
if max_sleeptime < sleeptime:
log.debug("max_sleeptime %d less than sleeptime %d" % (
max_sleeptime, sleeptime))
n = 1
for _ in retrier(attempts=attempts, sleeptime=sleeptime,
max_sleeptime=max_sleeptime, sleepscale=sleepscale,
jitter=jitter):
try:
logfn = log.info if n != 1 else log.debug
logfn(log_attempt_format, n)
return action(*args, **kwargs)
except retry_exceptions:
log.debug("retry: Caught exception: ", exc_info=True)
if cleanup:
cleanup()
if n == attempts:
log.info("retry: Giving up on %s" % action_name)
raise
continue
finally:
n += 1
def retriable(*retry_args, **retry_kwargs):
"""
A decorator factory for retry(). Wrap your function in @retriable(...) to
give it retry powers!
Arguments:
Same as for `retry`, with the exception of `action`, `args`, and `kwargs`,
which are left to the normal function definition.
Returns:
A function decorator
Example:
>>> count = 0
>>> @retriable(sleeptime=0, jitter=0)
... def foo():
... global count
... count += 1
... print(count)
... if count < 3:
... raise ValueError("count too small")
... return "success!"
>>> foo()
1
2
3
'success!'
"""
def _retriable_factory(func):
@wraps(func)
def _retriable_wrapper(*args, **kwargs):
return retry(func, args=args, kwargs=kwargs, *retry_args,
**retry_kwargs)
return _retriable_wrapper
return _retriable_factory
@contextmanager
def retrying(func, *retry_args, **retry_kwargs):
"""
A context manager for wrapping functions with retry functionality.
Arguments:
func (callable): the function to wrap
other arguments as per `retry`
Returns:
A context manager that returns retriable(func) on __enter__
Example:
>>> count = 0
>>> def foo():
... global count
... count += 1
... print(count)
... if count < 3:
... raise ValueError("count too small")
... return "success!"
>>> with retrying(foo, sleeptime=0, jitter=0) as f:
... f()
1
2
3
'success!'
"""
yield retriable(*retry_args, **retry_kwargs)(func)
|
