diff options
Diffstat (limited to 'python/redo')
| -rw-r--r-- | python/redo/PKG-INFO | 10 | ||||
| -rw-r--r-- | python/redo/README | 4 | ||||
| -rw-r--r-- | python/redo/redo/__init__.py | 240 | ||||
| -rw-r--r-- | python/redo/redo/cmd.py | 53 | ||||
| -rw-r--r-- | python/redo/setup.cfg | 8 | ||||
| -rw-r--r-- | python/redo/setup.py | 18 |
6 files changed, 333 insertions, 0 deletions
diff --git a/python/redo/PKG-INFO b/python/redo/PKG-INFO new file mode 100644 index 000000000..1f2f84d85 --- /dev/null +++ b/python/redo/PKG-INFO @@ -0,0 +1,10 @@ +Metadata-Version: 1.0 +Name: redo +Version: 1.6 +Summary: Utilities to retry Python callables. +Home-page: https://github.com/bhearsum/redo +Author: Ben Hearsum +Author-email: ben@hearsum.ca +License: UNKNOWN +Description: UNKNOWN +Platform: UNKNOWN diff --git a/python/redo/README b/python/redo/README new file mode 100644 index 000000000..d2247eb55 --- /dev/null +++ b/python/redo/README @@ -0,0 +1,4 @@ +Redo - Utilities to retry Python callables +****************************************** + +Redo provides various means to add seamless retriability to any Python callable. Redo includes a plain function (redo.retry), a decorator (redo.retriable), and a context manager (redo.retrying) to enable you to integrate it in the best possible way for your project. As a bonus, a standalone interface is also included ("retry"). For details and sample invocations have a look at the docstrings in redo/__init__.py. diff --git a/python/redo/redo/__init__.py b/python/redo/redo/__init__.py new file mode 100644 index 000000000..a124eeaaa --- /dev/null +++ b/python/redo/redo/__init__.py @@ -0,0 +1,240 @@ +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** + +import time +from functools import wraps +from contextlib import contextmanager +import logging +import random +log = logging.getLogger(__name__) + + +def retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=1.5, jitter=1): + """ + A generator function that sleeps between retries, handles exponential + backoff and jitter. The action you are retrying is meant to run after + retrier yields. + + At each iteration, we sleep for sleeptime + random.randint(-jitter, jitter). + Afterwards sleeptime is multiplied by sleepscale for the next iteration. + + Args: + attempts (int): maximum number of times to try; defaults to 5 + sleeptime (float): how many seconds to sleep between tries; defaults to + 60s (one minute) + max_sleeptime (float): the longest we'll sleep, in seconds; defaults to + 300s (five minutes) + sleepscale (float): how much to multiply the sleep time by each + iteration; defaults to 1.5 + jitter (int): random jitter to introduce to sleep time each iteration. + the amount is chosen at random between [-jitter, +jitter] + defaults to 1 + + Yields: + None, a maximum of `attempts` number of times + + Example: + >>> n = 0 + >>> for _ in retrier(sleeptime=0, jitter=0): + ... if n == 3: + ... # We did the thing! + ... break + ... n += 1 + >>> n + 3 + + >>> n = 0 + >>> for _ in retrier(sleeptime=0, jitter=0): + ... if n == 6: + ... # We did the thing! + ... break + ... n += 1 + ... else: + ... print("max tries hit") + max tries hit + """ + jitter = jitter or 0 # py35 barfs on the next line if jitter is None + if jitter > sleeptime: + # To prevent negative sleep times + raise Exception('jitter ({}) must be less than sleep time ({})'.format(jitter, sleeptime)) + + sleeptime_real = sleeptime + for _ in range(attempts): + log.debug("attempt %i/%i", _ + 1, attempts) + + yield sleeptime_real + + if jitter: + sleeptime_real = sleeptime + random.randint(-jitter, jitter) + # our jitter should scale along with the sleeptime + jitter = int(jitter * sleepscale) + else: + sleeptime_real = sleeptime + + sleeptime *= sleepscale + + if sleeptime_real > max_sleeptime: + sleeptime_real = max_sleeptime + + # Don't need to sleep the last time + if _ < attempts - 1: + log.debug("sleeping for %.2fs (attempt %i/%i)", sleeptime_real, _ + 1, attempts) + time.sleep(sleeptime_real) + + +def retry(action, attempts=5, sleeptime=60, max_sleeptime=5 * 60, + sleepscale=1.5, jitter=1, retry_exceptions=(Exception,), + cleanup=None, args=(), kwargs={}): + """ + Calls an action function until it succeeds, or we give up. + + Args: + action (callable): the function to retry + attempts (int): maximum number of times to try; defaults to 5 + sleeptime (float): how many seconds to sleep between tries; defaults to + 60s (one minute) + max_sleeptime (float): the longest we'll sleep, in seconds; defaults to + 300s (five minutes) + sleepscale (float): how much to multiply the sleep time by each + iteration; defaults to 1.5 + jitter (int): random jitter to introduce to sleep time each iteration. + the amount is chosen at random between [-jitter, +jitter] + defaults to 1 + retry_exceptions (tuple): tuple of exceptions to be caught. If other + exceptions are raised by action(), then these + are immediately re-raised to the caller. + cleanup (callable): optional; called if one of `retry_exceptions` is + caught. No arguments are passed to the cleanup + function; if your cleanup requires arguments, + consider using functools.partial or a lambda + function. + args (tuple): positional arguments to call `action` with + kwargs (dict): keyword arguments to call `action` with + + Returns: + Whatever action(*args, **kwargs) returns + + Raises: + Whatever action(*args, **kwargs) raises. `retry_exceptions` are caught + up until the last attempt, in which case they are re-raised. + + Example: + >>> count = 0 + >>> def foo(): + ... global count + ... count += 1 + ... print(count) + ... if count < 3: + ... raise ValueError("count is too small!") + ... return "success!" + >>> retry(foo, sleeptime=0, jitter=0) + 1 + 2 + 3 + 'success!' + """ + assert callable(action) + assert not cleanup or callable(cleanup) + + action_name = getattr(action, '__name__', action) + if args or kwargs: + log_attempt_format = ("retry: calling %s with args: %s," + " kwargs: %s, attempt #%%d" + % (action_name, args, kwargs)) + else: + log_attempt_format = ("retry: calling %s, attempt #%%d" + % action_name) + + if max_sleeptime < sleeptime: + log.debug("max_sleeptime %d less than sleeptime %d" % ( + max_sleeptime, sleeptime)) + + n = 1 + for _ in retrier(attempts=attempts, sleeptime=sleeptime, + max_sleeptime=max_sleeptime, sleepscale=sleepscale, + jitter=jitter): + try: + logfn = log.info if n != 1 else log.debug + logfn(log_attempt_format, n) + return action(*args, **kwargs) + except retry_exceptions: + log.debug("retry: Caught exception: ", exc_info=True) + if cleanup: + cleanup() + if n == attempts: + log.info("retry: Giving up on %s" % action_name) + raise + continue + finally: + n += 1 + + +def retriable(*retry_args, **retry_kwargs): + """ + A decorator factory for retry(). Wrap your function in @retriable(...) to + give it retry powers! + + Arguments: + Same as for `retry`, with the exception of `action`, `args`, and `kwargs`, + which are left to the normal function definition. + + Returns: + A function decorator + + Example: + >>> count = 0 + >>> @retriable(sleeptime=0, jitter=0) + ... def foo(): + ... global count + ... count += 1 + ... print(count) + ... if count < 3: + ... raise ValueError("count too small") + ... return "success!" + >>> foo() + 1 + 2 + 3 + 'success!' + """ + def _retriable_factory(func): + @wraps(func) + def _retriable_wrapper(*args, **kwargs): + return retry(func, args=args, kwargs=kwargs, *retry_args, + **retry_kwargs) + return _retriable_wrapper + return _retriable_factory + + +@contextmanager +def retrying(func, *retry_args, **retry_kwargs): + """ + A context manager for wrapping functions with retry functionality. + + Arguments: + func (callable): the function to wrap + other arguments as per `retry` + + Returns: + A context manager that returns retriable(func) on __enter__ + + Example: + >>> count = 0 + >>> def foo(): + ... global count + ... count += 1 + ... print(count) + ... if count < 3: + ... raise ValueError("count too small") + ... return "success!" + >>> with retrying(foo, sleeptime=0, jitter=0) as f: + ... f() + 1 + 2 + 3 + 'success!' + """ + yield retriable(*retry_args, **retry_kwargs)(func) diff --git a/python/redo/redo/cmd.py b/python/redo/redo/cmd.py new file mode 100644 index 000000000..afd98e744 --- /dev/null +++ b/python/redo/redo/cmd.py @@ -0,0 +1,53 @@ +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +import logging +from subprocess import check_call, CalledProcessError +import sys + +from redo import retrying + +log = logging.getLogger(__name__) + + +def main(): + from argparse import ArgumentParser + + parser = ArgumentParser() + parser.add_argument( + "-a", "--attempts", type=int, default=5, + help="How many times to retry.") + parser.add_argument( + "-s", "--sleeptime", type=int, default=60, + help="How long to sleep between attempts. Sleeptime doubles after each attempt.") + parser.add_argument( + "-m", "--max-sleeptime", type=int, default=5*60, + help="Maximum length of time to sleep between attempts (limits backoff length).") + parser.add_argument("-v", "--verbose", action="store_true", default=False) + parser.add_argument("cmd", nargs="+", help="Command to run. Eg: wget http://blah") + + args = parser.parse_args() + + if args.verbose: + logging.basicConfig(level=logging.INFO) + logging.getLogger("retry").setLevel(logging.INFO) + else: + logging.basicConfig(level=logging.ERROR) + logging.getLogger("retry").setLevel(logging.ERROR) + + try: + with retrying(check_call, attempts=args.attempts, sleeptime=args.sleeptime, + max_sleeptime=args.max_sleeptime, + retry_exceptions=(CalledProcessError,)) as r_check_call: + r_check_call(args.cmd) + except KeyboardInterrupt: + sys.exit(-1) + except Exception as e: + log.error("Unable to run command after %d attempts" % args.attempts, exc_info=True) + rc = getattr(e, "returncode", -2) + sys.exit(rc) + +if __name__ == "__main__": + main() diff --git a/python/redo/setup.cfg b/python/redo/setup.cfg new file mode 100644 index 000000000..6c71b612d --- /dev/null +++ b/python/redo/setup.cfg @@ -0,0 +1,8 @@ +[wheel] +universal = 1 + +[egg_info] +tag_build = +tag_date = 0 +tag_svn_revision = 0 + diff --git a/python/redo/setup.py b/python/redo/setup.py new file mode 100644 index 000000000..a1e57d7f0 --- /dev/null +++ b/python/redo/setup.py @@ -0,0 +1,18 @@ +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + + +setup( + name="redo", + version="1.6", + description="Utilities to retry Python callables.", + author="Ben Hearsum", + author_email="ben@hearsum.ca", + packages=["redo"], + entry_points={ + "console_scripts": ["retry = redo.cmd:main"], + }, + url="https://github.com/bhearsum/redo", +) |