diff options
Diffstat (limited to 'third_party/python/redo')
8 files changed, 373 insertions, 0 deletions
diff --git a/third_party/python/redo/redo-2.0.3.dist-info/AUTHORS b/third_party/python/redo/redo-2.0.3.dist-info/AUTHORS new file mode 100644 index 0000000000..b2e24333c0 --- /dev/null +++ b/third_party/python/redo/redo-2.0.3.dist-info/AUTHORS @@ -0,0 +1,7 @@ +Rail Aliiev (https://github.com/rail) +Chris AtLee (https://github.com/catlee) +Ben Hearsum (https://github.com/bhearsum) +John Hopkins (https://github.com/jhopkinsmoz) +Justin Wood (https://github.com/callek) +Terry Chia (https://github.com/Ayrx) +Mr. Deathless (https://github.com/mrdeathless) diff --git a/third_party/python/redo/redo-2.0.3.dist-info/METADATA b/third_party/python/redo/redo-2.0.3.dist-info/METADATA new file mode 100644 index 0000000000..e43a46e13a --- /dev/null +++ b/third_party/python/redo/redo-2.0.3.dist-info/METADATA @@ -0,0 +1,13 @@ +Metadata-Version: 2.1 +Name: redo +Version: 2.0.3 +Summary: Utilities to retry Python callables. +Home-page: https://github.com/bhearsum/redo +Author: Ben Hearsum +Author-email: ben@hearsum.ca +License: UNKNOWN +Platform: UNKNOWN + +UNKNOWN + + diff --git a/third_party/python/redo/redo-2.0.3.dist-info/RECORD b/third_party/python/redo/redo-2.0.3.dist-info/RECORD new file mode 100644 index 0000000000..d8f82bf7da --- /dev/null +++ b/third_party/python/redo/redo-2.0.3.dist-info/RECORD @@ -0,0 +1,8 @@ +redo/__init__.py,sha256=6VZUeFfbFkBJ_lxY_cJWk0S8mgSkrSRIwVniVm_sKsw,8518 +redo/cmd.py,sha256=F1axa3CVChlIvrSnq4xZZIyZ4M4wnnZjpv8wy46ugS4,2085 +redo-2.0.3.dist-info/AUTHORS,sha256=uIuTIaIlfQwklq75eg8VTjdnzENPlN_WKxa1UxQWTtQ,290 +redo-2.0.3.dist-info/METADATA,sha256=0DOrbjh62qccs3wFTgTxP9kQ1S4cphhUnupdfv0_6ms,233 +redo-2.0.3.dist-info/WHEEL,sha256=_wJFdOYk7i3xxT8ElOkUJvOdOvfNGbR9g-bf6UQT6sU,110 +redo-2.0.3.dist-info/entry_points.txt,sha256=ftcg9P_jTwZ9bYYDKB-s-5eIY6mHGiRHvd_HAGc7UPc,41 +redo-2.0.3.dist-info/top_level.txt,sha256=o1qXhN94bANfd7pz4zervaf8ytHEG_UVfhFZKMmmdvo,5 +redo-2.0.3.dist-info/RECORD,, diff --git a/third_party/python/redo/redo-2.0.3.dist-info/WHEEL b/third_party/python/redo/redo-2.0.3.dist-info/WHEEL new file mode 100644 index 0000000000..c4bde30377 --- /dev/null +++ b/third_party/python/redo/redo-2.0.3.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.32.3) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/third_party/python/redo/redo-2.0.3.dist-info/entry_points.txt b/third_party/python/redo/redo-2.0.3.dist-info/entry_points.txt new file mode 100644 index 0000000000..44eccdcfca --- /dev/null +++ b/third_party/python/redo/redo-2.0.3.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +retry = redo.cmd:main + diff --git a/third_party/python/redo/redo-2.0.3.dist-info/top_level.txt b/third_party/python/redo/redo-2.0.3.dist-info/top_level.txt new file mode 100644 index 0000000000..f49789cbab --- /dev/null +++ b/third_party/python/redo/redo-2.0.3.dist-info/top_level.txt @@ -0,0 +1 @@ +redo diff --git a/third_party/python/redo/redo/__init__.py b/third_party/python/redo/redo/__init__.py new file mode 100644 index 0000000000..9814805990 --- /dev/null +++ b/third_party/python/redo/redo/__init__.py @@ -0,0 +1,265 @@ +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** + +import time +from functools import wraps +from contextlib import contextmanager +import logging +import random + +log = logging.getLogger(__name__) + + +def retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=1.5, jitter=1): + """ + A generator function that sleeps between retries, handles exponential + backoff and jitter. The action you are retrying is meant to run after + retrier yields. + + At each iteration, we sleep for sleeptime + random.uniform(-jitter, jitter). + Afterwards sleeptime is multiplied by sleepscale for the next iteration. + + Args: + attempts (int): maximum number of times to try; defaults to 5 + sleeptime (float): how many seconds to sleep between tries; defaults to + 10 seconds + max_sleeptime (float): the longest we'll sleep, in seconds; defaults to + 300s (five minutes) + sleepscale (float): how much to multiply the sleep time by each + iteration; defaults to 1.5 + jitter (float): random jitter to introduce to sleep time each iteration. + the amount is chosen at random between [-jitter, +jitter] + defaults to 1 + + Yields: + None, a maximum of `attempts` number of times + + Example: + >>> n = 0 + >>> for _ in retrier(sleeptime=0, jitter=0): + ... if n == 3: + ... # We did the thing! + ... break + ... n += 1 + >>> n + 3 + + >>> n = 0 + >>> for _ in retrier(sleeptime=0, jitter=0): + ... if n == 6: + ... # We did the thing! + ... break + ... n += 1 + ... else: + ... print("max tries hit") + max tries hit + """ + jitter = jitter or 0 # py35 barfs on the next line if jitter is None + if jitter > sleeptime: + # To prevent negative sleep times + raise Exception( + "jitter ({}) must be less than sleep time ({})".format(jitter, sleeptime) + ) + + sleeptime_real = sleeptime + for _ in range(attempts): + log.debug("attempt %i/%i", _ + 1, attempts) + + yield sleeptime_real + + if jitter: + sleeptime_real = sleeptime + random.uniform(-jitter, jitter) + # our jitter should scale along with the sleeptime + jitter = jitter * sleepscale + else: + sleeptime_real = sleeptime + + sleeptime *= sleepscale + + if sleeptime_real > max_sleeptime: + sleeptime_real = max_sleeptime + + # Don't need to sleep the last time + if _ < attempts - 1: + log.debug( + "sleeping for %.2fs (attempt %i/%i)", sleeptime_real, _ + 1, attempts + ) + time.sleep(sleeptime_real) + + +def retry( + action, + attempts=5, + sleeptime=60, + max_sleeptime=5 * 60, + sleepscale=1.5, + jitter=1, + retry_exceptions=(Exception,), + cleanup=None, + args=(), + kwargs={}, + log_args=True, +): + """ + Calls an action function until it succeeds, or we give up. + + Args: + action (callable): the function to retry + attempts (int): maximum number of times to try; defaults to 5 + sleeptime (float): how many seconds to sleep between tries; defaults to + 60s (one minute) + max_sleeptime (float): the longest we'll sleep, in seconds; defaults to + 300s (five minutes) + sleepscale (float): how much to multiply the sleep time by each + iteration; defaults to 1.5 + jitter (float): random jitter to introduce to sleep time each iteration. + the amount is chosen at random between [-jitter, +jitter] + defaults to 1 + retry_exceptions (tuple): tuple of exceptions to be caught. If other + exceptions are raised by action(), then these + are immediately re-raised to the caller. + cleanup (callable): optional; called if one of `retry_exceptions` is + caught. No arguments are passed to the cleanup + function; if your cleanup requires arguments, + consider using functools.partial or a lambda + function. + args (tuple): positional arguments to call `action` with + kwargs (dict): keyword arguments to call `action` with + log_args (bool): whether or not to include args and kwargs in log + messages. Defaults to True. + + Returns: + Whatever action(*args, **kwargs) returns + + Raises: + Whatever action(*args, **kwargs) raises. `retry_exceptions` are caught + up until the last attempt, in which case they are re-raised. + + Example: + >>> count = 0 + >>> def foo(): + ... global count + ... count += 1 + ... print(count) + ... if count < 3: + ... raise ValueError("count is too small!") + ... return "success!" + >>> retry(foo, sleeptime=0, jitter=0) + 1 + 2 + 3 + 'success!' + """ + assert callable(action) + assert not cleanup or callable(cleanup) + + action_name = getattr(action, "__name__", action) + if log_args and (args or kwargs): + log_attempt_args = ( + "retry: calling %s with args: %s," " kwargs: %s, attempt #%d", + action_name, + args, + kwargs, + ) + else: + log_attempt_args = ("retry: calling %s, attempt #%d", action_name) + + if max_sleeptime < sleeptime: + log.debug("max_sleeptime %d less than sleeptime %d", max_sleeptime, sleeptime) + + n = 1 + for _ in retrier( + attempts=attempts, + sleeptime=sleeptime, + max_sleeptime=max_sleeptime, + sleepscale=sleepscale, + jitter=jitter, + ): + try: + logfn = log.info if n != 1 else log.debug + logfn_args = log_attempt_args + (n,) + logfn(*logfn_args) + return action(*args, **kwargs) + except retry_exceptions: + log.debug("retry: Caught exception: ", exc_info=True) + if cleanup: + cleanup() + if n == attempts: + log.info("retry: Giving up on %s", action_name) + raise + continue + finally: + n += 1 + + +def retriable(*retry_args, **retry_kwargs): + """ + A decorator factory for retry(). Wrap your function in @retriable(...) to + give it retry powers! + + Arguments: + Same as for `retry`, with the exception of `action`, `args`, and `kwargs`, + which are left to the normal function definition. + + Returns: + A function decorator + + Example: + >>> count = 0 + >>> @retriable(sleeptime=0, jitter=0) + ... def foo(): + ... global count + ... count += 1 + ... print(count) + ... if count < 3: + ... raise ValueError("count too small") + ... return "success!" + >>> foo() + 1 + 2 + 3 + 'success!' + """ + + def _retriable_factory(func): + @wraps(func) + def _retriable_wrapper(*args, **kwargs): + return retry(func, args=args, kwargs=kwargs, *retry_args, **retry_kwargs) + + return _retriable_wrapper + + return _retriable_factory + + +@contextmanager +def retrying(func, *retry_args, **retry_kwargs): + """ + A context manager for wrapping functions with retry functionality. + + Arguments: + func (callable): the function to wrap + other arguments as per `retry` + + Returns: + A context manager that returns retriable(func) on __enter__ + + Example: + >>> count = 0 + >>> def foo(): + ... global count + ... count += 1 + ... print(count) + ... if count < 3: + ... raise ValueError("count too small") + ... return "success!" + >>> with retrying(foo, sleeptime=0, jitter=0) as f: + ... f() + 1 + 2 + 3 + 'success!' + """ + yield retriable(*retry_args, **retry_kwargs)(func) diff --git a/third_party/python/redo/redo/cmd.py b/third_party/python/redo/redo/cmd.py new file mode 100644 index 0000000000..aeb65dbb3e --- /dev/null +++ b/third_party/python/redo/redo/cmd.py @@ -0,0 +1,70 @@ +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +import logging +from subprocess import check_call, CalledProcessError +import sys + +from redo import retrying + +log = logging.getLogger(__name__) + + +def main(argv): + from argparse import ArgumentParser, REMAINDER + + parser = ArgumentParser() + parser.add_argument( + "-a", "--attempts", type=int, default=5, help="How many times to retry." + ) + parser.add_argument( + "-s", + "--sleeptime", + type=int, + default=60, + help="How long to sleep between attempts. Sleeptime doubles after each attempt.", + ) + parser.add_argument( + "-m", + "--max-sleeptime", + type=int, + default=5 * 60, + help="Maximum length of time to sleep between attempts (limits backoff length).", + ) + parser.add_argument("-v", "--verbose", action="store_true", default=False) + parser.add_argument( + "cmd", nargs=REMAINDER, help="Command to run. Eg: wget http://blah" + ) + + args = parser.parse_args(argv[1:]) + + if args.verbose: + logging.basicConfig(level=logging.INFO) + logging.getLogger("retry").setLevel(logging.INFO) + else: + logging.basicConfig(level=logging.ERROR) + logging.getLogger("retry").setLevel(logging.ERROR) + + try: + with retrying( + check_call, + attempts=args.attempts, + sleeptime=args.sleeptime, + max_sleeptime=args.max_sleeptime, + retry_exceptions=(CalledProcessError,), + ) as r_check_call: + r_check_call(args.cmd) + except KeyboardInterrupt: + sys.exit(-1) + except Exception as e: + log.error( + "Unable to run command after %d attempts" % args.attempts, exc_info=True + ) + rc = getattr(e, "returncode", -2) + sys.exit(rc) + + +if __name__ == "__main__": + main(sys.argv) |