summaryrefslogtreecommitdiffstats
path: root/third_party/python/redo
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--third_party/python/redo/redo-2.0.3.dist-info/AUTHORS7
-rw-r--r--third_party/python/redo/redo-2.0.3.dist-info/METADATA13
-rw-r--r--third_party/python/redo/redo-2.0.3.dist-info/RECORD8
-rw-r--r--third_party/python/redo/redo-2.0.3.dist-info/WHEEL6
-rw-r--r--third_party/python/redo/redo-2.0.3.dist-info/entry_points.txt3
-rw-r--r--third_party/python/redo/redo-2.0.3.dist-info/top_level.txt1
-rw-r--r--third_party/python/redo/redo/__init__.py265
-rw-r--r--third_party/python/redo/redo/cmd.py70
8 files changed, 373 insertions, 0 deletions
diff --git a/third_party/python/redo/redo-2.0.3.dist-info/AUTHORS b/third_party/python/redo/redo-2.0.3.dist-info/AUTHORS
new file mode 100644
index 0000000000..b2e24333c0
--- /dev/null
+++ b/third_party/python/redo/redo-2.0.3.dist-info/AUTHORS
@@ -0,0 +1,7 @@
+Rail Aliiev (https://github.com/rail)
+Chris AtLee (https://github.com/catlee)
+Ben Hearsum (https://github.com/bhearsum)
+John Hopkins (https://github.com/jhopkinsmoz)
+Justin Wood (https://github.com/callek)
+Terry Chia (https://github.com/Ayrx)
+Mr. Deathless (https://github.com/mrdeathless)
diff --git a/third_party/python/redo/redo-2.0.3.dist-info/METADATA b/third_party/python/redo/redo-2.0.3.dist-info/METADATA
new file mode 100644
index 0000000000..e43a46e13a
--- /dev/null
+++ b/third_party/python/redo/redo-2.0.3.dist-info/METADATA
@@ -0,0 +1,13 @@
+Metadata-Version: 2.1
+Name: redo
+Version: 2.0.3
+Summary: Utilities to retry Python callables.
+Home-page: https://github.com/bhearsum/redo
+Author: Ben Hearsum
+Author-email: ben@hearsum.ca
+License: UNKNOWN
+Platform: UNKNOWN
+
+UNKNOWN
+
+
diff --git a/third_party/python/redo/redo-2.0.3.dist-info/RECORD b/third_party/python/redo/redo-2.0.3.dist-info/RECORD
new file mode 100644
index 0000000000..d8f82bf7da
--- /dev/null
+++ b/third_party/python/redo/redo-2.0.3.dist-info/RECORD
@@ -0,0 +1,8 @@
+redo/__init__.py,sha256=6VZUeFfbFkBJ_lxY_cJWk0S8mgSkrSRIwVniVm_sKsw,8518
+redo/cmd.py,sha256=F1axa3CVChlIvrSnq4xZZIyZ4M4wnnZjpv8wy46ugS4,2085
+redo-2.0.3.dist-info/AUTHORS,sha256=uIuTIaIlfQwklq75eg8VTjdnzENPlN_WKxa1UxQWTtQ,290
+redo-2.0.3.dist-info/METADATA,sha256=0DOrbjh62qccs3wFTgTxP9kQ1S4cphhUnupdfv0_6ms,233
+redo-2.0.3.dist-info/WHEEL,sha256=_wJFdOYk7i3xxT8ElOkUJvOdOvfNGbR9g-bf6UQT6sU,110
+redo-2.0.3.dist-info/entry_points.txt,sha256=ftcg9P_jTwZ9bYYDKB-s-5eIY6mHGiRHvd_HAGc7UPc,41
+redo-2.0.3.dist-info/top_level.txt,sha256=o1qXhN94bANfd7pz4zervaf8ytHEG_UVfhFZKMmmdvo,5
+redo-2.0.3.dist-info/RECORD,,
diff --git a/third_party/python/redo/redo-2.0.3.dist-info/WHEEL b/third_party/python/redo/redo-2.0.3.dist-info/WHEEL
new file mode 100644
index 0000000000..c4bde30377
--- /dev/null
+++ b/third_party/python/redo/redo-2.0.3.dist-info/WHEEL
@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.32.3)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+
diff --git a/third_party/python/redo/redo-2.0.3.dist-info/entry_points.txt b/third_party/python/redo/redo-2.0.3.dist-info/entry_points.txt
new file mode 100644
index 0000000000..44eccdcfca
--- /dev/null
+++ b/third_party/python/redo/redo-2.0.3.dist-info/entry_points.txt
@@ -0,0 +1,3 @@
+[console_scripts]
+retry = redo.cmd:main
+
diff --git a/third_party/python/redo/redo-2.0.3.dist-info/top_level.txt b/third_party/python/redo/redo-2.0.3.dist-info/top_level.txt
new file mode 100644
index 0000000000..f49789cbab
--- /dev/null
+++ b/third_party/python/redo/redo-2.0.3.dist-info/top_level.txt
@@ -0,0 +1 @@
+redo
diff --git a/third_party/python/redo/redo/__init__.py b/third_party/python/redo/redo/__init__.py
new file mode 100644
index 0000000000..9814805990
--- /dev/null
+++ b/third_party/python/redo/redo/__init__.py
@@ -0,0 +1,265 @@
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+
+import time
+from functools import wraps
+from contextlib import contextmanager
+import logging
+import random
+
+log = logging.getLogger(__name__)
+
+
+def retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=1.5, jitter=1):
+ """
+ A generator function that sleeps between retries, handles exponential
+ backoff and jitter. The action you are retrying is meant to run after
+ retrier yields.
+
+ At each iteration, we sleep for sleeptime + random.uniform(-jitter, jitter).
+ Afterwards sleeptime is multiplied by sleepscale for the next iteration.
+
+ Args:
+ attempts (int): maximum number of times to try; defaults to 5
+ sleeptime (float): how many seconds to sleep between tries; defaults to
+ 10 seconds
+ max_sleeptime (float): the longest we'll sleep, in seconds; defaults to
+ 300s (five minutes)
+ sleepscale (float): how much to multiply the sleep time by each
+ iteration; defaults to 1.5
+ jitter (float): random jitter to introduce to sleep time each iteration.
+ the amount is chosen at random between [-jitter, +jitter]
+ defaults to 1
+
+ Yields:
+ None, a maximum of `attempts` number of times
+
+ Example:
+ >>> n = 0
+ >>> for _ in retrier(sleeptime=0, jitter=0):
+ ... if n == 3:
+ ... # We did the thing!
+ ... break
+ ... n += 1
+ >>> n
+ 3
+
+ >>> n = 0
+ >>> for _ in retrier(sleeptime=0, jitter=0):
+ ... if n == 6:
+ ... # We did the thing!
+ ... break
+ ... n += 1
+ ... else:
+ ... print("max tries hit")
+ max tries hit
+ """
+ jitter = jitter or 0 # py35 barfs on the next line if jitter is None
+ if jitter > sleeptime:
+ # To prevent negative sleep times
+ raise Exception(
+ "jitter ({}) must be less than sleep time ({})".format(jitter, sleeptime)
+ )
+
+ sleeptime_real = sleeptime
+ for _ in range(attempts):
+ log.debug("attempt %i/%i", _ + 1, attempts)
+
+ yield sleeptime_real
+
+ if jitter:
+ sleeptime_real = sleeptime + random.uniform(-jitter, jitter)
+ # our jitter should scale along with the sleeptime
+ jitter = jitter * sleepscale
+ else:
+ sleeptime_real = sleeptime
+
+ sleeptime *= sleepscale
+
+ if sleeptime_real > max_sleeptime:
+ sleeptime_real = max_sleeptime
+
+ # Don't need to sleep the last time
+ if _ < attempts - 1:
+ log.debug(
+ "sleeping for %.2fs (attempt %i/%i)", sleeptime_real, _ + 1, attempts
+ )
+ time.sleep(sleeptime_real)
+
+
+def retry(
+ action,
+ attempts=5,
+ sleeptime=60,
+ max_sleeptime=5 * 60,
+ sleepscale=1.5,
+ jitter=1,
+ retry_exceptions=(Exception,),
+ cleanup=None,
+ args=(),
+ kwargs={},
+ log_args=True,
+):
+ """
+ Calls an action function until it succeeds, or we give up.
+
+ Args:
+ action (callable): the function to retry
+ attempts (int): maximum number of times to try; defaults to 5
+ sleeptime (float): how many seconds to sleep between tries; defaults to
+ 60s (one minute)
+ max_sleeptime (float): the longest we'll sleep, in seconds; defaults to
+ 300s (five minutes)
+ sleepscale (float): how much to multiply the sleep time by each
+ iteration; defaults to 1.5
+ jitter (float): random jitter to introduce to sleep time each iteration.
+ the amount is chosen at random between [-jitter, +jitter]
+ defaults to 1
+ retry_exceptions (tuple): tuple of exceptions to be caught. If other
+ exceptions are raised by action(), then these
+ are immediately re-raised to the caller.
+ cleanup (callable): optional; called if one of `retry_exceptions` is
+ caught. No arguments are passed to the cleanup
+ function; if your cleanup requires arguments,
+ consider using functools.partial or a lambda
+ function.
+ args (tuple): positional arguments to call `action` with
+ kwargs (dict): keyword arguments to call `action` with
+ log_args (bool): whether or not to include args and kwargs in log
+ messages. Defaults to True.
+
+ Returns:
+ Whatever action(*args, **kwargs) returns
+
+ Raises:
+ Whatever action(*args, **kwargs) raises. `retry_exceptions` are caught
+ up until the last attempt, in which case they are re-raised.
+
+ Example:
+ >>> count = 0
+ >>> def foo():
+ ... global count
+ ... count += 1
+ ... print(count)
+ ... if count < 3:
+ ... raise ValueError("count is too small!")
+ ... return "success!"
+ >>> retry(foo, sleeptime=0, jitter=0)
+ 1
+ 2
+ 3
+ 'success!'
+ """
+ assert callable(action)
+ assert not cleanup or callable(cleanup)
+
+ action_name = getattr(action, "__name__", action)
+ if log_args and (args or kwargs):
+ log_attempt_args = (
+ "retry: calling %s with args: %s," " kwargs: %s, attempt #%d",
+ action_name,
+ args,
+ kwargs,
+ )
+ else:
+ log_attempt_args = ("retry: calling %s, attempt #%d", action_name)
+
+ if max_sleeptime < sleeptime:
+ log.debug("max_sleeptime %d less than sleeptime %d", max_sleeptime, sleeptime)
+
+ n = 1
+ for _ in retrier(
+ attempts=attempts,
+ sleeptime=sleeptime,
+ max_sleeptime=max_sleeptime,
+ sleepscale=sleepscale,
+ jitter=jitter,
+ ):
+ try:
+ logfn = log.info if n != 1 else log.debug
+ logfn_args = log_attempt_args + (n,)
+ logfn(*logfn_args)
+ return action(*args, **kwargs)
+ except retry_exceptions:
+ log.debug("retry: Caught exception: ", exc_info=True)
+ if cleanup:
+ cleanup()
+ if n == attempts:
+ log.info("retry: Giving up on %s", action_name)
+ raise
+ continue
+ finally:
+ n += 1
+
+
+def retriable(*retry_args, **retry_kwargs):
+ """
+ A decorator factory for retry(). Wrap your function in @retriable(...) to
+ give it retry powers!
+
+ Arguments:
+ Same as for `retry`, with the exception of `action`, `args`, and `kwargs`,
+ which are left to the normal function definition.
+
+ Returns:
+ A function decorator
+
+ Example:
+ >>> count = 0
+ >>> @retriable(sleeptime=0, jitter=0)
+ ... def foo():
+ ... global count
+ ... count += 1
+ ... print(count)
+ ... if count < 3:
+ ... raise ValueError("count too small")
+ ... return "success!"
+ >>> foo()
+ 1
+ 2
+ 3
+ 'success!'
+ """
+
+ def _retriable_factory(func):
+ @wraps(func)
+ def _retriable_wrapper(*args, **kwargs):
+ return retry(func, args=args, kwargs=kwargs, *retry_args, **retry_kwargs)
+
+ return _retriable_wrapper
+
+ return _retriable_factory
+
+
+@contextmanager
+def retrying(func, *retry_args, **retry_kwargs):
+ """
+ A context manager for wrapping functions with retry functionality.
+
+ Arguments:
+ func (callable): the function to wrap
+ other arguments as per `retry`
+
+ Returns:
+ A context manager that returns retriable(func) on __enter__
+
+ Example:
+ >>> count = 0
+ >>> def foo():
+ ... global count
+ ... count += 1
+ ... print(count)
+ ... if count < 3:
+ ... raise ValueError("count too small")
+ ... return "success!"
+ >>> with retrying(foo, sleeptime=0, jitter=0) as f:
+ ... f()
+ 1
+ 2
+ 3
+ 'success!'
+ """
+ yield retriable(*retry_args, **retry_kwargs)(func)
diff --git a/third_party/python/redo/redo/cmd.py b/third_party/python/redo/redo/cmd.py
new file mode 100644
index 0000000000..aeb65dbb3e
--- /dev/null
+++ b/third_party/python/redo/redo/cmd.py
@@ -0,0 +1,70 @@
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+import logging
+from subprocess import check_call, CalledProcessError
+import sys
+
+from redo import retrying
+
+log = logging.getLogger(__name__)
+
+
+def main(argv):
+ from argparse import ArgumentParser, REMAINDER
+
+ parser = ArgumentParser()
+ parser.add_argument(
+ "-a", "--attempts", type=int, default=5, help="How many times to retry."
+ )
+ parser.add_argument(
+ "-s",
+ "--sleeptime",
+ type=int,
+ default=60,
+ help="How long to sleep between attempts. Sleeptime doubles after each attempt.",
+ )
+ parser.add_argument(
+ "-m",
+ "--max-sleeptime",
+ type=int,
+ default=5 * 60,
+ help="Maximum length of time to sleep between attempts (limits backoff length).",
+ )
+ parser.add_argument("-v", "--verbose", action="store_true", default=False)
+ parser.add_argument(
+ "cmd", nargs=REMAINDER, help="Command to run. Eg: wget http://blah"
+ )
+
+ args = parser.parse_args(argv[1:])
+
+ if args.verbose:
+ logging.basicConfig(level=logging.INFO)
+ logging.getLogger("retry").setLevel(logging.INFO)
+ else:
+ logging.basicConfig(level=logging.ERROR)
+ logging.getLogger("retry").setLevel(logging.ERROR)
+
+ try:
+ with retrying(
+ check_call,
+ attempts=args.attempts,
+ sleeptime=args.sleeptime,
+ max_sleeptime=args.max_sleeptime,
+ retry_exceptions=(CalledProcessError,),
+ ) as r_check_call:
+ r_check_call(args.cmd)
+ except KeyboardInterrupt:
+ sys.exit(-1)
+ except Exception as e:
+ log.error(
+ "Unable to run command after %d attempts" % args.attempts, exc_info=True
+ )
+ rc = getattr(e, "returncode", -2)
+ sys.exit(rc)
+
+
+if __name__ == "__main__":
+ main(sys.argv)