Adding upstream version 110.0.1.upstream/110.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

20 files changed, 1931 insertions, 0 deletions

diff --git a/third_party/python/dlmanager/README.rst b/third_party/python/dlmanager/README.rst
new file mode 100644
index 0000000000..e8db528fa2
--- /dev/null
+++ b/third_party/python/dlmanager/README.rst
@@ -0,0 +1,59 @@
+.. image:: https://badge.fury.io/py/dlmanager.svg
+    :target: https://pypi.python.org/pypi/dlmanager
+
+.. image:: https://readthedocs.org/projects/dlmanager/badge/?version=latest
+    :target: http://dlmanager.readthedocs.org/en/latest/?badge=latest
+    :alt: Documentation Status
+
+.. image:: https://travis-ci.org/parkouss/dlmanager.svg?branch=master
+    :target: https://travis-ci.org/parkouss/dlmanager
+
+.. image:: https://codecov.io/github/parkouss/dlmanager/coverage.svg?branch=master
+    :target: https://codecov.io/github/parkouss/dlmanager?branch=master
+
+dlmanager
+=========
+
+**dlmanager** is Python 2 and 3 download manager library, with the following
+features:
+
+- Download files in background and in parallel
+- Cancel downloads
+- store downloads in a given directory, avoiding re-downloading files
+- Limit the size of this directory, removing oldest files
+
+
+Example
+-------
+
+.. code-block:: python
+
+  from dlmanager import DownloadManager, PersistLimit
+
+  manager = DownloadManager(
+      "dlmanager-destir",
+      persist_limit=PersistLimit(
+          size_limit=1073741824,  #  1 GB max
+          file_limit=10,  # force to keep 10 files even if size_limit is reached
+      )
+  )
+
+  # Start downloads in background
+  # Note that if files are already present, this is a no-op.
+  manager.download(url1)
+  manager.download(url2)
+
+  # Wait for completion
+  try:
+      manager.wait()
+  except:
+      manager.cancel()
+      raise
+
+
+Installation
+------------
+
+Use pip: ::
+
+  pip install -U dlmanager
diff --git a/third_party/python/dlmanager/check.py b/third_party/python/dlmanager/check.py
new file mode 100755
index 0000000000..bcc842305e
--- /dev/null
+++ b/third_party/python/dlmanager/check.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+"""
+Run flake8 checks and tests.
+"""
+
+import os
+import argparse
+import pipes
+import shutil
+import tempfile
+
+from subprocess import check_call
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('-C', '--with-coverage', action='store_true',
+                        help="Generate coverage data from the tests run")
+    parser.add_argument('-H', '--cover-html', action='store_true',
+                        help='generate html files to see test coverage')
+    return parser.parse_args()
+
+
+def run(cmd, **kwargs):
+    msg = 'Running: |%s|' % ' '.join(pipes.quote(c) for c in cmd)
+    if kwargs.get('cwd'):
+        msg += ' in %s' % kwargs['cwd']
+    print(msg)
+    check_call(cmd, **kwargs)
+
+
+def rm(path):
+    if os.path.isfile(path):
+        os.unlink(path)
+    elif os.path.isdir(path):
+        shutil.rmtree(path)
+
+
+if __name__ == '__main__':
+    options = parse_args()
+
+    here = os.path.dirname(os.path.abspath(__file__))
+    os.chdir(here)
+
+    run(['flake8', 'dlmanager', 'tests', 'setup.py', __file__])
+
+    if options.with_coverage:
+        rm('.coverage')
+        test_run_cmd = ['coverage', 'run']
+    else:
+        test_run_cmd = ['python']
+
+    tmpdir = tempfile.gettempdir()
+    tmpfiles = set(os.listdir(tmpdir))
+    run(test_run_cmd + ['setup.py', 'test'])
+
+    remaining_tmpfiles = tmpfiles - set(os.listdir(tmpdir))
+    assert not remaining_tmpfiles, "tests leaked some temp files: %s" % (
+        ", ".join("`%s`" % os.path.join(tmpdir, f) for f in remaining_tmpfiles)
+    )
+
+    if options.with_coverage and options.cover_html:
+        rm('htmlcov')
+        run(['coverage', 'html'])
+        print("See coverage: |firefox %s|"
+              % os.path.join(here, 'htmlcov', 'index.html'))
diff --git a/third_party/python/dlmanager/dlmanager/__init__.py b/third_party/python/dlmanager/dlmanager/__init__.py
new file mode 100644
index 0000000000..0890af484a
--- /dev/null
+++ b/third_party/python/dlmanager/dlmanager/__init__.py
@@ -0,0 +1,18 @@
+import logging
+
+__version__ = "0.1.1"
+
+
+try:  # Python 2.7+
+    from logging import NullHandler
+except ImportError:
+    class NullHandler(logging.Handler):
+        def emit(self, record):
+            pass
+
+# Set default logging handler to avoid "No handler found" warnings.
+logging.getLogger(__name__).addHandler(NullHandler())
+
+# exported api
+from dlmanager.manager import Download, DownloadInterrupt, DownloadManager  # noqa
+from dlmanager.persist_limit import PersistLimit  # noqa
diff --git a/third_party/python/dlmanager/dlmanager/fs.py b/third_party/python/dlmanager/dlmanager/fs.py
new file mode 100644
index 0000000000..8908b5efce
--- /dev/null
+++ b/third_party/python/dlmanager/dlmanager/fs.py
@@ -0,0 +1,116 @@
+import errno
+import logging
+import os
+import shutil
+import stat
+import time
+
+"""
+File system utilities, copied from mozfile.
+"""
+
+LOG = logging.getLogger(__name__)
+
+
+def _call_windows_retry(func, args=(), retry_max=5, retry_delay=0.5):
+    """
+    It's possible to see spurious errors on Windows due to various things
+    keeping a handle to the directory open (explorer, virus scanners, etc)
+    So we try a few times if it fails with a known error.
+    """
+    retry_count = 0
+    while True:
+        try:
+            func(*args)
+        except OSError as e:
+            # Error codes are defined in:
+            # http://docs.python.org/2/library/errno.html#module-errno
+            if e.errno not in (errno.EACCES, errno.ENOTEMPTY):
+                raise
+
+            if retry_count == retry_max:
+                raise
+
+            retry_count += 1
+
+            LOG.info('%s() failed for "%s". Reason: %s (%s). Retrying...',
+                     func.__name__, args, e.strerror, e.errno)
+            time.sleep(retry_delay)
+        else:
+            # If no exception has been thrown it should be done
+            break
+
+
+def remove(path):
+    """Removes the specified file, link, or directory tree.
+
+    This is a replacement for shutil.rmtree that works better under
+    windows. It does the following things:
+
+     - check path access for the current user before trying to remove
+     - retry operations on some known errors due to various things keeping
+       a handle on file paths - like explorer, virus scanners, etc. The
+       known errors are errno.EACCES and errno.ENOTEMPTY, and it will
+       retry up to 5 five times with a delay of 0.5 seconds between each
+       attempt.
+
+    Note that no error will be raised if the given path does not exists.
+
+    :param path: path to be removed
+    """
+
+    def _call_with_windows_retry(*args, **kwargs):
+        try:
+            _call_windows_retry(*args, **kwargs)
+        except OSError as e:
+            # The file or directory to be removed doesn't exist anymore
+            if e.errno != errno.ENOENT:
+                raise
+
+    def _update_permissions(path):
+        """Sets specified pemissions depending on filetype"""
+        if os.path.islink(path):
+            # Path is a symlink which we don't have to modify
+            # because it should already have all the needed permissions
+            return
+
+        stats = os.stat(path)
+
+        if os.path.isfile(path):
+            mode = stats.st_mode | stat.S_IWUSR
+        elif os.path.isdir(path):
+            mode = stats.st_mode | stat.S_IWUSR | stat.S_IXUSR
+        else:
+            # Not supported type
+            return
+
+        _call_with_windows_retry(os.chmod, (path, mode))
+
+    if not os.path.exists(path):
+        return
+
+    if os.path.isfile(path) or os.path.islink(path):
+        # Verify the file or link is read/write for the current user
+        _update_permissions(path)
+        _call_with_windows_retry(os.remove, (path,))
+
+    elif os.path.isdir(path):
+        # Verify the directory is read/write/execute for the current user
+        _update_permissions(path)
+
+        # We're ensuring that every nested item has writable permission.
+        for root, dirs, files in os.walk(path):
+            for entry in dirs + files:
+                _update_permissions(os.path.join(root, entry))
+        _call_with_windows_retry(shutil.rmtree, (path,))
+
+
+def move(src, dst):
+    """
+    Move a file or directory path.
+
+    This is a replacement for shutil.move that works better under windows,
+    retrying operations on some known errors due to various things keeping
+    a handle on file paths.
+    """
+    _call_windows_retry(shutil.move, (src, dst))
diff --git a/third_party/python/dlmanager/dlmanager/manager.py b/third_party/python/dlmanager/dlmanager/manager.py
new file mode 100644
index 0000000000..3dce3b7838
--- /dev/null
+++ b/third_party/python/dlmanager/dlmanager/manager.py
@@ -0,0 +1,323 @@
+import os
+import requests
+import six
+import sys
+import tempfile
+import threading
+
+from contextlib import closing
+from six.moves.urllib.parse import urlparse
+
+from dlmanager import fs
+from dlmanager.persist_limit import PersistLimit
+
+
+class DownloadInterrupt(Exception):
+    "Raised when a download is interrupted."
+
+
+class Download(object):
+    """
+    Download is reponsible of downloading one file in the background.
+
+    Example of use: ::
+
+      dl = Download(url, dest)
+      dl.start()
+      dl.wait() # this will block until completion / cancel / error
+
+    If a download fail or is canceled, the temporary dest is removed from
+    the disk.
+
+    Usually, Downloads are created by using :meth:`DownloadManager.download`.
+
+    :param url: the url of the file to download
+    :param dest: the local file path destination
+    :param finished_callback: a callback that will be called in the thread
+                              when the thread work is done. Takes the download
+                              instance as a parameter.
+    :param chunk_size: size of the chunk that will be read. The thread can
+                        not be stopped while we are reading that chunk size.
+    :param session: a requests.Session instance that will do do the real
+                    downloading work. If None, `requests` module is used.
+    :param progress: A callable to report the progress (default to None).
+                     see :meth:`set_progress`.
+    """
+    def __init__(self, url, dest, finished_callback=None,
+                 chunk_size=16 * 1024, session=None, progress=None):
+        self.thread = threading.Thread(
+            target=self._download,
+            args=(url, dest, finished_callback, chunk_size,
+                  session or requests)
+        )
+        self._lock = threading.Lock()
+        self.__url = url
+        self.__dest = dest
+        self.__progress = progress
+        self.__canceled = False
+        self.__error = None
+
+    def start(self):
+        """
+        Start the thread that will do the download.
+        """
+        self.thread.start()
+
+    def cancel(self):
+        """
+        Cancel a previously started download.
+        """
+        self.__canceled = True
+
+    def is_canceled(self):
+        """
+        Returns True if we canceled this download.
+        """
+        return self.__canceled
+
+    def is_running(self):
+        """
+        Returns True if the downloading thread is running.
+        """
+        return self.thread.is_alive()
+
+    def wait(self, raise_if_error=True):
+        """
+        Block until the downloading thread is finished.
+
+        :param raise_if_error: if True (the default), :meth:`raise_if_error`
+                               will be called and raise an error if any.
+        """
+        while self.thread.is_alive():
+            try:
+                # in case of exception here (like KeyboardInterrupt),
+                # cancel the task.
+                self.thread.join(0.02)
+            except:
+                self.cancel()
+                raise
+        # this will raise exception that may happen inside the thread.
+        if raise_if_error:
+            self.raise_if_error()
+
+    def error(self):
+        """
+        Returns None or a tuple of three values (type, value, traceback)
+        that give information about the exception.
+        """
+        return self.__error
+
+    def raise_if_error(self):
+        """
+        Raise an error if any. If the download was canceled, raise
+        :class:`DownloadInterrupt`.
+        """
+        if self.__error:
+            six.reraise(*self.__error)
+        if self.__canceled:
+            raise DownloadInterrupt()
+
+    def set_progress(self, progress):
+        """
+        set a callable to report the progress of the download, or None to
+        disable any report.
+
+        The callable must take three parameters (download, current, total).
+        Note that this method is thread safe, you can call it during a
+        download.
+        """
+        with self._lock:
+            self.__progress = progress
+
+    def get_dest(self):
+        """
+        Returns the dest.
+        """
+        return self.__dest
+
+    def get_url(self):
+        """
+        Returns the url.
+        """
+        return self.__url
+
+    def _update_progress(self, current, total):
+        with self._lock:
+            if self.__progress:
+                self.__progress(self, current, total)
+
+    def _download(self, url, dest, finished_callback, chunk_size, session):
+        # save the file under a temporary name
+        # this allow to not use a broken file in case things went really bad
+        # while downloading the file (ie the python interpreter is killed
+        # abruptly)
+        temp = None
+        bytes_so_far = 0
+        try:
+            with closing(session.get(url, stream=True)) as response:
+                total_size = response.headers.get('Content-length', '').strip()
+                total_size = int(total_size) if total_size else None
+                self._update_progress(bytes_so_far, total_size)
+                # we use NamedTemporaryFile as raw open() call was causing
+                # issues on windows - see:
+                # https://bugzilla.mozilla.org/show_bug.cgi?id=1185756
+                with tempfile.NamedTemporaryFile(
+                        delete=False,
+                        suffix='.tmp',
+                        dir=os.path.dirname(dest)) as temp:
+                    for chunk in response.iter_content(chunk_size):
+                        if self.is_canceled():
+                            break
+                        if chunk:
+                            temp.write(chunk)
+                        bytes_so_far += len(chunk)
+                        self._update_progress(bytes_so_far, total_size)
+            response.raise_for_status()
+        except:
+            self.__error = sys.exc_info()
+        try:
+            if temp is None:
+                pass  # not even opened the temp file, nothing to do
+            elif self.is_canceled() or self.__error:
+                fs.remove(temp.name)
+            else:
+                # if all goes well, then rename the file to the real dest
+                fs.remove(dest)  # just in case it already existed
+                fs.move(temp.name, dest)
+        finally:
+            if finished_callback:
+                finished_callback(self)
+
+
+class DownloadManager(object):
+    """
+    DownloadManager is responsible of starting and managing downloads inside
+    a given directory. It will download a file only if a given filename
+    is not already there.
+
+    Note that background downloads needs to be stopped. For example, if
+    you have an exception while a download is occuring, python will only
+    exit when the download will finish. To get rid of that, there is a
+    possible idiom: ::
+
+      def download_things(manager):
+          # do things with the manager
+          manager.download(url1, f1)
+          manager.download(url2, f2)
+          ...
+
+      manager = DownloadManager(destdir)
+      try:
+          download_things(manager)
+      finally:
+          # ensure we cancel all background downloads to ask the end
+          # of possible remainings threads
+          manager.cancel()
+
+    :param destdir: a directory where files are downloaded. It will be created
+                    if it does not exists.
+    :param session: a requests session. If None, one will be created for you.
+    :param persist_limit: an instance of :class:`PersistLimit`, to allow
+                          limiting the size of the download dir. Defaults
+                          to None, meaning no limit.
+    """
+    def __init__(self, destdir, session=None, persist_limit=None):
+        self.destdir = destdir
+        self.session = session or requests.Session()
+        self._downloads = {}
+        self._lock = threading.Lock()
+        self.persist_limit = persist_limit or PersistLimit(0)
+        self.persist_limit.register_dir_content(self.destdir)
+
+        # if persist folder does not exist, create it
+        if not os.path.isdir(destdir):
+            os.makedirs(destdir)
+
+    def get_dest(self, fname):
+        return os.path.join(self.destdir, fname)
+
+    def cancel(self, cancel_if=None):
+        """
+        Cancel downloads, if any.
+
+        if cancel_if is given, it must be a callable that take the download
+        instance as parameter, and return True if the download needs to be
+        canceled.
+
+        Note that download threads won't be stopped directly.
+        """
+        with self._lock:
+            for download in six.itervalues(self._downloads):
+                if cancel_if is None or cancel_if(download):
+                    if download.is_running():
+                        download.cancel()
+
+    def wait(self, raise_if_error=True):
+        """
+        Wait for all downloads to be finished.
+        """
+        for download in self._downloads.values():
+            download.wait(raise_if_error=raise_if_error)
+
+    def download(self, url, fname=None, progress=None):
+        """
+        Returns a started :class:`Download` instance, or None if fname is
+        already present in destdir.
+
+        if a download is already running for the given fname, it is just
+        returned. Else the download is created, started and returned.
+
+        :param url: url of the file to download.
+        :param fname: name to give for the downloaded file. If None, it will
+                      be the name extracted in the url.
+        :param progress: a callable to report the download progress, or None.
+                         See :meth:`Download.set_progress`.
+        """
+        if fname is None:
+            fname = urlparse(url).path.split('/')[-1]
+        dest = self.get_dest(fname)
+        with self._lock:
+            # if we are downloading, returns the instance
+            if dest in self._downloads:
+                dl = self._downloads[dest]
+                if progress:
+                    dl.set_progress(progress)
+                return dl
+
+        if os.path.exists(dest):
+            return None
+
+        # else create the download (will be automatically removed of
+        # the list on completion) start it, and returns that.
+        with self._lock:
+            download = Download(url, dest,
+                                session=self.session,
+                                finished_callback=self._download_finished,
+                                progress=progress)
+            self._downloads[dest] = download
+            download.start()
+            self._download_started(download)
+            return download
+
+    def _download_started(self, dl):
+        """
+        Useful when sub-classing. Report the start event of a download.
+
+        :param dl: The :class:`Download` instance.
+        """
+        pass
+
+    def _download_finished(self, dl):
+        """
+        Useful when sub-classing. Report the end of a download.
+
+        Note that this is executed in the download thread. Also, you should
+        make sure to call the base implementation.
+
+        :param dl: The :class:`Download` instance.
+        """
+        with self._lock:
+            dest = dl.get_dest()
+            del self._downloads[dest]
+            self.persist_limit.register_file(dest)
+            self.persist_limit.remove_old_files()
diff --git a/third_party/python/dlmanager/dlmanager/persist_limit.py b/third_party/python/dlmanager/dlmanager/persist_limit.py
new file mode 100644
index 0000000000..03a1829f70
--- /dev/null
+++ b/third_party/python/dlmanager/dlmanager/persist_limit.py
@@ -0,0 +1,65 @@
+import os
+import stat
+
+from collections import namedtuple
+from glob import glob
+
+from dlmanager import fs
+
+
+File = namedtuple('File', ('path', 'stat'))
+
+
+class PersistLimit(object):
+    """
+    Keep a list of files, removing the oldest ones when the size_limit
+    is reached.
+
+    The access time of a file is used to determine the oldests, e.g. the
+    last time a file was read.
+
+    :param size_limit: the size limit in bytes. A value of 0 means no limit.
+    :param file_limit: even if the size limit is reached, this force
+                       to keep at least *file_limit* files.
+    """
+    def __init__(self, size_limit, file_limit=5):
+        self.size_limit = size_limit
+        self.file_limit = file_limit
+        self.files = []
+        self._files_size = 0
+
+    def register_file(self, path):
+        """
+        register a single file.
+        """
+        try:
+            fstat = os.stat(path)
+        except OSError:
+            # file do not exists probably, just skip it
+            # note this happen when backgound files are canceled
+            return
+        if stat.S_ISREG(fstat.st_mode):
+            self.files.append(File(path=path, stat=fstat))
+            self._files_size += fstat.st_size
+
+    def register_dir_content(self, directory, pattern="*"):
+        """
+        Register every files in a directory that match *pattern*.
+        """
+        for path in glob(os.path.join(directory, pattern)):
+            self.register_file(path)
+
+    def remove_old_files(self):
+        """
+        remove oldest registered files.
+        """
+        if self.size_limit <= 0 or self.file_limit <= 0:
+            return
+        # sort by creation time, oldest first
+        files = sorted(self.files, key=lambda f: f.stat.st_atime)
+        while len(files) > self.file_limit and \
+                self._files_size >= self.size_limit:
+            f = files.pop(0)
+            fs.remove(f.path)
+            self._files_size -= f.stat.st_size
+        self.files = files
diff --git a/third_party/python/dlmanager/doc/Makefile b/third_party/python/dlmanager/doc/Makefile
new file mode 100644
index 0000000000..6b477bf459
--- /dev/null
+++ b/third_party/python/dlmanager/doc/Makefile
@@ -0,0 +1,216 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  applehelp  to make an Apple Help Book"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+	@echo "  coverage   to run coverage check of the documentation (if enabled)"
+
+.PHONY: clean
+clean:
+	rm -rf $(BUILDDIR)/*
+
+.PHONY: html
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+.PHONY: dirhtml
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+.PHONY: singlehtml
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+.PHONY: pickle
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+.PHONY: json
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+.PHONY: htmlhelp
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+.PHONY: qthelp
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/dlmanager.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/dlmanager.qhc"
+
+.PHONY: applehelp
+applehelp:
+	$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
+	@echo
+	@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
+	@echo "N.B. You won't be able to view it unless you put it in" \
+	      "~/Library/Documentation/Help or install it in your application" \
+	      "bundle."
+
+.PHONY: devhelp
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/dlmanager"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/dlmanager"
+	@echo "# devhelp"
+
+.PHONY: epub
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+.PHONY: latex
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+.PHONY: latexpdf
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+.PHONY: latexpdfja
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+.PHONY: text
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+.PHONY: man
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+.PHONY: texinfo
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+.PHONY: info
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+.PHONY: gettext
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+.PHONY: changes
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+.PHONY: linkcheck
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+.PHONY: doctest
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+.PHONY: coverage
+coverage:
+	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
+	@echo "Testing of coverage in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/coverage/python.txt."
+
+.PHONY: xml
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+.PHONY: pseudoxml
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
diff --git a/third_party/python/dlmanager/doc/api.rst b/third_party/python/dlmanager/doc/api.rst
new file mode 100644
index 0000000000..295ce7c1fa
--- /dev/null
+++ b/third_party/python/dlmanager/doc/api.rst
@@ -0,0 +1,25 @@
+API
+===
+
+DownloadManager
+---------------
+
+.. currentmodule:: dlmanager
+
+.. autoclass:: DownloadManager
+   :members:
+
+Download
+--------
+
+.. autoclass:: Download
+   :members:
+
+.. autoclass:: DownloadInterrupt
+   :members:
+
+PersistLimit
+------------
+
+.. autoclass:: PersistLimit
+   :members:
diff --git a/third_party/python/dlmanager/doc/conf.py b/third_party/python/dlmanager/doc/conf.py
new file mode 100644
index 0000000000..80bb5172d2
--- /dev/null
+++ b/third_party/python/dlmanager/doc/conf.py
@@ -0,0 +1,289 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# dlmanager documentation build configuration file, created by
+# sphinx-quickstart on Fri Feb 19 11:22:21 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('..'))
+
+from dlmanager import __version__
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.viewcode',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'dlmanager'
+copyright = u'2016, Julien Pagès'
+author = u'Julien Pagès'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = __version__
+# The full version, including alpha/beta/rc tags.
+release = version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Language to be used for generating the HTML full-text search index.
+# Sphinx supports the following languages:
+#   'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
+#   'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr'
+#html_search_language = 'en'
+
+# A dictionary with options for the search language support, empty by default.
+# Now only 'ja' uses this config value
+#html_search_options = {'type': 'default'}
+
+# The name of a javascript file (relative to the configuration directory) that
+# implements a search results scorer. If empty, the default will be used.
+#html_search_scorer = 'scorer.js'
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'dlmanagerdoc'
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+
+# Latex figure (float) alignment
+#'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'dlmanager.tex', 'dlmanager Documentation',
+     'Julien Pagès', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'dlmanager', 'dlmanager Documentation',
+     [author], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'dlmanager', 'dlmanager Documentation',
+     author, 'dlmanager', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#texinfo_no_detailmenu = False
diff --git a/third_party/python/dlmanager/doc/index.rst b/third_party/python/dlmanager/doc/index.rst
new file mode 100644
index 0000000000..c585e573ad
--- /dev/null
+++ b/third_party/python/dlmanager/doc/index.rst
@@ -0,0 +1,26 @@
+.. dlmanager documentation master file, created by
+   sphinx-quickstart on Fri Feb 19 11:22:21 2016.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to dlmanager's documentation!
+=====================================
+
+**dlmanager** is a Python 2 and 3 download manager library. It is hosted
+`on github <https://github.com/parkouss/dlmanager>`_.
+
+Contents:
+
+.. toctree::
+   :maxdepth: 2
+
+   api
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+
diff --git a/third_party/python/dlmanager/doc/make.bat b/third_party/python/dlmanager/doc/make.bat
new file mode 100644
index 0000000000..5bcee17fab
--- /dev/null
+++ b/third_party/python/dlmanager/doc/make.bat
@@ -0,0 +1,263 @@
+@ECHO OFF

+

+REM Command file for Sphinx documentation

+

+if "%SPHINXBUILD%" == "" (

+	set SPHINXBUILD=sphinx-build

+)

+set BUILDDIR=_build

+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .

+set I18NSPHINXOPTS=%SPHINXOPTS% .

+if NOT "%PAPER%" == "" (

+	set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%

+	set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%

+)

+

+if "%1" == "" goto help

+

+if "%1" == "help" (

+	:help

+	echo.Please use `make ^<target^>` where ^<target^> is one of

+	echo.  html       to make standalone HTML files

+	echo.  dirhtml    to make HTML files named index.html in directories

+	echo.  singlehtml to make a single large HTML file

+	echo.  pickle     to make pickle files

+	echo.  json       to make JSON files

+	echo.  htmlhelp   to make HTML files and a HTML help project

+	echo.  qthelp     to make HTML files and a qthelp project

+	echo.  devhelp    to make HTML files and a Devhelp project

+	echo.  epub       to make an epub

+	echo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter

+	echo.  text       to make text files

+	echo.  man        to make manual pages

+	echo.  texinfo    to make Texinfo files

+	echo.  gettext    to make PO message catalogs

+	echo.  changes    to make an overview over all changed/added/deprecated items

+	echo.  xml        to make Docutils-native XML files

+	echo.  pseudoxml  to make pseudoxml-XML files for display purposes

+	echo.  linkcheck  to check all external links for integrity

+	echo.  doctest    to run all doctests embedded in the documentation if enabled

+	echo.  coverage   to run coverage check of the documentation if enabled

+	goto end

+)

+

+if "%1" == "clean" (

+	for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i

+	del /q /s %BUILDDIR%\*

+	goto end

+)

+

+

+REM Check if sphinx-build is available and fallback to Python version if any

+%SPHINXBUILD% 1>NUL 2>NUL

+if errorlevel 9009 goto sphinx_python

+goto sphinx_ok

+

+:sphinx_python

+

+set SPHINXBUILD=python -m sphinx.__init__

+%SPHINXBUILD% 2> nul

+if errorlevel 9009 (

+	echo.

+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx

+	echo.installed, then set the SPHINXBUILD environment variable to point

+	echo.to the full path of the 'sphinx-build' executable. Alternatively you

+	echo.may add the Sphinx directory to PATH.

+	echo.

+	echo.If you don't have Sphinx installed, grab it from

+	echo.http://sphinx-doc.org/

+	exit /b 1

+)

+

+:sphinx_ok

+

+

+if "%1" == "html" (

+	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished. The HTML pages are in %BUILDDIR%/html.

+	goto end

+)

+

+if "%1" == "dirhtml" (

+	%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.

+	goto end

+)

+

+if "%1" == "singlehtml" (

+	%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.

+	goto end

+)

+

+if "%1" == "pickle" (

+	%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished; now you can process the pickle files.

+	goto end

+)

+

+if "%1" == "json" (

+	%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished; now you can process the JSON files.

+	goto end

+)

+

+if "%1" == "htmlhelp" (

+	%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished; now you can run HTML Help Workshop with the ^

+.hhp project file in %BUILDDIR%/htmlhelp.

+	goto end

+)

+

+if "%1" == "qthelp" (

+	%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished; now you can run "qcollectiongenerator" with the ^

+.qhcp project file in %BUILDDIR%/qthelp, like this:

+	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\dlmanager.qhcp

+	echo.To view the help file:

+	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\dlmanager.ghc

+	goto end

+)

+

+if "%1" == "devhelp" (

+	%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished.

+	goto end

+)

+

+if "%1" == "epub" (

+	%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished. The epub file is in %BUILDDIR%/epub.

+	goto end

+)

+

+if "%1" == "latex" (

+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.

+	goto end

+)

+

+if "%1" == "latexpdf" (

+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex

+	cd %BUILDDIR%/latex

+	make all-pdf

+	cd %~dp0

+	echo.

+	echo.Build finished; the PDF files are in %BUILDDIR%/latex.

+	goto end

+)

+

+if "%1" == "latexpdfja" (

+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex

+	cd %BUILDDIR%/latex

+	make all-pdf-ja

+	cd %~dp0

+	echo.

+	echo.Build finished; the PDF files are in %BUILDDIR%/latex.

+	goto end

+)

+

+if "%1" == "text" (

+	%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished. The text files are in %BUILDDIR%/text.

+	goto end

+)

+

+if "%1" == "man" (

+	%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished. The manual pages are in %BUILDDIR%/man.

+	goto end

+)

+

+if "%1" == "texinfo" (

+	%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.

+	goto end

+)

+

+if "%1" == "gettext" (

+	%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished. The message catalogs are in %BUILDDIR%/locale.

+	goto end

+)

+

+if "%1" == "changes" (

+	%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.The overview file is in %BUILDDIR%/changes.

+	goto end

+)

+

+if "%1" == "linkcheck" (

+	%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Link check complete; look for any errors in the above output ^

+or in %BUILDDIR%/linkcheck/output.txt.

+	goto end

+)

+

+if "%1" == "doctest" (

+	%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Testing of doctests in the sources finished, look at the ^

+results in %BUILDDIR%/doctest/output.txt.

+	goto end

+)

+

+if "%1" == "coverage" (

+	%SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Testing of coverage in the sources finished, look at the ^

+results in %BUILDDIR%/coverage/python.txt.

+	goto end

+)

+

+if "%1" == "xml" (

+	%SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished. The XML files are in %BUILDDIR%/xml.

+	goto end

+)

+

+if "%1" == "pseudoxml" (

+	%SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml

+	if errorlevel 1 exit /b 1

+	echo.

+	echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.

+	goto end

+)

+

+:end

diff --git a/third_party/python/dlmanager/examples/dl_progressbar.py b/third_party/python/dlmanager/examples/dl_progressbar.py
new file mode 100644
index 0000000000..98c36d55b6
--- /dev/null
+++ b/third_party/python/dlmanager/examples/dl_progressbar.py
@@ -0,0 +1,41 @@
+import argparse
+
+# for python 3, use https://github.com/coagulant/progressbar-python3
+from progressbar import ProgressBar, Percentage, RotatingMarker, ETA, \
+    FileTransferSpeed, Bar
+
+from six.moves.urllib.parse import urlparse
+
+from dlmanager import Download
+
+
+def parse_args(argv=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("url", help="url to download")
+    return parser.parse_args(argv)
+
+
+def download_file(url, dest=None):
+    if dest is None:
+        dest = urlparse(url).path.split('/')[-1]
+
+    widgets = ['Download: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
+               ' ', ETA(), ' ', FileTransferSpeed()]
+    bar = ProgressBar(widgets=widgets).start()
+
+    def download_progress(_, current, total):
+        bar.maxval = total
+        bar.update(current)
+
+    dl = Download(url, dest, progress=download_progress)
+    dl.start()
+    dl.wait()
+    bar.finish()
+
+
+if __name__ == '__main__':
+    options = parse_args()
+    try:
+        download_file(options.url)
+    except KeyboardInterrupt:
+        print("\nInterrupted.")
diff --git a/third_party/python/dlmanager/examples/dl_tqdm.py b/third_party/python/dlmanager/examples/dl_tqdm.py
new file mode 100644
index 0000000000..a4e458a415
--- /dev/null
+++ b/third_party/python/dlmanager/examples/dl_tqdm.py
@@ -0,0 +1,45 @@
+import argparse
+import tqdm
+
+from six.moves.urllib.parse import urlparse
+
+from dlmanager import Download
+
+
+def parse_args(argv=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("url", help="url to download")
+    return parser.parse_args(argv)
+
+
+def download_progress(bar):
+    last_b = [0]
+
+    def inner(_, current, total):
+        if total is not None:
+            bar.total = total
+        delta = current - last_b[0]
+        last_b[0] = current
+
+        if delta > 0:
+            bar.update(delta)
+    return inner
+
+
+def download_file(url, dest=None):
+    if dest is None:
+        dest = urlparse(url).path.split('/')[-1]
+
+    with tqdm.tqdm(unit='B', unit_scale=True, miniters=1, dynamic_ncols=True,
+                   desc=dest) as bar:
+        dl = Download(url, dest, progress=download_progress(bar))
+        dl.start()
+        dl.wait()
+
+
+if __name__ == '__main__':
+    options = parse_args()
+    try:
+        download_file(options.url)
+    except KeyboardInterrupt:
+        print("\nInterrupted.")
diff --git a/third_party/python/dlmanager/requirements.txt b/third_party/python/dlmanager/requirements.txt
new file mode 100644
index 0000000000..640e3d44a6
--- /dev/null
+++ b/third_party/python/dlmanager/requirements.txt
@@ -0,0 +1,2 @@
+requests
+six
diff --git a/third_party/python/dlmanager/setup.cfg b/third_party/python/dlmanager/setup.cfg
new file mode 100644
index 0000000000..3c6e79cf31
--- /dev/null
+++ b/third_party/python/dlmanager/setup.cfg
@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal=1
diff --git a/third_party/python/dlmanager/setup.py b/third_party/python/dlmanager/setup.py
new file mode 100644
index 0000000000..b2a8fd392d
--- /dev/null
+++ b/third_party/python/dlmanager/setup.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+import os
+import re
+import sys
+from setuptools import setup
+from setuptools.command.test import test as TestCommand
+
+HERE = os.path.dirname(os.path.realpath(__file__))
+
+
+class PyTest(TestCommand):
+    """
+    Run py.test with the "python setup.py test command"
+    """
+    user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")]
+
+    def initialize_options(self):
+        TestCommand.initialize_options(self)
+        self.pytest_args = ''
+
+    def finalize_options(self):
+        TestCommand.finalize_options(self)
+        self.pytest_args += (' ' + self.distribution.test_suite)
+
+    def run_tests(self):
+        import pytest
+        errno = pytest.main(self.pytest_args)
+        sys.exit(errno)
+
+
+def read(*parts):
+    with open(os.path.join(HERE, *parts)) as f:
+        return f.read()
+
+
+def parse_requirements(data, exclude=()):
+    return [line for line in data.splitlines()
+            if line and not line.startswith("#") and line not in exclude]
+
+
+def version():
+    return re.findall(r"__version__ = \"([\d.]+)\"",
+                      read("dlmanager", "__init__.py"))[0]
+
+setup(
+    name="dlmanager",
+    version=version(),
+    description="download manager library",
+    long_description=read("README.rst"),
+    author="Julien Pagès",
+    author_email="j.parkouss@gmail.com",
+    url="http://github.com/parkouss/dlmanager",
+    license="GPL/LGPL",
+    install_requires=parse_requirements(read("requirements.txt")),
+    cmdclass={'test': PyTest},
+    tests_require=parse_requirements(read("requirements.txt"),
+                                     exclude=("-e .",)),
+    test_suite='tests',
+)
diff --git a/third_party/python/dlmanager/test-requirements.txt b/third_party/python/dlmanager/test-requirements.txt
new file mode 100644
index 0000000000..a4db4b7672
--- /dev/null
+++ b/third_party/python/dlmanager/test-requirements.txt
@@ -0,0 +1,7 @@
+-e .
+mock
+pytest
+pytest-mock
+flake8
+coverage
+unittest2; python_version < '2.7'
diff --git a/third_party/python/dlmanager/tests/__init__.py b/third_party/python/dlmanager/tests/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/third_party/python/dlmanager/tests/__init__.py
diff --git a/third_party/python/dlmanager/tests/test_manager.py b/third_party/python/dlmanager/tests/test_manager.py
new file mode 100644
index 0000000000..f0ade9021f
--- /dev/null
+++ b/third_party/python/dlmanager/tests/test_manager.py
@@ -0,0 +1,251 @@
+try:
+    import unittest2 as unittest  # python < 2.7 compat
+except ImportError:
+    import unittest
+import tempfile
+import shutil
+import os
+import time
+import six
+from mock import Mock
+
+from dlmanager import manager as download_manager
+
+
+def mock_session():
+    response = Mock()
+    session = Mock(get=Mock(return_value=response))
+    return session, response
+
+
+def mock_response(response, data, wait=0):
+    data = six.b(data)
+
+    def iter_content(chunk_size=4):
+        rest = data
+        while rest:
+            time.sleep(wait)
+            chunk = rest[:chunk_size]
+            rest = rest[chunk_size:]
+            yield chunk
+
+    response.headers = {'Content-length': str(len(data))}
+    response.iter_content = iter_content
+
+
+class TestDownload(unittest.TestCase):
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tempdir)
+        self.finished = Mock()
+        self.session, self.session_response = mock_session()
+        self.tempfile = os.path.join(self.tempdir, 'dest')
+        self.dl = download_manager.Download('http://url', self.tempfile,
+                                            finished_callback=self.finished,
+                                            chunk_size=4,
+                                            session=self.session)
+
+    def test_creation(self):
+        self.assertFalse(self.dl.is_canceled())
+        self.assertFalse(self.dl.is_running())
+        self.assertIsNone(self.dl.error())
+        self.assertEquals(self.dl.get_url(), 'http://url')
+        self.assertEquals(self.dl.get_dest(), self.tempfile)
+
+    def create_response(self, data, wait=0):
+        mock_response(self.session_response, data, wait)
+
+    def test_download(self):
+        self.create_response('1234' * 4, 0.01)
+
+        # no file present yet
+        self.assertFalse(os.path.exists(self.tempfile))
+
+        self.dl.start()
+        self.assertTrue(self.dl.is_running())
+        self.dl.wait()
+
+        self.assertFalse(self.dl.is_running())
+        self.finished.assert_called_with(self.dl)
+        # file has been downloaded
+        with open(self.tempfile) as f:
+            self.assertEquals(f.read(), '1234' * 4)
+
+    def test_download_cancel(self):
+        self.create_response('1234' * 1000, wait=0.01)
+
+        start = time.time()
+        self.dl.start()
+        time.sleep(0.1)
+        self.dl.cancel()
+
+        with self.assertRaises(download_manager.DownloadInterrupt):
+            self.dl.wait()
+
+        self.assertTrue(self.dl.is_canceled())
+
+        # response generation should have taken 1000 * 0.01 = 10 seconds.
+        # since we canceled, this must be lower.
+        self.assertTrue((time.time() - start) < 1.0)
+
+        # file was deleted
+        self.assertFalse(os.path.exists(self.tempfile))
+        # finished callback was called
+        self.finished.assert_called_with(self.dl)
+
+    def test_download_with_progress(self):
+        data = []
+
+        def update_progress(_dl, current, total):
+            data.append((_dl, current, total))
+
+        self.create_response('1234' * 4)
+
+        self.dl.set_progress(update_progress)
+        self.dl.start()
+        self.dl.wait()
+
+        self.assertEquals(data, [
+            (self.dl, 0, 16),
+            (self.dl, 4, 16),
+            (self.dl, 8, 16),
+            (self.dl, 12, 16),
+            (self.dl, 16, 16),
+        ])
+        # file has been downloaded
+        with open(self.tempfile) as f:
+            self.assertEquals(f.read(), '1234' * 4)
+        # finished callback was called
+        self.finished.assert_called_with(self.dl)
+
+    def test_download_error_in_thread(self):
+        self.session_response.headers = {'Content-length': '24'}
+        self.session_response.iter_content.side_effect = IOError
+
+        self.dl.start()
+        with self.assertRaises(IOError):
+            self.dl.wait()
+
+        self.assertEquals(self.dl.error()[0], IOError)
+        # finished callback was called
+        self.finished.assert_called_with(self.dl)
+
+    def test_wait_does_not_block_on_exception(self):
+        # this test the case when a user may hit CTRL-C for example
+        # during a dl.wait() call.
+        self.create_response('1234' * 1000, wait=0.01)
+
+        original_join = self.dl.thread.join
+        it = iter('123')
+
+        def join(timeout=None):
+            next(it)  # will throw StopIteration after a few calls
+            original_join(timeout)
+
+        self.dl.thread.join = join
+
+        start = time.time()
+        self.dl.start()
+
+        with self.assertRaises(StopIteration):
+            self.dl.wait()
+
+        self.assertTrue(self.dl.is_canceled())
+        # wait for the thread to finish
+        original_join()
+
+        # response generation should have taken 1000 * 0.01 = 10 seconds.
+        # since we got an error, this must be lower.
+        self.assertTrue((time.time() - start) < 1.0)
+
+        # file was deleted
+        self.assertFalse(os.path.exists(self.tempfile))
+        # finished callback was called
+        self.finished.assert_called_with(self.dl)
+
+
+class TestDownloadManager(unittest.TestCase):
+    def setUp(self):
+        self.tempdir = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tempdir)
+
+        self.dl_manager = download_manager.DownloadManager(self.tempdir)
+
+    def do_download(self, url, fname, data, wait=0):
+        session, response = mock_session()
+        mock_response(response, data, wait)
+        # patch the session, so the download will use that
+        self.dl_manager.session = session
+        return self.dl_manager.download(url, fname)
+
+    def test_download(self):
+        dl1 = self.do_download('http://foo', 'foo', 'hello' * 4, wait=0.02)
+        self.assertIsInstance(dl1, download_manager.Download)
+        self.assertTrue(dl1.is_running())
+
+        # with the same fname, no new download is started. The same instance
+        # is returned since the download is running.
+        dl2 = self.do_download('http://bar', 'foo', 'hello2' * 4, wait=0.02)
+        self.assertEquals(dl1, dl2)
+
+        # starting a download with another fname will trigger a new download
+        dl3 = self.do_download('http://bar', 'foo2', 'hello you' * 4)
+        self.assertIsInstance(dl3, download_manager.Download)
+        self.assertNotEquals(dl3, dl1)
+
+        # let's wait for the downloads to finish
+        dl3.wait()
+        dl1.wait()
+
+        # now if we try to download a fname that exists, None is returned
+        dl4 = self.do_download('http://bar', 'foo', 'hello2' * 4, wait=0.02)
+        self.assertIsNone(dl4)
+
+        # downloaded files are what is expected
+        def content(fname):
+            with open(os.path.join(self.tempdir, fname)) as f:
+                return f.read()
+        self.assertEquals(content('foo'), 'hello' * 4)
+        self.assertEquals(content('foo2'), 'hello you' * 4)
+
+        # download instances are removed from the manager (internal test)
+        self.assertEquals(self.dl_manager._downloads, {})
+
+    def test_cancel(self):
+        dl1 = self.do_download('http://foo', 'foo', 'foo' * 50000, wait=0.02)
+        dl2 = self.do_download('http://foo', 'bar', 'bar' * 50000, wait=0.02)
+        dl3 = self.do_download('http://foo', 'foobar', 'foobar' * 4)
+
+        # let's cancel only one
+        def cancel_if(dl):
+            if os.path.basename(dl.get_dest()) == 'foo':
+                return True
+        self.dl_manager.cancel(cancel_if=cancel_if)
+
+        self.assertTrue(dl1.is_canceled())
+        self.assertFalse(dl2.is_canceled())
+        self.assertFalse(dl3.is_canceled())
+
+        # wait for dl3
+        dl3.wait()
+
+        # cancel everything
+        self.dl_manager.cancel()
+
+        self.assertTrue(dl1.is_canceled())
+        self.assertTrue(dl2.is_canceled())
+        # dl3 is not canceled since it finished before
+        self.assertFalse(dl3.is_canceled())
+
+        # wait for the completion of dl1 and dl2 threads
+        dl1.wait(raise_if_error=False)
+        dl2.wait(raise_if_error=False)
+
+        # at the end, only dl3 has been downloaded
+        self.assertEquals(os.listdir(self.tempdir), ["foobar"])
+
+        with open(os.path.join(self.tempdir, 'foobar')) as f:
+            self.assertEquals(f.read(), 'foobar' * 4)
+
+        # download instances are removed from the manager (internal test)
+        self.assertEquals(self.dl_manager._downloads, {})
diff --git a/third_party/python/dlmanager/tests/test_persist_limit.py b/third_party/python/dlmanager/tests/test_persist_limit.py
new file mode 100644
index 0000000000..1d899a46f2
--- /dev/null
+++ b/third_party/python/dlmanager/tests/test_persist_limit.py
@@ -0,0 +1,56 @@
+import pytest
+import os
+import tempfile
+import time
+import six
+
+from dlmanager import fs
+from dlmanager.persist_limit import PersistLimit
+
+
+class TempCreator(object):
+    def __init__(self):
+        self.tempdir = tempfile.mkdtemp()
+
+    def list(self):
+        return os.listdir(self.tempdir)
+
+    def create_file(self, name, size, delay):
+        fname = os.path.join(self.tempdir, name)
+        with open(fname, 'wb') as f:
+            f.write(six.b('a' * size))
+        # equivalent to touch, but we apply a delay for the test
+        atime = time.time() + delay
+        os.utime(fname, (atime, atime))
+
+
+@pytest.yield_fixture
+def temp():
+    tmp = TempCreator()
+    yield tmp
+    fs.remove(tmp.tempdir)
+
+
+@pytest.mark.parametrize("size_limit,file_limit,files", [
+    # limit_file is always respected
+    (10, 5, "bcdef"),
+    (10, 3, "def"),
+    # if size_limit or file_limit is 0, nothing is removed
+    (0, 5, "abcdef"),
+    (5, 0, "abcdef"),
+    # limit_size works
+    (35, 1, "def"),
+])
+def test_persist_limit(temp, size_limit, file_limit, files):
+    temp.create_file("a", 10, -6)
+    temp.create_file("b", 10, -5)
+    temp.create_file("c", 10, -4)
+    temp.create_file("d", 10, -3)
+    temp.create_file("e", 10, -2)
+    temp.create_file("f", 10, -1)
+
+    persist_limit = PersistLimit(size_limit, file_limit)
+    persist_limit.register_dir_content(temp.tempdir)
+    persist_limit.remove_old_files()
+
+    assert ''.join(sorted(temp.list())) == ''.join(sorted(files))