diff options
Diffstat (limited to 'third_party/python/dlmanager')
20 files changed, 1931 insertions, 0 deletions
diff --git a/third_party/python/dlmanager/README.rst b/third_party/python/dlmanager/README.rst new file mode 100644 index 0000000000..e8db528fa2 --- /dev/null +++ b/third_party/python/dlmanager/README.rst @@ -0,0 +1,59 @@ +.. image:: https://badge.fury.io/py/dlmanager.svg + :target: https://pypi.python.org/pypi/dlmanager + +.. image:: https://readthedocs.org/projects/dlmanager/badge/?version=latest + :target: http://dlmanager.readthedocs.org/en/latest/?badge=latest + :alt: Documentation Status + +.. image:: https://travis-ci.org/parkouss/dlmanager.svg?branch=master + :target: https://travis-ci.org/parkouss/dlmanager + +.. image:: https://codecov.io/github/parkouss/dlmanager/coverage.svg?branch=master + :target: https://codecov.io/github/parkouss/dlmanager?branch=master + +dlmanager +========= + +**dlmanager** is Python 2 and 3 download manager library, with the following +features: + +- Download files in background and in parallel +- Cancel downloads +- store downloads in a given directory, avoiding re-downloading files +- Limit the size of this directory, removing oldest files + + +Example +------- + +.. code-block:: python + + from dlmanager import DownloadManager, PersistLimit + + manager = DownloadManager( + "dlmanager-destir", + persist_limit=PersistLimit( + size_limit=1073741824, # 1 GB max + file_limit=10, # force to keep 10 files even if size_limit is reached + ) + ) + + # Start downloads in background + # Note that if files are already present, this is a no-op. + manager.download(url1) + manager.download(url2) + + # Wait for completion + try: + manager.wait() + except: + manager.cancel() + raise + + +Installation +------------ + +Use pip: :: + + pip install -U dlmanager diff --git a/third_party/python/dlmanager/check.py b/third_party/python/dlmanager/check.py new file mode 100755 index 0000000000..bcc842305e --- /dev/null +++ b/third_party/python/dlmanager/check.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +""" +Run flake8 checks and tests. +""" + +import os +import argparse +import pipes +import shutil +import tempfile + +from subprocess import check_call + + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument('-C', '--with-coverage', action='store_true', + help="Generate coverage data from the tests run") + parser.add_argument('-H', '--cover-html', action='store_true', + help='generate html files to see test coverage') + return parser.parse_args() + + +def run(cmd, **kwargs): + msg = 'Running: |%s|' % ' '.join(pipes.quote(c) for c in cmd) + if kwargs.get('cwd'): + msg += ' in %s' % kwargs['cwd'] + print(msg) + check_call(cmd, **kwargs) + + +def rm(path): + if os.path.isfile(path): + os.unlink(path) + elif os.path.isdir(path): + shutil.rmtree(path) + + +if __name__ == '__main__': + options = parse_args() + + here = os.path.dirname(os.path.abspath(__file__)) + os.chdir(here) + + run(['flake8', 'dlmanager', 'tests', 'setup.py', __file__]) + + if options.with_coverage: + rm('.coverage') + test_run_cmd = ['coverage', 'run'] + else: + test_run_cmd = ['python'] + + tmpdir = tempfile.gettempdir() + tmpfiles = set(os.listdir(tmpdir)) + run(test_run_cmd + ['setup.py', 'test']) + + remaining_tmpfiles = tmpfiles - set(os.listdir(tmpdir)) + assert not remaining_tmpfiles, "tests leaked some temp files: %s" % ( + ", ".join("`%s`" % os.path.join(tmpdir, f) for f in remaining_tmpfiles) + ) + + if options.with_coverage and options.cover_html: + rm('htmlcov') + run(['coverage', 'html']) + print("See coverage: |firefox %s|" + % os.path.join(here, 'htmlcov', 'index.html')) diff --git a/third_party/python/dlmanager/dlmanager/__init__.py b/third_party/python/dlmanager/dlmanager/__init__.py new file mode 100644 index 0000000000..0890af484a --- /dev/null +++ b/third_party/python/dlmanager/dlmanager/__init__.py @@ -0,0 +1,18 @@ +import logging + +__version__ = "0.1.1" + + +try: # Python 2.7+ + from logging import NullHandler +except ImportError: + class NullHandler(logging.Handler): + def emit(self, record): + pass + +# Set default logging handler to avoid "No handler found" warnings. +logging.getLogger(__name__).addHandler(NullHandler()) + +# exported api +from dlmanager.manager import Download, DownloadInterrupt, DownloadManager # noqa +from dlmanager.persist_limit import PersistLimit # noqa diff --git a/third_party/python/dlmanager/dlmanager/fs.py b/third_party/python/dlmanager/dlmanager/fs.py new file mode 100644 index 0000000000..8908b5efce --- /dev/null +++ b/third_party/python/dlmanager/dlmanager/fs.py @@ -0,0 +1,116 @@ +import errno +import logging +import os +import shutil +import stat +import time + +""" +File system utilities, copied from mozfile. +""" + +LOG = logging.getLogger(__name__) + + +def _call_windows_retry(func, args=(), retry_max=5, retry_delay=0.5): + """ + It's possible to see spurious errors on Windows due to various things + keeping a handle to the directory open (explorer, virus scanners, etc) + So we try a few times if it fails with a known error. + """ + retry_count = 0 + while True: + try: + func(*args) + except OSError as e: + # Error codes are defined in: + # http://docs.python.org/2/library/errno.html#module-errno + if e.errno not in (errno.EACCES, errno.ENOTEMPTY): + raise + + if retry_count == retry_max: + raise + + retry_count += 1 + + LOG.info('%s() failed for "%s". Reason: %s (%s). Retrying...', + func.__name__, args, e.strerror, e.errno) + time.sleep(retry_delay) + else: + # If no exception has been thrown it should be done + break + + +def remove(path): + """Removes the specified file, link, or directory tree. + + This is a replacement for shutil.rmtree that works better under + windows. It does the following things: + + - check path access for the current user before trying to remove + - retry operations on some known errors due to various things keeping + a handle on file paths - like explorer, virus scanners, etc. The + known errors are errno.EACCES and errno.ENOTEMPTY, and it will + retry up to 5 five times with a delay of 0.5 seconds between each + attempt. + + Note that no error will be raised if the given path does not exists. + + :param path: path to be removed + """ + + def _call_with_windows_retry(*args, **kwargs): + try: + _call_windows_retry(*args, **kwargs) + except OSError as e: + # The file or directory to be removed doesn't exist anymore + if e.errno != errno.ENOENT: + raise + + def _update_permissions(path): + """Sets specified pemissions depending on filetype""" + if os.path.islink(path): + # Path is a symlink which we don't have to modify + # because it should already have all the needed permissions + return + + stats = os.stat(path) + + if os.path.isfile(path): + mode = stats.st_mode | stat.S_IWUSR + elif os.path.isdir(path): + mode = stats.st_mode | stat.S_IWUSR | stat.S_IXUSR + else: + # Not supported type + return + + _call_with_windows_retry(os.chmod, (path, mode)) + + if not os.path.exists(path): + return + + if os.path.isfile(path) or os.path.islink(path): + # Verify the file or link is read/write for the current user + _update_permissions(path) + _call_with_windows_retry(os.remove, (path,)) + + elif os.path.isdir(path): + # Verify the directory is read/write/execute for the current user + _update_permissions(path) + + # We're ensuring that every nested item has writable permission. + for root, dirs, files in os.walk(path): + for entry in dirs + files: + _update_permissions(os.path.join(root, entry)) + _call_with_windows_retry(shutil.rmtree, (path,)) + + +def move(src, dst): + """ + Move a file or directory path. + + This is a replacement for shutil.move that works better under windows, + retrying operations on some known errors due to various things keeping + a handle on file paths. + """ + _call_windows_retry(shutil.move, (src, dst)) diff --git a/third_party/python/dlmanager/dlmanager/manager.py b/third_party/python/dlmanager/dlmanager/manager.py new file mode 100644 index 0000000000..3dce3b7838 --- /dev/null +++ b/third_party/python/dlmanager/dlmanager/manager.py @@ -0,0 +1,323 @@ +import os +import requests +import six +import sys +import tempfile +import threading + +from contextlib import closing +from six.moves.urllib.parse import urlparse + +from dlmanager import fs +from dlmanager.persist_limit import PersistLimit + + +class DownloadInterrupt(Exception): + "Raised when a download is interrupted." + + +class Download(object): + """ + Download is reponsible of downloading one file in the background. + + Example of use: :: + + dl = Download(url, dest) + dl.start() + dl.wait() # this will block until completion / cancel / error + + If a download fail or is canceled, the temporary dest is removed from + the disk. + + Usually, Downloads are created by using :meth:`DownloadManager.download`. + + :param url: the url of the file to download + :param dest: the local file path destination + :param finished_callback: a callback that will be called in the thread + when the thread work is done. Takes the download + instance as a parameter. + :param chunk_size: size of the chunk that will be read. The thread can + not be stopped while we are reading that chunk size. + :param session: a requests.Session instance that will do do the real + downloading work. If None, `requests` module is used. + :param progress: A callable to report the progress (default to None). + see :meth:`set_progress`. + """ + def __init__(self, url, dest, finished_callback=None, + chunk_size=16 * 1024, session=None, progress=None): + self.thread = threading.Thread( + target=self._download, + args=(url, dest, finished_callback, chunk_size, + session or requests) + ) + self._lock = threading.Lock() + self.__url = url + self.__dest = dest + self.__progress = progress + self.__canceled = False + self.__error = None + + def start(self): + """ + Start the thread that will do the download. + """ + self.thread.start() + + def cancel(self): + """ + Cancel a previously started download. + """ + self.__canceled = True + + def is_canceled(self): + """ + Returns True if we canceled this download. + """ + return self.__canceled + + def is_running(self): + """ + Returns True if the downloading thread is running. + """ + return self.thread.is_alive() + + def wait(self, raise_if_error=True): + """ + Block until the downloading thread is finished. + + :param raise_if_error: if True (the default), :meth:`raise_if_error` + will be called and raise an error if any. + """ + while self.thread.is_alive(): + try: + # in case of exception here (like KeyboardInterrupt), + # cancel the task. + self.thread.join(0.02) + except: + self.cancel() + raise + # this will raise exception that may happen inside the thread. + if raise_if_error: + self.raise_if_error() + + def error(self): + """ + Returns None or a tuple of three values (type, value, traceback) + that give information about the exception. + """ + return self.__error + + def raise_if_error(self): + """ + Raise an error if any. If the download was canceled, raise + :class:`DownloadInterrupt`. + """ + if self.__error: + six.reraise(*self.__error) + if self.__canceled: + raise DownloadInterrupt() + + def set_progress(self, progress): + """ + set a callable to report the progress of the download, or None to + disable any report. + + The callable must take three parameters (download, current, total). + Note that this method is thread safe, you can call it during a + download. + """ + with self._lock: + self.__progress = progress + + def get_dest(self): + """ + Returns the dest. + """ + return self.__dest + + def get_url(self): + """ + Returns the url. + """ + return self.__url + + def _update_progress(self, current, total): + with self._lock: + if self.__progress: + self.__progress(self, current, total) + + def _download(self, url, dest, finished_callback, chunk_size, session): + # save the file under a temporary name + # this allow to not use a broken file in case things went really bad + # while downloading the file (ie the python interpreter is killed + # abruptly) + temp = None + bytes_so_far = 0 + try: + with closing(session.get(url, stream=True)) as response: + total_size = response.headers.get('Content-length', '').strip() + total_size = int(total_size) if total_size else None + self._update_progress(bytes_so_far, total_size) + # we use NamedTemporaryFile as raw open() call was causing + # issues on windows - see: + # https://bugzilla.mozilla.org/show_bug.cgi?id=1185756 + with tempfile.NamedTemporaryFile( + delete=False, + suffix='.tmp', + dir=os.path.dirname(dest)) as temp: + for chunk in response.iter_content(chunk_size): + if self.is_canceled(): + break + if chunk: + temp.write(chunk) + bytes_so_far += len(chunk) + self._update_progress(bytes_so_far, total_size) + response.raise_for_status() + except: + self.__error = sys.exc_info() + try: + if temp is None: + pass # not even opened the temp file, nothing to do + elif self.is_canceled() or self.__error: + fs.remove(temp.name) + else: + # if all goes well, then rename the file to the real dest + fs.remove(dest) # just in case it already existed + fs.move(temp.name, dest) + finally: + if finished_callback: + finished_callback(self) + + +class DownloadManager(object): + """ + DownloadManager is responsible of starting and managing downloads inside + a given directory. It will download a file only if a given filename + is not already there. + + Note that background downloads needs to be stopped. For example, if + you have an exception while a download is occuring, python will only + exit when the download will finish. To get rid of that, there is a + possible idiom: :: + + def download_things(manager): + # do things with the manager + manager.download(url1, f1) + manager.download(url2, f2) + ... + + manager = DownloadManager(destdir) + try: + download_things(manager) + finally: + # ensure we cancel all background downloads to ask the end + # of possible remainings threads + manager.cancel() + + :param destdir: a directory where files are downloaded. It will be created + if it does not exists. + :param session: a requests session. If None, one will be created for you. + :param persist_limit: an instance of :class:`PersistLimit`, to allow + limiting the size of the download dir. Defaults + to None, meaning no limit. + """ + def __init__(self, destdir, session=None, persist_limit=None): + self.destdir = destdir + self.session = session or requests.Session() + self._downloads = {} + self._lock = threading.Lock() + self.persist_limit = persist_limit or PersistLimit(0) + self.persist_limit.register_dir_content(self.destdir) + + # if persist folder does not exist, create it + if not os.path.isdir(destdir): + os.makedirs(destdir) + + def get_dest(self, fname): + return os.path.join(self.destdir, fname) + + def cancel(self, cancel_if=None): + """ + Cancel downloads, if any. + + if cancel_if is given, it must be a callable that take the download + instance as parameter, and return True if the download needs to be + canceled. + + Note that download threads won't be stopped directly. + """ + with self._lock: + for download in six.itervalues(self._downloads): + if cancel_if is None or cancel_if(download): + if download.is_running(): + download.cancel() + + def wait(self, raise_if_error=True): + """ + Wait for all downloads to be finished. + """ + for download in self._downloads.values(): + download.wait(raise_if_error=raise_if_error) + + def download(self, url, fname=None, progress=None): + """ + Returns a started :class:`Download` instance, or None if fname is + already present in destdir. + + if a download is already running for the given fname, it is just + returned. Else the download is created, started and returned. + + :param url: url of the file to download. + :param fname: name to give for the downloaded file. If None, it will + be the name extracted in the url. + :param progress: a callable to report the download progress, or None. + See :meth:`Download.set_progress`. + """ + if fname is None: + fname = urlparse(url).path.split('/')[-1] + dest = self.get_dest(fname) + with self._lock: + # if we are downloading, returns the instance + if dest in self._downloads: + dl = self._downloads[dest] + if progress: + dl.set_progress(progress) + return dl + + if os.path.exists(dest): + return None + + # else create the download (will be automatically removed of + # the list on completion) start it, and returns that. + with self._lock: + download = Download(url, dest, + session=self.session, + finished_callback=self._download_finished, + progress=progress) + self._downloads[dest] = download + download.start() + self._download_started(download) + return download + + def _download_started(self, dl): + """ + Useful when sub-classing. Report the start event of a download. + + :param dl: The :class:`Download` instance. + """ + pass + + def _download_finished(self, dl): + """ + Useful when sub-classing. Report the end of a download. + + Note that this is executed in the download thread. Also, you should + make sure to call the base implementation. + + :param dl: The :class:`Download` instance. + """ + with self._lock: + dest = dl.get_dest() + del self._downloads[dest] + self.persist_limit.register_file(dest) + self.persist_limit.remove_old_files() diff --git a/third_party/python/dlmanager/dlmanager/persist_limit.py b/third_party/python/dlmanager/dlmanager/persist_limit.py new file mode 100644 index 0000000000..03a1829f70 --- /dev/null +++ b/third_party/python/dlmanager/dlmanager/persist_limit.py @@ -0,0 +1,65 @@ +import os +import stat + +from collections import namedtuple +from glob import glob + +from dlmanager import fs + + +File = namedtuple('File', ('path', 'stat')) + + +class PersistLimit(object): + """ + Keep a list of files, removing the oldest ones when the size_limit + is reached. + + The access time of a file is used to determine the oldests, e.g. the + last time a file was read. + + :param size_limit: the size limit in bytes. A value of 0 means no limit. + :param file_limit: even if the size limit is reached, this force + to keep at least *file_limit* files. + """ + def __init__(self, size_limit, file_limit=5): + self.size_limit = size_limit + self.file_limit = file_limit + self.files = [] + self._files_size = 0 + + def register_file(self, path): + """ + register a single file. + """ + try: + fstat = os.stat(path) + except OSError: + # file do not exists probably, just skip it + # note this happen when backgound files are canceled + return + if stat.S_ISREG(fstat.st_mode): + self.files.append(File(path=path, stat=fstat)) + self._files_size += fstat.st_size + + def register_dir_content(self, directory, pattern="*"): + """ + Register every files in a directory that match *pattern*. + """ + for path in glob(os.path.join(directory, pattern)): + self.register_file(path) + + def remove_old_files(self): + """ + remove oldest registered files. + """ + if self.size_limit <= 0 or self.file_limit <= 0: + return + # sort by creation time, oldest first + files = sorted(self.files, key=lambda f: f.stat.st_atime) + while len(files) > self.file_limit and \ + self._files_size >= self.size_limit: + f = files.pop(0) + fs.remove(f.path) + self._files_size -= f.stat.st_size + self.files = files diff --git a/third_party/python/dlmanager/doc/Makefile b/third_party/python/dlmanager/doc/Makefile new file mode 100644 index 0000000000..6b477bf459 --- /dev/null +++ b/third_party/python/dlmanager/doc/Makefile @@ -0,0 +1,216 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + +.PHONY: clean +clean: + rm -rf $(BUILDDIR)/* + +.PHONY: html +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +.PHONY: dirhtml +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +.PHONY: singlehtml +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +.PHONY: pickle +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +.PHONY: json +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +.PHONY: htmlhelp +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +.PHONY: qthelp +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/dlmanager.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/dlmanager.qhc" + +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/dlmanager" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/dlmanager" + @echo "# devhelp" + +.PHONY: epub +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +.PHONY: latex +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +.PHONY: latexpdf +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: latexpdfja +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +.PHONY: text +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +.PHONY: man +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +.PHONY: texinfo +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +.PHONY: info +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +.PHONY: gettext +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +.PHONY: changes +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +.PHONY: linkcheck +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +.PHONY: doctest +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +.PHONY: pseudoxml +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/third_party/python/dlmanager/doc/api.rst b/third_party/python/dlmanager/doc/api.rst new file mode 100644 index 0000000000..295ce7c1fa --- /dev/null +++ b/third_party/python/dlmanager/doc/api.rst @@ -0,0 +1,25 @@ +API +=== + +DownloadManager +--------------- + +.. currentmodule:: dlmanager + +.. autoclass:: DownloadManager + :members: + +Download +-------- + +.. autoclass:: Download + :members: + +.. autoclass:: DownloadInterrupt + :members: + +PersistLimit +------------ + +.. autoclass:: PersistLimit + :members: diff --git a/third_party/python/dlmanager/doc/conf.py b/third_party/python/dlmanager/doc/conf.py new file mode 100644 index 0000000000..80bb5172d2 --- /dev/null +++ b/third_party/python/dlmanager/doc/conf.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# dlmanager documentation build configuration file, created by +# sphinx-quickstart on Fri Feb 19 11:22:21 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('..')) + +from dlmanager import __version__ + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'dlmanager' +copyright = u'2016, Julien Pagès' +author = u'Julien Pagès' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = __version__ +# The full version, including alpha/beta/rc tags. +release = version + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'dlmanagerdoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'dlmanager.tex', 'dlmanager Documentation', + 'Julien Pagès', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'dlmanager', 'dlmanager Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'dlmanager', 'dlmanager Documentation', + author, 'dlmanager', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False diff --git a/third_party/python/dlmanager/doc/index.rst b/third_party/python/dlmanager/doc/index.rst new file mode 100644 index 0000000000..c585e573ad --- /dev/null +++ b/third_party/python/dlmanager/doc/index.rst @@ -0,0 +1,26 @@ +.. dlmanager documentation master file, created by + sphinx-quickstart on Fri Feb 19 11:22:21 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to dlmanager's documentation! +===================================== + +**dlmanager** is a Python 2 and 3 download manager library. It is hosted +`on github <https://github.com/parkouss/dlmanager>`_. + +Contents: + +.. toctree:: + :maxdepth: 2 + + api + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/third_party/python/dlmanager/doc/make.bat b/third_party/python/dlmanager/doc/make.bat new file mode 100644 index 0000000000..5bcee17fab --- /dev/null +++ b/third_party/python/dlmanager/doc/make.bat @@ -0,0 +1,263 @@ +@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+ set SPHINXBUILD=sphinx-build
+)
+set BUILDDIR=_build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
+set I18NSPHINXOPTS=%SPHINXOPTS% .
+if NOT "%PAPER%" == "" (
+ set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+ set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+ :help
+ echo.Please use `make ^<target^>` where ^<target^> is one of
+ echo. html to make standalone HTML files
+ echo. dirhtml to make HTML files named index.html in directories
+ echo. singlehtml to make a single large HTML file
+ echo. pickle to make pickle files
+ echo. json to make JSON files
+ echo. htmlhelp to make HTML files and a HTML help project
+ echo. qthelp to make HTML files and a qthelp project
+ echo. devhelp to make HTML files and a Devhelp project
+ echo. epub to make an epub
+ echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+ echo. text to make text files
+ echo. man to make manual pages
+ echo. texinfo to make Texinfo files
+ echo. gettext to make PO message catalogs
+ echo. changes to make an overview over all changed/added/deprecated items
+ echo. xml to make Docutils-native XML files
+ echo. pseudoxml to make pseudoxml-XML files for display purposes
+ echo. linkcheck to check all external links for integrity
+ echo. doctest to run all doctests embedded in the documentation if enabled
+ echo. coverage to run coverage check of the documentation if enabled
+ goto end
+)
+
+if "%1" == "clean" (
+ for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+ del /q /s %BUILDDIR%\*
+ goto end
+)
+
+
+REM Check if sphinx-build is available and fallback to Python version if any
+%SPHINXBUILD% 1>NUL 2>NUL
+if errorlevel 9009 goto sphinx_python
+goto sphinx_ok
+
+:sphinx_python
+
+set SPHINXBUILD=python -m sphinx.__init__
+%SPHINXBUILD% 2> nul
+if errorlevel 9009 (
+ echo.
+ echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+ echo.installed, then set the SPHINXBUILD environment variable to point
+ echo.to the full path of the 'sphinx-build' executable. Alternatively you
+ echo.may add the Sphinx directory to PATH.
+ echo.
+ echo.If you don't have Sphinx installed, grab it from
+ echo.http://sphinx-doc.org/
+ exit /b 1
+)
+
+:sphinx_ok
+
+
+if "%1" == "html" (
+ %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+ goto end
+)
+
+if "%1" == "dirhtml" (
+ %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+ goto end
+)
+
+if "%1" == "singlehtml" (
+ %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
+ goto end
+)
+
+if "%1" == "pickle" (
+ %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the pickle files.
+ goto end
+)
+
+if "%1" == "json" (
+ %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the JSON files.
+ goto end
+)
+
+if "%1" == "htmlhelp" (
+ %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+ goto end
+)
+
+if "%1" == "qthelp" (
+ %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+ echo.^> qcollectiongenerator %BUILDDIR%\qthelp\dlmanager.qhcp
+ echo.To view the help file:
+ echo.^> assistant -collectionFile %BUILDDIR%\qthelp\dlmanager.ghc
+ goto end
+)
+
+if "%1" == "devhelp" (
+ %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished.
+ goto end
+)
+
+if "%1" == "epub" (
+ %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The epub file is in %BUILDDIR%/epub.
+ goto end
+)
+
+if "%1" == "latex" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "latexpdf" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ cd %BUILDDIR%/latex
+ make all-pdf
+ cd %~dp0
+ echo.
+ echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "latexpdfja" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ cd %BUILDDIR%/latex
+ make all-pdf-ja
+ cd %~dp0
+ echo.
+ echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "text" (
+ %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The text files are in %BUILDDIR%/text.
+ goto end
+)
+
+if "%1" == "man" (
+ %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The manual pages are in %BUILDDIR%/man.
+ goto end
+)
+
+if "%1" == "texinfo" (
+ %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
+ goto end
+)
+
+if "%1" == "gettext" (
+ %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
+ goto end
+)
+
+if "%1" == "changes" (
+ %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.The overview file is in %BUILDDIR%/changes.
+ goto end
+)
+
+if "%1" == "linkcheck" (
+ %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+ goto end
+)
+
+if "%1" == "doctest" (
+ %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+ goto end
+)
+
+if "%1" == "coverage" (
+ %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Testing of coverage in the sources finished, look at the ^
+results in %BUILDDIR%/coverage/python.txt.
+ goto end
+)
+
+if "%1" == "xml" (
+ %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The XML files are in %BUILDDIR%/xml.
+ goto end
+)
+
+if "%1" == "pseudoxml" (
+ %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
+ goto end
+)
+
+:end
diff --git a/third_party/python/dlmanager/examples/dl_progressbar.py b/third_party/python/dlmanager/examples/dl_progressbar.py new file mode 100644 index 0000000000..98c36d55b6 --- /dev/null +++ b/third_party/python/dlmanager/examples/dl_progressbar.py @@ -0,0 +1,41 @@ +import argparse + +# for python 3, use https://github.com/coagulant/progressbar-python3 +from progressbar import ProgressBar, Percentage, RotatingMarker, ETA, \ + FileTransferSpeed, Bar + +from six.moves.urllib.parse import urlparse + +from dlmanager import Download + + +def parse_args(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("url", help="url to download") + return parser.parse_args(argv) + + +def download_file(url, dest=None): + if dest is None: + dest = urlparse(url).path.split('/')[-1] + + widgets = ['Download: ', Percentage(), ' ', Bar(marker=RotatingMarker()), + ' ', ETA(), ' ', FileTransferSpeed()] + bar = ProgressBar(widgets=widgets).start() + + def download_progress(_, current, total): + bar.maxval = total + bar.update(current) + + dl = Download(url, dest, progress=download_progress) + dl.start() + dl.wait() + bar.finish() + + +if __name__ == '__main__': + options = parse_args() + try: + download_file(options.url) + except KeyboardInterrupt: + print("\nInterrupted.") diff --git a/third_party/python/dlmanager/examples/dl_tqdm.py b/third_party/python/dlmanager/examples/dl_tqdm.py new file mode 100644 index 0000000000..a4e458a415 --- /dev/null +++ b/third_party/python/dlmanager/examples/dl_tqdm.py @@ -0,0 +1,45 @@ +import argparse +import tqdm + +from six.moves.urllib.parse import urlparse + +from dlmanager import Download + + +def parse_args(argv=None): + parser = argparse.ArgumentParser() + parser.add_argument("url", help="url to download") + return parser.parse_args(argv) + + +def download_progress(bar): + last_b = [0] + + def inner(_, current, total): + if total is not None: + bar.total = total + delta = current - last_b[0] + last_b[0] = current + + if delta > 0: + bar.update(delta) + return inner + + +def download_file(url, dest=None): + if dest is None: + dest = urlparse(url).path.split('/')[-1] + + with tqdm.tqdm(unit='B', unit_scale=True, miniters=1, dynamic_ncols=True, + desc=dest) as bar: + dl = Download(url, dest, progress=download_progress(bar)) + dl.start() + dl.wait() + + +if __name__ == '__main__': + options = parse_args() + try: + download_file(options.url) + except KeyboardInterrupt: + print("\nInterrupted.") diff --git a/third_party/python/dlmanager/requirements.txt b/third_party/python/dlmanager/requirements.txt new file mode 100644 index 0000000000..640e3d44a6 --- /dev/null +++ b/third_party/python/dlmanager/requirements.txt @@ -0,0 +1,2 @@ +requests +six diff --git a/third_party/python/dlmanager/setup.cfg b/third_party/python/dlmanager/setup.cfg new file mode 100644 index 0000000000..3c6e79cf31 --- /dev/null +++ b/third_party/python/dlmanager/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/third_party/python/dlmanager/setup.py b/third_party/python/dlmanager/setup.py new file mode 100644 index 0000000000..b2a8fd392d --- /dev/null +++ b/third_party/python/dlmanager/setup.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +import os +import re +import sys +from setuptools import setup +from setuptools.command.test import test as TestCommand + +HERE = os.path.dirname(os.path.realpath(__file__)) + + +class PyTest(TestCommand): + """ + Run py.test with the "python setup.py test command" + """ + user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] + + def initialize_options(self): + TestCommand.initialize_options(self) + self.pytest_args = '' + + def finalize_options(self): + TestCommand.finalize_options(self) + self.pytest_args += (' ' + self.distribution.test_suite) + + def run_tests(self): + import pytest + errno = pytest.main(self.pytest_args) + sys.exit(errno) + + +def read(*parts): + with open(os.path.join(HERE, *parts)) as f: + return f.read() + + +def parse_requirements(data, exclude=()): + return [line for line in data.splitlines() + if line and not line.startswith("#") and line not in exclude] + + +def version(): + return re.findall(r"__version__ = \"([\d.]+)\"", + read("dlmanager", "__init__.py"))[0] + +setup( + name="dlmanager", + version=version(), + description="download manager library", + long_description=read("README.rst"), + author="Julien Pagès", + author_email="j.parkouss@gmail.com", + url="http://github.com/parkouss/dlmanager", + license="GPL/LGPL", + install_requires=parse_requirements(read("requirements.txt")), + cmdclass={'test': PyTest}, + tests_require=parse_requirements(read("requirements.txt"), + exclude=("-e .",)), + test_suite='tests', +) diff --git a/third_party/python/dlmanager/test-requirements.txt b/third_party/python/dlmanager/test-requirements.txt new file mode 100644 index 0000000000..a4db4b7672 --- /dev/null +++ b/third_party/python/dlmanager/test-requirements.txt @@ -0,0 +1,7 @@ +-e . +mock +pytest +pytest-mock +flake8 +coverage +unittest2; python_version < '2.7' diff --git a/third_party/python/dlmanager/tests/__init__.py b/third_party/python/dlmanager/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/third_party/python/dlmanager/tests/__init__.py diff --git a/third_party/python/dlmanager/tests/test_manager.py b/third_party/python/dlmanager/tests/test_manager.py new file mode 100644 index 0000000000..f0ade9021f --- /dev/null +++ b/third_party/python/dlmanager/tests/test_manager.py @@ -0,0 +1,251 @@ +try: + import unittest2 as unittest # python < 2.7 compat +except ImportError: + import unittest +import tempfile +import shutil +import os +import time +import six +from mock import Mock + +from dlmanager import manager as download_manager + + +def mock_session(): + response = Mock() + session = Mock(get=Mock(return_value=response)) + return session, response + + +def mock_response(response, data, wait=0): + data = six.b(data) + + def iter_content(chunk_size=4): + rest = data + while rest: + time.sleep(wait) + chunk = rest[:chunk_size] + rest = rest[chunk_size:] + yield chunk + + response.headers = {'Content-length': str(len(data))} + response.iter_content = iter_content + + +class TestDownload(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tempdir) + self.finished = Mock() + self.session, self.session_response = mock_session() + self.tempfile = os.path.join(self.tempdir, 'dest') + self.dl = download_manager.Download('http://url', self.tempfile, + finished_callback=self.finished, + chunk_size=4, + session=self.session) + + def test_creation(self): + self.assertFalse(self.dl.is_canceled()) + self.assertFalse(self.dl.is_running()) + self.assertIsNone(self.dl.error()) + self.assertEquals(self.dl.get_url(), 'http://url') + self.assertEquals(self.dl.get_dest(), self.tempfile) + + def create_response(self, data, wait=0): + mock_response(self.session_response, data, wait) + + def test_download(self): + self.create_response('1234' * 4, 0.01) + + # no file present yet + self.assertFalse(os.path.exists(self.tempfile)) + + self.dl.start() + self.assertTrue(self.dl.is_running()) + self.dl.wait() + + self.assertFalse(self.dl.is_running()) + self.finished.assert_called_with(self.dl) + # file has been downloaded + with open(self.tempfile) as f: + self.assertEquals(f.read(), '1234' * 4) + + def test_download_cancel(self): + self.create_response('1234' * 1000, wait=0.01) + + start = time.time() + self.dl.start() + time.sleep(0.1) + self.dl.cancel() + + with self.assertRaises(download_manager.DownloadInterrupt): + self.dl.wait() + + self.assertTrue(self.dl.is_canceled()) + + # response generation should have taken 1000 * 0.01 = 10 seconds. + # since we canceled, this must be lower. + self.assertTrue((time.time() - start) < 1.0) + + # file was deleted + self.assertFalse(os.path.exists(self.tempfile)) + # finished callback was called + self.finished.assert_called_with(self.dl) + + def test_download_with_progress(self): + data = [] + + def update_progress(_dl, current, total): + data.append((_dl, current, total)) + + self.create_response('1234' * 4) + + self.dl.set_progress(update_progress) + self.dl.start() + self.dl.wait() + + self.assertEquals(data, [ + (self.dl, 0, 16), + (self.dl, 4, 16), + (self.dl, 8, 16), + (self.dl, 12, 16), + (self.dl, 16, 16), + ]) + # file has been downloaded + with open(self.tempfile) as f: + self.assertEquals(f.read(), '1234' * 4) + # finished callback was called + self.finished.assert_called_with(self.dl) + + def test_download_error_in_thread(self): + self.session_response.headers = {'Content-length': '24'} + self.session_response.iter_content.side_effect = IOError + + self.dl.start() + with self.assertRaises(IOError): + self.dl.wait() + + self.assertEquals(self.dl.error()[0], IOError) + # finished callback was called + self.finished.assert_called_with(self.dl) + + def test_wait_does_not_block_on_exception(self): + # this test the case when a user may hit CTRL-C for example + # during a dl.wait() call. + self.create_response('1234' * 1000, wait=0.01) + + original_join = self.dl.thread.join + it = iter('123') + + def join(timeout=None): + next(it) # will throw StopIteration after a few calls + original_join(timeout) + + self.dl.thread.join = join + + start = time.time() + self.dl.start() + + with self.assertRaises(StopIteration): + self.dl.wait() + + self.assertTrue(self.dl.is_canceled()) + # wait for the thread to finish + original_join() + + # response generation should have taken 1000 * 0.01 = 10 seconds. + # since we got an error, this must be lower. + self.assertTrue((time.time() - start) < 1.0) + + # file was deleted + self.assertFalse(os.path.exists(self.tempfile)) + # finished callback was called + self.finished.assert_called_with(self.dl) + + +class TestDownloadManager(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tempdir) + + self.dl_manager = download_manager.DownloadManager(self.tempdir) + + def do_download(self, url, fname, data, wait=0): + session, response = mock_session() + mock_response(response, data, wait) + # patch the session, so the download will use that + self.dl_manager.session = session + return self.dl_manager.download(url, fname) + + def test_download(self): + dl1 = self.do_download('http://foo', 'foo', 'hello' * 4, wait=0.02) + self.assertIsInstance(dl1, download_manager.Download) + self.assertTrue(dl1.is_running()) + + # with the same fname, no new download is started. The same instance + # is returned since the download is running. + dl2 = self.do_download('http://bar', 'foo', 'hello2' * 4, wait=0.02) + self.assertEquals(dl1, dl2) + + # starting a download with another fname will trigger a new download + dl3 = self.do_download('http://bar', 'foo2', 'hello you' * 4) + self.assertIsInstance(dl3, download_manager.Download) + self.assertNotEquals(dl3, dl1) + + # let's wait for the downloads to finish + dl3.wait() + dl1.wait() + + # now if we try to download a fname that exists, None is returned + dl4 = self.do_download('http://bar', 'foo', 'hello2' * 4, wait=0.02) + self.assertIsNone(dl4) + + # downloaded files are what is expected + def content(fname): + with open(os.path.join(self.tempdir, fname)) as f: + return f.read() + self.assertEquals(content('foo'), 'hello' * 4) + self.assertEquals(content('foo2'), 'hello you' * 4) + + # download instances are removed from the manager (internal test) + self.assertEquals(self.dl_manager._downloads, {}) + + def test_cancel(self): + dl1 = self.do_download('http://foo', 'foo', 'foo' * 50000, wait=0.02) + dl2 = self.do_download('http://foo', 'bar', 'bar' * 50000, wait=0.02) + dl3 = self.do_download('http://foo', 'foobar', 'foobar' * 4) + + # let's cancel only one + def cancel_if(dl): + if os.path.basename(dl.get_dest()) == 'foo': + return True + self.dl_manager.cancel(cancel_if=cancel_if) + + self.assertTrue(dl1.is_canceled()) + self.assertFalse(dl2.is_canceled()) + self.assertFalse(dl3.is_canceled()) + + # wait for dl3 + dl3.wait() + + # cancel everything + self.dl_manager.cancel() + + self.assertTrue(dl1.is_canceled()) + self.assertTrue(dl2.is_canceled()) + # dl3 is not canceled since it finished before + self.assertFalse(dl3.is_canceled()) + + # wait for the completion of dl1 and dl2 threads + dl1.wait(raise_if_error=False) + dl2.wait(raise_if_error=False) + + # at the end, only dl3 has been downloaded + self.assertEquals(os.listdir(self.tempdir), ["foobar"]) + + with open(os.path.join(self.tempdir, 'foobar')) as f: + self.assertEquals(f.read(), 'foobar' * 4) + + # download instances are removed from the manager (internal test) + self.assertEquals(self.dl_manager._downloads, {}) diff --git a/third_party/python/dlmanager/tests/test_persist_limit.py b/third_party/python/dlmanager/tests/test_persist_limit.py new file mode 100644 index 0000000000..1d899a46f2 --- /dev/null +++ b/third_party/python/dlmanager/tests/test_persist_limit.py @@ -0,0 +1,56 @@ +import pytest +import os +import tempfile +import time +import six + +from dlmanager import fs +from dlmanager.persist_limit import PersistLimit + + +class TempCreator(object): + def __init__(self): + self.tempdir = tempfile.mkdtemp() + + def list(self): + return os.listdir(self.tempdir) + + def create_file(self, name, size, delay): + fname = os.path.join(self.tempdir, name) + with open(fname, 'wb') as f: + f.write(six.b('a' * size)) + # equivalent to touch, but we apply a delay for the test + atime = time.time() + delay + os.utime(fname, (atime, atime)) + + +@pytest.yield_fixture +def temp(): + tmp = TempCreator() + yield tmp + fs.remove(tmp.tempdir) + + +@pytest.mark.parametrize("size_limit,file_limit,files", [ + # limit_file is always respected + (10, 5, "bcdef"), + (10, 3, "def"), + # if size_limit or file_limit is 0, nothing is removed + (0, 5, "abcdef"), + (5, 0, "abcdef"), + # limit_size works + (35, 1, "def"), +]) +def test_persist_limit(temp, size_limit, file_limit, files): + temp.create_file("a", 10, -6) + temp.create_file("b", 10, -5) + temp.create_file("c", 10, -4) + temp.create_file("d", 10, -3) + temp.create_file("e", 10, -2) + temp.create_file("f", 10, -1) + + persist_limit = PersistLimit(size_limit, file_limit) + persist_limit.register_dir_content(temp.tempdir) + persist_limit.remove_old_files() + + assert ''.join(sorted(temp.list())) == ''.join(sorted(files)) |