diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-03-02 08:20:07 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-03-02 08:20:07 +0000 |
| commit | 3d2c9fd003c14a4969f383cd5eb0966b7b6a3d7b (patch) | |
| tree | 96212b1fc6b9515e6bb63a5fc7869cb1da01d36d | |
| download | tqdm-3d2c9fd003c14a4969f383cd5eb0966b7b6a3d7b.tar.xz tqdm-3d2c9fd003c14a4969f383cd5eb0966b7b6a3d7b.zip | |
Adding upstream version 4.64.1.upstream/4.64.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
89 files changed, 16770 insertions, 0 deletions
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..95b4a82 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,62 @@ +default_language_version: + python: python3 +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-toml + - id: check-merge-conflict + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: mixed-line-ending + - id: sort-simple-yaml + - id: trailing-whitespace + exclude: ^README.rst$ +- repo: local + hooks: + - id: todo + name: Check TODO + language: pygrep + entry: WIP + args: [-i] + types: [text] + exclude: ^(.pre-commit-config.yaml|.github/workflows/test.yml)$ + - id: pytest + name: pytest quick + language: python + entry: pytest + args: [-qq, --durations=1, -k=not slow] + types: [python] + pass_filenames: false + additional_dependencies: + - numpy + - pandas + - pytest-timeout + - pytest-asyncio +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.9.2 + hooks: + - id: flake8 + args: [-j8] + additional_dependencies: + - flake8-broken-line + - flake8-bugbear + - flake8-comprehensions + - flake8-debugger + - flake8-isort + - flake8-string-format + - flake8-type-annotations +- repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort +- repo: https://github.com/kynan/nbstripout + rev: 0.5.0 + hooks: + - id: nbstripout + args: [--keep-count, --keep-output] diff --git a/.zenodo.json b/.zenodo.json new file mode 100644 index 0000000..17bcb62 --- /dev/null +++ b/.zenodo.json @@ -0,0 +1,11 @@ +{ + "title": "tqdm: A fast, Extensible Progress Bar for Python and CLI", + "keywords": [ + "progressbar", "progressmeter", "progress-bar", "meter", "rate", "eta", + "console", "terminal", "time", "progress", "bar", "gui", "python", + "parallel", "cli", "utilities", "shell", "batch"], + "related_identifiers": [ + {"identifier": "10.21105/joss.01277", "relation": "cites"}], + "contributors": [ + {"name": "tqdm developers", "type": "Other", "affiliation": "tqdm"}] +} diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..2fa5a68 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,26 @@ +International law supersedes all codes of conduct, including this one. +Authors of codes of conduct presuming the authority to create new laws +should be brought to justice. + +This being said, if you still insist on wringing advice from us: +Be kind. Be professional. Be friendly. Be respectful. Focus on the project. +[Report inappropriate conduct](https://help.github.com/en/articles/reporting-abuse-or-spam). + +Contributions which fail to conform to these guidelines may be reworded, +censored, and/or reported as abuse without notice. + +## The long explanation + +We celebrate the fact that all contributions come from different humans with +their own unique attributes, background and history. However we also place a +strong emphasis on neutrality and a simple focus on the project's goals. +Contributions are thus particularly welcome when references to gender, +ethnicity, religion and so on are avoided as much as possible. +This is for inclusiveness, professionality, and also privacy and security. +This likely means that apart from usernames (which frequently unavoidably hint +about the author's background) there should be no personal information included +in contributions. + +Note that we have no contract with contributors. Introducing contributor licence +agreements (CLAs) would allow us to enforce additional rules but would also be a +scary barrier for new contributors. We don't want to be scary. We love you! diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4cd86d0 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,369 @@ +# HOW TO CONTRIBUTE TO TQDM + +**TL;DR: Skip to [QUICK DEV SUMMARY]** + +This file describes how to + +- contribute changes to the project, and +- upload released to the PyPI repository. + +Most of the management commands have been directly placed inside the +Makefile: + +``` +make [<alias>] # on UNIX-like environments +python setup.py make [<alias>] # if make is unavailable +``` + +The latter depends on [`py-make>=0.1.0`](https://github.com/tqdm/py-make). + +Use the alias `help` (or leave blank) to list all available aliases. + + +## HOW TO COMMIT CONTRIBUTIONS + +Contributions to the project are made using the "Fork & Pull" model. The +typical steps would be: + +1. create an account on [github](https://github.com) +2. fork [`tqdm`](https://github.com/tqdm/tqdm) +3. make a local clone: `git clone https://github.com/your_account/tqdm.git` +4. make changes on the local copy +5. test (see below) and commit changes `git commit -a -m "my message"` +6. `push` to your GitHub account: `git push origin` +7. create a Pull Request (PR) from your GitHub fork +(go to your fork's webpage and click on "Pull Request." +You can then add a message to describe your proposal.) + + +## WHAT CODE LAYOUT SHOULD I FOLLOW? + +Don't worry too much - maintainers can help reorganise contributions. +However it would be helpful to bear in mind: + +- The standard core of `tqdm`, i.e. [`tqdm.std.tqdm`](tqdm/std.py) + + must have no dependencies apart from pure python built-in standard libraries + + must have negligible impact on performance + + should have 100% coverage by unit tests + + should be appropriately commented + + should have well-formatted docstrings for functions + * under 76 chars (incl. initial spaces) to avoid linebreaks in terminal pagers + * use two spaces between variable name and colon, specify a type, and most likely state that it's optional: `VAR<space><space>:<space>TYPE[, optional]` + * use [default: ...] for default values of keyword arguments + + will not break backward compatibility unless there is a very good reason + * e.g. breaking py26 compatibility purely in favour of readability (such as converting `dict(a=1)` to `{'a': 1}`) is not a good enough reason + + API changes should be discussed carefully + + remember, with millions of downloads per month, `tqdm` must be extremely fast and reliable +- Any other kind of change may be included in a (possibly new) submodule + + submodules are likely single python files under the main [tqdm/](tqdm/) directory + + submodules extending `tqdm.std.tqdm` or any other module (e.g. [`tqdm.notebook.tqdm`](tqdm/notebook.py), [`tqdm.gui.tqdm`](tqdm/gui.py)) + + CLI wrapper `tqdm.cli` + * if a newly added `tqdm.std.tqdm` option is not supported by the CLI, append to `tqdm.cli.UNSUPPORTED_OPTS` + + can implement anything from experimental new features to support for third-party libraries such as `pandas`, `numpy`, etc. + + submodule maturity + * alpha: experimental; missing unit tests, comments, and/or feedback; raises `tqdm.TqdmExperimentalWarning` + * beta: well-used; commented, perhaps still missing tests + * stable: >10 users; commented, 80% coverage +- `.meta/` + + A "hidden" folder containing helper utilities not strictly part of the `tqdm` distribution itself + + +## TESTING + +Once again, don't worry too much - tests are automated online, and maintainers +can also help. + +To test functionality (such as before submitting a Pull +Request), there are a number of unit tests. + +### Standard unit tests + +The standard way to run the tests: + +- install `tox` +- `cd` to the root of the `tqdm` directory (in the same folder as this file) +- run the following command: + +``` +[python setup.py] make test +# or: +tox --skip-missing-interpreters +``` + +This will build the module and run the tests in a virtual environment. +Errors and coverage rates will be output to the console/log. (Ignore missing +interpreters errors - these are due to the local machine missing certain +versions of Python.) + +Note: to install all versions of the Python interpreter that are specified +in [tox.ini](https://github.com/tqdm/tqdm/blob/master/tox.ini), +you can use `MiniConda` to install a minimal setup. You must also make sure +that each distribution has an alias to call the Python interpreter: +`python27` for Python 2.7's interpreter, `python32` for Python 3.2's, etc. + +### Alternative unit tests with pytest + +Alternatively, use `pytest` to run the tests just for the current Python version: + +- install test requirements: `[python setup.py] make install_test` +- run the following command: + +``` +[python setup.py] make alltests +``` + + + +# MANAGE A NEW RELEASE + +This section is intended for the project's maintainers and describes +how to build and upload a new release. Once again, +`[python setup.py] make [<alias>]` will help. +Also consider `pip install`ing development utilities: +`[python setup.py] make install_build` at a minimum, or a more thorough `conda env create`. + + +## Pre-commit Hook + +It's probably a good idea to use the `pre-commit` (`pip install pre-commit`) helper. + +Run `pre-commit install` for convenient local sanity-checking. + + +## Semantic Versioning + +The `tqdm` repository managers should: + +- follow the [Semantic Versioning](https://semver.org) convention for tagging + + +## Checking setup.py + +To check that the `setup.py`/`setup.cfg`/`pyproject.toml` file is compliant with PyPI +requirements (e.g. version number; reStructuredText in `README.rst`) use: + +``` +[python setup.py] make testsetup +``` + +To upload just metadata (including overwriting mistakenly uploaded metadata) +to PyPI, use: + +``` +[python setup.py] make pypimeta +``` + + +## Merging Pull Requests + +This section describes how to cleanly merge PRs. + +### 1 Rebase + +From your project repository, merge and test +(replace `pr-branch-name` as appropriate): + +``` +git fetch origin +git checkout -b pr-branch-name origin/pr-branch-name +git rebase master +``` + +If there are conflicts: + +``` +git mergetool +git rebase --continue +``` + +### 2 Push + +Update branch with the rebased history: + +``` +git push origin pr-branch-name --force +``` + +Non maintainers can stop here. + +Note: NEVER just `git push --force` (this will push all local branches, +overwriting remotes). + +### 3 Merge + +``` +git checkout master +git merge --no-ff pr-branch-name +``` + +### 4 Test + +``` +[python setup.py] make alltests +``` + +### 5 Push to master + +``` +git push origin master +``` + + +## Building a Release and Uploading to PyPI + +Formally publishing requires additional steps: testing and tagging. + +### Test + +Ensure that all online CI tests have passed. + +### Tag + +- ensure the version has been tagged. +The tag format is `v{major}.{minor}.{patch}`, for example: `v4.4.1`. +The current commit's tag is used in the version checking process. +If the current commit is not tagged appropriately, the version will +display as `v{major}.{minor}.{patch}.dev{N}+g{commit_hash}`. + +### Upload + +GitHub Actions (GHA) CI should automatically do this after pushing tags. +Manual instructions are given below in case of failure. + +Build `tqdm` into a distributable python package: + +``` +[python setup.py] make build +``` + +This will generate several builds in the `dist/` folder. On non-windows +machines the windows `exe` installer may fail to build. This is normal. + +Finally, upload everything to PyPI. This can be done easily using the +[twine](https://github.com/pypa/twine) module: + +``` +[python setup.py] make pypi +``` + +Also, the new release can (should) be added to GitHub by creating a new +release from the [web interface](https://github.com/tqdm/tqdm/releases); +uploading packages from the `dist/` folder +created by `[python setup.py] make build`. +The [wiki] can be automatically updated with GitHub release notes by +running `make` within the wiki repository. + +[wiki]: https://github.com/tqdm/tqdm/wiki + +Docker images may be uploaded to <https://hub.docker.com/r/tqdm/tqdm>. +Assuming `docker` is +[installed](https://docs.docker.com/install/linux/docker-ce/ubuntu/): + +``` +make -B docker +docker login +docker push tqdm/tqdm:latest +docker push tqdm/tqdm:$(docker run -i --rm tqdm/tqdm -v) +``` + +Snaps may be uploaded to <https://snapcraft.io/tqdm>. +Assuming `snapcraft` is installed (`snap install snapcraft --classic --beta`): + +``` +make snap +snapcraft login +snapcraft push tqdm*.snap --release stable +``` + +### Notes + +- you can also test on the PyPI test servers `test.pypi.org` +before the real deployment +- in case of a mistake, you can delete an uploaded release on PyPI, but you +cannot re-upload another with the same version number +- in case of a mistake in the metadata on PyPI (e.g. bad README), +updating just the metadata is possible: `[python setup.py] make pypimeta` + + +## Updating Websites + +The most important file is `.readme.rst`, which should always be kept up-to-date +and in sync with the in-line source documentation. This will affect all of the +following: + +- `README.rst` (generated by `mkdocs.py` during `make build`) +- The [main repository site](https://github.com/tqdm/tqdm) which automatically + serves the latest `README.rst` as well as links to all of GitHub's features. + This is the preferred online referral link for `tqdm`. +- The [PyPI mirror](https://pypi.org/project/tqdm) which automatically + serves the latest release built from `README.rst` as well as links to past + releases. +- Many external web crawlers. + +Additionally (less maintained), there exists: + +- A [wiki] which is publicly editable. +- The [gh-pages project] which is built from the + [gh-pages branch](https://github.com/tqdm/tqdm/tree/gh-pages), which is + built using [asv](https://github.com/airspeed-velocity/asv). +- The [gh-pages root] which is built from a separate + [github.io repo](https://github.com/tqdm/tqdm.github.io). + +[gh-pages project]: https://tqdm.github.io/tqdm/ +[gh-pages root]: https://tqdm.github.io/ + + +## Helper Bots + +There are some helpers in +[.github/workflows](https://github.com/tqdm/tqdm/tree/master/.github/workflows) +to assist with maintenance. + +- Comment Bot + + allows maintainers to write `/tag vM.m.p commit_hash` in an issue/PR to create a tag +- Post Release + + automatically updates the [wiki] + + automatically updates the [gh-pages root] +- Benchmark + + automatically updates the [gh-pages project] + + +## QUICK DEV SUMMARY + +For experienced devs, once happy with local master, follow the steps below. +Much is automated so really it's steps 1-5, then 11(a). + +1. test (`[python setup.py] make alltests` or rely on `pre-commit`) +2. `git commit [--amend] # -m "bump version"` +3. `git push` +4. wait for tests to pass + a) in case of failure, fix and go back to (1) +5. `git tag vM.m.p && git push --tags` or comment `/tag vM.m.p commit_hash` +6. **`[AUTO:GHA]`** `[python setup.py] make distclean` +7. **`[AUTO:GHA]`** `[python setup.py] make build` +8. **`[AUTO:GHA]`** upload to PyPI. either: + a) `[python setup.py] make pypi`, or + b) `twine upload -s -i $(git config user.signingkey) dist/tqdm-*` +9. **`[AUTO:GHA]`** upload to docker hub: + a) `make -B docker` + b) `docker push tqdm/tqdm:latest` + c) `docker push tqdm/tqdm:$(docker run -i --rm tqdm/tqdm -v)` +10. **`[AUTO:GHA]`** upload to snapcraft: + a) `make snap`, and + b) `snapcraft push tqdm*.snap --release stable` +11. Wait for GHA to draft a new release on <https://github.com/tqdm/tqdm/releases> + a) replace the commit history with helpful release notes, and click publish + b) **`[AUTO:GHA]`** attach `dist/tqdm-*` binaries + (usually only `*.whl*`) +12. **`[SUB][AUTO:GHA-rel]`** run `make` in the `wiki` submodule to update release notes +13. **`[SUB][AUTO:GHA-rel]`** run `make deploy` in the `docs` submodule to update website +14. **`[SUB][AUTO:GHA-rel]`** accept the automated PR in the `feedstock` submodule to update conda +15. **`[AUTO:GHA-rel]`** update the [gh-pages project] benchmarks + a) `[python setup.py] make testasvfull` + b) `asv gh-pages` + +Key: + +- **`[AUTO:GHA]`**: GitHub Actions CI should automatically do this after `git push --tags` (5) +- **`[AUTO:GHA-rel]`**: GitHub Actions CI should automatically do this after release (11a) +- **`[SUB]`**: Requires one-time `make submodules` to clone `docs`, `wiki`, and `feedstock` diff --git a/DEMO.ipynb b/DEMO.ipynb new file mode 100644 index 0000000..b892af4 --- /dev/null +++ b/DEMO.ipynb @@ -0,0 +1,890 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "<h1 align=\"center\">tqdm</h1>\n", + "<img src=\"https://img.tqdm.ml/logo.gif\" align=\"left\" />\n", + "\n", + "[](https://pypi.org/project/tqdm)|[](https://tqdm.github.io/releases)|[](https://anaconda.org/conda-forge/tqdm)|[](https://hub.docker.com/r/tqdm/tqdm)|[](https://snapcraft.io/tqdm)\n", + "-|-|-|-|-\n", + "\n", + "[](https://github.com/tqdm/tqdm/actions?query=workflow%3ATest)|[](https://coveralls.io/github/tqdm/tqdm)|[](https://codecov.io/gh/tqdm/tqdm)|[](https://www.codacy.com/gh/tqdm/tqdm/dashboard)|[](https://libraries.io/pypi/tqdm)|[](https://pepy.tech/project/tqdm)\n", + "-|-|-|-|-|-\n", + "\n", + "[](https://doi.org/10.5281/zenodo.595120)|[](https://raw.githubusercontent.com/tqdm/tqdm/master/LICENCE)|[](https://www.openhub.net/p/tqdm?ref=Thin+badge)|[](https://mybinder.org/v2/gh/tqdm/tqdm/master?filepath=DEMO.ipynb)|[](https://github.com/vinta/awesome-python)\n", + "-|-|-|-|-\n", + "\n", + "`tqdm` derives from the Arabic word *taqaddum* (تقدّم) which can mean\n", + "\"progress,\" and is an abbreviation for \"I love you so much\" in Spanish\n", + "(*te quiero demasiado*).\n", + "\n", + "Instantly make your loops show a smart progress meter - just wrap any\n", + "iterable with `tqdm(iterable)`, and you're done!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm import tqdm\n", + "for i in tqdm(range(10000)):\n", + " pass" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`trange(N)` can be also used as a convenient shortcut for\n", + "`tqdm(xrange(N))`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm import trange\n", + "for i in trange(10000, unit_scale=True, desc=\"hello\", unit=\"epoch\"):\n", + " pass" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "|[](https://tqdm.github.io/video) [](https://tqdm.github.io/PyData2019/slides.html) [](https://tqdm.github.io/merch)\n", + "-|-\n", + "\n", + "It can also be executed as a module with pipes:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! seq 9999999 | tqdm --bytes | wc -l" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```sh\n", + "tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` > backup.tgz\n", + " 44%|██████████████▊ | 153M/352M [00:14<00:18, 11.0MB/s]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Overhead is low -- about 60ns per iteration (80ns with `tqdm.gui`), and\n", + "is unit tested against performance regression. By comparison, the\n", + "well-established\n", + "[ProgressBar](https://github.com/niltonvolpato/python-progressbar) has\n", + "an 800ns/iter overhead.\n", + "\n", + "In addition to its low overhead, `tqdm` uses smart algorithms to predict\n", + "the remaining time and to skip unnecessary iteration displays, which\n", + "allows for a negligible overhead in most cases.\n", + "\n", + "`tqdm` works on any platform (Linux, Windows, Mac, FreeBSD, NetBSD,\n", + "Solaris/SunOS), in any console or in a GUI, and is also friendly with\n", + "IPython/Jupyter notebooks.\n", + "\n", + "`tqdm` does not require any dependencies (not even `curses`!), just\n", + "Python and an environment supporting `carriage return \\r` and\n", + "`line feed \\n` control characters.\n", + "\n", + "---\n", + "\n", + "## Usage\n", + "\n", + "`tqdm` is very versatile and can be used in a number of ways.\n", + "The three main ones are given below.\n", + "\n", + "### Iterable-based\n", + "\n", + "Wrap `tqdm()` around any iterable:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm import tqdm\n", + "from time import sleep" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "text = \"\"\n", + "for char in tqdm([\"a\", \"b\", \"c\", \"d\"]):\n", + " sleep(0.25)\n", + " text = text + char" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`trange(i)` is a special optimised instance of `tqdm(range(i))`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm import trange\n", + "\n", + "for i in trange(100):\n", + " sleep(0.01)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Instantiation outside of the loop allows for manual control over `tqdm()`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pbar = tqdm([\"a\", \"b\", \"c\", \"d\"])\n", + "for char in pbar:\n", + " sleep(0.25)\n", + " pbar.set_description(\"Processing %s\" % char)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Manual\n", + "\n", + "Manual control of `tqdm()` updates using a `with` statement:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with tqdm(total=100) as pbar:\n", + " for i in range(10):\n", + " sleep(0.1)\n", + " pbar.update(10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If the optional variable `total` (or an iterable with `len()`) is\n", + "provided, predictive stats are displayed.\n", + "\n", + "`with` is also optional (you can just assign `tqdm()` to a variable,\n", + "but in this case don't forget to `del` or `close()` at the end:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pbar = tqdm(total=100)\n", + "for i in range(10):\n", + " sleep(0.1)\n", + " pbar.update(10)\n", + "pbar.close()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Module" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Perhaps the most wonderful use of `tqdm` is in a script or on the\n", + "command line. Simply inserting `tqdm` (or `python -m tqdm`) between\n", + "pipes will pass through all `stdin` to `stdout` while printing progress\n", + "to `stderr`.\n", + "\n", + "The example below demonstrated counting the number of lines in all\n", + "Python files in the current directory, with timing information included." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! time find . -name '*.py' -type f -exec cat \\{} \\; | wc -l" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! time find . -name '*.py' -type f -exec cat \\{} \\; | tqdm | wc -l" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that the usual arguments for `tqdm` can also be specified." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! find . -name '*.py' -type f -exec cat \\{} \\; | tqdm --unit loc --unit-scale --total 4104300 --null" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Backing up a large directory?\n", + "\n", + "```sh\n", + "tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` > backup.tgz\n", + " 44%|██████████████▊ | 153M/352M [00:14<00:18, 11.0MB/s]\n", + "```\n", + "\n", + "This can be beautified further:\n", + "\n", + "```sh\n", + "BYTES=\"$(du -sb docs/ | cut -f1)\"\n", + "tar -cf - docs/ \\\n", + " | tqdm --bytes --total \"$BYTES\" --desc Processing | gzip \\\n", + " | tqdm --bytes --total \"$BYTES\" --desc Compressed --position 1 \\\n", + " > ~/backup.tgz\n", + "Processing: 100%|██████████████████████| 352M/352M [00:14<00:00, 30.2MB/s]\n", + "Compressed: 42%|█████████▎ | 148M/352M [00:14<00:19, 10.9MB/s]\n", + "```\n", + "\n", + "Or done on a file level using 7-zip:\n", + "\n", + "```sh\n", + "7z a -bd -r backup.7z docs/ | grep Compressing \\\n", + " | tqdm --total $(find docs/ -type f | wc -l) --unit files \\\n", + " | grep -v Compressing\n", + "100%|██████████████████████████▉| 15327/15327 [01:00<00:00, 712.96files/s]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Documentation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tqdm?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! tqdm --help" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Examples and Advance Usage\n", + "\n", + "- See the [examples](https://github.com/tqdm/tqdm/tree/master/examples)\n", + " folder;\n", + "- import the module and run `help()`;\n", + "- consult the [wiki](https://github.com/tqdm/tqdm/wiki)\n", + " - this has an\n", + " [excellent article](https://github.com/tqdm/tqdm/wiki/How-to-make-a-great-Progress-Bar)\n", + " on how to make a **great** progressbar;\n", + "- check out the [slides from PyData London](https://tqdm.github.io/PyData2019/slides.html), or\n", + "- run this file!\n", + "\n", + "### Description and additional stats\n", + "\n", + "Custom information can be displayed and updated dynamically on `tqdm` bars\n", + "with the `desc` and `postfix` arguments:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm import tqdm, trange\n", + "from random import random, randint\n", + "from time import sleep\n", + "\n", + "with trange(10) as t:\n", + " for i in t:\n", + " # Description will be displayed on the left\n", + " t.set_description('GEN %i' % i)\n", + " # Postfix will be displayed on the right,\n", + " # formatted automatically based on argument's datatype\n", + " t.set_postfix(loss=random(), gen=randint(1,999), str='h',\n", + " lst=[1, 2])\n", + " sleep(0.1)\n", + "\n", + "with tqdm(total=10, bar_format=\"{postfix[0]} {postfix[1][value]:>8.2g}\",\n", + " postfix=[\"Batch\", dict(value=0)]) as t:\n", + " for i in range(10):\n", + " sleep(0.1)\n", + " t.postfix[1][\"value\"] = i / 2\n", + " t.update()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Points to remember when using `{postfix[...]}` in the `bar_format` string:\n", + "\n", + "- `postfix` also needs to be passed as an initial argument in a\n", + " compatible format, and\n", + "- `postfix` will be auto-converted to a string if it is a `dict`-like\n", + " object. To prevent this behaviour, insert an extra item into the\n", + " dictionary where the key is not a string.\n", + "\n", + "Additional `bar_format` parameters may also be defined by overriding\n", + "`format_dict`, and the bar itself may be modified using `ascii`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm import tqdm\n", + "from time import sleep\n", + "\n", + "class TqdmExtraFormat(tqdm):\n", + " \"\"\"Provides a `total_time` format parameter\"\"\"\n", + " @property\n", + " def format_dict(self):\n", + " d = super(TqdmExtraFormat, self).format_dict\n", + " total_time = d[\"elapsed\"] * (d[\"total\"] or 0) / max(d[\"n\"], 1)\n", + " d.update(total_time=self.format_interval(total_time) + \" in total\")\n", + " return d\n", + "\n", + "for i in TqdmExtraFormat(\n", + " range(9), ascii=\" .oO0\",\n", + " bar_format=\"{total_time}: {percentage:.0f}%|{bar}{r_bar}\"):\n", + " if i == 4:\n", + " break" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that `{bar}` also supports a format specifier `[width][type]`.\n", + "\n", + "- `width`\n", + " + unspecified (default): automatic to fill `ncols`\n", + " + `int >= 0`: fixed width overriding `ncols` logic\n", + " + `int < 0`: subtract from the automatic default\n", + "- `type`\n", + " + `a`: ascii (`ascii=True` override)\n", + " + `u`: unicode (`ascii=False` override)\n", + " + `b`: blank (`ascii=\" \"` override)\n", + "\n", + "This means a fixed bar with right-justified text may be created by\n", + "using: `bar_format=\"{l_bar}{bar:10}|{bar:-10b}right-justified\"`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Nested progress bars\n", + "\n", + "`tqdm` supports nested progress bars. Here's an example:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm.auto import trange\n", + "from time import sleep\n", + "\n", + "for i in trange(4, desc='1st loop'):\n", + " for j in trange(5, desc='2nd loop'):\n", + " for k in trange(50, desc='3rd loop', leave=False):\n", + " sleep(0.01)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For manual control over positioning (e.g. for multi-processing use),\n", + "you may specify `position=n` where `n=0` for the outermost bar, `n=1`\n", + "for the next, and so on. However, it's best to check if tqdm can work\n", + "without manual position first.\n", + "\n", + "```python\n", + "from time import sleep\n", + "from tqdm import trange, tqdm\n", + "from multiprocessing import Pool, RLock, freeze_support\n", + "\n", + "L = list(range(9))\n", + "\n", + "def progresser(n):\n", + " interval = 0.001 / (n + 2)\n", + " total = 5000\n", + " text = \"#{}, est. {:<04.2}s\".format(n, interval * total)\n", + " for _ in trange(total, desc=text, position=n):\n", + " sleep(interval)\n", + "\n", + "if __name__ == '__main__':\n", + " freeze_support() # for Windows support\n", + " tqdm.set_lock(RLock()) # for managing output contention\n", + " p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),))\n", + " p.map(progresser, L)\n", + "```\n", + "\n", + "Note that in Python 3, `tqdm.write` is thread-safe:\n", + "\n", + "```python\n", + "from time import sleep\n", + "from tqdm import tqdm, trange\n", + "from concurrent.futures import ThreadPoolExecutor\n", + "\n", + "L = list(range(9))\n", + "\n", + "def progresser(n):\n", + " interval = 0.001 / (n + 2)\n", + " total = 5000\n", + " text = \"#{}, est. {:<04.2}s\".format(n, interval * total)\n", + " for _ in trange(total, desc=text):\n", + " sleep(interval).auto\n", + " if n == 6:\n", + " tqdm.write(\"n == 6 completed.\")\n", + " tqdm.write(\"`tqdm.write()` is thread-safe in py3!\")\n", + "\n", + "if __name__ == '__main__':\n", + " with ThreadPoolExecutor() as p:\n", + " p.map(progresser, L)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Hooks and callbacks\n", + "\n", + "`tqdm` can easily support callbacks/hooks and manual updates.\n", + "Here's an example with `urllib`:\n", + "\n", + "**`urllib.urlretrieve` documentation**\n", + "\n", + "> [...]\n", + "> If present, the hook function will be called once\n", + "> on establishment of the network connection and once after each block read\n", + "> thereafter. The hook will be passed three arguments; a count of blocks\n", + "> transferred so far, a block size in bytes, and the total size of the file.\n", + "> [...]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import urllib, os\n", + "from tqdm import tqdm\n", + "urllib = getattr(urllib, 'request', urllib)\n", + "\n", + "class TqdmUpTo(tqdm):\n", + " \"\"\"Provides `update_to(n)` which uses `tqdm.update(delta_n)`.\"\"\"\n", + " def update_to(self, b=1, bsize=1, tsize=None):\n", + " \"\"\"\n", + " b : int, optional\n", + " Number of blocks transferred so far [default: 1].\n", + " bsize : int, optional\n", + " Size of each block (in tqdm units) [default: 1].\n", + " tsize : int, optional\n", + " Total size (in tqdm units). If [default: None] remains unchanged.\n", + " \"\"\"\n", + " if tsize is not None:\n", + " self.total = tsize\n", + " return self.update(b * bsize - self.n) # also sets self.n = b * bsize\n", + "\n", + "eg_link = \"https://caspersci.uk.to/matryoshka.zip\"\n", + "with TqdmUpTo(unit='B', unit_scale=True, unit_divisor=1024, miniters=1,\n", + " desc=eg_link.split('/')[-1]) as t: # all optional kwargs\n", + " urllib.urlretrieve(eg_link, filename=os.devnull,\n", + " reporthook=t.update_to, data=None)\n", + " t.total = t.n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Inspired by [twine#242](https://github.com/pypa/twine/pull/242).\n", + "Functional alternative in\n", + "[examples/tqdm_wget.py](https://github.com/tqdm/tqdm/blob/master/examples/tqdm_wget.py).\n", + "\n", + "It is recommend to use `miniters=1` whenever there is potentially large\n", + "differences in iteration speed (e.g. downloading a file over a patchy\n", + "connection).\n", + "\n", + "**Wrapping read/write methods**\n", + "\n", + "To measure throughput through a file-like object's `read` or `write`\n", + "methods, use `CallbackIOWrapper`:\n", + "\n", + "```python\n", + "from tqdm import tqdm\n", + "from tqdm.utils import CallbackIOWrapper\n", + "\n", + "with tqdm(total=file_obj.size,\n", + " unit='B', unit_scale=True, unit_divisor=1024) as t:\n", + " fobj = CallbackIOWrapper(t.update, file_obj, \"read\")\n", + " while True:\n", + " chunk = fobj.read(chunk_size)\n", + " if not chunk:\n", + " break\n", + " t.reset()\n", + " # ... continue to use `t` for something else\n", + "```\n", + "\n", + "Alternatively, use the even simpler `wrapattr` convenience function,\n", + "which would condense both the `urllib` and `CallbackIOWrapper` examples\n", + "down to:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import urllib, os\n", + "from tqdm import tqdm\n", + "\n", + "eg_link = \"https://caspersci.uk.to/matryoshka.zip\"\n", + "response = getattr(urllib, 'request', urllib).urlopen(eg_link)\n", + "with tqdm.wrapattr(open(os.devnull, \"wb\"), \"write\",\n", + " miniters=1, desc=eg_link.split('/')[-1],\n", + " total=getattr(response, 'length', None)) as fout:\n", + " for chunk in response:\n", + " fout.write(chunk)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `requests` equivalent is nearly identical:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import requests, os\n", + "from tqdm import tqdm\n", + "\n", + "eg_link = \"https://caspersci.uk.to/matryoshka.zip\"\n", + "response = requests.get(eg_link, stream=True)\n", + "with tqdm.wrapattr(open(os.devnull, \"wb\"), \"write\",\n", + " miniters=1, desc=eg_link.split('/')[-1],\n", + " total=int(response.headers.get('content-length', 0))) as fout:\n", + " for chunk in response.iter_content(chunk_size=4096):\n", + " fout.write(chunk)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Pandas Integration\n", + "\n", + "Due to popular demand we've added support for `pandas` -- here's an example\n", + "for `DataFrame.progress_apply` and `DataFrameGroupBy.progress_apply`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "from tqdm import tqdm\n", + "\n", + "df = pd.DataFrame(np.random.randint(0, 100, (100000, 6)))\n", + "\n", + "# Register `pandas.progress_apply` and `pandas.Series.map_apply` with `tqdm`\n", + "# (can use `tqdm.gui.tqdm`, `tqdm.notebook.tqdm`, optional kwargs, etc.)\n", + "tqdm.pandas(desc=\"my bar!\")\n", + "\n", + "# Now you can use `progress_apply` instead of `apply`\n", + "# and `progress_map` instead of `map`\n", + "df.progress_apply(lambda x: x**2)\n", + "# can also groupby:\n", + "# df.groupby(0).progress_apply(lambda x: x**2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In case you're interested in how this works (and how to modify it for\n", + "your own callbacks), see the\n", + "[examples](https://github.com/tqdm/tqdm/tree/master/examples) folder or\n", + "import the module and run `help()`.\n", + "\n", + "### Keras Integration\n", + "\n", + "A `keras` callback is also available:\n", + "\n", + "```python\n", + "from tqdm.keras import TqdmCallback\n", + "\n", + "...\n", + "\n", + "model.fit(..., verbose=0, callbacks=[TqdmCallback()])\n", + "```\n", + "\n", + "### IPython/Jupyter Integration\n", + "\n", + "IPython/Jupyter is supported via the `tqdm.notebook` submodule:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm.notebook import trange, tqdm\n", + "from time import sleep\n", + "\n", + "for i in trange(3, desc='1st loop'):\n", + " for j in tqdm(range(100), desc='2nd loop'):\n", + " sleep(0.01)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In addition to `tqdm` features, the submodule provides a native Jupyter\n", + "widget (compatible with IPython v1-v4 and Jupyter), fully working nested\n", + "bars and colour hints (blue: normal, green: completed, red:\n", + "error/interrupt, light blue: no ETA); as demonstrated below.\n", + "\n", + "\n", + "\n", + "The `notebook` version supports percentage or pixels for overall width\n", + "(e.g.: `ncols='100%'` or `ncols='480px'`).\n", + "\n", + "It is also possible to let `tqdm` automatically choose between console\n", + "or notebook versions by using the `autonotebook` submodule:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm.autonotebook import tqdm\n", + "tqdm.pandas()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that this will issue a `TqdmExperimentalWarning` if run in a\n", + "notebook since it is not meant to be possible to distinguish between\n", + "`jupyter notebook` and `jupyter console`. Use `auto` instead of\n", + "`autonotebook` to suppress this warning.\n", + "\n", + "Note that notebooks will display the bar in the cell where it was\n", + "created. This may be a different cell from the one where it is used. If\n", + "this is not desired, the creation of the bar must be delayed/moved to\n", + "the cell where it is desired to be displayed.\n", + "\n", + "Another possibility is to have a single bar (near the top of the\n", + "notebook) which is constantly re-used (using `reset()` rather than\n", + "`close()`). For this reason, the notebook version (unlike the CLI\n", + "version) does not automatically call `close()` upon `Exception`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm.notebook import tqdm\n", + "pbar = tqdm()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# different cell\n", + "iterable = range(100)\n", + "pbar.reset(total=len(iterable)) # initialise with new `total`\n", + "for i in iterable:\n", + " pbar.update()\n", + "pbar.refresh() # force print final status but don't `close()`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Writing messages\n", + "\n", + "This is a work in progress (see\n", + "[#737](https://github.com/tqdm/tqdm/issues/737)).\n", + "\n", + "Since `tqdm` uses a simple printing mechanism to display progress bars,\n", + "you should not write any message in the terminal using `print()` while a\n", + "progressbar is open.\n", + "\n", + "To write messages in the terminal without any collision with `tqdm` bar\n", + "display, a `.write()` method is provided:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tqdm.auto import tqdm, trange\n", + "from time import sleep\n", + "\n", + "bar = trange(10)\n", + "for i in bar:\n", + " # Print using tqdm class method .write()\n", + " sleep(0.1)\n", + " if not (i % 3):\n", + " tqdm.write(\"Done task %i\" % i)\n", + " # Can also use bar.write()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "By default, this will print to standard output `sys.stdout`. but you can\n", + "specify any file-like object using the `file` argument. For example,\n", + "this can be used to redirect the messages writing to a log file or class.\n", + "\n", + "[](https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&a=plot&r=https://github.com/tqdm/tqdm&l=https://img.tqdm.ml/favicon.png&f=https://img.tqdm.ml/logo.gif&style=social)|(Since 19 May 2016)\n", + "-|-" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Do your own experiments here 👇\n", + "\n", + "Try `tqdm` youself by adding your code below and running your own experiments." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import tqdm\n", + "\n", + "# your code here\n", + "tqdm." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython" + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} @@ -0,0 +1,49 @@ +`tqdm` is a product of collaborative work. +Unless otherwise stated, all authors (see commit logs) retain copyright +for their respective work, and release the work under the MIT licence +(text below). + +Exceptions or notable authors are listed below +in reverse chronological order: + +* files: * + MPLv2.0 2015-2021 (c) Casper da Costa-Luis + [casperdcl](https://github.com/casperdcl). +* files: tqdm/_tqdm.py + MIT 2016 (c) [PR #96] on behalf of Google Inc. +* files: tqdm/_tqdm.py setup.py README.rst MANIFEST.in .gitignore + MIT 2013 (c) Noam Yorav-Raphael, original author. + +[PR #96]: https://github.com/tqdm/tqdm/pull/96 + + +Mozilla Public Licence (MPL) v. 2.0 - Exhibit A +----------------------------------------------- + +This Source Code Form is subject to the terms of the +Mozilla Public License, v. 2.0. +If a copy of the MPL was not distributed with this project, +You can obtain one at https://mozilla.org/MPL/2.0/. + + +MIT License (MIT) +----------------- + +Copyright (c) 2013 noamraph + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..796f8b0 --- /dev/null +++ b/Makefile @@ -0,0 +1,191 @@ +# IMPORTANT: for compatibility with `python setup.py make [alias]`, ensure: +# 1. Every alias is preceded by @[+]make (eg: @make alias) +# 2. A maximum of one @make alias or command per line +# see: https://github.com/tqdm/py-make/issues/1 + +.PHONY: + alltests + all + flake8 + test + pytest + testsetup + testnb + testcoverage + testperf + testtimer + distclean + coverclean + prebuildclean + clean + toxclean + install_dev + install + build + buildupload + pypi + snap + docker + help + none + run + +help: + @python setup.py make -p + +alltests: + @+make testcoverage + @+make testperf + @+make flake8 + @+make testsetup + +all: + @+make alltests + @+make build + +flake8: + @+pre-commit run -a flake8 + @+pre-commit run -a nbstripout + +test: + TOX_SKIP_ENV=perf tox --skip-missing-interpreters -p all + tox -e perf + +pytest: + pytest + +testsetup: + @make README.rst + @make tqdm/tqdm.1 + @make tqdm/completion.sh + python setup.py check --metadata --restructuredtext --strict + python setup.py make none + +testnb: + pytest tests_notebook.ipynb --nbval --nbval-current-env -W=ignore --nbval-sanitize-with=setup.cfg --cov=tqdm.notebook --cov-report=term + +testcoverage: + @make coverclean + pytest tests_notebook.ipynb --cov=tqdm --cov-report= --nbval --nbval-current-env --nbval-sanitize-with=setup.cfg -W=ignore + pytest -k "not perf" --cov=tqdm --cov-report=xml --cov-report=term --cov-append --cov-fail-under=80 + +testperf: + # do not use coverage (which is extremely slow) + pytest -k perf + +testtimer: + pytest + +# another performance test, to check evolution across commits +testasv: + # Test only the last 3 commits (quick test) + asv run -j 8 HEAD~3..HEAD + @make viewasv + +testasvfull: + # Test all the commits since the beginning (full test) + asv run --skip-existing-commits -j 8 v1.0.0..HEAD + @make testasv + +viewasv: + asv publish + asv preview + +tqdm/tqdm.1: .meta/.tqdm.1.md tqdm/cli.py tqdm/std.py + # TODO: add to mkdocs.py + python -m tqdm --help | tail -n+5 |\ + sed -r -e 's/\\/\\\\/g' \ + -e 's/^ (--.*)=<(.*)> : (.*)$$/\n\\\1=*\2*\n: \3./' \ + -e 's/^ (--.*) : (.*)$$/\n\\\1\n: \2./' \ + -e 's/ (-.*, )(--.*) /\n\1\\\2\n: /' |\ + cat "$<" - |\ + pandoc -o "$@" -s -t man + +tqdm/completion.sh: .meta/mkcompletion.py tqdm/std.py tqdm/cli.py + @python .meta/mkcompletion.py + +README.rst: .meta/.readme.rst tqdm/std.py tqdm/cli.py + @python .meta/mkdocs.py + +snapcraft.yaml: .meta/mksnap.py + @python .meta/mksnap.py + +.dockerignore: + @+python -c "fd=open('.dockerignore', 'w'); fd.write('*\n!dist/*.whl\n')" + +Dockerfile: + @+python -c 'fd=open("Dockerfile", "w"); fd.write("FROM python:3.8-alpine\nCOPY dist/*.whl .\nRUN pip install -U $$(ls ./*.whl) && rm ./*.whl\nENTRYPOINT [\"tqdm\"]\n")' + +distclean: + @+make coverclean + @+make prebuildclean + @+make clean +prebuildclean: + @+python -c "import shutil; shutil.rmtree('build', True)" + @+python -c "import shutil; shutil.rmtree('dist', True)" + @+python -c "import shutil; shutil.rmtree('tqdm.egg-info', True)" + @+python -c "import shutil; shutil.rmtree('.eggs', True)" + @+python -c "import os; os.remove('tqdm/_dist_ver.py') if os.path.exists('tqdm/_dist_ver.py') else None" +coverclean: + @+python -c "import os; os.remove('.coverage') if os.path.exists('.coverage') else None" + @+python -c "import os, glob; [os.remove(i) for i in glob.glob('.coverage.*')]" + @+python -c "import shutil; shutil.rmtree('tests/__pycache__', True)" + @+python -c "import shutil; shutil.rmtree('benchmarks/__pycache__', True)" + @+python -c "import shutil; shutil.rmtree('tqdm/__pycache__', True)" + @+python -c "import shutil; shutil.rmtree('tqdm/contrib/__pycache__', True)" + @+python -c "import shutil; shutil.rmtree('tqdm/examples/__pycache__', True)" +clean: + @+python -c "import os, glob; [os.remove(i) for i in glob.glob('*.py[co]')]" + @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tests/*.py[co]')]" + @+python -c "import os, glob; [os.remove(i) for i in glob.glob('benchmarks/*.py[co]')]" + @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/*.py[co]')]" + @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/contrib/*.py[co]')]" + @+python -c "import os, glob; [os.remove(i) for i in glob.glob('tqdm/examples/*.py[co]')]" +toxclean: + @+python -c "import shutil; shutil.rmtree('.tox', True)" + + +submodules: + git clone git@github.com:tqdm/tqdm.wiki wiki + git clone git@github.com:tqdm/tqdm.github.io docs + git clone git@github.com:conda-forge/tqdm-feedstock feedstock + cd feedstock && git remote add autotick-bot git@github.com:regro-cf-autotick-bot/tqdm-feedstock + +install: + python setup.py install +install_dev: + python setup.py develop --uninstall + python setup.py develop +install_build: + python -m pip install -r .meta/requirements-dev.txt +install_test: + python -m pip install -r .meta/requirements-test.txt + pre-commit install + +build: + @make prebuildclean + @make testsetup + python setup.py sdist bdist_wheel + # python setup.py bdist_wininst + +pypi: + twine upload dist/* + +buildupload: + @make build + @make pypi + +snap: + @make -B snapcraft.yaml + snapcraft +docker: + @make build + @make .dockerignore + @make Dockerfile + docker build . -t tqdm/tqdm + docker tag tqdm/tqdm:latest tqdm/tqdm:$(shell docker run -i --rm tqdm/tqdm -v) +none: + # used for unit testing + +run: + python -Om tqdm --help diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..d796709 --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,1581 @@ +Metadata-Version: 2.1 +Name: tqdm +Version: 4.64.1 +Summary: Fast, Extensible Progress Meter +Home-page: https://tqdm.github.io +Maintainer: tqdm developers +Maintainer-email: python.tqdm@gmail.com +License: MPLv2.0, MIT Licences +Project-URL: Changelog, https://tqdm.github.io/releases +Project-URL: Source, https://github.com/tqdm/tqdm +Project-URL: Wiki, https://github.com/tqdm/tqdm/wiki +Keywords: progressbar,progressmeter,progress,bar,meter,rate,eta,console,terminal,time +Platform: any +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Environment :: MacOS X +Classifier: Environment :: Other Environment +Classifier: Environment :: Win32 (MS Windows) +Classifier: Environment :: X11 Applications +Classifier: Framework :: IPython +Classifier: Framework :: Jupyter +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Education +Classifier: Intended Audience :: End Users/Desktop +Classifier: Intended Audience :: Other Audience +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: MIT License +Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0) +Classifier: Operating System :: MacOS +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: Microsoft +Classifier: Operating System :: Microsoft :: MS-DOS +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX +Classifier: Operating System :: POSIX :: BSD +Classifier: Operating System :: POSIX :: BSD :: FreeBSD +Classifier: Operating System :: POSIX :: Linux +Classifier: Operating System :: POSIX :: SunOS/Solaris +Classifier: Operating System :: Unix +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: Implementation +Classifier: Programming Language :: Python :: Implementation :: IronPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Programming Language :: Unix Shell +Classifier: Topic :: Desktop Environment +Classifier: Topic :: Education :: Computer Aided Instruction (CAI) +Classifier: Topic :: Education :: Testing +Classifier: Topic :: Office/Business +Classifier: Topic :: Other/Nonlisted Topic +Classifier: Topic :: Software Development :: Build Tools +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Software Development :: Pre-processors +Classifier: Topic :: Software Development :: User Interfaces +Classifier: Topic :: System :: Installation/Setup +Classifier: Topic :: System :: Logging +Classifier: Topic :: System :: Monitoring +Classifier: Topic :: System :: Shells +Classifier: Topic :: Terminals +Classifier: Topic :: Utilities +Provides: tqdm +Requires-Python: !=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7 +Description-Content-Type: text/x-rst +Provides-Extra: dev +Provides-Extra: slack +Provides-Extra: telegram +Provides-Extra: notebook +License-File: LICENCE + +|Logo| + +tqdm +==== + +|Py-Versions| |Versions| |Conda-Forge-Status| |Docker| |Snapcraft| + +|Build-Status| |Coverage-Status| |Branch-Coverage-Status| |Codacy-Grade| |Libraries-Rank| |PyPI-Downloads| + +|LICENCE| |OpenHub-Status| |binder-demo| |awesome-python| + +``tqdm`` derives from the Arabic word *taqaddum* (تقدّم) which can mean "progress," +and is an abbreviation for "I love you so much" in Spanish (*te quiero demasiado*). + +Instantly make your loops show a smart progress meter - just wrap any +iterable with ``tqdm(iterable)``, and you're done! + +.. code:: python + + from tqdm import tqdm + for i in tqdm(range(10000)): + ... + +``76%|████████████████████████ | 7568/10000 [00:33<00:10, 229.00it/s]`` + +``trange(N)`` can be also used as a convenient shortcut for +``tqdm(range(N))``. + +|Screenshot| + |Video| |Slides| |Merch| + +It can also be executed as a module with pipes: + +.. code:: sh + + $ seq 9999999 | tqdm --bytes | wc -l + 75.2MB [00:00, 217MB/s] + 9999999 + + $ tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` \ + > backup.tgz + 32%|██████████▍ | 8.89G/27.9G [00:42<01:31, 223MB/s] + +Overhead is low -- about 60ns per iteration (80ns with ``tqdm.gui``), and is +unit tested against performance regression. +By comparison, the well-established +`ProgressBar <https://github.com/niltonvolpato/python-progressbar>`__ has +an 800ns/iter overhead. + +In addition to its low overhead, ``tqdm`` uses smart algorithms to predict +the remaining time and to skip unnecessary iteration displays, which allows +for a negligible overhead in most cases. + +``tqdm`` works on any platform +(Linux, Windows, Mac, FreeBSD, NetBSD, Solaris/SunOS), +in any console or in a GUI, and is also friendly with IPython/Jupyter notebooks. + +``tqdm`` does not require any dependencies (not even ``curses``!), just +Python and an environment supporting ``carriage return \r`` and +``line feed \n`` control characters. + +------------------------------------------ + +.. contents:: Table of contents + :backlinks: top + :local: + + +Installation +------------ + +Latest PyPI stable release +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +|Versions| |PyPI-Downloads| |Libraries-Dependents| + +.. code:: sh + + pip install tqdm + +Latest development release on GitHub +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +|GitHub-Status| |GitHub-Stars| |GitHub-Commits| |GitHub-Forks| |GitHub-Updated| + +Pull and install pre-release ``devel`` branch: + +.. code:: sh + + pip install "git+https://github.com/tqdm/tqdm.git@devel#egg=tqdm" + +Latest Conda release +~~~~~~~~~~~~~~~~~~~~ + +|Conda-Forge-Status| + +.. code:: sh + + conda install -c conda-forge tqdm + +Latest Snapcraft release +~~~~~~~~~~~~~~~~~~~~~~~~ + +|Snapcraft| + +There are 3 channels to choose from: + +.. code:: sh + + snap install tqdm # implies --stable, i.e. latest tagged release + snap install tqdm --candidate # master branch + snap install tqdm --edge # devel branch + +Note that ``snap`` binaries are purely for CLI use (not ``import``-able), and +automatically set up ``bash`` tab-completion. + +Latest Docker release +~~~~~~~~~~~~~~~~~~~~~ + +|Docker| + +.. code:: sh + + docker pull tqdm/tqdm + docker run -i --rm tqdm/tqdm --help + +Other +~~~~~ + +There are other (unofficial) places where ``tqdm`` may be downloaded, particularly for CLI use: + +|Repology| + +.. |Repology| image:: https://repology.org/badge/tiny-repos/python:tqdm.svg + :target: https://repology.org/project/python:tqdm/versions + +Changelog +--------- + +The list of all changes is available either on GitHub's Releases: +|GitHub-Status|, on the +`wiki <https://github.com/tqdm/tqdm/wiki/Releases>`__, or on the +`website <https://tqdm.github.io/releases>`__. + + +Usage +----- + +``tqdm`` is very versatile and can be used in a number of ways. +The three main ones are given below. + +Iterable-based +~~~~~~~~~~~~~~ + +Wrap ``tqdm()`` around any iterable: + +.. code:: python + + from tqdm import tqdm + from time import sleep + + text = "" + for char in tqdm(["a", "b", "c", "d"]): + sleep(0.25) + text = text + char + +``trange(i)`` is a special optimised instance of ``tqdm(range(i))``: + +.. code:: python + + from tqdm import trange + + for i in trange(100): + sleep(0.01) + +Instantiation outside of the loop allows for manual control over ``tqdm()``: + +.. code:: python + + pbar = tqdm(["a", "b", "c", "d"]) + for char in pbar: + sleep(0.25) + pbar.set_description("Processing %s" % char) + +Manual +~~~~~~ + +Manual control of ``tqdm()`` updates using a ``with`` statement: + +.. code:: python + + with tqdm(total=100) as pbar: + for i in range(10): + sleep(0.1) + pbar.update(10) + +If the optional variable ``total`` (or an iterable with ``len()``) is +provided, predictive stats are displayed. + +``with`` is also optional (you can just assign ``tqdm()`` to a variable, +but in this case don't forget to ``del`` or ``close()`` at the end: + +.. code:: python + + pbar = tqdm(total=100) + for i in range(10): + sleep(0.1) + pbar.update(10) + pbar.close() + +Module +~~~~~~ + +Perhaps the most wonderful use of ``tqdm`` is in a script or on the command +line. Simply inserting ``tqdm`` (or ``python -m tqdm``) between pipes will pass +through all ``stdin`` to ``stdout`` while printing progress to ``stderr``. + +The example below demonstrate counting the number of lines in all Python files +in the current directory, with timing information included. + +.. code:: sh + + $ time find . -name '*.py' -type f -exec cat \{} \; | wc -l + 857365 + + real 0m3.458s + user 0m0.274s + sys 0m3.325s + + $ time find . -name '*.py' -type f -exec cat \{} \; | tqdm | wc -l + 857366it [00:03, 246471.31it/s] + 857365 + + real 0m3.585s + user 0m0.862s + sys 0m3.358s + +Note that the usual arguments for ``tqdm`` can also be specified. + +.. code:: sh + + $ find . -name '*.py' -type f -exec cat \{} \; | + tqdm --unit loc --unit_scale --total 857366 >> /dev/null + 100%|█████████████████████████████████| 857K/857K [00:04<00:00, 246Kloc/s] + +Backing up a large directory? + +.. code:: sh + + $ tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` \ + > backup.tgz + 44%|██████████████▊ | 153M/352M [00:14<00:18, 11.0MB/s] + +This can be beautified further: + +.. code:: sh + + $ BYTES="$(du -sb docs/ | cut -f1)" + $ tar -cf - docs/ \ + | tqdm --bytes --total "$BYTES" --desc Processing | gzip \ + | tqdm --bytes --total "$BYTES" --desc Compressed --position 1 \ + > ~/backup.tgz + Processing: 100%|██████████████████████| 352M/352M [00:14<00:00, 30.2MB/s] + Compressed: 42%|█████████▎ | 148M/352M [00:14<00:19, 10.9MB/s] + +Or done on a file level using 7-zip: + +.. code:: sh + + $ 7z a -bd -r backup.7z docs/ | grep Compressing \ + | tqdm --total $(find docs/ -type f | wc -l) --unit files \ + | grep -v Compressing + 100%|██████████████████████████▉| 15327/15327 [01:00<00:00, 712.96files/s] + +Pre-existing CLI programs already outputting basic progress information will +benefit from ``tqdm``'s ``--update`` and ``--update_to`` flags: + +.. code:: sh + + $ seq 3 0.1 5 | tqdm --total 5 --update_to --null + 100%|████████████████████████████████████| 5.0/5 [00:00<00:00, 9673.21it/s] + $ seq 10 | tqdm --update --null # 1 + 2 + ... + 10 = 55 iterations + 55it [00:00, 90006.52it/s] + +FAQ and Known Issues +-------------------- + +|GitHub-Issues| + +The most common issues relate to excessive output on multiple lines, instead +of a neat one-line progress bar. + +- Consoles in general: require support for carriage return (``CR``, ``\r``). +- Nested progress bars: + + * Consoles in general: require support for moving cursors up to the + previous line. For example, + `IDLE <https://github.com/tqdm/tqdm/issues/191#issuecomment-230168030>`__, + `ConEmu <https://github.com/tqdm/tqdm/issues/254>`__ and + `PyCharm <https://github.com/tqdm/tqdm/issues/203>`__ (also + `here <https://github.com/tqdm/tqdm/issues/208>`__, + `here <https://github.com/tqdm/tqdm/issues/307>`__, and + `here <https://github.com/tqdm/tqdm/issues/454#issuecomment-335416815>`__) + lack full support. + * Windows: additionally may require the Python module ``colorama`` + to ensure nested bars stay within their respective lines. + +- Unicode: + + * Environments which report that they support unicode will have solid smooth + progressbars. The fallback is an ``ascii``-only bar. + * Windows consoles often only partially support unicode and thus + `often require explicit ascii=True <https://github.com/tqdm/tqdm/issues/454#issuecomment-335416815>`__ + (also `here <https://github.com/tqdm/tqdm/issues/499>`__). This is due to + either normal-width unicode characters being incorrectly displayed as + "wide", or some unicode characters not rendering. + +- Wrapping generators: + + * Generator wrapper functions tend to hide the length of iterables. + ``tqdm`` does not. + * Replace ``tqdm(enumerate(...))`` with ``enumerate(tqdm(...))`` or + ``tqdm(enumerate(x), total=len(x), ...)``. + The same applies to ``numpy.ndenumerate``. + * Replace ``tqdm(zip(a, b))`` with ``zip(tqdm(a), b)`` or even + ``zip(tqdm(a), tqdm(b))``. + * The same applies to ``itertools``. + * Some useful convenience functions can be found under ``tqdm.contrib``. + +- `Hanging pipes in python2 <https://github.com/tqdm/tqdm/issues/359>`__: + when using ``tqdm`` on the CLI, you may need to use Python 3.5+ for correct + buffering. +- `No intermediate output in docker-compose <https://github.com/tqdm/tqdm/issues/771>`__: + use ``docker-compose run`` instead of ``docker-compose up`` and ``tty: true``. + +If you come across any other difficulties, browse and file |GitHub-Issues|. + +Documentation +------------- + +|Py-Versions| |README-Hits| (Since 19 May 2016) + +.. code:: python + + class tqdm(): + """ + Decorate an iterable object, returning an iterator which acts exactly + like the original iterable, but prints a dynamically updating + progressbar every time a value is requested. + """ + + def __init__(self, iterable=None, desc=None, total=None, leave=True, + file=None, ncols=None, mininterval=0.1, + maxinterval=10.0, miniters=None, ascii=None, disable=False, + unit='it', unit_scale=False, dynamic_ncols=False, + smoothing=0.3, bar_format=None, initial=0, position=None, + postfix=None, unit_divisor=1000): + +Parameters +~~~~~~~~~~ + +* iterable : iterable, optional + Iterable to decorate with a progressbar. + Leave blank to manually manage the updates. +* desc : str, optional + Prefix for the progressbar. +* total : int or float, optional + The number of expected iterations. If unspecified, + len(iterable) is used if possible. If float("inf") or as a last + resort, only basic progress statistics are displayed + (no ETA, no progressbar). + If ``gui`` is True and this parameter needs subsequent updating, + specify an initial arbitrary large positive number, + e.g. 9e9. +* leave : bool, optional + If [default: True], keeps all traces of the progressbar + upon termination of iteration. + If ``None``, will leave only if ``position`` is ``0``. +* file : ``io.TextIOWrapper`` or ``io.StringIO``, optional + Specifies where to output the progress messages + (default: sys.stderr). Uses ``file.write(str)`` and ``file.flush()`` + methods. For encoding, see ``write_bytes``. +* ncols : int, optional + The width of the entire output message. If specified, + dynamically resizes the progressbar to stay within this bound. + If unspecified, attempts to use environment width. The + fallback is a meter width of 10 and no limit for the counter and + statistics. If 0, will not print any meter (only stats). +* mininterval : float, optional + Minimum progress display update interval [default: 0.1] seconds. +* maxinterval : float, optional + Maximum progress display update interval [default: 10] seconds. + Automatically adjusts ``miniters`` to correspond to ``mininterval`` + after long display update lag. Only works if ``dynamic_miniters`` + or monitor thread is enabled. +* miniters : int or float, optional + Minimum progress display update interval, in iterations. + If 0 and ``dynamic_miniters``, will automatically adjust to equal + ``mininterval`` (more CPU efficient, good for tight loops). + If > 0, will skip display of specified number of iterations. + Tweak this and ``mininterval`` to get very efficient loops. + If your progress is erratic with both fast and slow iterations + (network, skipping items, etc) you should set miniters=1. +* ascii : bool or str, optional + If unspecified or False, use unicode (smooth blocks) to fill + the meter. The fallback is to use ASCII characters " 123456789#". +* disable : bool, optional + Whether to disable the entire progressbar wrapper + [default: False]. If set to None, disable on non-TTY. +* unit : str, optional + String that will be used to define the unit of each iteration + [default: it]. +* unit_scale : bool or int or float, optional + If 1 or True, the number of iterations will be reduced/scaled + automatically and a metric prefix following the + International System of Units standard will be added + (kilo, mega, etc.) [default: False]. If any other non-zero + number, will scale ``total`` and ``n``. +* dynamic_ncols : bool, optional + If set, constantly alters ``ncols`` and ``nrows`` to the + environment (allowing for window resizes) [default: False]. +* smoothing : float, optional + Exponential moving average smoothing factor for speed estimates + (ignored in GUI mode). Ranges from 0 (average speed) to 1 + (current/instantaneous speed) [default: 0.3]. +* bar_format : str, optional + Specify a custom bar string formatting. May impact performance. + [default: '{l_bar}{bar}{r_bar}'], where + l_bar='{desc}: {percentage:3.0f}%|' and + r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, ' + '{rate_fmt}{postfix}]' + Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, + percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, + rate, rate_fmt, rate_noinv, rate_noinv_fmt, + rate_inv, rate_inv_fmt, postfix, unit_divisor, + remaining, remaining_s, eta. + Note that a trailing ": " is automatically removed after {desc} + if the latter is empty. +* initial : int or float, optional + The initial counter value. Useful when restarting a progress + bar [default: 0]. If using float, consider specifying ``{n:.3f}`` + or similar in ``bar_format``, or specifying ``unit_scale``. +* position : int, optional + Specify the line offset to print this bar (starting from 0) + Automatic if unspecified. + Useful to manage multiple bars at once (eg, from threads). +* postfix : dict or ``*``, optional + Specify additional stats to display at the end of the bar. + Calls ``set_postfix(**postfix)`` if possible (dict). +* unit_divisor : float, optional + [default: 1000], ignored unless ``unit_scale`` is True. +* write_bytes : bool, optional + If (default: None) and ``file`` is unspecified, + bytes will be written in Python 2. If ``True`` will also write + bytes. In all other cases will default to unicode. +* lock_args : tuple, optional + Passed to ``refresh`` for intermediate output + (initialisation, iterating, and updating). +* nrows : int, optional + The screen height. If specified, hides nested bars outside this + bound. If unspecified, attempts to use environment height. + The fallback is 20. +* colour : str, optional + Bar colour (e.g. 'green', '#00ff00'). +* delay : float, optional + Don't display until [default: 0] seconds have elapsed. + +Extra CLI Options +~~~~~~~~~~~~~~~~~ + +* delim : chr, optional + Delimiting character [default: '\n']. Use '\0' for null. + N.B.: on Windows systems, Python converts '\n' to '\r\n'. +* buf_size : int, optional + String buffer size in bytes [default: 256] + used when ``delim`` is specified. +* bytes : bool, optional + If true, will count bytes, ignore ``delim``, and default + ``unit_scale`` to True, ``unit_divisor`` to 1024, and ``unit`` to 'B'. +* tee : bool, optional + If true, passes ``stdin`` to both ``stderr`` and ``stdout``. +* update : bool, optional + If true, will treat input as newly elapsed iterations, + i.e. numbers to pass to ``update()``. Note that this is slow + (~2e5 it/s) since every input must be decoded as a number. +* update_to : bool, optional + If true, will treat input as total elapsed iterations, + i.e. numbers to assign to ``self.n``. Note that this is slow + (~2e5 it/s) since every input must be decoded as a number. +* null : bool, optional + If true, will discard input (no stdout). +* manpath : str, optional + Directory in which to install tqdm man pages. +* comppath : str, optional + Directory in which to place tqdm completion. +* log : str, optional + CRITICAL|FATAL|ERROR|WARN(ING)|[default: 'INFO']|DEBUG|NOTSET. + +Returns +~~~~~~~ + +* out : decorated iterator. + +.. code:: python + + class tqdm(): + def update(self, n=1): + """ + Manually update the progress bar, useful for streams + such as reading files. + E.g.: + >>> t = tqdm(total=filesize) # Initialise + >>> for current_buffer in stream: + ... ... + ... t.update(len(current_buffer)) + >>> t.close() + The last line is highly recommended, but possibly not necessary if + ``t.update()`` will be called in such a way that ``filesize`` will be + exactly reached and printed. + + Parameters + ---------- + n : int or float, optional + Increment to add to the internal counter of iterations + [default: 1]. If using float, consider specifying ``{n:.3f}`` + or similar in ``bar_format``, or specifying ``unit_scale``. + + Returns + ------- + out : bool or None + True if a ``display()`` was triggered. + """ + + def close(self): + """Cleanup and (if leave=False) close the progressbar.""" + + def clear(self, nomove=False): + """Clear current bar display.""" + + def refresh(self): + """ + Force refresh the display of this bar. + + Parameters + ---------- + nolock : bool, optional + If ``True``, does not lock. + If [default: ``False``]: calls ``acquire()`` on internal lock. + lock_args : tuple, optional + Passed to internal lock's ``acquire()``. + If specified, will only ``display()`` if ``acquire()`` returns ``True``. + """ + + def unpause(self): + """Restart tqdm timer from last print time.""" + + def reset(self, total=None): + """ + Resets to 0 iterations for repeated use. + + Consider combining with ``leave=True``. + + Parameters + ---------- + total : int or float, optional. Total to use for the new bar. + """ + + def set_description(self, desc=None, refresh=True): + """ + Set/modify description of the progress bar. + + Parameters + ---------- + desc : str, optional + refresh : bool, optional + Forces refresh [default: True]. + """ + + def set_postfix(self, ordered_dict=None, refresh=True, **tqdm_kwargs): + """ + Set/modify postfix (additional stats) + with automatic formatting based on datatype. + + Parameters + ---------- + ordered_dict : dict or OrderedDict, optional + refresh : bool, optional + Forces refresh [default: True]. + kwargs : dict, optional + """ + + @classmethod + def write(cls, s, file=sys.stdout, end="\n"): + """Print a message via tqdm (without overlap with bars).""" + + @property + def format_dict(self): + """Public API for read-only member access.""" + + def display(self, msg=None, pos=None): + """ + Use ``self.sp`` to display ``msg`` in the specified ``pos``. + + Consider overloading this function when inheriting to use e.g.: + ``self.some_frontend(**self.format_dict)`` instead of ``self.sp``. + + Parameters + ---------- + msg : str, optional. What to display (default: ``repr(self)``). + pos : int, optional. Position to ``moveto`` + (default: ``abs(self.pos)``). + """ + + @classmethod + @contextmanager + def wrapattr(cls, stream, method, total=None, bytes=True, **tqdm_kwargs): + """ + stream : file-like object. + method : str, "read" or "write". The result of ``read()`` and + the first argument of ``write()`` should have a ``len()``. + + >>> with tqdm.wrapattr(file_obj, "read", total=file_obj.size) as fobj: + ... while True: + ... chunk = fobj.read(chunk_size) + ... if not chunk: + ... break + """ + + @classmethod + def pandas(cls, *targs, **tqdm_kwargs): + """Registers the current `tqdm` class with `pandas`.""" + + def trange(*args, **tqdm_kwargs): + """ + A shortcut for `tqdm(xrange(*args), **tqdm_kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + +Convenience Functions +~~~~~~~~~~~~~~~~~~~~~ + +.. code:: python + + def tqdm.contrib.tenumerate(iterable, start=0, total=None, + tqdm_class=tqdm.auto.tqdm, **tqdm_kwargs): + """Equivalent of `numpy.ndenumerate` or builtin `enumerate`.""" + + def tqdm.contrib.tzip(iter1, *iter2plus, **tqdm_kwargs): + """Equivalent of builtin `zip`.""" + + def tqdm.contrib.tmap(function, *sequences, **tqdm_kwargs): + """Equivalent of builtin `map`.""" + +Submodules +~~~~~~~~~~ + +.. code:: python + + class tqdm.notebook.tqdm(tqdm.tqdm): + """IPython/Jupyter Notebook widget.""" + + class tqdm.auto.tqdm(tqdm.tqdm): + """Automatically chooses beween `tqdm.notebook` and `tqdm.tqdm`.""" + + class tqdm.asyncio.tqdm(tqdm.tqdm): + """Asynchronous version.""" + @classmethod + def as_completed(cls, fs, *, loop=None, timeout=None, total=None, + **tqdm_kwargs): + """Wrapper for `asyncio.as_completed`.""" + + class tqdm.gui.tqdm(tqdm.tqdm): + """Matplotlib GUI version.""" + + class tqdm.tk.tqdm(tqdm.tqdm): + """Tkinter GUI version.""" + + class tqdm.rich.tqdm(tqdm.tqdm): + """`rich.progress` version.""" + + class tqdm.keras.TqdmCallback(keras.callbacks.Callback): + """Keras callback for epoch and batch progress.""" + + class tqdm.dask.TqdmCallback(dask.callbacks.Callback): + """Dask callback for task progress.""" + + +``contrib`` ++++++++++++ + +The ``tqdm.contrib`` package also contains experimental modules: + +- ``tqdm.contrib.itertools``: Thin wrappers around ``itertools`` +- ``tqdm.contrib.concurrent``: Thin wrappers around ``concurrent.futures`` +- ``tqdm.contrib.slack``: Posts to `Slack <https://slack.com>`__ bots +- ``tqdm.contrib.discord``: Posts to `Discord <https://discord.com>`__ bots +- ``tqdm.contrib.telegram``: Posts to `Telegram <https://telegram.org>`__ bots +- ``tqdm.contrib.bells``: Automagically enables all optional features + + * ``auto``, ``pandas``, ``slack``, ``discord``, ``telegram`` + +Examples and Advanced Usage +--------------------------- + +- See the `examples <https://github.com/tqdm/tqdm/tree/master/examples>`__ + folder; +- import the module and run ``help()``; +- consult the `wiki <https://github.com/tqdm/tqdm/wiki>`__; + + * this has an + `excellent article <https://github.com/tqdm/tqdm/wiki/How-to-make-a-great-Progress-Bar>`__ + on how to make a **great** progressbar; + +- check out the `slides from PyData London <https://tqdm.github.io/PyData2019/slides.html>`__, or +- run the |binder-demo|. + +Description and additional stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Custom information can be displayed and updated dynamically on ``tqdm`` bars +with the ``desc`` and ``postfix`` arguments: + +.. code:: python + + from tqdm import tqdm, trange + from random import random, randint + from time import sleep + + with trange(10) as t: + for i in t: + # Description will be displayed on the left + t.set_description('GEN %i' % i) + # Postfix will be displayed on the right, + # formatted automatically based on argument's datatype + t.set_postfix(loss=random(), gen=randint(1,999), str='h', + lst=[1, 2]) + sleep(0.1) + + with tqdm(total=10, bar_format="{postfix[0]} {postfix[1][value]:>8.2g}", + postfix=["Batch", dict(value=0)]) as t: + for i in range(10): + sleep(0.1) + t.postfix[1]["value"] = i / 2 + t.update() + +Points to remember when using ``{postfix[...]}`` in the ``bar_format`` string: + +- ``postfix`` also needs to be passed as an initial argument in a compatible + format, and +- ``postfix`` will be auto-converted to a string if it is a ``dict``-like + object. To prevent this behaviour, insert an extra item into the dictionary + where the key is not a string. + +Additional ``bar_format`` parameters may also be defined by overriding +``format_dict``, and the bar itself may be modified using ``ascii``: + +.. code:: python + + from tqdm import tqdm + class TqdmExtraFormat(tqdm): + """Provides a `total_time` format parameter""" + @property + def format_dict(self): + d = super(TqdmExtraFormat, self).format_dict + total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1) + d.update(total_time=self.format_interval(total_time) + " in total") + return d + + for i in TqdmExtraFormat( + range(9), ascii=" .oO0", + bar_format="{total_time}: {percentage:.0f}%|{bar}{r_bar}"): + if i == 4: + break + +.. code:: + + 00:00 in total: 44%|0000. | 4/9 [00:00<00:00, 962.93it/s] + +Note that ``{bar}`` also supports a format specifier ``[width][type]``. + +- ``width`` + + * unspecified (default): automatic to fill ``ncols`` + * ``int >= 0``: fixed width overriding ``ncols`` logic + * ``int < 0``: subtract from the automatic default + +- ``type`` + + * ``a``: ascii (``ascii=True`` override) + * ``u``: unicode (``ascii=False`` override) + * ``b``: blank (``ascii=" "`` override) + +This means a fixed bar with right-justified text may be created by using: +``bar_format="{l_bar}{bar:10}|{bar:-10b}right-justified"`` + +Nested progress bars +~~~~~~~~~~~~~~~~~~~~ + +``tqdm`` supports nested progress bars. Here's an example: + +.. code:: python + + from tqdm.auto import trange + from time import sleep + + for i in trange(4, desc='1st loop'): + for j in trange(5, desc='2nd loop'): + for k in trange(50, desc='3rd loop', leave=False): + sleep(0.01) + +For manual control over positioning (e.g. for multi-processing use), +you may specify ``position=n`` where ``n=0`` for the outermost bar, +``n=1`` for the next, and so on. +However, it's best to check if ``tqdm`` can work without manual ``position`` +first. + +.. code:: python + + from time import sleep + from tqdm import trange, tqdm + from multiprocessing import Pool, RLock, freeze_support + + L = list(range(9)) + + def progresser(n): + interval = 0.001 / (n + 2) + total = 5000 + text = "#{}, est. {:<04.2}s".format(n, interval * total) + for _ in trange(total, desc=text, position=n): + sleep(interval) + + if __name__ == '__main__': + freeze_support() # for Windows support + tqdm.set_lock(RLock()) # for managing output contention + p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) + p.map(progresser, L) + +Note that in Python 3, ``tqdm.write`` is thread-safe: + +.. code:: python + + from time import sleep + from tqdm import tqdm, trange + from concurrent.futures import ThreadPoolExecutor + + L = list(range(9)) + + def progresser(n): + interval = 0.001 / (n + 2) + total = 5000 + text = "#{}, est. {:<04.2}s".format(n, interval * total) + for _ in trange(total, desc=text): + sleep(interval) + if n == 6: + tqdm.write("n == 6 completed.") + tqdm.write("`tqdm.write()` is thread-safe in py3!") + + if __name__ == '__main__': + with ThreadPoolExecutor() as p: + p.map(progresser, L) + +Hooks and callbacks +~~~~~~~~~~~~~~~~~~~ + +``tqdm`` can easily support callbacks/hooks and manual updates. +Here's an example with ``urllib``: + +**``urllib.urlretrieve`` documentation** + + | [...] + | If present, the hook function will be called once + | on establishment of the network connection and once after each block read + | thereafter. The hook will be passed three arguments; a count of blocks + | transferred so far, a block size in bytes, and the total size of the file. + | [...] + +.. code:: python + + import urllib, os + from tqdm import tqdm + urllib = getattr(urllib, 'request', urllib) + + class TqdmUpTo(tqdm): + """Provides `update_to(n)` which uses `tqdm.update(delta_n)`.""" + def update_to(self, b=1, bsize=1, tsize=None): + """ + b : int, optional + Number of blocks transferred so far [default: 1]. + bsize : int, optional + Size of each block (in tqdm units) [default: 1]. + tsize : int, optional + Total size (in tqdm units). If [default: None] remains unchanged. + """ + if tsize is not None: + self.total = tsize + return self.update(b * bsize - self.n) # also sets self.n = b * bsize + + eg_link = "https://caspersci.uk.to/matryoshka.zip" + with TqdmUpTo(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, + desc=eg_link.split('/')[-1]) as t: # all optional kwargs + urllib.urlretrieve(eg_link, filename=os.devnull, + reporthook=t.update_to, data=None) + t.total = t.n + +Inspired by `twine#242 <https://github.com/pypa/twine/pull/242>`__. +Functional alternative in +`examples/tqdm_wget.py <https://github.com/tqdm/tqdm/blob/master/examples/tqdm_wget.py>`__. + +It is recommend to use ``miniters=1`` whenever there is potentially +large differences in iteration speed (e.g. downloading a file over +a patchy connection). + +**Wrapping read/write methods** + +To measure throughput through a file-like object's ``read`` or ``write`` +methods, use ``CallbackIOWrapper``: + +.. code:: python + + from tqdm.auto import tqdm + from tqdm.utils import CallbackIOWrapper + + with tqdm(total=file_obj.size, + unit='B', unit_scale=True, unit_divisor=1024) as t: + fobj = CallbackIOWrapper(t.update, file_obj, "read") + while True: + chunk = fobj.read(chunk_size) + if not chunk: + break + t.reset() + # ... continue to use `t` for something else + +Alternatively, use the even simpler ``wrapattr`` convenience function, +which would condense both the ``urllib`` and ``CallbackIOWrapper`` examples +down to: + +.. code:: python + + import urllib, os + from tqdm import tqdm + + eg_link = "https://caspersci.uk.to/matryoshka.zip" + response = getattr(urllib, 'request', urllib).urlopen(eg_link) + with tqdm.wrapattr(open(os.devnull, "wb"), "write", + miniters=1, desc=eg_link.split('/')[-1], + total=getattr(response, 'length', None)) as fout: + for chunk in response: + fout.write(chunk) + +The ``requests`` equivalent is nearly identical: + +.. code:: python + + import requests, os + from tqdm import tqdm + + eg_link = "https://caspersci.uk.to/matryoshka.zip" + response = requests.get(eg_link, stream=True) + with tqdm.wrapattr(open(os.devnull, "wb"), "write", + miniters=1, desc=eg_link.split('/')[-1], + total=int(response.headers.get('content-length', 0))) as fout: + for chunk in response.iter_content(chunk_size=4096): + fout.write(chunk) + +**Custom callback** + +``tqdm`` is known for intelligently skipping unnecessary displays. To make a +custom callback take advantage of this, simply use the return value of +``update()``. This is set to ``True`` if a ``display()`` was triggered. + +.. code:: python + + from tqdm.auto import tqdm as std_tqdm + + def external_callback(*args, **kwargs): + ... + + class TqdmExt(std_tqdm): + def update(self, n=1): + displayed = super(TqdmExt, self).update(n) + if displayed: + external_callback(**self.format_dict) + return displayed + +``asyncio`` +~~~~~~~~~~~ + +Note that ``break`` isn't currently caught by asynchronous iterators. +This means that ``tqdm`` cannot clean up after itself in this case: + +.. code:: python + + from tqdm.asyncio import tqdm + + async for i in tqdm(range(9)): + if i == 2: + break + +Instead, either call ``pbar.close()`` manually or use the context manager syntax: + +.. code:: python + + from tqdm.asyncio import tqdm + + with tqdm(range(9)) as pbar: + async for i in pbar: + if i == 2: + break + +Pandas Integration +~~~~~~~~~~~~~~~~~~ + +Due to popular demand we've added support for ``pandas`` -- here's an example +for ``DataFrame.progress_apply`` and ``DataFrameGroupBy.progress_apply``: + +.. code:: python + + import pandas as pd + import numpy as np + from tqdm import tqdm + + df = pd.DataFrame(np.random.randint(0, 100, (100000, 6))) + + # Register `pandas.progress_apply` and `pandas.Series.map_apply` with `tqdm` + # (can use `tqdm.gui.tqdm`, `tqdm.notebook.tqdm`, optional kwargs, etc.) + tqdm.pandas(desc="my bar!") + + # Now you can use `progress_apply` instead of `apply` + # and `progress_map` instead of `map` + df.progress_apply(lambda x: x**2) + # can also groupby: + # df.groupby(0).progress_apply(lambda x: x**2) + +In case you're interested in how this works (and how to modify it for your +own callbacks), see the +`examples <https://github.com/tqdm/tqdm/tree/master/examples>`__ +folder or import the module and run ``help()``. + +Keras Integration +~~~~~~~~~~~~~~~~~ + +A ``keras`` callback is also available: + +.. code:: python + + from tqdm.keras import TqdmCallback + + ... + + model.fit(..., verbose=0, callbacks=[TqdmCallback()]) + +Dask Integration +~~~~~~~~~~~~~~~~ + +A ``dask`` callback is also available: + +.. code:: python + + from tqdm.dask import TqdmCallback + + with TqdmCallback(desc="compute"): + ... + arr.compute() + + # or use callback globally + cb = TqdmCallback(desc="global") + cb.register() + arr.compute() + +IPython/Jupyter Integration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +IPython/Jupyter is supported via the ``tqdm.notebook`` submodule: + +.. code:: python + + from tqdm.notebook import trange, tqdm + from time import sleep + + for i in trange(3, desc='1st loop'): + for j in tqdm(range(100), desc='2nd loop'): + sleep(0.01) + +In addition to ``tqdm`` features, the submodule provides a native Jupyter +widget (compatible with IPython v1-v4 and Jupyter), fully working nested bars +and colour hints (blue: normal, green: completed, red: error/interrupt, +light blue: no ETA); as demonstrated below. + +|Screenshot-Jupyter1| +|Screenshot-Jupyter2| +|Screenshot-Jupyter3| + +The ``notebook`` version supports percentage or pixels for overall width +(e.g.: ``ncols='100%'`` or ``ncols='480px'``). + +It is also possible to let ``tqdm`` automatically choose between +console or notebook versions by using the ``autonotebook`` submodule: + +.. code:: python + + from tqdm.autonotebook import tqdm + tqdm.pandas() + +Note that this will issue a ``TqdmExperimentalWarning`` if run in a notebook +since it is not meant to be possible to distinguish between ``jupyter notebook`` +and ``jupyter console``. Use ``auto`` instead of ``autonotebook`` to suppress +this warning. + +Note that notebooks will display the bar in the cell where it was created. +This may be a different cell from the one where it is used. +If this is not desired, either + +- delay the creation of the bar to the cell where it must be displayed, or +- create the bar with ``display=False``, and in a later cell call + ``display(bar.container)``: + +.. code:: python + + from tqdm.notebook import tqdm + pbar = tqdm(..., display=False) + +.. code:: python + + # different cell + display(pbar.container) + +The ``keras`` callback has a ``display()`` method which can be used likewise: + +.. code:: python + + from tqdm.keras import TqdmCallback + cbk = TqdmCallback(display=False) + +.. code:: python + + # different cell + cbk.display() + model.fit(..., verbose=0, callbacks=[cbk]) + +Another possibility is to have a single bar (near the top of the notebook) +which is constantly re-used (using ``reset()`` rather than ``close()``). +For this reason, the notebook version (unlike the CLI version) does not +automatically call ``close()`` upon ``Exception``. + +.. code:: python + + from tqdm.notebook import tqdm + pbar = tqdm() + +.. code:: python + + # different cell + iterable = range(100) + pbar.reset(total=len(iterable)) # initialise with new `total` + for i in iterable: + pbar.update() + pbar.refresh() # force print final status but don't `close()` + +Custom Integration +~~~~~~~~~~~~~~~~~~ + +To change the default arguments (such as making ``dynamic_ncols=True``), +simply use built-in Python magic: + +.. code:: python + + from functools import partial + from tqdm import tqdm as std_tqdm + tqdm = partial(std_tqdm, dynamic_ncols=True) + +For further customisation, +``tqdm`` may be inherited from to create custom callbacks (as with the +``TqdmUpTo`` example `above <#hooks-and-callbacks>`__) or for custom frontends +(e.g. GUIs such as notebook or plotting packages). In the latter case: + +1. ``def __init__()`` to call ``super().__init__(..., gui=True)`` to disable + terminal ``status_printer`` creation. +2. Redefine: ``close()``, ``clear()``, ``display()``. + +Consider overloading ``display()`` to use e.g. +``self.frontend(**self.format_dict)`` instead of ``self.sp(repr(self))``. + +Some submodule examples of inheritance: + +- `tqdm/notebook.py <https://github.com/tqdm/tqdm/blob/master/tqdm/notebook.py>`__ +- `tqdm/gui.py <https://github.com/tqdm/tqdm/blob/master/tqdm/gui.py>`__ +- `tqdm/tk.py <https://github.com/tqdm/tqdm/blob/master/tqdm/tk.py>`__ +- `tqdm/contrib/slack.py <https://github.com/tqdm/tqdm/blob/master/tqdm/contrib/slack.py>`__ +- `tqdm/contrib/discord.py <https://github.com/tqdm/tqdm/blob/master/tqdm/contrib/discord.py>`__ +- `tqdm/contrib/telegram.py <https://github.com/tqdm/tqdm/blob/master/tqdm/contrib/telegram.py>`__ + +Dynamic Monitor/Meter +~~~~~~~~~~~~~~~~~~~~~ + +You can use a ``tqdm`` as a meter which is not monotonically increasing. +This could be because ``n`` decreases (e.g. a CPU usage monitor) or ``total`` +changes. + +One example would be recursively searching for files. The ``total`` is the +number of objects found so far, while ``n`` is the number of those objects which +are files (rather than folders): + +.. code:: python + + from tqdm import tqdm + import os.path + + def find_files_recursively(path, show_progress=True): + files = [] + # total=1 assumes `path` is a file + t = tqdm(total=1, unit="file", disable=not show_progress) + if not os.path.exists(path): + raise IOError("Cannot find:" + path) + + def append_found_file(f): + files.append(f) + t.update() + + def list_found_dir(path): + """returns os.listdir(path) assuming os.path.isdir(path)""" + listing = os.listdir(path) + # subtract 1 since a "file" we found was actually this directory + t.total += len(listing) - 1 + # fancy way to give info without forcing a refresh + t.set_postfix(dir=path[-10:], refresh=False) + t.update(0) # may trigger a refresh + return listing + + def recursively_search(path): + if os.path.isdir(path): + for f in list_found_dir(path): + recursively_search(os.path.join(path, f)) + else: + append_found_file(path) + + recursively_search(path) + t.set_postfix(dir=path) + t.close() + return files + +Using ``update(0)`` is a handy way to let ``tqdm`` decide when to trigger a +display refresh to avoid console spamming. + +Writing messages +~~~~~~~~~~~~~~~~ + +This is a work in progress (see +`#737 <https://github.com/tqdm/tqdm/issues/737>`__). + +Since ``tqdm`` uses a simple printing mechanism to display progress bars, +you should not write any message in the terminal using ``print()`` while +a progressbar is open. + +To write messages in the terminal without any collision with ``tqdm`` bar +display, a ``.write()`` method is provided: + +.. code:: python + + from tqdm.auto import tqdm, trange + from time import sleep + + bar = trange(10) + for i in bar: + # Print using tqdm class method .write() + sleep(0.1) + if not (i % 3): + tqdm.write("Done task %i" % i) + # Can also use bar.write() + +By default, this will print to standard output ``sys.stdout``. but you can +specify any file-like object using the ``file`` argument. For example, this +can be used to redirect the messages writing to a log file or class. + +Redirecting writing +~~~~~~~~~~~~~~~~~~~ + +If using a library that can print messages to the console, editing the library +by replacing ``print()`` with ``tqdm.write()`` may not be desirable. +In that case, redirecting ``sys.stdout`` to ``tqdm.write()`` is an option. + +To redirect ``sys.stdout``, create a file-like class that will write +any input string to ``tqdm.write()``, and supply the arguments +``file=sys.stdout, dynamic_ncols=True``. + +A reusable canonical example is given below: + +.. code:: python + + from time import sleep + import contextlib + import sys + from tqdm import tqdm + from tqdm.contrib import DummyTqdmFile + + + @contextlib.contextmanager + def std_out_err_redirect_tqdm(): + orig_out_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = map(DummyTqdmFile, orig_out_err) + yield orig_out_err[0] + # Relay exceptions + except Exception as exc: + raise exc + # Always restore sys.stdout/err if necessary + finally: + sys.stdout, sys.stderr = orig_out_err + + def some_fun(i): + print("Fee, fi, fo,".split()[i]) + + # Redirect stdout to tqdm.write() (don't forget the `as save_stdout`) + with std_out_err_redirect_tqdm() as orig_stdout: + # tqdm needs the original stdout + # and dynamic_ncols=True to autodetect console width + for i in tqdm(range(3), file=orig_stdout, dynamic_ncols=True): + sleep(.5) + some_fun(i) + + # After the `with`, printing is restored + print("Done!") + +Redirecting ``logging`` +~~~~~~~~~~~~~~~~~~~~~~~ + +Similar to ``sys.stdout``/``sys.stderr`` as detailed above, console ``logging`` +may also be redirected to ``tqdm.write()``. + +Warning: if also redirecting ``sys.stdout``/``sys.stderr``, make sure to +redirect ``logging`` first if needed. + +Helper methods are available in ``tqdm.contrib.logging``. For example: + +.. code:: python + + import logging + from tqdm import trange + from tqdm.contrib.logging import logging_redirect_tqdm + + LOG = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + with logging_redirect_tqdm(): + for i in trange(9): + if i == 4: + LOG.info("console logging redirected to `tqdm.write()`") + # logging restored + +Monitoring thread, intervals and miniters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``tqdm`` implements a few tricks to increase efficiency and reduce overhead. + +- Avoid unnecessary frequent bar refreshing: ``mininterval`` defines how long + to wait between each refresh. ``tqdm`` always gets updated in the background, + but it will display only every ``mininterval``. +- Reduce number of calls to check system clock/time. +- ``mininterval`` is more intuitive to configure than ``miniters``. + A clever adjustment system ``dynamic_miniters`` will automatically adjust + ``miniters`` to the amount of iterations that fit into time ``mininterval``. + Essentially, ``tqdm`` will check if it's time to print without actually + checking time. This behaviour can be still be bypassed by manually setting + ``miniters``. + +However, consider a case with a combination of fast and slow iterations. +After a few fast iterations, ``dynamic_miniters`` will set ``miniters`` to a +large number. When iteration rate subsequently slows, ``miniters`` will +remain large and thus reduce display update frequency. To address this: + +- ``maxinterval`` defines the maximum time between display refreshes. + A concurrent monitoring thread checks for overdue updates and forces one + where necessary. + +The monitoring thread should not have a noticeable overhead, and guarantees +updates at least every 10 seconds by default. +This value can be directly changed by setting the ``monitor_interval`` of +any ``tqdm`` instance (i.e. ``t = tqdm.tqdm(...); t.monitor_interval = 2``). +The monitor thread may be disabled application-wide by setting +``tqdm.tqdm.monitor_interval = 0`` before instantiation of any ``tqdm`` bar. + + +Merch +----- + +You can buy `tqdm branded merch <https://tqdm.github.io/merch>`__ now! + +Contributions +------------- + +|GitHub-Commits| |GitHub-Issues| |GitHub-PRs| |OpenHub-Status| |GitHub-Contributions| |CII Best Practices| + +All source code is hosted on `GitHub <https://github.com/tqdm/tqdm>`__. +Contributions are welcome. + +See the +`CONTRIBUTING <https://github.com/tqdm/tqdm/blob/master/CONTRIBUTING.md>`__ +file for more information. + +Developers who have made significant contributions, ranked by *SLoC* +(surviving lines of code, +`git fame <https://github.com/casperdcl/git-fame>`__ ``-wMC --excl '\.(png|gif|jpg)$'``), +are: + +==================== ======================================================== ==== ================================ +Name ID SLoC Notes +==================== ======================================================== ==== ================================ +Casper da Costa-Luis `casperdcl <https://github.com/casperdcl>`__ ~78% primary maintainer |Gift-Casper| +Stephen Larroque `lrq3000 <https://github.com/lrq3000>`__ ~10% team member +Martin Zugnoni `martinzugnoni <https://github.com/martinzugnoni>`__ ~4% +Daniel Ecer `de-code <https://github.com/de-code>`__ ~2% +Richard Sheridan `richardsheridan <https://github.com/richardsheridan>`__ ~1% +Guangshuo Chen `chengs <https://github.com/chengs>`__ ~1% +Kyle Altendorf `altendky <https://github.com/altendky>`__ <1% +Matthew Stevens `mjstevens777 <https://github.com/mjstevens777>`__ <1% +Hadrien Mary `hadim <https://github.com/hadim>`__ <1% team member +Noam Yorav-Raphael `noamraph <https://github.com/noamraph>`__ <1% original author +Mikhail Korobov `kmike <https://github.com/kmike>`__ <1% team member +==================== ======================================================== ==== ================================ + +Ports to Other Languages +~~~~~~~~~~~~~~~~~~~~~~~~ + +A list is available on +`this wiki page <https://github.com/tqdm/tqdm/wiki/tqdm-ports>`__. + + +LICENCE +------- + +Open Source (OSI approved): |LICENCE| + +Citation information: |DOI| + +|README-Hits| (Since 19 May 2016) + +.. |Logo| image:: https://img.tqdm.ml/logo.gif +.. |Screenshot| image:: https://img.tqdm.ml/tqdm.gif +.. |Video| image:: https://img.tqdm.ml/video.jpg + :target: https://tqdm.github.io/video +.. |Slides| image:: https://img.tqdm.ml/slides.jpg + :target: https://tqdm.github.io/PyData2019/slides.html +.. |Merch| image:: https://img.tqdm.ml/merch.jpg + :target: https://tqdm.github.io/merch +.. |Build-Status| image:: https://img.shields.io/github/workflow/status/tqdm/tqdm/Test/master?logo=GitHub + :target: https://github.com/tqdm/tqdm/actions?query=workflow%3ATest +.. |Coverage-Status| image:: https://img.shields.io/coveralls/github/tqdm/tqdm/master?logo=coveralls + :target: https://coveralls.io/github/tqdm/tqdm +.. |Branch-Coverage-Status| image:: https://codecov.io/gh/tqdm/tqdm/branch/master/graph/badge.svg + :target: https://codecov.io/gh/tqdm/tqdm +.. |Codacy-Grade| image:: https://app.codacy.com/project/badge/Grade/3f965571598f44549c7818f29cdcf177 + :target: https://www.codacy.com/gh/tqdm/tqdm/dashboard +.. |CII Best Practices| image:: https://bestpractices.coreinfrastructure.org/projects/3264/badge + :target: https://bestpractices.coreinfrastructure.org/projects/3264 +.. |GitHub-Status| image:: https://img.shields.io/github/tag/tqdm/tqdm.svg?maxAge=86400&logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/releases +.. |GitHub-Forks| image:: https://img.shields.io/github/forks/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/network +.. |GitHub-Stars| image:: https://img.shields.io/github/stars/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/stargazers +.. |GitHub-Commits| image:: https://img.shields.io/github/commit-activity/y/tqdm/tqdm.svg?logo=git&logoColor=white + :target: https://github.com/tqdm/tqdm/graphs/commit-activity +.. |GitHub-Issues| image:: https://img.shields.io/github/issues-closed/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/issues?q= +.. |GitHub-PRs| image:: https://img.shields.io/github/issues-pr-closed/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/pulls +.. |GitHub-Contributions| image:: https://img.shields.io/github/contributors/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/graphs/contributors +.. |GitHub-Updated| image:: https://img.shields.io/github/last-commit/tqdm/tqdm/master.svg?logo=github&logoColor=white&label=pushed + :target: https://github.com/tqdm/tqdm/pulse +.. |Gift-Casper| image:: https://img.shields.io/badge/dynamic/json.svg?color=ff69b4&label=gifts%20received&prefix=%C2%A3&query=%24..sum&url=https%3A%2F%2Fcaspersci.uk.to%2Fgifts.json + :target: https://cdcl.ml/sponsor +.. |Versions| image:: https://img.shields.io/pypi/v/tqdm.svg + :target: https://tqdm.github.io/releases +.. |PyPI-Downloads| image:: https://img.shields.io/pypi/dm/tqdm.svg?label=pypi%20downloads&logo=PyPI&logoColor=white + :target: https://pepy.tech/project/tqdm +.. |Py-Versions| image:: https://img.shields.io/pypi/pyversions/tqdm.svg?logo=python&logoColor=white + :target: https://pypi.org/project/tqdm +.. |Conda-Forge-Status| image:: https://img.shields.io/conda/v/conda-forge/tqdm.svg?label=conda-forge&logo=conda-forge + :target: https://anaconda.org/conda-forge/tqdm +.. |Snapcraft| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft + :target: https://snapcraft.io/tqdm +.. |Docker| image:: https://img.shields.io/badge/docker-pull-blue.svg?logo=docker&logoColor=white + :target: https://hub.docker.com/r/tqdm/tqdm +.. |Libraries-Rank| image:: https://img.shields.io/librariesio/sourcerank/pypi/tqdm.svg?logo=koding&logoColor=white + :target: https://libraries.io/pypi/tqdm +.. |Libraries-Dependents| image:: https://img.shields.io/librariesio/dependent-repos/pypi/tqdm.svg?logo=koding&logoColor=white + :target: https://github.com/tqdm/tqdm/network/dependents +.. |OpenHub-Status| image:: https://www.openhub.net/p/tqdm/widgets/project_thin_badge?format=gif + :target: https://www.openhub.net/p/tqdm?ref=Thin+badge +.. |awesome-python| image:: https://awesome.re/mentioned-badge.svg + :target: https://github.com/vinta/awesome-python +.. |LICENCE| image:: https://img.shields.io/pypi/l/tqdm.svg + :target: https://raw.githubusercontent.com/tqdm/tqdm/master/LICENCE +.. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.595120-blue.svg + :target: https://doi.org/10.5281/zenodo.595120 +.. |binder-demo| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/tqdm/tqdm/master?filepath=DEMO.ipynb +.. |Screenshot-Jupyter1| image:: https://img.tqdm.ml/jupyter-1.gif +.. |Screenshot-Jupyter2| image:: https://img.tqdm.ml/jupyter-2.gif +.. |Screenshot-Jupyter3| image:: https://img.tqdm.ml/jupyter-3.gif +.. |README-Hits| image:: https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&style=social&r=https://github.com/tqdm/tqdm&l=https://img.tqdm.ml/favicon.png&f=https://img.tqdm.ml/logo.gif + :target: https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&a=plot&r=https://github.com/tqdm/tqdm&l=https://img.tqdm.ml/favicon.png&f=https://img.tqdm.ml/logo.gif&style=social diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..a45f98d --- /dev/null +++ b/README.rst @@ -0,0 +1,1503 @@ +|Logo| + +tqdm +==== + +|Py-Versions| |Versions| |Conda-Forge-Status| |Docker| |Snapcraft| + +|Build-Status| |Coverage-Status| |Branch-Coverage-Status| |Codacy-Grade| |Libraries-Rank| |PyPI-Downloads| + +|LICENCE| |OpenHub-Status| |binder-demo| |awesome-python| + +``tqdm`` derives from the Arabic word *taqaddum* (تقدّم) which can mean "progress," +and is an abbreviation for "I love you so much" in Spanish (*te quiero demasiado*). + +Instantly make your loops show a smart progress meter - just wrap any +iterable with ``tqdm(iterable)``, and you're done! + +.. code:: python + + from tqdm import tqdm + for i in tqdm(range(10000)): + ... + +``76%|████████████████████████ | 7568/10000 [00:33<00:10, 229.00it/s]`` + +``trange(N)`` can be also used as a convenient shortcut for +``tqdm(range(N))``. + +|Screenshot| + |Video| |Slides| |Merch| + +It can also be executed as a module with pipes: + +.. code:: sh + + $ seq 9999999 | tqdm --bytes | wc -l + 75.2MB [00:00, 217MB/s] + 9999999 + + $ tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` \ + > backup.tgz + 32%|██████████▍ | 8.89G/27.9G [00:42<01:31, 223MB/s] + +Overhead is low -- about 60ns per iteration (80ns with ``tqdm.gui``), and is +unit tested against performance regression. +By comparison, the well-established +`ProgressBar <https://github.com/niltonvolpato/python-progressbar>`__ has +an 800ns/iter overhead. + +In addition to its low overhead, ``tqdm`` uses smart algorithms to predict +the remaining time and to skip unnecessary iteration displays, which allows +for a negligible overhead in most cases. + +``tqdm`` works on any platform +(Linux, Windows, Mac, FreeBSD, NetBSD, Solaris/SunOS), +in any console or in a GUI, and is also friendly with IPython/Jupyter notebooks. + +``tqdm`` does not require any dependencies (not even ``curses``!), just +Python and an environment supporting ``carriage return \r`` and +``line feed \n`` control characters. + +------------------------------------------ + +.. contents:: Table of contents + :backlinks: top + :local: + + +Installation +------------ + +Latest PyPI stable release +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +|Versions| |PyPI-Downloads| |Libraries-Dependents| + +.. code:: sh + + pip install tqdm + +Latest development release on GitHub +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +|GitHub-Status| |GitHub-Stars| |GitHub-Commits| |GitHub-Forks| |GitHub-Updated| + +Pull and install pre-release ``devel`` branch: + +.. code:: sh + + pip install "git+https://github.com/tqdm/tqdm.git@devel#egg=tqdm" + +Latest Conda release +~~~~~~~~~~~~~~~~~~~~ + +|Conda-Forge-Status| + +.. code:: sh + + conda install -c conda-forge tqdm + +Latest Snapcraft release +~~~~~~~~~~~~~~~~~~~~~~~~ + +|Snapcraft| + +There are 3 channels to choose from: + +.. code:: sh + + snap install tqdm # implies --stable, i.e. latest tagged release + snap install tqdm --candidate # master branch + snap install tqdm --edge # devel branch + +Note that ``snap`` binaries are purely for CLI use (not ``import``-able), and +automatically set up ``bash`` tab-completion. + +Latest Docker release +~~~~~~~~~~~~~~~~~~~~~ + +|Docker| + +.. code:: sh + + docker pull tqdm/tqdm + docker run -i --rm tqdm/tqdm --help + +Other +~~~~~ + +There are other (unofficial) places where ``tqdm`` may be downloaded, particularly for CLI use: + +|Repology| + +.. |Repology| image:: https://repology.org/badge/tiny-repos/python:tqdm.svg + :target: https://repology.org/project/python:tqdm/versions + +Changelog +--------- + +The list of all changes is available either on GitHub's Releases: +|GitHub-Status|, on the +`wiki <https://github.com/tqdm/tqdm/wiki/Releases>`__, or on the +`website <https://tqdm.github.io/releases>`__. + + +Usage +----- + +``tqdm`` is very versatile and can be used in a number of ways. +The three main ones are given below. + +Iterable-based +~~~~~~~~~~~~~~ + +Wrap ``tqdm()`` around any iterable: + +.. code:: python + + from tqdm import tqdm + from time import sleep + + text = "" + for char in tqdm(["a", "b", "c", "d"]): + sleep(0.25) + text = text + char + +``trange(i)`` is a special optimised instance of ``tqdm(range(i))``: + +.. code:: python + + from tqdm import trange + + for i in trange(100): + sleep(0.01) + +Instantiation outside of the loop allows for manual control over ``tqdm()``: + +.. code:: python + + pbar = tqdm(["a", "b", "c", "d"]) + for char in pbar: + sleep(0.25) + pbar.set_description("Processing %s" % char) + +Manual +~~~~~~ + +Manual control of ``tqdm()`` updates using a ``with`` statement: + +.. code:: python + + with tqdm(total=100) as pbar: + for i in range(10): + sleep(0.1) + pbar.update(10) + +If the optional variable ``total`` (or an iterable with ``len()``) is +provided, predictive stats are displayed. + +``with`` is also optional (you can just assign ``tqdm()`` to a variable, +but in this case don't forget to ``del`` or ``close()`` at the end: + +.. code:: python + + pbar = tqdm(total=100) + for i in range(10): + sleep(0.1) + pbar.update(10) + pbar.close() + +Module +~~~~~~ + +Perhaps the most wonderful use of ``tqdm`` is in a script or on the command +line. Simply inserting ``tqdm`` (or ``python -m tqdm``) between pipes will pass +through all ``stdin`` to ``stdout`` while printing progress to ``stderr``. + +The example below demonstrate counting the number of lines in all Python files +in the current directory, with timing information included. + +.. code:: sh + + $ time find . -name '*.py' -type f -exec cat \{} \; | wc -l + 857365 + + real 0m3.458s + user 0m0.274s + sys 0m3.325s + + $ time find . -name '*.py' -type f -exec cat \{} \; | tqdm | wc -l + 857366it [00:03, 246471.31it/s] + 857365 + + real 0m3.585s + user 0m0.862s + sys 0m3.358s + +Note that the usual arguments for ``tqdm`` can also be specified. + +.. code:: sh + + $ find . -name '*.py' -type f -exec cat \{} \; | + tqdm --unit loc --unit_scale --total 857366 >> /dev/null + 100%|█████████████████████████████████| 857K/857K [00:04<00:00, 246Kloc/s] + +Backing up a large directory? + +.. code:: sh + + $ tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` \ + > backup.tgz + 44%|██████████████▊ | 153M/352M [00:14<00:18, 11.0MB/s] + +This can be beautified further: + +.. code:: sh + + $ BYTES="$(du -sb docs/ | cut -f1)" + $ tar -cf - docs/ \ + | tqdm --bytes --total "$BYTES" --desc Processing | gzip \ + | tqdm --bytes --total "$BYTES" --desc Compressed --position 1 \ + > ~/backup.tgz + Processing: 100%|██████████████████████| 352M/352M [00:14<00:00, 30.2MB/s] + Compressed: 42%|█████████▎ | 148M/352M [00:14<00:19, 10.9MB/s] + +Or done on a file level using 7-zip: + +.. code:: sh + + $ 7z a -bd -r backup.7z docs/ | grep Compressing \ + | tqdm --total $(find docs/ -type f | wc -l) --unit files \ + | grep -v Compressing + 100%|██████████████████████████▉| 15327/15327 [01:00<00:00, 712.96files/s] + +Pre-existing CLI programs already outputting basic progress information will +benefit from ``tqdm``'s ``--update`` and ``--update_to`` flags: + +.. code:: sh + + $ seq 3 0.1 5 | tqdm --total 5 --update_to --null + 100%|████████████████████████████████████| 5.0/5 [00:00<00:00, 9673.21it/s] + $ seq 10 | tqdm --update --null # 1 + 2 + ... + 10 = 55 iterations + 55it [00:00, 90006.52it/s] + +FAQ and Known Issues +-------------------- + +|GitHub-Issues| + +The most common issues relate to excessive output on multiple lines, instead +of a neat one-line progress bar. + +- Consoles in general: require support for carriage return (``CR``, ``\r``). +- Nested progress bars: + + * Consoles in general: require support for moving cursors up to the + previous line. For example, + `IDLE <https://github.com/tqdm/tqdm/issues/191#issuecomment-230168030>`__, + `ConEmu <https://github.com/tqdm/tqdm/issues/254>`__ and + `PyCharm <https://github.com/tqdm/tqdm/issues/203>`__ (also + `here <https://github.com/tqdm/tqdm/issues/208>`__, + `here <https://github.com/tqdm/tqdm/issues/307>`__, and + `here <https://github.com/tqdm/tqdm/issues/454#issuecomment-335416815>`__) + lack full support. + * Windows: additionally may require the Python module ``colorama`` + to ensure nested bars stay within their respective lines. + +- Unicode: + + * Environments which report that they support unicode will have solid smooth + progressbars. The fallback is an ``ascii``-only bar. + * Windows consoles often only partially support unicode and thus + `often require explicit ascii=True <https://github.com/tqdm/tqdm/issues/454#issuecomment-335416815>`__ + (also `here <https://github.com/tqdm/tqdm/issues/499>`__). This is due to + either normal-width unicode characters being incorrectly displayed as + "wide", or some unicode characters not rendering. + +- Wrapping generators: + + * Generator wrapper functions tend to hide the length of iterables. + ``tqdm`` does not. + * Replace ``tqdm(enumerate(...))`` with ``enumerate(tqdm(...))`` or + ``tqdm(enumerate(x), total=len(x), ...)``. + The same applies to ``numpy.ndenumerate``. + * Replace ``tqdm(zip(a, b))`` with ``zip(tqdm(a), b)`` or even + ``zip(tqdm(a), tqdm(b))``. + * The same applies to ``itertools``. + * Some useful convenience functions can be found under ``tqdm.contrib``. + +- `Hanging pipes in python2 <https://github.com/tqdm/tqdm/issues/359>`__: + when using ``tqdm`` on the CLI, you may need to use Python 3.5+ for correct + buffering. +- `No intermediate output in docker-compose <https://github.com/tqdm/tqdm/issues/771>`__: + use ``docker-compose run`` instead of ``docker-compose up`` and ``tty: true``. + +If you come across any other difficulties, browse and file |GitHub-Issues|. + +Documentation +------------- + +|Py-Versions| |README-Hits| (Since 19 May 2016) + +.. code:: python + + class tqdm(): + """ + Decorate an iterable object, returning an iterator which acts exactly + like the original iterable, but prints a dynamically updating + progressbar every time a value is requested. + """ + + def __init__(self, iterable=None, desc=None, total=None, leave=True, + file=None, ncols=None, mininterval=0.1, + maxinterval=10.0, miniters=None, ascii=None, disable=False, + unit='it', unit_scale=False, dynamic_ncols=False, + smoothing=0.3, bar_format=None, initial=0, position=None, + postfix=None, unit_divisor=1000): + +Parameters +~~~~~~~~~~ + +* iterable : iterable, optional + Iterable to decorate with a progressbar. + Leave blank to manually manage the updates. +* desc : str, optional + Prefix for the progressbar. +* total : int or float, optional + The number of expected iterations. If unspecified, + len(iterable) is used if possible. If float("inf") or as a last + resort, only basic progress statistics are displayed + (no ETA, no progressbar). + If ``gui`` is True and this parameter needs subsequent updating, + specify an initial arbitrary large positive number, + e.g. 9e9. +* leave : bool, optional + If [default: True], keeps all traces of the progressbar + upon termination of iteration. + If ``None``, will leave only if ``position`` is ``0``. +* file : ``io.TextIOWrapper`` or ``io.StringIO``, optional + Specifies where to output the progress messages + (default: sys.stderr). Uses ``file.write(str)`` and ``file.flush()`` + methods. For encoding, see ``write_bytes``. +* ncols : int, optional + The width of the entire output message. If specified, + dynamically resizes the progressbar to stay within this bound. + If unspecified, attempts to use environment width. The + fallback is a meter width of 10 and no limit for the counter and + statistics. If 0, will not print any meter (only stats). +* mininterval : float, optional + Minimum progress display update interval [default: 0.1] seconds. +* maxinterval : float, optional + Maximum progress display update interval [default: 10] seconds. + Automatically adjusts ``miniters`` to correspond to ``mininterval`` + after long display update lag. Only works if ``dynamic_miniters`` + or monitor thread is enabled. +* miniters : int or float, optional + Minimum progress display update interval, in iterations. + If 0 and ``dynamic_miniters``, will automatically adjust to equal + ``mininterval`` (more CPU efficient, good for tight loops). + If > 0, will skip display of specified number of iterations. + Tweak this and ``mininterval`` to get very efficient loops. + If your progress is erratic with both fast and slow iterations + (network, skipping items, etc) you should set miniters=1. +* ascii : bool or str, optional + If unspecified or False, use unicode (smooth blocks) to fill + the meter. The fallback is to use ASCII characters " 123456789#". +* disable : bool, optional + Whether to disable the entire progressbar wrapper + [default: False]. If set to None, disable on non-TTY. +* unit : str, optional + String that will be used to define the unit of each iteration + [default: it]. +* unit_scale : bool or int or float, optional + If 1 or True, the number of iterations will be reduced/scaled + automatically and a metric prefix following the + International System of Units standard will be added + (kilo, mega, etc.) [default: False]. If any other non-zero + number, will scale ``total`` and ``n``. +* dynamic_ncols : bool, optional + If set, constantly alters ``ncols`` and ``nrows`` to the + environment (allowing for window resizes) [default: False]. +* smoothing : float, optional + Exponential moving average smoothing factor for speed estimates + (ignored in GUI mode). Ranges from 0 (average speed) to 1 + (current/instantaneous speed) [default: 0.3]. +* bar_format : str, optional + Specify a custom bar string formatting. May impact performance. + [default: '{l_bar}{bar}{r_bar}'], where + l_bar='{desc}: {percentage:3.0f}%|' and + r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, ' + '{rate_fmt}{postfix}]' + Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, + percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, + rate, rate_fmt, rate_noinv, rate_noinv_fmt, + rate_inv, rate_inv_fmt, postfix, unit_divisor, + remaining, remaining_s, eta. + Note that a trailing ": " is automatically removed after {desc} + if the latter is empty. +* initial : int or float, optional + The initial counter value. Useful when restarting a progress + bar [default: 0]. If using float, consider specifying ``{n:.3f}`` + or similar in ``bar_format``, or specifying ``unit_scale``. +* position : int, optional + Specify the line offset to print this bar (starting from 0) + Automatic if unspecified. + Useful to manage multiple bars at once (eg, from threads). +* postfix : dict or ``*``, optional + Specify additional stats to display at the end of the bar. + Calls ``set_postfix(**postfix)`` if possible (dict). +* unit_divisor : float, optional + [default: 1000], ignored unless ``unit_scale`` is True. +* write_bytes : bool, optional + If (default: None) and ``file`` is unspecified, + bytes will be written in Python 2. If ``True`` will also write + bytes. In all other cases will default to unicode. +* lock_args : tuple, optional + Passed to ``refresh`` for intermediate output + (initialisation, iterating, and updating). +* nrows : int, optional + The screen height. If specified, hides nested bars outside this + bound. If unspecified, attempts to use environment height. + The fallback is 20. +* colour : str, optional + Bar colour (e.g. 'green', '#00ff00'). +* delay : float, optional + Don't display until [default: 0] seconds have elapsed. + +Extra CLI Options +~~~~~~~~~~~~~~~~~ + +* delim : chr, optional + Delimiting character [default: '\n']. Use '\0' for null. + N.B.: on Windows systems, Python converts '\n' to '\r\n'. +* buf_size : int, optional + String buffer size in bytes [default: 256] + used when ``delim`` is specified. +* bytes : bool, optional + If true, will count bytes, ignore ``delim``, and default + ``unit_scale`` to True, ``unit_divisor`` to 1024, and ``unit`` to 'B'. +* tee : bool, optional + If true, passes ``stdin`` to both ``stderr`` and ``stdout``. +* update : bool, optional + If true, will treat input as newly elapsed iterations, + i.e. numbers to pass to ``update()``. Note that this is slow + (~2e5 it/s) since every input must be decoded as a number. +* update_to : bool, optional + If true, will treat input as total elapsed iterations, + i.e. numbers to assign to ``self.n``. Note that this is slow + (~2e5 it/s) since every input must be decoded as a number. +* null : bool, optional + If true, will discard input (no stdout). +* manpath : str, optional + Directory in which to install tqdm man pages. +* comppath : str, optional + Directory in which to place tqdm completion. +* log : str, optional + CRITICAL|FATAL|ERROR|WARN(ING)|[default: 'INFO']|DEBUG|NOTSET. + +Returns +~~~~~~~ + +* out : decorated iterator. + +.. code:: python + + class tqdm(): + def update(self, n=1): + """ + Manually update the progress bar, useful for streams + such as reading files. + E.g.: + >>> t = tqdm(total=filesize) # Initialise + >>> for current_buffer in stream: + ... ... + ... t.update(len(current_buffer)) + >>> t.close() + The last line is highly recommended, but possibly not necessary if + ``t.update()`` will be called in such a way that ``filesize`` will be + exactly reached and printed. + + Parameters + ---------- + n : int or float, optional + Increment to add to the internal counter of iterations + [default: 1]. If using float, consider specifying ``{n:.3f}`` + or similar in ``bar_format``, or specifying ``unit_scale``. + + Returns + ------- + out : bool or None + True if a ``display()`` was triggered. + """ + + def close(self): + """Cleanup and (if leave=False) close the progressbar.""" + + def clear(self, nomove=False): + """Clear current bar display.""" + + def refresh(self): + """ + Force refresh the display of this bar. + + Parameters + ---------- + nolock : bool, optional + If ``True``, does not lock. + If [default: ``False``]: calls ``acquire()`` on internal lock. + lock_args : tuple, optional + Passed to internal lock's ``acquire()``. + If specified, will only ``display()`` if ``acquire()`` returns ``True``. + """ + + def unpause(self): + """Restart tqdm timer from last print time.""" + + def reset(self, total=None): + """ + Resets to 0 iterations for repeated use. + + Consider combining with ``leave=True``. + + Parameters + ---------- + total : int or float, optional. Total to use for the new bar. + """ + + def set_description(self, desc=None, refresh=True): + """ + Set/modify description of the progress bar. + + Parameters + ---------- + desc : str, optional + refresh : bool, optional + Forces refresh [default: True]. + """ + + def set_postfix(self, ordered_dict=None, refresh=True, **tqdm_kwargs): + """ + Set/modify postfix (additional stats) + with automatic formatting based on datatype. + + Parameters + ---------- + ordered_dict : dict or OrderedDict, optional + refresh : bool, optional + Forces refresh [default: True]. + kwargs : dict, optional + """ + + @classmethod + def write(cls, s, file=sys.stdout, end="\n"): + """Print a message via tqdm (without overlap with bars).""" + + @property + def format_dict(self): + """Public API for read-only member access.""" + + def display(self, msg=None, pos=None): + """ + Use ``self.sp`` to display ``msg`` in the specified ``pos``. + + Consider overloading this function when inheriting to use e.g.: + ``self.some_frontend(**self.format_dict)`` instead of ``self.sp``. + + Parameters + ---------- + msg : str, optional. What to display (default: ``repr(self)``). + pos : int, optional. Position to ``moveto`` + (default: ``abs(self.pos)``). + """ + + @classmethod + @contextmanager + def wrapattr(cls, stream, method, total=None, bytes=True, **tqdm_kwargs): + """ + stream : file-like object. + method : str, "read" or "write". The result of ``read()`` and + the first argument of ``write()`` should have a ``len()``. + + >>> with tqdm.wrapattr(file_obj, "read", total=file_obj.size) as fobj: + ... while True: + ... chunk = fobj.read(chunk_size) + ... if not chunk: + ... break + """ + + @classmethod + def pandas(cls, *targs, **tqdm_kwargs): + """Registers the current `tqdm` class with `pandas`.""" + + def trange(*args, **tqdm_kwargs): + """ + A shortcut for `tqdm(xrange(*args), **tqdm_kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + +Convenience Functions +~~~~~~~~~~~~~~~~~~~~~ + +.. code:: python + + def tqdm.contrib.tenumerate(iterable, start=0, total=None, + tqdm_class=tqdm.auto.tqdm, **tqdm_kwargs): + """Equivalent of `numpy.ndenumerate` or builtin `enumerate`.""" + + def tqdm.contrib.tzip(iter1, *iter2plus, **tqdm_kwargs): + """Equivalent of builtin `zip`.""" + + def tqdm.contrib.tmap(function, *sequences, **tqdm_kwargs): + """Equivalent of builtin `map`.""" + +Submodules +~~~~~~~~~~ + +.. code:: python + + class tqdm.notebook.tqdm(tqdm.tqdm): + """IPython/Jupyter Notebook widget.""" + + class tqdm.auto.tqdm(tqdm.tqdm): + """Automatically chooses beween `tqdm.notebook` and `tqdm.tqdm`.""" + + class tqdm.asyncio.tqdm(tqdm.tqdm): + """Asynchronous version.""" + @classmethod + def as_completed(cls, fs, *, loop=None, timeout=None, total=None, + **tqdm_kwargs): + """Wrapper for `asyncio.as_completed`.""" + + class tqdm.gui.tqdm(tqdm.tqdm): + """Matplotlib GUI version.""" + + class tqdm.tk.tqdm(tqdm.tqdm): + """Tkinter GUI version.""" + + class tqdm.rich.tqdm(tqdm.tqdm): + """`rich.progress` version.""" + + class tqdm.keras.TqdmCallback(keras.callbacks.Callback): + """Keras callback for epoch and batch progress.""" + + class tqdm.dask.TqdmCallback(dask.callbacks.Callback): + """Dask callback for task progress.""" + + +``contrib`` ++++++++++++ + +The ``tqdm.contrib`` package also contains experimental modules: + +- ``tqdm.contrib.itertools``: Thin wrappers around ``itertools`` +- ``tqdm.contrib.concurrent``: Thin wrappers around ``concurrent.futures`` +- ``tqdm.contrib.slack``: Posts to `Slack <https://slack.com>`__ bots +- ``tqdm.contrib.discord``: Posts to `Discord <https://discord.com>`__ bots +- ``tqdm.contrib.telegram``: Posts to `Telegram <https://telegram.org>`__ bots +- ``tqdm.contrib.bells``: Automagically enables all optional features + + * ``auto``, ``pandas``, ``slack``, ``discord``, ``telegram`` + +Examples and Advanced Usage +--------------------------- + +- See the `examples <https://github.com/tqdm/tqdm/tree/master/examples>`__ + folder; +- import the module and run ``help()``; +- consult the `wiki <https://github.com/tqdm/tqdm/wiki>`__; + + * this has an + `excellent article <https://github.com/tqdm/tqdm/wiki/How-to-make-a-great-Progress-Bar>`__ + on how to make a **great** progressbar; + +- check out the `slides from PyData London <https://tqdm.github.io/PyData2019/slides.html>`__, or +- run the |binder-demo|. + +Description and additional stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Custom information can be displayed and updated dynamically on ``tqdm`` bars +with the ``desc`` and ``postfix`` arguments: + +.. code:: python + + from tqdm import tqdm, trange + from random import random, randint + from time import sleep + + with trange(10) as t: + for i in t: + # Description will be displayed on the left + t.set_description('GEN %i' % i) + # Postfix will be displayed on the right, + # formatted automatically based on argument's datatype + t.set_postfix(loss=random(), gen=randint(1,999), str='h', + lst=[1, 2]) + sleep(0.1) + + with tqdm(total=10, bar_format="{postfix[0]} {postfix[1][value]:>8.2g}", + postfix=["Batch", dict(value=0)]) as t: + for i in range(10): + sleep(0.1) + t.postfix[1]["value"] = i / 2 + t.update() + +Points to remember when using ``{postfix[...]}`` in the ``bar_format`` string: + +- ``postfix`` also needs to be passed as an initial argument in a compatible + format, and +- ``postfix`` will be auto-converted to a string if it is a ``dict``-like + object. To prevent this behaviour, insert an extra item into the dictionary + where the key is not a string. + +Additional ``bar_format`` parameters may also be defined by overriding +``format_dict``, and the bar itself may be modified using ``ascii``: + +.. code:: python + + from tqdm import tqdm + class TqdmExtraFormat(tqdm): + """Provides a `total_time` format parameter""" + @property + def format_dict(self): + d = super(TqdmExtraFormat, self).format_dict + total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1) + d.update(total_time=self.format_interval(total_time) + " in total") + return d + + for i in TqdmExtraFormat( + range(9), ascii=" .oO0", + bar_format="{total_time}: {percentage:.0f}%|{bar}{r_bar}"): + if i == 4: + break + +.. code:: + + 00:00 in total: 44%|0000. | 4/9 [00:00<00:00, 962.93it/s] + +Note that ``{bar}`` also supports a format specifier ``[width][type]``. + +- ``width`` + + * unspecified (default): automatic to fill ``ncols`` + * ``int >= 0``: fixed width overriding ``ncols`` logic + * ``int < 0``: subtract from the automatic default + +- ``type`` + + * ``a``: ascii (``ascii=True`` override) + * ``u``: unicode (``ascii=False`` override) + * ``b``: blank (``ascii=" "`` override) + +This means a fixed bar with right-justified text may be created by using: +``bar_format="{l_bar}{bar:10}|{bar:-10b}right-justified"`` + +Nested progress bars +~~~~~~~~~~~~~~~~~~~~ + +``tqdm`` supports nested progress bars. Here's an example: + +.. code:: python + + from tqdm.auto import trange + from time import sleep + + for i in trange(4, desc='1st loop'): + for j in trange(5, desc='2nd loop'): + for k in trange(50, desc='3rd loop', leave=False): + sleep(0.01) + +For manual control over positioning (e.g. for multi-processing use), +you may specify ``position=n`` where ``n=0`` for the outermost bar, +``n=1`` for the next, and so on. +However, it's best to check if ``tqdm`` can work without manual ``position`` +first. + +.. code:: python + + from time import sleep + from tqdm import trange, tqdm + from multiprocessing import Pool, RLock, freeze_support + + L = list(range(9)) + + def progresser(n): + interval = 0.001 / (n + 2) + total = 5000 + text = "#{}, est. {:<04.2}s".format(n, interval * total) + for _ in trange(total, desc=text, position=n): + sleep(interval) + + if __name__ == '__main__': + freeze_support() # for Windows support + tqdm.set_lock(RLock()) # for managing output contention + p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) + p.map(progresser, L) + +Note that in Python 3, ``tqdm.write`` is thread-safe: + +.. code:: python + + from time import sleep + from tqdm import tqdm, trange + from concurrent.futures import ThreadPoolExecutor + + L = list(range(9)) + + def progresser(n): + interval = 0.001 / (n + 2) + total = 5000 + text = "#{}, est. {:<04.2}s".format(n, interval * total) + for _ in trange(total, desc=text): + sleep(interval) + if n == 6: + tqdm.write("n == 6 completed.") + tqdm.write("`tqdm.write()` is thread-safe in py3!") + + if __name__ == '__main__': + with ThreadPoolExecutor() as p: + p.map(progresser, L) + +Hooks and callbacks +~~~~~~~~~~~~~~~~~~~ + +``tqdm`` can easily support callbacks/hooks and manual updates. +Here's an example with ``urllib``: + +**``urllib.urlretrieve`` documentation** + + | [...] + | If present, the hook function will be called once + | on establishment of the network connection and once after each block read + | thereafter. The hook will be passed three arguments; a count of blocks + | transferred so far, a block size in bytes, and the total size of the file. + | [...] + +.. code:: python + + import urllib, os + from tqdm import tqdm + urllib = getattr(urllib, 'request', urllib) + + class TqdmUpTo(tqdm): + """Provides `update_to(n)` which uses `tqdm.update(delta_n)`.""" + def update_to(self, b=1, bsize=1, tsize=None): + """ + b : int, optional + Number of blocks transferred so far [default: 1]. + bsize : int, optional + Size of each block (in tqdm units) [default: 1]. + tsize : int, optional + Total size (in tqdm units). If [default: None] remains unchanged. + """ + if tsize is not None: + self.total = tsize + return self.update(b * bsize - self.n) # also sets self.n = b * bsize + + eg_link = "https://caspersci.uk.to/matryoshka.zip" + with TqdmUpTo(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, + desc=eg_link.split('/')[-1]) as t: # all optional kwargs + urllib.urlretrieve(eg_link, filename=os.devnull, + reporthook=t.update_to, data=None) + t.total = t.n + +Inspired by `twine#242 <https://github.com/pypa/twine/pull/242>`__. +Functional alternative in +`examples/tqdm_wget.py <https://github.com/tqdm/tqdm/blob/master/examples/tqdm_wget.py>`__. + +It is recommend to use ``miniters=1`` whenever there is potentially +large differences in iteration speed (e.g. downloading a file over +a patchy connection). + +**Wrapping read/write methods** + +To measure throughput through a file-like object's ``read`` or ``write`` +methods, use ``CallbackIOWrapper``: + +.. code:: python + + from tqdm.auto import tqdm + from tqdm.utils import CallbackIOWrapper + + with tqdm(total=file_obj.size, + unit='B', unit_scale=True, unit_divisor=1024) as t: + fobj = CallbackIOWrapper(t.update, file_obj, "read") + while True: + chunk = fobj.read(chunk_size) + if not chunk: + break + t.reset() + # ... continue to use `t` for something else + +Alternatively, use the even simpler ``wrapattr`` convenience function, +which would condense both the ``urllib`` and ``CallbackIOWrapper`` examples +down to: + +.. code:: python + + import urllib, os + from tqdm import tqdm + + eg_link = "https://caspersci.uk.to/matryoshka.zip" + response = getattr(urllib, 'request', urllib).urlopen(eg_link) + with tqdm.wrapattr(open(os.devnull, "wb"), "write", + miniters=1, desc=eg_link.split('/')[-1], + total=getattr(response, 'length', None)) as fout: + for chunk in response: + fout.write(chunk) + +The ``requests`` equivalent is nearly identical: + +.. code:: python + + import requests, os + from tqdm import tqdm + + eg_link = "https://caspersci.uk.to/matryoshka.zip" + response = requests.get(eg_link, stream=True) + with tqdm.wrapattr(open(os.devnull, "wb"), "write", + miniters=1, desc=eg_link.split('/')[-1], + total=int(response.headers.get('content-length', 0))) as fout: + for chunk in response.iter_content(chunk_size=4096): + fout.write(chunk) + +**Custom callback** + +``tqdm`` is known for intelligently skipping unnecessary displays. To make a +custom callback take advantage of this, simply use the return value of +``update()``. This is set to ``True`` if a ``display()`` was triggered. + +.. code:: python + + from tqdm.auto import tqdm as std_tqdm + + def external_callback(*args, **kwargs): + ... + + class TqdmExt(std_tqdm): + def update(self, n=1): + displayed = super(TqdmExt, self).update(n) + if displayed: + external_callback(**self.format_dict) + return displayed + +``asyncio`` +~~~~~~~~~~~ + +Note that ``break`` isn't currently caught by asynchronous iterators. +This means that ``tqdm`` cannot clean up after itself in this case: + +.. code:: python + + from tqdm.asyncio import tqdm + + async for i in tqdm(range(9)): + if i == 2: + break + +Instead, either call ``pbar.close()`` manually or use the context manager syntax: + +.. code:: python + + from tqdm.asyncio import tqdm + + with tqdm(range(9)) as pbar: + async for i in pbar: + if i == 2: + break + +Pandas Integration +~~~~~~~~~~~~~~~~~~ + +Due to popular demand we've added support for ``pandas`` -- here's an example +for ``DataFrame.progress_apply`` and ``DataFrameGroupBy.progress_apply``: + +.. code:: python + + import pandas as pd + import numpy as np + from tqdm import tqdm + + df = pd.DataFrame(np.random.randint(0, 100, (100000, 6))) + + # Register `pandas.progress_apply` and `pandas.Series.map_apply` with `tqdm` + # (can use `tqdm.gui.tqdm`, `tqdm.notebook.tqdm`, optional kwargs, etc.) + tqdm.pandas(desc="my bar!") + + # Now you can use `progress_apply` instead of `apply` + # and `progress_map` instead of `map` + df.progress_apply(lambda x: x**2) + # can also groupby: + # df.groupby(0).progress_apply(lambda x: x**2) + +In case you're interested in how this works (and how to modify it for your +own callbacks), see the +`examples <https://github.com/tqdm/tqdm/tree/master/examples>`__ +folder or import the module and run ``help()``. + +Keras Integration +~~~~~~~~~~~~~~~~~ + +A ``keras`` callback is also available: + +.. code:: python + + from tqdm.keras import TqdmCallback + + ... + + model.fit(..., verbose=0, callbacks=[TqdmCallback()]) + +Dask Integration +~~~~~~~~~~~~~~~~ + +A ``dask`` callback is also available: + +.. code:: python + + from tqdm.dask import TqdmCallback + + with TqdmCallback(desc="compute"): + ... + arr.compute() + + # or use callback globally + cb = TqdmCallback(desc="global") + cb.register() + arr.compute() + +IPython/Jupyter Integration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +IPython/Jupyter is supported via the ``tqdm.notebook`` submodule: + +.. code:: python + + from tqdm.notebook import trange, tqdm + from time import sleep + + for i in trange(3, desc='1st loop'): + for j in tqdm(range(100), desc='2nd loop'): + sleep(0.01) + +In addition to ``tqdm`` features, the submodule provides a native Jupyter +widget (compatible with IPython v1-v4 and Jupyter), fully working nested bars +and colour hints (blue: normal, green: completed, red: error/interrupt, +light blue: no ETA); as demonstrated below. + +|Screenshot-Jupyter1| +|Screenshot-Jupyter2| +|Screenshot-Jupyter3| + +The ``notebook`` version supports percentage or pixels for overall width +(e.g.: ``ncols='100%'`` or ``ncols='480px'``). + +It is also possible to let ``tqdm`` automatically choose between +console or notebook versions by using the ``autonotebook`` submodule: + +.. code:: python + + from tqdm.autonotebook import tqdm + tqdm.pandas() + +Note that this will issue a ``TqdmExperimentalWarning`` if run in a notebook +since it is not meant to be possible to distinguish between ``jupyter notebook`` +and ``jupyter console``. Use ``auto`` instead of ``autonotebook`` to suppress +this warning. + +Note that notebooks will display the bar in the cell where it was created. +This may be a different cell from the one where it is used. +If this is not desired, either + +- delay the creation of the bar to the cell where it must be displayed, or +- create the bar with ``display=False``, and in a later cell call + ``display(bar.container)``: + +.. code:: python + + from tqdm.notebook import tqdm + pbar = tqdm(..., display=False) + +.. code:: python + + # different cell + display(pbar.container) + +The ``keras`` callback has a ``display()`` method which can be used likewise: + +.. code:: python + + from tqdm.keras import TqdmCallback + cbk = TqdmCallback(display=False) + +.. code:: python + + # different cell + cbk.display() + model.fit(..., verbose=0, callbacks=[cbk]) + +Another possibility is to have a single bar (near the top of the notebook) +which is constantly re-used (using ``reset()`` rather than ``close()``). +For this reason, the notebook version (unlike the CLI version) does not +automatically call ``close()`` upon ``Exception``. + +.. code:: python + + from tqdm.notebook import tqdm + pbar = tqdm() + +.. code:: python + + # different cell + iterable = range(100) + pbar.reset(total=len(iterable)) # initialise with new `total` + for i in iterable: + pbar.update() + pbar.refresh() # force print final status but don't `close()` + +Custom Integration +~~~~~~~~~~~~~~~~~~ + +To change the default arguments (such as making ``dynamic_ncols=True``), +simply use built-in Python magic: + +.. code:: python + + from functools import partial + from tqdm import tqdm as std_tqdm + tqdm = partial(std_tqdm, dynamic_ncols=True) + +For further customisation, +``tqdm`` may be inherited from to create custom callbacks (as with the +``TqdmUpTo`` example `above <#hooks-and-callbacks>`__) or for custom frontends +(e.g. GUIs such as notebook or plotting packages). In the latter case: + +1. ``def __init__()`` to call ``super().__init__(..., gui=True)`` to disable + terminal ``status_printer`` creation. +2. Redefine: ``close()``, ``clear()``, ``display()``. + +Consider overloading ``display()`` to use e.g. +``self.frontend(**self.format_dict)`` instead of ``self.sp(repr(self))``. + +Some submodule examples of inheritance: + +- `tqdm/notebook.py <https://github.com/tqdm/tqdm/blob/master/tqdm/notebook.py>`__ +- `tqdm/gui.py <https://github.com/tqdm/tqdm/blob/master/tqdm/gui.py>`__ +- `tqdm/tk.py <https://github.com/tqdm/tqdm/blob/master/tqdm/tk.py>`__ +- `tqdm/contrib/slack.py <https://github.com/tqdm/tqdm/blob/master/tqdm/contrib/slack.py>`__ +- `tqdm/contrib/discord.py <https://github.com/tqdm/tqdm/blob/master/tqdm/contrib/discord.py>`__ +- `tqdm/contrib/telegram.py <https://github.com/tqdm/tqdm/blob/master/tqdm/contrib/telegram.py>`__ + +Dynamic Monitor/Meter +~~~~~~~~~~~~~~~~~~~~~ + +You can use a ``tqdm`` as a meter which is not monotonically increasing. +This could be because ``n`` decreases (e.g. a CPU usage monitor) or ``total`` +changes. + +One example would be recursively searching for files. The ``total`` is the +number of objects found so far, while ``n`` is the number of those objects which +are files (rather than folders): + +.. code:: python + + from tqdm import tqdm + import os.path + + def find_files_recursively(path, show_progress=True): + files = [] + # total=1 assumes `path` is a file + t = tqdm(total=1, unit="file", disable=not show_progress) + if not os.path.exists(path): + raise IOError("Cannot find:" + path) + + def append_found_file(f): + files.append(f) + t.update() + + def list_found_dir(path): + """returns os.listdir(path) assuming os.path.isdir(path)""" + listing = os.listdir(path) + # subtract 1 since a "file" we found was actually this directory + t.total += len(listing) - 1 + # fancy way to give info without forcing a refresh + t.set_postfix(dir=path[-10:], refresh=False) + t.update(0) # may trigger a refresh + return listing + + def recursively_search(path): + if os.path.isdir(path): + for f in list_found_dir(path): + recursively_search(os.path.join(path, f)) + else: + append_found_file(path) + + recursively_search(path) + t.set_postfix(dir=path) + t.close() + return files + +Using ``update(0)`` is a handy way to let ``tqdm`` decide when to trigger a +display refresh to avoid console spamming. + +Writing messages +~~~~~~~~~~~~~~~~ + +This is a work in progress (see +`#737 <https://github.com/tqdm/tqdm/issues/737>`__). + +Since ``tqdm`` uses a simple printing mechanism to display progress bars, +you should not write any message in the terminal using ``print()`` while +a progressbar is open. + +To write messages in the terminal without any collision with ``tqdm`` bar +display, a ``.write()`` method is provided: + +.. code:: python + + from tqdm.auto import tqdm, trange + from time import sleep + + bar = trange(10) + for i in bar: + # Print using tqdm class method .write() + sleep(0.1) + if not (i % 3): + tqdm.write("Done task %i" % i) + # Can also use bar.write() + +By default, this will print to standard output ``sys.stdout``. but you can +specify any file-like object using the ``file`` argument. For example, this +can be used to redirect the messages writing to a log file or class. + +Redirecting writing +~~~~~~~~~~~~~~~~~~~ + +If using a library that can print messages to the console, editing the library +by replacing ``print()`` with ``tqdm.write()`` may not be desirable. +In that case, redirecting ``sys.stdout`` to ``tqdm.write()`` is an option. + +To redirect ``sys.stdout``, create a file-like class that will write +any input string to ``tqdm.write()``, and supply the arguments +``file=sys.stdout, dynamic_ncols=True``. + +A reusable canonical example is given below: + +.. code:: python + + from time import sleep + import contextlib + import sys + from tqdm import tqdm + from tqdm.contrib import DummyTqdmFile + + + @contextlib.contextmanager + def std_out_err_redirect_tqdm(): + orig_out_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = map(DummyTqdmFile, orig_out_err) + yield orig_out_err[0] + # Relay exceptions + except Exception as exc: + raise exc + # Always restore sys.stdout/err if necessary + finally: + sys.stdout, sys.stderr = orig_out_err + + def some_fun(i): + print("Fee, fi, fo,".split()[i]) + + # Redirect stdout to tqdm.write() (don't forget the `as save_stdout`) + with std_out_err_redirect_tqdm() as orig_stdout: + # tqdm needs the original stdout + # and dynamic_ncols=True to autodetect console width + for i in tqdm(range(3), file=orig_stdout, dynamic_ncols=True): + sleep(.5) + some_fun(i) + + # After the `with`, printing is restored + print("Done!") + +Redirecting ``logging`` +~~~~~~~~~~~~~~~~~~~~~~~ + +Similar to ``sys.stdout``/``sys.stderr`` as detailed above, console ``logging`` +may also be redirected to ``tqdm.write()``. + +Warning: if also redirecting ``sys.stdout``/``sys.stderr``, make sure to +redirect ``logging`` first if needed. + +Helper methods are available in ``tqdm.contrib.logging``. For example: + +.. code:: python + + import logging + from tqdm import trange + from tqdm.contrib.logging import logging_redirect_tqdm + + LOG = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + with logging_redirect_tqdm(): + for i in trange(9): + if i == 4: + LOG.info("console logging redirected to `tqdm.write()`") + # logging restored + +Monitoring thread, intervals and miniters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``tqdm`` implements a few tricks to increase efficiency and reduce overhead. + +- Avoid unnecessary frequent bar refreshing: ``mininterval`` defines how long + to wait between each refresh. ``tqdm`` always gets updated in the background, + but it will display only every ``mininterval``. +- Reduce number of calls to check system clock/time. +- ``mininterval`` is more intuitive to configure than ``miniters``. + A clever adjustment system ``dynamic_miniters`` will automatically adjust + ``miniters`` to the amount of iterations that fit into time ``mininterval``. + Essentially, ``tqdm`` will check if it's time to print without actually + checking time. This behaviour can be still be bypassed by manually setting + ``miniters``. + +However, consider a case with a combination of fast and slow iterations. +After a few fast iterations, ``dynamic_miniters`` will set ``miniters`` to a +large number. When iteration rate subsequently slows, ``miniters`` will +remain large and thus reduce display update frequency. To address this: + +- ``maxinterval`` defines the maximum time between display refreshes. + A concurrent monitoring thread checks for overdue updates and forces one + where necessary. + +The monitoring thread should not have a noticeable overhead, and guarantees +updates at least every 10 seconds by default. +This value can be directly changed by setting the ``monitor_interval`` of +any ``tqdm`` instance (i.e. ``t = tqdm.tqdm(...); t.monitor_interval = 2``). +The monitor thread may be disabled application-wide by setting +``tqdm.tqdm.monitor_interval = 0`` before instantiation of any ``tqdm`` bar. + + +Merch +----- + +You can buy `tqdm branded merch <https://tqdm.github.io/merch>`__ now! + +Contributions +------------- + +|GitHub-Commits| |GitHub-Issues| |GitHub-PRs| |OpenHub-Status| |GitHub-Contributions| |CII Best Practices| + +All source code is hosted on `GitHub <https://github.com/tqdm/tqdm>`__. +Contributions are welcome. + +See the +`CONTRIBUTING <https://github.com/tqdm/tqdm/blob/master/CONTRIBUTING.md>`__ +file for more information. + +Developers who have made significant contributions, ranked by *SLoC* +(surviving lines of code, +`git fame <https://github.com/casperdcl/git-fame>`__ ``-wMC --excl '\.(png|gif|jpg)$'``), +are: + +==================== ======================================================== ==== ================================ +Name ID SLoC Notes +==================== ======================================================== ==== ================================ +Casper da Costa-Luis `casperdcl <https://github.com/casperdcl>`__ ~78% primary maintainer |Gift-Casper| +Stephen Larroque `lrq3000 <https://github.com/lrq3000>`__ ~10% team member +Martin Zugnoni `martinzugnoni <https://github.com/martinzugnoni>`__ ~4% +Daniel Ecer `de-code <https://github.com/de-code>`__ ~2% +Richard Sheridan `richardsheridan <https://github.com/richardsheridan>`__ ~1% +Guangshuo Chen `chengs <https://github.com/chengs>`__ ~1% +Kyle Altendorf `altendky <https://github.com/altendky>`__ <1% +Matthew Stevens `mjstevens777 <https://github.com/mjstevens777>`__ <1% +Hadrien Mary `hadim <https://github.com/hadim>`__ <1% team member +Noam Yorav-Raphael `noamraph <https://github.com/noamraph>`__ <1% original author +Mikhail Korobov `kmike <https://github.com/kmike>`__ <1% team member +==================== ======================================================== ==== ================================ + +Ports to Other Languages +~~~~~~~~~~~~~~~~~~~~~~~~ + +A list is available on +`this wiki page <https://github.com/tqdm/tqdm/wiki/tqdm-ports>`__. + + +LICENCE +------- + +Open Source (OSI approved): |LICENCE| + +Citation information: |DOI| + +|README-Hits| (Since 19 May 2016) + +.. |Logo| image:: https://img.tqdm.ml/logo.gif +.. |Screenshot| image:: https://img.tqdm.ml/tqdm.gif +.. |Video| image:: https://img.tqdm.ml/video.jpg + :target: https://tqdm.github.io/video +.. |Slides| image:: https://img.tqdm.ml/slides.jpg + :target: https://tqdm.github.io/PyData2019/slides.html +.. |Merch| image:: https://img.tqdm.ml/merch.jpg + :target: https://tqdm.github.io/merch +.. |Build-Status| image:: https://img.shields.io/github/workflow/status/tqdm/tqdm/Test/master?logo=GitHub + :target: https://github.com/tqdm/tqdm/actions?query=workflow%3ATest +.. |Coverage-Status| image:: https://img.shields.io/coveralls/github/tqdm/tqdm/master?logo=coveralls + :target: https://coveralls.io/github/tqdm/tqdm +.. |Branch-Coverage-Status| image:: https://codecov.io/gh/tqdm/tqdm/branch/master/graph/badge.svg + :target: https://codecov.io/gh/tqdm/tqdm +.. |Codacy-Grade| image:: https://app.codacy.com/project/badge/Grade/3f965571598f44549c7818f29cdcf177 + :target: https://www.codacy.com/gh/tqdm/tqdm/dashboard +.. |CII Best Practices| image:: https://bestpractices.coreinfrastructure.org/projects/3264/badge + :target: https://bestpractices.coreinfrastructure.org/projects/3264 +.. |GitHub-Status| image:: https://img.shields.io/github/tag/tqdm/tqdm.svg?maxAge=86400&logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/releases +.. |GitHub-Forks| image:: https://img.shields.io/github/forks/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/network +.. |GitHub-Stars| image:: https://img.shields.io/github/stars/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/stargazers +.. |GitHub-Commits| image:: https://img.shields.io/github/commit-activity/y/tqdm/tqdm.svg?logo=git&logoColor=white + :target: https://github.com/tqdm/tqdm/graphs/commit-activity +.. |GitHub-Issues| image:: https://img.shields.io/github/issues-closed/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/issues?q= +.. |GitHub-PRs| image:: https://img.shields.io/github/issues-pr-closed/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/pulls +.. |GitHub-Contributions| image:: https://img.shields.io/github/contributors/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/graphs/contributors +.. |GitHub-Updated| image:: https://img.shields.io/github/last-commit/tqdm/tqdm/master.svg?logo=github&logoColor=white&label=pushed + :target: https://github.com/tqdm/tqdm/pulse +.. |Gift-Casper| image:: https://img.shields.io/badge/dynamic/json.svg?color=ff69b4&label=gifts%20received&prefix=%C2%A3&query=%24..sum&url=https%3A%2F%2Fcaspersci.uk.to%2Fgifts.json + :target: https://cdcl.ml/sponsor +.. |Versions| image:: https://img.shields.io/pypi/v/tqdm.svg + :target: https://tqdm.github.io/releases +.. |PyPI-Downloads| image:: https://img.shields.io/pypi/dm/tqdm.svg?label=pypi%20downloads&logo=PyPI&logoColor=white + :target: https://pepy.tech/project/tqdm +.. |Py-Versions| image:: https://img.shields.io/pypi/pyversions/tqdm.svg?logo=python&logoColor=white + :target: https://pypi.org/project/tqdm +.. |Conda-Forge-Status| image:: https://img.shields.io/conda/v/conda-forge/tqdm.svg?label=conda-forge&logo=conda-forge + :target: https://anaconda.org/conda-forge/tqdm +.. |Snapcraft| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft + :target: https://snapcraft.io/tqdm +.. |Docker| image:: https://img.shields.io/badge/docker-pull-blue.svg?logo=docker&logoColor=white + :target: https://hub.docker.com/r/tqdm/tqdm +.. |Libraries-Rank| image:: https://img.shields.io/librariesio/sourcerank/pypi/tqdm.svg?logo=koding&logoColor=white + :target: https://libraries.io/pypi/tqdm +.. |Libraries-Dependents| image:: https://img.shields.io/librariesio/dependent-repos/pypi/tqdm.svg?logo=koding&logoColor=white + :target: https://github.com/tqdm/tqdm/network/dependents +.. |OpenHub-Status| image:: https://www.openhub.net/p/tqdm/widgets/project_thin_badge?format=gif + :target: https://www.openhub.net/p/tqdm?ref=Thin+badge +.. |awesome-python| image:: https://awesome.re/mentioned-badge.svg + :target: https://github.com/vinta/awesome-python +.. |LICENCE| image:: https://img.shields.io/pypi/l/tqdm.svg + :target: https://raw.githubusercontent.com/tqdm/tqdm/master/LICENCE +.. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.595120-blue.svg + :target: https://doi.org/10.5281/zenodo.595120 +.. |binder-demo| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/tqdm/tqdm/master?filepath=DEMO.ipynb +.. |Screenshot-Jupyter1| image:: https://img.tqdm.ml/jupyter-1.gif +.. |Screenshot-Jupyter2| image:: https://img.tqdm.ml/jupyter-2.gif +.. |Screenshot-Jupyter3| image:: https://img.tqdm.ml/jupyter-3.gif +.. |README-Hits| image:: https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&style=social&r=https://github.com/tqdm/tqdm&l=https://img.tqdm.ml/favicon.png&f=https://img.tqdm.ml/logo.gif + :target: https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&a=plot&r=https://github.com/tqdm/tqdm&l=https://img.tqdm.ml/favicon.png&f=https://img.tqdm.ml/logo.gif&style=social diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..871e3e8 --- /dev/null +++ b/environment.yml @@ -0,0 +1,46 @@ +# development environment +name: tqdm +channels: +- conda-forge +- defaults +dependencies: +# base +- python=3 +- pip +- ipykernel +- ipywidgets +- setuptools +- setuptools_scm +- toml +# test env managers +- pre-commit +- tox +- asv +# tests (native) +- pytest +- pytest-cov +- pytest-timeout +- pytest-asyncio # [py>=3.7] +- nbval +- coverage +# extras +- dask # dask +- matplotlib # gui +- numpy # pandas, keras, contrib.tenumerate +- pandas +- tensorflow # keras +- slack-sdk # contrib.slack +- requests # contrib.telegram +- rich # rich +- argopt # `cd wiki && pymake` +- twine # `pymake pypi` +- wheel # `setup.py bdist_wheel` +# `cd docs && pymake` +- mkdocs-material +- pydoc-markdown +- pygments +- pymdown-extensions +- pip: + - py-make >=0.1.0 # `setup.py make/pymake` + - mkdocs-minify-plugin # `cd docs && pymake` + - git+https://github.com/tqdm/jsmin@python3-only#egg=jsmin # `cd docs && pymake` diff --git a/examples/7zx.py b/examples/7zx.py new file mode 100644 index 0000000..3d15254 --- /dev/null +++ b/examples/7zx.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +"""Usage: + 7zx.py [--help | options] <zipfiles>... + +Options: + -h, --help Print this help and exit + -v, --version Print version and exit + -c, --compressed Use compressed (instead of uncompressed) file sizes + -s, --silent Do not print one row per zip file + -y, --yes Assume yes to all queries (for extraction) + -D=<level>, --debug=<level> + Print various types of debugging information. Choices: + CRITICAL|FATAL + ERROR + WARN(ING) + [default: INFO] + DEBUG + NOTSET + -d, --debug-trace Print lots of debugging information (-D NOTSET) +""" +from __future__ import print_function + +import io +import logging +import os +import pty +import re +import subprocess # nosec + +from argopt import argopt + +from tqdm import tqdm + +__author__ = "Casper da Costa-Luis <casper.dcl@physics.org>" +__licence__ = "MPLv2.0" +__version__ = "0.2.2" +__license__ = __licence__ + +RE_SCN = re.compile(r"([0-9]+)\s+([0-9]+)\s+(.*)$", flags=re.M) + + +def main(): + args = argopt(__doc__, version=__version__).parse_args() + if args.debug_trace: + args.debug = "NOTSET" + logging.basicConfig(level=getattr(logging, args.debug, logging.INFO), + format='%(levelname)s:%(message)s') + log = logging.getLogger(__name__) + log.debug(args) + + # Get compressed sizes + zips = {} + for fn in args.zipfiles: + info = subprocess.check_output(["7z", "l", fn]).strip() # nosec + finfo = RE_SCN.findall(info) # size|compressed|name + + # builtin test: last line should be total sizes + log.debug(finfo) + totals = map(int, finfo[-1][:2]) + # log.debug(totals) + for s in range(2): # size|compressed totals + totals_s = sum(map(int, (inf[s] for inf in finfo[:-1]))) + if totals_s != totals[s]: + log.warn("%s: individual total %d != 7z total %d", + fn, totals_s, totals[s]) + fcomp = {n: int(c if args.compressed else u) for (u, c, n) in finfo[:-1]} + # log.debug(fcomp) + # zips : {'zipname' : {'filename' : int(size)}} + zips[fn] = fcomp + + # Extract + cmd7zx = ["7z", "x", "-bd"] + if args.yes: + cmd7zx += ["-y"] + log.info("Extracting from %d file(s)", len(zips)) + with tqdm(total=sum(sum(fcomp.values()) for fcomp in zips.values()), + unit="B", unit_scale=True) as tall: + for fn, fcomp in zips.items(): + md, sd = pty.openpty() + ex = subprocess.Popen( # nosec + cmd7zx + [fn], + bufsize=1, + stdout=md, # subprocess.PIPE, + stderr=subprocess.STDOUT) + os.close(sd) + with io.open(md, mode="rU", buffering=1) as m: + with tqdm(total=sum(fcomp.values()), disable=len(zips) < 2, + leave=False, unit="B", unit_scale=True) as t: + if not hasattr(t, "start_t"): # disabled + t.start_t = tall._time() + while True: + try: + l_raw = m.readline() + except IOError: + break + ln = l_raw.strip() + if ln.startswith("Extracting"): + exname = ln[len("Extracting"):].lstrip() + s = fcomp.get(exname, 0) # 0 is likely folders + t.update(s) + tall.update(s) + elif ln: + if not any( + ln.startswith(i) + for i in ("7-Zip ", "p7zip Version ", + "Everything is Ok", "Folders: ", + "Files: ", "Size: ", "Compressed: ")): + if ln.startswith("Processing archive: "): + if not args.silent: + t.write(t.format_interval( + t.start_t - tall.start_t) + ' ' + + ln.replace("Processing archive: ", "")) + else: + t.write(ln) + ex.wait() + + +main.__doc__ = __doc__ + +if __name__ == "__main__": + main() diff --git a/examples/async_coroutines.py b/examples/async_coroutines.py new file mode 100644 index 0000000..40f4f24 --- /dev/null +++ b/examples/async_coroutines.py @@ -0,0 +1,38 @@ +""" +Asynchronous examples using `asyncio`, `async` and `await` on `python>=3.7`. +""" +import asyncio + +from tqdm.asyncio import tqdm, trange + + +def count(start=0, step=1): + i = start + while True: + new_start = yield i + if new_start is None: + i += step + else: + i = new_start + + +async def main(): + N = int(1e6) + async for row in tqdm(trange(N, desc="inner"), desc="outer"): + if row >= N: + break + with tqdm(count(), desc="coroutine", total=N + 2) as pbar: + async for row in pbar: + if row == N: + pbar.send(-10) + elif row < 0: + assert row == -9 + break + # should be ~1sec rather than ~50s due to async scheduling + for i in tqdm.as_completed([asyncio.sleep(0.01 * i) + for i in range(100, 0, -1)], desc="as_completed"): + await i + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/coroutine_pipe.py b/examples/coroutine_pipe.py new file mode 100644 index 0000000..deb498a --- /dev/null +++ b/examples/coroutine_pipe.py @@ -0,0 +1,69 @@ +""" +Inserting `tqdm` as a "pipe" in a chain of coroutines. +Not to be confused with `asyncio.coroutine`. +""" +from functools import wraps + +from tqdm.auto import tqdm + + +def autonext(func): + @wraps(func) + def inner(*args, **kwargs): + res = func(*args, **kwargs) + next(res) + return res + return inner + + +@autonext +def tqdm_pipe(target, **tqdm_kwargs): + """ + Coroutine chain pipe `send()`ing to `target`. + + This: + >>> r = receiver() + >>> p = producer(r) + >>> next(r) + >>> next(p) + + Becomes: + >>> r = receiver() + >>> t = tqdm.pipe(r) + >>> p = producer(t) + >>> next(r) + >>> next(p) + """ + with tqdm(**tqdm_kwargs) as pbar: + while True: + obj = (yield) + target.send(obj) + pbar.update() + + +def source(target): + for i in ["foo", "bar", "baz", "pythonista", "python", "py"]: + target.send(i) + target.close() + + +@autonext +def grep(pattern, target): + while True: + line = (yield) + if pattern in line: + target.send(line) + + +@autonext +def sink(): + while True: + line = (yield) + tqdm.write(line) + + +if __name__ == "__main__": + source( + tqdm_pipe( + grep('python', + sink()))) diff --git a/examples/include_no_requirements.py b/examples/include_no_requirements.py new file mode 100644 index 0000000..c51a85c --- /dev/null +++ b/examples/include_no_requirements.py @@ -0,0 +1,11 @@ +# How to import tqdm in any frontend without enforcing it as a dependency +try: + from tqdm.auto import tqdm +except ImportError: + + def tqdm(*args, **kwargs): + if args: + return args[0] + return kwargs.get('iterable', None) + +__all__ = ['tqdm'] diff --git a/examples/pandas_progress_apply.py b/examples/pandas_progress_apply.py new file mode 100644 index 0000000..4fc8f6b --- /dev/null +++ b/examples/pandas_progress_apply.py @@ -0,0 +1,29 @@ +import numpy as np +import pandas as pd + +from tqdm.auto import tqdm + +df = pd.DataFrame(np.random.randint(0, 100, (100000, 6))) + +# Register `pandas.progress_apply` and `pandas.Series.map_apply` with `tqdm` +# (can use `tqdm.gui.tqdm`, `tqdm.notebook.tqdm`, optional kwargs, etc.) +tqdm.pandas(desc="my bar!") + +# Now you can use `progress_apply` instead of `apply` +# and `progress_map` instead of `map` +df.progress_apply(lambda x: x**2) +# can also groupby: +# df.groupby(0).progress_apply(lambda x: x**2) + +# -- Source code for `tqdm_pandas` (really simple!) +# def tqdm_pandas(t): +# from pandas.core.frame import DataFrame +# def inner(df, func, *args, **kwargs): +# t.total = groups.size // len(groups) +# def wrapper(*args, **kwargs): +# t.update(1) +# return func(*args, **kwargs) +# result = df.apply(wrapper, *args, **kwargs) +# t.close() +# return result +# DataFrame.progress_apply = inner diff --git a/examples/paper.bib b/examples/paper.bib new file mode 100644 index 0000000..13901e5 --- /dev/null +++ b/examples/paper.bib @@ -0,0 +1,252 @@ +@phdthesis{tqdm-ar, + author="Maḥmūd Alī Ġūl", + title="Early Southern Arabian Languages and Classical Arabic Sources: A Critical Examination of Literary and Lexicographical Sources by Comparison with the Inscriptions", + school="{SOAS} University of London", + year="1963" +} +@misc{tqdm-es, + year="2009", + title="¿Lenguaje sms que significa esto?", + url="https://es.answers.yahoo.com/question/index?qid=20090405052137AAF2YBo&guccounter=1", + author="{Yahoo Answers}" +} +@misc{pypi, + year="2019", + author="{Python Package Index ({PyPI})}", + publisher="Python Software Foundation", + title="{tqdm}", + url="https://pypi.org/project/tqdm/" +} +@misc{conda, + author="Anaconda", + year="2019", + title="{tqdm} :: Anaconda Cloud", + url="https://anaconda.org/conda-forge/tqdm" +} +@misc{docker, + year="2019", + author="{Docker Inc.}", + title="{tqdm}/{tqdm} - Docker Hub", + url="https://hub.docker.com/r/tqdm/tqdm" +} +@misc{snapcraft, + year="2019", + author="Snapcraft", + title="Installing {tqdm} for Linux using the Snap Store", + url="https://snapcraft.io/tqdm" +} +@article{zenodo, + year="2019", + author="Casper O. {da Costa-Luis} and {{tqdm} developers}", + title="{tqdm} stable", + publisher="Zenodo", + doi="10.5281/zenodo.595120" +} +@misc{notebooks, + year="2019", + author="{Notebooks {AI}}", + title="{tqdm}", + url="https://notebooks.ai/demo/gh/tqdm/tqdm" +} +@misc{binder, + year="2019", + author="Binder", + title="{tqdm}", + url="https://mybinder.org/v2/gh/tqdm/tqdm/master?filepath=DEMO.ipynb" +} +@misc{stdout, + year="2019", + author="{Stack Overflow}", + title="Why is printing to stdout so slow? Can it be sped up?", + url="https://stackoverflow.com/questions/3857052/why-is-printing-to-stdout-so-slow-can-it-be-sped-up" +} +@misc{pypi-downloads, + year="2019", + author="{Python Packaging Authority ({PyPA})}", + publisher="Python Software Foundation", + title="Analyzing {PyPI} package downloads -- Python Packaging User Guide", + url="https://packaging.python.org/guides/analyzing-pypi-package-downloads/" +} +@misc{keras, + year="2019", + author="Ben", + title="Keras integration with {tqdm} progress bars", + url="https://github.com/bstriner/keras-tqdm" +} +@misc{tqdm-results, + year="2019", + author="GitHub", + title="{tqdm} Code Results", + url="https://github.com/search?q=tqdm&type=Code" +} +@misc{tqdm-dependents, + year="2019", + author="GitHub", + title="{tqdm} dependents", + url="https://github.com/tqdm/tqdm/network/dependents" +} +@misc{lib-io, + year="2019", + author="Libraries.io", + title="{tqdm} on {PyPI}", + url="https://libraries.io/pypi/tqdm" +} +@misc{sourcerank, + year="2019", + author="Libraries.io", + title="SourceRank Breakdown for {tqdm}", + url="https://libraries.io/pypi/tqdm/sourcerank" +} +@misc{sourcerank-descending, + year="2019", + author="Libraries.io", + title="Libraries - The Open Source Discovery Service", + url="https://libraries.io/search?order=desc&platforms=PyPI&sort=rank" +} +@misc{stars, + year="2019", + author="GitHub", + title="{tqdm} Stargazers", + url="https://github.com/tqdm/tqdm/stargazers" +} +@misc{stars-hist, + year="2019", + author="{timqian}", + title="Star history", + url="https://timqian.com/star-history/#tqdm/tqdm" +} +@misc{trend-hist, + year="2018", + month="June", + day="19", + author="Nihey Takizawa", + title="GitHub Trending History", + url="https://github.com/nihey/trending-history/blob/master/histories/Python.md" +} +@misc{hits, + year="2019", + title="{tqdm} hits", + url="https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&a=plot", + author="Casper O. {da Costa-Luis}" +} +@book{miller, + year="2017", + author="Preston Miller and Chapin Bryce", + title="Python Digital Forensics Cookbook: Effective Python recipes for digital investigations", + publisher="Packt Publishing Ltd", + isbn="9781783987474" +} +@book{boxel, + year="2017", + author="Dan {Van Boxel}", + title="Hands-On Deep Learning with TensorFlow", + publisher="Packt Publishing", + isbn="9781787125827" +} +@incollection{nandy, + year="2018", + author="Abhishek Nandy and Manisha Biswas", + title="Reinforcement Learning with Keras, TensorFlow, and ChainerRL", + booktitle="Reinforcement Learning : With Open AI, TensorFlow and Keras Using Python", + publisher="Apress", + isbn="9781484232859", + pages="129--153", + doi="10.1007/978-1-4842-3285-9_5" +} +@journal{stein, + year="2019", + author="Helge S. Stein and Dan Guevarra and Paul F. Newhouse and Edwin Soedarmadji and John M. Gregoire", + title="Machine learning of optical properties of materials -- predicting spectra from images and images from spectra", + journal="Chemical Science", + volume="10", + issue="1", + pages="47--55", + doi="10.1039/C8SC03077D" +} +@journal{cook, + year="2018", + author="Neil J. Cook and Aleks Scholz and Ray Jayawardhana", + title="Very Low-mass Stars and Brown Dwarfs in Upper Scorpius Using Gaia DR1: Mass Function, Disks, and Kinematics", + journal="The Astronomical Journal", + volume="154", + issue="6", + pages="256", + doi="10.3847/1538-3881/aa9751", + url="https://arxiv.org/abs/1710.11625" +} +@journal{madhikar, + year="2018", + author="Pranav Madhikar and Jan Åström and Jan Westerholm and Mikko Karttunen", + title="CellSim3D: GPU accelerated software for simulations of cellular growth and division in three dimensions", + journal="Computer Physics Communications", + volume="232", + pages="206--213", + doi="10.1016/j.cpc.2018.05.024" +} +@journal{palmer, + year="2018", + author="Geraint I. Palmer and Vincent A. Knight and Paul R. Harper and Asyl L. Hawa", + title="Ciw: An open-source discrete event simulation library", + journal="Journal of Simulation", + pages="1--15", + doi="10.1080/17477778.2018.1473909" +} +@journal{knight, + year="2016", + author="Vincent Knight and Owen Campbell and Marc Harper and Karol Langner and James Campbell and Thomas Campbell and Alex Carney and Martin Chorley and Cameron Davidson-Pilon and Kristian Glass and Nikoleta Glynatsi and Tomáš Ehrlich and Martin Jones and Georgios Koutsovoulos and Holly Tibble and Müller Jochen and Geraint Palmer and Piotr Petunov and Paul Slavin and Timothy Standen and Luis Visintini and Karl Molden", + title="An open reproducible framework for the study of the iterated prisoner's dilemma", + journal="Journal of Open Research Software", + volume="4", + doi="10.5334/jors.125", + url="https://arxiv.org/abs/1604.00896", + issn="2049-9647" +} +@article{moriwaki, + title={Mordred: a molecular descriptor calculator}, + author={Moriwaki, Hirotomo and Tian, Yu-Shi and Kawashita, Norihito and Takagi, Tatsuya}, + doi={10.1186/s13321-018-0258-y}, + number={1}, + volume={10}, + month={February}, + year={2018}, + journal={Journal of cheminformatics}, + issn={1758-2946}, + pages={4} +} +@article{jackson, + title={3D for the people: multi-camera motion capture in the field with consumer-grade cameras and open source software}, + author={Jackson, Brandon E and Evangelista, Dennis J and Ray, Dylan D and hedrick, Tyson L}, + doi={10.1242/bio.018713}, + number={9}, + volume={5}, + month={September}, + year={2016}, + journal={Biology open}, + issn={2046-6390}, + pages={1334--1342} +} +@misc{travis, + year="2019", + author="{Travis {CI}}", + title="tqdm/tqdm build status", + url="https://travis-ci.org/tqdm/tqdm" +} +@misc{code-review, + year="2018", + author="Wikipedia", + title="List of tools for code review", + url="https://en.wikipedia.org/wiki/List_of_tools_for_code_review" +} +@misc{asv, + year="2019", + author="{{tqdm} developers}", + title="airspeed velocity", + url="https://tqdm.github.io/tqdm/" +} +@misc{licence, + year="2019", + author="{{tqdm} developers}", + title="{tqdm} Licence", + url="https://github.com/tqdm/tqdm/blob/master/LICENCE", + publisher="GitHub" +} diff --git a/examples/paper.md b/examples/paper.md new file mode 100644 index 0000000..d62a298 --- /dev/null +++ b/examples/paper.md @@ -0,0 +1,169 @@ +--- +title: '`tqdm`: A Fast, Extensible Progress Meter for Python and CLI' +tags: + - progressbar + - progressmeter + - progress-bar + - meter + - rate + - eta + - console + - terminal + - time + - progress + - bar + - gui + - python + - parallel + - cli + - utilities + - shell + - batch +authors: + - name: Casper O da Costa-Luis + orcid: 0000-0002-7211-1557 + affiliation: 1 +affiliations: + - name: "Independent (Non-affiliated)" + index: 1 +date: 16 February 2019 +bibliography: paper.bib +--- + +# Introduction + +**`tqdm`** is a progress bar library designed to be fast and extensible. It is +written in Python, though ports in other languages are available. `tqdm` means +**progress** in Arabic (*taqadum* [@tqdm-ar]) and is an abbreviation for +**I love you so much** in Spanish (*te quiero demasiado* [@tqdm-es]). + +It is a common programming problem to have iterative operations where progress +monitoring is desirable or advantageous. Including statements within a `for` loop to `print` out the current iteration number is a common strategy. However, there are many improvements which could be made in such a scenario: + +- preventing excessive printing, such as only displaying every $n$^th^ + iteration; +- displaying iteration rate; +- displaying elapsed and estimated completion times, and +- showing all of the above on one continuously updating line. + +Addressing all these issues may well take up more developer time and effort than +the rest of the content of the loop. Any changes to iteration rates or attempts +to re-use the printing logic in a different loop may well result in suboptimal +display rates -- displaying every $n$^th^ iteration may be too (in)frequent -- +requiring manual adjustment of $n$ to fix. + +`tqdm` addresses all of these problems once and for all, taking advantage of +Pythonic patterns to make it a trivial task to add visually appealing, +customisable progress bars without any significant performance degradation even +in the most demanding of scenarios. + +`tqdm` is intended to be used in frontends (giving end users a visual indication +of progress of computations or data transfer). It is also useful for developers +for debugging purposes, both as a profiling tool and also as a way of displaying +logging information of an iterative task (such as error during training of +machine learning algorithms). Due to its ease of use, the library is also an +ideal candidate for inclusion in Python educational courses. For general (not +necessarily Python) purposes, the command-line interface (CLI) mode further +presents a useful tool for CLI users and system administrators monitoring data +flow through pipes. + +# Features + +Exhaustive documentation may be found on the project's [home +page](https://github.com/tqdm/tqdm/#documentation). + +The two basic use cases are within Python code and within a CLI: + +## Python Iterable Wrapper + +`tqdm`'s primary (and original) use is as a wrapper around Python iterables. A +simple case would be: + +```python +from tqdm import tqdm +from time import sleep +for i in tqdm(range(100)): + sleep(0.1) +100%|#########################################| 100/100 [00:10<00:00, 9.95it/s] +``` + +Supported features include: + +- Display customisation via arguments such as `desc`, `postfix` and `bar_format` +- Automatic limiting of display updates to avoid slowing down due to excessive + iteration rates [@stdout] +- Automatic detection of console width to fill the display +- Automatic use of Unicode to render smooth-filling progress bars on supported + terminals +- Support for custom rendering frontends, including: + * Command-line interface + * *Jupyter* HTML notebooks + * `matplotlib` +- Support for custom hooks/callbacks, including: + * `pandas` + * `keras` [@keras] + +## Command-line Interface (CLI) + +A CLI is also provided, where `tqdm` may be used a pipe: + +```sh + # count lines of text in all *.txt files +$ cat *.txt | wc -l +1075075 + # same but with continuously updating progress information +$ cat *.txt | python3 -m tqdm --unit loc --unit_scale | wc -l +1.08Mloc [00:07, 142kloc/s] + # same if `total` is known +$ cat *.txt | python3 -m tqdm --unit loc --unit_scale --total 1075075 | wc -l +100%|#####################################| 1.08/1.08M [00:07<00:00, 142kloc/s] +1075075 +``` + +# Availability + +The package supports both Python versions 2 and 3, and is available for download +via `conda` [@conda], `pip` [@pypi], `snap` [@snapcraft], `docker` [@docker], +and *Zenodo* [@zenodo]. +Web-based Jupyter interactive demonstrations are also available +[@notebooks;@binder] + +Unit tests are run at least weekly on cloud-based continuous integration +[@travis], with code style and security issues checked on +[Codacy](https://app.codacy.com/project/tqdm/tqdm/dashboard) [@code-review]. +Coverage is reported on [Coveralls](https://coveralls.io/github/tqdm/tqdm) and +[Codecov](https://codecov.io/gh/tqdm/tqdm), and performance is monitored against +regression [@asv]. + +# Impact + +As of January 2019, `tqdm` has accumulated over 20 million downloads +[@pypi-downloads], and 315 thousand code inclusions [@tqdm-results]. Dependants +of `tqdm` include 23 thousand repositories [@tqdm-dependents] and 7 thousand +libraries [@lib-io]. `tqdm` has a SourceRank of 22 [@sourcerank], placing it in +the world's top 20 Python packages as of early 2019 [@sourcerank-descending]. + +The source code of `tqdm` is hosted on GitHub, where it has received over 9 +thousand stars [@stars;@stars-hist], and was top trending repository during a +period in December 2015 [@trend-hist]. The documentation has received over 500 +thousand hits [@hits], with highest rates during weekdays. Historical reading +rates have also trended upwards at the end of holiday periods. This implies +widespread use in commercial and academic settings. +[OpenHub](https://www.openhub.net/p/tqdm) valuates the work according to the +constructive cost model (COCOMO) as being worth approximately $50,000. + +The library has also been used in several textbooks [@miller;@boxel;@nandy] and +peer-reviewed scientific publications +[@stein;@cook;@madhikar;@palmer;@knight;@moriwaki;@jackson]. +The [`tqdm` wiki](https://github.com/tqdm/tqdm/wiki) also lists other references +in public media. + +# Licence + +`tqdm`'s source code is OSS, and all versions are archived at the DOI +[10.5281/zenodo.595120](https://doi.org/10.5281/zenodo.595120). The primary +maintainer [Casper da Costa-Luis](https://github.com/casperdcl) releases +contributions under the terms of the MPLv2.0, while all other contributions are +released under the terms of the MIT licence [@licence]. + +# References diff --git a/examples/parallel_bars.py b/examples/parallel_bars.py new file mode 100644 index 0000000..498fd61 --- /dev/null +++ b/examples/parallel_bars.py @@ -0,0 +1,61 @@ +from __future__ import print_function + +import sys +from concurrent.futures import ThreadPoolExecutor +from functools import partial +from multiprocessing import Pool, RLock, freeze_support +from random import random +from threading import RLock as TRLock +from time import sleep + +from tqdm.auto import tqdm, trange +from tqdm.contrib.concurrent import process_map, thread_map + +NUM_SUBITERS = 9 +PY2 = sys.version_info[:1] <= (2,) + + +def progresser(n, auto_position=True, write_safe=False, blocking=True, progress=False): + interval = random() * 0.002 / (NUM_SUBITERS - n + 2) # nosec + total = 5000 + text = "#{0}, est. {1:<04.2}s".format(n, interval * total) + for _ in trange(total, desc=text, disable=not progress, + lock_args=None if blocking else (False,), + position=None if auto_position else n): + sleep(interval) + # NB: may not clear instances with higher `position` upon completion + # since this worker may not know about other bars #796 + if write_safe: + # we think we know about other bars (currently only py3 threading) + if n == 6: + tqdm.write("n == 6 completed") + return n + 1 + + +if __name__ == '__main__': + freeze_support() # for Windows support + L = list(range(NUM_SUBITERS))[::-1] + + print("Simple thread mapping") + thread_map(partial(progresser, write_safe=not PY2), L, max_workers=4) + + print("Simple process mapping") + process_map(partial(progresser), L, max_workers=4) + + print("Manual nesting") + for i in trange(16, desc="1"): + for _ in trange(16, desc="2 @ %d" % i, leave=i % 2): + sleep(0.01) + + print("Multi-processing") + tqdm.set_lock(RLock()) + p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) + p.map(partial(progresser, progress=True), L) + + print("Multi-threading") + tqdm.set_lock(TRLock()) + pool_args = {} + if not PY2: + pool_args.update(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) + with ThreadPoolExecutor(**pool_args) as p: + p.map(partial(progresser, progress=True, write_safe=not PY2, blocking=False), L) diff --git a/examples/redirect_print.py b/examples/redirect_print.py new file mode 100644 index 0000000..0f9721e --- /dev/null +++ b/examples/redirect_print.py @@ -0,0 +1,52 @@ +"""Redirecting writing + +If using a library that can print messages to the console, editing the library +by replacing `print()` with `tqdm.write()` may not be desirable. +In that case, redirecting `sys.stdout` to `tqdm.write()` is an option. + +To redirect `sys.stdout`, create a file-like class that will write +any input string to `tqdm.write()`, and supply the arguments +`file=sys.stdout, dynamic_ncols=True`. + +A reusable canonical example is given below: +""" +from __future__ import print_function + +import contextlib +import sys +from time import sleep + +from tqdm import tqdm +from tqdm.contrib import DummyTqdmFile + + +@contextlib.contextmanager +def std_out_err_redirect_tqdm(): + orig_out_err = sys.stdout, sys.stderr + try: + # sys.stdout = sys.stderr = DummyTqdmFile(orig_out_err[0]) + sys.stdout, sys.stderr = map(DummyTqdmFile, orig_out_err) + yield orig_out_err[0] + # Relay exceptions + except Exception as exc: + raise exc + # Always restore sys.stdout/err if necessary + finally: + sys.stdout, sys.stderr = orig_out_err + + +def some_fun(i): + print("Fee, fi, fo,".split()[i]) + + +# Redirect stdout to tqdm.write() +with std_out_err_redirect_tqdm() as orig_stdout: + # tqdm needs the original stdout + # and dynamic_ncols=True to autodetect console width + for i in tqdm(range(3), file=orig_stdout, dynamic_ncols=True): + # order of the following two lines should not matter + some_fun(i) + sleep(.5) + +# After the `with`, printing is restored +print("Done!") diff --git a/examples/simple_examples.py b/examples/simple_examples.py new file mode 100644 index 0000000..f3401d3 --- /dev/null +++ b/examples/simple_examples.py @@ -0,0 +1,65 @@ +""" +# Simple tqdm examples and profiling + +# Benchmark +for i in _range(int(1e8)): + pass + +# Basic demo +import tqdm +for i in tqdm.trange(int(1e8)): + pass + +# Some decorations +import tqdm +for i in tqdm.trange(int(1e8), miniters=int(1e6), ascii=True, + desc="cool", dynamic_ncols=True): + pass + +# Nested bars +from tqdm import trange +for i in trange(10): + for j in trange(int(1e7), leave=False, unit_scale=True): + pass + +# Experimental GUI demo +import tqdm +for i in tqdm.tgrange(int(1e8)): + pass + +# Comparison to https://code.google.com/p/python-progressbar/ +try: + from progressbar.progressbar import ProgressBar +except ImportError: + pass +else: + for i in ProgressBar()(_range(int(1e8))): + pass + +# Dynamic miniters benchmark +from tqdm import trange +for i in trange(int(1e8), miniters=None, mininterval=0.1, smoothing=0): + pass + +# Fixed miniters benchmark +from tqdm import trange +for i in trange(int(1e8), miniters=4500000, mininterval=0.1, smoothing=0): + pass +""" + +import re +from time import sleep +from timeit import timeit + +# Simple demo +from tqdm import trange + +for _ in trange(16, leave=True): + sleep(0.1) + +# Profiling/overhead tests +stmts = filter(None, re.split(r'\n\s*#.*?\n', __doc__)) +for s in stmts: + print(s.replace('import tqdm\n', '')) + print(timeit(stmt='try:\n\t_range = xrange' + '\nexcept:\n\t_range = range\n' + s, number=1), 'seconds') diff --git a/examples/tqdm_requests.py b/examples/tqdm_requests.py new file mode 100644 index 0000000..5d03594 --- /dev/null +++ b/examples/tqdm_requests.py @@ -0,0 +1,49 @@ +"""An example of wrapping manual tqdm updates for `requests.get`. +See also: tqdm_wget.py. + +Usage: + tqdm_requests.py [options] + +Options: +-h, --help + Print this help message and exit +-u URL, --url URL : string, optional + The url to fetch. + [default: https://caspersci.uk.to/matryoshka.zip] +-o FILE, --output FILE : string, optional + The local file path in which to save the url [default: /dev/null]. +""" + +from os import devnull + +import requests +from docopt import docopt + +from tqdm.auto import tqdm + +opts = docopt(__doc__) + +eg_link = opts['--url'] +eg_file = eg_link.replace('/', ' ').split()[-1] +eg_out = opts['--output'].replace("/dev/null", devnull) + +response = requests.get(eg_link, stream=True) +with open(eg_out, "wb") as fout: + with tqdm( + # all optional kwargs + unit='B', unit_scale=True, unit_divisor=1024, miniters=1, + desc=eg_file, total=int(response.headers.get('content-length', 0)) + ) as pbar: + for chunk in response.iter_content(chunk_size=4096): + fout.write(chunk) + pbar.update(len(chunk)) + +# Even simpler progress by wrapping the output file's `write()` +response = requests.get(eg_link, stream=True) +with tqdm.wrapattr( + open(eg_out, "wb"), "write", + unit='B', unit_scale=True, unit_divisor=1024, miniters=1, + desc=eg_file, total=int(response.headers.get('content-length', 0)) +) as fout: + for chunk in response.iter_content(chunk_size=4096): + fout.write(chunk) diff --git a/examples/tqdm_wget.py b/examples/tqdm_wget.py new file mode 100644 index 0000000..8663e5a --- /dev/null +++ b/examples/tqdm_wget.py @@ -0,0 +1,113 @@ +"""An example of wrapping manual tqdm updates for `urllib` reporthook. +See also: tqdm_requests.py. + +# `urllib.urlretrieve` documentation +> If present, the hook function will be called once +> on establishment of the network connection and once after each block read +> thereafter. The hook will be passed three arguments; a count of blocks +> transferred so far, a block size in bytes, and the total size of the file. + +Usage: + tqdm_wget.py [options] + +Options: +-h, --help + Print this help message and exit +-u URL, --url URL : string, optional + The url to fetch. + [default: https://caspersci.uk.to/matryoshka.zip] +-o FILE, --output FILE : string, optional + The local file path in which to save the url [default: /dev/null]. +""" + +try: + from urllib import request as urllib +except ImportError: # py2 + import urllib +from os import devnull + +from docopt import docopt + +from tqdm.auto import tqdm + + +def my_hook(t): + """Wraps tqdm instance. + + Don't forget to close() or __exit__() + the tqdm instance once you're done with it (easiest using `with` syntax). + + Example + ------- + + >>> with tqdm(...) as t: + ... reporthook = my_hook(t) + ... urllib.urlretrieve(..., reporthook=reporthook) + + """ + last_b = [0] + + def update_to(b=1, bsize=1, tsize=None): + """ + b : int, optional + Number of blocks transferred so far [default: 1]. + bsize : int, optional + Size of each block (in tqdm units) [default: 1]. + tsize : int, optional + Total size (in tqdm units). If [default: None] or -1, + remains unchanged. + """ + if tsize not in (None, -1): + t.total = tsize + displayed = t.update((b - last_b[0]) * bsize) + last_b[0] = b + return displayed + + return update_to + + +class TqdmUpTo(tqdm): + """Alternative Class-based version of the above. + + Provides `update_to(n)` which uses `tqdm.update(delta_n)`. + + Inspired by [twine#242](https://github.com/pypa/twine/pull/242), + [here](https://github.com/pypa/twine/commit/42e55e06). + """ + + def update_to(self, b=1, bsize=1, tsize=None): + """ + b : int, optional + Number of blocks transferred so far [default: 1]. + bsize : int, optional + Size of each block (in tqdm units) [default: 1]. + tsize : int, optional + Total size (in tqdm units). If [default: None] remains unchanged. + """ + if tsize is not None: + self.total = tsize + return self.update(b * bsize - self.n) # also sets self.n = b * bsize + + +opts = docopt(__doc__) + +eg_link = opts['--url'] +eg_file = eg_link.replace('/', ' ').split()[-1] +eg_out = opts['--output'].replace("/dev/null", devnull) +# with tqdm(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, +# desc=eg_file) as t: # all optional kwargs +# urllib.urlretrieve(eg_link, filename=eg_out, +# reporthook=my_hook(t), data=None) +with TqdmUpTo(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, + desc=eg_file) as t: # all optional kwargs + urllib.urlretrieve( # nosec + eg_link, filename=eg_out, reporthook=t.update_to, data=None) + t.total = t.n + +# Even simpler progress by wrapping the output file's `write()` +response = urllib.urlopen(eg_link) # nosec +with tqdm.wrapattr(open(eg_out, "wb"), "write", + miniters=1, desc=eg_file, + total=getattr(response, 'length', None)) as fout: + for chunk in response: + fout.write(chunk) diff --git a/examples/wrapping_generators.py b/examples/wrapping_generators.py new file mode 100644 index 0000000..65c85bf --- /dev/null +++ b/examples/wrapping_generators.py @@ -0,0 +1,15 @@ +import numpy as np + +from tqdm.contrib import tenumerate, tmap, tzip + +for _ in tenumerate(range(int(1e6)), desc="builtin enumerate"): + pass + +for _ in tenumerate(np.random.random((999, 999)), desc="numpy.ndenumerate"): + pass + +for _ in tzip(np.arange(1e6), np.arange(1e6) + 1, desc="builtin zip"): + pass + +mapped = tmap(lambda x: x + 1, np.arange(1e6), desc="builtin map") +assert (np.arange(1e6) + 1 == list(mapped)).all() diff --git a/logo.png b/logo.png Binary files differnew file mode 100644 index 0000000..ea3cb31 --- /dev/null +++ b/logo.png diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..3eb7bbc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=3.4"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "tqdm/_dist_ver.py" +write_to_template = "__version__ = '{version}'\n" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..4fb34f3 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,159 @@ +[metadata] +name = tqdm +url = https://tqdm.github.io +project_urls = + Changelog=https://tqdm.github.io/releases + Source=https://github.com/tqdm/tqdm + Wiki=https://github.com/tqdm/tqdm/wiki +maintainer = tqdm developers +maintainer_email = python.tqdm@gmail.com +license = MPLv2.0, MIT Licences +license_file = LICENCE +description = Fast, Extensible Progress Meter +long_description = file: README.rst +long_description_content_type = text/x-rst +keywords = progressbar, progressmeter, progress, bar, meter, rate, eta, console, terminal, time +platforms = any +provides = tqdm +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Console + Environment :: MacOS X + Environment :: Other Environment + Environment :: Win32 (MS Windows) + Environment :: X11 Applications + Framework :: IPython + Framework :: Jupyter + Intended Audience :: Developers + Intended Audience :: Education + Intended Audience :: End Users/Desktop + Intended Audience :: Other Audience + Intended Audience :: System Administrators + License :: OSI Approved :: MIT License + License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0) + Operating System :: MacOS + Operating System :: MacOS :: MacOS X + Operating System :: Microsoft + Operating System :: Microsoft :: MS-DOS + Operating System :: Microsoft :: Windows + Operating System :: POSIX + Operating System :: POSIX :: BSD + Operating System :: POSIX :: BSD :: FreeBSD + Operating System :: POSIX :: Linux + Operating System :: POSIX :: SunOS/Solaris + Operating System :: Unix + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: Implementation + Programming Language :: Python :: Implementation :: IronPython + Programming Language :: Python :: Implementation :: PyPy + Programming Language :: Unix Shell + Topic :: Desktop Environment + Topic :: Education :: Computer Aided Instruction (CAI) + Topic :: Education :: Testing + Topic :: Office/Business + Topic :: Other/Nonlisted Topic + Topic :: Software Development :: Build Tools + Topic :: Software Development :: Libraries + Topic :: Software Development :: Libraries :: Python Modules + Topic :: Software Development :: Pre-processors + Topic :: Software Development :: User Interfaces + Topic :: System :: Installation/Setup + Topic :: System :: Logging + Topic :: System :: Monitoring + Topic :: System :: Shells + Topic :: Terminals + Topic :: Utilities + +[options] +setup_requires = setuptools>=42; setuptools_scm[toml]>=3.4 +python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* +install_requires = + colorama; platform_system == 'Windows' + importlib_resources; python_version < "3.7" +tests_require = tox +include_package_data = True +packages = find: + +[options.extras_require] +dev = py-make>=0.1.0; twine; wheel +slack = slack-sdk +telegram = requests +notebook = ipywidgets>=6 + +[options.entry_points] +console_scripts = + tqdm=tqdm.cli:main + +[options.packages.find] +exclude = benchmarks, tests + +[bdist_wheel] +universal = 1 + +[flake8] +max_line_length = 99 +exclude = .asv,.eggs,.tox,.ipynb_checkpoints,build,dist,.git,__pycache__ + +[pydocstyle] +add_ignore = D400,D415 + +[yapf] +coalesce_brackets = True +column_limit = 99 +each_dict_entry_on_separate_line = False +i18n_comment = NOQA +space_between_ending_comma_and_closing_bracket = False +split_before_named_assigns = False +split_before_closing_bracket = False + +[isort] +line_length = 99 +multi_line_output = 4 +known_first_party = tqdm,tests + +[tool:pytest] +timeout = 30 +log_level = INFO +markers = + asyncio + slow +python_files = tests_*.py tests_*.ipynb +testpaths = tests +addopts = -v --tb=short -rxs -W=error --durations=0 --durations-min=0.1 --asyncio-mode=strict + +[regex1] +regex = (?<= )[\s\d.]+(it/s|s/it) +replace = ??.??it/s + +[regex2] +regex = 00:0[01]<00:0[01] +replace = 00:00<00:00 + +[coverage:run] +branch = True +include = tqdm/* +omit = + tqdm/contrib/bells.py + tqdm/contrib/slack.py + tqdm/contrib/discord.py + tqdm/contrib/telegram.py + tqdm/contrib/utils_worker.py +relative_files = True +disable_warnings = include-ignored + +[coverage:report] +show_missing = True + +[egg_info] +tag_build = +tag_date = 0 + diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..89dadf5 --- /dev/null +++ b/setup.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import sys +from os import path + +from setuptools import setup + +src_dir = path.abspath(path.dirname(__file__)) +if sys.argv[1].lower().strip() == 'make': # exec Makefile commands + import pymake + fpath = path.join(src_dir, 'Makefile') + pymake.main(['-f', fpath] + sys.argv[2:]) + # Stop to avoid setup.py raising non-standard command error + sys.exit(0) + +setup(use_scm_version=True) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/tests/__init__.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..6717044 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,41 @@ +"""Shared pytest config.""" +import sys + +from pytest import fixture + +from tqdm import tqdm + + +@fixture(autouse=True) +def pretest_posttest(): + """Fixture for all tests ensuring environment cleanup""" + try: + sys.setswitchinterval(1) + except AttributeError: + sys.setcheckinterval(100) # deprecated + + if getattr(tqdm, "_instances", False): + n = len(tqdm._instances) + if n: + tqdm._instances.clear() + raise EnvironmentError( + "{0} `tqdm` instances still in existence PRE-test".format(n)) + yield + if getattr(tqdm, "_instances", False): + n = len(tqdm._instances) + if n: + tqdm._instances.clear() + raise EnvironmentError( + "{0} `tqdm` instances still in existence POST-test".format(n)) + + +if sys.version_info[0] > 2: + @fixture + def capsysbin(capsysbinary): + """alias for capsysbinary (py3)""" + return capsysbinary +else: + @fixture + def capsysbin(capsys): + """alias for capsys (py2)""" + return capsys diff --git a/tests/py37_asyncio.py b/tests/py37_asyncio.py new file mode 100644 index 0000000..8bf61e7 --- /dev/null +++ b/tests/py37_asyncio.py @@ -0,0 +1,128 @@ +import asyncio +from functools import partial +from sys import platform +from time import time + +from tqdm.asyncio import tarange, tqdm_asyncio + +from .tests_tqdm import StringIO, closing, mark + +tqdm = partial(tqdm_asyncio, miniters=0, mininterval=0) +trange = partial(tarange, miniters=0, mininterval=0) +as_completed = partial(tqdm_asyncio.as_completed, miniters=0, mininterval=0) +gather = partial(tqdm_asyncio.gather, miniters=0, mininterval=0) + + +def count(start=0, step=1): + i = start + while True: + new_start = yield i + if new_start is None: + i += step + else: + i = new_start + + +async def acount(*args, **kwargs): + for i in count(*args, **kwargs): + yield i + + +@mark.asyncio +async def test_break(): + """Test asyncio break""" + pbar = tqdm(count()) + async for _ in pbar: + break + pbar.close() + + +@mark.asyncio +async def test_generators(capsys): + """Test asyncio generators""" + with tqdm(count(), desc="counter") as pbar: + async for i in pbar: + if i >= 8: + break + _, err = capsys.readouterr() + assert '9it' in err + + with tqdm(acount(), desc="async_counter") as pbar: + async for i in pbar: + if i >= 8: + break + _, err = capsys.readouterr() + assert '9it' in err + + +@mark.asyncio +async def test_range(): + """Test asyncio range""" + with closing(StringIO()) as our_file: + async for _ in tqdm(range(9), desc="range", file=our_file): + pass + assert '9/9' in our_file.getvalue() + our_file.seek(0) + our_file.truncate() + + async for _ in trange(9, desc="trange", file=our_file): + pass + assert '9/9' in our_file.getvalue() + + +@mark.asyncio +async def test_nested(): + """Test asyncio nested""" + with closing(StringIO()) as our_file: + async for _ in tqdm(trange(9, desc="inner", file=our_file), + desc="outer", file=our_file): + pass + assert 'inner: 100%' in our_file.getvalue() + assert 'outer: 100%' in our_file.getvalue() + + +@mark.asyncio +async def test_coroutines(): + """Test asyncio coroutine.send""" + with closing(StringIO()) as our_file: + with tqdm(count(), file=our_file) as pbar: + async for i in pbar: + if i == 9: + pbar.send(-10) + elif i < 0: + assert i == -9 + break + assert '10it' in our_file.getvalue() + + +@mark.slow +@mark.asyncio +@mark.parametrize("tol", [0.2 if platform.startswith("darwin") else 0.1]) +async def test_as_completed(capsys, tol): + """Test asyncio as_completed""" + for retry in range(3): + t = time() + skew = time() - t + for i in as_completed([asyncio.sleep(0.01 * i) for i in range(30, 0, -1)]): + await i + t = time() - t - 2 * skew + try: + assert 0.3 * (1 - tol) < t < 0.3 * (1 + tol), t + _, err = capsys.readouterr() + assert '30/30' in err + except AssertionError: + if retry == 2: + raise + + +async def double(i): + return i * 2 + + +@mark.asyncio +async def test_gather(capsys): + """Test asyncio gather""" + res = await gather(*map(double, range(30))) + _, err = capsys.readouterr() + assert '30/30' in err + assert res == list(range(0, 30 * 2, 2)) diff --git a/tests/tests_asyncio.py b/tests/tests_asyncio.py new file mode 100644 index 0000000..6f08926 --- /dev/null +++ b/tests/tests_asyncio.py @@ -0,0 +1,11 @@ +"""Tests `tqdm.asyncio` on `python>=3.7`.""" +import sys + +if sys.version_info[:2] > (3, 6): + from .py37_asyncio import * # NOQA, pylint: disable=wildcard-import +else: + from .tests_tqdm import skip + try: + skip("async not supported", allow_module_level=True) + except TypeError: + pass diff --git a/tests/tests_concurrent.py b/tests/tests_concurrent.py new file mode 100644 index 0000000..5cd439c --- /dev/null +++ b/tests/tests_concurrent.py @@ -0,0 +1,49 @@ +""" +Tests for `tqdm.contrib.concurrent`. +""" +from pytest import warns + +from tqdm.contrib.concurrent import process_map, thread_map + +from .tests_tqdm import StringIO, TqdmWarning, closing, importorskip, mark, skip + + +def incr(x): + """Dummy function""" + return x + 1 + + +def test_thread_map(): + """Test contrib.concurrent.thread_map""" + with closing(StringIO()) as our_file: + a = range(9) + b = [i + 1 for i in a] + try: + assert thread_map(lambda x: x + 1, a, file=our_file) == b + except ImportError as err: + skip(str(err)) + assert thread_map(incr, a, file=our_file) == b + + +def test_process_map(): + """Test contrib.concurrent.process_map""" + with closing(StringIO()) as our_file: + a = range(9) + b = [i + 1 for i in a] + try: + assert process_map(incr, a, file=our_file) == b + except ImportError as err: + skip(str(err)) + + +@mark.parametrize("iterables,should_warn", [([], False), (['x'], False), ([()], False), + (['x', ()], False), (['x' * 1001], True), + (['x' * 100, ('x',) * 1001], True)]) +def test_chunksize_warning(iterables, should_warn): + """Test contrib.concurrent.process_map chunksize warnings""" + patch = importorskip('unittest.mock').patch + with patch('tqdm.contrib.concurrent._executor_map'): + if should_warn: + warns(TqdmWarning, process_map, incr, *iterables) + else: + process_map(incr, *iterables) diff --git a/tests/tests_contrib.py b/tests/tests_contrib.py new file mode 100644 index 0000000..69a1cad --- /dev/null +++ b/tests/tests_contrib.py @@ -0,0 +1,71 @@ +""" +Tests for `tqdm.contrib`. +""" +import sys + +import pytest + +from tqdm import tqdm +from tqdm.contrib import tenumerate, tmap, tzip + +from .tests_tqdm import StringIO, closing, importorskip + + +def incr(x): + """Dummy function""" + return x + 1 + + +@pytest.mark.parametrize("tqdm_kwargs", [{}, {"tqdm_class": tqdm}]) +def test_enumerate(tqdm_kwargs): + """Test contrib.tenumerate""" + with closing(StringIO()) as our_file: + a = range(9) + assert list(tenumerate(a, file=our_file, **tqdm_kwargs)) == list(enumerate(a)) + assert list(tenumerate(a, 42, file=our_file, **tqdm_kwargs)) == list( + enumerate(a, 42) + ) + with closing(StringIO()) as our_file: + _ = list(tenumerate(iter(a), file=our_file, **tqdm_kwargs)) + assert "100%" not in our_file.getvalue() + with closing(StringIO()) as our_file: + _ = list(tenumerate(iter(a), file=our_file, total=len(a), **tqdm_kwargs)) + assert "100%" in our_file.getvalue() + + +def test_enumerate_numpy(): + """Test contrib.tenumerate(numpy.ndarray)""" + np = importorskip("numpy") + with closing(StringIO()) as our_file: + a = np.random.random((42, 7)) + assert list(tenumerate(a, file=our_file)) == list(np.ndenumerate(a)) + + +@pytest.mark.parametrize("tqdm_kwargs", [{}, {"tqdm_class": tqdm}]) +def test_zip(tqdm_kwargs): + """Test contrib.tzip""" + with closing(StringIO()) as our_file: + a = range(9) + b = [i + 1 for i in a] + if sys.version_info[:1] < (3,): + assert tzip(a, b, file=our_file, **tqdm_kwargs) == zip(a, b) + else: + gen = tzip(a, b, file=our_file, **tqdm_kwargs) + assert gen != list(zip(a, b)) + assert list(gen) == list(zip(a, b)) + + +@pytest.mark.parametrize("tqdm_kwargs", [{}, {"tqdm_class": tqdm}]) +def test_map(tqdm_kwargs): + """Test contrib.tmap""" + with closing(StringIO()) as our_file: + a = range(9) + b = [i + 1 for i in a] + if sys.version_info[:1] < (3,): + assert tmap(lambda x: x + 1, a, file=our_file, **tqdm_kwargs) == map( + incr, a + ) + else: + gen = tmap(lambda x: x + 1, a, file=our_file, **tqdm_kwargs) + assert gen != b + assert list(gen) == b diff --git a/tests/tests_contrib_logging.py b/tests/tests_contrib_logging.py new file mode 100644 index 0000000..6f675dd --- /dev/null +++ b/tests/tests_contrib_logging.py @@ -0,0 +1,173 @@ +# pylint: disable=missing-module-docstring, missing-class-docstring +# pylint: disable=missing-function-docstring, no-self-use +from __future__ import absolute_import + +import logging +import logging.handlers +import sys +from io import StringIO + +import pytest + +from tqdm import tqdm +from tqdm.contrib.logging import _get_first_found_console_logging_handler +from tqdm.contrib.logging import _TqdmLoggingHandler as TqdmLoggingHandler +from tqdm.contrib.logging import logging_redirect_tqdm, tqdm_logging_redirect + +from .tests_tqdm import importorskip + +LOGGER = logging.getLogger(__name__) + +TEST_LOGGING_FORMATTER = logging.Formatter() + + +class CustomTqdm(tqdm): + messages = [] + + @classmethod + def write(cls, s, **__): # pylint: disable=arguments-differ + CustomTqdm.messages.append(s) + + +class ErrorRaisingTqdm(tqdm): + exception_class = RuntimeError + + @classmethod + def write(cls, s, **__): # pylint: disable=arguments-differ + raise ErrorRaisingTqdm.exception_class('fail fast') + + +class TestTqdmLoggingHandler: + def test_should_call_tqdm_write(self): + CustomTqdm.messages = [] + logger = logging.Logger('test') + logger.handlers = [TqdmLoggingHandler(CustomTqdm)] + logger.info('test') + assert CustomTqdm.messages == ['test'] + + def test_should_call_handle_error_if_exception_was_thrown(self): + patch = importorskip('unittest.mock').patch + logger = logging.Logger('test') + ErrorRaisingTqdm.exception_class = RuntimeError + handler = TqdmLoggingHandler(ErrorRaisingTqdm) + logger.handlers = [handler] + with patch.object(handler, 'handleError') as mock: + logger.info('test') + assert mock.called + + @pytest.mark.parametrize('exception_class', [ + KeyboardInterrupt, + SystemExit + ]) + def test_should_not_swallow_certain_exceptions(self, exception_class): + logger = logging.Logger('test') + ErrorRaisingTqdm.exception_class = exception_class + handler = TqdmLoggingHandler(ErrorRaisingTqdm) + logger.handlers = [handler] + with pytest.raises(exception_class): + logger.info('test') + + +class TestGetFirstFoundConsoleLoggingHandler: + def test_should_return_none_for_no_handlers(self): + assert _get_first_found_console_logging_handler([]) is None + + def test_should_return_none_without_stream_handler(self): + handler = logging.handlers.MemoryHandler(capacity=1) + assert _get_first_found_console_logging_handler([handler]) is None + + def test_should_return_none_for_stream_handler_not_stdout_or_stderr(self): + handler = logging.StreamHandler(StringIO()) + assert _get_first_found_console_logging_handler([handler]) is None + + def test_should_return_stream_handler_if_stream_is_stdout(self): + handler = logging.StreamHandler(sys.stdout) + assert _get_first_found_console_logging_handler([handler]) == handler + + def test_should_return_stream_handler_if_stream_is_stderr(self): + handler = logging.StreamHandler(sys.stderr) + assert _get_first_found_console_logging_handler([handler]) == handler + + +class TestRedirectLoggingToTqdm: + def test_should_add_and_remove_tqdm_handler(self): + logger = logging.Logger('test') + with logging_redirect_tqdm(loggers=[logger]): + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], TqdmLoggingHandler) + assert not logger.handlers + + def test_should_remove_and_restore_console_handlers(self): + logger = logging.Logger('test') + stderr_console_handler = logging.StreamHandler(sys.stderr) + stdout_console_handler = logging.StreamHandler(sys.stderr) + logger.handlers = [stderr_console_handler, stdout_console_handler] + with logging_redirect_tqdm(loggers=[logger]): + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], TqdmLoggingHandler) + assert logger.handlers == [stderr_console_handler, stdout_console_handler] + + def test_should_inherit_console_logger_formatter(self): + logger = logging.Logger('test') + formatter = logging.Formatter('custom: %(message)s') + console_handler = logging.StreamHandler(sys.stderr) + console_handler.setFormatter(formatter) + logger.handlers = [console_handler] + with logging_redirect_tqdm(loggers=[logger]): + assert logger.handlers[0].formatter == formatter + + def test_should_not_remove_stream_handlers_not_for_stdout_or_stderr(self): + logger = logging.Logger('test') + stream_handler = logging.StreamHandler(StringIO()) + logger.addHandler(stream_handler) + with logging_redirect_tqdm(loggers=[logger]): + assert len(logger.handlers) == 2 + assert logger.handlers[0] == stream_handler + assert isinstance(logger.handlers[1], TqdmLoggingHandler) + assert logger.handlers == [stream_handler] + + +class TestTqdmWithLoggingRedirect: + def test_should_add_and_remove_handler_from_root_logger_by_default(self): + original_handlers = list(logging.root.handlers) + with tqdm_logging_redirect(total=1) as pbar: + assert isinstance(logging.root.handlers[-1], TqdmLoggingHandler) + LOGGER.info('test') + pbar.update(1) + assert logging.root.handlers == original_handlers + + def test_should_add_and_remove_handler_from_custom_logger(self): + logger = logging.Logger('test') + with tqdm_logging_redirect(total=1, loggers=[logger]) as pbar: + assert len(logger.handlers) == 1 + assert isinstance(logger.handlers[0], TqdmLoggingHandler) + logger.info('test') + pbar.update(1) + assert not logger.handlers + + def test_should_not_fail_with_logger_without_console_handler(self): + logger = logging.Logger('test') + logger.handlers = [] + with tqdm_logging_redirect(total=1, loggers=[logger]): + logger.info('test') + assert not logger.handlers + + def test_should_format_message(self): + logger = logging.Logger('test') + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(logging.Formatter( + r'prefix:%(message)s' + )) + logger.handlers = [console_handler] + CustomTqdm.messages = [] + with tqdm_logging_redirect(loggers=[logger], tqdm_class=CustomTqdm): + logger.info('test') + assert CustomTqdm.messages == ['prefix:test'] + + def test_use_root_logger_by_default_and_write_to_custom_tqdm(self): + logger = logging.root + CustomTqdm.messages = [] + with tqdm_logging_redirect(total=1, tqdm_class=CustomTqdm) as pbar: + assert isinstance(pbar, CustomTqdm) + logger.info('test') + assert CustomTqdm.messages == ['test'] diff --git a/tests/tests_dask.py b/tests/tests_dask.py new file mode 100644 index 0000000..8bf4b64 --- /dev/null +++ b/tests/tests_dask.py @@ -0,0 +1,20 @@ +from __future__ import division + +from time import sleep + +from .tests_tqdm import importorskip, mark + +pytestmark = mark.slow + + +def test_dask(capsys): + """Test tqdm.dask.TqdmCallback""" + ProgressBar = importorskip('tqdm.dask').TqdmCallback + dask = importorskip('dask') + + schedule = [dask.delayed(sleep)(i / 10) for i in range(5)] + with ProgressBar(desc="computing"): + dask.compute(schedule) + _, err = capsys.readouterr() + assert "computing: " in err + assert '5/5' in err diff --git a/tests/tests_gui.py b/tests/tests_gui.py new file mode 100644 index 0000000..dddd918 --- /dev/null +++ b/tests/tests_gui.py @@ -0,0 +1,7 @@ +"""Test `tqdm.gui`.""" +from .tests_tqdm import importorskip + + +def test_gui_import(): + """Test `tqdm.gui` import""" + importorskip('tqdm.gui') diff --git a/tests/tests_itertools.py b/tests/tests_itertools.py new file mode 100644 index 0000000..bfb6eb2 --- /dev/null +++ b/tests/tests_itertools.py @@ -0,0 +1,26 @@ +""" +Tests for `tqdm.contrib.itertools`. +""" +import itertools as it + +from tqdm.contrib.itertools import product + +from .tests_tqdm import StringIO, closing + + +class NoLenIter(object): + def __init__(self, iterable): + self._it = iterable + + def __iter__(self): + for i in self._it: + yield i + + +def test_product(): + """Test contrib.itertools.product""" + with closing(StringIO()) as our_file: + a = range(9) + assert list(product(a, a[::-1], file=our_file)) == list(it.product(a, a[::-1])) + + assert list(product(a, NoLenIter(a), file=our_file)) == list(it.product(a, NoLenIter(a))) diff --git a/tests/tests_keras.py b/tests/tests_keras.py new file mode 100644 index 0000000..220f946 --- /dev/null +++ b/tests/tests_keras.py @@ -0,0 +1,93 @@ +from __future__ import division + +from .tests_tqdm import importorskip, mark + +pytestmark = mark.slow + + +@mark.filterwarnings("ignore:.*:DeprecationWarning") +def test_keras(capsys): + """Test tqdm.keras.TqdmCallback""" + TqdmCallback = importorskip('tqdm.keras').TqdmCallback + np = importorskip('numpy') + try: + import keras as K + except ImportError: + K = importorskip('tensorflow.keras') + + # 1D autoencoder + dtype = np.float32 + model = K.models.Sequential([ + K.layers.InputLayer((1, 1), dtype=dtype), K.layers.Conv1D(1, 1)]) + model.compile("adam", "mse") + x = np.random.rand(100, 1, 1).astype(dtype) + batch_size = 10 + batches = len(x) / batch_size + epochs = 5 + + # just epoch (no batch) progress + model.fit( + x, + x, + epochs=epochs, + batch_size=batch_size, + verbose=False, + callbacks=[ + TqdmCallback( + epochs, + desc="training", + data_size=len(x), + batch_size=batch_size, + verbose=0)]) + _, res = capsys.readouterr() + assert "training: " in res + assert "{epochs}/{epochs}".format(epochs=epochs) in res + assert "{batches}/{batches}".format(batches=batches) not in res + + # full (epoch and batch) progress + model.fit( + x, + x, + epochs=epochs, + batch_size=batch_size, + verbose=False, + callbacks=[ + TqdmCallback( + epochs, + desc="training", + data_size=len(x), + batch_size=batch_size, + verbose=2)]) + _, res = capsys.readouterr() + assert "training: " in res + assert "{epochs}/{epochs}".format(epochs=epochs) in res + assert "{batches}/{batches}".format(batches=batches) in res + + # auto-detect epochs and batches + model.fit( + x, + x, + epochs=epochs, + batch_size=batch_size, + verbose=False, + callbacks=[TqdmCallback(desc="training", verbose=2)]) + _, res = capsys.readouterr() + assert "training: " in res + assert "{epochs}/{epochs}".format(epochs=epochs) in res + assert "{batches}/{batches}".format(batches=batches) in res + + # continue training (start from epoch != 0) + initial_epoch = 3 + model.fit( + x, + x, + initial_epoch=initial_epoch, + epochs=epochs, + batch_size=batch_size, + verbose=False, + callbacks=[TqdmCallback(desc="training", verbose=0, + miniters=1, mininterval=0, maxinterval=0)]) + _, res = capsys.readouterr() + assert "training: " in res + assert "{epochs}/{epochs}".format(epochs=initial_epoch - 1) not in res + assert "{epochs}/{epochs}".format(epochs=epochs) in res diff --git a/tests/tests_main.py b/tests/tests_main.py new file mode 100644 index 0000000..0523cc7 --- /dev/null +++ b/tests/tests_main.py @@ -0,0 +1,245 @@ +"""Test CLI usage.""" +import logging +import subprocess # nosec +import sys +from functools import wraps +from os import linesep + +from tqdm.cli import TqdmKeyError, TqdmTypeError, main +from tqdm.utils import IS_WIN + +from .tests_tqdm import BytesIO, _range, closing, mark, raises + + +def restore_sys(func): + """Decorates `func(capsysbin)` to save & restore `sys.(stdin|argv)`.""" + @wraps(func) + def inner(capsysbin): + """function requiring capsysbin which may alter `sys.(stdin|argv)`""" + _SYS = sys.stdin, sys.argv + try: + res = func(capsysbin) + finally: + sys.stdin, sys.argv = _SYS + return res + + return inner + + +def norm(bytestr): + """Normalise line endings.""" + return bytestr if linesep == "\n" else bytestr.replace(linesep.encode(), b"\n") + + +@mark.slow +def test_pipes(): + """Test command line pipes""" + ls_out = subprocess.check_output(['ls']) # nosec + ls = subprocess.Popen(['ls'], stdout=subprocess.PIPE) # nosec + res = subprocess.Popen( # nosec + [sys.executable, '-c', 'from tqdm.cli import main; main()'], + stdin=ls.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = res.communicate() + assert ls.poll() == 0 + + # actual test: + assert norm(ls_out) == norm(out) + assert b"it/s" in err + assert b"Error" not in err + + +if sys.version_info[:2] >= (3, 8): + test_pipes = mark.filterwarnings("ignore:unclosed file:ResourceWarning")( + test_pipes) + + +def test_main_import(): + """Test main CLI import""" + N = 123 + _SYS = sys.stdin, sys.argv + # test direct import + sys.stdin = [str(i).encode() for i in _range(N)] + sys.argv = ['', '--desc', 'Test CLI import', + '--ascii', 'True', '--unit_scale', 'True'] + try: + import tqdm.__main__ # NOQA, pylint: disable=unused-variable + finally: + sys.stdin, sys.argv = _SYS + + +@restore_sys +def test_main_bytes(capsysbin): + """Test CLI --bytes""" + N = 123 + + # test --delim + IN_DATA = '\0'.join(map(str, _range(N))).encode() + with closing(BytesIO()) as sys.stdin: + sys.stdin.write(IN_DATA) + # sys.stdin.write(b'\xff') # TODO + sys.stdin.seek(0) + main(sys.stderr, ['--desc', 'Test CLI delim', '--ascii', 'True', + '--delim', r'\0', '--buf_size', '64']) + out, err = capsysbin.readouterr() + assert out == IN_DATA + assert str(N) + "it" in err.decode("U8") + + # test --bytes + IN_DATA = IN_DATA.replace(b'\0', b'\n') + with closing(BytesIO()) as sys.stdin: + sys.stdin.write(IN_DATA) + sys.stdin.seek(0) + main(sys.stderr, ['--ascii', '--bytes=True', '--unit_scale', 'False']) + out, err = capsysbin.readouterr() + assert out == IN_DATA + assert str(len(IN_DATA)) + "B" in err.decode("U8") + + +@mark.skipif(sys.version_info[0] == 2, reason="no caplog on py2") +def test_main_log(capsysbin, caplog): + """Test CLI --log""" + _SYS = sys.stdin, sys.argv + N = 123 + sys.stdin = [(str(i) + '\n').encode() for i in _range(N)] + IN_DATA = b''.join(sys.stdin) + try: + with caplog.at_level(logging.INFO): + main(sys.stderr, ['--log', 'INFO']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA and b"123/123" in err + assert not caplog.record_tuples + with caplog.at_level(logging.DEBUG): + main(sys.stderr, ['--log', 'DEBUG']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA and b"123/123" in err + assert caplog.record_tuples + finally: + sys.stdin, sys.argv = _SYS + + +@restore_sys +def test_main(capsysbin): + """Test misc CLI options""" + N = 123 + sys.stdin = [(str(i) + '\n').encode() for i in _range(N)] + IN_DATA = b''.join(sys.stdin) + + # test --tee + main(sys.stderr, ['--mininterval', '0', '--miniters', '1']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA and b"123/123" in err + assert N <= len(err.split(b"\r")) < N + 5 + + len_err = len(err) + main(sys.stderr, ['--tee', '--mininterval', '0', '--miniters', '1']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA and b"123/123" in err + # spaces to clear intermediate lines could increase length + assert len_err + len(norm(out)) <= len(err) + + # test --null + main(sys.stderr, ['--null']) + out, err = capsysbin.readouterr() + assert not out and b"123/123" in err + + # test integer --update + main(sys.stderr, ['--update']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA + assert (str(N // 2 * N) + "it").encode() in err, "expected arithmetic sum formula" + + # test integer --update_to + main(sys.stderr, ['--update-to']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA + assert (str(N - 1) + "it").encode() in err + assert (str(N) + "it").encode() not in err + + with closing(BytesIO()) as sys.stdin: + sys.stdin.write(IN_DATA.replace(b'\n', b'D')) + + # test integer --update --delim + sys.stdin.seek(0) + main(sys.stderr, ['--update', '--delim', 'D']) + out, err = capsysbin.readouterr() + assert out == IN_DATA.replace(b'\n', b'D') + assert (str(N // 2 * N) + "it").encode() in err, "expected arithmetic sum" + + # test integer --update_to --delim + sys.stdin.seek(0) + main(sys.stderr, ['--update-to', '--delim', 'D']) + out, err = capsysbin.readouterr() + assert out == IN_DATA.replace(b'\n', b'D') + assert (str(N - 1) + "it").encode() in err + assert (str(N) + "it").encode() not in err + + # test float --update_to + sys.stdin = [(str(i / 2.0) + '\n').encode() for i in _range(N)] + IN_DATA = b''.join(sys.stdin) + main(sys.stderr, ['--update-to']) + out, err = capsysbin.readouterr() + assert norm(out) == IN_DATA + assert (str((N - 1) / 2.0) + "it").encode() in err + assert (str(N / 2.0) + "it").encode() not in err + + +@mark.slow +@mark.skipif(IS_WIN, reason="no manpages on windows") +def test_manpath(tmp_path): + """Test CLI --manpath""" + man = tmp_path / "tqdm.1" + assert not man.exists() + with raises(SystemExit): + main(argv=['--manpath', str(tmp_path)]) + assert man.is_file() + + +@mark.slow +@mark.skipif(IS_WIN, reason="no completion on windows") +def test_comppath(tmp_path): + """Test CLI --comppath""" + man = tmp_path / "tqdm_completion.sh" + assert not man.exists() + with raises(SystemExit): + main(argv=['--comppath', str(tmp_path)]) + assert man.is_file() + + # check most important options appear + script = man.read_text() + opts = {'--help', '--desc', '--total', '--leave', '--ncols', '--ascii', + '--dynamic_ncols', '--position', '--bytes', '--nrows', '--delim', + '--manpath', '--comppath'} + assert all(args in script for args in opts) + + +@restore_sys +def test_exceptions(capsysbin): + """Test CLI Exceptions""" + N = 123 + sys.stdin = [str(i) + '\n' for i in _range(N)] + IN_DATA = ''.join(sys.stdin).encode() + + with raises(TqdmKeyError, match="bad_arg_u_ment"): + main(sys.stderr, argv=['-ascii', '-unit_scale', '--bad_arg_u_ment', 'foo']) + out, _ = capsysbin.readouterr() + assert norm(out) == IN_DATA + + with raises(TqdmTypeError, match="invalid_bool_value"): + main(sys.stderr, argv=['-ascii', '-unit_scale', 'invalid_bool_value']) + out, _ = capsysbin.readouterr() + assert norm(out) == IN_DATA + + with raises(TqdmTypeError, match="invalid_int_value"): + main(sys.stderr, argv=['-ascii', '--total', 'invalid_int_value']) + out, _ = capsysbin.readouterr() + assert norm(out) == IN_DATA + + with raises(TqdmKeyError, match="Can only have one of --"): + main(sys.stderr, argv=['--update', '--update_to']) + out, _ = capsysbin.readouterr() + assert norm(out) == IN_DATA + + # test SystemExits + for i in ('-h', '--help', '-v', '--version'): + with raises(SystemExit): + main(argv=[i]) diff --git a/tests/tests_notebook.py b/tests/tests_notebook.py new file mode 100644 index 0000000..004d7e5 --- /dev/null +++ b/tests/tests_notebook.py @@ -0,0 +1,7 @@ +from tqdm.notebook import tqdm as tqdm_notebook + + +def test_notebook_disabled_description(): + """Test that set_description works for disabled tqdm_notebook""" + with tqdm_notebook(1, disable=True) as t: + t.set_description("description") diff --git a/tests/tests_pandas.py b/tests/tests_pandas.py new file mode 100644 index 0000000..334a97c --- /dev/null +++ b/tests/tests_pandas.py @@ -0,0 +1,219 @@ +from tqdm import tqdm + +from .tests_tqdm import StringIO, closing, importorskip, mark, skip + +pytestmark = mark.slow + +random = importorskip('numpy.random') +rand = random.rand +randint = random.randint +pd = importorskip('pandas') + + +def test_pandas_setup(): + """Test tqdm.pandas()""" + with closing(StringIO()) as our_file: + tqdm.pandas(file=our_file, leave=True, ascii=True, total=123) + series = pd.Series(randint(0, 50, (100,))) + series.progress_apply(lambda x: x + 10) + res = our_file.getvalue() + assert '100/123' in res + + +def test_pandas_rolling_expanding(): + """Test pandas.(Series|DataFrame).(rolling|expanding)""" + with closing(StringIO()) as our_file: + tqdm.pandas(file=our_file, leave=True, ascii=True) + + series = pd.Series(randint(0, 50, (123,))) + res1 = series.rolling(10).progress_apply(lambda x: 1, raw=True) + res2 = series.rolling(10).apply(lambda x: 1, raw=True) + assert res1.equals(res2) + + res3 = series.expanding(10).progress_apply(lambda x: 2, raw=True) + res4 = series.expanding(10).apply(lambda x: 2, raw=True) + assert res3.equals(res4) + + expects = ['114it'] # 123-10+1 + for exres in expects: + our_file.seek(0) + if our_file.getvalue().count(exres) < 2: + our_file.seek(0) + raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( + exres + " at least twice.", our_file.read())) + + +def test_pandas_series(): + """Test pandas.Series.progress_apply and .progress_map""" + with closing(StringIO()) as our_file: + tqdm.pandas(file=our_file, leave=True, ascii=True) + + series = pd.Series(randint(0, 50, (123,))) + res1 = series.progress_apply(lambda x: x + 10) + res2 = series.apply(lambda x: x + 10) + assert res1.equals(res2) + + res3 = series.progress_map(lambda x: x + 10) + res4 = series.map(lambda x: x + 10) + assert res3.equals(res4) + + expects = ['100%', '123/123'] + for exres in expects: + our_file.seek(0) + if our_file.getvalue().count(exres) < 2: + our_file.seek(0) + raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( + exres + " at least twice.", our_file.read())) + + +def test_pandas_data_frame(): + """Test pandas.DataFrame.progress_apply and .progress_applymap""" + with closing(StringIO()) as our_file: + tqdm.pandas(file=our_file, leave=True, ascii=True) + df = pd.DataFrame(randint(0, 50, (100, 200))) + + def task_func(x): + return x + 1 + + # applymap + res1 = df.progress_applymap(task_func) + res2 = df.applymap(task_func) + assert res1.equals(res2) + + # apply unhashable + res1 = [] + df.progress_apply(res1.extend) + assert len(res1) == df.size + + # apply + for axis in [0, 1, 'index', 'columns']: + res3 = df.progress_apply(task_func, axis=axis) + res4 = df.apply(task_func, axis=axis) + assert res3.equals(res4) + + our_file.seek(0) + if our_file.read().count('100%') < 3: + our_file.seek(0) + raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( + '100% at least three times', our_file.read())) + + # apply_map, apply axis=0, apply axis=1 + expects = ['20000/20000', '200/200', '100/100'] + for exres in expects: + our_file.seek(0) + if our_file.getvalue().count(exres) < 1: + our_file.seek(0) + raise AssertionError("\nExpected:\n{0}\nIn:\n {1}\n".format( + exres + " at least once.", our_file.read())) + + +def test_pandas_groupby_apply(): + """Test pandas.DataFrame.groupby(...).progress_apply""" + with closing(StringIO()) as our_file: + tqdm.pandas(file=our_file, leave=False, ascii=True) + + df = pd.DataFrame(randint(0, 50, (500, 3))) + df.groupby(0).progress_apply(lambda x: None) + + dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc')) + dfs.groupby(['a']).progress_apply(lambda x: None) + + df2 = df = pd.DataFrame({'a': randint(1, 8, 10000), 'b': rand(10000)}) + res1 = df2.groupby("a").apply(max) + res2 = df2.groupby("a").progress_apply(max) + assert res1.equals(res2) + + our_file.seek(0) + + # don't expect final output since no `leave` and + # high dynamic `miniters` + nexres = '100%|##########|' + if nexres in our_file.read(): + our_file.seek(0) + raise AssertionError("\nDid not expect:\n{0}\nIn:{1}\n".format( + nexres, our_file.read())) + + with closing(StringIO()) as our_file: + tqdm.pandas(file=our_file, leave=True, ascii=True) + + dfs = pd.DataFrame(randint(0, 50, (500, 3)), columns=list('abc')) + dfs.loc[0] = [2, 1, 1] + dfs['d'] = 100 + + expects = ['500/500', '1/1', '4/4', '2/2'] + dfs.groupby(dfs.index).progress_apply(lambda x: None) + dfs.groupby('d').progress_apply(lambda x: None) + dfs.groupby(dfs.columns, axis=1).progress_apply(lambda x: None) + dfs.groupby([2, 2, 1, 1], axis=1).progress_apply(lambda x: None) + + our_file.seek(0) + if our_file.read().count('100%') < 4: + our_file.seek(0) + raise AssertionError("\nExpected:\n{0}\nIn:\n{1}\n".format( + '100% at least four times', our_file.read())) + + for exres in expects: + our_file.seek(0) + if our_file.getvalue().count(exres) < 1: + our_file.seek(0) + raise AssertionError("\nExpected:\n{0}\nIn:\n {1}\n".format( + exres + " at least once.", our_file.read())) + + +def test_pandas_leave(): + """Test pandas with `leave=True`""" + with closing(StringIO()) as our_file: + df = pd.DataFrame(randint(0, 100, (1000, 6))) + tqdm.pandas(file=our_file, leave=True, ascii=True) + df.groupby(0).progress_apply(lambda x: None) + + our_file.seek(0) + + exres = '100%|##########| 100/100' + if exres not in our_file.read(): + our_file.seek(0) + raise AssertionError("\nExpected:\n{0}\nIn:{1}\n".format( + exres, our_file.read())) + + +def test_pandas_apply_args_deprecation(): + """Test warning info in + `pandas.Dataframe(Series).progress_apply(func, *args)`""" + try: + from tqdm import tqdm_pandas + except ImportError as err: + skip(str(err)) + + with closing(StringIO()) as our_file: + tqdm_pandas(tqdm(file=our_file, leave=False, ascii=True, ncols=20)) + df = pd.DataFrame(randint(0, 50, (500, 3))) + df.progress_apply(lambda x: None, 1) # 1 shall cause a warning + # Check deprecation message + res = our_file.getvalue() + assert all(i in res for i in ( + "TqdmDeprecationWarning", "not supported", + "keyword arguments instead")) + + +def test_pandas_deprecation(): + """Test bar object instance as argument deprecation""" + try: + from tqdm import tqdm_pandas + except ImportError as err: + skip(str(err)) + + with closing(StringIO()) as our_file: + tqdm_pandas(tqdm(file=our_file, leave=False, ascii=True, ncols=20)) + df = pd.DataFrame(randint(0, 50, (500, 3))) + df.groupby(0).progress_apply(lambda x: None) + # Check deprecation message + assert "TqdmDeprecationWarning" in our_file.getvalue() + assert "instead of `tqdm_pandas(tqdm(...))`" in our_file.getvalue() + + with closing(StringIO()) as our_file: + tqdm_pandas(tqdm, file=our_file, leave=False, ascii=True, ncols=20) + df = pd.DataFrame(randint(0, 50, (500, 3))) + df.groupby(0).progress_apply(lambda x: None) + # Check deprecation message + assert "TqdmDeprecationWarning" in our_file.getvalue() + assert "instead of `tqdm_pandas(tqdm, ...)`" in our_file.getvalue() diff --git a/tests/tests_perf.py b/tests/tests_perf.py new file mode 100644 index 0000000..552a169 --- /dev/null +++ b/tests/tests_perf.py @@ -0,0 +1,325 @@ +from __future__ import division, print_function + +import sys +from contextlib import contextmanager +from functools import wraps +from time import sleep, time + +# Use relative/cpu timer to have reliable timings when there is a sudden load +try: + from time import process_time +except ImportError: + from time import clock + process_time = clock + +from tqdm import tqdm, trange + +from .tests_tqdm import _range, importorskip, mark, patch_lock, skip + +pytestmark = mark.slow + + +def cpu_sleep(t): + """Sleep the given amount of cpu time""" + start = process_time() + while (process_time() - start) < t: + pass + + +def checkCpuTime(sleeptime=0.2): + """Check if cpu time works correctly""" + if checkCpuTime.passed: + return True + # First test that sleeping does not consume cputime + start1 = process_time() + sleep(sleeptime) + t1 = process_time() - start1 + + # secondly check by comparing to cpusleep (where we actually do something) + start2 = process_time() + cpu_sleep(sleeptime) + t2 = process_time() - start2 + + if abs(t1) < 0.0001 and t1 < t2 / 10: + checkCpuTime.passed = True + return True + skip("cpu time not reliable on this machine") + + +checkCpuTime.passed = False + + +@contextmanager +def relative_timer(): + """yields a context timer function which stops ticking on exit""" + start = process_time() + + def elapser(): + return process_time() - start + + yield lambda: elapser() + spent = elapser() + + def elapser(): # NOQA + return spent + + +def retry_on_except(n=3, check_cpu_time=True): + """decroator for retrying `n` times before raising Exceptions""" + def wrapper(func): + """actual decorator""" + @wraps(func) + def test_inner(*args, **kwargs): + """may skip if `check_cpu_time` fails""" + for i in range(1, n + 1): + try: + if check_cpu_time: + checkCpuTime() + func(*args, **kwargs) + except Exception: + if i >= n: + raise + else: + return + return test_inner + return wrapper + + +def simple_progress(iterable=None, total=None, file=sys.stdout, desc='', + leave=False, miniters=1, mininterval=0.1, width=60): + """Simple progress bar reproducing tqdm's major features""" + n = [0] # use a closure + start_t = [time()] + last_n = [0] + last_t = [0] + if iterable is not None: + total = len(iterable) + + def format_interval(t): + mins, s = divmod(int(t), 60) + h, m = divmod(mins, 60) + if h: + return '{0:d}:{1:02d}:{2:02d}'.format(h, m, s) + else: + return '{0:02d}:{1:02d}'.format(m, s) + + def update_and_print(i=1): + n[0] += i + if (n[0] - last_n[0]) >= miniters: + last_n[0] = n[0] + + if (time() - last_t[0]) >= mininterval: + last_t[0] = time() # last_t[0] == current time + + spent = last_t[0] - start_t[0] + spent_fmt = format_interval(spent) + rate = n[0] / spent if spent > 0 else 0 + rate_fmt = "%.2fs/it" % (1.0 / rate) if 0.0 < rate < 1.0 else "%.2fit/s" % rate + + frac = n[0] / total + percentage = int(frac * 100) + eta = (total - n[0]) / rate if rate > 0 else 0 + eta_fmt = format_interval(eta) + + # full_bar = "#" * int(frac * width) + barfill = " " * int((1.0 - frac) * width) + bar_length, frac_bar_length = divmod(int(frac * width * 10), 10) + full_bar = '#' * bar_length + frac_bar = chr(48 + frac_bar_length) if frac_bar_length else ' ' + + file.write("\r%s %i%%|%s%s%s| %i/%i [%s<%s, %s]" % + (desc, percentage, full_bar, frac_bar, barfill, n[0], + total, spent_fmt, eta_fmt, rate_fmt)) + + if n[0] == total and leave: + file.write("\n") + file.flush() + + def update_and_yield(): + for elt in iterable: + yield elt + update_and_print() + + update_and_print(0) + if iterable is not None: + return update_and_yield() + else: + return update_and_print + + +def assert_performance(thresh, name_left, time_left, name_right, time_right): + """raises if time_left > thresh * time_right""" + if time_left > thresh * time_right: + raise ValueError( + ('{name[0]}: {time[0]:f}, ' + '{name[1]}: {time[1]:f}, ' + 'ratio {ratio:f} > {thresh:f}').format( + name=(name_left, name_right), + time=(time_left, time_right), + ratio=time_left / time_right, thresh=thresh)) + + +@retry_on_except() +def test_iter_basic_overhead(): + """Test overhead of iteration based tqdm""" + total = int(1e6) + + a = 0 + with trange(total) as t: + with relative_timer() as time_tqdm: + for i in t: + a += i + assert a == (total ** 2 - total) / 2.0 + + a = 0 + with relative_timer() as time_bench: + for i in _range(total): + a += i + sys.stdout.write(str(a)) + + assert_performance(3, 'trange', time_tqdm(), 'range', time_bench()) + + +@retry_on_except() +def test_manual_basic_overhead(): + """Test overhead of manual tqdm""" + total = int(1e6) + + with tqdm(total=total * 10, leave=True) as t: + a = 0 + with relative_timer() as time_tqdm: + for i in _range(total): + a += i + t.update(10) + + a = 0 + with relative_timer() as time_bench: + for i in _range(total): + a += i + sys.stdout.write(str(a)) + + assert_performance(5, 'tqdm', time_tqdm(), 'range', time_bench()) + + +def worker(total, blocking=True): + def incr_bar(x): + for _ in trange(total, lock_args=None if blocking else (False,), + miniters=1, mininterval=0, maxinterval=0): + pass + return x + 1 + return incr_bar + + +@retry_on_except() +@patch_lock(thread=True) +def test_lock_args(): + """Test overhead of nonblocking threads""" + ThreadPoolExecutor = importorskip('concurrent.futures').ThreadPoolExecutor + + total = 16 + subtotal = 10000 + + with ThreadPoolExecutor() as pool: + sys.stderr.write('block ... ') + sys.stderr.flush() + with relative_timer() as time_tqdm: + res = list(pool.map(worker(subtotal, True), range(total))) + assert sum(res) == sum(range(total)) + total + sys.stderr.write('noblock ... ') + sys.stderr.flush() + with relative_timer() as time_noblock: + res = list(pool.map(worker(subtotal, False), range(total))) + assert sum(res) == sum(range(total)) + total + + assert_performance(0.5, 'noblock', time_noblock(), 'tqdm', time_tqdm()) + + +@retry_on_except(10) +def test_iter_overhead_hard(): + """Test overhead of iteration based tqdm (hard)""" + total = int(1e5) + + a = 0 + with trange(total, leave=True, miniters=1, + mininterval=0, maxinterval=0) as t: + with relative_timer() as time_tqdm: + for i in t: + a += i + assert a == (total ** 2 - total) / 2.0 + + a = 0 + with relative_timer() as time_bench: + for i in _range(total): + a += i + sys.stdout.write(("%i" % a) * 40) + + assert_performance(130, 'trange', time_tqdm(), 'range', time_bench()) + + +@retry_on_except(10) +def test_manual_overhead_hard(): + """Test overhead of manual tqdm (hard)""" + total = int(1e5) + + with tqdm(total=total * 10, leave=True, miniters=1, + mininterval=0, maxinterval=0) as t: + a = 0 + with relative_timer() as time_tqdm: + for i in _range(total): + a += i + t.update(10) + + a = 0 + with relative_timer() as time_bench: + for i in _range(total): + a += i + sys.stdout.write(("%i" % a) * 40) + + assert_performance(130, 'tqdm', time_tqdm(), 'range', time_bench()) + + +@retry_on_except(10) +def test_iter_overhead_simplebar_hard(): + """Test overhead of iteration based tqdm vs simple progress bar (hard)""" + total = int(1e4) + + a = 0 + with trange(total, leave=True, miniters=1, + mininterval=0, maxinterval=0) as t: + with relative_timer() as time_tqdm: + for i in t: + a += i + assert a == (total ** 2 - total) / 2.0 + + a = 0 + s = simple_progress(_range(total), leave=True, + miniters=1, mininterval=0) + with relative_timer() as time_bench: + for i in s: + a += i + + assert_performance(10, 'trange', time_tqdm(), 'simple_progress', time_bench()) + + +@retry_on_except(10) +def test_manual_overhead_simplebar_hard(): + """Test overhead of manual tqdm vs simple progress bar (hard)""" + total = int(1e4) + + with tqdm(total=total * 10, leave=True, miniters=1, + mininterval=0, maxinterval=0) as t: + a = 0 + with relative_timer() as time_tqdm: + for i in _range(total): + a += i + t.update(10) + + simplebar_update = simple_progress(total=total * 10, leave=True, + miniters=1, mininterval=0) + a = 0 + with relative_timer() as time_bench: + for i in _range(total): + a += i + simplebar_update(10) + + assert_performance(10, 'tqdm', time_tqdm(), 'simple_progress', time_bench()) diff --git a/tests/tests_rich.py b/tests/tests_rich.py new file mode 100644 index 0000000..c75e246 --- /dev/null +++ b/tests/tests_rich.py @@ -0,0 +1,10 @@ +"""Test `tqdm.rich`.""" +import sys + +from .tests_tqdm import importorskip, mark + + +@mark.skipif(sys.version_info[:3] < (3, 6, 1), reason="`rich` needs py>=3.6.1") +def test_rich_import(): + """Test `tqdm.rich` import""" + importorskip('tqdm.rich') diff --git a/tests/tests_synchronisation.py b/tests/tests_synchronisation.py new file mode 100644 index 0000000..7ee55fb --- /dev/null +++ b/tests/tests_synchronisation.py @@ -0,0 +1,224 @@ +from __future__ import division + +import sys +from functools import wraps +from threading import Event +from time import sleep, time + +from tqdm import TMonitor, tqdm, trange + +from .tests_perf import retry_on_except +from .tests_tqdm import StringIO, closing, importorskip, patch_lock, skip + + +class Time(object): + """Fake time class class providing an offset""" + offset = 0 + + @classmethod + def reset(cls): + """zeroes internal offset""" + cls.offset = 0 + + @classmethod + def time(cls): + """time.time() + offset""" + return time() + cls.offset + + @staticmethod + def sleep(dur): + """identical to time.sleep()""" + sleep(dur) + + @classmethod + def fake_sleep(cls, dur): + """adds `dur` to internal offset""" + cls.offset += dur + sleep(0.000001) # sleep to allow interrupt (instead of pass) + + +def FakeEvent(): + """patched `threading.Event` where `wait()` uses `Time.fake_sleep()`""" + event = Event() # not a class in py2 so can't inherit + + def wait(timeout=None): + """uses Time.fake_sleep""" + if timeout is not None: + Time.fake_sleep(timeout) + return event.is_set() + + event.wait = wait + return event + + +def patch_sleep(func): + """Temporarily makes TMonitor use Time.fake_sleep""" + @wraps(func) + def inner(*args, **kwargs): + """restores TMonitor on completion regardless of Exceptions""" + TMonitor._test["time"] = Time.time + TMonitor._test["Event"] = FakeEvent + if tqdm.monitor: + assert not tqdm.monitor.get_instances() + tqdm.monitor.exit() + del tqdm.monitor + tqdm.monitor = None + try: + return func(*args, **kwargs) + finally: + # Check that class var monitor is deleted if no instance left + tqdm.monitor_interval = 10 + if tqdm.monitor: + assert not tqdm.monitor.get_instances() + tqdm.monitor.exit() + del tqdm.monitor + tqdm.monitor = None + TMonitor._test.pop("Event") + TMonitor._test.pop("time") + + return inner + + +def cpu_timify(t, timer=Time): + """Force tqdm to use the specified timer instead of system-wide time""" + t._time = timer.time + t._sleep = timer.fake_sleep + t.start_t = t.last_print_t = t._time() + return timer + + +class FakeTqdm(object): + _instances = set() + get_lock = tqdm.get_lock + + +def incr(x): + return x + 1 + + +def incr_bar(x): + with closing(StringIO()) as our_file: + for _ in trange(x, lock_args=(False,), file=our_file): + pass + return incr(x) + + +@patch_sleep +def test_monitor_thread(): + """Test dummy monitoring thread""" + monitor = TMonitor(FakeTqdm, 10) + # Test if alive, then killed + assert monitor.report() + monitor.exit() + assert not monitor.report() + assert not monitor.is_alive() + del monitor + + +@patch_sleep +def test_monitoring_and_cleanup(): + """Test for stalled tqdm instance and monitor deletion""" + # Note: should fix miniters for these tests, else with dynamic_miniters + # it's too complicated to handle with monitoring update and maxinterval... + maxinterval = tqdm.monitor_interval + assert maxinterval == 10 + total = 1000 + + with closing(StringIO()) as our_file: + with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, + maxinterval=maxinterval) as t: + cpu_timify(t, Time) + # Do a lot of iterations in a small timeframe + # (smaller than monitor interval) + Time.fake_sleep(maxinterval / 10) # monitor won't wake up + t.update(500) + # check that our fixed miniters is still there + assert t.miniters <= 500 # TODO: should really be == 500 + # Then do 1 it after monitor interval, so that monitor kicks in + Time.fake_sleep(maxinterval) + t.update(1) + # Wait for the monitor to get out of sleep's loop and update tqdm. + timeend = Time.time() + while not (t.monitor.woken >= timeend and t.miniters == 1): + Time.fake_sleep(1) # Force awake up if it woken too soon + assert t.miniters == 1 # check that monitor corrected miniters + # Note: at this point, there may be a race condition: monitor saved + # current woken time but Time.sleep() happen just before monitor + # sleep. To fix that, either sleep here or increase time in a loop + # to ensure that monitor wakes up at some point. + + # Try again but already at miniters = 1 so nothing will be done + Time.fake_sleep(maxinterval) + t.update(2) + timeend = Time.time() + while t.monitor.woken < timeend: + Time.fake_sleep(1) # Force awake if it woken too soon + # Wait for the monitor to get out of sleep's loop and update + # tqdm + assert t.miniters == 1 # check that monitor corrected miniters + + +@patch_sleep +def test_monitoring_multi(): + """Test on multiple bars, one not needing miniters adjustment""" + # Note: should fix miniters for these tests, else with dynamic_miniters + # it's too complicated to handle with monitoring update and maxinterval... + maxinterval = tqdm.monitor_interval + assert maxinterval == 10 + total = 1000 + + with closing(StringIO()) as our_file: + with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, + maxinterval=maxinterval) as t1: + # Set high maxinterval for t2 so monitor does not need to adjust it + with tqdm(total=total, file=our_file, miniters=500, mininterval=0.1, + maxinterval=1E5) as t2: + cpu_timify(t1, Time) + cpu_timify(t2, Time) + # Do a lot of iterations in a small timeframe + Time.fake_sleep(maxinterval / 10) + t1.update(500) + t2.update(500) + assert t1.miniters <= 500 # TODO: should really be == 500 + assert t2.miniters == 500 + # Then do 1 it after monitor interval, so that monitor kicks in + Time.fake_sleep(maxinterval) + t1.update(1) + t2.update(1) + # Wait for the monitor to get out of sleep and update tqdm + timeend = Time.time() + while not (t1.monitor.woken >= timeend and t1.miniters == 1): + Time.fake_sleep(1) + assert t1.miniters == 1 # check that monitor corrected miniters + assert t2.miniters == 500 # check that t2 was not adjusted + + +def test_imap(): + """Test multiprocessing.Pool""" + try: + from multiprocessing import Pool + except ImportError as err: + skip(str(err)) + + pool = Pool() + res = list(tqdm(pool.imap(incr, range(100)), disable=True)) + pool.close() + assert res[-1] == 100 + + +# py2: locks won't propagate to incr_bar so may cause `AttributeError` +@retry_on_except(n=3 if sys.version_info < (3,) else 1, check_cpu_time=False) +@patch_lock(thread=True) +def test_threadpool(): + """Test concurrent.futures.ThreadPoolExecutor""" + ThreadPoolExecutor = importorskip('concurrent.futures').ThreadPoolExecutor + + with ThreadPoolExecutor(8) as pool: + try: + res = list(tqdm(pool.map(incr_bar, range(100)), disable=True)) + except AttributeError: + if sys.version_info < (3,): + skip("not supported on py2") + else: + raise + assert sum(res) == sum(range(1, 101)) diff --git a/tests/tests_tk.py b/tests/tests_tk.py new file mode 100644 index 0000000..9aa645c --- /dev/null +++ b/tests/tests_tk.py @@ -0,0 +1,7 @@ +"""Test `tqdm.tk`.""" +from .tests_tqdm import importorskip + + +def test_tk_import(): + """Test `tqdm.tk` import""" + importorskip('tqdm.tk') diff --git a/tests/tests_tqdm.py b/tests/tests_tqdm.py new file mode 100644 index 0000000..bba457a --- /dev/null +++ b/tests/tests_tqdm.py @@ -0,0 +1,1996 @@ +# -*- coding: utf-8 -*- +# Advice: use repr(our_file.read()) to print the full output of tqdm +# (else '\r' will replace the previous lines and you'll see only the latest. +from __future__ import print_function + +import csv +import os +import re +import sys +from contextlib import contextmanager +from functools import wraps +from warnings import catch_warnings, simplefilter + +from pytest import importorskip, mark, raises, skip + +from tqdm import TqdmDeprecationWarning, TqdmWarning, tqdm, trange +from tqdm.contrib import DummyTqdmFile +from tqdm.std import EMA, Bar + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +from io import IOBase # to support unicode strings +from io import BytesIO + + +class DeprecationError(Exception): + pass + + +# Ensure we can use `with closing(...) as ... :` syntax +if getattr(StringIO, '__exit__', False) and getattr(StringIO, '__enter__', False): + def closing(arg): + return arg +else: + from contextlib import closing + +try: + _range = xrange +except NameError: + _range = range + +try: + _unicode = unicode +except NameError: + _unicode = str + +nt_and_no_colorama = False +if os.name == 'nt': + try: + import colorama # NOQA + except ImportError: + nt_and_no_colorama = True + +# Regex definitions +# List of control characters +CTRLCHR = [r'\r', r'\n', r'\x1b\[A'] # Need to escape [ for regex +# Regular expressions compilation +RE_rate = re.compile(r'[^\d](\d[.\d]+)it/s') +RE_ctrlchr = re.compile("(%s)" % '|'.join(CTRLCHR)) # Match control chars +RE_ctrlchr_excl = re.compile('|'.join(CTRLCHR)) # Match and exclude ctrl chars +RE_pos = re.compile(r'([\r\n]+((pos\d+) bar:\s+\d+%|\s{3,6})?[^\r\n]*)') + + +def pos_line_diff(res_list, expected_list, raise_nonempty=True): + """ + Return differences between two bar output lists. + To be used with `RE_pos` + """ + res = [(r, e) for r, e in zip(res_list, expected_list) + for pos in [len(e) - len(e.lstrip('\n'))] # bar position + if r != e # simple comparison + if not r.startswith(e) # start matches + or not ( + # move up at end (maybe less due to closing bars) + any(r.endswith(end + i * '\x1b[A') for i in range(pos + 1) + for end in [ + ']', # bar + ' ']) # cleared + or '100%' in r # completed bar + or r == '\n') # final bar + or r[(-1 - pos) * len('\x1b[A'):] == '\x1b[A'] # too many moves up + if raise_nonempty and (res or len(res_list) != len(expected_list)): + if len(res_list) < len(expected_list): + res.extend([(None, e) for e in expected_list[len(res_list):]]) + elif len(res_list) > len(expected_list): + res.extend([(r, None) for r in res_list[len(expected_list):]]) + raise AssertionError( + "Got => Expected\n" + '\n'.join('%r => %r' % i for i in res)) + return res + + +class DiscreteTimer(object): + """Virtual discrete time manager, to precisely control time for tests""" + def __init__(self): + self.t = 0.0 + + def sleep(self, t): + """Sleep = increment the time counter (almost no CPU used)""" + self.t += t + + def time(self): + """Get the current time""" + return self.t + + +def cpu_timify(t, timer=None): + """Force tqdm to use the specified timer instead of system-wide time()""" + if timer is None: + timer = DiscreteTimer() + t._time = timer.time + t._sleep = timer.sleep + t.start_t = t.last_print_t = t._time() + return timer + + +class UnicodeIO(IOBase): + """Unicode version of StringIO""" + def __init__(self, *args, **kwargs): + super(UnicodeIO, self).__init__(*args, **kwargs) + self.encoding = 'U8' # io.StringIO supports unicode, but no encoding + self.text = '' + self.cursor = 0 + + def __len__(self): + return len(self.text) + + def seek(self, offset): + self.cursor = offset + + def tell(self): + return self.cursor + + def write(self, s): + self.text = self.text[:self.cursor] + s + self.text[self.cursor + len(s):] + self.cursor += len(s) + + def read(self, n=-1): + _cur = self.cursor + self.cursor = len(self) if n < 0 else min(_cur + n, len(self)) + return self.text[_cur:self.cursor] + + def getvalue(self): + return self.text + + +def get_bar(all_bars, i=None): + """Get a specific update from a whole bar traceback""" + # Split according to any used control characters + bars_split = RE_ctrlchr_excl.split(all_bars) + bars_split = list(filter(None, bars_split)) # filter out empty splits + return bars_split if i is None else bars_split[i] + + +def progressbar_rate(bar_str): + return float(RE_rate.search(bar_str).group(1)) + + +def squash_ctrlchars(s): + """Apply control characters in a string just like a terminal display""" + curline = 0 + lines = [''] # state of fake terminal + for nextctrl in filter(None, RE_ctrlchr.split(s)): + # apply control chars + if nextctrl == '\r': + # go to line beginning (simplified here: just empty the string) + lines[curline] = '' + elif nextctrl == '\n': + if curline >= len(lines) - 1: + # wrap-around creates newline + lines.append('') + # move cursor down + curline += 1 + elif nextctrl == '\x1b[A': + # move cursor up + if curline > 0: + curline -= 1 + else: + raise ValueError("Cannot go further up") + else: + # print message on current line + lines[curline] += nextctrl + return lines + + +def test_format_interval(): + """Test time interval format""" + format_interval = tqdm.format_interval + + assert format_interval(60) == '01:00' + assert format_interval(6160) == '1:42:40' + assert format_interval(238113) == '66:08:33' + + +def test_format_num(): + """Test number format""" + format_num = tqdm.format_num + + assert float(format_num(1337)) == 1337 + assert format_num(int(1e6)) == '1e+6' + assert format_num(1239876) == '1' '239' '876' + + +def test_format_meter(): + """Test statistics and progress bar formatting""" + try: + unich = unichr + except NameError: + unich = chr + + format_meter = tqdm.format_meter + + assert format_meter(0, 1000, 13) == " 0%| | 0/1000 [00:13<?, ?it/s]" + # If not implementing any changes to _tqdm.py, set prefix='desc' + # or else ": : " will be in output, so assertion should change + assert format_meter(0, 1000, 13, ncols=68, prefix='desc: ') == ( + "desc: 0%| | 0/1000 [00:13<?, ?it/s]") + assert format_meter(231, 1000, 392) == (" 23%|" + unich(0x2588) * 2 + unich(0x258e) + + " | 231/1000 [06:32<21:44, 1.70s/it]") + assert format_meter(10000, 1000, 13) == "10000it [00:13, 769.23it/s]" + assert format_meter(231, 1000, 392, ncols=56, ascii=True) == " 23%|" + '#' * 3 + '6' + ( + " | 231/1000 [06:32<21:44, 1.70s/it]") + assert format_meter(100000, 1000, 13, unit_scale=True, + unit='iB') == "100kiB [00:13, 7.69kiB/s]" + assert format_meter(100, 1000, 12, ncols=0, + rate=7.33) == " 10% 100/1000 [00:12<02:02, 7.33it/s]" + # ncols is small, l_bar is too large + # l_bar gets chopped + # no bar + # no r_bar + # 10/12 stars since ncols is 10 + assert format_meter( + 0, 1000, 13, ncols=10, + bar_format="************{bar:10}$$$$$$$$$$") == "**********" + # n_cols allows for l_bar and some of bar + # l_bar displays + # bar gets chopped + # no r_bar + # all 12 stars and 8/10 bar parts + assert format_meter( + 0, 1000, 13, ncols=20, + bar_format="************{bar:10}$$$$$$$$$$") == "************ " + # n_cols allows for l_bar, bar, and some of r_bar + # l_bar displays + # bar displays + # r_bar gets chopped + # all 12 stars and 10 bar parts, but only 8/10 dollar signs + assert format_meter( + 0, 1000, 13, ncols=30, + bar_format="************{bar:10}$$$$$$$$$$") == "************ $$$$$$$$" + # trim left ANSI; escape is before trim zone + # we only know it has ANSI codes, so we append an END code anyway + assert format_meter( + 0, 1000, 13, ncols=10, bar_format="*****\033[22m****\033[0m***{bar:10}$$$$$$$$$$" + ) == "*****\033[22m****\033[0m*\033[0m" + # trim left ANSI; escape is at trim zone + assert format_meter( + 0, 1000, 13, ncols=10, + bar_format="*****\033[22m*****\033[0m**{bar:10}$$$$$$$$$$") == "*****\033[22m*****\033[0m" + # trim left ANSI; escape is after trim zone + assert format_meter( + 0, 1000, 13, ncols=10, + bar_format="*****\033[22m******\033[0m*{bar:10}$$$$$$$$$$") == "*****\033[22m*****\033[0m" + # Check that bar_format correctly adapts {bar} size to the rest + assert format_meter( + 20, 100, 12, ncols=13, rate=8.1, + bar_format=r'{l_bar}{bar}|{n_fmt}/{total_fmt}') == " 20%|" + unich(0x258f) + "|20/100" + assert format_meter( + 20, 100, 12, ncols=14, rate=8.1, + bar_format=r'{l_bar}{bar}|{n_fmt}/{total_fmt}') == " 20%|" + unich(0x258d) + " |20/100" + # Check wide characters + if sys.version_info >= (3,): + assert format_meter(0, 1000, 13, ncols=68, prefix='ｆｕｌｌｗｉｄｔｈ: ') == ( + "ｆｕｌｌｗｉｄｔｈ: 0%| | 0/1000 [00:13<?, ?it/s]") + assert format_meter(0, 1000, 13, ncols=68, prefix='ニッポン [ﾆｯﾎﾟﾝ]: ') == ( + "ニッポン [ﾆｯﾎﾟﾝ]: 0%| | 0/1000 [00:13<?, ?it/s]") + # Check that bar_format can print only {bar} or just one side + assert format_meter(20, 100, 12, ncols=2, rate=8.1, + bar_format=r'{bar}') == unich(0x258d) + " " + assert format_meter(20, 100, 12, ncols=7, rate=8.1, + bar_format=r'{l_bar}{bar}') == " 20%|" + unich(0x258d) + " " + assert format_meter(20, 100, 12, ncols=6, rate=8.1, + bar_format=r'{bar}|test') == unich(0x258f) + "|test" + + +def test_ansi_escape_codes(): + """Test stripping of ANSI escape codes""" + ansi = {'BOLD': '\033[1m', 'RED': '\033[91m', 'END': '\033[0m'} + desc_raw = '{BOLD}{RED}Colored{END} description' + ncols = 123 + + desc_stripped = desc_raw.format(BOLD='', RED='', END='') + meter = tqdm.format_meter(0, 100, 0, ncols=ncols, prefix=desc_stripped) + assert len(meter) == ncols + + desc = desc_raw.format(**ansi) + meter = tqdm.format_meter(0, 100, 0, ncols=ncols, prefix=desc) + # `format_meter` inserts an extra END for safety + ansi_len = len(desc) - len(desc_stripped) + len(ansi['END']) + assert len(meter) == ncols + ansi_len + + +def test_si_format(): + """Test SI unit prefixes""" + format_meter = tqdm.format_meter + + assert '9.00 ' in format_meter(1, 9, 1, unit_scale=True, unit='B') + assert '99.0 ' in format_meter(1, 99, 1, unit_scale=True) + assert '999 ' in format_meter(1, 999, 1, unit_scale=True) + assert '9.99k ' in format_meter(1, 9994, 1, unit_scale=True) + assert '10.0k ' in format_meter(1, 9999, 1, unit_scale=True) + assert '99.5k ' in format_meter(1, 99499, 1, unit_scale=True) + assert '100k ' in format_meter(1, 99999, 1, unit_scale=True) + assert '1.00M ' in format_meter(1, 999999, 1, unit_scale=True) + assert '1.00G ' in format_meter(1, 999999999, 1, unit_scale=True) + assert '1.00T ' in format_meter(1, 999999999999, 1, unit_scale=True) + assert '1.00P ' in format_meter(1, 999999999999999, 1, unit_scale=True) + assert '1.00E ' in format_meter(1, 999999999999999999, 1, unit_scale=True) + assert '1.00Z ' in format_meter(1, 999999999999999999999, 1, unit_scale=True) + assert '1.0Y ' in format_meter(1, 999999999999999999999999, 1, unit_scale=True) + assert '10.0Y ' in format_meter(1, 9999999999999999999999999, 1, unit_scale=True) + assert '100.0Y ' in format_meter(1, 99999999999999999999999999, 1, unit_scale=True) + assert '1000.0Y ' in format_meter(1, 999999999999999999999999999, 1, + unit_scale=True) + + +def test_bar_formatspec(): + """Test Bar.__format__ spec""" + assert "{0:5a}".format(Bar(0.3)) == "#5 " + assert "{0:2}".format(Bar(0.5, charset=" .oO0")) == "0 " + assert "{0:2a}".format(Bar(0.5, charset=" .oO0")) == "# " + assert "{0:-6a}".format(Bar(0.5, 10)) == '## ' + assert "{0:2b}".format(Bar(0.5, 10)) == ' ' + + +def test_all_defaults(): + """Test default kwargs""" + with closing(UnicodeIO()) as our_file: + with tqdm(range(10), file=our_file) as progressbar: + assert len(progressbar) == 10 + for _ in progressbar: + pass + # restore stdout/stderr output for `nosetest` interface + # try: + # sys.stderr.write('\x1b[A') + # except: + # pass + sys.stderr.write('\rTest default kwargs ... ') + + +class WriteTypeChecker(BytesIO): + """File-like to assert the expected type is written""" + def __init__(self, expected_type): + super(WriteTypeChecker, self).__init__() + self.expected_type = expected_type + + def write(self, s): + assert isinstance(s, self.expected_type) + + +def test_native_string_io_for_default_file(): + """Native strings written to unspecified files""" + stderr = sys.stderr + try: + sys.stderr = WriteTypeChecker(expected_type=type('')) + for _ in tqdm(range(3)): + pass + sys.stderr.encoding = None # py2 behaviour + for _ in tqdm(range(3)): + pass + finally: + sys.stderr = stderr + + +def test_unicode_string_io_for_specified_file(): + """Unicode strings written to specified files""" + for _ in tqdm(range(3), file=WriteTypeChecker(expected_type=type(u''))): + pass + + +def test_write_bytes(): + """Test write_bytes argument with and without `file`""" + # specified file (and bytes) + for _ in tqdm(range(3), file=WriteTypeChecker(expected_type=type(b'')), + write_bytes=True): + pass + # unspecified file (and unicode) + stderr = sys.stderr + try: + sys.stderr = WriteTypeChecker(expected_type=type(u'')) + for _ in tqdm(range(3), write_bytes=False): + pass + finally: + sys.stderr = stderr + + +def test_iterate_over_csv_rows(): + """Test csv iterator""" + # Create a test csv pseudo file + with closing(StringIO()) as test_csv_file: + writer = csv.writer(test_csv_file) + for _ in _range(3): + writer.writerow(['test'] * 3) + test_csv_file.seek(0) + + # Test that nothing fails if we iterate over rows + reader = csv.DictReader(test_csv_file, fieldnames=('row1', 'row2', 'row3')) + with closing(StringIO()) as our_file: + for _ in tqdm(reader, file=our_file): + pass + + +def test_file_output(): + """Test output to arbitrary file-like objects""" + with closing(StringIO()) as our_file: + for i in tqdm(_range(3), file=our_file): + if i == 1: + our_file.seek(0) + assert '0/3' in our_file.read() + + +def test_leave_option(): + """Test `leave=True` always prints info about the last iteration""" + with closing(StringIO()) as our_file: + for _ in tqdm(_range(3), file=our_file, leave=True): + pass + res = our_file.getvalue() + assert '| 3/3 ' in res + assert '\n' == res[-1] # not '\r' + + with closing(StringIO()) as our_file2: + for _ in tqdm(_range(3), file=our_file2, leave=False): + pass + assert '| 3/3 ' not in our_file2.getvalue() + + +def test_trange(): + """Test trange""" + with closing(StringIO()) as our_file: + for _ in trange(3, file=our_file, leave=True): + pass + assert '| 3/3 ' in our_file.getvalue() + + with closing(StringIO()) as our_file2: + for _ in trange(3, file=our_file2, leave=False): + pass + assert '| 3/3 ' not in our_file2.getvalue() + + +def test_min_interval(): + """Test mininterval""" + with closing(StringIO()) as our_file: + for _ in tqdm(_range(3), file=our_file, mininterval=1e-10): + pass + assert " 0%| | 0/3 [00:00<" in our_file.getvalue() + + +def test_max_interval(): + """Test maxinterval""" + total = 100 + bigstep = 10 + smallstep = 5 + + # Test without maxinterval + timer = DiscreteTimer() + with closing(StringIO()) as our_file: + with closing(StringIO()) as our_file2: + # with maxinterval but higher than loop sleep time + t = tqdm(total=total, file=our_file, miniters=None, mininterval=0, + smoothing=1, maxinterval=1e-2) + cpu_timify(t, timer) + + # without maxinterval + t2 = tqdm(total=total, file=our_file2, miniters=None, mininterval=0, + smoothing=1, maxinterval=None) + cpu_timify(t2, timer) + + assert t.dynamic_miniters + assert t2.dynamic_miniters + + # Increase 10 iterations at once + t.update(bigstep) + t2.update(bigstep) + # The next iterations should not trigger maxinterval (step 10) + for _ in _range(4): + t.update(smallstep) + t2.update(smallstep) + timer.sleep(1e-5) + t.close() # because PyPy doesn't gc immediately + t2.close() # as above + + assert "25%" not in our_file2.getvalue() + assert "25%" not in our_file.getvalue() + + # Test with maxinterval effect + timer = DiscreteTimer() + with closing(StringIO()) as our_file: + with tqdm(total=total, file=our_file, miniters=None, mininterval=0, + smoothing=1, maxinterval=1e-4) as t: + cpu_timify(t, timer) + + # Increase 10 iterations at once + t.update(bigstep) + # The next iterations should trigger maxinterval (step 5) + for _ in _range(4): + t.update(smallstep) + timer.sleep(1e-2) + + assert "25%" in our_file.getvalue() + + # Test iteration based tqdm with maxinterval effect + timer = DiscreteTimer() + with closing(StringIO()) as our_file: + with tqdm(_range(total), file=our_file, miniters=None, + mininterval=1e-5, smoothing=1, maxinterval=1e-4) as t2: + cpu_timify(t2, timer) + + for i in t2: + if i >= (bigstep - 1) and ((i - (bigstep - 1)) % smallstep) == 0: + timer.sleep(1e-2) + if i >= 3 * bigstep: + break + + assert "15%" in our_file.getvalue() + + # Test different behavior with and without mininterval + timer = DiscreteTimer() + total = 1000 + mininterval = 0.1 + maxinterval = 10 + with closing(StringIO()) as our_file: + with tqdm(total=total, file=our_file, miniters=None, smoothing=1, + mininterval=mininterval, maxinterval=maxinterval) as tm1: + with tqdm(total=total, file=our_file, miniters=None, smoothing=1, + mininterval=0, maxinterval=maxinterval) as tm2: + + cpu_timify(tm1, timer) + cpu_timify(tm2, timer) + + # Fast iterations, check if dynamic_miniters triggers + timer.sleep(mininterval) # to force update for t1 + tm1.update(total / 2) + tm2.update(total / 2) + assert int(tm1.miniters) == tm2.miniters == total / 2 + + # Slow iterations, check different miniters if mininterval + timer.sleep(maxinterval * 2) + tm1.update(total / 2) + tm2.update(total / 2) + res = [tm1.miniters, tm2.miniters] + assert res == [(total / 2) * mininterval / (maxinterval * 2), + (total / 2) * maxinterval / (maxinterval * 2)] + + # Same with iterable based tqdm + timer1 = DiscreteTimer() # need 2 timers for each bar because zip not work + timer2 = DiscreteTimer() + total = 100 + mininterval = 0.1 + maxinterval = 10 + with closing(StringIO()) as our_file: + t1 = tqdm(_range(total), file=our_file, miniters=None, smoothing=1, + mininterval=mininterval, maxinterval=maxinterval) + t2 = tqdm(_range(total), file=our_file, miniters=None, smoothing=1, + mininterval=0, maxinterval=maxinterval) + + cpu_timify(t1, timer1) + cpu_timify(t2, timer2) + + for i in t1: + if i == ((total / 2) - 2): + timer1.sleep(mininterval) + if i == (total - 1): + timer1.sleep(maxinterval * 2) + + for i in t2: + if i == ((total / 2) - 2): + timer2.sleep(mininterval) + if i == (total - 1): + timer2.sleep(maxinterval * 2) + + assert t1.miniters == 0.255 + assert t2.miniters == 0.5 + + t1.close() + t2.close() + + +def test_delay(): + """Test delay""" + timer = DiscreteTimer() + with closing(StringIO()) as our_file: + t = tqdm(total=2, file=our_file, leave=True, delay=3) + cpu_timify(t, timer) + timer.sleep(2) + t.update(1) + assert not our_file.getvalue() + timer.sleep(2) + t.update(1) + assert our_file.getvalue() + t.close() + + +def test_min_iters(): + """Test miniters""" + with closing(StringIO()) as our_file: + for _ in tqdm(_range(3), file=our_file, leave=True, mininterval=0, miniters=2): + pass + + out = our_file.getvalue() + assert '| 0/3 ' in out + assert '| 1/3 ' not in out + assert '| 2/3 ' in out + assert '| 3/3 ' in out + + with closing(StringIO()) as our_file: + for _ in tqdm(_range(3), file=our_file, leave=True, mininterval=0, miniters=1): + pass + + out = our_file.getvalue() + assert '| 0/3 ' in out + assert '| 1/3 ' in out + assert '| 2/3 ' in out + assert '| 3/3 ' in out + + +def test_dynamic_min_iters(): + """Test purely dynamic miniters (and manual updates and __del__)""" + with closing(StringIO()) as our_file: + total = 10 + t = tqdm(total=total, file=our_file, miniters=None, mininterval=0, smoothing=1) + + t.update() + # Increase 3 iterations + t.update(3) + # The next two iterations should be skipped because of dynamic_miniters + t.update() + t.update() + # The third iteration should be displayed + t.update() + + out = our_file.getvalue() + assert t.dynamic_miniters + t.__del__() # simulate immediate del gc + + assert ' 0%| | 0/10 [00:00<' in out + assert '40%' in out + assert '50%' not in out + assert '60%' not in out + assert '70%' in out + + # Check with smoothing=0, miniters should be set to max update seen so far + with closing(StringIO()) as our_file: + total = 10 + t = tqdm(total=total, file=our_file, miniters=None, mininterval=0, smoothing=0) + + t.update() + t.update(2) + t.update(5) # this should be stored as miniters + t.update(1) + + out = our_file.getvalue() + assert all(i in out for i in ("0/10", "1/10", "3/10")) + assert "2/10" not in out + assert t.dynamic_miniters and not t.smoothing + assert t.miniters == 5 + t.close() + + # Check iterable based tqdm + with closing(StringIO()) as our_file: + t = tqdm(_range(10), file=our_file, miniters=None, mininterval=None, + smoothing=0.5) + for _ in t: + pass + assert t.dynamic_miniters + + # No smoothing + with closing(StringIO()) as our_file: + t = tqdm(_range(10), file=our_file, miniters=None, mininterval=None, + smoothing=0) + for _ in t: + pass + assert t.dynamic_miniters + + # No dynamic_miniters (miniters is fixed manually) + with closing(StringIO()) as our_file: + t = tqdm(_range(10), file=our_file, miniters=1, mininterval=None) + for _ in t: + pass + assert not t.dynamic_miniters + + +def test_big_min_interval(): + """Test large mininterval""" + with closing(StringIO()) as our_file: + for _ in tqdm(_range(2), file=our_file, mininterval=1E10): + pass + assert '50%' not in our_file.getvalue() + + with closing(StringIO()) as our_file: + with tqdm(_range(2), file=our_file, mininterval=1E10) as t: + t.update() + t.update() + assert '50%' not in our_file.getvalue() + + +def test_smoothed_dynamic_min_iters(): + """Test smoothed dynamic miniters""" + timer = DiscreteTimer() + + with closing(StringIO()) as our_file: + with tqdm(total=100, file=our_file, miniters=None, mininterval=1, + smoothing=0.5, maxinterval=0) as t: + cpu_timify(t, timer) + + # Increase 10 iterations at once + timer.sleep(1) + t.update(10) + # The next iterations should be partially skipped + for _ in _range(2): + timer.sleep(1) + t.update(4) + for _ in _range(20): + timer.sleep(1) + t.update() + + assert t.dynamic_miniters + out = our_file.getvalue() + assert ' 0%| | 0/100 [00:00<' in out + assert '20%' in out + assert '23%' not in out + assert '25%' in out + assert '26%' not in out + assert '28%' in out + + +def test_smoothed_dynamic_min_iters_with_min_interval(): + """Test smoothed dynamic miniters with mininterval""" + timer = DiscreteTimer() + + # In this test, `miniters` should gradually decline + total = 100 + + with closing(StringIO()) as our_file: + # Test manual updating tqdm + with tqdm(total=total, file=our_file, miniters=None, mininterval=1e-3, + smoothing=1, maxinterval=0) as t: + cpu_timify(t, timer) + + t.update(10) + timer.sleep(1e-2) + for _ in _range(4): + t.update() + timer.sleep(1e-2) + out = our_file.getvalue() + assert t.dynamic_miniters + + with closing(StringIO()) as our_file: + # Test iteration-based tqdm + with tqdm(_range(total), file=our_file, miniters=None, + mininterval=0.01, smoothing=1, maxinterval=0) as t2: + cpu_timify(t2, timer) + + for i in t2: + if i >= 10: + timer.sleep(0.1) + if i >= 14: + break + out2 = our_file.getvalue() + + assert t.dynamic_miniters + assert ' 0%| | 0/100 [00:00<' in out + assert '11%' in out and '11%' in out2 + # assert '12%' not in out and '12%' in out2 + assert '13%' in out and '13%' in out2 + assert '14%' in out and '14%' in out2 + + +@mark.slow +def test_rlock_creation(): + """Test that importing tqdm does not create multiprocessing objects.""" + mp = importorskip('multiprocessing') + if not hasattr(mp, 'get_context'): + skip("missing multiprocessing.get_context") + + # Use 'spawn' instead of 'fork' so that the process does not inherit any + # globals that have been constructed by running other tests + ctx = mp.get_context('spawn') + with ctx.Pool(1) as pool: + # The pool will propagate the error if the target method fails + pool.apply(_rlock_creation_target) + + +def _rlock_creation_target(): + """Check that the RLock has not been constructed.""" + import multiprocessing as mp + patch = importorskip('unittest.mock').patch + + # Patch the RLock class/method but use the original implementation + with patch('multiprocessing.RLock', wraps=mp.RLock) as rlock_mock: + # Importing the module should not create a lock + from tqdm import tqdm + assert rlock_mock.call_count == 0 + # Creating a progress bar should initialize the lock + with closing(StringIO()) as our_file: + with tqdm(file=our_file) as _: # NOQA + pass + assert rlock_mock.call_count == 1 + # Creating a progress bar again should reuse the lock + with closing(StringIO()) as our_file: + with tqdm(file=our_file) as _: # NOQA + pass + assert rlock_mock.call_count == 1 + + +def test_disable(): + """Test disable""" + with closing(StringIO()) as our_file: + for _ in tqdm(_range(3), file=our_file, disable=True): + pass + assert our_file.getvalue() == '' + + with closing(StringIO()) as our_file: + progressbar = tqdm(total=3, file=our_file, miniters=1, disable=True) + progressbar.update(3) + progressbar.close() + assert our_file.getvalue() == '' + + +def test_infinite_total(): + """Test treatment of infinite total""" + with closing(StringIO()) as our_file: + for _ in tqdm(_range(3), file=our_file, total=float("inf")): + pass + + +def test_nototal(): + """Test unknown total length""" + with closing(StringIO()) as our_file: + for _ in tqdm(iter(range(10)), file=our_file, unit_scale=10): + pass + assert "100it" in our_file.getvalue() + + with closing(StringIO()) as our_file: + for _ in tqdm(iter(range(10)), file=our_file, + bar_format="{l_bar}{bar}{r_bar}"): + pass + assert "10/?" in our_file.getvalue() + + +def test_unit(): + """Test SI unit prefix""" + with closing(StringIO()) as our_file: + for _ in tqdm(_range(3), file=our_file, miniters=1, unit="bytes"): + pass + assert 'bytes/s' in our_file.getvalue() + + +def test_ascii(): + """Test ascii/unicode bar""" + # Test ascii autodetection + with closing(StringIO()) as our_file: + with tqdm(total=10, file=our_file, ascii=None) as t: + assert t.ascii # TODO: this may fail in the future + + # Test ascii bar + with closing(StringIO()) as our_file: + for _ in tqdm(_range(3), total=15, file=our_file, miniters=1, + mininterval=0, ascii=True): + pass + res = our_file.getvalue().strip("\r").split("\r") + assert '7%|6' in res[1] + assert '13%|#3' in res[2] + assert '20%|##' in res[3] + + # Test unicode bar + with closing(UnicodeIO()) as our_file: + with tqdm(total=15, file=our_file, ascii=False, mininterval=0) as t: + for _ in _range(3): + t.update() + res = our_file.getvalue().strip("\r").split("\r") + assert u"7%|\u258b" in res[1] + assert u"13%|\u2588\u258e" in res[2] + assert u"20%|\u2588\u2588" in res[3] + + # Test custom bar + for bars in [" .oO0", " #"]: + with closing(StringIO()) as our_file: + for _ in tqdm(_range(len(bars) - 1), file=our_file, miniters=1, + mininterval=0, ascii=bars, ncols=27): + pass + res = our_file.getvalue().strip("\r").split("\r") + for b, line in zip(bars, res): + assert '|' + b + '|' in line + + +def test_update(): + """Test manual creation and updates""" + res = None + with closing(StringIO()) as our_file: + with tqdm(total=2, file=our_file, miniters=1, mininterval=0) as progressbar: + assert len(progressbar) == 2 + progressbar.update(2) + assert '| 2/2' in our_file.getvalue() + progressbar.desc = 'dynamically notify of 4 increments in total' + progressbar.total = 4 + progressbar.update(-1) + progressbar.update(2) + res = our_file.getvalue() + assert '| 3/4 ' in res + assert 'dynamically notify of 4 increments in total' in res + + +def test_close(): + """Test manual creation and closure and n_instances""" + + # With `leave` option + with closing(StringIO()) as our_file: + progressbar = tqdm(total=3, file=our_file, miniters=10) + progressbar.update(3) + assert '| 3/3 ' not in our_file.getvalue() # Should be blank + assert len(tqdm._instances) == 1 + progressbar.close() + assert len(tqdm._instances) == 0 + assert '| 3/3 ' in our_file.getvalue() + + # Without `leave` option + with closing(StringIO()) as our_file: + progressbar = tqdm(total=3, file=our_file, miniters=10, leave=False) + progressbar.update(3) + progressbar.close() + assert '| 3/3 ' not in our_file.getvalue() # Should be blank + + # With all updates + with closing(StringIO()) as our_file: + assert len(tqdm._instances) == 0 + with tqdm(total=3, file=our_file, miniters=0, mininterval=0, + leave=True) as progressbar: + assert len(tqdm._instances) == 1 + progressbar.update(3) + res = our_file.getvalue() + assert '| 3/3 ' in res # Should be blank + assert '\n' not in res + # close() called + assert len(tqdm._instances) == 0 + + exres = res.rsplit(', ', 1)[0] + res = our_file.getvalue() + assert res[-1] == '\n' + if not res.startswith(exres): + raise AssertionError("\n<<< Expected:\n{0}\n>>> Got:\n{1}\n===".format( + exres + ', ...it/s]\n', our_file.getvalue())) + + # Closing after the output stream has closed + with closing(StringIO()) as our_file: + t = tqdm(total=2, file=our_file) + t.update() + t.update() + t.close() + + +def test_ema(): + """Test exponential weighted average""" + ema = EMA(0.01) + assert round(ema(10), 2) == 10 + assert round(ema(1), 2) == 5.48 + assert round(ema(), 2) == 5.48 + assert round(ema(1), 2) == 3.97 + assert round(ema(1), 2) == 3.22 + + +def test_smoothing(): + """Test exponential weighted average smoothing""" + timer = DiscreteTimer() + + # -- Test disabling smoothing + with closing(StringIO()) as our_file: + with tqdm(_range(3), file=our_file, smoothing=None, leave=True) as t: + cpu_timify(t, timer) + + for _ in t: + pass + assert '| 3/3 ' in our_file.getvalue() + + # -- Test smoothing + # 1st case: no smoothing (only use average) + with closing(StringIO()) as our_file2: + with closing(StringIO()) as our_file: + t = tqdm(_range(3), file=our_file2, smoothing=None, leave=True, + miniters=1, mininterval=0) + cpu_timify(t, timer) + + with tqdm(_range(3), file=our_file, smoothing=None, leave=True, + miniters=1, mininterval=0) as t2: + cpu_timify(t2, timer) + + for i in t2: + # Sleep more for first iteration and + # see how quickly rate is updated + if i == 0: + timer.sleep(0.01) + else: + # Need to sleep in all iterations + # to calculate smoothed rate + # (else delta_t is 0!) + timer.sleep(0.001) + t.update() + n_old = len(tqdm._instances) + t.close() + assert len(tqdm._instances) == n_old - 1 + # Get result for iter-based bar + a = progressbar_rate(get_bar(our_file.getvalue(), 3)) + # Get result for manually updated bar + a2 = progressbar_rate(get_bar(our_file2.getvalue(), 3)) + + # 2nd case: use max smoothing (= instant rate) + with closing(StringIO()) as our_file2: + with closing(StringIO()) as our_file: + t = tqdm(_range(3), file=our_file2, smoothing=1, leave=True, + miniters=1, mininterval=0) + cpu_timify(t, timer) + + with tqdm(_range(3), file=our_file, smoothing=1, leave=True, + miniters=1, mininterval=0) as t2: + cpu_timify(t2, timer) + + for i in t2: + if i == 0: + timer.sleep(0.01) + else: + timer.sleep(0.001) + t.update() + t.close() + # Get result for iter-based bar + b = progressbar_rate(get_bar(our_file.getvalue(), 3)) + # Get result for manually updated bar + b2 = progressbar_rate(get_bar(our_file2.getvalue(), 3)) + + # 3rd case: use medium smoothing + with closing(StringIO()) as our_file2: + with closing(StringIO()) as our_file: + t = tqdm(_range(3), file=our_file2, smoothing=0.5, leave=True, + miniters=1, mininterval=0) + cpu_timify(t, timer) + + t2 = tqdm(_range(3), file=our_file, smoothing=0.5, leave=True, + miniters=1, mininterval=0) + cpu_timify(t2, timer) + + for i in t2: + if i == 0: + timer.sleep(0.01) + else: + timer.sleep(0.001) + t.update() + t2.close() + t.close() + # Get result for iter-based bar + c = progressbar_rate(get_bar(our_file.getvalue(), 3)) + # Get result for manually updated bar + c2 = progressbar_rate(get_bar(our_file2.getvalue(), 3)) + + # Check that medium smoothing's rate is between no and max smoothing rates + assert a <= c <= b + assert a2 <= c2 <= b2 + + +@mark.skipif(nt_and_no_colorama, reason="Windows without colorama") +def test_deprecated_nested(): + """Test nested progress bars""" + # TODO: test degradation on windows without colorama? + + # Artificially test nested loop printing + # Without leave + our_file = StringIO() + try: + tqdm(total=2, file=our_file, nested=True) + except TqdmDeprecationWarning: + if """`nested` is deprecated and automated. +Use `position` instead for manual control.""" not in our_file.getvalue(): + raise + else: + raise DeprecationError("Should not allow nested kwarg") + + +def test_bar_format(): + """Test custom bar formatting""" + with closing(StringIO()) as our_file: + bar_format = ('{l_bar}{bar}|{n_fmt}/{total_fmt}-{n}/{total}' + '{percentage}{rate}{rate_fmt}{elapsed}{remaining}') + for _ in trange(2, file=our_file, leave=True, bar_format=bar_format): + pass + out = our_file.getvalue() + assert "\r 0%| |0/2-0/20.0None?it/s00:00?\r" in out + + # Test unicode string auto conversion + with closing(StringIO()) as our_file: + bar_format = r'hello world' + with tqdm(ascii=False, bar_format=bar_format, file=our_file) as t: + assert isinstance(t.bar_format, _unicode) + + +def test_custom_format(): + """Test adding additional derived format arguments""" + class TqdmExtraFormat(tqdm): + """Provides a `total_time` format parameter""" + @property + def format_dict(self): + d = super(TqdmExtraFormat, self).format_dict + total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1) + d.update(total_time=self.format_interval(total_time) + " in total") + return d + + with closing(StringIO()) as our_file: + for _ in TqdmExtraFormat( + range(10), file=our_file, + bar_format="{total_time}: {percentage:.0f}%|{bar}{r_bar}"): + pass + assert "00:00 in total" in our_file.getvalue() + + +def test_eta(capsys): + """Test eta bar_format""" + from datetime import datetime as dt + for _ in trange(999, miniters=1, mininterval=0, leave=True, + bar_format='{l_bar}{eta:%Y-%m-%d}'): + pass + _, err = capsys.readouterr() + assert "\r100%|{eta:%Y-%m-%d}\n".format(eta=dt.now()) in err + + +def test_unpause(): + """Test unpause""" + timer = DiscreteTimer() + with closing(StringIO()) as our_file: + t = trange(10, file=our_file, leave=True, mininterval=0) + cpu_timify(t, timer) + timer.sleep(0.01) + t.update() + timer.sleep(0.01) + t.update() + timer.sleep(0.1) # longer wait time + t.unpause() + timer.sleep(0.01) + t.update() + timer.sleep(0.01) + t.update() + t.close() + r_before = progressbar_rate(get_bar(our_file.getvalue(), 2)) + r_after = progressbar_rate(get_bar(our_file.getvalue(), 3)) + assert r_before == r_after + + +def test_disabled_unpause(capsys): + """Test disabled unpause""" + with tqdm(total=10, disable=True) as t: + t.update() + t.unpause() + t.update() + print(t) + out, err = capsys.readouterr() + assert not err + assert out == ' 0%| | 0/10 [00:00<?, ?it/s]\n' + + +def test_reset(): + """Test resetting a bar for re-use""" + with closing(StringIO()) as our_file: + with tqdm(total=10, file=our_file, + miniters=1, mininterval=0, maxinterval=0) as t: + t.update(9) + t.reset() + t.update() + t.reset(total=12) + t.update(10) + assert '| 1/10' in our_file.getvalue() + assert '| 10/12' in our_file.getvalue() + + +def test_disabled_reset(capsys): + """Test disabled reset""" + with tqdm(total=10, disable=True) as t: + t.update(9) + t.reset() + t.update() + t.reset(total=12) + t.update(10) + print(t) + out, err = capsys.readouterr() + assert not err + assert out == ' 0%| | 0/12 [00:00<?, ?it/s]\n' + + +@mark.skipif(nt_and_no_colorama, reason="Windows without colorama") +def test_position(): + """Test positioned progress bars""" + # Artificially test nested loop printing + # Without leave + our_file = StringIO() + kwargs = {'file': our_file, 'miniters': 1, 'mininterval': 0, 'maxinterval': 0} + t = tqdm(total=2, desc='pos2 bar', leave=False, position=2, **kwargs) + t.update() + t.close() + out = our_file.getvalue() + res = [m[0] for m in RE_pos.findall(out)] + exres = ['\n\n\rpos2 bar: 0%', + '\n\n\rpos2 bar: 50%', + '\n\n\r '] + + pos_line_diff(res, exres) + + # Test iteration-based tqdm positioning + our_file = StringIO() + kwargs["file"] = our_file + for _ in trange(2, desc='pos0 bar', position=0, **kwargs): + for _ in trange(2, desc='pos1 bar', position=1, **kwargs): + for _ in trange(2, desc='pos2 bar', position=2, **kwargs): + pass + out = our_file.getvalue() + res = [m[0] for m in RE_pos.findall(out)] + exres = ['\rpos0 bar: 0%', + '\n\rpos1 bar: 0%', + '\n\n\rpos2 bar: 0%', + '\n\n\rpos2 bar: 50%', + '\n\n\rpos2 bar: 100%', + '\rpos2 bar: 100%', + '\n\n\rpos1 bar: 50%', + '\n\n\rpos2 bar: 0%', + '\n\n\rpos2 bar: 50%', + '\n\n\rpos2 bar: 100%', + '\rpos2 bar: 100%', + '\n\n\rpos1 bar: 100%', + '\rpos1 bar: 100%', + '\n\rpos0 bar: 50%', + '\n\rpos1 bar: 0%', + '\n\n\rpos2 bar: 0%', + '\n\n\rpos2 bar: 50%', + '\n\n\rpos2 bar: 100%', + '\rpos2 bar: 100%', + '\n\n\rpos1 bar: 50%', + '\n\n\rpos2 bar: 0%', + '\n\n\rpos2 bar: 50%', + '\n\n\rpos2 bar: 100%', + '\rpos2 bar: 100%', + '\n\n\rpos1 bar: 100%', + '\rpos1 bar: 100%', + '\n\rpos0 bar: 100%', + '\rpos0 bar: 100%', + '\n'] + pos_line_diff(res, exres) + + # Test manual tqdm positioning + our_file = StringIO() + kwargs["file"] = our_file + kwargs["total"] = 2 + t1 = tqdm(desc='pos0 bar', position=0, **kwargs) + t2 = tqdm(desc='pos1 bar', position=1, **kwargs) + t3 = tqdm(desc='pos2 bar', position=2, **kwargs) + for _ in _range(2): + t1.update() + t3.update() + t2.update() + out = our_file.getvalue() + res = [m[0] for m in RE_pos.findall(out)] + exres = ['\rpos0 bar: 0%', + '\n\rpos1 bar: 0%', + '\n\n\rpos2 bar: 0%', + '\rpos0 bar: 50%', + '\n\n\rpos2 bar: 50%', + '\n\rpos1 bar: 50%', + '\rpos0 bar: 100%', + '\n\n\rpos2 bar: 100%', + '\n\rpos1 bar: 100%'] + pos_line_diff(res, exres) + t1.close() + t2.close() + t3.close() + + # Test auto repositioning of bars when a bar is prematurely closed + # tqdm._instances.clear() # reset number of instances + with closing(StringIO()) as our_file: + t1 = tqdm(total=10, file=our_file, desc='1.pos0 bar', mininterval=0) + t2 = tqdm(total=10, file=our_file, desc='2.pos1 bar', mininterval=0) + t3 = tqdm(total=10, file=our_file, desc='3.pos2 bar', mininterval=0) + res = [m[0] for m in RE_pos.findall(our_file.getvalue())] + exres = ['\r1.pos0 bar: 0%', + '\n\r2.pos1 bar: 0%', + '\n\n\r3.pos2 bar: 0%'] + pos_line_diff(res, exres) + + t2.close() + t4 = tqdm(total=10, file=our_file, desc='4.pos2 bar', mininterval=0) + t1.update(1) + t3.update(1) + t4.update(1) + res = [m[0] for m in RE_pos.findall(our_file.getvalue())] + exres = ['\r1.pos0 bar: 0%', + '\n\r2.pos1 bar: 0%', + '\n\n\r3.pos2 bar: 0%', + '\r2.pos1 bar: 0%', + '\n\n\r4.pos2 bar: 0%', + '\r1.pos0 bar: 10%', + '\n\n\r3.pos2 bar: 10%', + '\n\r4.pos2 bar: 10%'] + pos_line_diff(res, exres) + t4.close() + t3.close() + t1.close() + + +def test_set_description(): + """Test set description""" + with closing(StringIO()) as our_file: + with tqdm(desc='Hello', file=our_file) as t: + assert t.desc == 'Hello' + t.set_description_str('World') + assert t.desc == 'World' + t.set_description() + assert t.desc == '' + t.set_description('Bye') + assert t.desc == 'Bye: ' + assert "World" in our_file.getvalue() + + # without refresh + with closing(StringIO()) as our_file: + with tqdm(desc='Hello', file=our_file) as t: + assert t.desc == 'Hello' + t.set_description_str('World', False) + assert t.desc == 'World' + t.set_description(None, False) + assert t.desc == '' + assert "World" not in our_file.getvalue() + + # unicode + with closing(StringIO()) as our_file: + with tqdm(total=10, file=our_file) as t: + t.set_description(u"\xe1\xe9\xed\xf3\xfa") + + +def test_deprecated_gui(): + """Test internal GUI properties""" + # Check: StatusPrinter iff gui is disabled + with closing(StringIO()) as our_file: + t = tqdm(total=2, gui=True, file=our_file, miniters=1, mininterval=0) + assert not hasattr(t, "sp") + try: + t.update(1) + except TqdmDeprecationWarning as e: + if ( + 'Please use `tqdm.gui.tqdm(...)` instead of `tqdm(..., gui=True)`' + not in our_file.getvalue() + ): + raise e + else: + raise DeprecationError('Should not allow manual gui=True without' + ' overriding __iter__() and update()') + finally: + t._instances.clear() + # t.close() + # len(tqdm._instances) += 1 # undo the close() decrement + + t = tqdm(_range(3), gui=True, file=our_file, miniters=1, mininterval=0) + try: + for _ in t: + pass + except TqdmDeprecationWarning as e: + if ( + 'Please use `tqdm.gui.tqdm(...)` instead of `tqdm(..., gui=True)`' + not in our_file.getvalue() + ): + raise e + else: + raise DeprecationError('Should not allow manual gui=True without' + ' overriding __iter__() and update()') + finally: + t._instances.clear() + # t.close() + # len(tqdm._instances) += 1 # undo the close() decrement + + with tqdm(total=1, gui=False, file=our_file) as t: + assert hasattr(t, "sp") + + +def test_cmp(): + """Test comparison functions""" + with closing(StringIO()) as our_file: + t0 = tqdm(total=10, file=our_file) + t1 = tqdm(total=10, file=our_file) + t2 = tqdm(total=10, file=our_file) + + assert t0 < t1 + assert t2 >= t0 + assert t0 <= t2 + + t3 = tqdm(total=10, file=our_file) + t4 = tqdm(total=10, file=our_file) + t5 = tqdm(total=10, file=our_file) + t5.close() + t6 = tqdm(total=10, file=our_file) + + assert t3 != t4 + assert t3 > t2 + assert t5 == t6 + t6.close() + t4.close() + t3.close() + t2.close() + t1.close() + t0.close() + + +def test_repr(): + """Test representation""" + with closing(StringIO()) as our_file: + with tqdm(total=10, ascii=True, file=our_file) as t: + assert str(t) == ' 0%| | 0/10 [00:00<?, ?it/s]' + + +def test_clear(): + """Test clearing bar display""" + with closing(StringIO()) as our_file: + t1 = tqdm(total=10, file=our_file, desc='pos0 bar', bar_format='{l_bar}') + t2 = trange(10, file=our_file, desc='pos1 bar', bar_format='{l_bar}') + before = squash_ctrlchars(our_file.getvalue()) + t2.clear() + t1.clear() + after = squash_ctrlchars(our_file.getvalue()) + t1.close() + t2.close() + assert before == ['pos0 bar: 0%|', 'pos1 bar: 0%|'] + assert after == ['', ''] + + +def test_clear_disabled(): + """Test disabled clear""" + with closing(StringIO()) as our_file: + with tqdm(total=10, file=our_file, desc='pos0 bar', disable=True, + bar_format='{l_bar}') as t: + t.clear() + assert our_file.getvalue() == '' + + +def test_refresh(): + """Test refresh bar display""" + with closing(StringIO()) as our_file: + t1 = tqdm(total=10, file=our_file, desc='pos0 bar', + bar_format='{l_bar}', mininterval=999, miniters=999) + t2 = tqdm(total=10, file=our_file, desc='pos1 bar', + bar_format='{l_bar}', mininterval=999, miniters=999) + t1.update() + t2.update() + before = squash_ctrlchars(our_file.getvalue()) + t1.refresh() + t2.refresh() + after = squash_ctrlchars(our_file.getvalue()) + t1.close() + t2.close() + + # Check that refreshing indeed forced the display to use realtime state + assert before == [u'pos0 bar: 0%|', u'pos1 bar: 0%|'] + assert after == [u'pos0 bar: 10%|', u'pos1 bar: 10%|'] + + +def test_disabled_repr(capsys): + """Test disabled repr""" + with tqdm(total=10, disable=True) as t: + str(t) + t.update() + print(t) + out, err = capsys.readouterr() + assert not err + assert out == ' 0%| | 0/10 [00:00<?, ?it/s]\n' + + +def test_disabled_refresh(): + """Test disabled refresh""" + with closing(StringIO()) as our_file: + with tqdm(total=10, file=our_file, desc='pos0 bar', disable=True, + bar_format='{l_bar}', mininterval=999, miniters=999) as t: + t.update() + t.refresh() + + assert our_file.getvalue() == '' + + +def test_write(): + """Test write messages""" + s = "Hello world" + with closing(StringIO()) as our_file: + # Change format to keep only left part w/o bar and it/s rate + t1 = tqdm(total=10, file=our_file, desc='pos0 bar', + bar_format='{l_bar}', mininterval=0, miniters=1) + t2 = trange(10, file=our_file, desc='pos1 bar', bar_format='{l_bar}', + mininterval=0, miniters=1) + t3 = tqdm(total=10, file=our_file, desc='pos2 bar', + bar_format='{l_bar}', mininterval=0, miniters=1) + t1.update() + t2.update() + t3.update() + before = our_file.getvalue() + + # Write msg and see if bars are correctly redrawn below the msg + t1.write(s, file=our_file) # call as an instance method + tqdm.write(s, file=our_file) # call as a class method + after = our_file.getvalue() + + t1.close() + t2.close() + t3.close() + + before_squashed = squash_ctrlchars(before) + after_squashed = squash_ctrlchars(after) + + assert after_squashed == [s, s] + before_squashed + + # Check that no bar clearing if different file + with closing(StringIO()) as our_file_bar: + with closing(StringIO()) as our_file_write: + t1 = tqdm(total=10, file=our_file_bar, desc='pos0 bar', + bar_format='{l_bar}', mininterval=0, miniters=1) + + t1.update() + before_bar = our_file_bar.getvalue() + + tqdm.write(s, file=our_file_write) + + after_bar = our_file_bar.getvalue() + t1.close() + + assert before_bar == after_bar + + # Test stdout/stderr anti-mixup strategy + # Backup stdout/stderr + stde = sys.stderr + stdo = sys.stdout + # Mock stdout/stderr + with closing(StringIO()) as our_stderr: + with closing(StringIO()) as our_stdout: + sys.stderr = our_stderr + sys.stdout = our_stdout + t1 = tqdm(total=10, file=sys.stderr, desc='pos0 bar', + bar_format='{l_bar}', mininterval=0, miniters=1) + + t1.update() + before_err = sys.stderr.getvalue() + before_out = sys.stdout.getvalue() + + tqdm.write(s, file=sys.stdout) + after_err = sys.stderr.getvalue() + after_out = sys.stdout.getvalue() + + t1.close() + + assert before_err == '\rpos0 bar: 0%|\rpos0 bar: 10%|' + assert before_out == '' + after_err_res = [m[0] for m in RE_pos.findall(after_err)] + exres = ['\rpos0 bar: 0%|', + '\rpos0 bar: 10%|', + '\r ', + '\r\rpos0 bar: 10%|'] + pos_line_diff(after_err_res, exres) + assert after_out == s + '\n' + # Restore stdout and stderr + sys.stderr = stde + sys.stdout = stdo + + +def test_len(): + """Test advance len (numpy array shape)""" + np = importorskip('numpy') + with closing(StringIO()) as f: + with tqdm(np.zeros((3, 4)), file=f) as t: + assert len(t) == 3 + + +def test_autodisable_disable(): + """Test autodisable will disable on non-TTY""" + with closing(StringIO()) as our_file: + with tqdm(total=10, disable=None, file=our_file) as t: + t.update(3) + assert our_file.getvalue() == '' + + +def test_autodisable_enable(): + """Test autodisable will not disable on TTY""" + with closing(StringIO()) as our_file: + our_file.isatty = lambda: True + with tqdm(total=10, disable=None, file=our_file) as t: + t.update() + assert our_file.getvalue() != '' + + +def test_deprecation_exception(): + def test_TqdmDeprecationWarning(): + with closing(StringIO()) as our_file: + raise (TqdmDeprecationWarning('Test!', fp_write=getattr( + our_file, 'write', sys.stderr.write))) + + def test_TqdmDeprecationWarning_nofpwrite(): + raise TqdmDeprecationWarning('Test!', fp_write=None) + + raises(TqdmDeprecationWarning, test_TqdmDeprecationWarning) + raises(Exception, test_TqdmDeprecationWarning_nofpwrite) + + +def test_postfix(): + """Test postfix""" + postfix = {'float': 0.321034, 'gen': 543, 'str': 'h', 'lst': [2]} + postfix_order = (('w', 'w'), ('a', 0)) # no need for OrderedDict + expected = ['float=0.321', 'gen=543', 'lst=[2]', 'str=h'] + expected_order = ['w=w', 'a=0', 'float=0.321', 'gen=543', 'lst=[2]', 'str=h'] + + # Test postfix set at init + with closing(StringIO()) as our_file: + with tqdm(total=10, file=our_file, desc='pos0 bar', + bar_format='{r_bar}', postfix=postfix) as t1: + t1.refresh() + out = our_file.getvalue() + + # Test postfix set after init + with closing(StringIO()) as our_file: + with trange(10, file=our_file, desc='pos1 bar', bar_format='{r_bar}', + postfix=None) as t2: + t2.set_postfix(**postfix) + t2.refresh() + out2 = our_file.getvalue() + + # Order of items in dict may change, so need a loop to check per item + for res in expected: + assert res in out + assert res in out2 + + # Test postfix (with ordered dict and no refresh) set after init + with closing(StringIO()) as our_file: + with trange(10, file=our_file, desc='pos2 bar', bar_format='{r_bar}', + postfix=None) as t3: + t3.set_postfix(postfix_order, False, **postfix) + t3.refresh() # explicit external refresh + out3 = our_file.getvalue() + + out3 = out3[1:-1].split(', ')[3:] + assert out3 == expected_order + + # Test postfix (with ordered dict and refresh) set after init + with closing(StringIO()) as our_file: + with trange(10, file=our_file, desc='pos2 bar', + bar_format='{r_bar}', postfix=None) as t4: + t4.set_postfix(postfix_order, True, **postfix) + t4.refresh() # double refresh + out4 = our_file.getvalue() + + assert out4.count('\r') > out3.count('\r') + assert out4.count(", ".join(expected_order)) == 2 + + # Test setting postfix string directly + with closing(StringIO()) as our_file: + with trange(10, file=our_file, desc='pos2 bar', bar_format='{r_bar}', + postfix=None) as t5: + t5.set_postfix_str("Hello", False) + t5.set_postfix_str("World") + out5 = our_file.getvalue() + + assert "Hello" not in out5 + out5 = out5[1:-1].split(', ')[3:] + assert out5 == ["World"] + + +def test_postfix_direct(): + """Test directly assigning non-str objects to postfix""" + with closing(StringIO()) as our_file: + with tqdm(total=10, file=our_file, miniters=1, mininterval=0, + bar_format="{postfix[0][name]} {postfix[1]:>5.2f}", + postfix=[{'name': "foo"}, 42]) as t: + for i in range(10): + if i % 2: + t.postfix[0]["name"] = "abcdefghij"[i] + else: + t.postfix[1] = i + t.update() + res = our_file.getvalue() + assert "f 6.00" in res + assert "h 6.00" in res + assert "h 8.00" in res + assert "j 8.00" in res + + +@contextmanager +def std_out_err_redirect_tqdm(tqdm_file=sys.stderr): + orig_out_err = sys.stdout, sys.stderr + try: + sys.stdout = sys.stderr = DummyTqdmFile(tqdm_file) + yield orig_out_err[0] + # Relay exceptions + except Exception as exc: + raise exc + # Always restore sys.stdout/err if necessary + finally: + sys.stdout, sys.stderr = orig_out_err + + +def test_file_redirection(): + """Test redirection of output""" + with closing(StringIO()) as our_file: + # Redirect stdout to tqdm.write() + with std_out_err_redirect_tqdm(tqdm_file=our_file): + with tqdm(total=3) as pbar: + print("Such fun") + pbar.update(1) + print("Such", "fun") + pbar.update(1) + print("Such ", end="") + print("fun") + pbar.update(1) + res = our_file.getvalue() + assert res.count("Such fun\n") == 3 + assert "0/3" in res + assert "3/3" in res + + +def test_external_write(): + """Test external write mode""" + with closing(StringIO()) as our_file: + # Redirect stdout to tqdm.write() + for _ in trange(3, file=our_file): + del tqdm._lock # classmethod should be able to recreate lock + with tqdm.external_write_mode(file=our_file): + our_file.write("Such fun\n") + res = our_file.getvalue() + assert res.count("Such fun\n") == 3 + assert "0/3" in res + assert "3/3" in res + + +def test_unit_scale(): + """Test numeric `unit_scale`""" + with closing(StringIO()) as our_file: + for _ in tqdm(_range(9), unit_scale=9, file=our_file, + miniters=1, mininterval=0): + pass + out = our_file.getvalue() + assert '81/81' in out + + +def patch_lock(thread=True): + """decorator replacing tqdm's lock with vanilla threading/multiprocessing""" + try: + if thread: + from threading import RLock + else: + from multiprocessing import RLock + lock = RLock() + except (ImportError, OSError) as err: + skip(str(err)) + + def outer(func): + """actual decorator""" + @wraps(func) + def inner(*args, **kwargs): + """set & reset lock even if exceptions occur""" + default_lock = tqdm.get_lock() + try: + tqdm.set_lock(lock) + return func(*args, **kwargs) + finally: + tqdm.set_lock(default_lock) + return inner + return outer + + +@patch_lock(thread=False) +def test_threading(): + """Test multiprocess/thread-realted features""" + pass # TODO: test interleaved output #445 + + +def test_bool(): + """Test boolean cast""" + def internal(our_file, disable): + kwargs = {'file': our_file, 'disable': disable} + with trange(10, **kwargs) as t: + assert t + with trange(0, **kwargs) as t: + assert not t + with tqdm(total=10, **kwargs) as t: + assert bool(t) + with tqdm(total=0, **kwargs) as t: + assert not bool(t) + with tqdm([], **kwargs) as t: + assert not t + with tqdm([0], **kwargs) as t: + assert t + with tqdm(iter([]), **kwargs) as t: + assert t + with tqdm(iter([1, 2, 3]), **kwargs) as t: + assert t + with tqdm(**kwargs) as t: + try: + print(bool(t)) + except TypeError: + pass + else: + raise TypeError("Expected bool(tqdm()) to fail") + + # test with and without disable + with closing(StringIO()) as our_file: + internal(our_file, False) + internal(our_file, True) + + +def backendCheck(module): + """Test tqdm-like module fallback""" + tn = module.tqdm + tr = module.trange + + with closing(StringIO()) as our_file: + with tn(total=10, file=our_file) as t: + assert len(t) == 10 + with tr(1337) as t: + assert len(t) == 1337 + + +def test_auto(): + """Test auto fallback""" + from tqdm import auto, autonotebook + backendCheck(autonotebook) + backendCheck(auto) + + +def test_wrapattr(): + """Test wrapping file-like objects""" + data = "a twenty-char string" + + with closing(StringIO()) as our_file: + with closing(StringIO()) as writer: + with tqdm.wrapattr(writer, "write", file=our_file, bytes=True) as wrap: + wrap.write(data) + res = writer.getvalue() + assert data == res + res = our_file.getvalue() + assert '%.1fB [' % len(data) in res + + with closing(StringIO()) as our_file: + with closing(StringIO()) as writer: + with tqdm.wrapattr(writer, "write", file=our_file, bytes=False) as wrap: + wrap.write(data) + res = our_file.getvalue() + assert '%dit [' % len(data) in res + + +def test_float_progress(): + """Test float totals""" + with closing(StringIO()) as our_file: + with trange(10, total=9.6, file=our_file) as t: + with catch_warnings(record=True) as w: + simplefilter("always", category=TqdmWarning) + for i in t: + if i < 9: + assert not w + assert w + assert "clamping frac" in str(w[-1].message) + + +def test_screen_shape(): + """Test screen shape""" + # ncols + with closing(StringIO()) as our_file: + with trange(10, file=our_file, ncols=50) as t: + list(t) + + res = our_file.getvalue() + assert all(len(i) == 50 for i in get_bar(res)) + + # no second/third bar, leave=False + with closing(StringIO()) as our_file: + kwargs = {'file': our_file, 'ncols': 50, 'nrows': 2, 'miniters': 0, + 'mininterval': 0, 'leave': False} + with trange(10, desc="one", **kwargs) as t1: + with trange(10, desc="two", **kwargs) as t2: + with trange(10, desc="three", **kwargs) as t3: + list(t3) + list(t2) + list(t1) + + res = our_file.getvalue() + assert "one" in res + assert "two" not in res + assert "three" not in res + assert "\n\n" not in res + assert "more hidden" in res + # double-check ncols + assert all(len(i) == 50 for i in get_bar(res) + if i.strip() and "more hidden" not in i) + + # all bars, leave=True + with closing(StringIO()) as our_file: + kwargs = {'file': our_file, 'ncols': 50, 'nrows': 2, + 'miniters': 0, 'mininterval': 0} + with trange(10, desc="one", **kwargs) as t1: + with trange(10, desc="two", **kwargs) as t2: + assert "two" not in our_file.getvalue() + with trange(10, desc="three", **kwargs) as t3: + assert "three" not in our_file.getvalue() + list(t3) + list(t2) + list(t1) + + res = our_file.getvalue() + assert "one" in res + assert "two" in res + assert "three" in res + assert "\n\n" not in res + assert "more hidden" in res + # double-check ncols + assert all(len(i) == 50 for i in get_bar(res) + if i.strip() and "more hidden" not in i) + + # second bar becomes first, leave=False + with closing(StringIO()) as our_file: + kwargs = {'file': our_file, 'ncols': 50, 'nrows': 2, 'miniters': 0, + 'mininterval': 0, 'leave': False} + t1 = tqdm(total=10, desc="one", **kwargs) + with tqdm(total=10, desc="two", **kwargs) as t2: + t1.update() + t2.update() + t1.close() + res = our_file.getvalue() + assert "one" in res + assert "two" not in res + assert "more hidden" in res + t2.update() + + res = our_file.getvalue() + assert "two" in res + + +def test_initial(): + """Test `initial`""" + with closing(StringIO()) as our_file: + for _ in tqdm(_range(9), initial=10, total=19, file=our_file, + miniters=1, mininterval=0): + pass + out = our_file.getvalue() + assert '10/19' in out + assert '19/19' in out + + +def test_colour(): + """Test `colour`""" + with closing(StringIO()) as our_file: + for _ in tqdm(_range(9), file=our_file, colour="#beefed"): + pass + out = our_file.getvalue() + assert '\x1b[38;2;%d;%d;%dm' % (0xbe, 0xef, 0xed) in out + + with catch_warnings(record=True) as w: + simplefilter("always", category=TqdmWarning) + with tqdm(total=1, file=our_file, colour="charm") as t: + assert w + t.update() + assert "Unknown colour" in str(w[-1].message) + + with closing(StringIO()) as our_file2: + for _ in tqdm(_range(9), file=our_file2, colour="blue"): + pass + out = our_file2.getvalue() + assert '\x1b[34m' in out + + +def test_closed(): + """Test writing to closed file""" + with closing(StringIO()) as our_file: + for i in trange(9, file=our_file, miniters=1, mininterval=0): + if i == 5: + our_file.close() + + +def test_reversed(capsys): + """Test reversed()""" + for _ in reversed(tqdm(_range(9))): + pass + out, err = capsys.readouterr() + assert not out + assert ' 0%' in err + assert '100%' in err + + +def test_contains(capsys): + """Test __contains__ doesn't iterate""" + with tqdm(list(range(9))) as t: + assert 9 not in t + assert all(i in t for i in _range(9)) + out, err = capsys.readouterr() + assert not out + assert ' 0%' in err + assert '100%' not in err diff --git a/tests/tests_version.py b/tests/tests_version.py new file mode 100644 index 0000000..495c797 --- /dev/null +++ b/tests/tests_version.py @@ -0,0 +1,14 @@ +"""Test `tqdm.__version__`.""" +import re +from ast import literal_eval + + +def test_version(): + """Test version string""" + from tqdm import __version__ + version_parts = re.split('[.-]', __version__) + if __version__ != "UNKNOWN": + assert 3 <= len(version_parts), "must have at least Major.minor.patch" + assert all( + isinstance(literal_eval(i), int) for i in version_parts[:3] + ), "Version Major.minor.patch must be 3 integers" diff --git a/tests_notebook.ipynb b/tests_notebook.ipynb new file mode 100644 index 0000000..1eec5ca --- /dev/null +++ b/tests_notebook.ipynb @@ -0,0 +1,515 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This file is part of the [test suite](./tests) and will be moved there when [nbval#116](https://github.com/computationalmodelling/nbval/issues/116#issuecomment-793148404) is fixed.\n", + "\n", + "See [DEMO.ipynb](DEMO.ipynb) instead for notebook examples." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from functools import partial\n", + "from time import sleep\n", + "\n", + "from tqdm.notebook import tqdm_notebook\n", + "from tqdm.notebook import tnrange\n", + "\n", + "# avoid displaying widgets by default (pollutes output cells)\n", + "tqdm = partial(tqdm_notebook, display=False)\n", + "trange = partial(tnrange, display=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Help on function display in module tqdm.notebook:\n", + "\n", + "display(self, msg=None, pos=None, close=False, bar_style=None, check_delay=True)\n", + " Use `self.sp` to display `msg` in the specified `pos`.\n", + " \n", + " Consider overloading this function when inheriting to use e.g.:\n", + " `self.some_frontend(**self.format_dict)` instead of `self.sp`.\n", + " \n", + " Parameters\n", + " ----------\n", + " msg : str, optional. What to display (default: `repr(self)`).\n", + " pos : int, optional. Position to `moveto`\n", + " (default: `abs(self.pos)`).\n", + "\n" + ] + } + ], + "source": [ + "help(tqdm_notebook.display)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7c18c038bf964b55941e228503292506", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/9 [00:00<?, ?it/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0\n", + "1\n", + "2\n", + "3\n", + "4\n", + "5\n", + "6\n", + "7\n", + "8\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e29668be41ca4e40b16fb98572b333a5", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/9 [00:00<?, ?it/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# NBVAL_TEST_NAME: basic use\n", + "with tqdm_notebook(range(9)) as t:\n", + " for i in t:\n", + " print(i)\n", + "assert t.container.children[1].bar_style == 'success'\n", + "\n", + "t = tqdm_notebook(total=9)\n", + "t.update()\n", + "t.refresh()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 11%|█ | 1/9 [00:00<00:00, 17.73it/s]\n", + " 20%|██ | 1/5 [00:00<00:00, 341.50it/s]\n" + ] + } + ], + "source": [ + "# NBVAL_TEST_NAME: reset\n", + "print(t)\n", + "t.reset(total=5)\n", + "t.update(1)\n", + "print(t)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# NBVAL_TEST_NAME: bar_style\n", + "assert t.container.children[1].bar_style != 'danger'\n", + "t.close()\n", + "assert t.container.children[1].bar_style == 'danger'" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 0%| | 0/8 [00:00<?, ?it/s]\n", + " 0%| | 0/8 [00:00<?, ?it/s]\n", + "1\n", + " 0%| | 0/8 [00:00<?, ?it/s]\n", + " 0%| | 0/8 [00:00<?, ?it/s]\n" + ] + } + ], + "source": [ + "# NBVAL_TEST_NAME: repr\n", + "with trange(1, 9) as t:\n", + " print(t)\n", + " print(t.container)\n", + " it = iter(t)\n", + " print(next(it))\n", + " print(t)\n", + " print(t.container)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "t = trange(9)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 0%| | 0/9 [00:00<?, ?it/s]\n", + " 0%| | 0/9 [00:00<?, ?it/s]\n" + ] + } + ], + "source": [ + "# NBVAL_TEST_NAME: display pre\n", + "print(t)\n", + "print(t.container)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "for i in t:\n", + " pass" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "100%|██████████| 9/9 [00:00<00:00, 132.23it/s]\n", + "100%|##########| 9/9 [00:00<00:00, 131.02it/s]\n" + ] + } + ], + "source": [ + "# NBVAL_TEST_NAME: display post\n", + "print(t)\n", + "print(t.container)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "no total: 0it [00:00, ?it/s]\n", + "no total: 1it [00:00, 47.83it/s]\n" + ] + } + ], + "source": [ + "# NBVAL_TEST_NAME: no total\n", + "with tqdm(desc=\"no total\") as t:\n", + " print(t)\n", + " t.update()\n", + " print(t)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 0%| | 0/9 [00:00<?, ?it/s]\n", + " 11%|███▍ | 1/9 [00:00<00:00, 45.06it/s]\n" + ] + } + ], + "source": [ + "# NBVAL_TEST_NAME: ncols\n", + "with trange(9, ncols=66) as t:\n", + " print(t)\n", + " for i in t:\n", + " if i == 1:\n", + " break\n", + " print(t)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 0%| | 0/1 [00:00<?, ?it/s]\n", + "100%|██████████| 1/1 [00:00<00:00, 54.60it/s]\n", + " 0%| | 0/9 [00:00<?, ?it/s]\n", + " 11%|█ | 1/9 [00:00<00:00, 57.43it/s]\n" + ] + } + ], + "source": [ + "# NBVAL_TEST_NAME: leave\n", + "def is_hidden(widget):\n", + " return ('hidden', False, None) == (\n", + " getattr(getattr(widget, \"layout\", None), \"visibility\", 'visible'), # ipyw>=8\n", + " getattr(widget, \"visible\", False), getattr(widget, \"_ipython_display_\", None)) # ipyw<8\n", + "\n", + "assert not is_hidden(t.container)\n", + "for total in (1, 9):\n", + " with tqdm(total=total, leave=False) as t:\n", + " print(t)\n", + " t.update()\n", + " print(t)\n", + " assert total != 1 or is_hidden(t.container)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0it [00:00, ?it/s]\n", + "1it [00:00, 47.87it/s]\n" + ] + } + ], + "source": [ + "# NBVAL_TEST_NAME: no total\n", + "with tqdm() as t:\n", + " print(t)\n", + " t.update()\n", + " print(t)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "disable: None\n", + " 0%| | 0/1 [00:00<?, ?it/s]\n", + "100%|██████████| 1/1 [00:00<00:00, 53.24it/s]\n", + " 0%| | 0/9 [00:00<?, ?it/s]\n", + " 11%|█ | 1/9 [00:00<00:00, 1752.01it/s]\n", + "0it [00:00, ?it/s]\n", + "1it [00:00, 35.88it/s]\n", + " 0%| | 0/1 [00:00<?, ?it/s]\n", + "100%|██████████| 1/1 [00:00<00:00, 1880.85it/s]\n", + "disable: True\n", + " 0%| | 0/1 [00:00<?, ?it/s]\n", + " 0%| | 0/1 [00:00<?, ?it/s]\n", + " 0%| | 0/9 [00:00<?, ?it/s]\n", + " 0%| | 0/9 [00:00<?, ?it/s]\n", + "0it [00:00, ?it/s]\n", + "0it [00:00, ?it/s]\n", + " 0%| | 0/1 [00:00<?, ?it/s]\n", + " 0%| | 0/1 [00:00<?, ?it/s]\n" + ] + } + ], + "source": [ + "# NBVAL_TEST_NAME: reset and disable\n", + "for disable in (None, True):\n", + " print(\"disable:\", disable)\n", + " with tqdm(total=1, disable=disable) as t:\n", + " print(t)\n", + " t.update()\n", + " print(t)\n", + "\n", + " t.reset(total=9)\n", + " print(t)\n", + " t.update()\n", + " print(t)\n", + " with tqdm(disable=disable) as t:\n", + " print(t)\n", + " t.update()\n", + " print(t)\n", + "\n", + " t.reset(total=1)\n", + " print(t)\n", + " t.update()\n", + " print(t)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 0%|| 0/1 [00:00<?, ?it/s]\n", + "100%|| 1/1 [00:00<00:00, 32.57it/s]\n", + " 0%| \n", + "100%|██████████\n" + ] + } + ], + "source": [ + "# NBVAL_TEST_NAME: bar_format\n", + "with tqdm(total=1, bar_format='{l_bar}{r_bar}') as t:\n", + " print(t)\n", + " t.update()\n", + " print(t)\n", + "\n", + "with tqdm(total=1, bar_format='{l_bar}{bar}') as t:\n", + " print(t)\n", + " t.update()\n", + " print(t)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 0%|\u001b[33m \u001b[0m| 0/1 [00:00<?, ?it/s]\n", + "100%|\u001b[33m██████████\u001b[0m| 1/1 [00:00<00:00, 47.14it/s]\n" + ] + } + ], + "source": [ + "# NBVAL_TEST_NAME: colour\n", + "assert t.colour != 'yellow'\n", + "with tqdm(total=1, colour='yellow') as t:\n", + " print(t)\n", + " t.update()\n", + " print(t)\n", + " assert t.colour == 'yellow'" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "# NBVAL_TEST_NAME: delay no trigger\n", + "with tqdm_notebook(total=1, delay=10) as t:\n", + " t.update()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "fe102eedbb4f437783fbd0cff32f6613", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "100%|##########| 1/1 [00:00<00:00, 7.68it/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# NBVAL_TEST_NAME: delay trigger\n", + "with tqdm_notebook(total=1, delay=0.1) as t:\n", + " sleep(0.1)\n", + " t.update()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:tqdm]", + "language": "python", + "name": "conda-env-tqdm-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython" + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} @@ -0,0 +1,88 @@ +# Tox (https://tox.testrun.org/) is a tool for running tests +# in multiple virtualenvs. This configuration file will run the +# test suite on all supported python versions. To use it, "pip install tox" +# and then run "tox" from this directory. + +[tox] +# deprecation warning: py{27,py2,34,35,36} +envlist=py{27,34,35,36,37,38,39,310,py2,py3}{,-tf}{,-keras}, perf, setup.py +isolated_build=True + +[gh-actions] +python= + 2.7: py27 + 3.5: py35 + 3.6: py36 + 3.7: py37 + 3.8: py38 + 3.9: py39 + 3.10: py310 + pypy-2.7: pypy2 + pypy-3.7: pypy3 +[gh-actions:env] +PLATFORM= + ubuntu: tf-keras + +[core] +deps= + pytest + py3{4,5,6}: pytest<7 + pytest-cov + pytest-timeout + py3{7,8,9,10}: pytest-asyncio + py3{6,7,8,9,10}: ipywidgets + py3{7,8,9,10}: git+https://github.com/casperdcl/nbval.git@master#egg=nbval + coverage + coveralls + codecov +commands= + - coveralls + codecov -X pycov -e TOXENV + - codacy report -l Python -r coverage.xml --partial + +[testenv] +passenv=TOXENV CI GITHUB_* CODECOV_* COVERALLS_* CODACY_* HOME +deps= + {[core]deps} + cython + dask[delayed] + matplotlib + numpy + pandas + tf: tensorflow!=2.5.0 + !py27-keras: keras + py27-keras: keras<2.5 + py35-keras: keras<2.7 + py27-tf: protobuf<3.18 + py3{6,7,8,9,10}: rich +commands= + py3{4,5,6}: pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" -o addopts= -v --tb=short -rxs -W=error --durations=0 --durations-min=0.1 + py3{7,8,9,10}: pytest --cov=tqdm --cov-report= tests_notebook.ipynb --nbval --nbval-current-env -W=ignore --nbval-sanitize-with=setup.cfg + py3{7,8,9,10}: pytest --cov=tqdm --cov-report=xml --cov-report=term --cov-append -k "not perf" + {[core]commands} +allowlist_externals=codacy + +[testenv:py{27,py2}{,-tf}{,-keras}] +commands= + pytest --cov=tqdm --cov-report=xml --cov-report=term -k "not perf" -o addopts= -v --tb=short -rxs -W=error --durations=10 + {[core]commands} + +# no cython/numpy/pandas +[testenv:py{34,py2,py3}] +deps={[core]deps} + +[testenv:perf] +deps= + pytest + pytest-timeout + pytest-asyncio +commands=pytest -k perf + +[testenv:setup.py] +deps= + docutils + pygments + py-make>=0.1.0 +commands= + {envpython} setup.py check --restructuredtext --metadata --strict + {envpython} setup.py make none diff --git a/tqdm.egg-info/PKG-INFO b/tqdm.egg-info/PKG-INFO new file mode 100644 index 0000000..d796709 --- /dev/null +++ b/tqdm.egg-info/PKG-INFO @@ -0,0 +1,1581 @@ +Metadata-Version: 2.1 +Name: tqdm +Version: 4.64.1 +Summary: Fast, Extensible Progress Meter +Home-page: https://tqdm.github.io +Maintainer: tqdm developers +Maintainer-email: python.tqdm@gmail.com +License: MPLv2.0, MIT Licences +Project-URL: Changelog, https://tqdm.github.io/releases +Project-URL: Source, https://github.com/tqdm/tqdm +Project-URL: Wiki, https://github.com/tqdm/tqdm/wiki +Keywords: progressbar,progressmeter,progress,bar,meter,rate,eta,console,terminal,time +Platform: any +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Console +Classifier: Environment :: MacOS X +Classifier: Environment :: Other Environment +Classifier: Environment :: Win32 (MS Windows) +Classifier: Environment :: X11 Applications +Classifier: Framework :: IPython +Classifier: Framework :: Jupyter +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Education +Classifier: Intended Audience :: End Users/Desktop +Classifier: Intended Audience :: Other Audience +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: MIT License +Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0) +Classifier: Operating System :: MacOS +Classifier: Operating System :: MacOS :: MacOS X +Classifier: Operating System :: Microsoft +Classifier: Operating System :: Microsoft :: MS-DOS +Classifier: Operating System :: Microsoft :: Windows +Classifier: Operating System :: POSIX +Classifier: Operating System :: POSIX :: BSD +Classifier: Operating System :: POSIX :: BSD :: FreeBSD +Classifier: Operating System :: POSIX :: Linux +Classifier: Operating System :: POSIX :: SunOS/Solaris +Classifier: Operating System :: Unix +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: Implementation +Classifier: Programming Language :: Python :: Implementation :: IronPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Programming Language :: Unix Shell +Classifier: Topic :: Desktop Environment +Classifier: Topic :: Education :: Computer Aided Instruction (CAI) +Classifier: Topic :: Education :: Testing +Classifier: Topic :: Office/Business +Classifier: Topic :: Other/Nonlisted Topic +Classifier: Topic :: Software Development :: Build Tools +Classifier: Topic :: Software Development :: Libraries +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Software Development :: Pre-processors +Classifier: Topic :: Software Development :: User Interfaces +Classifier: Topic :: System :: Installation/Setup +Classifier: Topic :: System :: Logging +Classifier: Topic :: System :: Monitoring +Classifier: Topic :: System :: Shells +Classifier: Topic :: Terminals +Classifier: Topic :: Utilities +Provides: tqdm +Requires-Python: !=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7 +Description-Content-Type: text/x-rst +Provides-Extra: dev +Provides-Extra: slack +Provides-Extra: telegram +Provides-Extra: notebook +License-File: LICENCE + +|Logo| + +tqdm +==== + +|Py-Versions| |Versions| |Conda-Forge-Status| |Docker| |Snapcraft| + +|Build-Status| |Coverage-Status| |Branch-Coverage-Status| |Codacy-Grade| |Libraries-Rank| |PyPI-Downloads| + +|LICENCE| |OpenHub-Status| |binder-demo| |awesome-python| + +``tqdm`` derives from the Arabic word *taqaddum* (تقدّم) which can mean "progress," +and is an abbreviation for "I love you so much" in Spanish (*te quiero demasiado*). + +Instantly make your loops show a smart progress meter - just wrap any +iterable with ``tqdm(iterable)``, and you're done! + +.. code:: python + + from tqdm import tqdm + for i in tqdm(range(10000)): + ... + +``76%|████████████████████████ | 7568/10000 [00:33<00:10, 229.00it/s]`` + +``trange(N)`` can be also used as a convenient shortcut for +``tqdm(range(N))``. + +|Screenshot| + |Video| |Slides| |Merch| + +It can also be executed as a module with pipes: + +.. code:: sh + + $ seq 9999999 | tqdm --bytes | wc -l + 75.2MB [00:00, 217MB/s] + 9999999 + + $ tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` \ + > backup.tgz + 32%|██████████▍ | 8.89G/27.9G [00:42<01:31, 223MB/s] + +Overhead is low -- about 60ns per iteration (80ns with ``tqdm.gui``), and is +unit tested against performance regression. +By comparison, the well-established +`ProgressBar <https://github.com/niltonvolpato/python-progressbar>`__ has +an 800ns/iter overhead. + +In addition to its low overhead, ``tqdm`` uses smart algorithms to predict +the remaining time and to skip unnecessary iteration displays, which allows +for a negligible overhead in most cases. + +``tqdm`` works on any platform +(Linux, Windows, Mac, FreeBSD, NetBSD, Solaris/SunOS), +in any console or in a GUI, and is also friendly with IPython/Jupyter notebooks. + +``tqdm`` does not require any dependencies (not even ``curses``!), just +Python and an environment supporting ``carriage return \r`` and +``line feed \n`` control characters. + +------------------------------------------ + +.. contents:: Table of contents + :backlinks: top + :local: + + +Installation +------------ + +Latest PyPI stable release +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +|Versions| |PyPI-Downloads| |Libraries-Dependents| + +.. code:: sh + + pip install tqdm + +Latest development release on GitHub +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +|GitHub-Status| |GitHub-Stars| |GitHub-Commits| |GitHub-Forks| |GitHub-Updated| + +Pull and install pre-release ``devel`` branch: + +.. code:: sh + + pip install "git+https://github.com/tqdm/tqdm.git@devel#egg=tqdm" + +Latest Conda release +~~~~~~~~~~~~~~~~~~~~ + +|Conda-Forge-Status| + +.. code:: sh + + conda install -c conda-forge tqdm + +Latest Snapcraft release +~~~~~~~~~~~~~~~~~~~~~~~~ + +|Snapcraft| + +There are 3 channels to choose from: + +.. code:: sh + + snap install tqdm # implies --stable, i.e. latest tagged release + snap install tqdm --candidate # master branch + snap install tqdm --edge # devel branch + +Note that ``snap`` binaries are purely for CLI use (not ``import``-able), and +automatically set up ``bash`` tab-completion. + +Latest Docker release +~~~~~~~~~~~~~~~~~~~~~ + +|Docker| + +.. code:: sh + + docker pull tqdm/tqdm + docker run -i --rm tqdm/tqdm --help + +Other +~~~~~ + +There are other (unofficial) places where ``tqdm`` may be downloaded, particularly for CLI use: + +|Repology| + +.. |Repology| image:: https://repology.org/badge/tiny-repos/python:tqdm.svg + :target: https://repology.org/project/python:tqdm/versions + +Changelog +--------- + +The list of all changes is available either on GitHub's Releases: +|GitHub-Status|, on the +`wiki <https://github.com/tqdm/tqdm/wiki/Releases>`__, or on the +`website <https://tqdm.github.io/releases>`__. + + +Usage +----- + +``tqdm`` is very versatile and can be used in a number of ways. +The three main ones are given below. + +Iterable-based +~~~~~~~~~~~~~~ + +Wrap ``tqdm()`` around any iterable: + +.. code:: python + + from tqdm import tqdm + from time import sleep + + text = "" + for char in tqdm(["a", "b", "c", "d"]): + sleep(0.25) + text = text + char + +``trange(i)`` is a special optimised instance of ``tqdm(range(i))``: + +.. code:: python + + from tqdm import trange + + for i in trange(100): + sleep(0.01) + +Instantiation outside of the loop allows for manual control over ``tqdm()``: + +.. code:: python + + pbar = tqdm(["a", "b", "c", "d"]) + for char in pbar: + sleep(0.25) + pbar.set_description("Processing %s" % char) + +Manual +~~~~~~ + +Manual control of ``tqdm()`` updates using a ``with`` statement: + +.. code:: python + + with tqdm(total=100) as pbar: + for i in range(10): + sleep(0.1) + pbar.update(10) + +If the optional variable ``total`` (or an iterable with ``len()``) is +provided, predictive stats are displayed. + +``with`` is also optional (you can just assign ``tqdm()`` to a variable, +but in this case don't forget to ``del`` or ``close()`` at the end: + +.. code:: python + + pbar = tqdm(total=100) + for i in range(10): + sleep(0.1) + pbar.update(10) + pbar.close() + +Module +~~~~~~ + +Perhaps the most wonderful use of ``tqdm`` is in a script or on the command +line. Simply inserting ``tqdm`` (or ``python -m tqdm``) between pipes will pass +through all ``stdin`` to ``stdout`` while printing progress to ``stderr``. + +The example below demonstrate counting the number of lines in all Python files +in the current directory, with timing information included. + +.. code:: sh + + $ time find . -name '*.py' -type f -exec cat \{} \; | wc -l + 857365 + + real 0m3.458s + user 0m0.274s + sys 0m3.325s + + $ time find . -name '*.py' -type f -exec cat \{} \; | tqdm | wc -l + 857366it [00:03, 246471.31it/s] + 857365 + + real 0m3.585s + user 0m0.862s + sys 0m3.358s + +Note that the usual arguments for ``tqdm`` can also be specified. + +.. code:: sh + + $ find . -name '*.py' -type f -exec cat \{} \; | + tqdm --unit loc --unit_scale --total 857366 >> /dev/null + 100%|█████████████████████████████████| 857K/857K [00:04<00:00, 246Kloc/s] + +Backing up a large directory? + +.. code:: sh + + $ tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` \ + > backup.tgz + 44%|██████████████▊ | 153M/352M [00:14<00:18, 11.0MB/s] + +This can be beautified further: + +.. code:: sh + + $ BYTES="$(du -sb docs/ | cut -f1)" + $ tar -cf - docs/ \ + | tqdm --bytes --total "$BYTES" --desc Processing | gzip \ + | tqdm --bytes --total "$BYTES" --desc Compressed --position 1 \ + > ~/backup.tgz + Processing: 100%|██████████████████████| 352M/352M [00:14<00:00, 30.2MB/s] + Compressed: 42%|█████████▎ | 148M/352M [00:14<00:19, 10.9MB/s] + +Or done on a file level using 7-zip: + +.. code:: sh + + $ 7z a -bd -r backup.7z docs/ | grep Compressing \ + | tqdm --total $(find docs/ -type f | wc -l) --unit files \ + | grep -v Compressing + 100%|██████████████████████████▉| 15327/15327 [01:00<00:00, 712.96files/s] + +Pre-existing CLI programs already outputting basic progress information will +benefit from ``tqdm``'s ``--update`` and ``--update_to`` flags: + +.. code:: sh + + $ seq 3 0.1 5 | tqdm --total 5 --update_to --null + 100%|████████████████████████████████████| 5.0/5 [00:00<00:00, 9673.21it/s] + $ seq 10 | tqdm --update --null # 1 + 2 + ... + 10 = 55 iterations + 55it [00:00, 90006.52it/s] + +FAQ and Known Issues +-------------------- + +|GitHub-Issues| + +The most common issues relate to excessive output on multiple lines, instead +of a neat one-line progress bar. + +- Consoles in general: require support for carriage return (``CR``, ``\r``). +- Nested progress bars: + + * Consoles in general: require support for moving cursors up to the + previous line. For example, + `IDLE <https://github.com/tqdm/tqdm/issues/191#issuecomment-230168030>`__, + `ConEmu <https://github.com/tqdm/tqdm/issues/254>`__ and + `PyCharm <https://github.com/tqdm/tqdm/issues/203>`__ (also + `here <https://github.com/tqdm/tqdm/issues/208>`__, + `here <https://github.com/tqdm/tqdm/issues/307>`__, and + `here <https://github.com/tqdm/tqdm/issues/454#issuecomment-335416815>`__) + lack full support. + * Windows: additionally may require the Python module ``colorama`` + to ensure nested bars stay within their respective lines. + +- Unicode: + + * Environments which report that they support unicode will have solid smooth + progressbars. The fallback is an ``ascii``-only bar. + * Windows consoles often only partially support unicode and thus + `often require explicit ascii=True <https://github.com/tqdm/tqdm/issues/454#issuecomment-335416815>`__ + (also `here <https://github.com/tqdm/tqdm/issues/499>`__). This is due to + either normal-width unicode characters being incorrectly displayed as + "wide", or some unicode characters not rendering. + +- Wrapping generators: + + * Generator wrapper functions tend to hide the length of iterables. + ``tqdm`` does not. + * Replace ``tqdm(enumerate(...))`` with ``enumerate(tqdm(...))`` or + ``tqdm(enumerate(x), total=len(x), ...)``. + The same applies to ``numpy.ndenumerate``. + * Replace ``tqdm(zip(a, b))`` with ``zip(tqdm(a), b)`` or even + ``zip(tqdm(a), tqdm(b))``. + * The same applies to ``itertools``. + * Some useful convenience functions can be found under ``tqdm.contrib``. + +- `Hanging pipes in python2 <https://github.com/tqdm/tqdm/issues/359>`__: + when using ``tqdm`` on the CLI, you may need to use Python 3.5+ for correct + buffering. +- `No intermediate output in docker-compose <https://github.com/tqdm/tqdm/issues/771>`__: + use ``docker-compose run`` instead of ``docker-compose up`` and ``tty: true``. + +If you come across any other difficulties, browse and file |GitHub-Issues|. + +Documentation +------------- + +|Py-Versions| |README-Hits| (Since 19 May 2016) + +.. code:: python + + class tqdm(): + """ + Decorate an iterable object, returning an iterator which acts exactly + like the original iterable, but prints a dynamically updating + progressbar every time a value is requested. + """ + + def __init__(self, iterable=None, desc=None, total=None, leave=True, + file=None, ncols=None, mininterval=0.1, + maxinterval=10.0, miniters=None, ascii=None, disable=False, + unit='it', unit_scale=False, dynamic_ncols=False, + smoothing=0.3, bar_format=None, initial=0, position=None, + postfix=None, unit_divisor=1000): + +Parameters +~~~~~~~~~~ + +* iterable : iterable, optional + Iterable to decorate with a progressbar. + Leave blank to manually manage the updates. +* desc : str, optional + Prefix for the progressbar. +* total : int or float, optional + The number of expected iterations. If unspecified, + len(iterable) is used if possible. If float("inf") or as a last + resort, only basic progress statistics are displayed + (no ETA, no progressbar). + If ``gui`` is True and this parameter needs subsequent updating, + specify an initial arbitrary large positive number, + e.g. 9e9. +* leave : bool, optional + If [default: True], keeps all traces of the progressbar + upon termination of iteration. + If ``None``, will leave only if ``position`` is ``0``. +* file : ``io.TextIOWrapper`` or ``io.StringIO``, optional + Specifies where to output the progress messages + (default: sys.stderr). Uses ``file.write(str)`` and ``file.flush()`` + methods. For encoding, see ``write_bytes``. +* ncols : int, optional + The width of the entire output message. If specified, + dynamically resizes the progressbar to stay within this bound. + If unspecified, attempts to use environment width. The + fallback is a meter width of 10 and no limit for the counter and + statistics. If 0, will not print any meter (only stats). +* mininterval : float, optional + Minimum progress display update interval [default: 0.1] seconds. +* maxinterval : float, optional + Maximum progress display update interval [default: 10] seconds. + Automatically adjusts ``miniters`` to correspond to ``mininterval`` + after long display update lag. Only works if ``dynamic_miniters`` + or monitor thread is enabled. +* miniters : int or float, optional + Minimum progress display update interval, in iterations. + If 0 and ``dynamic_miniters``, will automatically adjust to equal + ``mininterval`` (more CPU efficient, good for tight loops). + If > 0, will skip display of specified number of iterations. + Tweak this and ``mininterval`` to get very efficient loops. + If your progress is erratic with both fast and slow iterations + (network, skipping items, etc) you should set miniters=1. +* ascii : bool or str, optional + If unspecified or False, use unicode (smooth blocks) to fill + the meter. The fallback is to use ASCII characters " 123456789#". +* disable : bool, optional + Whether to disable the entire progressbar wrapper + [default: False]. If set to None, disable on non-TTY. +* unit : str, optional + String that will be used to define the unit of each iteration + [default: it]. +* unit_scale : bool or int or float, optional + If 1 or True, the number of iterations will be reduced/scaled + automatically and a metric prefix following the + International System of Units standard will be added + (kilo, mega, etc.) [default: False]. If any other non-zero + number, will scale ``total`` and ``n``. +* dynamic_ncols : bool, optional + If set, constantly alters ``ncols`` and ``nrows`` to the + environment (allowing for window resizes) [default: False]. +* smoothing : float, optional + Exponential moving average smoothing factor for speed estimates + (ignored in GUI mode). Ranges from 0 (average speed) to 1 + (current/instantaneous speed) [default: 0.3]. +* bar_format : str, optional + Specify a custom bar string formatting. May impact performance. + [default: '{l_bar}{bar}{r_bar}'], where + l_bar='{desc}: {percentage:3.0f}%|' and + r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, ' + '{rate_fmt}{postfix}]' + Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, + percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, + rate, rate_fmt, rate_noinv, rate_noinv_fmt, + rate_inv, rate_inv_fmt, postfix, unit_divisor, + remaining, remaining_s, eta. + Note that a trailing ": " is automatically removed after {desc} + if the latter is empty. +* initial : int or float, optional + The initial counter value. Useful when restarting a progress + bar [default: 0]. If using float, consider specifying ``{n:.3f}`` + or similar in ``bar_format``, or specifying ``unit_scale``. +* position : int, optional + Specify the line offset to print this bar (starting from 0) + Automatic if unspecified. + Useful to manage multiple bars at once (eg, from threads). +* postfix : dict or ``*``, optional + Specify additional stats to display at the end of the bar. + Calls ``set_postfix(**postfix)`` if possible (dict). +* unit_divisor : float, optional + [default: 1000], ignored unless ``unit_scale`` is True. +* write_bytes : bool, optional + If (default: None) and ``file`` is unspecified, + bytes will be written in Python 2. If ``True`` will also write + bytes. In all other cases will default to unicode. +* lock_args : tuple, optional + Passed to ``refresh`` for intermediate output + (initialisation, iterating, and updating). +* nrows : int, optional + The screen height. If specified, hides nested bars outside this + bound. If unspecified, attempts to use environment height. + The fallback is 20. +* colour : str, optional + Bar colour (e.g. 'green', '#00ff00'). +* delay : float, optional + Don't display until [default: 0] seconds have elapsed. + +Extra CLI Options +~~~~~~~~~~~~~~~~~ + +* delim : chr, optional + Delimiting character [default: '\n']. Use '\0' for null. + N.B.: on Windows systems, Python converts '\n' to '\r\n'. +* buf_size : int, optional + String buffer size in bytes [default: 256] + used when ``delim`` is specified. +* bytes : bool, optional + If true, will count bytes, ignore ``delim``, and default + ``unit_scale`` to True, ``unit_divisor`` to 1024, and ``unit`` to 'B'. +* tee : bool, optional + If true, passes ``stdin`` to both ``stderr`` and ``stdout``. +* update : bool, optional + If true, will treat input as newly elapsed iterations, + i.e. numbers to pass to ``update()``. Note that this is slow + (~2e5 it/s) since every input must be decoded as a number. +* update_to : bool, optional + If true, will treat input as total elapsed iterations, + i.e. numbers to assign to ``self.n``. Note that this is slow + (~2e5 it/s) since every input must be decoded as a number. +* null : bool, optional + If true, will discard input (no stdout). +* manpath : str, optional + Directory in which to install tqdm man pages. +* comppath : str, optional + Directory in which to place tqdm completion. +* log : str, optional + CRITICAL|FATAL|ERROR|WARN(ING)|[default: 'INFO']|DEBUG|NOTSET. + +Returns +~~~~~~~ + +* out : decorated iterator. + +.. code:: python + + class tqdm(): + def update(self, n=1): + """ + Manually update the progress bar, useful for streams + such as reading files. + E.g.: + >>> t = tqdm(total=filesize) # Initialise + >>> for current_buffer in stream: + ... ... + ... t.update(len(current_buffer)) + >>> t.close() + The last line is highly recommended, but possibly not necessary if + ``t.update()`` will be called in such a way that ``filesize`` will be + exactly reached and printed. + + Parameters + ---------- + n : int or float, optional + Increment to add to the internal counter of iterations + [default: 1]. If using float, consider specifying ``{n:.3f}`` + or similar in ``bar_format``, or specifying ``unit_scale``. + + Returns + ------- + out : bool or None + True if a ``display()`` was triggered. + """ + + def close(self): + """Cleanup and (if leave=False) close the progressbar.""" + + def clear(self, nomove=False): + """Clear current bar display.""" + + def refresh(self): + """ + Force refresh the display of this bar. + + Parameters + ---------- + nolock : bool, optional + If ``True``, does not lock. + If [default: ``False``]: calls ``acquire()`` on internal lock. + lock_args : tuple, optional + Passed to internal lock's ``acquire()``. + If specified, will only ``display()`` if ``acquire()`` returns ``True``. + """ + + def unpause(self): + """Restart tqdm timer from last print time.""" + + def reset(self, total=None): + """ + Resets to 0 iterations for repeated use. + + Consider combining with ``leave=True``. + + Parameters + ---------- + total : int or float, optional. Total to use for the new bar. + """ + + def set_description(self, desc=None, refresh=True): + """ + Set/modify description of the progress bar. + + Parameters + ---------- + desc : str, optional + refresh : bool, optional + Forces refresh [default: True]. + """ + + def set_postfix(self, ordered_dict=None, refresh=True, **tqdm_kwargs): + """ + Set/modify postfix (additional stats) + with automatic formatting based on datatype. + + Parameters + ---------- + ordered_dict : dict or OrderedDict, optional + refresh : bool, optional + Forces refresh [default: True]. + kwargs : dict, optional + """ + + @classmethod + def write(cls, s, file=sys.stdout, end="\n"): + """Print a message via tqdm (without overlap with bars).""" + + @property + def format_dict(self): + """Public API for read-only member access.""" + + def display(self, msg=None, pos=None): + """ + Use ``self.sp`` to display ``msg`` in the specified ``pos``. + + Consider overloading this function when inheriting to use e.g.: + ``self.some_frontend(**self.format_dict)`` instead of ``self.sp``. + + Parameters + ---------- + msg : str, optional. What to display (default: ``repr(self)``). + pos : int, optional. Position to ``moveto`` + (default: ``abs(self.pos)``). + """ + + @classmethod + @contextmanager + def wrapattr(cls, stream, method, total=None, bytes=True, **tqdm_kwargs): + """ + stream : file-like object. + method : str, "read" or "write". The result of ``read()`` and + the first argument of ``write()`` should have a ``len()``. + + >>> with tqdm.wrapattr(file_obj, "read", total=file_obj.size) as fobj: + ... while True: + ... chunk = fobj.read(chunk_size) + ... if not chunk: + ... break + """ + + @classmethod + def pandas(cls, *targs, **tqdm_kwargs): + """Registers the current `tqdm` class with `pandas`.""" + + def trange(*args, **tqdm_kwargs): + """ + A shortcut for `tqdm(xrange(*args), **tqdm_kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + +Convenience Functions +~~~~~~~~~~~~~~~~~~~~~ + +.. code:: python + + def tqdm.contrib.tenumerate(iterable, start=0, total=None, + tqdm_class=tqdm.auto.tqdm, **tqdm_kwargs): + """Equivalent of `numpy.ndenumerate` or builtin `enumerate`.""" + + def tqdm.contrib.tzip(iter1, *iter2plus, **tqdm_kwargs): + """Equivalent of builtin `zip`.""" + + def tqdm.contrib.tmap(function, *sequences, **tqdm_kwargs): + """Equivalent of builtin `map`.""" + +Submodules +~~~~~~~~~~ + +.. code:: python + + class tqdm.notebook.tqdm(tqdm.tqdm): + """IPython/Jupyter Notebook widget.""" + + class tqdm.auto.tqdm(tqdm.tqdm): + """Automatically chooses beween `tqdm.notebook` and `tqdm.tqdm`.""" + + class tqdm.asyncio.tqdm(tqdm.tqdm): + """Asynchronous version.""" + @classmethod + def as_completed(cls, fs, *, loop=None, timeout=None, total=None, + **tqdm_kwargs): + """Wrapper for `asyncio.as_completed`.""" + + class tqdm.gui.tqdm(tqdm.tqdm): + """Matplotlib GUI version.""" + + class tqdm.tk.tqdm(tqdm.tqdm): + """Tkinter GUI version.""" + + class tqdm.rich.tqdm(tqdm.tqdm): + """`rich.progress` version.""" + + class tqdm.keras.TqdmCallback(keras.callbacks.Callback): + """Keras callback for epoch and batch progress.""" + + class tqdm.dask.TqdmCallback(dask.callbacks.Callback): + """Dask callback for task progress.""" + + +``contrib`` ++++++++++++ + +The ``tqdm.contrib`` package also contains experimental modules: + +- ``tqdm.contrib.itertools``: Thin wrappers around ``itertools`` +- ``tqdm.contrib.concurrent``: Thin wrappers around ``concurrent.futures`` +- ``tqdm.contrib.slack``: Posts to `Slack <https://slack.com>`__ bots +- ``tqdm.contrib.discord``: Posts to `Discord <https://discord.com>`__ bots +- ``tqdm.contrib.telegram``: Posts to `Telegram <https://telegram.org>`__ bots +- ``tqdm.contrib.bells``: Automagically enables all optional features + + * ``auto``, ``pandas``, ``slack``, ``discord``, ``telegram`` + +Examples and Advanced Usage +--------------------------- + +- See the `examples <https://github.com/tqdm/tqdm/tree/master/examples>`__ + folder; +- import the module and run ``help()``; +- consult the `wiki <https://github.com/tqdm/tqdm/wiki>`__; + + * this has an + `excellent article <https://github.com/tqdm/tqdm/wiki/How-to-make-a-great-Progress-Bar>`__ + on how to make a **great** progressbar; + +- check out the `slides from PyData London <https://tqdm.github.io/PyData2019/slides.html>`__, or +- run the |binder-demo|. + +Description and additional stats +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Custom information can be displayed and updated dynamically on ``tqdm`` bars +with the ``desc`` and ``postfix`` arguments: + +.. code:: python + + from tqdm import tqdm, trange + from random import random, randint + from time import sleep + + with trange(10) as t: + for i in t: + # Description will be displayed on the left + t.set_description('GEN %i' % i) + # Postfix will be displayed on the right, + # formatted automatically based on argument's datatype + t.set_postfix(loss=random(), gen=randint(1,999), str='h', + lst=[1, 2]) + sleep(0.1) + + with tqdm(total=10, bar_format="{postfix[0]} {postfix[1][value]:>8.2g}", + postfix=["Batch", dict(value=0)]) as t: + for i in range(10): + sleep(0.1) + t.postfix[1]["value"] = i / 2 + t.update() + +Points to remember when using ``{postfix[...]}`` in the ``bar_format`` string: + +- ``postfix`` also needs to be passed as an initial argument in a compatible + format, and +- ``postfix`` will be auto-converted to a string if it is a ``dict``-like + object. To prevent this behaviour, insert an extra item into the dictionary + where the key is not a string. + +Additional ``bar_format`` parameters may also be defined by overriding +``format_dict``, and the bar itself may be modified using ``ascii``: + +.. code:: python + + from tqdm import tqdm + class TqdmExtraFormat(tqdm): + """Provides a `total_time` format parameter""" + @property + def format_dict(self): + d = super(TqdmExtraFormat, self).format_dict + total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1) + d.update(total_time=self.format_interval(total_time) + " in total") + return d + + for i in TqdmExtraFormat( + range(9), ascii=" .oO0", + bar_format="{total_time}: {percentage:.0f}%|{bar}{r_bar}"): + if i == 4: + break + +.. code:: + + 00:00 in total: 44%|0000. | 4/9 [00:00<00:00, 962.93it/s] + +Note that ``{bar}`` also supports a format specifier ``[width][type]``. + +- ``width`` + + * unspecified (default): automatic to fill ``ncols`` + * ``int >= 0``: fixed width overriding ``ncols`` logic + * ``int < 0``: subtract from the automatic default + +- ``type`` + + * ``a``: ascii (``ascii=True`` override) + * ``u``: unicode (``ascii=False`` override) + * ``b``: blank (``ascii=" "`` override) + +This means a fixed bar with right-justified text may be created by using: +``bar_format="{l_bar}{bar:10}|{bar:-10b}right-justified"`` + +Nested progress bars +~~~~~~~~~~~~~~~~~~~~ + +``tqdm`` supports nested progress bars. Here's an example: + +.. code:: python + + from tqdm.auto import trange + from time import sleep + + for i in trange(4, desc='1st loop'): + for j in trange(5, desc='2nd loop'): + for k in trange(50, desc='3rd loop', leave=False): + sleep(0.01) + +For manual control over positioning (e.g. for multi-processing use), +you may specify ``position=n`` where ``n=0`` for the outermost bar, +``n=1`` for the next, and so on. +However, it's best to check if ``tqdm`` can work without manual ``position`` +first. + +.. code:: python + + from time import sleep + from tqdm import trange, tqdm + from multiprocessing import Pool, RLock, freeze_support + + L = list(range(9)) + + def progresser(n): + interval = 0.001 / (n + 2) + total = 5000 + text = "#{}, est. {:<04.2}s".format(n, interval * total) + for _ in trange(total, desc=text, position=n): + sleep(interval) + + if __name__ == '__main__': + freeze_support() # for Windows support + tqdm.set_lock(RLock()) # for managing output contention + p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),)) + p.map(progresser, L) + +Note that in Python 3, ``tqdm.write`` is thread-safe: + +.. code:: python + + from time import sleep + from tqdm import tqdm, trange + from concurrent.futures import ThreadPoolExecutor + + L = list(range(9)) + + def progresser(n): + interval = 0.001 / (n + 2) + total = 5000 + text = "#{}, est. {:<04.2}s".format(n, interval * total) + for _ in trange(total, desc=text): + sleep(interval) + if n == 6: + tqdm.write("n == 6 completed.") + tqdm.write("`tqdm.write()` is thread-safe in py3!") + + if __name__ == '__main__': + with ThreadPoolExecutor() as p: + p.map(progresser, L) + +Hooks and callbacks +~~~~~~~~~~~~~~~~~~~ + +``tqdm`` can easily support callbacks/hooks and manual updates. +Here's an example with ``urllib``: + +**``urllib.urlretrieve`` documentation** + + | [...] + | If present, the hook function will be called once + | on establishment of the network connection and once after each block read + | thereafter. The hook will be passed three arguments; a count of blocks + | transferred so far, a block size in bytes, and the total size of the file. + | [...] + +.. code:: python + + import urllib, os + from tqdm import tqdm + urllib = getattr(urllib, 'request', urllib) + + class TqdmUpTo(tqdm): + """Provides `update_to(n)` which uses `tqdm.update(delta_n)`.""" + def update_to(self, b=1, bsize=1, tsize=None): + """ + b : int, optional + Number of blocks transferred so far [default: 1]. + bsize : int, optional + Size of each block (in tqdm units) [default: 1]. + tsize : int, optional + Total size (in tqdm units). If [default: None] remains unchanged. + """ + if tsize is not None: + self.total = tsize + return self.update(b * bsize - self.n) # also sets self.n = b * bsize + + eg_link = "https://caspersci.uk.to/matryoshka.zip" + with TqdmUpTo(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, + desc=eg_link.split('/')[-1]) as t: # all optional kwargs + urllib.urlretrieve(eg_link, filename=os.devnull, + reporthook=t.update_to, data=None) + t.total = t.n + +Inspired by `twine#242 <https://github.com/pypa/twine/pull/242>`__. +Functional alternative in +`examples/tqdm_wget.py <https://github.com/tqdm/tqdm/blob/master/examples/tqdm_wget.py>`__. + +It is recommend to use ``miniters=1`` whenever there is potentially +large differences in iteration speed (e.g. downloading a file over +a patchy connection). + +**Wrapping read/write methods** + +To measure throughput through a file-like object's ``read`` or ``write`` +methods, use ``CallbackIOWrapper``: + +.. code:: python + + from tqdm.auto import tqdm + from tqdm.utils import CallbackIOWrapper + + with tqdm(total=file_obj.size, + unit='B', unit_scale=True, unit_divisor=1024) as t: + fobj = CallbackIOWrapper(t.update, file_obj, "read") + while True: + chunk = fobj.read(chunk_size) + if not chunk: + break + t.reset() + # ... continue to use `t` for something else + +Alternatively, use the even simpler ``wrapattr`` convenience function, +which would condense both the ``urllib`` and ``CallbackIOWrapper`` examples +down to: + +.. code:: python + + import urllib, os + from tqdm import tqdm + + eg_link = "https://caspersci.uk.to/matryoshka.zip" + response = getattr(urllib, 'request', urllib).urlopen(eg_link) + with tqdm.wrapattr(open(os.devnull, "wb"), "write", + miniters=1, desc=eg_link.split('/')[-1], + total=getattr(response, 'length', None)) as fout: + for chunk in response: + fout.write(chunk) + +The ``requests`` equivalent is nearly identical: + +.. code:: python + + import requests, os + from tqdm import tqdm + + eg_link = "https://caspersci.uk.to/matryoshka.zip" + response = requests.get(eg_link, stream=True) + with tqdm.wrapattr(open(os.devnull, "wb"), "write", + miniters=1, desc=eg_link.split('/')[-1], + total=int(response.headers.get('content-length', 0))) as fout: + for chunk in response.iter_content(chunk_size=4096): + fout.write(chunk) + +**Custom callback** + +``tqdm`` is known for intelligently skipping unnecessary displays. To make a +custom callback take advantage of this, simply use the return value of +``update()``. This is set to ``True`` if a ``display()`` was triggered. + +.. code:: python + + from tqdm.auto import tqdm as std_tqdm + + def external_callback(*args, **kwargs): + ... + + class TqdmExt(std_tqdm): + def update(self, n=1): + displayed = super(TqdmExt, self).update(n) + if displayed: + external_callback(**self.format_dict) + return displayed + +``asyncio`` +~~~~~~~~~~~ + +Note that ``break`` isn't currently caught by asynchronous iterators. +This means that ``tqdm`` cannot clean up after itself in this case: + +.. code:: python + + from tqdm.asyncio import tqdm + + async for i in tqdm(range(9)): + if i == 2: + break + +Instead, either call ``pbar.close()`` manually or use the context manager syntax: + +.. code:: python + + from tqdm.asyncio import tqdm + + with tqdm(range(9)) as pbar: + async for i in pbar: + if i == 2: + break + +Pandas Integration +~~~~~~~~~~~~~~~~~~ + +Due to popular demand we've added support for ``pandas`` -- here's an example +for ``DataFrame.progress_apply`` and ``DataFrameGroupBy.progress_apply``: + +.. code:: python + + import pandas as pd + import numpy as np + from tqdm import tqdm + + df = pd.DataFrame(np.random.randint(0, 100, (100000, 6))) + + # Register `pandas.progress_apply` and `pandas.Series.map_apply` with `tqdm` + # (can use `tqdm.gui.tqdm`, `tqdm.notebook.tqdm`, optional kwargs, etc.) + tqdm.pandas(desc="my bar!") + + # Now you can use `progress_apply` instead of `apply` + # and `progress_map` instead of `map` + df.progress_apply(lambda x: x**2) + # can also groupby: + # df.groupby(0).progress_apply(lambda x: x**2) + +In case you're interested in how this works (and how to modify it for your +own callbacks), see the +`examples <https://github.com/tqdm/tqdm/tree/master/examples>`__ +folder or import the module and run ``help()``. + +Keras Integration +~~~~~~~~~~~~~~~~~ + +A ``keras`` callback is also available: + +.. code:: python + + from tqdm.keras import TqdmCallback + + ... + + model.fit(..., verbose=0, callbacks=[TqdmCallback()]) + +Dask Integration +~~~~~~~~~~~~~~~~ + +A ``dask`` callback is also available: + +.. code:: python + + from tqdm.dask import TqdmCallback + + with TqdmCallback(desc="compute"): + ... + arr.compute() + + # or use callback globally + cb = TqdmCallback(desc="global") + cb.register() + arr.compute() + +IPython/Jupyter Integration +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +IPython/Jupyter is supported via the ``tqdm.notebook`` submodule: + +.. code:: python + + from tqdm.notebook import trange, tqdm + from time import sleep + + for i in trange(3, desc='1st loop'): + for j in tqdm(range(100), desc='2nd loop'): + sleep(0.01) + +In addition to ``tqdm`` features, the submodule provides a native Jupyter +widget (compatible with IPython v1-v4 and Jupyter), fully working nested bars +and colour hints (blue: normal, green: completed, red: error/interrupt, +light blue: no ETA); as demonstrated below. + +|Screenshot-Jupyter1| +|Screenshot-Jupyter2| +|Screenshot-Jupyter3| + +The ``notebook`` version supports percentage or pixels for overall width +(e.g.: ``ncols='100%'`` or ``ncols='480px'``). + +It is also possible to let ``tqdm`` automatically choose between +console or notebook versions by using the ``autonotebook`` submodule: + +.. code:: python + + from tqdm.autonotebook import tqdm + tqdm.pandas() + +Note that this will issue a ``TqdmExperimentalWarning`` if run in a notebook +since it is not meant to be possible to distinguish between ``jupyter notebook`` +and ``jupyter console``. Use ``auto`` instead of ``autonotebook`` to suppress +this warning. + +Note that notebooks will display the bar in the cell where it was created. +This may be a different cell from the one where it is used. +If this is not desired, either + +- delay the creation of the bar to the cell where it must be displayed, or +- create the bar with ``display=False``, and in a later cell call + ``display(bar.container)``: + +.. code:: python + + from tqdm.notebook import tqdm + pbar = tqdm(..., display=False) + +.. code:: python + + # different cell + display(pbar.container) + +The ``keras`` callback has a ``display()`` method which can be used likewise: + +.. code:: python + + from tqdm.keras import TqdmCallback + cbk = TqdmCallback(display=False) + +.. code:: python + + # different cell + cbk.display() + model.fit(..., verbose=0, callbacks=[cbk]) + +Another possibility is to have a single bar (near the top of the notebook) +which is constantly re-used (using ``reset()`` rather than ``close()``). +For this reason, the notebook version (unlike the CLI version) does not +automatically call ``close()`` upon ``Exception``. + +.. code:: python + + from tqdm.notebook import tqdm + pbar = tqdm() + +.. code:: python + + # different cell + iterable = range(100) + pbar.reset(total=len(iterable)) # initialise with new `total` + for i in iterable: + pbar.update() + pbar.refresh() # force print final status but don't `close()` + +Custom Integration +~~~~~~~~~~~~~~~~~~ + +To change the default arguments (such as making ``dynamic_ncols=True``), +simply use built-in Python magic: + +.. code:: python + + from functools import partial + from tqdm import tqdm as std_tqdm + tqdm = partial(std_tqdm, dynamic_ncols=True) + +For further customisation, +``tqdm`` may be inherited from to create custom callbacks (as with the +``TqdmUpTo`` example `above <#hooks-and-callbacks>`__) or for custom frontends +(e.g. GUIs such as notebook or plotting packages). In the latter case: + +1. ``def __init__()`` to call ``super().__init__(..., gui=True)`` to disable + terminal ``status_printer`` creation. +2. Redefine: ``close()``, ``clear()``, ``display()``. + +Consider overloading ``display()`` to use e.g. +``self.frontend(**self.format_dict)`` instead of ``self.sp(repr(self))``. + +Some submodule examples of inheritance: + +- `tqdm/notebook.py <https://github.com/tqdm/tqdm/blob/master/tqdm/notebook.py>`__ +- `tqdm/gui.py <https://github.com/tqdm/tqdm/blob/master/tqdm/gui.py>`__ +- `tqdm/tk.py <https://github.com/tqdm/tqdm/blob/master/tqdm/tk.py>`__ +- `tqdm/contrib/slack.py <https://github.com/tqdm/tqdm/blob/master/tqdm/contrib/slack.py>`__ +- `tqdm/contrib/discord.py <https://github.com/tqdm/tqdm/blob/master/tqdm/contrib/discord.py>`__ +- `tqdm/contrib/telegram.py <https://github.com/tqdm/tqdm/blob/master/tqdm/contrib/telegram.py>`__ + +Dynamic Monitor/Meter +~~~~~~~~~~~~~~~~~~~~~ + +You can use a ``tqdm`` as a meter which is not monotonically increasing. +This could be because ``n`` decreases (e.g. a CPU usage monitor) or ``total`` +changes. + +One example would be recursively searching for files. The ``total`` is the +number of objects found so far, while ``n`` is the number of those objects which +are files (rather than folders): + +.. code:: python + + from tqdm import tqdm + import os.path + + def find_files_recursively(path, show_progress=True): + files = [] + # total=1 assumes `path` is a file + t = tqdm(total=1, unit="file", disable=not show_progress) + if not os.path.exists(path): + raise IOError("Cannot find:" + path) + + def append_found_file(f): + files.append(f) + t.update() + + def list_found_dir(path): + """returns os.listdir(path) assuming os.path.isdir(path)""" + listing = os.listdir(path) + # subtract 1 since a "file" we found was actually this directory + t.total += len(listing) - 1 + # fancy way to give info without forcing a refresh + t.set_postfix(dir=path[-10:], refresh=False) + t.update(0) # may trigger a refresh + return listing + + def recursively_search(path): + if os.path.isdir(path): + for f in list_found_dir(path): + recursively_search(os.path.join(path, f)) + else: + append_found_file(path) + + recursively_search(path) + t.set_postfix(dir=path) + t.close() + return files + +Using ``update(0)`` is a handy way to let ``tqdm`` decide when to trigger a +display refresh to avoid console spamming. + +Writing messages +~~~~~~~~~~~~~~~~ + +This is a work in progress (see +`#737 <https://github.com/tqdm/tqdm/issues/737>`__). + +Since ``tqdm`` uses a simple printing mechanism to display progress bars, +you should not write any message in the terminal using ``print()`` while +a progressbar is open. + +To write messages in the terminal without any collision with ``tqdm`` bar +display, a ``.write()`` method is provided: + +.. code:: python + + from tqdm.auto import tqdm, trange + from time import sleep + + bar = trange(10) + for i in bar: + # Print using tqdm class method .write() + sleep(0.1) + if not (i % 3): + tqdm.write("Done task %i" % i) + # Can also use bar.write() + +By default, this will print to standard output ``sys.stdout``. but you can +specify any file-like object using the ``file`` argument. For example, this +can be used to redirect the messages writing to a log file or class. + +Redirecting writing +~~~~~~~~~~~~~~~~~~~ + +If using a library that can print messages to the console, editing the library +by replacing ``print()`` with ``tqdm.write()`` may not be desirable. +In that case, redirecting ``sys.stdout`` to ``tqdm.write()`` is an option. + +To redirect ``sys.stdout``, create a file-like class that will write +any input string to ``tqdm.write()``, and supply the arguments +``file=sys.stdout, dynamic_ncols=True``. + +A reusable canonical example is given below: + +.. code:: python + + from time import sleep + import contextlib + import sys + from tqdm import tqdm + from tqdm.contrib import DummyTqdmFile + + + @contextlib.contextmanager + def std_out_err_redirect_tqdm(): + orig_out_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = map(DummyTqdmFile, orig_out_err) + yield orig_out_err[0] + # Relay exceptions + except Exception as exc: + raise exc + # Always restore sys.stdout/err if necessary + finally: + sys.stdout, sys.stderr = orig_out_err + + def some_fun(i): + print("Fee, fi, fo,".split()[i]) + + # Redirect stdout to tqdm.write() (don't forget the `as save_stdout`) + with std_out_err_redirect_tqdm() as orig_stdout: + # tqdm needs the original stdout + # and dynamic_ncols=True to autodetect console width + for i in tqdm(range(3), file=orig_stdout, dynamic_ncols=True): + sleep(.5) + some_fun(i) + + # After the `with`, printing is restored + print("Done!") + +Redirecting ``logging`` +~~~~~~~~~~~~~~~~~~~~~~~ + +Similar to ``sys.stdout``/``sys.stderr`` as detailed above, console ``logging`` +may also be redirected to ``tqdm.write()``. + +Warning: if also redirecting ``sys.stdout``/``sys.stderr``, make sure to +redirect ``logging`` first if needed. + +Helper methods are available in ``tqdm.contrib.logging``. For example: + +.. code:: python + + import logging + from tqdm import trange + from tqdm.contrib.logging import logging_redirect_tqdm + + LOG = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + with logging_redirect_tqdm(): + for i in trange(9): + if i == 4: + LOG.info("console logging redirected to `tqdm.write()`") + # logging restored + +Monitoring thread, intervals and miniters +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``tqdm`` implements a few tricks to increase efficiency and reduce overhead. + +- Avoid unnecessary frequent bar refreshing: ``mininterval`` defines how long + to wait between each refresh. ``tqdm`` always gets updated in the background, + but it will display only every ``mininterval``. +- Reduce number of calls to check system clock/time. +- ``mininterval`` is more intuitive to configure than ``miniters``. + A clever adjustment system ``dynamic_miniters`` will automatically adjust + ``miniters`` to the amount of iterations that fit into time ``mininterval``. + Essentially, ``tqdm`` will check if it's time to print without actually + checking time. This behaviour can be still be bypassed by manually setting + ``miniters``. + +However, consider a case with a combination of fast and slow iterations. +After a few fast iterations, ``dynamic_miniters`` will set ``miniters`` to a +large number. When iteration rate subsequently slows, ``miniters`` will +remain large and thus reduce display update frequency. To address this: + +- ``maxinterval`` defines the maximum time between display refreshes. + A concurrent monitoring thread checks for overdue updates and forces one + where necessary. + +The monitoring thread should not have a noticeable overhead, and guarantees +updates at least every 10 seconds by default. +This value can be directly changed by setting the ``monitor_interval`` of +any ``tqdm`` instance (i.e. ``t = tqdm.tqdm(...); t.monitor_interval = 2``). +The monitor thread may be disabled application-wide by setting +``tqdm.tqdm.monitor_interval = 0`` before instantiation of any ``tqdm`` bar. + + +Merch +----- + +You can buy `tqdm branded merch <https://tqdm.github.io/merch>`__ now! + +Contributions +------------- + +|GitHub-Commits| |GitHub-Issues| |GitHub-PRs| |OpenHub-Status| |GitHub-Contributions| |CII Best Practices| + +All source code is hosted on `GitHub <https://github.com/tqdm/tqdm>`__. +Contributions are welcome. + +See the +`CONTRIBUTING <https://github.com/tqdm/tqdm/blob/master/CONTRIBUTING.md>`__ +file for more information. + +Developers who have made significant contributions, ranked by *SLoC* +(surviving lines of code, +`git fame <https://github.com/casperdcl/git-fame>`__ ``-wMC --excl '\.(png|gif|jpg)$'``), +are: + +==================== ======================================================== ==== ================================ +Name ID SLoC Notes +==================== ======================================================== ==== ================================ +Casper da Costa-Luis `casperdcl <https://github.com/casperdcl>`__ ~78% primary maintainer |Gift-Casper| +Stephen Larroque `lrq3000 <https://github.com/lrq3000>`__ ~10% team member +Martin Zugnoni `martinzugnoni <https://github.com/martinzugnoni>`__ ~4% +Daniel Ecer `de-code <https://github.com/de-code>`__ ~2% +Richard Sheridan `richardsheridan <https://github.com/richardsheridan>`__ ~1% +Guangshuo Chen `chengs <https://github.com/chengs>`__ ~1% +Kyle Altendorf `altendky <https://github.com/altendky>`__ <1% +Matthew Stevens `mjstevens777 <https://github.com/mjstevens777>`__ <1% +Hadrien Mary `hadim <https://github.com/hadim>`__ <1% team member +Noam Yorav-Raphael `noamraph <https://github.com/noamraph>`__ <1% original author +Mikhail Korobov `kmike <https://github.com/kmike>`__ <1% team member +==================== ======================================================== ==== ================================ + +Ports to Other Languages +~~~~~~~~~~~~~~~~~~~~~~~~ + +A list is available on +`this wiki page <https://github.com/tqdm/tqdm/wiki/tqdm-ports>`__. + + +LICENCE +------- + +Open Source (OSI approved): |LICENCE| + +Citation information: |DOI| + +|README-Hits| (Since 19 May 2016) + +.. |Logo| image:: https://img.tqdm.ml/logo.gif +.. |Screenshot| image:: https://img.tqdm.ml/tqdm.gif +.. |Video| image:: https://img.tqdm.ml/video.jpg + :target: https://tqdm.github.io/video +.. |Slides| image:: https://img.tqdm.ml/slides.jpg + :target: https://tqdm.github.io/PyData2019/slides.html +.. |Merch| image:: https://img.tqdm.ml/merch.jpg + :target: https://tqdm.github.io/merch +.. |Build-Status| image:: https://img.shields.io/github/workflow/status/tqdm/tqdm/Test/master?logo=GitHub + :target: https://github.com/tqdm/tqdm/actions?query=workflow%3ATest +.. |Coverage-Status| image:: https://img.shields.io/coveralls/github/tqdm/tqdm/master?logo=coveralls + :target: https://coveralls.io/github/tqdm/tqdm +.. |Branch-Coverage-Status| image:: https://codecov.io/gh/tqdm/tqdm/branch/master/graph/badge.svg + :target: https://codecov.io/gh/tqdm/tqdm +.. |Codacy-Grade| image:: https://app.codacy.com/project/badge/Grade/3f965571598f44549c7818f29cdcf177 + :target: https://www.codacy.com/gh/tqdm/tqdm/dashboard +.. |CII Best Practices| image:: https://bestpractices.coreinfrastructure.org/projects/3264/badge + :target: https://bestpractices.coreinfrastructure.org/projects/3264 +.. |GitHub-Status| image:: https://img.shields.io/github/tag/tqdm/tqdm.svg?maxAge=86400&logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/releases +.. |GitHub-Forks| image:: https://img.shields.io/github/forks/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/network +.. |GitHub-Stars| image:: https://img.shields.io/github/stars/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/stargazers +.. |GitHub-Commits| image:: https://img.shields.io/github/commit-activity/y/tqdm/tqdm.svg?logo=git&logoColor=white + :target: https://github.com/tqdm/tqdm/graphs/commit-activity +.. |GitHub-Issues| image:: https://img.shields.io/github/issues-closed/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/issues?q= +.. |GitHub-PRs| image:: https://img.shields.io/github/issues-pr-closed/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/pulls +.. |GitHub-Contributions| image:: https://img.shields.io/github/contributors/tqdm/tqdm.svg?logo=github&logoColor=white + :target: https://github.com/tqdm/tqdm/graphs/contributors +.. |GitHub-Updated| image:: https://img.shields.io/github/last-commit/tqdm/tqdm/master.svg?logo=github&logoColor=white&label=pushed + :target: https://github.com/tqdm/tqdm/pulse +.. |Gift-Casper| image:: https://img.shields.io/badge/dynamic/json.svg?color=ff69b4&label=gifts%20received&prefix=%C2%A3&query=%24..sum&url=https%3A%2F%2Fcaspersci.uk.to%2Fgifts.json + :target: https://cdcl.ml/sponsor +.. |Versions| image:: https://img.shields.io/pypi/v/tqdm.svg + :target: https://tqdm.github.io/releases +.. |PyPI-Downloads| image:: https://img.shields.io/pypi/dm/tqdm.svg?label=pypi%20downloads&logo=PyPI&logoColor=white + :target: https://pepy.tech/project/tqdm +.. |Py-Versions| image:: https://img.shields.io/pypi/pyversions/tqdm.svg?logo=python&logoColor=white + :target: https://pypi.org/project/tqdm +.. |Conda-Forge-Status| image:: https://img.shields.io/conda/v/conda-forge/tqdm.svg?label=conda-forge&logo=conda-forge + :target: https://anaconda.org/conda-forge/tqdm +.. |Snapcraft| image:: https://img.shields.io/badge/snap-install-82BEA0.svg?logo=snapcraft + :target: https://snapcraft.io/tqdm +.. |Docker| image:: https://img.shields.io/badge/docker-pull-blue.svg?logo=docker&logoColor=white + :target: https://hub.docker.com/r/tqdm/tqdm +.. |Libraries-Rank| image:: https://img.shields.io/librariesio/sourcerank/pypi/tqdm.svg?logo=koding&logoColor=white + :target: https://libraries.io/pypi/tqdm +.. |Libraries-Dependents| image:: https://img.shields.io/librariesio/dependent-repos/pypi/tqdm.svg?logo=koding&logoColor=white + :target: https://github.com/tqdm/tqdm/network/dependents +.. |OpenHub-Status| image:: https://www.openhub.net/p/tqdm/widgets/project_thin_badge?format=gif + :target: https://www.openhub.net/p/tqdm?ref=Thin+badge +.. |awesome-python| image:: https://awesome.re/mentioned-badge.svg + :target: https://github.com/vinta/awesome-python +.. |LICENCE| image:: https://img.shields.io/pypi/l/tqdm.svg + :target: https://raw.githubusercontent.com/tqdm/tqdm/master/LICENCE +.. |DOI| image:: https://img.shields.io/badge/DOI-10.5281/zenodo.595120-blue.svg + :target: https://doi.org/10.5281/zenodo.595120 +.. |binder-demo| image:: https://mybinder.org/badge_logo.svg + :target: https://mybinder.org/v2/gh/tqdm/tqdm/master?filepath=DEMO.ipynb +.. |Screenshot-Jupyter1| image:: https://img.tqdm.ml/jupyter-1.gif +.. |Screenshot-Jupyter2| image:: https://img.tqdm.ml/jupyter-2.gif +.. |Screenshot-Jupyter3| image:: https://img.tqdm.ml/jupyter-3.gif +.. |README-Hits| image:: https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&style=social&r=https://github.com/tqdm/tqdm&l=https://img.tqdm.ml/favicon.png&f=https://img.tqdm.ml/logo.gif + :target: https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&a=plot&r=https://github.com/tqdm/tqdm&l=https://img.tqdm.ml/favicon.png&f=https://img.tqdm.ml/logo.gif&style=social diff --git a/tqdm.egg-info/SOURCES.txt b/tqdm.egg-info/SOURCES.txt new file mode 100644 index 0000000..41d7b89 --- /dev/null +++ b/tqdm.egg-info/SOURCES.txt @@ -0,0 +1,88 @@ +.pre-commit-config.yaml +.zenodo.json +CODE_OF_CONDUCT.md +CONTRIBUTING.md +DEMO.ipynb +LICENCE +Makefile +README.rst +environment.yml +logo.png +pyproject.toml +setup.cfg +setup.py +tests_notebook.ipynb +tox.ini +examples/7zx.py +examples/async_coroutines.py +examples/coroutine_pipe.py +examples/include_no_requirements.py +examples/pandas_progress_apply.py +examples/paper.bib +examples/paper.md +examples/parallel_bars.py +examples/redirect_print.py +examples/simple_examples.py +examples/tqdm_requests.py +examples/tqdm_wget.py +examples/wrapping_generators.py +tests/__init__.py +tests/conftest.py +tests/py37_asyncio.py +tests/tests_asyncio.py +tests/tests_concurrent.py +tests/tests_contrib.py +tests/tests_contrib_logging.py +tests/tests_dask.py +tests/tests_gui.py +tests/tests_itertools.py +tests/tests_keras.py +tests/tests_main.py +tests/tests_notebook.py +tests/tests_pandas.py +tests/tests_perf.py +tests/tests_rich.py +tests/tests_synchronisation.py +tests/tests_tk.py +tests/tests_tqdm.py +tests/tests_version.py +tqdm/__init__.py +tqdm/__main__.py +tqdm/_dist_ver.py +tqdm/_main.py +tqdm/_monitor.py +tqdm/_tqdm.py +tqdm/_tqdm_gui.py +tqdm/_tqdm_notebook.py +tqdm/_tqdm_pandas.py +tqdm/_utils.py +tqdm/asyncio.py +tqdm/auto.py +tqdm/autonotebook.py +tqdm/cli.py +tqdm/completion.sh +tqdm/dask.py +tqdm/gui.py +tqdm/keras.py +tqdm/notebook.py +tqdm/rich.py +tqdm/std.py +tqdm/tk.py +tqdm/tqdm.1 +tqdm/utils.py +tqdm/version.py +tqdm.egg-info/PKG-INFO +tqdm.egg-info/SOURCES.txt +tqdm.egg-info/dependency_links.txt +tqdm.egg-info/entry_points.txt +tqdm.egg-info/requires.txt +tqdm.egg-info/top_level.txt +tqdm/contrib/__init__.py +tqdm/contrib/bells.py +tqdm/contrib/concurrent.py +tqdm/contrib/discord.py +tqdm/contrib/itertools.py +tqdm/contrib/logging.py +tqdm/contrib/slack.py +tqdm/contrib/telegram.py +tqdm/contrib/utils_worker.py
\ No newline at end of file diff --git a/tqdm.egg-info/dependency_links.txt b/tqdm.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tqdm.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/tqdm.egg-info/entry_points.txt b/tqdm.egg-info/entry_points.txt new file mode 100644 index 0000000..540e60f --- /dev/null +++ b/tqdm.egg-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +tqdm = tqdm.cli:main diff --git a/tqdm.egg-info/requires.txt b/tqdm.egg-info/requires.txt new file mode 100644 index 0000000..352f7a3 --- /dev/null +++ b/tqdm.egg-info/requires.txt @@ -0,0 +1,20 @@ + +[:platform_system == "Windows"] +colorama + +[:python_version < "3.7"] +importlib_resources + +[dev] +py-make>=0.1.0 +twine +wheel + +[notebook] +ipywidgets>=6 + +[slack] +slack-sdk + +[telegram] +requests diff --git a/tqdm.egg-info/top_level.txt b/tqdm.egg-info/top_level.txt new file mode 100644 index 0000000..78620c4 --- /dev/null +++ b/tqdm.egg-info/top_level.txt @@ -0,0 +1 @@ +tqdm diff --git a/tqdm/__init__.py b/tqdm/__init__.py new file mode 100644 index 0000000..a021d16 --- /dev/null +++ b/tqdm/__init__.py @@ -0,0 +1,41 @@ +from ._monitor import TMonitor, TqdmSynchronisationWarning +from ._tqdm_pandas import tqdm_pandas +from .cli import main # TODO: remove in v5.0.0 +from .gui import tqdm as tqdm_gui # TODO: remove in v5.0.0 +from .gui import trange as tgrange # TODO: remove in v5.0.0 +from .std import ( + TqdmDeprecationWarning, TqdmExperimentalWarning, TqdmKeyError, TqdmMonitorWarning, + TqdmTypeError, TqdmWarning, tqdm, trange) +from .version import __version__ + +__all__ = ['tqdm', 'tqdm_gui', 'trange', 'tgrange', 'tqdm_pandas', + 'tqdm_notebook', 'tnrange', 'main', 'TMonitor', + 'TqdmTypeError', 'TqdmKeyError', + 'TqdmWarning', 'TqdmDeprecationWarning', + 'TqdmExperimentalWarning', + 'TqdmMonitorWarning', 'TqdmSynchronisationWarning', + '__version__'] + + +def tqdm_notebook(*args, **kwargs): # pragma: no cover + """See tqdm.notebook.tqdm for full documentation""" + from warnings import warn + + from .notebook import tqdm as _tqdm_notebook + warn("This function will be removed in tqdm==5.0.0\n" + "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`", + TqdmDeprecationWarning, stacklevel=2) + return _tqdm_notebook(*args, **kwargs) + + +def tnrange(*args, **kwargs): # pragma: no cover + """ + A shortcut for `tqdm.notebook.tqdm(xrange(*args), **kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + from warnings import warn + + from .notebook import trange as _tnrange + warn("Please use `tqdm.notebook.trange` instead of `tqdm.tnrange`", + TqdmDeprecationWarning, stacklevel=2) + return _tnrange(*args, **kwargs) diff --git a/tqdm/__main__.py b/tqdm/__main__.py new file mode 100644 index 0000000..4e28416 --- /dev/null +++ b/tqdm/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +main() diff --git a/tqdm/_dist_ver.py b/tqdm/_dist_ver.py new file mode 100644 index 0000000..1c2d004 --- /dev/null +++ b/tqdm/_dist_ver.py @@ -0,0 +1 @@ +__version__ = '4.64.1' diff --git a/tqdm/_main.py b/tqdm/_main.py new file mode 100644 index 0000000..04fdeef --- /dev/null +++ b/tqdm/_main.py @@ -0,0 +1,9 @@ +from warnings import warn + +from .cli import * # NOQA +from .cli import __all__ # NOQA +from .std import TqdmDeprecationWarning + +warn("This function will be removed in tqdm==5.0.0\n" + "Please use `tqdm.cli.*` instead of `tqdm._main.*`", + TqdmDeprecationWarning, stacklevel=2) diff --git a/tqdm/_monitor.py b/tqdm/_monitor.py new file mode 100644 index 0000000..f71aa56 --- /dev/null +++ b/tqdm/_monitor.py @@ -0,0 +1,95 @@ +import atexit +from threading import Event, Thread, current_thread +from time import time +from warnings import warn + +__all__ = ["TMonitor", "TqdmSynchronisationWarning"] + + +class TqdmSynchronisationWarning(RuntimeWarning): + """tqdm multi-thread/-process errors which may cause incorrect nesting + but otherwise no adverse effects""" + pass + + +class TMonitor(Thread): + """ + Monitoring thread for tqdm bars. + Monitors if tqdm bars are taking too much time to display + and readjusts miniters automatically if necessary. + + Parameters + ---------- + tqdm_cls : class + tqdm class to use (can be core tqdm or a submodule). + sleep_interval : float + Time to sleep between monitoring checks. + """ + _test = {} # internal vars for unit testing + + def __init__(self, tqdm_cls, sleep_interval): + Thread.__init__(self) + self.daemon = True # kill thread when main killed (KeyboardInterrupt) + self.woken = 0 # last time woken up, to sync with monitor + self.tqdm_cls = tqdm_cls + self.sleep_interval = sleep_interval + self._time = self._test.get("time", time) + self.was_killed = self._test.get("Event", Event)() + atexit.register(self.exit) + self.start() + + def exit(self): + self.was_killed.set() + if self is not current_thread(): + self.join() + return self.report() + + def get_instances(self): + # returns a copy of started `tqdm_cls` instances + return [i for i in self.tqdm_cls._instances.copy() + # Avoid race by checking that the instance started + if hasattr(i, 'start_t')] + + def run(self): + cur_t = self._time() + while True: + # After processing and before sleeping, notify that we woke + # Need to be done just before sleeping + self.woken = cur_t + # Sleep some time... + self.was_killed.wait(self.sleep_interval) + # Quit if killed + if self.was_killed.is_set(): + return + # Then monitor! + # Acquire lock (to access _instances) + with self.tqdm_cls.get_lock(): + cur_t = self._time() + # Check tqdm instances are waiting too long to print + instances = self.get_instances() + for instance in instances: + # Check event in loop to reduce blocking time on exit + if self.was_killed.is_set(): + return + # Only if mininterval > 1 (else iterations are just slow) + # and last refresh exceeded maxinterval + if ( + instance.miniters > 1 + and (cur_t - instance.last_print_t) >= instance.maxinterval + ): + # force bypassing miniters on next iteration + # (dynamic_miniters adjusts mininterval automatically) + instance.miniters = 1 + # Refresh now! (works only for manual tqdm) + instance.refresh(nolock=True) + # Remove accidental long-lived strong reference + del instance + if instances != self.get_instances(): # pragma: nocover + warn("Set changed size during iteration" + + " (see https://github.com/tqdm/tqdm/issues/481)", + TqdmSynchronisationWarning, stacklevel=2) + # Remove accidental long-lived strong references + del instances + + def report(self): + return not self.was_killed.is_set() diff --git a/tqdm/_tqdm.py b/tqdm/_tqdm.py new file mode 100644 index 0000000..7fc4962 --- /dev/null +++ b/tqdm/_tqdm.py @@ -0,0 +1,9 @@ +from warnings import warn + +from .std import * # NOQA +from .std import __all__ # NOQA +from .std import TqdmDeprecationWarning + +warn("This function will be removed in tqdm==5.0.0\n" + "Please use `tqdm.std.*` instead of `tqdm._tqdm.*`", + TqdmDeprecationWarning, stacklevel=2) diff --git a/tqdm/_tqdm_gui.py b/tqdm/_tqdm_gui.py new file mode 100644 index 0000000..f32aa89 --- /dev/null +++ b/tqdm/_tqdm_gui.py @@ -0,0 +1,9 @@ +from warnings import warn + +from .gui import * # NOQA +from .gui import __all__ # NOQA +from .std import TqdmDeprecationWarning + +warn("This function will be removed in tqdm==5.0.0\n" + "Please use `tqdm.gui.*` instead of `tqdm._tqdm_gui.*`", + TqdmDeprecationWarning, stacklevel=2) diff --git a/tqdm/_tqdm_notebook.py b/tqdm/_tqdm_notebook.py new file mode 100644 index 0000000..f225fbf --- /dev/null +++ b/tqdm/_tqdm_notebook.py @@ -0,0 +1,9 @@ +from warnings import warn + +from .notebook import * # NOQA +from .notebook import __all__ # NOQA +from .std import TqdmDeprecationWarning + +warn("This function will be removed in tqdm==5.0.0\n" + "Please use `tqdm.notebook.*` instead of `tqdm._tqdm_notebook.*`", + TqdmDeprecationWarning, stacklevel=2) diff --git a/tqdm/_tqdm_pandas.py b/tqdm/_tqdm_pandas.py new file mode 100644 index 0000000..c4fe6ef --- /dev/null +++ b/tqdm/_tqdm_pandas.py @@ -0,0 +1,24 @@ +import sys + +__author__ = "github.com/casperdcl" +__all__ = ['tqdm_pandas'] + + +def tqdm_pandas(tclass, **tqdm_kwargs): + """ + Registers the given `tqdm` instance with + `pandas.core.groupby.DataFrameGroupBy.progress_apply`. + """ + from tqdm import TqdmDeprecationWarning + + if isinstance(tclass, type) or (getattr(tclass, '__name__', '').startswith( + 'tqdm_')): # delayed adapter case + TqdmDeprecationWarning( + "Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm, ...)`.", + fp_write=getattr(tqdm_kwargs.get('file', None), 'write', sys.stderr.write)) + tclass.pandas(**tqdm_kwargs) + else: + TqdmDeprecationWarning( + "Please use `tqdm.pandas(...)` instead of `tqdm_pandas(tqdm(...))`.", + fp_write=getattr(tclass.fp, 'write', sys.stderr.write)) + type(tclass).pandas(deprecated_t=tclass) diff --git a/tqdm/_utils.py b/tqdm/_utils.py new file mode 100644 index 0000000..2cf1090 --- /dev/null +++ b/tqdm/_utils.py @@ -0,0 +1,12 @@ +from warnings import warn + +from .std import TqdmDeprecationWarning +from .utils import ( # NOQA, pylint: disable=unused-import + CUR_OS, IS_NIX, IS_WIN, RE_ANSI, Comparable, FormatReplace, SimpleTextIOWrapper, _basestring, + _environ_cols_wrapper, _is_ascii, _is_utf, _range, _screen_shape_linux, _screen_shape_tput, + _screen_shape_windows, _screen_shape_wrapper, _supports_unicode, _term_move_up, _unich, + _unicode, colorama) + +warn("This function will be removed in tqdm==5.0.0\n" + "Please use `tqdm.utils.*` instead of `tqdm._utils.*`", + TqdmDeprecationWarning, stacklevel=2) diff --git a/tqdm/asyncio.py b/tqdm/asyncio.py new file mode 100644 index 0000000..97c5f88 --- /dev/null +++ b/tqdm/asyncio.py @@ -0,0 +1,93 @@ +""" +Asynchronous progressbar decorator for iterators. +Includes a default `range` iterator printing to `stderr`. + +Usage: +>>> from tqdm.asyncio import trange, tqdm +>>> async for i in trange(10): +... ... +""" +import asyncio +from sys import version_info + +from .std import tqdm as std_tqdm + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['tqdm_asyncio', 'tarange', 'tqdm', 'trange'] + + +class tqdm_asyncio(std_tqdm): + """ + Asynchronous-friendly version of tqdm (Python 3.6+). + """ + def __init__(self, iterable=None, *args, **kwargs): + super(tqdm_asyncio, self).__init__(iterable, *args, **kwargs) + self.iterable_awaitable = False + if iterable is not None: + if hasattr(iterable, "__anext__"): + self.iterable_next = iterable.__anext__ + self.iterable_awaitable = True + elif hasattr(iterable, "__next__"): + self.iterable_next = iterable.__next__ + else: + self.iterable_iterator = iter(iterable) + self.iterable_next = self.iterable_iterator.__next__ + + def __aiter__(self): + return self + + async def __anext__(self): + try: + if self.iterable_awaitable: + res = await self.iterable_next() + else: + res = self.iterable_next() + self.update() + return res + except StopIteration: + self.close() + raise StopAsyncIteration + except BaseException: + self.close() + raise + + def send(self, *args, **kwargs): + return self.iterable.send(*args, **kwargs) + + @classmethod + def as_completed(cls, fs, *, loop=None, timeout=None, total=None, **tqdm_kwargs): + """ + Wrapper for `asyncio.as_completed`. + """ + if total is None: + total = len(fs) + kwargs = {} + if version_info[:2] < (3, 10): + kwargs['loop'] = loop + yield from cls(asyncio.as_completed(fs, timeout=timeout, **kwargs), + total=total, **tqdm_kwargs) + + @classmethod + async def gather(cls, *fs, loop=None, timeout=None, total=None, **tqdm_kwargs): + """ + Wrapper for `asyncio.gather`. + """ + async def wrap_awaitable(i, f): + return i, await f + + ifs = [wrap_awaitable(i, f) for i, f in enumerate(fs)] + res = [await f for f in cls.as_completed(ifs, loop=loop, timeout=timeout, + total=total, **tqdm_kwargs)] + return [i for _, i in sorted(res)] + + +def tarange(*args, **kwargs): + """ + A shortcut for `tqdm.asyncio.tqdm(range(*args), **kwargs)`. + """ + return tqdm_asyncio(range(*args), **kwargs) + + +# Aliases +tqdm = tqdm_asyncio +trange = tarange diff --git a/tqdm/auto.py b/tqdm/auto.py new file mode 100644 index 0000000..cffca20 --- /dev/null +++ b/tqdm/auto.py @@ -0,0 +1,44 @@ +""" +Enables multiple commonly used features. + +Method resolution order: + +- `tqdm.autonotebook` without import warnings +- `tqdm.asyncio` on Python3.6+ +- `tqdm.std` base class + +Usage: +>>> from tqdm.auto import trange, tqdm +>>> for i in trange(10): +... ... +""" +import sys +import warnings + +from .std import TqdmExperimentalWarning + +with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=TqdmExperimentalWarning) + from .autonotebook import tqdm as notebook_tqdm + from .autonotebook import trange as notebook_trange + +if sys.version_info[:2] < (3, 6): + tqdm = notebook_tqdm + trange = notebook_trange +else: # Python3.6+ + from .asyncio import tqdm as asyncio_tqdm + from .std import tqdm as std_tqdm + + if notebook_tqdm != std_tqdm: + class tqdm(notebook_tqdm, asyncio_tqdm): # pylint: disable=inconsistent-mro + pass + else: + tqdm = asyncio_tqdm + + def trange(*args, **kwargs): + """ + A shortcut for `tqdm.auto.tqdm(range(*args), **kwargs)`. + """ + return tqdm(range(*args), **kwargs) + +__all__ = ["tqdm", "trange"] diff --git a/tqdm/autonotebook.py b/tqdm/autonotebook.py new file mode 100644 index 0000000..a09f2ec --- /dev/null +++ b/tqdm/autonotebook.py @@ -0,0 +1,29 @@ +""" +Automatically choose between `tqdm.notebook` and `tqdm.std`. + +Usage: +>>> from tqdm.autonotebook import trange, tqdm +>>> for i in trange(10): +... ... +""" +import sys +from warnings import warn + +try: + get_ipython = sys.modules['IPython'].get_ipython + if 'IPKernelApp' not in get_ipython().config: # pragma: no cover + raise ImportError("console") + from .notebook import WARN_NOIPYW, IProgress + if IProgress is None: + from .std import TqdmWarning + warn(WARN_NOIPYW, TqdmWarning, stacklevel=2) + raise ImportError('ipywidgets') +except Exception: + from .std import tqdm, trange +else: # pragma: no cover + from .notebook import tqdm, trange + from .std import TqdmExperimentalWarning + warn("Using `tqdm.autonotebook.tqdm` in notebook mode." + " Use `tqdm.tqdm` instead to force console mode" + " (e.g. in jupyter console)", TqdmExperimentalWarning, stacklevel=2) +__all__ = ["tqdm", "trange"] diff --git a/tqdm/cli.py b/tqdm/cli.py new file mode 100644 index 0000000..3ed25fb --- /dev/null +++ b/tqdm/cli.py @@ -0,0 +1,315 @@ +""" +Module version for monitoring CLI pipes (`... | python -m tqdm | ...`). +""" +import logging +import re +import sys +from ast import literal_eval as numeric + +from .std import TqdmKeyError, TqdmTypeError, tqdm +from .version import __version__ + +__all__ = ["main"] +log = logging.getLogger(__name__) + + +def cast(val, typ): + log.debug((val, typ)) + if " or " in typ: + for t in typ.split(" or "): + try: + return cast(val, t) + except TqdmTypeError: + pass + raise TqdmTypeError(val + ' : ' + typ) + + # sys.stderr.write('\ndebug | `val:type`: `' + val + ':' + typ + '`.\n') + if typ == 'bool': + if (val == 'True') or (val == ''): + return True + elif val == 'False': + return False + else: + raise TqdmTypeError(val + ' : ' + typ) + try: + return eval(typ + '("' + val + '")') + except Exception: + if typ == 'chr': + return chr(ord(eval('"' + val + '"'))).encode() + else: + raise TqdmTypeError(val + ' : ' + typ) + + +def posix_pipe(fin, fout, delim=b'\\n', buf_size=256, + callback=lambda float: None, callback_len=True): + """ + Params + ------ + fin : binary file with `read(buf_size : int)` method + fout : binary file with `write` (and optionally `flush`) methods. + callback : function(float), e.g.: `tqdm.update` + callback_len : If (default: True) do `callback(len(buffer))`. + Otherwise, do `callback(data) for data in buffer.split(delim)`. + """ + fp_write = fout.write + + if not delim: + while True: + tmp = fin.read(buf_size) + + # flush at EOF + if not tmp: + getattr(fout, 'flush', lambda: None)() + return + + fp_write(tmp) + callback(len(tmp)) + # return + + buf = b'' + len_delim = len(delim) + # n = 0 + while True: + tmp = fin.read(buf_size) + + # flush at EOF + if not tmp: + if buf: + fp_write(buf) + if callback_len: + # n += 1 + buf.count(delim) + callback(1 + buf.count(delim)) + else: + for i in buf.split(delim): + callback(i) + getattr(fout, 'flush', lambda: None)() + return # n + + while True: + i = tmp.find(delim) + if i < 0: + buf += tmp + break + fp_write(buf + tmp[:i + len(delim)]) + # n += 1 + callback(1 if callback_len else (buf + tmp[:i])) + buf = b'' + tmp = tmp[i + len_delim:] + + +# ((opt, type), ... ) +RE_OPTS = re.compile(r'\n {8}(\S+)\s{2,}:\s*([^,]+)') +# better split method assuming no positional args +RE_SHLEX = re.compile(r'\s*(?<!\S)--?([^\s=]+)(\s+|=|$)') + +# TODO: add custom support for some of the following? +UNSUPPORTED_OPTS = ('iterable', 'gui', 'out', 'file') + +# The 8 leading spaces are required for consistency +CLI_EXTRA_DOC = r""" + Extra CLI Options + ----------------- + name : type, optional + TODO: find out why this is needed. + delim : chr, optional + Delimiting character [default: '\n']. Use '\0' for null. + N.B.: on Windows systems, Python converts '\n' to '\r\n'. + buf_size : int, optional + String buffer size in bytes [default: 256] + used when `delim` is specified. + bytes : bool, optional + If true, will count bytes, ignore `delim`, and default + `unit_scale` to True, `unit_divisor` to 1024, and `unit` to 'B'. + tee : bool, optional + If true, passes `stdin` to both `stderr` and `stdout`. + update : bool, optional + If true, will treat input as newly elapsed iterations, + i.e. numbers to pass to `update()`. Note that this is slow + (~2e5 it/s) since every input must be decoded as a number. + update_to : bool, optional + If true, will treat input as total elapsed iterations, + i.e. numbers to assign to `self.n`. Note that this is slow + (~2e5 it/s) since every input must be decoded as a number. + null : bool, optional + If true, will discard input (no stdout). + manpath : str, optional + Directory in which to install tqdm man pages. + comppath : str, optional + Directory in which to place tqdm completion. + log : str, optional + CRITICAL|FATAL|ERROR|WARN(ING)|[default: 'INFO']|DEBUG|NOTSET. +""" + + +def main(fp=sys.stderr, argv=None): + """ + Parameters (internal use only) + --------- + fp : file-like object for tqdm + argv : list (default: sys.argv[1:]) + """ + if argv is None: + argv = sys.argv[1:] + try: + log_idx = argv.index('--log') + except ValueError: + for i in argv: + if i.startswith('--log='): + logLevel = i[len('--log='):] + break + else: + logLevel = 'INFO' + else: + # argv.pop(log_idx) + # logLevel = argv.pop(log_idx) + logLevel = argv[log_idx + 1] + logging.basicConfig(level=getattr(logging, logLevel), + format="%(levelname)s:%(module)s:%(lineno)d:%(message)s") + + d = tqdm.__init__.__doc__ + CLI_EXTRA_DOC + + opt_types = dict(RE_OPTS.findall(d)) + # opt_types['delim'] = 'chr' + + for o in UNSUPPORTED_OPTS: + opt_types.pop(o) + + log.debug(sorted(opt_types.items())) + + # d = RE_OPTS.sub(r' --\1=<\1> : \2', d) + split = RE_OPTS.split(d) + opt_types_desc = zip(split[1::3], split[2::3], split[3::3]) + d = ''.join(('\n --{0} : {2}{3}' if otd[1] == 'bool' else + '\n --{0}=<{1}> : {2}{3}').format( + otd[0].replace('_', '-'), otd[0], *otd[1:]) + for otd in opt_types_desc if otd[0] not in UNSUPPORTED_OPTS) + + help_short = "Usage:\n tqdm [--help | options]\n" + d = help_short + """ +Options: + -h, --help Print this help and exit. + -v, --version Print version and exit. +""" + d.strip('\n') + '\n' + + # opts = docopt(d, version=__version__) + if any(v in argv for v in ('-v', '--version')): + sys.stdout.write(__version__ + '\n') + sys.exit(0) + elif any(v in argv for v in ('-h', '--help')): + sys.stdout.write(d + '\n') + sys.exit(0) + elif argv and argv[0][:2] != '--': + sys.stderr.write( + "Error:Unknown argument:{0}\n{1}".format(argv[0], help_short)) + + argv = RE_SHLEX.split(' '.join(["tqdm"] + argv)) + opts = dict(zip(argv[1::3], argv[3::3])) + + log.debug(opts) + opts.pop('log', True) + + tqdm_args = {'file': fp} + try: + for (o, v) in opts.items(): + o = o.replace('-', '_') + try: + tqdm_args[o] = cast(v, opt_types[o]) + except KeyError as e: + raise TqdmKeyError(str(e)) + log.debug('args:' + str(tqdm_args)) + + delim_per_char = tqdm_args.pop('bytes', False) + update = tqdm_args.pop('update', False) + update_to = tqdm_args.pop('update_to', False) + if sum((delim_per_char, update, update_to)) > 1: + raise TqdmKeyError("Can only have one of --bytes --update --update_to") + except Exception: + fp.write("\nError:\n" + help_short) + stdin, stdout_write = sys.stdin, sys.stdout.write + for i in stdin: + stdout_write(i) + raise + else: + buf_size = tqdm_args.pop('buf_size', 256) + delim = tqdm_args.pop('delim', b'\\n') + tee = tqdm_args.pop('tee', False) + manpath = tqdm_args.pop('manpath', None) + comppath = tqdm_args.pop('comppath', None) + if tqdm_args.pop('null', False): + class stdout(object): + @staticmethod + def write(_): + pass + else: + stdout = sys.stdout + stdout = getattr(stdout, 'buffer', stdout) + stdin = getattr(sys.stdin, 'buffer', sys.stdin) + if manpath or comppath: + from os import path + from shutil import copyfile + try: # py<3.7 + import importlib_resources as resources + except ImportError: + from importlib import resources + + def cp(name, dst): + """copy resource `name` to `dst`""" + if hasattr(resources, 'files'): + copyfile(str(resources.files('tqdm') / name), dst) + else: # py<3.9 + with resources.path('tqdm', name) as src: + copyfile(str(src), dst) + log.info("written:%s", dst) + if manpath is not None: + cp('tqdm.1', path.join(manpath, 'tqdm.1')) + if comppath is not None: + cp('completion.sh', path.join(comppath, 'tqdm_completion.sh')) + sys.exit(0) + if tee: + stdout_write = stdout.write + fp_write = getattr(fp, 'buffer', fp).write + + class stdout(object): # pylint: disable=function-redefined + @staticmethod + def write(x): + with tqdm.external_write_mode(file=fp): + fp_write(x) + stdout_write(x) + if delim_per_char: + tqdm_args.setdefault('unit', 'B') + tqdm_args.setdefault('unit_scale', True) + tqdm_args.setdefault('unit_divisor', 1024) + log.debug(tqdm_args) + with tqdm(**tqdm_args) as t: + posix_pipe(stdin, stdout, '', buf_size, t.update) + elif delim == b'\\n': + log.debug(tqdm_args) + write = stdout.write + if update or update_to: + with tqdm(**tqdm_args) as t: + if update: + def callback(i): + t.update(numeric(i.decode())) + else: # update_to + def callback(i): + t.update(numeric(i.decode()) - t.n) + for i in stdin: + write(i) + callback(i) + else: + for i in tqdm(stdin, **tqdm_args): + write(i) + else: + log.debug(tqdm_args) + with tqdm(**tqdm_args) as t: + callback_len = False + if update: + def callback(i): + t.update(numeric(i.decode())) + elif update_to: + def callback(i): + t.update(numeric(i.decode()) - t.n) + else: + callback = t.update + callback_len = True + posix_pipe(stdin, stdout, delim, buf_size, callback, callback_len) diff --git a/tqdm/completion.sh b/tqdm/completion.sh new file mode 100755 index 0000000..9f61c7f --- /dev/null +++ b/tqdm/completion.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +_tqdm(){ + local cur prv + cur="${COMP_WORDS[COMP_CWORD]}" + prv="${COMP_WORDS[COMP_CWORD - 1]}" + + case ${prv} in + --bar_format|--buf_size|--colour|--comppath|--delay|--delim|--desc|--initial|--lock_args|--manpath|--maxinterval|--mininterval|--miniters|--ncols|--nrows|--position|--postfix|--smoothing|--total|--unit|--unit_divisor) + # await user input + ;; + "--log") + COMPREPLY=($(compgen -W 'CRITICAL FATAL ERROR WARN WARNING INFO DEBUG NOTSET' -- ${cur})) + ;; + *) + COMPREPLY=($(compgen -W '--ascii --bar_format --buf_size --bytes --colour --comppath --delay --delim --desc --disable --dynamic_ncols --help --initial --leave --lock_args --log --manpath --maxinterval --mininterval --miniters --ncols --nrows --null --position --postfix --smoothing --tee --total --unit --unit_divisor --unit_scale --update --update_to --version --write_bytes -h -v' -- ${cur})) + ;; + esac +} +complete -F _tqdm tqdm diff --git a/tqdm/contrib/__init__.py b/tqdm/contrib/__init__.py new file mode 100644 index 0000000..0b52177 --- /dev/null +++ b/tqdm/contrib/__init__.py @@ -0,0 +1,98 @@ +""" +Thin wrappers around common functions. + +Subpackages contain potentially unstable extensions. +""" +import sys +from functools import wraps + +from ..auto import tqdm as tqdm_auto +from ..std import tqdm +from ..utils import ObjectWrapper + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['tenumerate', 'tzip', 'tmap'] + + +class DummyTqdmFile(ObjectWrapper): + """Dummy file-like that will write to tqdm""" + + def __init__(self, wrapped): + super(DummyTqdmFile, self).__init__(wrapped) + self._buf = [] + + def write(self, x, nolock=False): + nl = b"\n" if isinstance(x, bytes) else "\n" + pre, sep, post = x.rpartition(nl) + if sep: + blank = type(nl)() + tqdm.write(blank.join(self._buf + [pre, sep]), + end=blank, file=self._wrapped, nolock=nolock) + self._buf = [post] + else: + self._buf.append(x) + + def __del__(self): + if self._buf: + blank = type(self._buf[0])() + try: + tqdm.write(blank.join(self._buf), end=blank, file=self._wrapped) + except (OSError, ValueError): + pass + + +def builtin_iterable(func): + """Wraps `func()` output in a `list()` in py2""" + if sys.version_info[:1] < (3,): + @wraps(func) + def inner(*args, **kwargs): + return list(func(*args, **kwargs)) + return inner + return func + + +def tenumerate(iterable, start=0, total=None, tqdm_class=tqdm_auto, **tqdm_kwargs): + """ + Equivalent of `numpy.ndenumerate` or builtin `enumerate`. + + Parameters + ---------- + tqdm_class : [default: tqdm.auto.tqdm]. + """ + try: + import numpy as np + except ImportError: + pass + else: + if isinstance(iterable, np.ndarray): + return tqdm_class(np.ndenumerate(iterable), total=total or iterable.size, + **tqdm_kwargs) + return enumerate(tqdm_class(iterable, total=total, **tqdm_kwargs), start) + + +@builtin_iterable +def tzip(iter1, *iter2plus, **tqdm_kwargs): + """ + Equivalent of builtin `zip`. + + Parameters + ---------- + tqdm_class : [default: tqdm.auto.tqdm]. + """ + kwargs = tqdm_kwargs.copy() + tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) + for i in zip(tqdm_class(iter1, **kwargs), *iter2plus): + yield i + + +@builtin_iterable +def tmap(function, *sequences, **tqdm_kwargs): + """ + Equivalent of builtin `map`. + + Parameters + ---------- + tqdm_class : [default: tqdm.auto.tqdm]. + """ + for i in tzip(*sequences, **tqdm_kwargs): + yield function(*i) diff --git a/tqdm/contrib/bells.py b/tqdm/contrib/bells.py new file mode 100644 index 0000000..5b8f4b9 --- /dev/null +++ b/tqdm/contrib/bells.py @@ -0,0 +1,26 @@ +""" +Even more features than `tqdm.auto` (all the bells & whistles): + +- `tqdm.auto` +- `tqdm.tqdm.pandas` +- `tqdm.contrib.telegram` + + uses `${TQDM_TELEGRAM_TOKEN}` and `${TQDM_TELEGRAM_CHAT_ID}` +- `tqdm.contrib.discord` + + uses `${TQDM_DISCORD_TOKEN}` and `${TQDM_DISCORD_CHANNEL_ID}` +""" +__all__ = ['tqdm', 'trange'] +import warnings +from os import getenv + +if getenv("TQDM_SLACK_TOKEN") and getenv("TQDM_SLACK_CHANNEL"): + from .slack import tqdm, trange +elif getenv("TQDM_TELEGRAM_TOKEN") and getenv("TQDM_TELEGRAM_CHAT_ID"): + from .telegram import tqdm, trange +elif getenv("TQDM_DISCORD_TOKEN") and getenv("TQDM_DISCORD_CHANNEL_ID"): + from .discord import tqdm, trange +else: + from ..auto import tqdm, trange + +with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=FutureWarning) + tqdm.pandas() diff --git a/tqdm/contrib/concurrent.py b/tqdm/contrib/concurrent.py new file mode 100644 index 0000000..ccb5e12 --- /dev/null +++ b/tqdm/contrib/concurrent.py @@ -0,0 +1,130 @@ +""" +Thin wrappers around `concurrent.futures`. +""" +from __future__ import absolute_import + +from contextlib import contextmanager + +from ..auto import tqdm as tqdm_auto +from ..std import TqdmWarning + +try: + from operator import length_hint +except ImportError: + def length_hint(it, default=0): + """Returns `len(it)`, falling back to `default`""" + try: + return len(it) + except TypeError: + return default +try: + from os import cpu_count +except ImportError: + try: + from multiprocessing import cpu_count + except ImportError: + def cpu_count(): + return 4 +import sys + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['thread_map', 'process_map'] + + +@contextmanager +def ensure_lock(tqdm_class, lock_name=""): + """get (create if necessary) and then restore `tqdm_class`'s lock""" + old_lock = getattr(tqdm_class, '_lock', None) # don't create a new lock + lock = old_lock or tqdm_class.get_lock() # maybe create a new lock + lock = getattr(lock, lock_name, lock) # maybe subtype + tqdm_class.set_lock(lock) + yield lock + if old_lock is None: + del tqdm_class._lock + else: + tqdm_class.set_lock(old_lock) + + +def _executor_map(PoolExecutor, fn, *iterables, **tqdm_kwargs): + """ + Implementation of `thread_map` and `process_map`. + + Parameters + ---------- + tqdm_class : [default: tqdm.auto.tqdm]. + max_workers : [default: min(32, cpu_count() + 4)]. + chunksize : [default: 1]. + lock_name : [default: "":str]. + """ + kwargs = tqdm_kwargs.copy() + if "total" not in kwargs: + kwargs["total"] = length_hint(iterables[0]) + tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) + max_workers = kwargs.pop("max_workers", min(32, cpu_count() + 4)) + chunksize = kwargs.pop("chunksize", 1) + lock_name = kwargs.pop("lock_name", "") + with ensure_lock(tqdm_class, lock_name=lock_name) as lk: + pool_kwargs = {'max_workers': max_workers} + sys_version = sys.version_info[:2] + if sys_version >= (3, 7): + # share lock in case workers are already using `tqdm` + pool_kwargs.update(initializer=tqdm_class.set_lock, initargs=(lk,)) + map_args = {} + if not (3, 0) < sys_version < (3, 5): + map_args.update(chunksize=chunksize) + with PoolExecutor(**pool_kwargs) as ex: + return list(tqdm_class(ex.map(fn, *iterables, **map_args), **kwargs)) + + +def thread_map(fn, *iterables, **tqdm_kwargs): + """ + Equivalent of `list(map(fn, *iterables))` + driven by `concurrent.futures.ThreadPoolExecutor`. + + Parameters + ---------- + tqdm_class : optional + `tqdm` class to use for bars [default: tqdm.auto.tqdm]. + max_workers : int, optional + Maximum number of workers to spawn; passed to + `concurrent.futures.ThreadPoolExecutor.__init__`. + [default: max(32, cpu_count() + 4)]. + """ + from concurrent.futures import ThreadPoolExecutor + return _executor_map(ThreadPoolExecutor, fn, *iterables, **tqdm_kwargs) + + +def process_map(fn, *iterables, **tqdm_kwargs): + """ + Equivalent of `list(map(fn, *iterables))` + driven by `concurrent.futures.ProcessPoolExecutor`. + + Parameters + ---------- + tqdm_class : optional + `tqdm` class to use for bars [default: tqdm.auto.tqdm]. + max_workers : int, optional + Maximum number of workers to spawn; passed to + `concurrent.futures.ProcessPoolExecutor.__init__`. + [default: min(32, cpu_count() + 4)]. + chunksize : int, optional + Size of chunks sent to worker processes; passed to + `concurrent.futures.ProcessPoolExecutor.map`. [default: 1]. + lock_name : str, optional + Member of `tqdm_class.get_lock()` to use [default: mp_lock]. + """ + from concurrent.futures import ProcessPoolExecutor + if iterables and "chunksize" not in tqdm_kwargs: + # default `chunksize=1` has poor performance for large iterables + # (most time spent dispatching items to workers). + longest_iterable_len = max(map(length_hint, iterables)) + if longest_iterable_len > 1000: + from warnings import warn + warn("Iterable length %d > 1000 but `chunksize` is not set." + " This may seriously degrade multiprocess performance." + " Set `chunksize=1` or more." % longest_iterable_len, + TqdmWarning, stacklevel=2) + if "lock_name" not in tqdm_kwargs: + tqdm_kwargs = tqdm_kwargs.copy() + tqdm_kwargs["lock_name"] = "mp_lock" + return _executor_map(ProcessPoolExecutor, fn, *iterables, **tqdm_kwargs) diff --git a/tqdm/contrib/discord.py b/tqdm/contrib/discord.py new file mode 100644 index 0000000..0edd35c --- /dev/null +++ b/tqdm/contrib/discord.py @@ -0,0 +1,125 @@ +""" +Sends updates to a Discord bot. + +Usage: +>>> from tqdm.contrib.discord import tqdm, trange +>>> for i in trange(10, token='{token}', channel_id='{channel_id}'): +... ... + + +""" +from __future__ import absolute_import + +import logging +from os import getenv + +try: + from disco.client import Client, ClientConfig +except ImportError: + raise ImportError("Please `pip install disco-py`") + +from ..auto import tqdm as tqdm_auto +from ..utils import _range +from .utils_worker import MonoWorker + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['DiscordIO', 'tqdm_discord', 'tdrange', 'tqdm', 'trange'] + + +class DiscordIO(MonoWorker): + """Non-blocking file-like IO using a Discord Bot.""" + def __init__(self, token, channel_id): + """Creates a new message in the given `channel_id`.""" + super(DiscordIO, self).__init__() + config = ClientConfig() + config.token = token + client = Client(config) + self.text = self.__class__.__name__ + try: + self.message = client.api.channels_messages_create(channel_id, self.text) + except Exception as e: + tqdm_auto.write(str(e)) + self.message = None + + def write(self, s): + """Replaces internal `message`'s text with `s`.""" + if not s: + s = "..." + s = s.replace('\r', '').strip() + if s == self.text: + return # skip duplicate message + message = self.message + if message is None: + return + self.text = s + try: + future = self.submit(message.edit, '`' + s + '`') + except Exception as e: + tqdm_auto.write(str(e)) + else: + return future + + +class tqdm_discord(tqdm_auto): + """ + Standard `tqdm.auto.tqdm` but also sends updates to a Discord Bot. + May take a few seconds to create (`__init__`). + + - create a discord bot (not public, no requirement of OAuth2 code + grant, only send message permissions) & invite it to a channel: + <https://discordpy.readthedocs.io/en/latest/discord.html> + - copy the bot `{token}` & `{channel_id}` and paste below + + >>> from tqdm.contrib.discord import tqdm, trange + >>> for i in tqdm(iterable, token='{token}', channel_id='{channel_id}'): + ... ... + """ + def __init__(self, *args, **kwargs): + """ + Parameters + ---------- + token : str, required. Discord token + [default: ${TQDM_DISCORD_TOKEN}]. + channel_id : int, required. Discord channel ID + [default: ${TQDM_DISCORD_CHANNEL_ID}]. + mininterval : float, optional. + Minimum of [default: 1.5] to avoid rate limit. + + See `tqdm.auto.tqdm.__init__` for other parameters. + """ + if not kwargs.get('disable'): + kwargs = kwargs.copy() + logging.getLogger("HTTPClient").setLevel(logging.WARNING) + self.dio = DiscordIO( + kwargs.pop('token', getenv("TQDM_DISCORD_TOKEN")), + kwargs.pop('channel_id', getenv("TQDM_DISCORD_CHANNEL_ID"))) + kwargs['mininterval'] = max(1.5, kwargs.get('mininterval', 1.5)) + super(tqdm_discord, self).__init__(*args, **kwargs) + + def display(self, **kwargs): + super(tqdm_discord, self).display(**kwargs) + fmt = self.format_dict + if fmt.get('bar_format', None): + fmt['bar_format'] = fmt['bar_format'].replace( + '<bar/>', '{bar:10u}').replace('{bar}', '{bar:10u}') + else: + fmt['bar_format'] = '{l_bar}{bar:10u}{r_bar}' + self.dio.write(self.format_meter(**fmt)) + + def clear(self, *args, **kwargs): + super(tqdm_discord, self).clear(*args, **kwargs) + if not self.disable: + self.dio.write("") + + +def tdrange(*args, **kwargs): + """ + A shortcut for `tqdm.contrib.discord.tqdm(xrange(*args), **kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + return tqdm_discord(_range(*args), **kwargs) + + +# Aliases +tqdm = tqdm_discord +trange = tdrange diff --git a/tqdm/contrib/itertools.py b/tqdm/contrib/itertools.py new file mode 100644 index 0000000..5f22505 --- /dev/null +++ b/tqdm/contrib/itertools.py @@ -0,0 +1,37 @@ +""" +Thin wrappers around `itertools`. +""" +from __future__ import absolute_import + +import itertools + +from ..auto import tqdm as tqdm_auto + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['product'] + + +def product(*iterables, **tqdm_kwargs): + """ + Equivalent of `itertools.product`. + + Parameters + ---------- + tqdm_class : [default: tqdm.auto.tqdm]. + """ + kwargs = tqdm_kwargs.copy() + tqdm_class = kwargs.pop("tqdm_class", tqdm_auto) + try: + lens = list(map(len, iterables)) + except TypeError: + total = None + else: + total = 1 + for i in lens: + total *= i + kwargs.setdefault("total", total) + with tqdm_class(**kwargs) as t: + it = itertools.product(*iterables) + for i in it: + yield i + t.update() diff --git a/tqdm/contrib/logging.py b/tqdm/contrib/logging.py new file mode 100644 index 0000000..cd9925e --- /dev/null +++ b/tqdm/contrib/logging.py @@ -0,0 +1,128 @@ +""" +Helper functionality for interoperability with stdlib `logging`. +""" +from __future__ import absolute_import + +import logging +import sys +from contextlib import contextmanager + +try: + from typing import Iterator, List, Optional, Type # pylint: disable=unused-import +except ImportError: + pass + +from ..std import tqdm as std_tqdm + + +class _TqdmLoggingHandler(logging.StreamHandler): + def __init__( + self, + tqdm_class=std_tqdm # type: Type[std_tqdm] + ): + super(_TqdmLoggingHandler, self).__init__() + self.tqdm_class = tqdm_class + + def emit(self, record): + try: + msg = self.format(record) + self.tqdm_class.write(msg, file=self.stream) + self.flush() + except (KeyboardInterrupt, SystemExit): + raise + except: # noqa pylint: disable=bare-except + self.handleError(record) + + +def _is_console_logging_handler(handler): + return (isinstance(handler, logging.StreamHandler) + and handler.stream in {sys.stdout, sys.stderr}) + + +def _get_first_found_console_logging_handler(handlers): + for handler in handlers: + if _is_console_logging_handler(handler): + return handler + + +@contextmanager +def logging_redirect_tqdm( + loggers=None, # type: Optional[List[logging.Logger]], + tqdm_class=std_tqdm # type: Type[std_tqdm] +): + # type: (...) -> Iterator[None] + """ + Context manager redirecting console logging to `tqdm.write()`, leaving + other logging handlers (e.g. log files) unaffected. + + Parameters + ---------- + loggers : list, optional + Which handlers to redirect (default: [logging.root]). + tqdm_class : optional + + Example + ------- + ```python + import logging + from tqdm import trange + from tqdm.contrib.logging import logging_redirect_tqdm + + LOG = logging.getLogger(__name__) + + if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + with logging_redirect_tqdm(): + for i in trange(9): + if i == 4: + LOG.info("console logging redirected to `tqdm.write()`") + # logging restored + ``` + """ + if loggers is None: + loggers = [logging.root] + original_handlers_list = [logger.handlers for logger in loggers] + try: + for logger in loggers: + tqdm_handler = _TqdmLoggingHandler(tqdm_class) + orig_handler = _get_first_found_console_logging_handler(logger.handlers) + if orig_handler is not None: + tqdm_handler.setFormatter(orig_handler.formatter) + tqdm_handler.stream = orig_handler.stream + logger.handlers = [ + handler for handler in logger.handlers + if not _is_console_logging_handler(handler)] + [tqdm_handler] + yield + finally: + for logger, original_handlers in zip(loggers, original_handlers_list): + logger.handlers = original_handlers + + +@contextmanager +def tqdm_logging_redirect( + *args, + # loggers=None, # type: Optional[List[logging.Logger]] + # tqdm=None, # type: Optional[Type[tqdm.tqdm]] + **kwargs +): + # type: (...) -> Iterator[None] + """ + Convenience shortcut for: + ```python + with tqdm_class(*args, **tqdm_kwargs) as pbar: + with logging_redirect_tqdm(loggers=loggers, tqdm_class=tqdm_class): + yield pbar + ``` + + Parameters + ---------- + tqdm_class : optional, (default: tqdm.std.tqdm). + loggers : optional, list. + **tqdm_kwargs : passed to `tqdm_class`. + """ + tqdm_kwargs = kwargs.copy() + loggers = tqdm_kwargs.pop('loggers', None) + tqdm_class = tqdm_kwargs.pop('tqdm_class', std_tqdm) + with tqdm_class(*args, **tqdm_kwargs) as pbar: + with logging_redirect_tqdm(loggers=loggers, tqdm_class=tqdm_class): + yield pbar diff --git a/tqdm/contrib/slack.py b/tqdm/contrib/slack.py new file mode 100644 index 0000000..b478d92 --- /dev/null +++ b/tqdm/contrib/slack.py @@ -0,0 +1,126 @@ +""" +Sends updates to a Slack app. + +Usage: +>>> from tqdm.contrib.slack import tqdm, trange +>>> for i in trange(10, token='{token}', channel='{channel}'): +... ... + + +""" +from __future__ import absolute_import + +import logging +from os import getenv + +try: + from slack_sdk import WebClient +except ImportError: + raise ImportError("Please `pip install slack-sdk`") + +from ..auto import tqdm as tqdm_auto +from ..utils import _range +from .utils_worker import MonoWorker + +__author__ = {"github.com/": ["0x2b3bfa0", "casperdcl"]} +__all__ = ['SlackIO', 'tqdm_slack', 'tsrange', 'tqdm', 'trange'] + + +class SlackIO(MonoWorker): + """Non-blocking file-like IO using a Slack app.""" + def __init__(self, token, channel): + """Creates a new message in the given `channel`.""" + super(SlackIO, self).__init__() + self.client = WebClient(token=token) + self.text = self.__class__.__name__ + try: + self.message = self.client.chat_postMessage(channel=channel, text=self.text) + except Exception as e: + tqdm_auto.write(str(e)) + self.message = None + + def write(self, s): + """Replaces internal `message`'s text with `s`.""" + if not s: + s = "..." + s = s.replace('\r', '').strip() + if s == self.text: + return # skip duplicate message + message = self.message + if message is None: + return + self.text = s + try: + future = self.submit(self.client.chat_update, channel=message['channel'], + ts=message['ts'], text='`' + s + '`') + except Exception as e: + tqdm_auto.write(str(e)) + else: + return future + + +class tqdm_slack(tqdm_auto): + """ + Standard `tqdm.auto.tqdm` but also sends updates to a Slack app. + May take a few seconds to create (`__init__`). + + - create a Slack app with the `chat:write` scope & invite it to a + channel: <https://api.slack.com/authentication/basics> + - copy the bot `{token}` & `{channel}` and paste below + >>> from tqdm.contrib.slack import tqdm, trange + >>> for i in tqdm(iterable, token='{token}', channel='{channel}'): + ... ... + """ + def __init__(self, *args, **kwargs): + """ + Parameters + ---------- + token : str, required. Slack token + [default: ${TQDM_SLACK_TOKEN}]. + channel : int, required. Slack channel + [default: ${TQDM_SLACK_CHANNEL}]. + mininterval : float, optional. + Minimum of [default: 1.5] to avoid rate limit. + + See `tqdm.auto.tqdm.__init__` for other parameters. + """ + if not kwargs.get('disable'): + kwargs = kwargs.copy() + logging.getLogger("HTTPClient").setLevel(logging.WARNING) + self.sio = SlackIO( + kwargs.pop('token', getenv("TQDM_SLACK_TOKEN")), + kwargs.pop('channel', getenv("TQDM_SLACK_CHANNEL"))) + kwargs['mininterval'] = max(1.5, kwargs.get('mininterval', 1.5)) + super(tqdm_slack, self).__init__(*args, **kwargs) + + def display(self, **kwargs): + super(tqdm_slack, self).display(**kwargs) + fmt = self.format_dict + if fmt.get('bar_format', None): + fmt['bar_format'] = fmt['bar_format'].replace( + '<bar/>', '`{bar:10}`').replace('{bar}', '`{bar:10u}`') + else: + fmt['bar_format'] = '{l_bar}`{bar:10}`{r_bar}' + if fmt['ascii'] is False: + fmt['ascii'] = [":black_square:", ":small_blue_diamond:", ":large_blue_diamond:", + ":large_blue_square:"] + fmt['ncols'] = 336 + self.sio.write(self.format_meter(**fmt)) + + def clear(self, *args, **kwargs): + super(tqdm_slack, self).clear(*args, **kwargs) + if not self.disable: + self.sio.write("") + + +def tsrange(*args, **kwargs): + """ + A shortcut for `tqdm.contrib.slack.tqdm(xrange(*args), **kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + return tqdm_slack(_range(*args), **kwargs) + + +# Aliases +tqdm = tqdm_slack +trange = tsrange diff --git a/tqdm/contrib/telegram.py b/tqdm/contrib/telegram.py new file mode 100644 index 0000000..99cbe8c --- /dev/null +++ b/tqdm/contrib/telegram.py @@ -0,0 +1,159 @@ +""" +Sends updates to a Telegram bot. + +Usage: +>>> from tqdm.contrib.telegram import tqdm, trange +>>> for i in trange(10, token='{token}', chat_id='{chat_id}'): +... ... + + +""" +from __future__ import absolute_import + +from os import getenv +from warnings import warn + +from requests import Session + +from ..auto import tqdm as tqdm_auto +from ..std import TqdmWarning +from ..utils import _range +from .utils_worker import MonoWorker + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['TelegramIO', 'tqdm_telegram', 'ttgrange', 'tqdm', 'trange'] + + +class TelegramIO(MonoWorker): + """Non-blocking file-like IO using a Telegram Bot.""" + API = 'https://api.telegram.org/bot' + + def __init__(self, token, chat_id): + """Creates a new message in the given `chat_id`.""" + super(TelegramIO, self).__init__() + self.token = token + self.chat_id = chat_id + self.session = Session() + self.text = self.__class__.__name__ + self.message_id + + @property + def message_id(self): + if hasattr(self, '_message_id'): + return self._message_id + try: + res = self.session.post( + self.API + '%s/sendMessage' % self.token, + data={'text': '`' + self.text + '`', 'chat_id': self.chat_id, + 'parse_mode': 'MarkdownV2'}).json() + except Exception as e: + tqdm_auto.write(str(e)) + else: + if res.get('error_code') == 429: + warn("Creation rate limit: try increasing `mininterval`.", + TqdmWarning, stacklevel=2) + else: + self._message_id = res['result']['message_id'] + return self._message_id + + def write(self, s): + """Replaces internal `message_id`'s text with `s`.""" + if not s: + s = "..." + s = s.replace('\r', '').strip() + if s == self.text: + return # avoid duplicate message Bot error + message_id = self.message_id + if message_id is None: + return + self.text = s + try: + future = self.submit( + self.session.post, self.API + '%s/editMessageText' % self.token, + data={'text': '`' + s + '`', 'chat_id': self.chat_id, + 'message_id': message_id, 'parse_mode': 'MarkdownV2'}) + except Exception as e: + tqdm_auto.write(str(e)) + else: + return future + + def delete(self): + """Deletes internal `message_id`.""" + try: + future = self.submit( + self.session.post, self.API + '%s/deleteMessage' % self.token, + data={'chat_id': self.chat_id, 'message_id': self.message_id}) + except Exception as e: + tqdm_auto.write(str(e)) + else: + return future + + +class tqdm_telegram(tqdm_auto): + """ + Standard `tqdm.auto.tqdm` but also sends updates to a Telegram Bot. + May take a few seconds to create (`__init__`). + + - create a bot <https://core.telegram.org/bots#6-botfather> + - copy its `{token}` + - add the bot to a chat and send it a message such as `/start` + - go to <https://api.telegram.org/bot`{token}`/getUpdates> to find out + the `{chat_id}` + - paste the `{token}` & `{chat_id}` below + + >>> from tqdm.contrib.telegram import tqdm, trange + >>> for i in tqdm(iterable, token='{token}', chat_id='{chat_id}'): + ... ... + """ + def __init__(self, *args, **kwargs): + """ + Parameters + ---------- + token : str, required. Telegram token + [default: ${TQDM_TELEGRAM_TOKEN}]. + chat_id : str, required. Telegram chat ID + [default: ${TQDM_TELEGRAM_CHAT_ID}]. + + See `tqdm.auto.tqdm.__init__` for other parameters. + """ + if not kwargs.get('disable'): + kwargs = kwargs.copy() + self.tgio = TelegramIO( + kwargs.pop('token', getenv('TQDM_TELEGRAM_TOKEN')), + kwargs.pop('chat_id', getenv('TQDM_TELEGRAM_CHAT_ID'))) + super(tqdm_telegram, self).__init__(*args, **kwargs) + + def display(self, **kwargs): + super(tqdm_telegram, self).display(**kwargs) + fmt = self.format_dict + if fmt.get('bar_format', None): + fmt['bar_format'] = fmt['bar_format'].replace( + '<bar/>', '{bar:10u}').replace('{bar}', '{bar:10u}') + else: + fmt['bar_format'] = '{l_bar}{bar:10u}{r_bar}' + self.tgio.write(self.format_meter(**fmt)) + + def clear(self, *args, **kwargs): + super(tqdm_telegram, self).clear(*args, **kwargs) + if not self.disable: + self.tgio.write("") + + def close(self): + if self.disable: + return + super(tqdm_telegram, self).close() + if not (self.leave or (self.leave is None and self.pos == 0)): + self.tgio.delete() + + +def ttgrange(*args, **kwargs): + """ + A shortcut for `tqdm.contrib.telegram.tqdm(xrange(*args), **kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + return tqdm_telegram(_range(*args), **kwargs) + + +# Aliases +tqdm = tqdm_telegram +trange = ttgrange diff --git a/tqdm/contrib/utils_worker.py b/tqdm/contrib/utils_worker.py new file mode 100644 index 0000000..17adda6 --- /dev/null +++ b/tqdm/contrib/utils_worker.py @@ -0,0 +1,40 @@ +""" +IO/concurrency helpers for `tqdm.contrib`. +""" +from __future__ import absolute_import + +from collections import deque +from concurrent.futures import ThreadPoolExecutor + +from ..auto import tqdm as tqdm_auto + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['MonoWorker'] + + +class MonoWorker(object): + """ + Supports one running task and one waiting task. + The waiting task is the most recent submitted (others are discarded). + """ + def __init__(self): + self.pool = ThreadPoolExecutor(max_workers=1) + self.futures = deque([], 2) + + def submit(self, func, *args, **kwargs): + """`func(*args, **kwargs)` may replace currently waiting task.""" + futures = self.futures + if len(futures) == futures.maxlen: + running = futures.popleft() + if not running.done(): + if len(futures): # clear waiting + waiting = futures.pop() + waiting.cancel() + futures.appendleft(running) # re-insert running + try: + waiting = self.pool.submit(func, *args, **kwargs) + except Exception as e: + tqdm_auto.write(str(e)) + else: + futures.append(waiting) + return waiting diff --git a/tqdm/dask.py b/tqdm/dask.py new file mode 100644 index 0000000..6fc7504 --- /dev/null +++ b/tqdm/dask.py @@ -0,0 +1,46 @@ +from __future__ import absolute_import + +from functools import partial + +from dask.callbacks import Callback + +from .auto import tqdm as tqdm_auto + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['TqdmCallback'] + + +class TqdmCallback(Callback): + """Dask callback for task progress.""" + def __init__(self, start=None, pretask=None, tqdm_class=tqdm_auto, + **tqdm_kwargs): + """ + Parameters + ---------- + tqdm_class : optional + `tqdm` class to use for bars [default: `tqdm.auto.tqdm`]. + tqdm_kwargs : optional + Any other arguments used for all bars. + """ + super(TqdmCallback, self).__init__(start=start, pretask=pretask) + if tqdm_kwargs: + tqdm_class = partial(tqdm_class, **tqdm_kwargs) + self.tqdm_class = tqdm_class + + def _start_state(self, _, state): + self.pbar = self.tqdm_class(total=sum( + len(state[k]) for k in ['ready', 'waiting', 'running', 'finished'])) + + def _posttask(self, *_, **__): + self.pbar.update() + + def _finish(self, *_, **__): + self.pbar.close() + + def display(self): + """Displays in the current cell in Notebooks.""" + container = getattr(self.bar, 'container', None) + if container is None: + return + from .notebook import display + display(container) diff --git a/tqdm/gui.py b/tqdm/gui.py new file mode 100644 index 0000000..4612701 --- /dev/null +++ b/tqdm/gui.py @@ -0,0 +1,191 @@ +""" +Matplotlib GUI progressbar decorator for iterators. + +Usage: +>>> from tqdm.gui import trange, tqdm +>>> for i in trange(10): +... ... +""" +# future division is important to divide integers and get as +# a result precise floating numbers (instead of truncated int) +from __future__ import absolute_import, division + +import re +from warnings import warn + +# to inherit from the tqdm class +from .std import TqdmExperimentalWarning +from .std import tqdm as std_tqdm +# import compatibility functions and utilities +from .utils import _range + +__author__ = {"github.com/": ["casperdcl", "lrq3000"]} +__all__ = ['tqdm_gui', 'tgrange', 'tqdm', 'trange'] + + +class tqdm_gui(std_tqdm): # pragma: no cover + """Experimental Matplotlib GUI version of tqdm!""" + # TODO: @classmethod: write() on GUI? + def __init__(self, *args, **kwargs): + from collections import deque + + import matplotlib as mpl + import matplotlib.pyplot as plt + kwargs = kwargs.copy() + kwargs['gui'] = True + colour = kwargs.pop('colour', 'g') + super(tqdm_gui, self).__init__(*args, **kwargs) + + if self.disable: + return + + warn("GUI is experimental/alpha", TqdmExperimentalWarning, stacklevel=2) + self.mpl = mpl + self.plt = plt + + # Remember if external environment uses toolbars + self.toolbar = self.mpl.rcParams['toolbar'] + self.mpl.rcParams['toolbar'] = 'None' + + self.mininterval = max(self.mininterval, 0.5) + self.fig, ax = plt.subplots(figsize=(9, 2.2)) + # self.fig.subplots_adjust(bottom=0.2) + total = self.__len__() # avoids TypeError on None #971 + if total is not None: + self.xdata = [] + self.ydata = [] + self.zdata = [] + else: + self.xdata = deque([]) + self.ydata = deque([]) + self.zdata = deque([]) + self.line1, = ax.plot(self.xdata, self.ydata, color='b') + self.line2, = ax.plot(self.xdata, self.zdata, color='k') + ax.set_ylim(0, 0.001) + if total is not None: + ax.set_xlim(0, 100) + ax.set_xlabel("percent") + self.fig.legend((self.line1, self.line2), ("cur", "est"), + loc='center right') + # progressbar + self.hspan = plt.axhspan(0, 0.001, xmin=0, xmax=0, color=colour) + else: + # ax.set_xlim(-60, 0) + ax.set_xlim(0, 60) + ax.invert_xaxis() + ax.set_xlabel("seconds") + ax.legend(("cur", "est"), loc='lower left') + ax.grid() + # ax.set_xlabel('seconds') + ax.set_ylabel((self.unit if self.unit else "it") + "/s") + if self.unit_scale: + plt.ticklabel_format(style='sci', axis='y', scilimits=(0, 0)) + ax.yaxis.get_offset_text().set_x(-0.15) + + # Remember if external environment is interactive + self.wasion = plt.isinteractive() + plt.ion() + self.ax = ax + + def close(self): + if self.disable: + return + + self.disable = True + + with self.get_lock(): + self._instances.remove(self) + + # Restore toolbars + self.mpl.rcParams['toolbar'] = self.toolbar + # Return to non-interactive mode + if not self.wasion: + self.plt.ioff() + if self.leave: + self.display() + else: + self.plt.close(self.fig) + + def clear(self, *_, **__): + pass + + def display(self, *_, **__): + n = self.n + cur_t = self._time() + elapsed = cur_t - self.start_t + delta_it = n - self.last_print_n + delta_t = cur_t - self.last_print_t + + # Inline due to multiple calls + total = self.total + xdata = self.xdata + ydata = self.ydata + zdata = self.zdata + ax = self.ax + line1 = self.line1 + line2 = self.line2 + # instantaneous rate + y = delta_it / delta_t + # overall rate + z = n / elapsed + # update line data + xdata.append(n * 100.0 / total if total else cur_t) + ydata.append(y) + zdata.append(z) + + # Discard old values + # xmin, xmax = ax.get_xlim() + # if (not total) and elapsed > xmin * 1.1: + if (not total) and elapsed > 66: + xdata.popleft() + ydata.popleft() + zdata.popleft() + + ymin, ymax = ax.get_ylim() + if y > ymax or z > ymax: + ymax = 1.1 * y + ax.set_ylim(ymin, ymax) + ax.figure.canvas.draw() + + if total: + line1.set_data(xdata, ydata) + line2.set_data(xdata, zdata) + try: + poly_lims = self.hspan.get_xy() + except AttributeError: + self.hspan = self.plt.axhspan(0, 0.001, xmin=0, xmax=0, color='g') + poly_lims = self.hspan.get_xy() + poly_lims[0, 1] = ymin + poly_lims[1, 1] = ymax + poly_lims[2] = [n / total, ymax] + poly_lims[3] = [poly_lims[2, 0], ymin] + if len(poly_lims) > 4: + poly_lims[4, 1] = ymin + self.hspan.set_xy(poly_lims) + else: + t_ago = [cur_t - i for i in xdata] + line1.set_data(t_ago, ydata) + line2.set_data(t_ago, zdata) + + d = self.format_dict + # remove {bar} + d['bar_format'] = (d['bar_format'] or "{l_bar}<bar/>{r_bar}").replace( + "{bar}", "<bar/>") + msg = self.format_meter(**d) + if '<bar/>' in msg: + msg = "".join(re.split(r'\|?<bar/>\|?', msg, 1)) + ax.set_title(msg, fontname="DejaVu Sans Mono", fontsize=11) + self.plt.pause(1e-9) + + +def tgrange(*args, **kwargs): + """ + A shortcut for `tqdm.gui.tqdm(xrange(*args), **kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + return tqdm_gui(_range(*args), **kwargs) + + +# Aliases +tqdm = tqdm_gui +trange = tgrange diff --git a/tqdm/keras.py b/tqdm/keras.py new file mode 100644 index 0000000..523e62e --- /dev/null +++ b/tqdm/keras.py @@ -0,0 +1,124 @@ +from __future__ import absolute_import, division + +from copy import copy +from functools import partial + +from .auto import tqdm as tqdm_auto + +try: + import keras +except (ImportError, AttributeError) as e: + try: + from tensorflow import keras + except ImportError: + raise e +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['TqdmCallback'] + + +class TqdmCallback(keras.callbacks.Callback): + """Keras callback for epoch and batch progress.""" + @staticmethod + def bar2callback(bar, pop=None, delta=(lambda logs: 1)): + def callback(_, logs=None): + n = delta(logs) + if logs: + if pop: + logs = copy(logs) + [logs.pop(i, 0) for i in pop] + bar.set_postfix(logs, refresh=False) + bar.update(n) + + return callback + + def __init__(self, epochs=None, data_size=None, batch_size=None, verbose=1, + tqdm_class=tqdm_auto, **tqdm_kwargs): + """ + Parameters + ---------- + epochs : int, optional + data_size : int, optional + Number of training pairs. + batch_size : int, optional + Number of training pairs per batch. + verbose : int + 0: epoch, 1: batch (transient), 2: batch. [default: 1]. + Will be set to `0` unless both `data_size` and `batch_size` + are given. + tqdm_class : optional + `tqdm` class to use for bars [default: `tqdm.auto.tqdm`]. + tqdm_kwargs : optional + Any other arguments used for all bars. + """ + if tqdm_kwargs: + tqdm_class = partial(tqdm_class, **tqdm_kwargs) + self.tqdm_class = tqdm_class + self.epoch_bar = tqdm_class(total=epochs, unit='epoch') + self.on_epoch_end = self.bar2callback(self.epoch_bar) + if data_size and batch_size: + self.batches = batches = (data_size + batch_size - 1) // batch_size + else: + self.batches = batches = None + self.verbose = verbose + if verbose == 1: + self.batch_bar = tqdm_class(total=batches, unit='batch', leave=False) + self.on_batch_end = self.bar2callback( + self.batch_bar, pop=['batch', 'size'], + delta=lambda logs: logs.get('size', 1)) + + def on_train_begin(self, *_, **__): + params = self.params.get + auto_total = params('epochs', params('nb_epoch', None)) + if auto_total is not None and auto_total != self.epoch_bar.total: + self.epoch_bar.reset(total=auto_total) + + def on_epoch_begin(self, epoch, *_, **__): + if self.epoch_bar.n < epoch: + ebar = self.epoch_bar + ebar.n = ebar.last_print_n = ebar.initial = epoch + if self.verbose: + params = self.params.get + total = params('samples', params( + 'nb_sample', params('steps', None))) or self.batches + if self.verbose == 2: + if hasattr(self, 'batch_bar'): + self.batch_bar.close() + self.batch_bar = self.tqdm_class( + total=total, unit='batch', leave=True, + unit_scale=1 / (params('batch_size', 1) or 1)) + self.on_batch_end = self.bar2callback( + self.batch_bar, pop=['batch', 'size'], + delta=lambda logs: logs.get('size', 1)) + elif self.verbose == 1: + self.batch_bar.unit_scale = 1 / (params('batch_size', 1) or 1) + self.batch_bar.reset(total=total) + else: + raise KeyError('Unknown verbosity') + + def on_train_end(self, *_, **__): + if self.verbose: + self.batch_bar.close() + self.epoch_bar.close() + + def display(self): + """Displays in the current cell in Notebooks.""" + container = getattr(self.epoch_bar, 'container', None) + if container is None: + return + from .notebook import display + display(container) + batch_bar = getattr(self, 'batch_bar', None) + if batch_bar is not None: + display(batch_bar.container) + + @staticmethod + def _implements_train_batch_hooks(): + return True + + @staticmethod + def _implements_test_batch_hooks(): + return True + + @staticmethod + def _implements_predict_batch_hooks(): + return True diff --git a/tqdm/notebook.py b/tqdm/notebook.py new file mode 100644 index 0000000..ffd0947 --- /dev/null +++ b/tqdm/notebook.py @@ -0,0 +1,329 @@ +""" +IPython/Jupyter Notebook progressbar decorator for iterators. +Includes a default `range` iterator printing to `stderr`. + +Usage: +>>> from tqdm.notebook import trange, tqdm +>>> for i in trange(10): +... ... +""" +# future division is important to divide integers and get as +# a result precise floating numbers (instead of truncated int) +from __future__ import absolute_import, division + +# import compatibility functions and utilities +import re +import sys +from weakref import proxy + +# to inherit from the tqdm class +from .std import tqdm as std_tqdm +from .utils import _range + +if True: # pragma: no cover + # import IPython/Jupyter base widget and display utilities + IPY = 0 + try: # IPython 4.x + import ipywidgets + IPY = 4 + except ImportError: # IPython 3.x / 2.x + IPY = 32 + import warnings + with warnings.catch_warnings(): + warnings.filterwarnings( + 'ignore', message=".*The `IPython.html` package has been deprecated.*") + try: + import IPython.html.widgets as ipywidgets # NOQA: F401 + except ImportError: + pass + + try: # IPython 4.x / 3.x + if IPY == 32: + from IPython.html.widgets import HTML + from IPython.html.widgets import FloatProgress as IProgress + from IPython.html.widgets import HBox + IPY = 3 + else: + from ipywidgets import HTML + from ipywidgets import FloatProgress as IProgress + from ipywidgets import HBox + except ImportError: + try: # IPython 2.x + from IPython.html.widgets import HTML + from IPython.html.widgets import ContainerWidget as HBox + from IPython.html.widgets import FloatProgressWidget as IProgress + IPY = 2 + except ImportError: + IPY = 0 + IProgress = None + HBox = object + + try: + from IPython.display import display # , clear_output + except ImportError: + pass + + # HTML encoding + try: # Py3 + from html import escape + except ImportError: # Py2 + from cgi import escape + +__author__ = {"github.com/": ["lrq3000", "casperdcl", "alexanderkuk"]} +__all__ = ['tqdm_notebook', 'tnrange', 'tqdm', 'trange'] +WARN_NOIPYW = ("IProgress not found. Please update jupyter and ipywidgets." + " See https://ipywidgets.readthedocs.io/en/stable" + "/user_install.html") + + +class TqdmHBox(HBox): + """`ipywidgets.HBox` with a pretty representation""" + def _json_(self, pretty=None): + pbar = getattr(self, 'pbar', None) + if pbar is None: + return {} + d = pbar.format_dict + if pretty is not None: + d["ascii"] = not pretty + return d + + def __repr__(self, pretty=False): + pbar = getattr(self, 'pbar', None) + if pbar is None: + return super(TqdmHBox, self).__repr__() + return pbar.format_meter(**self._json_(pretty)) + + def _repr_pretty_(self, pp, *_, **__): + pp.text(self.__repr__(True)) + + +class tqdm_notebook(std_tqdm): + """ + Experimental IPython/Jupyter Notebook widget using tqdm! + """ + @staticmethod + def status_printer(_, total=None, desc=None, ncols=None): + """ + Manage the printing of an IPython/Jupyter Notebook progress bar widget. + """ + # Fallback to text bar if there's no total + # DEPRECATED: replaced with an 'info' style bar + # if not total: + # return super(tqdm_notebook, tqdm_notebook).status_printer(file) + + # fp = file + + # Prepare IPython progress bar + if IProgress is None: # #187 #451 #558 #872 + raise ImportError(WARN_NOIPYW) + if total: + pbar = IProgress(min=0, max=total) + else: # No total? Show info style bar with no progress tqdm status + pbar = IProgress(min=0, max=1) + pbar.value = 1 + pbar.bar_style = 'info' + if ncols is None: + pbar.layout.width = "20px" + + ltext = HTML() + rtext = HTML() + if desc: + ltext.value = desc + container = TqdmHBox(children=[ltext, pbar, rtext]) + # Prepare layout + if ncols is not None: # use default style of ipywidgets + # ncols could be 100, "100px", "100%" + ncols = str(ncols) # ipywidgets only accepts string + try: + if int(ncols) > 0: # isnumeric and positive + ncols += 'px' + except ValueError: + pass + pbar.layout.flex = '2' + container.layout.width = ncols + container.layout.display = 'inline-flex' + container.layout.flex_flow = 'row wrap' + + return container + + def display(self, msg=None, pos=None, + # additional signals + close=False, bar_style=None, check_delay=True): + # Note: contrary to native tqdm, msg='' does NOT clear bar + # goal is to keep all infos if error happens so user knows + # at which iteration the loop failed. + + # Clear previous output (really necessary?) + # clear_output(wait=1) + + if not msg and not close: + d = self.format_dict + # remove {bar} + d['bar_format'] = (d['bar_format'] or "{l_bar}<bar/>{r_bar}").replace( + "{bar}", "<bar/>") + msg = self.format_meter(**d) + + ltext, pbar, rtext = self.container.children + pbar.value = self.n + + if msg: + # html escape special characters (like '&') + if '<bar/>' in msg: + left, right = map(escape, re.split(r'\|?<bar/>\|?', msg, 1)) + else: + left, right = '', escape(msg) + + # Update description + ltext.value = left + # never clear the bar (signal: msg='') + if right: + rtext.value = right + + # Change bar style + if bar_style: + # Hack-ish way to avoid the danger bar_style being overridden by + # success because the bar gets closed after the error... + if pbar.bar_style != 'danger' or bar_style != 'success': + pbar.bar_style = bar_style + + # Special signal to close the bar + if close and pbar.bar_style != 'danger': # hide only if no error + try: + self.container.close() + except AttributeError: + self.container.visible = False + self.container.layout.visibility = 'hidden' # IPYW>=8 + + if check_delay and self.delay > 0 and not self.displayed: + display(self.container) + self.displayed = True + + @property + def colour(self): + if hasattr(self, 'container'): + return self.container.children[-2].style.bar_color + + @colour.setter + def colour(self, bar_color): + if hasattr(self, 'container'): + self.container.children[-2].style.bar_color = bar_color + + def __init__(self, *args, **kwargs): + """ + Supports the usual `tqdm.tqdm` parameters as well as those listed below. + + Parameters + ---------- + display : Whether to call `display(self.container)` immediately + [default: True]. + """ + kwargs = kwargs.copy() + # Setup default output + file_kwarg = kwargs.get('file', sys.stderr) + if file_kwarg is sys.stderr or file_kwarg is None: + kwargs['file'] = sys.stdout # avoid the red block in IPython + + # Initialize parent class + avoid printing by using gui=True + kwargs['gui'] = True + # convert disable = None to False + kwargs['disable'] = bool(kwargs.get('disable', False)) + colour = kwargs.pop('colour', None) + display_here = kwargs.pop('display', True) + super(tqdm_notebook, self).__init__(*args, **kwargs) + if self.disable or not kwargs['gui']: + self.disp = lambda *_, **__: None + return + + # Get bar width + self.ncols = '100%' if self.dynamic_ncols else kwargs.get("ncols", None) + + # Replace with IPython progress bar display (with correct total) + unit_scale = 1 if self.unit_scale is True else self.unit_scale or 1 + total = self.total * unit_scale if self.total else self.total + self.container = self.status_printer(self.fp, total, self.desc, self.ncols) + self.container.pbar = proxy(self) + self.displayed = False + if display_here and self.delay <= 0: + display(self.container) + self.displayed = True + self.disp = self.display + self.colour = colour + + # Print initial bar state + if not self.disable: + self.display(check_delay=False) + + def __iter__(self): + try: + it = super(tqdm_notebook, self).__iter__() + for obj in it: + # return super(tqdm...) will not catch exception + yield obj + # NB: except ... [ as ...] breaks IPython async KeyboardInterrupt + except: # NOQA + self.disp(bar_style='danger') + raise + # NB: don't `finally: close()` + # since this could be a shared bar which the user will `reset()` + + def update(self, n=1): + try: + return super(tqdm_notebook, self).update(n=n) + # NB: except ... [ as ...] breaks IPython async KeyboardInterrupt + except: # NOQA + # cannot catch KeyboardInterrupt when using manual tqdm + # as the interrupt will most likely happen on another statement + self.disp(bar_style='danger') + raise + # NB: don't `finally: close()` + # since this could be a shared bar which the user will `reset()` + + def close(self): + if self.disable: + return + super(tqdm_notebook, self).close() + # Try to detect if there was an error or KeyboardInterrupt + # in manual mode: if n < total, things probably got wrong + if self.total and self.n < self.total: + self.disp(bar_style='danger', check_delay=False) + else: + if self.leave: + self.disp(bar_style='success', check_delay=False) + else: + self.disp(close=True, check_delay=False) + + def clear(self, *_, **__): + pass + + def reset(self, total=None): + """ + Resets to 0 iterations for repeated use. + + Consider combining with `leave=True`. + + Parameters + ---------- + total : int or float, optional. Total to use for the new bar. + """ + if self.disable: + return super(tqdm_notebook, self).reset(total=total) + _, pbar, _ = self.container.children + pbar.bar_style = '' + if total is not None: + pbar.max = total + if not self.total and self.ncols is None: # no longer unknown total + pbar.layout.width = None # reset width + return super(tqdm_notebook, self).reset(total=total) + + +def tnrange(*args, **kwargs): + """ + A shortcut for `tqdm.notebook.tqdm(xrange(*args), **kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + return tqdm_notebook(_range(*args), **kwargs) + + +# Aliases +tqdm = tqdm_notebook +trange = tnrange diff --git a/tqdm/rich.py b/tqdm/rich.py new file mode 100644 index 0000000..69893ff --- /dev/null +++ b/tqdm/rich.py @@ -0,0 +1,156 @@ +""" +`rich.progress` decorator for iterators. + +Usage: +>>> from tqdm.rich import trange, tqdm +>>> for i in trange(10): +... ... +""" +from __future__ import absolute_import + +from warnings import warn + +from rich.progress import ( + BarColumn, Progress, ProgressColumn, Text, TimeElapsedColumn, TimeRemainingColumn, filesize) + +from .std import TqdmExperimentalWarning +from .std import tqdm as std_tqdm +from .utils import _range + +__author__ = {"github.com/": ["casperdcl"]} +__all__ = ['tqdm_rich', 'trrange', 'tqdm', 'trange'] + + +class FractionColumn(ProgressColumn): + """Renders completed/total, e.g. '0.5/2.3 G'.""" + def __init__(self, unit_scale=False, unit_divisor=1000): + self.unit_scale = unit_scale + self.unit_divisor = unit_divisor + super().__init__() + + def render(self, task): + """Calculate common unit for completed and total.""" + completed = int(task.completed) + total = int(task.total) + if self.unit_scale: + unit, suffix = filesize.pick_unit_and_suffix( + total, + ["", "K", "M", "G", "T", "P", "E", "Z", "Y"], + self.unit_divisor, + ) + else: + unit, suffix = filesize.pick_unit_and_suffix(total, [""], 1) + precision = 0 if unit == 1 else 1 + return Text( + f"{completed/unit:,.{precision}f}/{total/unit:,.{precision}f} {suffix}", + style="progress.download") + + +class RateColumn(ProgressColumn): + """Renders human readable transfer speed.""" + def __init__(self, unit="", unit_scale=False, unit_divisor=1000): + self.unit = unit + self.unit_scale = unit_scale + self.unit_divisor = unit_divisor + super().__init__() + + def render(self, task): + """Show data transfer speed.""" + speed = task.speed + if speed is None: + return Text(f"? {self.unit}/s", style="progress.data.speed") + if self.unit_scale: + unit, suffix = filesize.pick_unit_and_suffix( + speed, + ["", "K", "M", "G", "T", "P", "E", "Z", "Y"], + self.unit_divisor, + ) + else: + unit, suffix = filesize.pick_unit_and_suffix(speed, [""], 1) + precision = 0 if unit == 1 else 1 + return Text(f"{speed/unit:,.{precision}f} {suffix}{self.unit}/s", + style="progress.data.speed") + + +class tqdm_rich(std_tqdm): # pragma: no cover + """Experimental rich.progress GUI version of tqdm!""" + # TODO: @classmethod: write()? + def __init__(self, *args, **kwargs): + """ + This class accepts the following parameters *in addition* to + the parameters accepted by `tqdm`. + + Parameters + ---------- + progress : tuple, optional + arguments for `rich.progress.Progress()`. + options : dict, optional + keyword arguments for `rich.progress.Progress()`. + """ + kwargs = kwargs.copy() + kwargs['gui'] = True + # convert disable = None to False + kwargs['disable'] = bool(kwargs.get('disable', False)) + progress = kwargs.pop('progress', None) + options = kwargs.pop('options', {}).copy() + super(tqdm_rich, self).__init__(*args, **kwargs) + + if self.disable: + return + + warn("rich is experimental/alpha", TqdmExperimentalWarning, stacklevel=2) + d = self.format_dict + if progress is None: + progress = ( + "[progress.description]{task.description}" + "[progress.percentage]{task.percentage:>4.0f}%", + BarColumn(bar_width=None), + FractionColumn( + unit_scale=d['unit_scale'], unit_divisor=d['unit_divisor']), + "[", TimeElapsedColumn(), "<", TimeRemainingColumn(), + ",", RateColumn(unit=d['unit'], unit_scale=d['unit_scale'], + unit_divisor=d['unit_divisor']), "]" + ) + options.setdefault('transient', not self.leave) + self._prog = Progress(*progress, **options) + self._prog.__enter__() + self._task_id = self._prog.add_task(self.desc or "", **d) + + def close(self): + if self.disable: + return + super(tqdm_rich, self).close() + self._prog.__exit__(None, None, None) + + def clear(self, *_, **__): + pass + + def display(self, *_, **__): + if not hasattr(self, '_prog'): + return + self._prog.update(self._task_id, completed=self.n, description=self.desc) + + def reset(self, total=None): + """ + Resets to 0 iterations for repeated use. + + Parameters + ---------- + total : int or float, optional. Total to use for the new bar. + """ + if hasattr(self, '_prog'): + self._prog.reset(total=total) + super(tqdm_rich, self).reset(total=total) + + +def trrange(*args, **kwargs): + """ + A shortcut for `tqdm.rich.tqdm(xrange(*args), **kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + return tqdm_rich(_range(*args), **kwargs) + + +# Aliases +tqdm = tqdm_rich +trange = trrange diff --git a/tqdm/std.py b/tqdm/std.py new file mode 100644 index 0000000..5f9dcca --- /dev/null +++ b/tqdm/std.py @@ -0,0 +1,1541 @@ +""" +Customisable progressbar decorator for iterators. +Includes a default `range` iterator printing to `stderr`. + +Usage: +>>> from tqdm import trange, tqdm +>>> for i in trange(10): +... ... +""" +from __future__ import absolute_import, division + +import sys +from collections import OrderedDict, defaultdict +from contextlib import contextmanager +from datetime import datetime, timedelta +from numbers import Number +from time import time +from warnings import warn +from weakref import WeakSet + +from ._monitor import TMonitor +from .utils import ( + CallbackIOWrapper, Comparable, DisableOnWriteError, FormatReplace, SimpleTextIOWrapper, + _basestring, _is_ascii, _range, _screen_shape_wrapper, _supports_unicode, _term_move_up, + _unich, _unicode, disp_len, disp_trim) + +__author__ = "https://github.com/tqdm/tqdm#contributions" +__all__ = ['tqdm', 'trange', + 'TqdmTypeError', 'TqdmKeyError', 'TqdmWarning', + 'TqdmExperimentalWarning', 'TqdmDeprecationWarning', + 'TqdmMonitorWarning'] + + +class TqdmTypeError(TypeError): + pass + + +class TqdmKeyError(KeyError): + pass + + +class TqdmWarning(Warning): + """base class for all tqdm warnings. + + Used for non-external-code-breaking errors, such as garbled printing. + """ + def __init__(self, msg, fp_write=None, *a, **k): + if fp_write is not None: + fp_write("\n" + self.__class__.__name__ + ": " + str(msg).rstrip() + '\n') + else: + super(TqdmWarning, self).__init__(msg, *a, **k) + + +class TqdmExperimentalWarning(TqdmWarning, FutureWarning): + """beta feature, unstable API and behaviour""" + pass + + +class TqdmDeprecationWarning(TqdmWarning, DeprecationWarning): + # not suppressed if raised + pass + + +class TqdmMonitorWarning(TqdmWarning, RuntimeWarning): + """tqdm monitor errors which do not affect external functionality""" + pass + + +def TRLock(*args, **kwargs): + """threading RLock""" + try: + from threading import RLock + return RLock(*args, **kwargs) + except (ImportError, OSError): # pragma: no cover + pass + + +class TqdmDefaultWriteLock(object): + """ + Provide a default write lock for thread and multiprocessing safety. + Works only on platforms supporting `fork` (so Windows is excluded). + You must initialise a `tqdm` or `TqdmDefaultWriteLock` instance + before forking in order for the write lock to work. + On Windows, you need to supply the lock from the parent to the children as + an argument to joblib or the parallelism lib you use. + """ + # global thread lock so no setup required for multithreading. + # NB: Do not create multiprocessing lock as it sets the multiprocessing + # context, disallowing `spawn()`/`forkserver()` + th_lock = TRLock() + + def __init__(self): + # Create global parallelism locks to avoid racing issues with parallel + # bars works only if fork available (Linux/MacOSX, but not Windows) + cls = type(self) + root_lock = cls.th_lock + if root_lock is not None: + root_lock.acquire() + cls.create_mp_lock() + self.locks = [lk for lk in [cls.mp_lock, cls.th_lock] if lk is not None] + if root_lock is not None: + root_lock.release() + + def acquire(self, *a, **k): + for lock in self.locks: + lock.acquire(*a, **k) + + def release(self): + for lock in self.locks[::-1]: # Release in inverse order of acquisition + lock.release() + + def __enter__(self): + self.acquire() + + def __exit__(self, *exc): + self.release() + + @classmethod + def create_mp_lock(cls): + if not hasattr(cls, 'mp_lock'): + try: + from multiprocessing import RLock + cls.mp_lock = RLock() + except (ImportError, OSError): # pragma: no cover + cls.mp_lock = None + + @classmethod + def create_th_lock(cls): + assert hasattr(cls, 'th_lock') + warn("create_th_lock not needed anymore", TqdmDeprecationWarning, stacklevel=2) + + +class Bar(object): + """ + `str.format`-able bar with format specifiers: `[width][type]` + + - `width` + + unspecified (default): use `self.default_len` + + `int >= 0`: overrides `self.default_len` + + `int < 0`: subtract from `self.default_len` + - `type` + + `a`: ascii (`charset=self.ASCII` override) + + `u`: unicode (`charset=self.UTF` override) + + `b`: blank (`charset=" "` override) + """ + ASCII = " 123456789#" + UTF = u" " + u''.join(map(_unich, range(0x258F, 0x2587, -1))) + BLANK = " " + COLOUR_RESET = '\x1b[0m' + COLOUR_RGB = '\x1b[38;2;%d;%d;%dm' + COLOURS = {'BLACK': '\x1b[30m', 'RED': '\x1b[31m', 'GREEN': '\x1b[32m', + 'YELLOW': '\x1b[33m', 'BLUE': '\x1b[34m', 'MAGENTA': '\x1b[35m', + 'CYAN': '\x1b[36m', 'WHITE': '\x1b[37m'} + + def __init__(self, frac, default_len=10, charset=UTF, colour=None): + if not 0 <= frac <= 1: + warn("clamping frac to range [0, 1]", TqdmWarning, stacklevel=2) + frac = max(0, min(1, frac)) + assert default_len > 0 + self.frac = frac + self.default_len = default_len + self.charset = charset + self.colour = colour + + @property + def colour(self): + return self._colour + + @colour.setter + def colour(self, value): + if not value: + self._colour = None + return + try: + if value.upper() in self.COLOURS: + self._colour = self.COLOURS[value.upper()] + elif value[0] == '#' and len(value) == 7: + self._colour = self.COLOUR_RGB % tuple( + int(i, 16) for i in (value[1:3], value[3:5], value[5:7])) + else: + raise KeyError + except (KeyError, AttributeError): + warn("Unknown colour (%s); valid choices: [hex (#00ff00), %s]" % ( + value, ", ".join(self.COLOURS)), + TqdmWarning, stacklevel=2) + self._colour = None + + def __format__(self, format_spec): + if format_spec: + _type = format_spec[-1].lower() + try: + charset = {'a': self.ASCII, 'u': self.UTF, 'b': self.BLANK}[_type] + except KeyError: + charset = self.charset + else: + format_spec = format_spec[:-1] + if format_spec: + N_BARS = int(format_spec) + if N_BARS < 0: + N_BARS += self.default_len + else: + N_BARS = self.default_len + else: + charset = self.charset + N_BARS = self.default_len + + nsyms = len(charset) - 1 + bar_length, frac_bar_length = divmod(int(self.frac * N_BARS * nsyms), nsyms) + + res = charset[-1] * bar_length + if bar_length < N_BARS: # whitespace padding + res = res + charset[frac_bar_length] + charset[0] * (N_BARS - bar_length - 1) + return self.colour + res + self.COLOUR_RESET if self.colour else res + + +class EMA(object): + """ + Exponential moving average: smoothing to give progressively lower + weights to older values. + + Parameters + ---------- + smoothing : float, optional + Smoothing factor in range [0, 1], [default: 0.3]. + Increase to give more weight to recent values. + Ranges from 0 (yields old value) to 1 (yields new value). + """ + def __init__(self, smoothing=0.3): + self.alpha = smoothing + self.last = 0 + self.calls = 0 + + def __call__(self, x=None): + """ + Parameters + ---------- + x : float + New value to include in EMA. + """ + beta = 1 - self.alpha + if x is not None: + self.last = self.alpha * x + beta * self.last + self.calls += 1 + return self.last / (1 - beta ** self.calls) if self.calls else self.last + + +class tqdm(Comparable): + """ + Decorate an iterable object, returning an iterator which acts exactly + like the original iterable, but prints a dynamically updating + progressbar every time a value is requested. + """ + + monitor_interval = 10 # set to 0 to disable the thread + monitor = None + _instances = WeakSet() + + @staticmethod + def format_sizeof(num, suffix='', divisor=1000): + """ + Formats a number (greater than unity) with SI Order of Magnitude + prefixes. + + Parameters + ---------- + num : float + Number ( >= 1) to format. + suffix : str, optional + Post-postfix [default: '']. + divisor : float, optional + Divisor between prefixes [default: 1000]. + + Returns + ------- + out : str + Number with Order of Magnitude SI unit postfix. + """ + for unit in ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z']: + if abs(num) < 999.5: + if abs(num) < 99.95: + if abs(num) < 9.995: + return '{0:1.2f}'.format(num) + unit + suffix + return '{0:2.1f}'.format(num) + unit + suffix + return '{0:3.0f}'.format(num) + unit + suffix + num /= divisor + return '{0:3.1f}Y'.format(num) + suffix + + @staticmethod + def format_interval(t): + """ + Formats a number of seconds as a clock time, [H:]MM:SS + + Parameters + ---------- + t : int + Number of seconds. + + Returns + ------- + out : str + [H:]MM:SS + """ + mins, s = divmod(int(t), 60) + h, m = divmod(mins, 60) + if h: + return '{0:d}:{1:02d}:{2:02d}'.format(h, m, s) + else: + return '{0:02d}:{1:02d}'.format(m, s) + + @staticmethod + def format_num(n): + """ + Intelligent scientific notation (.3g). + + Parameters + ---------- + n : int or float or Numeric + A Number. + + Returns + ------- + out : str + Formatted number. + """ + f = '{0:.3g}'.format(n).replace('+0', '+').replace('-0', '-') + n = str(n) + return f if len(f) < len(n) else n + + @staticmethod + def status_printer(file): + """ + Manage the printing and in-place updating of a line of characters. + Note that if the string is longer than a line, then in-place + updating may not work (it will print a new line at each refresh). + """ + fp = file + fp_flush = getattr(fp, 'flush', lambda: None) # pragma: no cover + if fp in (sys.stderr, sys.stdout): + getattr(sys.stderr, 'flush', lambda: None)() + getattr(sys.stdout, 'flush', lambda: None)() + + def fp_write(s): + fp.write(_unicode(s)) + fp_flush() + + last_len = [0] + + def print_status(s): + len_s = disp_len(s) + fp_write('\r' + s + (' ' * max(last_len[0] - len_s, 0))) + last_len[0] = len_s + + return print_status + + @staticmethod + def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, unit='it', + unit_scale=False, rate=None, bar_format=None, postfix=None, + unit_divisor=1000, initial=0, colour=None, **extra_kwargs): + """ + Return a string-based progress bar given some parameters + + Parameters + ---------- + n : int or float + Number of finished iterations. + total : int or float + The expected total number of iterations. If meaningless (None), + only basic progress statistics are displayed (no ETA). + elapsed : float + Number of seconds passed since start. + ncols : int, optional + The width of the entire output message. If specified, + dynamically resizes `{bar}` to stay within this bound + [default: None]. If `0`, will not print any bar (only stats). + The fallback is `{bar:10}`. + prefix : str, optional + Prefix message (included in total width) [default: '']. + Use as {desc} in bar_format string. + ascii : bool, optional or str, optional + If not set, use unicode (smooth blocks) to fill the meter + [default: False]. The fallback is to use ASCII characters + " 123456789#". + unit : str, optional + The iteration unit [default: 'it']. + unit_scale : bool or int or float, optional + If 1 or True, the number of iterations will be printed with an + appropriate SI metric prefix (k = 10^3, M = 10^6, etc.) + [default: False]. If any other non-zero number, will scale + `total` and `n`. + rate : float, optional + Manual override for iteration rate. + If [default: None], uses n/elapsed. + bar_format : str, optional + Specify a custom bar string formatting. May impact performance. + [default: '{l_bar}{bar}{r_bar}'], where + l_bar='{desc}: {percentage:3.0f}%|' and + r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, ' + '{rate_fmt}{postfix}]' + Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, + percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, + rate, rate_fmt, rate_noinv, rate_noinv_fmt, + rate_inv, rate_inv_fmt, postfix, unit_divisor, + remaining, remaining_s, eta. + Note that a trailing ": " is automatically removed after {desc} + if the latter is empty. + postfix : *, optional + Similar to `prefix`, but placed at the end + (e.g. for additional stats). + Note: postfix is usually a string (not a dict) for this method, + and will if possible be set to postfix = ', ' + postfix. + However other types are supported (#382). + unit_divisor : float, optional + [default: 1000], ignored unless `unit_scale` is True. + initial : int or float, optional + The initial counter value [default: 0]. + colour : str, optional + Bar colour (e.g. 'green', '#00ff00'). + + Returns + ------- + out : Formatted meter and stats, ready to display. + """ + + # sanity check: total + if total and n >= (total + 0.5): # allow float imprecision (#849) + total = None + + # apply custom scale if necessary + if unit_scale and unit_scale not in (True, 1): + if total: + total *= unit_scale + n *= unit_scale + if rate: + rate *= unit_scale # by default rate = self.avg_dn / self.avg_dt + unit_scale = False + + elapsed_str = tqdm.format_interval(elapsed) + + # if unspecified, attempt to use rate = average speed + # (we allow manual override since predicting time is an arcane art) + if rate is None and elapsed: + rate = (n - initial) / elapsed + inv_rate = 1 / rate if rate else None + format_sizeof = tqdm.format_sizeof + rate_noinv_fmt = ((format_sizeof(rate) if unit_scale else + '{0:5.2f}'.format(rate)) if rate else '?') + unit + '/s' + rate_inv_fmt = ( + (format_sizeof(inv_rate) if unit_scale else '{0:5.2f}'.format(inv_rate)) + if inv_rate else '?') + 's/' + unit + rate_fmt = rate_inv_fmt if inv_rate and inv_rate > 1 else rate_noinv_fmt + + if unit_scale: + n_fmt = format_sizeof(n, divisor=unit_divisor) + total_fmt = format_sizeof(total, divisor=unit_divisor) if total is not None else '?' + else: + n_fmt = str(n) + total_fmt = str(total) if total is not None else '?' + + try: + postfix = ', ' + postfix if postfix else '' + except TypeError: + pass + + remaining = (total - n) / rate if rate and total else 0 + remaining_str = tqdm.format_interval(remaining) if rate else '?' + try: + eta_dt = (datetime.now() + timedelta(seconds=remaining) + if rate and total else datetime.utcfromtimestamp(0)) + except OverflowError: + eta_dt = datetime.max + + # format the stats displayed to the left and right sides of the bar + if prefix: + # old prefix setup work around + bool_prefix_colon_already = (prefix[-2:] == ": ") + l_bar = prefix if bool_prefix_colon_already else prefix + ": " + else: + l_bar = '' + + r_bar = '| {0}/{1} [{2}<{3}, {4}{5}]'.format( + n_fmt, total_fmt, elapsed_str, remaining_str, rate_fmt, postfix) + + # Custom bar formatting + # Populate a dict with all available progress indicators + format_dict = dict( + # slight extension of self.format_dict + n=n, n_fmt=n_fmt, total=total, total_fmt=total_fmt, + elapsed=elapsed_str, elapsed_s=elapsed, + ncols=ncols, desc=prefix or '', unit=unit, + rate=inv_rate if inv_rate and inv_rate > 1 else rate, + rate_fmt=rate_fmt, rate_noinv=rate, + rate_noinv_fmt=rate_noinv_fmt, rate_inv=inv_rate, + rate_inv_fmt=rate_inv_fmt, + postfix=postfix, unit_divisor=unit_divisor, + colour=colour, + # plus more useful definitions + remaining=remaining_str, remaining_s=remaining, + l_bar=l_bar, r_bar=r_bar, eta=eta_dt, + **extra_kwargs) + + # total is known: we can predict some stats + if total: + # fractional and percentage progress + frac = n / total + percentage = frac * 100 + + l_bar += '{0:3.0f}%|'.format(percentage) + + if ncols == 0: + return l_bar[:-1] + r_bar[1:] + + format_dict.update(l_bar=l_bar) + if bar_format: + format_dict.update(percentage=percentage) + + # auto-remove colon for empty `desc` + if not prefix: + bar_format = bar_format.replace("{desc}: ", '') + else: + bar_format = "{l_bar}{bar}{r_bar}" + + full_bar = FormatReplace() + try: + nobar = bar_format.format(bar=full_bar, **format_dict) + except UnicodeEncodeError: + bar_format = _unicode(bar_format) + nobar = bar_format.format(bar=full_bar, **format_dict) + if not full_bar.format_called: + # no {bar}, we can just format and return + return nobar + + # Formatting progress bar space available for bar's display + full_bar = Bar(frac, + max(1, ncols - disp_len(nobar)) if ncols else 10, + charset=Bar.ASCII if ascii is True else ascii or Bar.UTF, + colour=colour) + if not _is_ascii(full_bar.charset) and _is_ascii(bar_format): + bar_format = _unicode(bar_format) + res = bar_format.format(bar=full_bar, **format_dict) + return disp_trim(res, ncols) if ncols else res + + elif bar_format: + # user-specified bar_format but no total + l_bar += '|' + format_dict.update(l_bar=l_bar, percentage=0) + full_bar = FormatReplace() + nobar = bar_format.format(bar=full_bar, **format_dict) + if not full_bar.format_called: + return nobar + full_bar = Bar(0, + max(1, ncols - disp_len(nobar)) if ncols else 10, + charset=Bar.BLANK, colour=colour) + res = bar_format.format(bar=full_bar, **format_dict) + return disp_trim(res, ncols) if ncols else res + else: + # no total: no progressbar, ETA, just progress stats + return '{0}{1}{2} [{3}, {4}{5}]'.format( + (prefix + ": ") if prefix else '', n_fmt, unit, elapsed_str, rate_fmt, postfix) + + def __new__(cls, *_, **__): + instance = object.__new__(cls) + with cls.get_lock(): # also constructs lock if non-existent + cls._instances.add(instance) + # create monitoring thread + if cls.monitor_interval and (cls.monitor is None + or not cls.monitor.report()): + try: + cls.monitor = TMonitor(cls, cls.monitor_interval) + except Exception as e: # pragma: nocover + warn("tqdm:disabling monitor support" + " (monitor_interval = 0) due to:\n" + str(e), + TqdmMonitorWarning, stacklevel=2) + cls.monitor_interval = 0 + return instance + + @classmethod + def _get_free_pos(cls, instance=None): + """Skips specified instance.""" + positions = {abs(inst.pos) for inst in cls._instances + if inst is not instance and hasattr(inst, "pos")} + return min(set(range(len(positions) + 1)).difference(positions)) + + @classmethod + def _decr_instances(cls, instance): + """ + Remove from list and reposition another unfixed bar + to fill the new gap. + + This means that by default (where all nested bars are unfixed), + order is not maintained but screen flicker/blank space is minimised. + (tqdm<=4.44.1 moved ALL subsequent unfixed bars up.) + """ + with cls._lock: + try: + cls._instances.remove(instance) + except KeyError: + # if not instance.gui: # pragma: no cover + # raise + pass # py2: maybe magically removed already + # else: + if not instance.gui: + last = (instance.nrows or 20) - 1 + # find unfixed (`pos >= 0`) overflow (`pos >= nrows - 1`) + instances = list(filter( + lambda i: hasattr(i, "pos") and last <= i.pos, + cls._instances)) + # set first found to current `pos` + if instances: + inst = min(instances, key=lambda i: i.pos) + inst.clear(nolock=True) + inst.pos = abs(instance.pos) + + @classmethod + def write(cls, s, file=None, end="\n", nolock=False): + """Print a message via tqdm (without overlap with bars).""" + fp = file if file is not None else sys.stdout + with cls.external_write_mode(file=file, nolock=nolock): + # Write the message + fp.write(s) + fp.write(end) + + @classmethod + @contextmanager + def external_write_mode(cls, file=None, nolock=False): + """ + Disable tqdm within context and refresh tqdm when exits. + Useful when writing to standard output stream + """ + fp = file if file is not None else sys.stdout + + try: + if not nolock: + cls.get_lock().acquire() + # Clear all bars + inst_cleared = [] + for inst in getattr(cls, '_instances', []): + # Clear instance if in the target output file + # or if write output + tqdm output are both either + # sys.stdout or sys.stderr (because both are mixed in terminal) + if hasattr(inst, "start_t") and (inst.fp == fp or all( + f in (sys.stdout, sys.stderr) for f in (fp, inst.fp))): + inst.clear(nolock=True) + inst_cleared.append(inst) + yield + # Force refresh display of bars we cleared + for inst in inst_cleared: + inst.refresh(nolock=True) + finally: + if not nolock: + cls._lock.release() + + @classmethod + def set_lock(cls, lock): + """Set the global lock.""" + cls._lock = lock + + @classmethod + def get_lock(cls): + """Get the global lock. Construct it if it does not exist.""" + if not hasattr(cls, '_lock'): + cls._lock = TqdmDefaultWriteLock() + return cls._lock + + @classmethod + def pandas(cls, **tqdm_kwargs): + """ + Registers the current `tqdm` class with + pandas.core. + ( frame.DataFrame + | series.Series + | groupby.(generic.)DataFrameGroupBy + | groupby.(generic.)SeriesGroupBy + ).progress_apply + + A new instance will be created every time `progress_apply` is called, + and each instance will automatically `close()` upon completion. + + Parameters + ---------- + tqdm_kwargs : arguments for the tqdm instance + + Examples + -------- + >>> import pandas as pd + >>> import numpy as np + >>> from tqdm import tqdm + >>> from tqdm.gui import tqdm as tqdm_gui + >>> + >>> df = pd.DataFrame(np.random.randint(0, 100, (100000, 6))) + >>> tqdm.pandas(ncols=50) # can use tqdm_gui, optional kwargs, etc + >>> # Now you can use `progress_apply` instead of `apply` + >>> df.groupby(0).progress_apply(lambda x: x**2) + + References + ---------- + <https://stackoverflow.com/questions/18603270/\ + progress-indicator-during-pandas-operations-python> + """ + from warnings import catch_warnings, simplefilter + + from pandas.core.frame import DataFrame + from pandas.core.series import Series + try: + with catch_warnings(): + simplefilter("ignore", category=FutureWarning) + from pandas import Panel + except ImportError: # pandas>=1.2.0 + Panel = None + Rolling, Expanding = None, None + try: # pandas>=1.0.0 + from pandas.core.window.rolling import _Rolling_and_Expanding + except ImportError: + try: # pandas>=0.18.0 + from pandas.core.window import _Rolling_and_Expanding + except ImportError: # pandas>=1.2.0 + try: # pandas>=1.2.0 + from pandas.core.window.expanding import Expanding + from pandas.core.window.rolling import Rolling + _Rolling_and_Expanding = Rolling, Expanding + except ImportError: # pragma: no cover + _Rolling_and_Expanding = None + try: # pandas>=0.25.0 + from pandas.core.groupby.generic import SeriesGroupBy # , NDFrameGroupBy + from pandas.core.groupby.generic import DataFrameGroupBy + except ImportError: # pragma: no cover + try: # pandas>=0.23.0 + from pandas.core.groupby.groupby import DataFrameGroupBy, SeriesGroupBy + except ImportError: + from pandas.core.groupby import DataFrameGroupBy, SeriesGroupBy + try: # pandas>=0.23.0 + from pandas.core.groupby.groupby import GroupBy + except ImportError: # pragma: no cover + from pandas.core.groupby import GroupBy + + try: # pandas>=0.23.0 + from pandas.core.groupby.groupby import PanelGroupBy + except ImportError: + try: + from pandas.core.groupby import PanelGroupBy + except ImportError: # pandas>=0.25.0 + PanelGroupBy = None + + tqdm_kwargs = tqdm_kwargs.copy() + deprecated_t = [tqdm_kwargs.pop('deprecated_t', None)] + + def inner_generator(df_function='apply'): + def inner(df, func, *args, **kwargs): + """ + Parameters + ---------- + df : (DataFrame|Series)[GroupBy] + Data (may be grouped). + func : function + To be applied on the (grouped) data. + **kwargs : optional + Transmitted to `df.apply()`. + """ + + # Precompute total iterations + total = tqdm_kwargs.pop("total", getattr(df, 'ngroups', None)) + if total is None: # not grouped + if df_function == 'applymap': + total = df.size + elif isinstance(df, Series): + total = len(df) + elif (_Rolling_and_Expanding is None or + not isinstance(df, _Rolling_and_Expanding)): + # DataFrame or Panel + axis = kwargs.get('axis', 0) + if axis == 'index': + axis = 0 + elif axis == 'columns': + axis = 1 + # when axis=0, total is shape[axis1] + total = df.size // df.shape[axis] + + # Init bar + if deprecated_t[0] is not None: + t = deprecated_t[0] + deprecated_t[0] = None + else: + t = cls(total=total, **tqdm_kwargs) + + if len(args) > 0: + # *args intentionally not supported (see #244, #299) + TqdmDeprecationWarning( + "Except func, normal arguments are intentionally" + + " not supported by" + + " `(DataFrame|Series|GroupBy).progress_apply`." + + " Use keyword arguments instead.", + fp_write=getattr(t.fp, 'write', sys.stderr.write)) + + try: # pandas>=1.3.0 + from pandas.core.common import is_builtin_func + except ImportError: + is_builtin_func = df._is_builtin_func + try: + func = is_builtin_func(func) + except TypeError: + pass + + # Define bar updating wrapper + def wrapper(*args, **kwargs): + # update tbar correctly + # it seems `pandas apply` calls `func` twice + # on the first column/row to decide whether it can + # take a fast or slow code path; so stop when t.total==t.n + t.update(n=1 if not t.total or t.n < t.total else 0) + return func(*args, **kwargs) + + # Apply the provided function (in **kwargs) + # on the df using our wrapper (which provides bar updating) + try: + return getattr(df, df_function)(wrapper, **kwargs) + finally: + t.close() + + return inner + + # Monkeypatch pandas to provide easy methods + # Enable custom tqdm progress in pandas! + Series.progress_apply = inner_generator() + SeriesGroupBy.progress_apply = inner_generator() + Series.progress_map = inner_generator('map') + SeriesGroupBy.progress_map = inner_generator('map') + + DataFrame.progress_apply = inner_generator() + DataFrameGroupBy.progress_apply = inner_generator() + DataFrame.progress_applymap = inner_generator('applymap') + + if Panel is not None: + Panel.progress_apply = inner_generator() + if PanelGroupBy is not None: + PanelGroupBy.progress_apply = inner_generator() + + GroupBy.progress_apply = inner_generator() + GroupBy.progress_aggregate = inner_generator('aggregate') + GroupBy.progress_transform = inner_generator('transform') + + if Rolling is not None and Expanding is not None: + Rolling.progress_apply = inner_generator() + Expanding.progress_apply = inner_generator() + elif _Rolling_and_Expanding is not None: + _Rolling_and_Expanding.progress_apply = inner_generator() + + def __init__(self, iterable=None, desc=None, total=None, leave=True, file=None, + ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None, + ascii=None, disable=False, unit='it', unit_scale=False, + dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0, + position=None, postfix=None, unit_divisor=1000, write_bytes=None, + lock_args=None, nrows=None, colour=None, delay=0, gui=False, + **kwargs): + """ + Parameters + ---------- + iterable : iterable, optional + Iterable to decorate with a progressbar. + Leave blank to manually manage the updates. + desc : str, optional + Prefix for the progressbar. + total : int or float, optional + The number of expected iterations. If unspecified, + len(iterable) is used if possible. If float("inf") or as a last + resort, only basic progress statistics are displayed + (no ETA, no progressbar). + If `gui` is True and this parameter needs subsequent updating, + specify an initial arbitrary large positive number, + e.g. 9e9. + leave : bool, optional + If [default: True], keeps all traces of the progressbar + upon termination of iteration. + If `None`, will leave only if `position` is `0`. + file : `io.TextIOWrapper` or `io.StringIO`, optional + Specifies where to output the progress messages + (default: sys.stderr). Uses `file.write(str)` and `file.flush()` + methods. For encoding, see `write_bytes`. + ncols : int, optional + The width of the entire output message. If specified, + dynamically resizes the progressbar to stay within this bound. + If unspecified, attempts to use environment width. The + fallback is a meter width of 10 and no limit for the counter and + statistics. If 0, will not print any meter (only stats). + mininterval : float, optional + Minimum progress display update interval [default: 0.1] seconds. + maxinterval : float, optional + Maximum progress display update interval [default: 10] seconds. + Automatically adjusts `miniters` to correspond to `mininterval` + after long display update lag. Only works if `dynamic_miniters` + or monitor thread is enabled. + miniters : int or float, optional + Minimum progress display update interval, in iterations. + If 0 and `dynamic_miniters`, will automatically adjust to equal + `mininterval` (more CPU efficient, good for tight loops). + If > 0, will skip display of specified number of iterations. + Tweak this and `mininterval` to get very efficient loops. + If your progress is erratic with both fast and slow iterations + (network, skipping items, etc) you should set miniters=1. + ascii : bool or str, optional + If unspecified or False, use unicode (smooth blocks) to fill + the meter. The fallback is to use ASCII characters " 123456789#". + disable : bool, optional + Whether to disable the entire progressbar wrapper + [default: False]. If set to None, disable on non-TTY. + unit : str, optional + String that will be used to define the unit of each iteration + [default: it]. + unit_scale : bool or int or float, optional + If 1 or True, the number of iterations will be reduced/scaled + automatically and a metric prefix following the + International System of Units standard will be added + (kilo, mega, etc.) [default: False]. If any other non-zero + number, will scale `total` and `n`. + dynamic_ncols : bool, optional + If set, constantly alters `ncols` and `nrows` to the + environment (allowing for window resizes) [default: False]. + smoothing : float, optional + Exponential moving average smoothing factor for speed estimates + (ignored in GUI mode). Ranges from 0 (average speed) to 1 + (current/instantaneous speed) [default: 0.3]. + bar_format : str, optional + Specify a custom bar string formatting. May impact performance. + [default: '{l_bar}{bar}{r_bar}'], where + l_bar='{desc}: {percentage:3.0f}%|' and + r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, ' + '{rate_fmt}{postfix}]' + Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, + percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, + rate, rate_fmt, rate_noinv, rate_noinv_fmt, + rate_inv, rate_inv_fmt, postfix, unit_divisor, + remaining, remaining_s, eta. + Note that a trailing ": " is automatically removed after {desc} + if the latter is empty. + initial : int or float, optional + The initial counter value. Useful when restarting a progress + bar [default: 0]. If using float, consider specifying `{n:.3f}` + or similar in `bar_format`, or specifying `unit_scale`. + position : int, optional + Specify the line offset to print this bar (starting from 0) + Automatic if unspecified. + Useful to manage multiple bars at once (eg, from threads). + postfix : dict or *, optional + Specify additional stats to display at the end of the bar. + Calls `set_postfix(**postfix)` if possible (dict). + unit_divisor : float, optional + [default: 1000], ignored unless `unit_scale` is True. + write_bytes : bool, optional + If (default: None) and `file` is unspecified, + bytes will be written in Python 2. If `True` will also write + bytes. In all other cases will default to unicode. + lock_args : tuple, optional + Passed to `refresh` for intermediate output + (initialisation, iterating, and updating). + nrows : int, optional + The screen height. If specified, hides nested bars outside this + bound. If unspecified, attempts to use environment height. + The fallback is 20. + colour : str, optional + Bar colour (e.g. 'green', '#00ff00'). + delay : float, optional + Don't display until [default: 0] seconds have elapsed. + gui : bool, optional + WARNING: internal parameter - do not use. + Use tqdm.gui.tqdm(...) instead. If set, will attempt to use + matplotlib animations for a graphical output [default: False]. + + Returns + ------- + out : decorated iterator. + """ + if write_bytes is None: + write_bytes = file is None and sys.version_info < (3,) + + if file is None: + file = sys.stderr + + if write_bytes: + # Despite coercing unicode into bytes, py2 sys.std* streams + # should have bytes written to them. + file = SimpleTextIOWrapper( + file, encoding=getattr(file, 'encoding', None) or 'utf-8') + + file = DisableOnWriteError(file, tqdm_instance=self) + + if disable is None and hasattr(file, "isatty") and not file.isatty(): + disable = True + + if total is None and iterable is not None: + try: + total = len(iterable) + except (TypeError, AttributeError): + total = None + if total == float("inf"): + # Infinite iterations, behave same as unknown + total = None + + if disable: + self.iterable = iterable + self.disable = disable + with self._lock: + self.pos = self._get_free_pos(self) + self._instances.remove(self) + self.n = initial + self.total = total + self.leave = leave + return + + if kwargs: + self.disable = True + with self._lock: + self.pos = self._get_free_pos(self) + self._instances.remove(self) + raise ( + TqdmDeprecationWarning( + "`nested` is deprecated and automated.\n" + "Use `position` instead for manual control.\n", + fp_write=getattr(file, 'write', sys.stderr.write)) + if "nested" in kwargs else + TqdmKeyError("Unknown argument(s): " + str(kwargs))) + + # Preprocess the arguments + if ( + (ncols is None or nrows is None) and (file in (sys.stderr, sys.stdout)) + ) or dynamic_ncols: # pragma: no cover + if dynamic_ncols: + dynamic_ncols = _screen_shape_wrapper() + if dynamic_ncols: + ncols, nrows = dynamic_ncols(file) + else: + _dynamic_ncols = _screen_shape_wrapper() + if _dynamic_ncols: + _ncols, _nrows = _dynamic_ncols(file) + if ncols is None: + ncols = _ncols + if nrows is None: + nrows = _nrows + + if miniters is None: + miniters = 0 + dynamic_miniters = True + else: + dynamic_miniters = False + + if mininterval is None: + mininterval = 0 + + if maxinterval is None: + maxinterval = 0 + + if ascii is None: + ascii = not _supports_unicode(file) + + if bar_format and ascii is not True and not _is_ascii(ascii): + # Convert bar format into unicode since terminal uses unicode + bar_format = _unicode(bar_format) + + if smoothing is None: + smoothing = 0 + + # Store the arguments + self.iterable = iterable + self.desc = desc or '' + self.total = total + self.leave = leave + self.fp = file + self.ncols = ncols + self.nrows = nrows + self.mininterval = mininterval + self.maxinterval = maxinterval + self.miniters = miniters + self.dynamic_miniters = dynamic_miniters + self.ascii = ascii + self.disable = disable + self.unit = unit + self.unit_scale = unit_scale + self.unit_divisor = unit_divisor + self.initial = initial + self.lock_args = lock_args + self.delay = delay + self.gui = gui + self.dynamic_ncols = dynamic_ncols + self.smoothing = smoothing + self._ema_dn = EMA(smoothing) + self._ema_dt = EMA(smoothing) + self._ema_miniters = EMA(smoothing) + self.bar_format = bar_format + self.postfix = None + self.colour = colour + self._time = time + if postfix: + try: + self.set_postfix(refresh=False, **postfix) + except TypeError: + self.postfix = postfix + + # Init the iterations counters + self.last_print_n = initial + self.n = initial + + # if nested, at initial sp() call we replace '\r' by '\n' to + # not overwrite the outer progress bar + with self._lock: + # mark fixed positions as negative + self.pos = self._get_free_pos(self) if position is None else -position + + if not gui: + # Initialize the screen printer + self.sp = self.status_printer(self.fp) + if delay <= 0: + self.refresh(lock_args=self.lock_args) + + # Init the time counter + self.last_print_t = self._time() + # NB: Avoid race conditions by setting start_t at the very end of init + self.start_t = self.last_print_t + + def __bool__(self): + if self.total is not None: + return self.total > 0 + if self.iterable is None: + raise TypeError('bool() undefined when iterable == total == None') + return bool(self.iterable) + + def __nonzero__(self): + return self.__bool__() + + def __len__(self): + return ( + self.total if self.iterable is None + else self.iterable.shape[0] if hasattr(self.iterable, "shape") + else len(self.iterable) if hasattr(self.iterable, "__len__") + else self.iterable.__length_hint__() if hasattr(self.iterable, "__length_hint__") + else getattr(self, "total", None)) + + def __reversed__(self): + try: + orig = self.iterable + except AttributeError: + raise TypeError("'tqdm' object is not reversible") + else: + self.iterable = reversed(self.iterable) + return self.__iter__() + finally: + self.iterable = orig + + def __contains__(self, item): + contains = getattr(self.iterable, '__contains__', None) + return contains(item) if contains is not None else item in self.__iter__() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + try: + self.close() + except AttributeError: + # maybe eager thread cleanup upon external error + if (exc_type, exc_value, traceback) == (None, None, None): + raise + warn("AttributeError ignored", TqdmWarning, stacklevel=2) + + def __del__(self): + self.close() + + def __str__(self): + return self.format_meter(**self.format_dict) + + @property + def _comparable(self): + return abs(getattr(self, "pos", 1 << 31)) + + def __hash__(self): + return id(self) + + def __iter__(self): + """Backward-compatibility to use: for x in tqdm(iterable)""" + + # Inlining instance variables as locals (speed optimisation) + iterable = self.iterable + + # If the bar is disabled, then just walk the iterable + # (note: keep this check outside the loop for performance) + if self.disable: + for obj in iterable: + yield obj + return + + mininterval = self.mininterval + last_print_t = self.last_print_t + last_print_n = self.last_print_n + min_start_t = self.start_t + self.delay + n = self.n + time = self._time + + try: + for obj in iterable: + yield obj + # Update and possibly print the progressbar. + # Note: does not call self.update(1) for speed optimisation. + n += 1 + + if n - last_print_n >= self.miniters: + cur_t = time() + dt = cur_t - last_print_t + if dt >= mininterval and cur_t >= min_start_t: + self.update(n - last_print_n) + last_print_n = self.last_print_n + last_print_t = self.last_print_t + finally: + self.n = n + self.close() + + def update(self, n=1): + """ + Manually update the progress bar, useful for streams + such as reading files. + E.g.: + >>> t = tqdm(total=filesize) # Initialise + >>> for current_buffer in stream: + ... ... + ... t.update(len(current_buffer)) + >>> t.close() + The last line is highly recommended, but possibly not necessary if + `t.update()` will be called in such a way that `filesize` will be + exactly reached and printed. + + Parameters + ---------- + n : int or float, optional + Increment to add to the internal counter of iterations + [default: 1]. If using float, consider specifying `{n:.3f}` + or similar in `bar_format`, or specifying `unit_scale`. + + Returns + ------- + out : bool or None + True if a `display()` was triggered. + """ + if self.disable: + return + + if n < 0: + self.last_print_n += n # for auto-refresh logic to work + self.n += n + + # check counter first to reduce calls to time() + if self.n - self.last_print_n >= self.miniters: + cur_t = self._time() + dt = cur_t - self.last_print_t + if dt >= self.mininterval and cur_t >= self.start_t + self.delay: + cur_t = self._time() + dn = self.n - self.last_print_n # >= n + if self.smoothing and dt and dn: + # EMA (not just overall average) + self._ema_dn(dn) + self._ema_dt(dt) + self.refresh(lock_args=self.lock_args) + if self.dynamic_miniters: + # If no `miniters` was specified, adjust automatically to the + # maximum iteration rate seen so far between two prints. + # e.g.: After running `tqdm.update(5)`, subsequent + # calls to `tqdm.update()` will only cause an update after + # at least 5 more iterations. + if self.maxinterval and dt >= self.maxinterval: + self.miniters = dn * (self.mininterval or self.maxinterval) / dt + elif self.smoothing: + # EMA miniters update + self.miniters = self._ema_miniters( + dn * (self.mininterval / dt if self.mininterval and dt + else 1)) + else: + # max iters between two prints + self.miniters = max(self.miniters, dn) + + # Store old values for next call + self.last_print_n = self.n + self.last_print_t = cur_t + return True + + def close(self): + """Cleanup and (if leave=False) close the progressbar.""" + if self.disable: + return + + # Prevent multiple closures + self.disable = True + + # decrement instance pos and remove from internal set + pos = abs(self.pos) + self._decr_instances(self) + + if self.last_print_t < self.start_t + self.delay: + # haven't ever displayed; nothing to clear + return + + # GUI mode + if getattr(self, 'sp', None) is None: + return + + # annoyingly, _supports_unicode isn't good enough + def fp_write(s): + self.fp.write(_unicode(s)) + + try: + fp_write('') + except ValueError as e: + if 'closed' in str(e): + return + raise # pragma: no cover + + leave = pos == 0 if self.leave is None else self.leave + + with self._lock: + if leave: + # stats for overall rate (no weighted average) + self._ema_dt = lambda: None + self.display(pos=0) + fp_write('\n') + else: + # clear previous display + if self.display(msg='', pos=pos) and not pos: + fp_write('\r') + + def clear(self, nolock=False): + """Clear current bar display.""" + if self.disable: + return + + if not nolock: + self._lock.acquire() + pos = abs(self.pos) + if pos < (self.nrows or 20): + self.moveto(pos) + self.sp('') + self.fp.write('\r') # place cursor back at the beginning of line + self.moveto(-pos) + if not nolock: + self._lock.release() + + def refresh(self, nolock=False, lock_args=None): + """ + Force refresh the display of this bar. + + Parameters + ---------- + nolock : bool, optional + If `True`, does not lock. + If [default: `False`]: calls `acquire()` on internal lock. + lock_args : tuple, optional + Passed to internal lock's `acquire()`. + If specified, will only `display()` if `acquire()` returns `True`. + """ + if self.disable: + return + + if not nolock: + if lock_args: + if not self._lock.acquire(*lock_args): + return False + else: + self._lock.acquire() + self.display() + if not nolock: + self._lock.release() + return True + + def unpause(self): + """Restart tqdm timer from last print time.""" + if self.disable: + return + cur_t = self._time() + self.start_t += cur_t - self.last_print_t + self.last_print_t = cur_t + + def reset(self, total=None): + """ + Resets to 0 iterations for repeated use. + + Consider combining with `leave=True`. + + Parameters + ---------- + total : int or float, optional. Total to use for the new bar. + """ + self.n = 0 + if total is not None: + self.total = total + if self.disable: + return + self.last_print_n = 0 + self.last_print_t = self.start_t = self._time() + self._ema_dn = EMA(self.smoothing) + self._ema_dt = EMA(self.smoothing) + self._ema_miniters = EMA(self.smoothing) + self.refresh() + + def set_description(self, desc=None, refresh=True): + """ + Set/modify description of the progress bar. + + Parameters + ---------- + desc : str, optional + refresh : bool, optional + Forces refresh [default: True]. + """ + self.desc = desc + ': ' if desc else '' + if refresh: + self.refresh() + + def set_description_str(self, desc=None, refresh=True): + """Set/modify description without ': ' appended.""" + self.desc = desc or '' + if refresh: + self.refresh() + + def set_postfix(self, ordered_dict=None, refresh=True, **kwargs): + """ + Set/modify postfix (additional stats) + with automatic formatting based on datatype. + + Parameters + ---------- + ordered_dict : dict or OrderedDict, optional + refresh : bool, optional + Forces refresh [default: True]. + kwargs : dict, optional + """ + # Sort in alphabetical order to be more deterministic + postfix = OrderedDict([] if ordered_dict is None else ordered_dict) + for key in sorted(kwargs.keys()): + postfix[key] = kwargs[key] + # Preprocess stats according to datatype + for key in postfix.keys(): + # Number: limit the length of the string + if isinstance(postfix[key], Number): + postfix[key] = self.format_num(postfix[key]) + # Else for any other type, try to get the string conversion + elif not isinstance(postfix[key], _basestring): + postfix[key] = str(postfix[key]) + # Else if it's a string, don't need to preprocess anything + # Stitch together to get the final postfix + self.postfix = ', '.join(key + '=' + postfix[key].strip() + for key in postfix.keys()) + if refresh: + self.refresh() + + def set_postfix_str(self, s='', refresh=True): + """ + Postfix without dictionary expansion, similar to prefix handling. + """ + self.postfix = str(s) + if refresh: + self.refresh() + + def moveto(self, n): + # TODO: private method + self.fp.write(_unicode('\n' * n + _term_move_up() * -n)) + getattr(self.fp, 'flush', lambda: None)() + + @property + def format_dict(self): + """Public API for read-only member access.""" + if self.disable and not hasattr(self, 'unit'): + return defaultdict(lambda: None, { + 'n': self.n, 'total': self.total, 'elapsed': 0, 'unit': 'it'}) + if self.dynamic_ncols: + self.ncols, self.nrows = self.dynamic_ncols(self.fp) + return { + 'n': self.n, 'total': self.total, + 'elapsed': self._time() - self.start_t if hasattr(self, 'start_t') else 0, + 'ncols': self.ncols, 'nrows': self.nrows, 'prefix': self.desc, + 'ascii': self.ascii, 'unit': self.unit, 'unit_scale': self.unit_scale, + 'rate': self._ema_dn() / self._ema_dt() if self._ema_dt() else None, + 'bar_format': self.bar_format, 'postfix': self.postfix, + 'unit_divisor': self.unit_divisor, 'initial': self.initial, + 'colour': self.colour} + + def display(self, msg=None, pos=None): + """ + Use `self.sp` to display `msg` in the specified `pos`. + + Consider overloading this function when inheriting to use e.g.: + `self.some_frontend(**self.format_dict)` instead of `self.sp`. + + Parameters + ---------- + msg : str, optional. What to display (default: `repr(self)`). + pos : int, optional. Position to `moveto` + (default: `abs(self.pos)`). + """ + if pos is None: + pos = abs(self.pos) + + nrows = self.nrows or 20 + if pos >= nrows - 1: + if pos >= nrows: + return False + if msg or msg is None: # override at `nrows - 1` + msg = " ... (more hidden) ..." + + if not hasattr(self, "sp"): + raise TqdmDeprecationWarning( + "Please use `tqdm.gui.tqdm(...)`" + " instead of `tqdm(..., gui=True)`\n", + fp_write=getattr(self.fp, 'write', sys.stderr.write)) + + if pos: + self.moveto(pos) + self.sp(self.__str__() if msg is None else msg) + if pos: + self.moveto(-pos) + return True + + @classmethod + @contextmanager + def wrapattr(cls, stream, method, total=None, bytes=True, **tqdm_kwargs): + """ + stream : file-like object. + method : str, "read" or "write". The result of `read()` and + the first argument of `write()` should have a `len()`. + + >>> with tqdm.wrapattr(file_obj, "read", total=file_obj.size) as fobj: + ... while True: + ... chunk = fobj.read(chunk_size) + ... if not chunk: + ... break + """ + with cls(total=total, **tqdm_kwargs) as t: + if bytes: + t.unit = "B" + t.unit_scale = True + t.unit_divisor = 1024 + yield CallbackIOWrapper(t.update, stream, method) + + +def trange(*args, **kwargs): + """ + A shortcut for tqdm(xrange(*args), **kwargs). + On Python3+ range is used instead of xrange. + """ + return tqdm(_range(*args), **kwargs) diff --git a/tqdm/tk.py b/tqdm/tk.py new file mode 100644 index 0000000..92adb51 --- /dev/null +++ b/tqdm/tk.py @@ -0,0 +1,207 @@ +""" +Tkinter GUI progressbar decorator for iterators. + +Usage: +>>> from tqdm.tk import trange, tqdm +>>> for i in trange(10): +... ... +""" +from __future__ import absolute_import, division + +import re +import sys +from warnings import warn + +try: + import tkinter + import tkinter.ttk as ttk +except ImportError: + import Tkinter as tkinter + import ttk as ttk + +from .std import TqdmExperimentalWarning, TqdmWarning +from .std import tqdm as std_tqdm +from .utils import _range + +__author__ = {"github.com/": ["richardsheridan", "casperdcl"]} +__all__ = ['tqdm_tk', 'ttkrange', 'tqdm', 'trange'] + + +class tqdm_tk(std_tqdm): # pragma: no cover + """ + Experimental Tkinter GUI version of tqdm! + + Note: Window interactivity suffers if `tqdm_tk` is not running within + a Tkinter mainloop and values are generated infrequently. In this case, + consider calling `tqdm_tk.refresh()` frequently in the Tk thread. + """ + + # TODO: @classmethod: write()? + + def __init__(self, *args, **kwargs): + """ + This class accepts the following parameters *in addition* to + the parameters accepted by `tqdm`. + + Parameters + ---------- + grab : bool, optional + Grab the input across all windows of the process. + tk_parent : `tkinter.Wm`, optional + Parent Tk window. + cancel_callback : Callable, optional + Create a cancel button and set `cancel_callback` to be called + when the cancel or window close button is clicked. + """ + kwargs = kwargs.copy() + kwargs['gui'] = True + # convert disable = None to False + kwargs['disable'] = bool(kwargs.get('disable', False)) + self._warn_leave = 'leave' in kwargs + grab = kwargs.pop('grab', False) + tk_parent = kwargs.pop('tk_parent', None) + self._cancel_callback = kwargs.pop('cancel_callback', None) + super(tqdm_tk, self).__init__(*args, **kwargs) + + if self.disable: + return + + if tk_parent is None: # Discover parent widget + try: + tk_parent = tkinter._default_root + except AttributeError: + raise AttributeError( + "`tk_parent` required when using `tkinter.NoDefaultRoot()`") + if tk_parent is None: # use new default root window as display + self._tk_window = tkinter.Tk() + else: # some other windows already exist + self._tk_window = tkinter.Toplevel() + else: + self._tk_window = tkinter.Toplevel(tk_parent) + + warn("GUI is experimental/alpha", TqdmExperimentalWarning, stacklevel=2) + self._tk_dispatching = self._tk_dispatching_helper() + + self._tk_window.protocol("WM_DELETE_WINDOW", self.cancel) + self._tk_window.wm_title(self.desc) + self._tk_window.wm_attributes("-topmost", 1) + self._tk_window.after(0, lambda: self._tk_window.wm_attributes("-topmost", 0)) + self._tk_n_var = tkinter.DoubleVar(self._tk_window, value=0) + self._tk_text_var = tkinter.StringVar(self._tk_window) + pbar_frame = ttk.Frame(self._tk_window, padding=5) + pbar_frame.pack() + _tk_label = ttk.Label(pbar_frame, textvariable=self._tk_text_var, + wraplength=600, anchor="center", justify="center") + _tk_label.pack() + self._tk_pbar = ttk.Progressbar( + pbar_frame, variable=self._tk_n_var, length=450) + if self.total is not None: + self._tk_pbar.configure(maximum=self.total) + else: + self._tk_pbar.configure(mode="indeterminate") + self._tk_pbar.pack() + if self._cancel_callback is not None: + _tk_button = ttk.Button(pbar_frame, text="Cancel", command=self.cancel) + _tk_button.pack() + if grab: + self._tk_window.grab_set() + + def close(self): + if self.disable: + return + + self.disable = True + + with self.get_lock(): + self._instances.remove(self) + + def _close(): + self._tk_window.after('idle', self._tk_window.destroy) + if not self._tk_dispatching: + self._tk_window.update() + + self._tk_window.protocol("WM_DELETE_WINDOW", _close) + + # if leave is set but we are self-dispatching, the left window is + # totally unresponsive unless the user manually dispatches + if not self.leave: + _close() + elif not self._tk_dispatching: + if self._warn_leave: + warn("leave flag ignored if not in tkinter mainloop", + TqdmWarning, stacklevel=2) + _close() + + def clear(self, *_, **__): + pass + + def display(self, *_, **__): + self._tk_n_var.set(self.n) + d = self.format_dict + # remove {bar} + d['bar_format'] = (d['bar_format'] or "{l_bar}<bar/>{r_bar}").replace( + "{bar}", "<bar/>") + msg = self.format_meter(**d) + if '<bar/>' in msg: + msg = "".join(re.split(r'\|?<bar/>\|?', msg, 1)) + self._tk_text_var.set(msg) + if not self._tk_dispatching: + self._tk_window.update() + + def set_description(self, desc=None, refresh=True): + self.set_description_str(desc, refresh) + + def set_description_str(self, desc=None, refresh=True): + self.desc = desc + if not self.disable: + self._tk_window.wm_title(desc) + if refresh and not self._tk_dispatching: + self._tk_window.update() + + def cancel(self): + """ + `cancel_callback()` followed by `close()` + when close/cancel buttons clicked. + """ + if self._cancel_callback is not None: + self._cancel_callback() + self.close() + + def reset(self, total=None): + """ + Resets to 0 iterations for repeated use. + + Parameters + ---------- + total : int or float, optional. Total to use for the new bar. + """ + if hasattr(self, '_tk_pbar'): + if total is None: + self._tk_pbar.configure(maximum=100, mode="indeterminate") + else: + self._tk_pbar.configure(maximum=total, mode="determinate") + super(tqdm_tk, self).reset(total=total) + + @staticmethod + def _tk_dispatching_helper(): + """determine if Tkinter mainloop is dispatching events""" + codes = {tkinter.mainloop.__code__, tkinter.Misc.mainloop.__code__} + for frame in sys._current_frames().values(): + while frame: + if frame.f_code in codes: + return True + frame = frame.f_back + return False + + +def ttkrange(*args, **kwargs): + """ + A shortcut for `tqdm.tk.tqdm(xrange(*args), **kwargs)`. + On Python3+, `range` is used instead of `xrange`. + """ + return tqdm_tk(_range(*args), **kwargs) + + +# Aliases +tqdm = tqdm_tk +trange = ttkrange diff --git a/tqdm/tqdm.1 b/tqdm/tqdm.1 new file mode 100644 index 0000000..0533198 --- /dev/null +++ b/tqdm/tqdm.1 @@ -0,0 +1,316 @@ +.\" Automatically generated by Pandoc 1.19.2 +.\" +.TH "TQDM" "1" "2015\-2021" "tqdm User Manuals" "" +.hy +.SH NAME +.PP +tqdm \- fast, extensible progress bar for Python and CLI +.SH SYNOPSIS +.PP +tqdm [\f[I]options\f[]] +.SH DESCRIPTION +.PP +See <https://github.com/tqdm/tqdm>. +Can be used as a pipe: +.IP +.nf +\f[C] +$\ #\ count\ lines\ of\ code +$\ cat\ *.py\ |\ tqdm\ |\ wc\ \-l +327it\ [00:00,\ 981773.38it/s] +327 + +$\ #\ find\ all\ files +$\ find\ .\ \-name\ "*.py"\ |\ tqdm\ |\ wc\ \-l +432it\ [00:00,\ 833842.30it/s] +432 + +#\ ...\ and\ more\ info +$\ find\ .\ \-name\ \[aq]*.py\[aq]\ \-exec\ wc\ \-l\ \\{}\ \\;\ \\ +\ \ |\ tqdm\ \-\-total\ 432\ \-\-unit\ files\ \-\-desc\ counting\ \\ +\ \ |\ awk\ \[aq]{\ sum\ +=\ $1\ };\ END\ {\ print\ sum\ }\[aq] +counting:\ 100%|█████████|\ 432/432\ [00:00<00:00,\ 794361.83files/s] +131998 +\f[] +.fi +.SH OPTIONS +.TP +.B \-h, \-\-help +Print this help and exit. +.RS +.RE +.TP +.B \-v, \-\-version +Print version and exit. +.RS +.RE +.TP +.B \-\-desc=\f[I]desc\f[] +str, optional. +Prefix for the progressbar. +.RS +.RE +.TP +.B \-\-total=\f[I]total\f[] +int or float, optional. +The number of expected iterations. +If unspecified, len(iterable) is used if possible. +If float("inf") or as a last resort, only basic progress statistics are +displayed (no ETA, no progressbar). +If \f[C]gui\f[] is True and this parameter needs subsequent updating, +specify an initial arbitrary large positive number, e.g. +9e9. +.RS +.RE +.TP +.B \-\-leave +bool, optional. +If [default: True], keeps all traces of the progressbar upon termination +of iteration. +If \f[C]None\f[], will leave only if \f[C]position\f[] is \f[C]0\f[]. +.RS +.RE +.TP +.B \-\-ncols=\f[I]ncols\f[] +int, optional. +The width of the entire output message. +If specified, dynamically resizes the progressbar to stay within this +bound. +If unspecified, attempts to use environment width. +The fallback is a meter width of 10 and no limit for the counter and +statistics. +If 0, will not print any meter (only stats). +.RS +.RE +.TP +.B \-\-mininterval=\f[I]mininterval\f[] +float, optional. +Minimum progress display update interval [default: 0.1] seconds. +.RS +.RE +.TP +.B \-\-maxinterval=\f[I]maxinterval\f[] +float, optional. +Maximum progress display update interval [default: 10] seconds. +Automatically adjusts \f[C]miniters\f[] to correspond to +\f[C]mininterval\f[] after long display update lag. +Only works if \f[C]dynamic_miniters\f[] or monitor thread is enabled. +.RS +.RE +.TP +.B \-\-miniters=\f[I]miniters\f[] +int or float, optional. +Minimum progress display update interval, in iterations. +If 0 and \f[C]dynamic_miniters\f[], will automatically adjust to equal +\f[C]mininterval\f[] (more CPU efficient, good for tight loops). +If > 0, will skip display of specified number of iterations. +Tweak this and \f[C]mininterval\f[] to get very efficient loops. +If your progress is erratic with both fast and slow iterations (network, +skipping items, etc) you should set miniters=1. +.RS +.RE +.TP +.B \-\-ascii=\f[I]ascii\f[] +bool or str, optional. +If unspecified or False, use unicode (smooth blocks) to fill the meter. +The fallback is to use ASCII characters " 123456789#". +.RS +.RE +.TP +.B \-\-disable +bool, optional. +Whether to disable the entire progressbar wrapper [default: False]. +If set to None, disable on non\-TTY. +.RS +.RE +.TP +.B \-\-unit=\f[I]unit\f[] +str, optional. +String that will be used to define the unit of each iteration [default: +it]. +.RS +.RE +.TP +.B \-\-unit\-scale=\f[I]unit_scale\f[] +bool or int or float, optional. +If 1 or True, the number of iterations will be reduced/scaled +automatically and a metric prefix following the International System of +Units standard will be added (kilo, mega, etc.) [default: False]. +If any other non\-zero number, will scale \f[C]total\f[] and \f[C]n\f[]. +.RS +.RE +.TP +.B \-\-dynamic\-ncols +bool, optional. +If set, constantly alters \f[C]ncols\f[] and \f[C]nrows\f[] to the +environment (allowing for window resizes) [default: False]. +.RS +.RE +.TP +.B \-\-smoothing=\f[I]smoothing\f[] +float, optional. +Exponential moving average smoothing factor for speed estimates (ignored +in GUI mode). +Ranges from 0 (average speed) to 1 (current/instantaneous speed) +[default: 0.3]. +.RS +.RE +.TP +.B \-\-bar\-format=\f[I]bar_format\f[] +str, optional. +Specify a custom bar string formatting. +May impact performance. +[default: \[aq]{l_bar}{bar}{r_bar}\[aq]], where l_bar=\[aq]{desc}: +{percentage:3.0f}%|\[aq] and r_bar=\[aq]| {n_fmt}/{total_fmt} +[{elapsed}<{remaining}, \[aq] \[aq]{rate_fmt}{postfix}]\[aq] Possible +vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, percentage, +elapsed, elapsed_s, ncols, nrows, desc, unit, rate, rate_fmt, +rate_noinv, rate_noinv_fmt, rate_inv, rate_inv_fmt, postfix, +unit_divisor, remaining, remaining_s, eta. +Note that a trailing ": " is automatically removed after {desc} if the +latter is empty. +.RS +.RE +.TP +.B \-\-initial=\f[I]initial\f[] +int or float, optional. +The initial counter value. +Useful when restarting a progress bar [default: 0]. +If using float, consider specifying \f[C]{n:.3f}\f[] or similar in +\f[C]bar_format\f[], or specifying \f[C]unit_scale\f[]. +.RS +.RE +.TP +.B \-\-position=\f[I]position\f[] +int, optional. +Specify the line offset to print this bar (starting from 0) Automatic if +unspecified. +Useful to manage multiple bars at once (eg, from threads). +.RS +.RE +.TP +.B \-\-postfix=\f[I]postfix\f[] +dict or *, optional. +Specify additional stats to display at the end of the bar. +Calls \f[C]set_postfix(**postfix)\f[] if possible (dict). +.RS +.RE +.TP +.B \-\-unit\-divisor=\f[I]unit_divisor\f[] +float, optional. +[default: 1000], ignored unless \f[C]unit_scale\f[] is True. +.RS +.RE +.TP +.B \-\-write\-bytes +bool, optional. +If (default: None) and \f[C]file\f[] is unspecified, bytes will be +written in Python 2. +If \f[C]True\f[] will also write bytes. +In all other cases will default to unicode. +.RS +.RE +.TP +.B \-\-lock\-args=\f[I]lock_args\f[] +tuple, optional. +Passed to \f[C]refresh\f[] for intermediate output (initialisation, +iterating, and updating). +.RS +.RE +.TP +.B \-\-nrows=\f[I]nrows\f[] +int, optional. +The screen height. +If specified, hides nested bars outside this bound. +If unspecified, attempts to use environment height. +The fallback is 20. +.RS +.RE +.TP +.B \-\-colour=\f[I]colour\f[] +str, optional. +Bar colour (e.g. +\[aq]green\[aq], \[aq]#00ff00\[aq]). +.RS +.RE +.TP +.B \-\-delay=\f[I]delay\f[] +float, optional. +Don\[aq]t display until [default: 0] seconds have elapsed. +.RS +.RE +.TP +.B \-\-delim=\f[I]delim\f[] +chr, optional. +Delimiting character [default: \[aq]\\n\[aq]]. +Use \[aq]\\0\[aq] for null. +N.B.: on Windows systems, Python converts \[aq]\\n\[aq] to +\[aq]\\r\\n\[aq]. +.RS +.RE +.TP +.B \-\-buf\-size=\f[I]buf_size\f[] +int, optional. +String buffer size in bytes [default: 256] used when \f[C]delim\f[] is +specified. +.RS +.RE +.TP +.B \-\-bytes +bool, optional. +If true, will count bytes, ignore \f[C]delim\f[], and default +\f[C]unit_scale\f[] to True, \f[C]unit_divisor\f[] to 1024, and +\f[C]unit\f[] to \[aq]B\[aq]. +.RS +.RE +.TP +.B \-\-tee +bool, optional. +If true, passes \f[C]stdin\f[] to both \f[C]stderr\f[] and +\f[C]stdout\f[]. +.RS +.RE +.TP +.B \-\-update +bool, optional. +If true, will treat input as newly elapsed iterations, i.e. +numbers to pass to \f[C]update()\f[]. +Note that this is slow (~2e5 it/s) since every input must be decoded as +a number. +.RS +.RE +.TP +.B \-\-update\-to +bool, optional. +If true, will treat input as total elapsed iterations, i.e. +numbers to assign to \f[C]self.n\f[]. +Note that this is slow (~2e5 it/s) since every input must be decoded as +a number. +.RS +.RE +.TP +.B \-\-null +bool, optional. +If true, will discard input (no stdout). +.RS +.RE +.TP +.B \-\-manpath=\f[I]manpath\f[] +str, optional. +Directory in which to install tqdm man pages. +.RS +.RE +.TP +.B \-\-comppath=\f[I]comppath\f[] +str, optional. +Directory in which to place tqdm completion. +.RS +.RE +.TP +.B \-\-log=\f[I]log\f[] +str, optional. +CRITICAL|FATAL|ERROR|WARN(ING)|[default: \[aq]INFO\[aq]]|DEBUG|NOTSET. +.RS +.RE +.SH AUTHORS +tqdm developers <https://github.com/tqdm>. diff --git a/tqdm/utils.py b/tqdm/utils.py new file mode 100644 index 0000000..0632b8d --- /dev/null +++ b/tqdm/utils.py @@ -0,0 +1,354 @@ +""" +General helpers required for `tqdm.std`. +""" +import os +import re +import sys +from functools import wraps +from warnings import warn +from weakref import proxy + +# py2/3 compat +try: + _range = xrange +except NameError: + _range = range + +try: + _unich = unichr +except NameError: + _unich = chr + +try: + _unicode = unicode +except NameError: + _unicode = str + +try: + _basestring = basestring +except NameError: + _basestring = str + +CUR_OS = sys.platform +IS_WIN = any(CUR_OS.startswith(i) for i in ['win32', 'cygwin']) +IS_NIX = any(CUR_OS.startswith(i) for i in ['aix', 'linux', 'darwin']) +RE_ANSI = re.compile(r"\x1b\[[;\d]*[A-Za-z]") + +try: + if IS_WIN: + import colorama + else: + raise ImportError +except ImportError: + colorama = None +else: + try: + colorama.init(strip=False) + except TypeError: + colorama.init() + + +class FormatReplace(object): + """ + >>> a = FormatReplace('something') + >>> "{:5d}".format(a) + 'something' + """ # NOQA: P102 + def __init__(self, replace=''): + self.replace = replace + self.format_called = 0 + + def __format__(self, _): + self.format_called += 1 + return self.replace + + +class Comparable(object): + """Assumes child has self._comparable attr/@property""" + def __lt__(self, other): + return self._comparable < other._comparable + + def __le__(self, other): + return (self < other) or (self == other) + + def __eq__(self, other): + return self._comparable == other._comparable + + def __ne__(self, other): + return not self == other + + def __gt__(self, other): + return not self <= other + + def __ge__(self, other): + return not self < other + + +class ObjectWrapper(object): + def __getattr__(self, name): + return getattr(self._wrapped, name) + + def __setattr__(self, name, value): + return setattr(self._wrapped, name, value) + + def wrapper_getattr(self, name): + """Actual `self.getattr` rather than self._wrapped.getattr""" + try: + return object.__getattr__(self, name) + except AttributeError: # py2 + return getattr(self, name) + + def wrapper_setattr(self, name, value): + """Actual `self.setattr` rather than self._wrapped.setattr""" + return object.__setattr__(self, name, value) + + def __init__(self, wrapped): + """ + Thin wrapper around a given object + """ + self.wrapper_setattr('_wrapped', wrapped) + + +class SimpleTextIOWrapper(ObjectWrapper): + """ + Change only `.write()` of the wrapped object by encoding the passed + value and passing the result to the wrapped object's `.write()` method. + """ + # pylint: disable=too-few-public-methods + def __init__(self, wrapped, encoding): + super(SimpleTextIOWrapper, self).__init__(wrapped) + self.wrapper_setattr('encoding', encoding) + + def write(self, s): + """ + Encode `s` and pass to the wrapped object's `.write()` method. + """ + return self._wrapped.write(s.encode(self.wrapper_getattr('encoding'))) + + def __eq__(self, other): + return self._wrapped == getattr(other, '_wrapped', other) + + +class DisableOnWriteError(ObjectWrapper): + """ + Disable the given `tqdm_instance` upon `write()` or `flush()` errors. + """ + @staticmethod + def disable_on_exception(tqdm_instance, func): + """ + Quietly set `tqdm_instance.miniters=inf` if `func` raises `errno=5`. + """ + tqdm_instance = proxy(tqdm_instance) + + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + except OSError as e: + if e.errno != 5: + raise + try: + tqdm_instance.miniters = float('inf') + except ReferenceError: + pass + except ValueError as e: + if 'closed' not in str(e): + raise + try: + tqdm_instance.miniters = float('inf') + except ReferenceError: + pass + return inner + + def __init__(self, wrapped, tqdm_instance): + super(DisableOnWriteError, self).__init__(wrapped) + if hasattr(wrapped, 'write'): + self.wrapper_setattr( + 'write', self.disable_on_exception(tqdm_instance, wrapped.write)) + if hasattr(wrapped, 'flush'): + self.wrapper_setattr( + 'flush', self.disable_on_exception(tqdm_instance, wrapped.flush)) + + def __eq__(self, other): + return self._wrapped == getattr(other, '_wrapped', other) + + +class CallbackIOWrapper(ObjectWrapper): + def __init__(self, callback, stream, method="read"): + """ + Wrap a given `file`-like object's `read()` or `write()` to report + lengths to the given `callback` + """ + super(CallbackIOWrapper, self).__init__(stream) + func = getattr(stream, method) + if method == "write": + @wraps(func) + def write(data, *args, **kwargs): + res = func(data, *args, **kwargs) + callback(len(data)) + return res + self.wrapper_setattr('write', write) + elif method == "read": + @wraps(func) + def read(*args, **kwargs): + data = func(*args, **kwargs) + callback(len(data)) + return data + self.wrapper_setattr('read', read) + else: + raise KeyError("Can only wrap read/write methods") + + +def _is_utf(encoding): + try: + u'\u2588\u2589'.encode(encoding) + except UnicodeEncodeError: + return False + except Exception: + try: + return encoding.lower().startswith('utf-') or ('U8' == encoding) + except Exception: + return False + else: + return True + + +def _supports_unicode(fp): + try: + return _is_utf(fp.encoding) + except AttributeError: + return False + + +def _is_ascii(s): + if isinstance(s, str): + for c in s: + if ord(c) > 255: + return False + return True + return _supports_unicode(s) + + +def _screen_shape_wrapper(): # pragma: no cover + """ + Return a function which returns console dimensions (width, height). + Supported: linux, osx, windows, cygwin. + """ + _screen_shape = None + if IS_WIN: + _screen_shape = _screen_shape_windows + if _screen_shape is None: + _screen_shape = _screen_shape_tput + if IS_NIX: + _screen_shape = _screen_shape_linux + return _screen_shape + + +def _screen_shape_windows(fp): # pragma: no cover + try: + import struct + from ctypes import create_string_buffer, windll + from sys import stdin, stdout + + io_handle = -12 # assume stderr + if fp == stdin: + io_handle = -10 + elif fp == stdout: + io_handle = -11 + + h = windll.kernel32.GetStdHandle(io_handle) + csbi = create_string_buffer(22) + res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) + if res: + (_bufx, _bufy, _curx, _cury, _wattr, left, top, right, bottom, + _maxx, _maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw) + return right - left, bottom - top # +1 + except Exception: # nosec + pass + return None, None + + +def _screen_shape_tput(*_): # pragma: no cover + """cygwin xterm (windows)""" + try: + import shlex + from subprocess import check_call # nosec + return [int(check_call(shlex.split('tput ' + i))) - 1 + for i in ('cols', 'lines')] + except Exception: # nosec + pass + return None, None + + +def _screen_shape_linux(fp): # pragma: no cover + + try: + from array import array + from fcntl import ioctl + from termios import TIOCGWINSZ + except ImportError: + return None, None + else: + try: + rows, cols = array('h', ioctl(fp, TIOCGWINSZ, '\0' * 8))[:2] + return cols, rows + except Exception: + try: + return [int(os.environ[i]) - 1 for i in ("COLUMNS", "LINES")] + except (KeyError, ValueError): + return None, None + + +def _environ_cols_wrapper(): # pragma: no cover + """ + Return a function which returns console width. + Supported: linux, osx, windows, cygwin. + """ + warn("Use `_screen_shape_wrapper()(file)[0]` instead of" + " `_environ_cols_wrapper()(file)`", DeprecationWarning, stacklevel=2) + shape = _screen_shape_wrapper() + if not shape: + return None + + @wraps(shape) + def inner(fp): + return shape(fp)[0] + + return inner + + +def _term_move_up(): # pragma: no cover + return '' if (os.name == 'nt') and (colorama is None) else '\x1b[A' + + +try: + # TODO consider using wcswidth third-party package for 0-width characters + from unicodedata import east_asian_width +except ImportError: + _text_width = len +else: + def _text_width(s): + return sum(2 if east_asian_width(ch) in 'FW' else 1 for ch in _unicode(s)) + + +def disp_len(data): + """ + Returns the real on-screen length of a string which may contain + ANSI control codes and wide chars. + """ + return _text_width(RE_ANSI.sub('', data)) + + +def disp_trim(data, length): + """ + Trim a string which may contain ANSI control characters. + """ + if len(data) == disp_len(data): + return data[:length] + + ansi_present = bool(RE_ANSI.search(data)) + while disp_len(data) > length: # carefully delete one char at a time + data = data[:-1] + if ansi_present and bool(RE_ANSI.search(data)): + # assume ANSI reset is required + return data if data.endswith("\033[0m") else data + "\033[0m" + return data diff --git a/tqdm/version.py b/tqdm/version.py new file mode 100644 index 0000000..11cbaea --- /dev/null +++ b/tqdm/version.py @@ -0,0 +1,9 @@ +"""`tqdm` version detector. Precedence: installed dist, git, 'UNKNOWN'.""" +try: + from ._dist_ver import __version__ +except ImportError: + try: + from setuptools_scm import get_version + __version__ = get_version(root='..', relative_to=__file__) + except (ImportError, LookupError): + __version__ = "UNKNOWN" |
