diff options
51 files changed, 4251 insertions, 0 deletions
diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..248c56c --- /dev/null +++ b/.coveragerc @@ -0,0 +1,5 @@ +[report] +exclude_lines = + pragma: no cover + def __repr__ + if TYPE_CHECKING: @@ -0,0 +1,3 @@ +[flake8] +max-line-length = 88 +extend-ignore = E203,E501 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e50ab93 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,62 @@ +name: CI +on: + pull_request: + push: + branches: [main] + +concurrency: + # prettier-ignore + group: >- + ${{ github.workflow }}- + ${{ github.ref_type }}- + ${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +jobs: + tests: + name: tests / ${{ matrix.os }} / ${{ matrix.python-version }} + runs-on: ${{ matrix.os }}-latest + + strategy: + matrix: + os: [Windows, Ubuntu, MacOS] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + include: + # Only run PyPy jobs, on Ubuntu. + - os: Ubuntu + python-version: pypy-3.7 + + steps: + - uses: actions/checkout@v3 + + # Get Python to test against + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + # Setup pip's cache + - name: Save date (for cache) + id: date + run: echo "::set-output name=date::$(date +%F)" + - name: Save pip cache dir + id: pip-cache-dir + run: echo "::set-output name=dir::$(pip cache dir)" + - name: pip cache + uses: actions/cache@v3 + with: + path: ${{ steps.pip-cache-dir.outputs.dir }} + key: pip-v1-${{ runner.os }}-${{ steps.date.outputs.date }} + restore-keys: pip-v1-${{ runner.os }} + + - run: pip install nox + + # prettier-ignore + - run: > + nox + -s test-${{ matrix.python-version }} + doctest-${{ matrix.python-version }} + --error-on-missing-interpreters + if: matrix.python-version != 'pypy-3.7' + + - run: nox --error-on-missing-interpreters -s test-pypy3 doctest-pypy3 + if: matrix.python-version == 'pypy-3.7' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea86f72 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*.pyc + +.nox/ +/build/ +/dist/ + +.coverage +.*cache diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..9d981a8 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,6 @@ +[settings] +line_length = 88 +known_first_party = installer +default_section = THIRDPARTY +multi_line_output = 3 +include_trailing_comma = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..cc782bd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,54 @@ +repos: + - repo: https://github.com/psf/black + rev: "22.10.0" + hooks: + - id: black + language_version: python3.8 + + - repo: https://github.com/PyCQA/isort + rev: "5.10.1" + hooks: + - id: isort + files: \.py$ + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: "v0.991" + hooks: + - id: mypy + exclude: docs/.*|tests/.*|noxfile.py + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: "v3.0.0-alpha.4" + hooks: + - id: prettier + args: [--prose-wrap, always] + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v4.4.0" + hooks: + - id: check-builtin-literals + - id: check-added-large-files + - id: check-case-conflict + - id: check-toml + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: forbid-new-submodules + - id: trailing-whitespace + + - repo: https://github.com/PyCQA/flake8 + rev: "6.0.0" + hooks: + - id: flake8 + + - repo: https://github.com/PyCQA/pydocstyle.git + rev: "6.1.1" + hooks: + - id: pydocstyle + files: src/.*\.py$ + + - repo: https://github.com/asottile/blacken-docs + rev: "v1.12.1" + hooks: + - id: blacken-docs + additional_dependencies: [black==21.9b0] diff --git a/.pydocstyle b/.pydocstyle new file mode 100644 index 0000000..4e920d3 --- /dev/null +++ b/.pydocstyle @@ -0,0 +1,2 @@ +[pydocstyle] +ignore = D105,D203,D213 diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..ab42605 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,12 @@ +version: 2 + +sphinx: + builder: htmldir + configuration: docs/conf.py + +python: + version: 3.8 + install: + - requirements: docs/requirements.txt + - method: pip + path: . diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..0e08bc3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,112 @@ +# Contributing + +Thank you for your interest in contributing to installer. We welcome all +contributions and greatly appreciate your effort! + +## Code of Conduct + +Everyone interacting in the pip project's codebases, issue trackers, chat rooms, +and mailing lists is expected to follow the [PyPA Code of Conduct][coc]. + +[coc]: https://www.pypa.io/en/latest/code-of-conduct/ + +## Bugs and Feature Requests + +If you have found any bugs or would like to request a new feature, please do +check if there is an existing issue already filed for the same, in the project's +GitHub [issue tracker]. If not, please file a new issue. + +If you want to help out by fixing bugs, choose an open issue in the [issue +tracker] to work on and claim it by posting a comment saying "I would like to +work on this.". Feel free to ask any doubts in the issue thread. + +While working on implementing the feature, please go ahead and file a pull +request. Filing a pull request early allows for getting feedback as early as +possible. + +[issue tracker]: https://github.com/pradyunsg/installer/issues + +## Pull Requests + +Pull Requests should be small to facilitate easier review. Keep them +self-contained, and limited in scope. Studies have shown that review quality +falls off as patch size grows. Sometimes this will result in many small PRs to +land a single large feature. + +Checklist: + +1. All pull requests _must_ be made against the `main` branch. +2. Include tests for any functionality you implement. Any contributions helping + improve existing tests are welcome. +3. Update documentation as necessary and provide documentation for any new + functionality. + +## Development + +[nox] is used to simplify invocation and usage of all the tooling used during +development. + +[nox]: https://github.com/theacodes/nox + +### Code Convention + +This codebase uses the following tools for enforcing a code convention: + +- [black] for code formatting +- [isort] for import sorting +- [mypy] for static type checking +- [pre-commit] for managing all the linters + +To run all the linters: + +```sh-session +$ nox -s lint +``` + +[black]: https://github.com/psf/black +[isort]: https://github.com/timothycrosley/isort +[mypy]: https://github.com/python/mypy +[pre-commit]: https://pre-commit.com/ + +### Testing + +This codebase uses [pytest] as the testing framework and [coverage] for +generating code coverage metrics. We enforce a strict 100% test coverage policy +for all code contributions, although [code coverage isn't everything]. + +To run all the tests: + +```sh-session +$ nox -s test +``` + +nox has been configured to forward any additional arguments it is given to +pytest. This enables the use of [pytest's rich CLI]. + +``` +$ # Using file name +$ nox -s test -- tests/*.py +$ # Using markers +$ nox -s test -- -m unit +$ # Using keywords +$ nox -s test -- -k "basic" +``` + +[pytest]: https://docs.pytest.org/en/stable/ +[coverage]: https://coverage.readthedocs.io/ +[code coverage isn't everything]: + https://bryanpendleton.blogspot.com/2011/02/code-coverage-isnt-everything-but-its.html +[pytest's rich cli]: + https://docs.pytest.org/en/latest/usage.html#specifying-tests-selecting-tests + +### Documentation + +This codebase uses [Sphinx] for generating documentation. + +To build the documentation: + +```sh-session +$ nox -s docs +``` + +[sphinx]: https://www.sphinx-doc.org/ @@ -0,0 +1,19 @@ +Copyright (c) 2020 Pradyun Gedam + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..5702cd9 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +# installer + +<!-- start readme-pitch --> + +This is a low-level library for installing a Python package from a +[wheel distribution](https://packaging.python.org/glossary/#term-Wheel). It +provides basic functionality and abstractions for handling wheels and installing +packages from wheels. + +- Logic for "unpacking" a wheel (i.e. installation). +- Abstractions for various parts of the unpacking process. +- Extensible simple implementations of the abstractions. +- Platform-independent Python script wrapper generation. + +<!-- end readme-pitch --> + +You can read more in the [documentation](https://installer.rtfd.io/). diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 0000000..a3dd063 --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,29 @@ +/*** GENERAL ***/ + +/* Make inline code blocks nicer to look at */ +code.literal { + border-radius: 0.3em; + padding: 0em 0.3em; +} + +div.highlight pre { + border-radius: 0.2em; + padding: 0.75em; + margin: 0 -0.5em; +} + +/*** API REFERENCE ***/ + +/* Space things out properly */ +dl > dd:last-child { + margin-bottom: 10px; +} + +/* Add a tiny dash of color to names of things */ +dt > .property { + color: #a02000; +} +.sig-name, +.sig-prename { + color: #0066bb; +} diff --git a/docs/api/__init__.md b/docs/api/__init__.md new file mode 100644 index 0000000..8a74ad3 --- /dev/null +++ b/docs/api/__init__.md @@ -0,0 +1,9 @@ +```{caution} +This API is not finalised, and may change in a patch version. +``` + +# `installer` + +```{eval-rst} +.. autofunction:: installer.install +``` diff --git a/docs/api/destinations.md b/docs/api/destinations.md new file mode 100644 index 0000000..780392d --- /dev/null +++ b/docs/api/destinations.md @@ -0,0 +1,16 @@ +```{caution} +This API is not finalised, and may change in a patch version. +``` + +# `installer.destinations` + +```{eval-rst} +.. automodule:: installer.destinations + +.. autoclass:: installer.destinations.WheelDestination + :members: + +.. autoclass:: installer.destinations.SchemeDictionaryDestination() + :members: + :special-members: __init__ +``` diff --git a/docs/api/records.md b/docs/api/records.md new file mode 100644 index 0000000..b0201fd --- /dev/null +++ b/docs/api/records.md @@ -0,0 +1,51 @@ +```{caution} +This API is not finalised, and may change in a patch version. +``` + +# `installer.records` + +```{eval-rst} +.. automodule:: installer.records +``` + +## Example + +```{doctest} pycon +>>> from installer.records import parse_record_file, RecordEntry +>>> lines = [ +... "file.py,sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI,3144", +... "distribution-1.0.dist-info/RECORD,,", +... ] +>>> records = parse_record_file(lines) +>>> li = list(records) +>>> len(li) +2 +>>> record_tuple = li[0] +>>> record_tuple +('file.py', 'sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI', '3144') +>>> record = RecordEntry.from_elements(*record_tuple) +>>> record +RecordEntry(path='file.py', hash_=Hash(name='sha256', value='AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI'), size=3144) +>>> record.path +'file.py' +>>> record.hash_ +Hash(name='sha256', value='AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI') +>>> record.size +3144 +>>> record.validate(b"...") +False +``` + +## Reference + +```{eval-rst} +.. autofunction:: installer.records.parse_record_file + +.. autoclass:: installer.records.RecordEntry() + :special-members: __init__ + :members: + +.. autoclass:: installer.records.Hash() + :special-members: __init__ + :members: +``` diff --git a/docs/api/scripts.md b/docs/api/scripts.md new file mode 100644 index 0000000..470d13e --- /dev/null +++ b/docs/api/scripts.md @@ -0,0 +1,19 @@ +```{caution} +This API is not finalised, and may change in a patch version. +``` + +# `installer.scripts` + +Provides the ability to generate executable launcher scripts, that are based on +[`simple_launcher`]. A description of how these scripts work is available in +simple_launcher's README. + +[`simple_launcher`]: https://bitbucket.org/vinay.sajip/simple_launcher/ + +```{eval-rst} +.. autoclass:: installer.scripts.InvalidScript() + +.. autoclass:: installer.scripts.Script() + :special-members: __init__ + :members: +``` diff --git a/docs/api/sources.md b/docs/api/sources.md new file mode 100644 index 0000000..7ee8306 --- /dev/null +++ b/docs/api/sources.md @@ -0,0 +1,10 @@ +```{caution} +This API is not finalised, and may change in a patch version. +``` + +# `installer.sources` + +```{eval-rst} +.. automodule:: installer.sources + :members: +``` diff --git a/docs/api/utils.md b/docs/api/utils.md new file mode 100644 index 0000000..53a4c10 --- /dev/null +++ b/docs/api/utils.md @@ -0,0 +1,10 @@ +```{caution} +This API is not finalised, and may change in a patch version. +``` + +# `installer.utils` + +```{eval-rst} +.. automodule:: installer.utils + :members: +``` diff --git a/docs/changelog.md b/docs/changelog.md new file mode 100644 index 0000000..2f9c494 --- /dev/null +++ b/docs/changelog.md @@ -0,0 +1,68 @@ +# Changelog + +## v0.6.0 (Dec 7, 2022) + +- Add support for Python 3.11 (#154) +- Encode hashes in `RECORD` files correctly (#141) +- Add `py.typed` marker file (#138) +- Implement `--prefix` option (#103) +- Fix the unbound `is_executable` (#115) +- Construct `RECORD` file using `csv.writer` (#118) +- Move away from `import installer.xyz` style imports (#110) +- Improve existing documentation content (typos, formatting) (#109) + +## v0.5.1 (Mar 11, 2022) + +- Change all names in `installer.__main__` to be underscore prefixed. +- Update project URL after move to the `pypa` organisation. +- Rewrite imports to be compatible with `vendoring`. + +## v0.5.0 (Feb 16, 2022) + +- Add a CLI, to install a wheel into the currently-running Python. +- Convert Windows paths to `/` separated when writing `RECORD`. +- Drop support for Python 3.6 and lower. +- Preserve the executable bit from wheels being installed. +- Write records in `RECORD` with relative paths. +- Improve API documentation. + +## v0.4.0 (Oct 13, 2021) + +- Pass schemes into {any}`WheelDestination.finalize_installation`. + +## v0.3.0 (Oct 11, 2021) + +- Add support for ARM 64 executables on Windows. +- Improve handling of wheels that contain entries for directories. + +## v0.2.3 (Jul 29, 2021) + +- Fix entry point handling in {any}`installer.install`. + +## v0.2.2 (May 15, 2021) + +- Teach {any}`SchemeDictionaryDestination` to create subfolders. + +## v0.2.1 (May 15, 2021) + +- Change {any}`parse_record_file` to yield the elements as a tuple, instead of + {any}`RecordEntry` objects. +- Implement {any}`WheelFile`, completing the end-to-end wheel installation + pipeline. +- Generate {any}`RecordEntry` for `RECORD` file in the {any}`installer.install`, + instead of requiring every `WheelDestination` implementation to do the exact + same thing. + +## v0.2.0 (May 3, 2021) + +- Initial release. + +--- + +Thank you to [Dan Ryan] and [Tzu-ping Chung] for the project name on PyPI. The +PyPI releases before 0.2.0 come from <https://github.com/sarugaku/installer> and +have been [yanked]. + +[dan ryan]: https://github.com/techalchemy +[tzu-ping chung]: https://github.com/uranusjr +[yanked]: https://www.python.org/dev/peps/pep-0592/#abstract diff --git a/docs/cli/installer.md b/docs/cli/installer.md new file mode 100644 index 0000000..d0a54e2 --- /dev/null +++ b/docs/cli/installer.md @@ -0,0 +1,9 @@ +# `python -m installer` + +This interface allows you to install a specific wheel into a Python interpreter. + +```{argparse} +:module: installer.__main__ +:func: _get_main_parser +:prog: python -m installer +``` diff --git a/docs/concepts.md b/docs/concepts.md new file mode 100644 index 0000000..e4b926e --- /dev/null +++ b/docs/concepts.md @@ -0,0 +1,38 @@ +# Concepts + +This library has two main abstractions: + +- {any}`WheelSource`: Serves as source of information about a wheel file. +- {any}`WheelDestination`: Handles all file writing and post-installation + processing. + +## WheelSource + +These objects represent a wheel file, abstracting away how the actual file is +stored or accessed. + +This allows the core install logic to be used with in-memory wheel files, or +unzipped-on-disk wheel, or with {any}`zipfile.ZipFile` objects from an on-disk +wheel, or something else entirely. + +This protocol/abstraction is designed to be implementable without a direct +dependency on this library. This allows for other libraries in the Python +packaging ecosystem to provide implementations of the protocol, allowing for +more code reuse opportunities. + +One of the benefits of this fully described interface is the possibility to +decouple the implementation of additional validation on wheels (such as +validating the RECORD entries in a wheel match the actual contents of the wheel, +or enforcing signing requirements) based on what the specific usecase demands. + +## WheelDestination + +These objects are responsible for handling the writing-to-filesystem +interactions, determining RECORD file entries and post-install actions (like +generating .pyc files). While this is a lot of responsibility, this was +explicitly provided to make it possible for custom `WheelDestination` +implementations to be more powerful and flexible. + +Most of these tasks can either be delegated to utilities provided in this +library (eg: script generation), or to the Python standard libary (eg: +generating `.pyc` files). diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..22bbe11 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,58 @@ +"""A sphinx documentation configuration file. +""" + +# -- Project information --------------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "installer" + +copyright = "2020, Pradyun Gedam" +author = "Pradyun Gedam" + +# -- General configuration ------------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "myst_parser", + "sphinxarg.ext", +] + +# -- Options for HTML output ----------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "furo" +html_title = project + +# -- Options for Autodoc -------------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration + +autodoc_member_order = "bysource" +autodoc_preserve_defaults = True + +# Keep the type hints outside the function signature, moving them to the +# descriptions of the relevant function/methods. +autodoc_typehints = "description" + +# Don't show the class signature with the class name. +autodoc_class_signature = "separated" + +# -- Options for intersphinx ---------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration + +intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "pypug": ("https://packaging.python.org", None), +} + +# -- Options for Markdown files -------------------------------------------------------- +# https://myst-parser.readthedocs.io/en/latest/sphinx/reference.html + +myst_enable_extensions = [ + "colon_fence", + "deflist", +] +myst_heading_anchors = 3 diff --git a/docs/development/design.md b/docs/development/design.md new file mode 100644 index 0000000..cb67cc3 --- /dev/null +++ b/docs/development/design.md @@ -0,0 +1,22 @@ +# Design and Scope + +## What this is for + +This project is born out of [this discussion][1]. Effectively, the volunteers +who maintain the Python packaging toolchain identified a need for a library in +the ecology that handles the details of "wheel -> installed package". This is +that library. + +There's also a need for “a fast tool to populate a package into an environment” +and this library can be used to build that. This package itself might also +"grow" a CLI, to provide just that functionality. + +[1]: https://discuss.python.org/t/3869/ + +## What is provided + +- Abstractions for installation of a wheel distribution. +- Utilities for writing concrete implementations of these abstractions. +- Concrete implementations of these abstraction, for the most common usecase. +- Utilities for handling wheel RECORD files. +- Utilities for generating Python script launchers. diff --git a/docs/development/index.md b/docs/development/index.md new file mode 100644 index 0000000..22d1611 --- /dev/null +++ b/docs/development/index.md @@ -0,0 +1,27 @@ +# Development + +Thank you for your interest in installer! ✨ + +installer is a volunteer maintained open source project, and we welcome +contributions of all forms. This section of installer's documentation serves as +a resource to help you to contribute to the project. + +```{toctree} +:hidden: + +workflow +design +``` + +<!-- prettier-ignore-start --> +[Code of Conduct] +: Applies within all community spaces. If you are not familiar with our Code of Conduct, take a minute to read it before starting with your first contribution. + +[Workflow](./workflow) +: Describes how to work on this project. Start here if you're a new contributor. + +[Design and Scope](./design) +: Describes what this project is for, and how that informs the design decisions made. +<!-- prettier-ignore-end --> + +[code of conduct]: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md diff --git a/docs/development/workflow.md b/docs/development/workflow.md new file mode 100644 index 0000000..0c4f9e7 --- /dev/null +++ b/docs/development/workflow.md @@ -0,0 +1,116 @@ +# Workflow + +This page describes the tooling used during development of this project. It also +serves as a reference for the various commands that you would use when working +on this project. + +## Overview + +This project uses the [GitHub Flow] for collaboration. The codebase is Python. + +- [flit] is used for automating development tasks. +- [nox] is used for automating development tasks. +- [pre-commit] is used for running the linters. +- [sphinx] is used for generating this documentation. +- [pytest] is used for running the automated tests. + +## Repository Layout + +The repository layout is pretty standard for a modern pure-Python project. + +- `CODE_OF_CONDUCT.md` +- `LICENSE` +- `README.md` +- `.nox/` -- Generated by [nox]. +- `dist/` -- Generated as part of the release process. +- `docs/` -- Sources for the documentation. +- `src/` + - `installer/` -- Actual source code for the package +- `tests/` -- Automated tests for the package. +- `noxfile.py` -- for [nox]. +- `pyproject.toml` -- for packaging and tooling configuration. + +## Initial Setup + +To work on this project, you need to have git 2.17+ and Python 3.7+. + +- Clone this project using git: + + ```sh + git clone https://github.com/pradyunsg/installer.git + cd installer + ``` + +- Install the project's main development dependencies: + + ```sh + pip install nox + ``` + +You're all set for working on this project. + +## Commands + +### Code Linting + +```sh +nox -s lint +``` + +Run the linters, as configured with [pre-commit]. + +### Testing + +```sh +nox -s test +``` + +Run the tests against all supported Python versions, if an interpreter for that +version is available locally. + +```sh +nox -s test-3.9 +``` + +Run the tests against Python 3.9. It is also possible to specify other supported +Python versions (like `3.7` or `pypy3`). + +### Documentation + +```sh +nox -s docs +``` + +Generate the documentation for installer into the `build/docs` folder. This +(mostly) does the same thing as `nox -s docs-live`, except it invokes +`sphinx-build` instead of [sphinx-autobuild]. + +```sh +nox -s docs-live +``` + +Serve this project's documentation locally, using [sphinx-autobuild]. This will +open the generated documentation page in your browser. + +The server also watches for changes made to the documentation (`docs/`), which +will trigger a rebuild. Once the build is completed, server will automagically +reload any open pages using livereload. + +## Release process + +- Update the changelog. +- Update the version number in `__init__.py`. +- Commit these changes. +- Create a signed git tag. +- Run `flit publish`. +- Update the version number in `__init__.py`. +- Commit these changes. +- Push tag and commits. + +[github flow]: https://guides.github.com/introduction/flow/ +[flit]: https://flit.readthedocs.io/en/stable/ +[nox]: https://nox.readthedocs.io/en/stable/ +[pytest]: https://docs.pytest.org/en/stable/ +[sphinx]: https://www.sphinx-doc.org/en/master/ +[sphinx-autobuild]: https://github.com/executablebooks/sphinx-autobuild +[pre-commit]: https://pre-commit.com/ diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..7846513 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,71 @@ +--- +hide-toc: true +--- + +# Welcome to installer's documentation + +```{include} ../README.md +:start-after: <!-- start readme-pitch --> +:end-before: <!-- end readme-pitch --> +``` + +```{toctree} +:hidden: + +concepts +``` + +```{toctree} +:caption: API reference +:hidden: +:glob: + +api/* +``` + +```{toctree} +:caption: CLI reference +:hidden: +:glob: + +cli/* +``` + +```{toctree} +:caption: Project +:hidden: + +development/index +changelog +license +GitHub <https://github.com/pradyunsg/installer> +PyPI <https://pypi.org/project/installer> +``` + +## Basic Usage + +```python +import sys +import sysconfig + +from installer import install +from installer.destinations import SchemeDictionaryDestination +from installer.sources import WheelFile + +# Handler for installation directories and writing into them. +destination = SchemeDictionaryDestination( + sysconfig.get_paths(), + interpreter=sys.executable, + script_kind="posix", +) + +with WheelFile.open("sampleproject-1.3.1-py2.py3-none-any.whl") as source: + install( + source=source, + destination=destination, + # Additional metadata that is generated by the installation tool. + additional_metadata={ + "INSTALLER": b"amazing-installer 0.1.0", + }, + ) +``` diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 0000000..7ec60a8 --- /dev/null +++ b/docs/license.md @@ -0,0 +1,7 @@ +# License + +This project's source code and documentation is under the following license: + +```{include} ../LICENSE + +``` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..c20ceec --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,4 @@ +sphinx +furo +myst-parser +sphinx-argparse diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000..eee4f7b --- /dev/null +++ b/noxfile.py @@ -0,0 +1,99 @@ +"""Development automation +""" +import os + +import nox + +nox.options.sessions = ["lint", "test", "doctest"] +nox.options.reuse_existing_virtualenvs = True + + +def _install_this_project_with_flit(session, *, extras=None, editable=False): + session.install("flit") + args = [] + if extras: + args.append("--extras") + args.append(",".join(extras)) + if editable: + args.append("--pth-file" if os.name == "nt" else "--symlink") + + session.run("flit", "install", "--deps=production", *args, silent=True) + + +@nox.session(python="3.11") +def lint(session): + session.install("pre-commit") + + if session.posargs: + args = session.posargs + elif "CI" in os.environ: + args = ["--show-diff-on-failure"] + else: + args = [] + + session.run("pre-commit", "run", "--all-files", *args) + + +@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11", "pypy3"]) +def test(session): + _install_this_project_with_flit(session, editable=True) + session.install("-r", "tests/requirements.txt") + + htmlcov_output = os.path.join(session.virtualenv.location, "htmlcov") + + session.run( + "pytest", + "--cov=installer", + "--cov-fail-under=100", + "--cov-report=term-missing", + f"--cov-report=html:{htmlcov_output}", + "--cov-context=test", + "-n", + "auto", + *session.posargs, + ) + + +@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11", "pypy3"]) +def doctest(session): + session.install(".") + session.install("-r", "docs/requirements.txt") + + session.run("sphinx-build", "-b", "doctest", "docs/", "build/doctest") + + +@nox.session(python="3.11", name="update-launchers") +def update_launchers(session): + session.install("httpx") + session.run("python", "tools/update_launchers.py") + + +# +# Documentation +# +@nox.session(python="3.11") +def docs(session): + _install_this_project_with_flit(session) + session.install("-r", "docs/requirements.txt") + + # Generate documentation into `build/docs` + session.run("sphinx-build", "-W", "-b", "html", "docs/", "build/docs") + + +@nox.session(name="docs-live", python="3.11") +def docs_live(session): + _install_this_project_with_flit(session, editable=True) + session.install("-r", "docs/requirements.txt") + session.install("sphinx-autobuild") + + # fmt: off + session.run( + "sphinx-autobuild", "docs/", "build/docs", + # Rebuild all files when rebuilding + "-a", + # Trigger rebuilds on code changes (for autodoc) + "--watch", "src/installer", + # Use a not-common high-numbered port + "--port", "8765", + ) + # fmt: on diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..ba0f48b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["flit_core >=3.2.0,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "installer" +authors = [ + { name = "Pradyun Gedam", email = "pradyunsg@gmail.com" }, +] +readme = "README.md" +classifiers = [ + "License :: OSI Approved :: MIT License", +] +requires-python = ">=3.7" +dynamic = [ + "version", + "description", +] + +[project.urls] +"GitHub" = "https://github.com/pypa/installer" diff --git a/src/installer/__init__.py b/src/installer/__init__.py new file mode 100644 index 0000000..aa8e244 --- /dev/null +++ b/src/installer/__init__.py @@ -0,0 +1,6 @@ +"""A library for installing Python wheels.""" + +__version__ = "0.6.0" +__all__ = ["install"] + +from installer._core import install # noqa diff --git a/src/installer/__main__.py b/src/installer/__main__.py new file mode 100644 index 0000000..51014b9 --- /dev/null +++ b/src/installer/__main__.py @@ -0,0 +1,98 @@ +"""Installer CLI.""" + +import argparse +import os.path +import sys +import sysconfig +from typing import Dict, Optional, Sequence + +import installer +from installer.destinations import SchemeDictionaryDestination +from installer.sources import WheelFile +from installer.utils import get_launcher_kind + + +def _get_main_parser() -> argparse.ArgumentParser: + """Construct the main parser.""" + parser = argparse.ArgumentParser() + parser.add_argument("wheel", type=str, help="wheel file to install") + parser.add_argument( + "--destdir", + "-d", + metavar="path", + type=str, + help="destination directory (prefix to prepend to each file)", + ) + parser.add_argument( + "--prefix", + "-p", + metavar="path", + type=str, + help="override prefix to install packages to", + ) + parser.add_argument( + "--compile-bytecode", + action="append", + metavar="level", + type=int, + choices=[0, 1, 2], + help="generate bytecode for the specified optimization level(s) (default=0, 1)", + ) + parser.add_argument( + "--no-compile-bytecode", + action="store_true", + help="don't generate bytecode for installed modules", + ) + return parser + + +def _get_scheme_dict( + distribution_name: str, prefix: Optional[str] = None +) -> Dict[str, str]: + """Calculate the scheme dictionary for the current Python environment.""" + vars = {} + if prefix is None: + installed_base = sysconfig.get_config_var("base") + assert installed_base + else: + vars["base"] = vars["platbase"] = installed_base = prefix + + scheme_dict = sysconfig.get_paths(vars=vars) + + # calculate 'headers' path, not currently in sysconfig - see + # https://bugs.python.org/issue44445. This is based on what distutils does. + # TODO: figure out original vs normalised distribution names + scheme_dict["headers"] = os.path.join( + sysconfig.get_path("include", vars={"installed_base": installed_base}), + distribution_name, + ) + + return scheme_dict + + +def _main(cli_args: Sequence[str], program: Optional[str] = None) -> None: + """Process arguments and perform the install.""" + parser = _get_main_parser() + if program: + parser.prog = program + args = parser.parse_args(cli_args) + + bytecode_levels = args.compile_bytecode + if args.no_compile_bytecode: + bytecode_levels = [] + elif not bytecode_levels: + bytecode_levels = [0, 1] + + with WheelFile.open(args.wheel) as source: + destination = SchemeDictionaryDestination( + scheme_dict=_get_scheme_dict(source.distribution, prefix=args.prefix), + interpreter=sys.executable, + script_kind=get_launcher_kind(), + bytecode_optimization_levels=bytecode_levels, + destdir=args.destdir, + ) + installer.install(source, destination, {}) + + +if __name__ == "__main__": # pragma: no cover + _main(sys.argv[1:], "python -m installer") diff --git a/src/installer/_core.py b/src/installer/_core.py new file mode 100644 index 0000000..9a02728 --- /dev/null +++ b/src/installer/_core.py @@ -0,0 +1,135 @@ +"""Core wheel installation logic.""" + +import posixpath +from io import BytesIO +from typing import Dict, Tuple, cast + +from installer.destinations import WheelDestination +from installer.exceptions import InvalidWheelSource +from installer.records import RecordEntry +from installer.sources import WheelSource +from installer.utils import SCHEME_NAMES, Scheme, parse_entrypoints, parse_metadata_file + +__all__ = ["install"] + + +def _process_WHEEL_file(source: WheelSource) -> Scheme: + """Process the WHEEL file, from ``source``. + + Returns the scheme that the archive root should go in. + """ + stream = source.read_dist_info("WHEEL") + metadata = parse_metadata_file(stream) + + # Ensure compatibility with this wheel version. + if not (metadata["Wheel-Version"] and metadata["Wheel-Version"].startswith("1.")): + message = "Incompatible Wheel-Version {}, only support version 1.x wheels." + raise InvalidWheelSource(source, message.format(metadata["Wheel-Version"])) + + # Determine where archive root should go. + if metadata["Root-Is-Purelib"] == "true": + return cast(Scheme, "purelib") + else: + return cast(Scheme, "platlib") + + +def _determine_scheme( + path: str, source: WheelSource, root_scheme: Scheme +) -> Tuple[Scheme, str]: + """Determine which scheme to place given path in, from source.""" + data_dir = source.data_dir + + # If it's in not `{distribution}-{version}.data`, then it's in root_scheme. + if posixpath.commonprefix([data_dir, path]) != data_dir: + return root_scheme, path + + # Figure out which scheme this goes to. + parts = [] + scheme_name = None + left = path + while True: + left, right = posixpath.split(left) + parts.append(right) + if left == source.data_dir: + scheme_name = right + break + + if scheme_name not in SCHEME_NAMES: + msg_fmt = "{path} is not contained in a valid .data subdirectory." + raise InvalidWheelSource(source, msg_fmt.format(path=path)) + + return cast(Scheme, scheme_name), posixpath.join(*reversed(parts[:-1])) + + +def install( + source: WheelSource, + destination: WheelDestination, + additional_metadata: Dict[str, bytes], +) -> None: + """Install wheel described by ``source`` into ``destination``. + + :param source: wheel to install. + :param destination: where to write the wheel. + :param additional_metadata: additional metadata files to generate, usually + generated by the caller. + + """ + root_scheme = _process_WHEEL_file(source) + + # RECORD handling + record_file_path = posixpath.join(source.dist_info_dir, "RECORD") + written_records = [] + + # Write the entry_points based scripts. + if "entry_points.txt" in source.dist_info_filenames: + entrypoints_text = source.read_dist_info("entry_points.txt") + for name, module, attr, section in parse_entrypoints(entrypoints_text): + record = destination.write_script( + name=name, + module=module, + attr=attr, + section=section, + ) + written_records.append((Scheme("scripts"), record)) + + # Write all the files from the wheel. + for record_elements, stream, is_executable in source.get_contents(): + source_record = RecordEntry.from_elements(*record_elements) + path = source_record.path + # Skip the RECORD, which is written at the end, based on this info. + if path == record_file_path: + continue + + # Figure out where to write this file. + scheme, destination_path = _determine_scheme( + path=path, + source=source, + root_scheme=root_scheme, + ) + record = destination.write_file( + scheme=scheme, + path=destination_path, + stream=stream, + is_executable=is_executable, + ) + written_records.append((scheme, record)) + + # Write all the installation-specific metadata + for filename, contents in additional_metadata.items(): + path = posixpath.join(source.dist_info_dir, filename) + + with BytesIO(contents) as other_stream: + record = destination.write_file( + scheme=root_scheme, + path=path, + stream=other_stream, + is_executable=False, + ) + written_records.append((root_scheme, record)) + + written_records.append((root_scheme, RecordEntry(record_file_path, None, None))) + destination.finalize_installation( + scheme=root_scheme, + record_file_path=record_file_path, + records=written_records, + ) diff --git a/src/installer/_scripts/__init__.py b/src/installer/_scripts/__init__.py new file mode 100644 index 0000000..0361a58 --- /dev/null +++ b/src/installer/_scripts/__init__.py @@ -0,0 +1 @@ +"""Internal package, containing launcher templates for ``installer.scripts``.""" diff --git a/src/installer/destinations.py b/src/installer/destinations.py new file mode 100644 index 0000000..a3c1967 --- /dev/null +++ b/src/installer/destinations.py @@ -0,0 +1,284 @@ +"""Handles all file writing and post-installation processing.""" + +import compileall +import io +import os +from pathlib import Path +from typing import ( + TYPE_CHECKING, + BinaryIO, + Collection, + Dict, + Iterable, + Optional, + Tuple, + Union, +) + +from installer.records import Hash, RecordEntry +from installer.scripts import Script +from installer.utils import ( + Scheme, + construct_record_file, + copyfileobj_with_hashing, + fix_shebang, + make_file_executable, +) + +if TYPE_CHECKING: + from installer.scripts import LauncherKind, ScriptSection + + +class WheelDestination: + """Handles writing the unpacked files, script generation and ``RECORD`` generation. + + Subclasses provide the concrete script generation logic, as well as the RECORD file + (re)writing. + """ + + def write_script( + self, name: str, module: str, attr: str, section: "ScriptSection" + ) -> RecordEntry: + """Write a script in the correct location to invoke given entry point. + + :param name: name of the script + :param module: module path, to load the entry point from + :param attr: final attribute access, for the entry point + :param section: Denotes the "entry point section" where this was specified. + Valid values are ``"gui"`` and ``"console"``. + :type section: str + + Example usage/behaviour:: + + >>> dest.write_script("pip", "pip._internal.cli", "main", "console") + + """ + raise NotImplementedError + + def write_file( + self, + scheme: Scheme, + path: Union[str, "os.PathLike[str]"], + stream: BinaryIO, + is_executable: bool, + ) -> RecordEntry: + """Write a file to correct ``path`` within the ``scheme``. + + :param scheme: scheme to write the file in (like "purelib", "platlib" etc). + :param path: path within that scheme + :param stream: contents of the file + :param is_executable: whether the file should be made executable + + The stream would be closed by the caller, after this call. + + Example usage/behaviour:: + + >>> with open("__init__.py") as stream: + ... dest.write_file("purelib", "pkg/__init__.py", stream) + + """ + raise NotImplementedError + + def finalize_installation( + self, + scheme: Scheme, + record_file_path: str, + records: Iterable[Tuple[Scheme, RecordEntry]], + ) -> None: + """Finalize installation, after all the files are written. + + Handles (re)writing of the ``RECORD`` file. + + :param scheme: scheme to write the ``RECORD`` file in + :param record_file_path: path of the ``RECORD`` file with that scheme + :param records: entries to write to the ``RECORD`` file + + Example usage/behaviour:: + + >>> dest.finalize_installation("purelib") + + """ + raise NotImplementedError + + +class SchemeDictionaryDestination(WheelDestination): + """Destination, based on a mapping of {scheme: file-system-path}.""" + + def __init__( + self, + scheme_dict: Dict[str, str], + interpreter: str, + script_kind: "LauncherKind", + hash_algorithm: str = "sha256", + bytecode_optimization_levels: Collection[int] = (), + destdir: Optional[str] = None, + ) -> None: + """Construct a ``SchemeDictionaryDestination`` object. + + :param scheme_dict: a mapping of {scheme: file-system-path} + :param interpreter: the interpreter to use for generating scripts + :param script_kind: the "kind" of launcher script to use + :param hash_algorithm: the hashing algorithm to use, which is a member + of :any:`hashlib.algorithms_available` (ideally from + :any:`hashlib.algorithms_guaranteed`). + :param bytecode_optimization_levels: Compile cached bytecode for + installed .py files with these optimization levels. The bytecode + is specific to the minor version of Python (e.g. 3.10) used to + generate it. + :param destdir: A staging directory in which to write all files. This + is expected to be the filesystem root at runtime, so embedded paths + will be written as though this was the root. + """ + self.scheme_dict = scheme_dict + self.interpreter = interpreter + self.script_kind = script_kind + self.hash_algorithm = hash_algorithm + self.bytecode_optimization_levels = bytecode_optimization_levels + self.destdir = destdir + + def _path_with_destdir(self, scheme: Scheme, path: str) -> str: + file = os.path.join(self.scheme_dict[scheme], path) + if self.destdir is not None: + file_path = Path(file) + rel_path = file_path.relative_to(file_path.anchor) + return os.path.join(self.destdir, rel_path) + return file + + def write_to_fs( + self, + scheme: Scheme, + path: str, + stream: BinaryIO, + is_executable: bool, + ) -> RecordEntry: + """Write contents of ``stream`` to the correct location on the filesystem. + + :param scheme: scheme to write the file in (like "purelib", "platlib" etc). + :param path: path within that scheme + :param stream: contents of the file + :param is_executable: whether the file should be made executable + + - Ensures that an existing file is not being overwritten. + - Hashes the written content, to determine the entry in the ``RECORD`` file. + """ + target_path = self._path_with_destdir(scheme, path) + if os.path.exists(target_path): + message = f"File already exists: {target_path}" + raise FileExistsError(message) + + parent_folder = os.path.dirname(target_path) + if not os.path.exists(parent_folder): + os.makedirs(parent_folder) + + with open(target_path, "wb") as f: + hash_, size = copyfileobj_with_hashing(stream, f, self.hash_algorithm) + + if is_executable: + make_file_executable(target_path) + + return RecordEntry(path, Hash(self.hash_algorithm, hash_), size) + + def write_file( + self, + scheme: Scheme, + path: Union[str, "os.PathLike[str]"], + stream: BinaryIO, + is_executable: bool, + ) -> RecordEntry: + """Write a file to correct ``path`` within the ``scheme``. + + :param scheme: scheme to write the file in (like "purelib", "platlib" etc). + :param path: path within that scheme + :param stream: contents of the file + :param is_executable: whether the file should be made executable + + - Changes the shebang for files in the "scripts" scheme. + - Uses :py:meth:`SchemeDictionaryDestination.write_to_fs` for the + filesystem interaction. + """ + path_ = os.fspath(path) + + if scheme == "scripts": + with fix_shebang(stream, self.interpreter) as stream_with_different_shebang: + return self.write_to_fs( + scheme, path_, stream_with_different_shebang, is_executable + ) + + return self.write_to_fs(scheme, path_, stream, is_executable) + + def write_script( + self, name: str, module: str, attr: str, section: "ScriptSection" + ) -> RecordEntry: + """Write a script to invoke an entrypoint. + + :param name: name of the script + :param module: module path, to load the entry point from + :param attr: final attribute access, for the entry point + :param section: Denotes the "entry point section" where this was specified. + Valid values are ``"gui"`` and ``"console"``. + :type section: str + + - Generates a launcher using :any:`Script.generate`. + - Writes to the "scripts" scheme. + - Uses :py:meth:`SchemeDictionaryDestination.write_to_fs` for the + filesystem interaction. + """ + script = Script(name, module, attr, section) + script_name, data = script.generate(self.interpreter, self.script_kind) + + with io.BytesIO(data) as stream: + entry = self.write_to_fs( + Scheme("scripts"), script_name, stream, is_executable=True + ) + + path = self._path_with_destdir(Scheme("scripts"), script_name) + mode = os.stat(path).st_mode + mode |= (mode & 0o444) >> 2 + os.chmod(path, mode) + + return entry + + def _compile_bytecode(self, scheme: Scheme, record: RecordEntry) -> None: + """Compile bytecode for a single .py file.""" + if scheme not in ("purelib", "platlib"): + return + + target_path = self._path_with_destdir(scheme, record.path) + dir_path_to_embed = os.path.dirname( # Without destdir + os.path.join(self.scheme_dict[scheme], record.path) + ) + for level in self.bytecode_optimization_levels: + compileall.compile_file( + target_path, optimize=level, quiet=1, ddir=dir_path_to_embed + ) + + def finalize_installation( + self, + scheme: Scheme, + record_file_path: str, + records: Iterable[Tuple[Scheme, RecordEntry]], + ) -> None: + """Finalize installation, by writing the ``RECORD`` file & compiling bytecode. + + :param scheme: scheme to write the ``RECORD`` file in + :param record_file_path: path of the ``RECORD`` file with that scheme + :param records: entries to write to the ``RECORD`` file + """ + + def prefix_for_scheme(file_scheme: str) -> Optional[str]: + if file_scheme == scheme: + return None + path = os.path.relpath( + self.scheme_dict[file_scheme], + start=self.scheme_dict[scheme], + ) + return path + "/" + + record_list = list(records) + with construct_record_file(record_list, prefix_for_scheme) as record_stream: + self.write_to_fs( + scheme, record_file_path, record_stream, is_executable=False + ) + + for scheme, record in record_list: + self._compile_bytecode(scheme, record) diff --git a/src/installer/exceptions.py b/src/installer/exceptions.py new file mode 100644 index 0000000..01f044a --- /dev/null +++ b/src/installer/exceptions.py @@ -0,0 +1,9 @@ +"""Errors raised from this package.""" + + +class InstallerError(Exception): + """All exceptions raised from this package's code.""" + + +class InvalidWheelSource(InstallerError): + """When a wheel source violates a contract, or is not supported.""" diff --git a/src/installer/py.typed b/src/installer/py.typed new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/installer/py.typed diff --git a/src/installer/records.py b/src/installer/records.py new file mode 100644 index 0000000..36c37d0 --- /dev/null +++ b/src/installer/records.py @@ -0,0 +1,217 @@ +"""Provides an object-oriented model for handling :pep:`376` RECORD files.""" + +import base64 +import csv +import hashlib +import os +from typing import Iterable, Iterator, Optional, Tuple, cast + +__all__ = [ + "Hash", + "RecordEntry", + "InvalidRecordEntry", + "parse_record_file", +] + + +class InvalidRecordEntry(Exception): + """Raised when a RecordEntry is not valid, due to improper element values or count.""" + + def __init__(self, elements, issues): # noqa: D107 + super().__init__(", ".join(issues)) + self.issues = issues + self.elements = elements + + def __repr__(self): + return "InvalidRecordEntry(elements={!r}, issues={!r})".format( + self.elements, self.issues + ) + + +class Hash: + """Represents the "hash" element of a RecordEntry.""" + + def __init__(self, name: str, value: str) -> None: + """Construct a ``Hash`` object. + + Most consumers should use :py:meth:`Hash.parse` instead, since no + validation or parsing is performed by this constructor. + + :param name: name of the hash function + :param value: hashed value + """ + self.name = name + self.value = value + + def __str__(self) -> str: + return f"{self.name}={self.value}" + + def __repr__(self) -> str: + return f"Hash(name={self.name!r}, value={self.value!r})" + + def __eq__(self, other): + if not isinstance(other, Hash): + return NotImplemented + return self.value == other.value and self.name == other.name + + def validate(self, data: bytes) -> bool: + """Validate that ``data`` matches this instance. + + :param data: Contents of the file. + :return: Whether ``data`` matches the hashed value. + """ + digest = hashlib.new(self.name, data).digest() + value = base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") + return self.value == value + + @classmethod + def parse(cls, h: str) -> "Hash": + """Build a Hash object, from a "name=value" string. + + This accepts a string of the format for the second element in a record, + as described in :pep:`376`. + + Typical usage:: + + Hash.parse("sha256=Y0sCextp4SQtQNU-MSs7SsdxD1W-gfKJtUlEbvZ3i-4") + + :param h: a name=value string + """ + name, value = h.split("=", 1) + return cls(name, value) + + +class RecordEntry: + """Represents a single record in a RECORD file. + + A list of :py:class:`RecordEntry` objects fully represents a RECORD file. + """ + + def __init__(self, path: str, hash_: Optional[Hash], size: Optional[int]) -> None: + r"""Construct a ``RecordEntry`` object. + + Most consumers should use :py:meth:`RecordEntry.from_elements`, since no + validation or parsing is performed by this constructor. + + :param path: file's path + :param hash\_: hash of the file's contents + :param size: file's size in bytes + """ + super().__init__() + + self.path = path + self.hash_ = hash_ + self.size = size + + def to_row(self, path_prefix: Optional[str] = None) -> Tuple[str, str, str]: + """Convert this into a 3-element tuple that can be written in a RECORD file. + + :param path_prefix: A prefix to attach to the path -- must end in `/` + :return: a (path, hash, size) row + """ + if path_prefix is not None: + assert path_prefix.endswith("/") + path = path_prefix + self.path + else: + path = self.path + + # Convert Windows paths to use / for consistency + if os.sep == "\\": + path = path.replace("\\", "/") # pragma: no cover + + return ( + path, + str(self.hash_ or ""), + str(self.size) if self.size is not None else "", + ) + + def __repr__(self) -> str: + return "RecordEntry(path={!r}, hash_={!r}, size={!r})".format( + self.path, self.hash_, self.size + ) + + def __eq__(self, other): + if not isinstance(other, RecordEntry): + return NotImplemented + return ( + self.path == other.path + and self.hash_ == other.hash_ + and self.size == other.size + ) + + def validate(self, data: bytes) -> bool: + """Validate that ``data`` matches this instance. + + :param data: Contents of the file corresponding to this instance. + :return: whether ``data`` matches hash and size. + """ + if self.size is not None and len(data) != self.size: + return False + + if self.hash_: + return self.hash_.validate(data) + + return True + + @classmethod + def from_elements(cls, path: str, hash_: str, size: str) -> "RecordEntry": + r"""Build a RecordEntry object, from values of the elements. + + Typical usage:: + + for row in parse_record_file(f): + record = RecordEntry.from_elements(row[0], row[1], row[2]) + + Meaning of each element is specified in :pep:`376`. + + :param path: first element (file's path) + :param hash\_: second element (hash of the file's contents) + :param size: third element (file's size in bytes) + :raises InvalidRecordEntry: if any element is invalid + """ + # Validate the passed values. + issues = [] + + if not path: + issues.append("`path` cannot be empty") + + if hash_: + try: + hash_value: Optional[Hash] = Hash.parse(hash_) + except ValueError: + issues.append("`hash` does not follow the required format") + else: + hash_value = None + + if size: + try: + size_value: Optional[int] = int(size) + except ValueError: + issues.append("`size` cannot be non-integer") + else: + size_value = None + + if issues: + raise InvalidRecordEntry(elements=(path, hash_, size), issues=issues) + + return cls(path=path, hash_=hash_value, size=size_value) + + +def parse_record_file(rows: Iterable[str]) -> Iterator[Tuple[str, str, str]]: + """Parse a :pep:`376` RECORD. + + Returns an iterable of 3-value tuples, that can be passed to + :any:`RecordEntry.from_elements`. + + :param rows: iterator providing lines of a RECORD (no trailing newlines). + """ + reader = csv.reader(rows, delimiter=",", quotechar='"', lineterminator="\n") + for row_index, elements in enumerate(reader): + if len(elements) != 3: + message = "Row Index {}: expected 3 elements, got {}".format( + row_index, len(elements) + ) + raise InvalidRecordEntry(elements=elements, issues=[message]) + + value = cast(Tuple[str, str, str], tuple(elements)) + yield value diff --git a/src/installer/scripts.py b/src/installer/scripts.py new file mode 100644 index 0000000..7e3c8fc --- /dev/null +++ b/src/installer/scripts.py @@ -0,0 +1,151 @@ +"""Generate executable scripts, on various platforms.""" + +import io +import shlex +import zipfile +from importlib.resources import read_binary +from typing import TYPE_CHECKING, Mapping, Optional, Tuple + +from installer import _scripts + +if TYPE_CHECKING: + from typing import Literal + + LauncherKind = Literal["posix", "win-ia32", "win-amd64", "win-arm", "win-arm64"] + ScriptSection = Literal["console", "gui"] + + +__all__ = ["InvalidScript", "Script"] + + +_ALLOWED_LAUNCHERS: Mapping[Tuple["ScriptSection", "LauncherKind"], str] = { + ("console", "win-ia32"): "t32.exe", + ("console", "win-amd64"): "t64.exe", + ("console", "win-arm"): "t_arm.exe", + ("console", "win-arm64"): "t64-arm.exe", + ("gui", "win-ia32"): "w32.exe", + ("gui", "win-amd64"): "w64.exe", + ("gui", "win-arm"): "w_arm.exe", + ("gui", "win-arm64"): "w64-arm.exe", +} + +_SCRIPT_TEMPLATE = """\ +# -*- coding: utf-8 -*- +import re +import sys +from {module} import {import_name} +if __name__ == "__main__": + sys.argv[0] = re.sub(r"(-script\\.pyw|\\.exe)?$", "", sys.argv[0]) + sys.exit({func_path}()) +""" + + +def _is_executable_simple(executable: bytes) -> bool: + if b" " in executable: + return False + shebang_length = len(executable) + 3 # Prefix #! and newline after. + # According to distlib, Darwin can handle up to 512 characters. But I want + # to avoid platform sniffing to make this as platform agnostic as possible. + # The "complex" script isn't that bad anyway. + return shebang_length <= 127 + + +def _build_shebang(executable: str, forlauncher: bool) -> bytes: + """Build a shebang line. + + The non-launcher cases are taken directly from distlib's implementation, + which tries its best to account for command length, spaces in path, etc. + + https://bitbucket.org/pypa/distlib/src/58cd5c6/distlib/scripts.py#lines-124 + """ + executable_bytes = executable.encode("utf-8") + if forlauncher: # The launcher can just use the command as-is. + return b"#!" + executable_bytes + if _is_executable_simple(executable_bytes): + return b"#!" + executable_bytes + + # Shebang support for an executable with a space in it is under-specified + # and platform-dependent, so we use a clever hack to generate a script to + # run in ``/bin/sh`` that should work on all reasonably modern platforms. + # Read the following message to understand how the hack works: + # https://github.com/pradyunsg/installer/pull/4#issuecomment-623668717 + + quoted = shlex.quote(executable).encode("utf-8") + # I don't understand a lick what this is trying to do. + return b"#!/bin/sh\n'''exec' " + quoted + b' "$0" "$@"\n' + b"' '''" + + +class InvalidScript(ValueError): + """Raised if the user provides incorrect script section or kind.""" + + +class Script: + """Describes a script based on an entry point declaration.""" + + __slots__ = ("name", "module", "attr", "section") + + def __init__( + self, name: str, module: str, attr: str, section: "ScriptSection" + ) -> None: + """Construct a Script object. + + :param name: name of the script + :param module: module path, to load the entry point from + :param attr: final attribute access, for the entry point + :param section: Denotes the "entry point section" where this was specified. + Valid values are ``"gui"`` and ``"console"``. + :type section: str + + """ + self.name = name + self.module = module + self.attr = attr + self.section = section + + def __repr__(self) -> str: + return "Script(name={!r}, module={!r}, attr={!r}".format( + self.name, + self.module, + self.attr, + ) + + def _get_launcher_data(self, kind: "LauncherKind") -> Optional[bytes]: + if kind == "posix": + return None + key = (self.section, kind) + try: + name = _ALLOWED_LAUNCHERS[key] + except KeyError: + error = f"{key!r} not in {sorted(_ALLOWED_LAUNCHERS)!r}" + raise InvalidScript(error) + return read_binary(_scripts, name) + + def generate(self, executable: str, kind: "LauncherKind") -> Tuple[str, bytes]: + """Generate a launcher for this script. + + :param executable: Path to the executable to invoke. + :param kind: Which launcher template should be used. + Valid values are ``"posix"``, ``"win-ia32"``, ``"win-amd64"`` and + ``"win-arm"``. + :type kind: str + + :raises InvalidScript: if no appropriate template is available. + :return: The name and contents of the launcher file. + """ + launcher = self._get_launcher_data(kind) + shebang = _build_shebang(executable, forlauncher=bool(launcher)) + code = _SCRIPT_TEMPLATE.format( + module=self.module, + import_name=self.attr.split(".")[0], + func_path=self.attr, + ).encode("utf-8") + + if launcher is None: + return (self.name, shebang + b"\n" + code) + + stream = io.BytesIO() + with zipfile.ZipFile(stream, "w") as zf: + zf.writestr("__main__.py", code) + name = f"{self.name}.exe" + data = launcher + shebang + b"\n" + stream.getvalue() + return (name, data) diff --git a/src/installer/sources.py b/src/installer/sources.py new file mode 100644 index 0000000..fa0bc34 --- /dev/null +++ b/src/installer/sources.py @@ -0,0 +1,170 @@ +"""Source of information about a wheel file.""" + +import os +import posixpath +import stat +import zipfile +from contextlib import contextmanager +from typing import BinaryIO, Iterator, List, Tuple, cast + +from installer.records import parse_record_file +from installer.utils import parse_wheel_filename + +WheelContentElement = Tuple[Tuple[str, str, str], BinaryIO, bool] + + +__all__ = ["WheelSource", "WheelFile"] + + +class WheelSource: + """Represents an installable wheel. + + This is an abstract class, whose methods have to be implemented by subclasses. + """ + + def __init__(self, distribution: str, version: str) -> None: + """Initialize a WheelSource object. + + :param distribution: distribution name (like ``urllib3``) + :param version: version associated with the wheel + """ + super().__init__() + self.distribution = distribution + self.version = version + + @property + def dist_info_dir(self): + """Name of the dist-info directory.""" + return f"{self.distribution}-{self.version}.dist-info" + + @property + def data_dir(self): + """Name of the data directory.""" + return f"{self.distribution}-{self.version}.data" + + @property + def dist_info_filenames(self) -> List[str]: + """Get names of all files in the dist-info directory. + + Sample usage/behaviour:: + + >>> wheel_source.dist_info_filenames + ['METADATA', 'WHEEL'] + """ + raise NotImplementedError + + def read_dist_info(self, filename: str) -> str: + """Get contents, from ``filename`` in the dist-info directory. + + Sample usage/behaviour:: + + >>> wheel_source.read_dist_info("METADATA") + ... + + :param filename: name of the file + """ + raise NotImplementedError + + def get_contents(self) -> Iterator[WheelContentElement]: + """Sequential access to all contents of the wheel (including dist-info files). + + This method should return an iterable. Each value from the iterable must be a + tuple containing 3 elements: + + - record: 3-value tuple, to pass to + :py:meth:`RecordEntry.from_elements <installer.records.RecordEntry.from_elements>`. + - stream: An :py:class:`io.BufferedReader` object, providing the contents of the + file at the location provided by the first element (path). + - is_executable: A boolean, representing whether the item has an executable bit. + + All paths must be relative to the root of the wheel. + + Sample usage/behaviour:: + + >>> iterable = wheel_source.get_contents() + >>> next(iterable) + (('pkg/__init__.py', '', '0'), <...>, False) + + This method may be called multiple times. Each iterable returned must + provide the same content upon reading from a specific file's stream. + """ + raise NotImplementedError + + +class WheelFile(WheelSource): + """Implements `WheelSource`, for an existing file from the filesystem. + + Example usage:: + + >>> with WheelFile.open("sampleproject-2.0.0-py3-none-any.whl") as source: + ... installer.install(source, destination) + """ + + def __init__(self, f: zipfile.ZipFile) -> None: + """Initialize a WheelFile object. + + :param f: An open zipfile, which will stay open as long as this object is used. + """ + self._zipfile = f + assert f.filename + + basename = os.path.basename(f.filename) + parsed_name = parse_wheel_filename(basename) + super().__init__( + version=parsed_name.version, + distribution=parsed_name.distribution, + ) + + @classmethod + @contextmanager + def open(cls, path: "os.PathLike[str]") -> Iterator["WheelFile"]: + """Create a wheelfile from a given path.""" + with zipfile.ZipFile(path) as f: + yield cls(f) + + @property + def dist_info_filenames(self) -> List[str]: + """Get names of all files in the dist-info directory.""" + base = self.dist_info_dir + return [ + name[len(base) + 1 :] + for name in self._zipfile.namelist() + if name[-1:] != "/" + if base == posixpath.commonprefix([name, base]) + ] + + def read_dist_info(self, filename: str) -> str: + """Get contents, from ``filename`` in the dist-info directory.""" + path = posixpath.join(self.dist_info_dir, filename) + return self._zipfile.read(path).decode("utf-8") + + def get_contents(self) -> Iterator[WheelContentElement]: + """Sequential access to all contents of the wheel (including dist-info files). + + This implementation requires that every file that is a part of the wheel + archive has a corresponding entry in RECORD. If they are not, an + :any:`AssertionError` will be raised. + """ + # Convert the record file into a useful mapping + record_lines = self.read_dist_info("RECORD").splitlines() + records = parse_record_file(record_lines) + record_mapping = {record[0]: record for record in records} + + for item in self._zipfile.infolist(): + if item.filename[-1:] == "/": # looks like a directory + continue + + record = record_mapping.pop(item.filename, None) + assert record is not None, "In {}, {} is not mentioned in RECORD".format( + self._zipfile.filename, + item.filename, + ) # should not happen for valid wheels + + # Borrowed from: + # https://github.com/pypa/pip/blob/0f21fb92/src/pip/_internal/utils/unpacking.py#L96-L100 + mode = item.external_attr >> 16 + is_executable = bool(mode and stat.S_ISREG(mode) and mode & 0o111) + + with self._zipfile.open(item) as stream: + stream_casted = cast("BinaryIO", stream) + yield record, stream_casted, is_executable diff --git a/src/installer/utils.py b/src/installer/utils.py new file mode 100644 index 0000000..7b1404d --- /dev/null +++ b/src/installer/utils.py @@ -0,0 +1,252 @@ +"""Utilities related to handling / interacting with wheel files.""" + +import base64 +import contextlib +import csv +import hashlib +import io +import os +import re +import sys +from collections import namedtuple +from configparser import ConfigParser +from email.message import Message +from email.parser import FeedParser +from typing import ( + TYPE_CHECKING, + BinaryIO, + Callable, + Iterable, + Iterator, + NewType, + Optional, + Tuple, + Union, + cast, +) + +from installer.records import RecordEntry + +if TYPE_CHECKING: + from installer.scripts import LauncherKind, ScriptSection + +Scheme = NewType("Scheme", str) +AllSchemes = Tuple[Scheme, ...] + +__all__ = [ + "parse_metadata_file", + "parse_wheel_filename", + "copyfileobj_with_hashing", + "get_launcher_kind", + "fix_shebang", + "construct_record_file", + "parse_entrypoints", + "make_file_executable", + "WheelFilename", + "SCHEME_NAMES", +] + +# Borrowed from https://github.com/python/cpython/blob/v3.9.1/Lib/shutil.py#L52 +_WINDOWS = os.name == "nt" +_COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 + +# According to https://www.python.org/dev/peps/pep-0427/#file-name-convention +_WHEEL_FILENAME_REGEX = re.compile( + r""" + ^ + (?P<distribution>.+?) + -(?P<version>.*?) + (?:-(?P<build_tag>\d[^-]*?))? + -(?P<tag>.+?-.+?-.+?) + \.whl + $ + """, + re.VERBOSE | re.UNICODE, +) +WheelFilename = namedtuple( + "WheelFilename", ["distribution", "version", "build_tag", "tag"] +) + +# Adapted from https://github.com/python/importlib_metadata/blob/v3.4.0/importlib_metadata/__init__.py#L90 # noqa +_ENTRYPOINT_REGEX = re.compile( + r""" + (?P<module>[\w.]+)\s* + (:\s*(?P<attrs>[\w.]+))\s* + (?P<extras>\[.*\])?\s*$ + """, + re.VERBOSE | re.UNICODE, +) + +# According to https://www.python.org/dev/peps/pep-0427/#id7 +SCHEME_NAMES = cast(AllSchemes, ("purelib", "platlib", "headers", "scripts", "data")) + + +def parse_metadata_file(contents: str) -> Message: + """Parse :pep:`376` ``PKG-INFO``-style metadata files. + + ``METADATA`` and ``WHEEL`` files (as per :pep:`427`) use the same syntax + and can also be parsed using this function. + + :param contents: The entire contents of the file + """ + feed_parser = FeedParser() + feed_parser.feed(contents) + return feed_parser.close() + + +def parse_wheel_filename(filename: str) -> WheelFilename: + """Parse a wheel filename, into it's various components. + + :param filename: The filename to parse + """ + wheel_info = _WHEEL_FILENAME_REGEX.match(filename) + if not wheel_info: + raise ValueError(f"Not a valid wheel filename: {filename}") + return WheelFilename(*wheel_info.groups()) + + +def copyfileobj_with_hashing( + source: BinaryIO, + dest: BinaryIO, + hash_algorithm: str, +) -> Tuple[str, int]: + """Copy a buffer while computing the content's hash and size. + + Copies the source buffer into the destination buffer while computing the + hash of the contents. Adapted from :any:`shutil.copyfileobj`. + + :param source: buffer holding the source data + :param dest: destination buffer + :param hash_algorithm: hashing algorithm + + :return: size, hash digest of the contents + """ + hasher = hashlib.new(hash_algorithm) + size = 0 + while True: + buf = source.read(_COPY_BUFSIZE) + if not buf: + break + hasher.update(buf) + dest.write(buf) + size += len(buf) + + return base64.urlsafe_b64encode(hasher.digest()).decode("ascii").rstrip("="), size + + +def get_launcher_kind() -> "LauncherKind": # pragma: no cover + """Get the launcher kind for the current machine.""" + if os.name != "nt": + return "posix" + + if "amd64" in sys.version.lower(): + return "win-amd64" + if "(arm64)" in sys.version.lower(): + return "win-arm64" + if "(arm)" in sys.version.lower(): + return "win-arm" + if sys.platform == "win32": + return "win-ia32" + + raise NotImplementedError("Unknown launcher kind for this machine") + + +@contextlib.contextmanager +def fix_shebang(stream: BinaryIO, interpreter: str) -> Iterator[BinaryIO]: + """Replace ``#!python`` shebang in a stream with the correct interpreter. + + :param stream: stream to modify + :param interpreter: "correct interpreter" to substitute the shebang with + + :returns: A context manager, that provides an appropriately modified stream. + """ + stream.seek(0) + if stream.read(8) == b"#!python": + new_stream = io.BytesIO() + # write our new shebang + new_stream.write(f"#!{interpreter}\n".encode()) + # copy the rest of the stream + stream.seek(0) + stream.readline() # skip first line + while True: + buf = stream.read(_COPY_BUFSIZE) + if not buf: + break + new_stream.write(buf) + new_stream.seek(0) + yield new_stream + new_stream.close() + else: + stream.seek(0) + yield stream + + +def construct_record_file( + records: Iterable[Tuple[Scheme, RecordEntry]], + prefix_for_scheme: Callable[[Scheme], Optional[str]] = lambda _: None, +) -> BinaryIO: + """Construct a RECORD file. + + :param records: + ``records`` as passed into :any:`WheelDestination.finalize_installation` + :param prefix_for_scheme: + function to get a prefix to add for RECORD entries, within a scheme + + :return: A stream that can be written to file. Must be closed by the caller. + """ + stream = io.TextIOWrapper( + io.BytesIO(), encoding="utf-8", write_through=True, newline="" + ) + writer = csv.writer(stream, delimiter=",", quotechar='"', lineterminator="\n") + for scheme, record in records: + writer.writerow(record.to_row(prefix_for_scheme(scheme))) + stream.seek(0) + return stream.detach() + + +def parse_entrypoints(text: str) -> Iterable[Tuple[str, str, str, "ScriptSection"]]: + """Parse ``entry_points.txt``-style files. + + :param text: entire contents of the file + :return: + name of the script, module to use, attribute to call, kind of script (cli / gui) + """ + # Borrowed from https://github.com/python/importlib_metadata/blob/v3.4.0/importlib_metadata/__init__.py#L115 # noqa + config = ConfigParser(delimiters="=") + config.optionxform = str # type: ignore + config.read_string(text) + + for section in config.sections(): + if section not in ["console_scripts", "gui_scripts"]: + continue + + for name, value in config.items(section): + assert isinstance(name, str) + match = _ENTRYPOINT_REGEX.match(value) + assert match + + module = match.group("module") + assert isinstance(module, str) + + attrs = match.group("attrs") + # TODO: make this a proper error, which can be caught. + assert attrs is not None + assert isinstance(attrs, str) + + script_section = cast("ScriptSection", section[: -len("_scripts")]) + + yield name, module, attrs, script_section + + +def _current_umask() -> int: + """Get the current umask which involves having to set it temporarily.""" + mask = os.umask(0) + os.umask(mask) + return mask + + +# Borrowed from: +# https://github.com/pypa/pip/blob/0f21fb92/src/pip/_internal/utils/unpacking.py#L93 +def make_file_executable(path: Union[str, "os.PathLike[str]"]) -> None: + """Make the file at the provided path executable.""" + os.chmod(path, (0o777 & ~_current_umask() | 0o111)) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..029cb8f --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,75 @@ +import textwrap +import zipfile + +import pytest + + +@pytest.fixture +def fancy_wheel(tmp_path): + path = tmp_path / "fancy-1.0.0-py2.py3-none-any.whl" + files = { + "fancy/": b"""""", + "fancy/__init__.py": b"""\ + def main(): + print("I'm fancy.") + """, + "fancy/__main__.py": b"""\ + if __name__ == "__main__": + from . import main + main() + """, + "fancy-1.0.0.data/data/fancy/": b"""""", + "fancy-1.0.0.data/data/fancy/data.py": b"""\ + # put me in data + """, + "fancy-1.0.0.dist-info/": b"""""", + "fancy-1.0.0.dist-info/top_level.txt": b"""\ + fancy + """, + "fancy-1.0.0.dist-info/entry_points.txt": b"""\ + [console_scripts] + fancy = fancy:main + + [gui_scripts] + fancy-gui = fancy:main + """, + "fancy-1.0.0.dist-info/WHEEL": b"""\ + Wheel-Version: 1.0 + Generator: magic (1.0.0) + Root-Is-Purelib: true + Tag: py3-none-any + """, + "fancy-1.0.0.dist-info/METADATA": b"""\ + Metadata-Version: 2.1 + Name: fancy + Version: 1.0.0 + Summary: A fancy package + Author: Agendaless Consulting + Author-email: nobody@example.com + License: MIT + Keywords: fancy amazing + Platform: UNKNOWN + Classifier: Intended Audience :: Developers + """, + # The RECORD file is indirectly validated by the WheelFile, since it only + # provides the items that are a part of the wheel. + "fancy-1.0.0.dist-info/RECORD": b"""\ + fancy/__init__.py,, + fancy/__main__.py,, + fancy-1.0.0.data/data/fancy/data.py,, + fancy-1.0.0.dist-info/top_level.txt,, + fancy-1.0.0.dist-info/entry_points.txt,, + fancy-1.0.0.dist-info/WHEEL,, + fancy-1.0.0.dist-info/METADATA,, + fancy-1.0.0.dist-info/RECORD,, + """, + } + + with zipfile.ZipFile(path, "w") as archive: + for name, indented_content in files.items(): + archive.writestr( + name, + textwrap.dedent(indented_content.decode("utf-8")).encode("utf-8"), + ) + + return path diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..78b1a59 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,3 @@ +pytest +pytest-cov +pytest-xdist diff --git a/tests/test_core.py b/tests/test_core.py new file mode 100644 index 0000000..1f3a44e --- /dev/null +++ b/tests/test_core.py @@ -0,0 +1,905 @@ +import hashlib +import textwrap +from io import BytesIO +from unittest import mock + +import pytest + +from installer import install +from installer.exceptions import InvalidWheelSource +from installer.records import RecordEntry +from installer.sources import WheelSource + + +# -------------------------------------------------------------------------------------- +# Helpers +# -------------------------------------------------------------------------------------- +def hash_and_size(data): + return hashlib.sha256(data).hexdigest(), len(data) + + +@pytest.fixture +def mock_destination(): + retval = mock.Mock() + + # A hacky approach to making sure we got the right objects going in. + def custom_write_file(scheme, path, stream, is_executable): + assert isinstance(stream, BytesIO) + return (path, scheme, 0) + + def custom_write_script(name, module, attr, section): + return (name, module, attr, section) + + retval.write_file.side_effect = custom_write_file + retval.write_script.side_effect = custom_write_script + + return retval + + +class FakeWheelSource(WheelSource): + def __init__(self, *, distribution, version, regular_files, dist_info_files): + super().__init__(distribution, version) + + self.dist_info_files = { + file: textwrap.dedent(content.decode("utf-8")) + for file, content in dist_info_files.items() + } + self.regular_files = { + file: textwrap.dedent(content.decode("utf-8")).encode("utf-8") + for file, content in regular_files.items() + } + + # Compute RECORD file. + _records = [record for record, _, _ in self.get_contents()] + self.dist_info_files["RECORD"] = "\n".join( + sorted( + ",".join([file, "sha256=" + hash_, str(size)]) + for file, hash_, size in _records + ) + ) + + @property + def dist_info_filenames(self): + return list(self.dist_info_files) + + def read_dist_info(self, filename): + return self.dist_info_files[filename] + + def get_contents(self): + # Sort for deterministic behaviour for Python versions that do not preserve + # insertion order for dictionaries. + for file, content in sorted(self.regular_files.items()): + hashed, size = hash_and_size(content) + record = (file, f"sha256={hashed}", str(size)) + with BytesIO(content) as stream: + yield record, stream, False + + # Sort for deterministic behaviour for Python versions that do not preserve + # insertion order for dictionaries. + for file, text in sorted(self.dist_info_files.items()): + content = text.encode("utf-8") + hashed, size = hash_and_size(content) + record = ( + self.dist_info_dir + "/" + file, + f"sha256={hashed}", + str(size), + ) + with BytesIO(content) as stream: + yield record, stream, False + + +# -------------------------------------------------------------------------------------- +# Actual Tests +# -------------------------------------------------------------------------------------- +class TestInstall: + def test_calls_destination_correctly(self, mock_destination): + # Create a fake wheel + source = FakeWheelSource( + distribution="fancy", + version="1.0.0", + regular_files={ + "fancy/__init__.py": b"""\ + def main(): + print("I'm a fancy package") + """, + "fancy/__main__.py": b"""\ + if __name__ == "__main__": + from . import main + main() + """, + }, + dist_info_files={ + "top_level.txt": b"""\ + fancy + """, + "entry_points.txt": b"""\ + [console_scripts] + fancy = fancy:main + + [gui_scripts] + fancy-gui = fancy:main + """, + "WHEEL": b"""\ + Wheel-Version: 1.0 + Generator: magic (1.0.0) + Root-Is-Purelib: true + Tag: py3-none-any + """, + "METADATA": b"""\ + Metadata-Version: 2.1 + Name: fancy + Version: 1.0.0 + Summary: A fancy package + Author: Agendaless Consulting + Author-email: nobody@example.com + License: MIT + Keywords: fancy amazing + Platform: UNKNOWN + Classifier: Intended Audience :: Developers + """, + }, + ) + + install( + source=source, + destination=mock_destination, + additional_metadata={ + "fun_file.txt": b"this should be in dist-info!", + }, + ) + + mock_destination.assert_has_calls( + [ + mock.call.write_script( + name="fancy", + module="fancy", + attr="main", + section="console", + ), + mock.call.write_script( + name="fancy-gui", + module="fancy", + attr="main", + section="gui", + ), + mock.call.write_file( + scheme="purelib", + path="fancy/__init__.py", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy/__main__.py", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/METADATA", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/WHEEL", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/entry_points.txt", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/top_level.txt", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/fun_file.txt", + stream=mock.ANY, + is_executable=False, + ), + mock.call.finalize_installation( + scheme="purelib", + record_file_path="fancy-1.0.0.dist-info/RECORD", + records=[ + ("scripts", ("fancy", "fancy", "main", "console")), + ("scripts", ("fancy-gui", "fancy", "main", "gui")), + ("purelib", ("fancy/__init__.py", "purelib", 0)), + ("purelib", ("fancy/__main__.py", "purelib", 0)), + ("purelib", ("fancy-1.0.0.dist-info/METADATA", "purelib", 0)), + ("purelib", ("fancy-1.0.0.dist-info/WHEEL", "purelib", 0)), + ( + "purelib", + ("fancy-1.0.0.dist-info/entry_points.txt", "purelib", 0), + ), + ( + "purelib", + ("fancy-1.0.0.dist-info/top_level.txt", "purelib", 0), + ), + ( + "purelib", + ("fancy-1.0.0.dist-info/fun_file.txt", "purelib", 0), + ), + ( + "purelib", + RecordEntry("fancy-1.0.0.dist-info/RECORD", None, None), + ), + ], + ), + ] + ) + + def test_no_entrypoints_is_ok(self, mock_destination): + # Create a fake wheel + source = FakeWheelSource( + distribution="fancy", + version="1.0.0", + regular_files={ + "fancy/__init__.py": b"""\ + def main(): + print("I'm a fancy package") + """, + "fancy/__main__.py": b"""\ + if __name__ == "__main__": + from . import main + main() + """, + }, + dist_info_files={ + "top_level.txt": b"""\ + fancy + """, + "WHEEL": b"""\ + Wheel-Version: 1.0 + Generator: magic (1.0.0) + Root-Is-Purelib: true + Tag: py3-none-any + """, + "METADATA": b"""\ + Metadata-Version: 2.1 + Name: fancy + Version: 1.0.0 + Summary: A fancy package + Author: Agendaless Consulting + Author-email: nobody@example.com + License: MIT + Keywords: fancy amazing + Platform: UNKNOWN + Classifier: Intended Audience :: Developers + """, + }, + ) + + install( + source=source, + destination=mock_destination, + additional_metadata={ + "fun_file.txt": b"this should be in dist-info!", + }, + ) + + mock_destination.assert_has_calls( + [ + mock.call.write_file( + scheme="purelib", + path="fancy/__init__.py", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy/__main__.py", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/METADATA", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/WHEEL", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/top_level.txt", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/fun_file.txt", + stream=mock.ANY, + is_executable=False, + ), + mock.call.finalize_installation( + scheme="purelib", + record_file_path="fancy-1.0.0.dist-info/RECORD", + records=[ + ("purelib", ("fancy/__init__.py", "purelib", 0)), + ("purelib", ("fancy/__main__.py", "purelib", 0)), + ("purelib", ("fancy-1.0.0.dist-info/METADATA", "purelib", 0)), + ("purelib", ("fancy-1.0.0.dist-info/WHEEL", "purelib", 0)), + ( + "purelib", + ("fancy-1.0.0.dist-info/top_level.txt", "purelib", 0), + ), + ( + "purelib", + ("fancy-1.0.0.dist-info/fun_file.txt", "purelib", 0), + ), + ( + "purelib", + RecordEntry("fancy-1.0.0.dist-info/RECORD", None, None), + ), + ], + ), + ] + ) + + def test_handles_platlib(self, mock_destination): + # Create a fake wheel + source = FakeWheelSource( + distribution="fancy", + version="1.0.0", + regular_files={ + "fancy/__init__.py": b"""\ + def main(): + print("I'm a fancy package") + """, + "fancy/__main__.py": b"""\ + if __name__ == "__main__": + from . import main + main() + """, + }, + dist_info_files={ + "top_level.txt": b"""\ + fancy + """, + "entry_points.txt": b"""\ + [console_scripts] + fancy = fancy:main + + [gui_scripts] + fancy-gui = fancy:main + """, + "WHEEL": b"""\ + Wheel-Version: 1.0 + Generator: magic (1.0.0) + Root-Is-Purelib: false + Tag: py3-none-any + """, + "METADATA": b"""\ + Metadata-Version: 2.1 + Name: fancy + Version: 1.0.0 + Summary: A fancy package + Author: Agendaless Consulting + Author-email: nobody@example.com + License: MIT + Keywords: fancy amazing + Platform: UNKNOWN + Classifier: Intended Audience :: Developers + """, + }, + ) + + install( + source=source, + destination=mock_destination, + additional_metadata={ + "fun_file.txt": b"this should be in dist-info!", + }, + ) + + mock_destination.assert_has_calls( + [ + mock.call.write_script( + name="fancy", + module="fancy", + attr="main", + section="console", + ), + mock.call.write_script( + name="fancy-gui", + module="fancy", + attr="main", + section="gui", + ), + mock.call.write_file( + scheme="platlib", + path="fancy/__init__.py", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="platlib", + path="fancy/__main__.py", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="platlib", + path="fancy-1.0.0.dist-info/METADATA", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="platlib", + path="fancy-1.0.0.dist-info/WHEEL", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="platlib", + path="fancy-1.0.0.dist-info/entry_points.txt", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="platlib", + path="fancy-1.0.0.dist-info/top_level.txt", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="platlib", + path="fancy-1.0.0.dist-info/fun_file.txt", + stream=mock.ANY, + is_executable=False, + ), + mock.call.finalize_installation( + scheme="platlib", + record_file_path="fancy-1.0.0.dist-info/RECORD", + records=[ + ("scripts", ("fancy", "fancy", "main", "console")), + ("scripts", ("fancy-gui", "fancy", "main", "gui")), + ("platlib", ("fancy/__init__.py", "platlib", 0)), + ("platlib", ("fancy/__main__.py", "platlib", 0)), + ("platlib", ("fancy-1.0.0.dist-info/METADATA", "platlib", 0)), + ("platlib", ("fancy-1.0.0.dist-info/WHEEL", "platlib", 0)), + ( + "platlib", + ("fancy-1.0.0.dist-info/entry_points.txt", "platlib", 0), + ), + ( + "platlib", + ("fancy-1.0.0.dist-info/top_level.txt", "platlib", 0), + ), + ( + "platlib", + ("fancy-1.0.0.dist-info/fun_file.txt", "platlib", 0), + ), + ( + "platlib", + RecordEntry("fancy-1.0.0.dist-info/RECORD", None, None), + ), + ], + ), + ] + ) + + def test_accepts_newer_minor_wheel_versions(self, mock_destination): + # Create a fake wheel + source = FakeWheelSource( + distribution="fancy", + version="1.0.0", + regular_files={ + "fancy/__init__.py": b"""\ + def main(): + print("I'm a fancy package") + """, + "fancy/__main__.py": b"""\ + if __name__ == "__main__": + from . import main + main() + """, + }, + dist_info_files={ + "top_level.txt": b"""\ + fancy + """, + "entry_points.txt": b"""\ + [console_scripts] + fancy = fancy:main + + [gui_scripts] + fancy-gui = fancy:main + """, + "WHEEL": b"""\ + Wheel-Version: 1.1 + Generator: magic (1.0.0) + Root-Is-Purelib: true + Tag: py3-none-any + """, + "METADATA": b"""\ + Metadata-Version: 2.1 + Name: fancy + Version: 1.0.0 + Summary: A fancy package + Author: Agendaless Consulting + Author-email: nobody@example.com + License: MIT + Keywords: fancy amazing + Platform: UNKNOWN + Classifier: Intended Audience :: Developers + """, + }, + ) + + install( + source=source, + destination=mock_destination, + additional_metadata={ + "fun_file.txt": b"this should be in dist-info!", + }, + ) + + # no assertions necessary, since we want to make sure this test didn't + # raises errors. + assert True + + def test_rejects_newer_major_wheel_versions(self, mock_destination): + # Create a fake wheel + source = FakeWheelSource( + distribution="fancy", + version="1.0.0", + regular_files={ + "fancy/__init__.py": b"""\ + def main(): + print("I'm a fancy package") + """, + "fancy/__main__.py": b"""\ + if __name__ == "__main__": + from . import main + main() + """, + }, + dist_info_files={ + "top_level.txt": b"""\ + fancy + """, + "entry_points.txt": b"""\ + [console_scripts] + fancy = fancy:main + + [gui_scripts] + fancy-gui = fancy:main + """, + "WHEEL": b"""\ + Wheel-Version: 2.0 + Generator: magic (1.0.0) + Root-Is-Purelib: true + Tag: py3-none-any + """, + "METADATA": b"""\ + Metadata-Version: 2.1 + Name: fancy + Version: 1.0.0 + Summary: A fancy package + Author: Agendaless Consulting + Author-email: nobody@example.com + License: MIT + Keywords: fancy amazing + Platform: UNKNOWN + Classifier: Intended Audience :: Developers + """, + }, + ) + + with pytest.raises(InvalidWheelSource) as ctx: + install( + source=source, + destination=mock_destination, + additional_metadata={ + "fun_file.txt": b"this should be in dist-info!", + }, + ) + + assert "Incompatible Wheel-Version" in str(ctx.value) + + def test_handles_data_properly(self, mock_destination): + # Create a fake wheel + source = FakeWheelSource( + distribution="fancy", + version="1.0.0", + regular_files={ + "fancy/__init__.py": b"""\ + # put me in purelib + """, + "fancy-1.0.0.data/purelib/fancy/purelib.py": b"""\ + # put me in purelib + """, + "fancy-1.0.0.data/platlib/fancy/platlib.py": b"""\ + # put me in platlib + """, + "fancy-1.0.0.data/scripts/fancy/scripts.py": b"""\ + # put me in scripts + """, + "fancy-1.0.0.data/headers/fancy/headers.py": b"""\ + # put me in headers + """, + "fancy-1.0.0.data/data/fancy/data.py": b"""\ + # put me in data + """, + }, + dist_info_files={ + "top_level.txt": b"""\ + fancy + """, + "entry_points.txt": b"""\ + [console_scripts] + fancy = fancy:main + + [gui_scripts] + fancy-gui = fancy:main + """, + "WHEEL": b"""\ + Wheel-Version: 1.0 + Generator: magic (1.0.0) + Root-Is-Purelib: true + Tag: py3-none-any + """, + "METADATA": b"""\ + Metadata-Version: 2.1 + Name: fancy + Version: 1.0.0 + Summary: A fancy package + Author: Agendaless Consulting + Author-email: nobody@example.com + License: MIT + Keywords: fancy amazing + Platform: UNKNOWN + Classifier: Intended Audience :: Developers + """, + }, + ) + + install( + source=source, + destination=mock_destination, + additional_metadata={}, + ) + + mock_destination.assert_has_calls( + [ + mock.call.write_script( + name="fancy", + module="fancy", + attr="main", + section="console", + ), + mock.call.write_script( + name="fancy-gui", + module="fancy", + attr="main", + section="gui", + ), + mock.call.write_file( + scheme="data", + path="fancy/data.py", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="headers", + path="fancy/headers.py", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="platlib", + path="fancy/platlib.py", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy/purelib.py", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="scripts", + path="fancy/scripts.py", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy/__init__.py", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/METADATA", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/WHEEL", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/entry_points.txt", + stream=mock.ANY, + is_executable=False, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/top_level.txt", + stream=mock.ANY, + is_executable=False, + ), + mock.call.finalize_installation( + scheme="purelib", + record_file_path="fancy-1.0.0.dist-info/RECORD", + records=[ + ("scripts", ("fancy", "fancy", "main", "console")), + ("scripts", ("fancy-gui", "fancy", "main", "gui")), + ("data", ("fancy/data.py", "data", 0)), + ("headers", ("fancy/headers.py", "headers", 0)), + ("platlib", ("fancy/platlib.py", "platlib", 0)), + ("purelib", ("fancy/purelib.py", "purelib", 0)), + ("scripts", ("fancy/scripts.py", "scripts", 0)), + ("purelib", ("fancy/__init__.py", "purelib", 0)), + ("purelib", ("fancy-1.0.0.dist-info/METADATA", "purelib", 0)), + ("purelib", ("fancy-1.0.0.dist-info/WHEEL", "purelib", 0)), + ( + "purelib", + ("fancy-1.0.0.dist-info/entry_points.txt", "purelib", 0), + ), + ( + "purelib", + ("fancy-1.0.0.dist-info/top_level.txt", "purelib", 0), + ), + ( + "purelib", + RecordEntry("fancy-1.0.0.dist-info/RECORD", None, None), + ), + ], + ), + ] + ) + + def test_errors_out_when_given_invalid_scheme_in_data(self, mock_destination): + # Create a fake wheel + source = FakeWheelSource( + distribution="fancy", + version="1.0.0", + regular_files={ + "fancy/__init__.py": b"""\ + # put me in purelib + """, + "fancy-1.0.0.data/purelib/fancy/purelib.py": b"""\ + # put me in purelib + """, + "fancy-1.0.0.data/invalid/fancy/invalid.py": b"""\ + # i am invalid + """, + }, + dist_info_files={ + "top_level.txt": b"""\ + fancy + """, + "entry_points.txt": b"""\ + [console_scripts] + fancy = fancy:main + + [gui_scripts] + fancy-gui = fancy:main + """, + "WHEEL": b"""\ + Wheel-Version: 1.0 + Generator: magic (1.0.0) + Root-Is-Purelib: true + Tag: py3-none-any + """, + "METADATA": b"""\ + Metadata-Version: 2.1 + Name: fancy + Version: 1.0.0 + Summary: A fancy package + Author: Agendaless Consulting + Author-email: nobody@example.com + License: MIT + Keywords: fancy amazing + Platform: UNKNOWN + Classifier: Intended Audience :: Developers + """, + }, + ) + + with pytest.raises(InvalidWheelSource) as ctx: + install( + source=source, + destination=mock_destination, + additional_metadata={}, + ) + + assert "fancy-1.0.0.data/invalid/fancy/invalid.py" in str(ctx.value) + + def test_ensure_non_executable_for_additional_metadata(self, mock_destination): + # Create a fake wheel + source = FakeWheelSource( + distribution="fancy", + version="1.0.0", + regular_files={ + "fancy/__init__.py": b"""\ + # put me in purelib + """, + }, + dist_info_files={ + "top_level.txt": b"""\ + fancy + """, + "WHEEL": b"""\ + Wheel-Version: 1.0 + Generator: magic (1.0.0) + Root-Is-Purelib: true + Tag: py3-none-any + """, + "METADATA": b"""\ + Metadata-Version: 2.1 + Name: fancy + Version: 1.0.0 + Summary: A fancy package + Author: Agendaless Consulting + Author-email: nobody@example.com + License: MIT + Keywords: fancy amazing + Platform: UNKNOWN + Classifier: Intended Audience :: Developers + """, + }, + ) + all_contents = list(source.get_contents()) + source.get_contents = lambda: ( + (*contents, True) for (*contents, _) in all_contents + ) + install( + source=source, + destination=mock_destination, + additional_metadata={ + "fun_file.txt": b"this should be in dist-info!", + }, + ) + + mock_destination.assert_has_calls( + [ + mock.call.write_file( + scheme="purelib", + path="fancy/__init__.py", + stream=mock.ANY, + is_executable=True, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/METADATA", + stream=mock.ANY, + is_executable=True, + ), + mock.call.write_file( + scheme="purelib", + path="fancy-1.0.0.dist-info/fun_file.txt", + stream=mock.ANY, + is_executable=False, + ), + ], + any_order=True, + ) diff --git a/tests/test_destinations.py b/tests/test_destinations.py new file mode 100644 index 0000000..225b009 --- /dev/null +++ b/tests/test_destinations.py @@ -0,0 +1,173 @@ +import io +import os.path + +import pytest + +from installer.destinations import SchemeDictionaryDestination, WheelDestination +from installer.records import RecordEntry +from installer.scripts import Script +from installer.utils import SCHEME_NAMES + + +class TestWheelDestination: + def test_takes_no_arguments(self): + WheelDestination() + + def test_raises_not_implemented_error(self): + destination = WheelDestination() + + with pytest.raises(NotImplementedError): + destination.write_script(name=None, module=None, attr=None, section=None) + + with pytest.raises(NotImplementedError): + destination.write_file( + scheme=None, path=None, stream=None, is_executable=False + ) + + with pytest.raises(NotImplementedError): + destination.finalize_installation( + scheme=None, + record_file_path=None, + records=None, + ) + + +class TestSchemeDictionaryDestination: + @pytest.fixture() + def destination(self, tmp_path): + scheme_dict = {} + for scheme in SCHEME_NAMES: + full_path = tmp_path / scheme + if not full_path.exists(): + full_path.mkdir() + scheme_dict[scheme] = str(full_path) + return SchemeDictionaryDestination(scheme_dict, "/my/python", "posix") + + @pytest.mark.parametrize( + ("scheme", "path", "data", "expected"), + [ + pytest.param( + "data", "my_data.bin", b"my data", b"my data", id="normal file" + ), + pytest.param( + "data", + "data_folder/my_data.bin", + b"my data", + b"my data", + id="normal file in subfolder", + ), + pytest.param( + "scripts", + "my_script.py", + b"#!python\nmy script", + b"#!/my/python\nmy script", + id="script file", + ), + pytest.param( + "scripts", + "script_folder/my_script.py", + b"#!python\nmy script", + b"#!/my/python\nmy script", + id="script file in subfolder", + ), + ], + ) + def test_write_file(self, destination, scheme, path, data, expected): + record = destination.write_file(scheme, path, io.BytesIO(data), False) + file_path = os.path.join(destination.scheme_dict[scheme], path) + with open(file_path, "rb") as f: + file_data = f.read() + + assert file_data == expected + assert record.path == path + + def test_write_record_duplicate(self, destination): + destination.write_file("data", "my_data.bin", io.BytesIO(b"my data"), False) + with pytest.raises(FileExistsError): + destination.write_file("data", "my_data.bin", io.BytesIO(b"my data"), False) + + def test_write_script(self, destination): + script_args = ("my_entrypoint", "my_module", "my_function", "console") + record = destination.write_script(*script_args) + file_path = os.path.join(destination.scheme_dict["scripts"], "my_entrypoint") + + assert os.path.isfile(file_path) + + with open(file_path, "rb") as f: + file_data = f.read() + name, expected_data = Script(*script_args).generate("/my/python", "posix") + + assert file_data == expected_data + assert record.path == "my_entrypoint" + + def test_finalize_write_record(self, destination): + records = [ + ( + "data", + destination.write_file( + "data", + "my_data1.bin", + io.BytesIO(b"my data 1"), + is_executable=False, + ), + ), + ( + "data", + destination.write_file( + "data", + "my_data2.bin", + io.BytesIO(b"my data 2"), + is_executable=False, + ), + ), + ( + "data", + destination.write_file( + "data", + "my_data3,my_data4.bin", + io.BytesIO(b"my data 3"), + is_executable=False, + ), + ), + ( + "scripts", + destination.write_file( + "scripts", + "my_script", + io.BytesIO(b"my script"), + is_executable=True, + ), + ), + ( + "scripts", + destination.write_file( + "scripts", + "my_script2", + io.BytesIO(b"#!python\nmy script"), + is_executable=False, + ), + ), + ( + "scripts", + destination.write_script( + "my_entrypoint", "my_module", "my_function", "console" + ), + ), + ("purelib", RecordEntry("RECORD", None, None)), + ] + + destination.finalize_installation("purelib", "RECORD", records) + file_path = os.path.join(destination.scheme_dict["purelib"], "RECORD") + + with open(file_path, "rb") as f: + data = f.read() + + assert data == ( + b"../data/my_data1.bin,sha256=NV0A-M4OPuqTsHjeD6Wth_-UqrpAAAdyplcustFZ8s4,9\n" + b"../data/my_data2.bin,sha256=lP7V8oWLqgyXCbdASNiPdsUogzPUZhht_7F8T5bC3eQ,9\n" + b'"../data/my_data3,my_data4.bin",sha256=18krruu1gr01x-WM_9ChSASoHv0mfRAV6-B2bd9sxpo,9\n' + b"../scripts/my_script,sha256=M60fWvUSMJkPtw2apUvjWWwOcnRPcVy_zO4-4lpH08o,9\n" + b"../scripts/my_script2,sha256=k9_997kTbTYQm7EXFLclVZL1m2N98rU90QX46XeMvjY,22\n" + b"../scripts/my_entrypoint,sha256=_p_9nwmeIeoMBfQ0akhr1KbKn3laDydg0J7cy0Fs6JI,216\n" + b"RECORD,,\n" + ) diff --git a/tests/test_main.py b/tests/test_main.py new file mode 100644 index 0000000..391a13d --- /dev/null +++ b/tests/test_main.py @@ -0,0 +1,69 @@ +import os + +from installer.__main__ import _get_scheme_dict as get_scheme_dict +from installer.__main__ import _main as main + + +def test_get_scheme_dict(): + d = get_scheme_dict(distribution_name="foo") + assert set(d.keys()) >= {"purelib", "platlib", "headers", "scripts", "data"} + + +def test_get_scheme_dict_prefix(): + d = get_scheme_dict(distribution_name="foo", prefix="/foo") + for key in ("purelib", "platlib", "headers", "scripts", "data"): + assert d[key].startswith( + f"{os.sep}foo" + ), f"{key} does not start with /foo: {d[key]}" + + +def test_main(fancy_wheel, tmp_path): + destdir = tmp_path / "dest" + + main([str(fancy_wheel), "-d", str(destdir)], "python -m installer") + + installed_py_files = destdir.rglob("*.py") + + assert {f.stem for f in installed_py_files} == {"__init__", "__main__", "data"} + + installed_pyc_files = destdir.rglob("*.pyc") + assert {f.name.split(".")[0] for f in installed_pyc_files} == { + "__init__", + "__main__", + } + + +def test_main_prefix(fancy_wheel, tmp_path): + destdir = tmp_path / "dest" + + main([str(fancy_wheel), "-d", str(destdir), "-p", "/foo"], "python -m installer") + + installed_py_files = list(destdir.rglob("*.py")) + + for f in installed_py_files: + assert str(f.parent).startswith( + str(destdir / "foo") + ), f"path does not respect destdir+prefix: {f}" + assert {f.stem for f in installed_py_files} == {"__init__", "__main__", "data"} + + installed_pyc_files = destdir.rglob("*.pyc") + assert {f.name.split(".")[0] for f in installed_pyc_files} == { + "__init__", + "__main__", + } + + +def test_main_no_pyc(fancy_wheel, tmp_path): + destdir = tmp_path / "dest" + + main( + [str(fancy_wheel), "-d", str(destdir), "--no-compile-bytecode"], + "python -m installer", + ) + + installed_py_files = destdir.rglob("*.py") + + assert {f.stem for f in installed_py_files} == {"__init__", "__main__", "data"} + + installed_pyc_files = destdir.rglob("*.pyc") + assert set(installed_pyc_files) == set() diff --git a/tests/test_records.py b/tests/test_records.py new file mode 100644 index 0000000..45427ae --- /dev/null +++ b/tests/test_records.py @@ -0,0 +1,275 @@ +import pytest + +from installer.records import Hash, InvalidRecordEntry, RecordEntry, parse_record_file + + +# +# pytest fixture witchcraft +# +@pytest.fixture() +def record_simple_list(): + return [ + "file.py,sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI,3144", + "distribution-1.0.dist-info/RECORD,,", + ] + + +@pytest.fixture() +def record_simple_iter(record_simple_list): + return iter(record_simple_list) + + +@pytest.fixture() +def record_simple_file(tmpdir, record_simple_list): + p = tmpdir.join("RECORD") + p.write("\n".join(record_simple_list)) + with open(str(p)) as f: + yield f + + +@pytest.fixture() +def record_input(request): + return request.getfixturevalue(request.param) + + +SAMPLE_RECORDS = [ + ( + "purelib", + ("test1.py", "sha256=Y0sCextp4SQtQNU-MSs7SsdxD1W-gfKJtUlEbvZ3i-4", 6), + b"test1\n", + True, + ), + ( + "purelib", + ("test2.py", "sha256=fW_Xd08Nh2JNptzxbQ09EEwxkedx--LznIau1LK_Gg8", 6), + b"test2\n", + True, + ), + ( + "purelib", + ("test3.py", "sha256=qwPDTx7OCCEf4qgDn9ZCQZmz9de1X_E7ETSzZHdsRcU", 6), + b"test3\n", + True, + ), + ( + "purelib", + ("test4.py", "sha256=Y0sCextp4SQtQNU-MSs7SsdxD1W-gfKJtUlEbvZ3i-4", 7), + b"test1\n", + False, + ), + ( + "purelib", + ( + "test5.py", + "sha256=Y0sCextp4SQtQNU-MSs7SsdxD1W-gfKJtUlEbvZ3i-4", + None, + ), + b"test1\n", + True, + ), + ("purelib", ("test6.py", None, None), b"test1\n", True), +] + + +# +# Actual Tests +# +class TestRecordEntry: + @pytest.mark.parametrize( + "path, hash_, size, caused_by", + [ + ("", "", "", ["path"]), + ("", "", "non-int", ["path", "size"]), + ("a.py", "", "non-int", ["size"]), + # Notice that we're explicitly allowing non-compliant hash values + ("a.py", "some-random-value", "non-int", ["size"]), + ], + ) + def test_invalid_elements(self, path, hash_, size, caused_by): + with pytest.raises(InvalidRecordEntry) as exc_info: + RecordEntry.from_elements(path, hash_, size) + + assert exc_info.value.elements == (path, hash_, size) + for word in caused_by: + assert word in str(exc_info.value) + + @pytest.mark.parametrize( + "path, hash_, size", + [ + ("a.py", "", ""), + ("a.py", "", "3144"), + ("a.py", "sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI", ""), + ("a.py", "sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI", "3144"), + ], + ) + def test_valid_elements(self, path, hash_, size): + RecordEntry.from_elements(path, hash_, size) + + @pytest.mark.parametrize( + ("scheme", "elements", "data", "passes_validation"), SAMPLE_RECORDS + ) + def test_populates_attributes_correctly( + self, scheme, elements, data, passes_validation + ): + path, hash_string, size = elements + + record = RecordEntry.from_elements(path, hash_string, size) + + assert record.path == path + assert record.size == size + + if record.hash_ is not None: + assert isinstance(record.hash_, Hash) + assert record.hash_.name == "sha256" + assert record.hash_.value == hash_string[len("sha256=") :] + + @pytest.mark.parametrize( + ("scheme", "elements", "data", "passes_validation"), SAMPLE_RECORDS + ) + def test_validation(self, scheme, elements, data, passes_validation): + record = RecordEntry.from_elements(*elements) + assert record.validate(data) == passes_validation + + @pytest.mark.parametrize( + ("scheme", "elements", "data", "passes_validation"), SAMPLE_RECORDS + ) + def test_string_representation(self, scheme, elements, data, passes_validation): + record = RecordEntry.from_elements(*elements) + + expected_row = tuple( + [(str(elem) if elem is not None else "") for elem in elements] + ) + assert record.to_row() == expected_row + + @pytest.mark.parametrize( + ("scheme", "elements", "data", "passes_validation"), SAMPLE_RECORDS + ) + def test_string_representation_with_prefix( + self, scheme, elements, data, passes_validation + ): + record = RecordEntry.from_elements(*elements) + + expected_row = tuple( + [ + (str(elem) if elem is not None else "") + for elem in ("prefix/" + elements[0], elements[1], elements[2]) + ] + ) + assert record.to_row("prefix/") == expected_row + + def test_equality(self): + record = RecordEntry.from_elements( + "file.py", + "sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI", + "3144", + ) + record_same = RecordEntry.from_elements( + "file.py", + "sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI", + "3144", + ) + record_different_name = RecordEntry.from_elements( + "file2.py", + "sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI", + "3144", + ) + record_different_hash_name = RecordEntry.from_elements( + "file.py", + "md5=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI", + "3144", + ) + record_different_hash_value = RecordEntry.from_elements( + "file.py", + "sha256=qwertyuiodfdsflkgshdlkjghrefawrwerwffsdfflk29", + "3144", + ) + record_different_size = RecordEntry.from_elements( + "file.py", + "sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI", + "10", + ) + + assert record == record_same + + assert record != "random string" + assert record != record_different_name + assert record != record_different_hash_name + assert record != record_different_hash_value + assert record != record_different_size + + # Ensure equality is based on current state + record_same.hash_ = None + assert record != record_same + + +class TestParseRecordFile: + def test_accepts_empty_iterable(self): + list(parse_record_file([])) + + @pytest.mark.parametrize( + "record_input", + ["record_simple_list", "record_simple_iter", "record_simple_file"], + indirect=True, + ) + def test_accepts_all_kinds_of_iterables(self, record_input): + """Should accepts any iterable, e.g. container, iterator, or file object.""" + records = list(parse_record_file(record_input)) + assert len(records) == 2 + + assert records == [ + ( + "file.py", + "sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI", + "3144", + ), + ("distribution-1.0.dist-info/RECORD", "", ""), + ] + + @pytest.mark.parametrize( + "line, element_count", + [ + pytest.param( + "file.py,sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI,3144,", + 4, + id="four", + ), + pytest.param( + "distribution-1.0.dist-info/RECORD,,,,", + 5, + id="five", + ), + ], + ) + def test_rejects_wrong_element_count(self, line, element_count): + with pytest.raises(InvalidRecordEntry) as exc_info: + list(parse_record_file([line])) + + message = f"expected 3 elements, got {element_count}" + assert message in str(exc_info.value) + + def test_shows_correct_row_number(self): + record_lines = [ + "file1.py,sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI,3144", + "file2.py,sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI,3144", + "file3.py,sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI,3144", + "distribution-1.0.dist-info/RECORD,,,,", + ] + with pytest.raises(InvalidRecordEntry) as exc_info: + list(parse_record_file(record_lines)) + + assert "Row Index 3" in str(exc_info.value) + + def test_parse_record_entry_with_comma(self): + record_lines = [ + '"file1,file2.txt",sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI,3144', + "distribution-1.0.dist-info/RECORD,,", + ] + records = list(parse_record_file(record_lines)) + assert records == [ + ( + "file1,file2.txt", + "sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\\_pNh2yI", + "3144", + ), + ("distribution-1.0.dist-info/RECORD", "", ""), + ] diff --git a/tests/test_scripts.py b/tests/test_scripts.py new file mode 100644 index 0000000..2da6577 --- /dev/null +++ b/tests/test_scripts.py @@ -0,0 +1,70 @@ +import io +import os +import zipfile + +import pytest + +from installer import _scripts +from installer.scripts import InvalidScript, Script + + +def test_script_generate_simple(): + script = Script("foo", "foo.bar", "baz.qux", section="console") + name, data = script.generate("/path/to/my/python", kind="posix") + + assert name == "foo" + assert data.startswith(b"#!/path/to/my/python\n") + assert b"\nfrom foo.bar import baz\n" in data + assert b"baz.qux()" in data + + +def test_script_generate_space_in_executable(): + script = Script("foo", "foo.bar", "baz.qux", section="console") + name, data = script.generate("/path to my/python", kind="posix") + + assert name == "foo" + assert data.startswith(b"#!/bin/sh\n") + assert b" '/path to my/python'" in data + assert b"\nfrom foo.bar import baz\n" in data + assert b"baz.qux()" in data + + +def _read_launcher_data(section, kind): + prefix = {"console": "t", "gui": "w"}[section] + suffix = {"win-ia32": "32", "win-amd64": "64", "win-arm": "_arm"}[kind] + filename = os.path.join( + os.path.dirname(os.path.abspath(_scripts.__file__)), + f"{prefix}{suffix}.exe", + ) + with open(filename, "rb") as f: + return f.read() + + +@pytest.mark.parametrize("section", ["console", "gui"]) +@pytest.mark.parametrize("kind", ["win-ia32", "win-amd64", "win-arm"]) +def test_script_generate_launcher(section, kind): + launcher_data = _read_launcher_data(section, kind) + + script = Script("foo", "foo.bar", "baz.qux", section=section) + name, data = script.generate("#!C:\\path to my\\python.exe\n", kind=kind) + + prefix_len = len(launcher_data) + len(b"#!C:\\path to my\\python.exe\n") + stream = io.BytesIO(data[prefix_len:]) + with zipfile.ZipFile(stream) as zf: + code = zf.read("__main__.py") + + assert name == "foo.exe" + assert data.startswith(launcher_data) + assert b"#!C:\\path to my\\python.exe\n" in data + assert b"\nfrom foo.bar import baz\n" in code + assert b"baz.qux()" in code + + +@pytest.mark.parametrize( + "section, kind", + [("nonexist", "win-ia32"), ("console", "nonexist"), ("nonexist", "nonexist")], +) +def test_script_generate_launcher_error(section, kind): + script = Script("foo", "foo.bar", "baz.qux", section=section) + with pytest.raises(InvalidScript): + script.generate("#!C:\\path to my\\python.exe\n", kind=kind) diff --git a/tests/test_sources.py b/tests/test_sources.py new file mode 100644 index 0000000..a79cc24 --- /dev/null +++ b/tests/test_sources.py @@ -0,0 +1,94 @@ +import posixpath +import zipfile + +import pytest + +from installer.records import parse_record_file +from installer.sources import WheelFile, WheelSource + + +class TestWheelSource: + def test_takes_two_arguments(self): + WheelSource("distribution", "version") + WheelSource(distribution="distribution", version="version") + + def test_correctly_computes_properties(self): + source = WheelSource(distribution="distribution", version="version") + + assert source.data_dir == "distribution-version.data" + assert source.dist_info_dir == "distribution-version.dist-info" + + def test_raises_not_implemented_error(self): + source = WheelSource(distribution="distribution", version="version") + + with pytest.raises(NotImplementedError): + source.dist_info_filenames + + with pytest.raises(NotImplementedError): + source.read_dist_info("METADATA") + + with pytest.raises(NotImplementedError): + source.get_contents() + + +class TestWheelFile: + def test_rejects_not_okay_name(self, tmp_path): + # Create an empty zipfile + path = tmp_path / "not_a_valid_name.whl" + with zipfile.ZipFile(str(path), "w"): + pass + + with pytest.raises(ValueError, match="Not a valid wheel filename: .+"): + with WheelFile.open(str(path)): + pass + + def test_provides_correct_dist_info_filenames(self, fancy_wheel): + with WheelFile.open(fancy_wheel) as source: + assert sorted(source.dist_info_filenames) == [ + "METADATA", + "RECORD", + "WHEEL", + "entry_points.txt", + "top_level.txt", + ] + + def test_correctly_reads_from_dist_info_files(self, fancy_wheel): + files = {} + with zipfile.ZipFile(fancy_wheel) as archive: + for file in archive.namelist(): + if ".dist-info" not in file: + continue + files[posixpath.basename(file)] = archive.read(file).decode("utf-8") + + got_files = {} + with WheelFile.open(fancy_wheel) as source: + for file in files: + got_files[file] = source.read_dist_info(file) + + assert got_files == files + + def test_provides_correct_contents(self, fancy_wheel): + # Know the contents of the wheel + files = {} + with zipfile.ZipFile(fancy_wheel) as archive: + for file in archive.namelist(): + if file[-1:] == "/": + continue + files[file] = archive.read(file) + + expected_record_lines = ( + files["fancy-1.0.0.dist-info/RECORD"].decode("utf-8").splitlines() + ) + expected_records = list(parse_record_file(expected_record_lines)) + + # Check that the object's output is appropriate + got_records = [] + got_files = {} + with WheelFile.open(fancy_wheel) as source: + for record_elements, stream, is_executable in source.get_contents(): + got_records.append(record_elements) + got_files[record_elements[0]] = stream.read() + assert not is_executable + + assert sorted(got_records) == sorted(expected_records) + assert got_files == files diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..bfcc089 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,232 @@ +"""Tests for installer.utils +""" + +import base64 +import hashlib +import textwrap +from email.message import Message +from io import BytesIO + +import pytest +from test_records import SAMPLE_RECORDS + +from installer.records import RecordEntry +from installer.utils import ( + WheelFilename, + construct_record_file, + copyfileobj_with_hashing, + fix_shebang, + parse_entrypoints, + parse_metadata_file, + parse_wheel_filename, +) + + +class TestParseMetadata: + def test_basics(self): + result = parse_metadata_file( + textwrap.dedent( + """\ + Name: package + Version: 1.0.0 + Multi-Use-Field: 1 + Multi-Use-Field: 2 + Multi-Use-Field: 3 + """ + ) + ) + assert isinstance(result, Message) + assert result.get("Name") == "package" + assert result.get("version") == "1.0.0" + assert result.get_all("MULTI-USE-FIELD") == ["1", "2", "3"] + + +class TestParseWheelFilename: + @pytest.mark.parametrize( + "string, expected", + [ + # Crafted package name w/ a "complex" version and build tag + ( + "package-1!1.0+abc.7-753-py3-none-any.whl", + WheelFilename("package", "1!1.0+abc.7", "753", "py3-none-any"), + ), + # Crafted package name w/ a "complex" version and no build tag + ( + "package-1!1.0+abc.7-py3-none-any.whl", + WheelFilename("package", "1!1.0+abc.7", None, "py3-none-any"), + ), + # Use real tensorflow wheel names + ( + "tensorflow-2.3.0-cp38-cp38-macosx_10_11_x86_64.whl", + WheelFilename( + "tensorflow", "2.3.0", None, "cp38-cp38-macosx_10_11_x86_64" + ), + ), + ( + "tensorflow-2.3.0-cp38-cp38-manylinux2010_x86_64.whl", + WheelFilename( + "tensorflow", "2.3.0", None, "cp38-cp38-manylinux2010_x86_64" + ), + ), + ( + "tensorflow-2.3.0-cp38-cp38-win_amd64.whl", + WheelFilename("tensorflow", "2.3.0", None, "cp38-cp38-win_amd64"), + ), + ], + ) + def test_valid_cases(self, string, expected): + got = parse_wheel_filename(string) + assert expected == got, (expected, got) + + @pytest.mark.parametrize( + "string", + [ + # Not ".whl" + "pip-20.0.0-py2.py3-none-any.zip", + # No tag + "pip-20.0.0.whl", + # Empty tag + "pip-20.0.0---.whl", + ], + ) + def test_invalid_cases(self, string): + with pytest.raises(ValueError): + parse_wheel_filename(string) + + +class TestCopyFileObjWithHashing: + def test_basic_functionality(self): + data = b"input data is this" + hash_ = ( + base64.urlsafe_b64encode(hashlib.sha256(data).digest()) + .decode("ascii") + .rstrip("=") + ) + size = len(data) + + with BytesIO(data) as source: + with BytesIO() as dest: + result = copyfileobj_with_hashing(source, dest, hash_algorithm="sha256") + written_data = dest.getvalue() + + assert result == (hash_, size) + assert written_data == data + + +class TestScript: + @pytest.mark.parametrize( + ("data", "expected"), + [ + pytest.param( + b"#!python\ntest", + b"#!/my/python\ntest", + id="python", + ), + pytest.param( + b"#!pythonw\ntest", + b"#!/my/python\ntest", + id="pythonw", + ), + pytest.param( + b"#!python something\ntest", + b"#!/my/python\ntest", + id="python-with-args", + ), + pytest.param( + b"#!python", + b"#!/my/python\n", + id="python-no-content", + ), + ], + ) + def test_replace_shebang(self, data, expected): + with BytesIO(data) as source: + with fix_shebang(source, "/my/python") as stream: + result = stream.read() + assert result == expected + + @pytest.mark.parametrize( + "data", + [ + b"#!py\ntest", + b"#!something\ntest", + b"#something\ntest", + b"#something", + b"something", + ], + ) + def test_keep_data(self, data): + with BytesIO(data) as source: + with fix_shebang(source, "/my/python") as stream: + result = stream.read() + assert result == data + + +class TestConstructRecord: + def test_construct(self): + records = [ + (scheme, RecordEntry.from_elements(*elements)) + for scheme, elements, _, _ in SAMPLE_RECORDS + ] + assert construct_record_file(records).read() == ( + b"test1.py,sha256=Y0sCextp4SQtQNU-MSs7SsdxD1W-gfKJtUlEbvZ3i-4,6\n" + b"test2.py,sha256=fW_Xd08Nh2JNptzxbQ09EEwxkedx--LznIau1LK_Gg8,6\n" + b"test3.py,sha256=qwPDTx7OCCEf4qgDn9ZCQZmz9de1X_E7ETSzZHdsRcU,6\n" + b"test4.py,sha256=Y0sCextp4SQtQNU-MSs7SsdxD1W-gfKJtUlEbvZ3i-4,7\n" + b"test5.py,sha256=Y0sCextp4SQtQNU-MSs7SsdxD1W-gfKJtUlEbvZ3i-4,\n" + b"test6.py,,\n" + ) + + +class TestParseEntryPoints: + @pytest.mark.parametrize( + ("script", "expected"), + [ + pytest.param("", [], id="empty"), + pytest.param( + """ + [foo] + foo = foo.bar + """, + [], + id="unrelated", + ), + pytest.param( + """ + [console_scripts] + package = package.__main__:package + """, + [ + ("package", "package.__main__", "package", "console"), + ], + id="cli", + ), + pytest.param( + """ + [gui_scripts] + package = package.__main__:package + """, + [ + ("package", "package.__main__", "package", "gui"), + ], + id="gui", + ), + pytest.param( + """ + [console_scripts] + magic-cli = magic.cli:main + + [gui_scripts] + magic-gui = magic.gui:main + """, + [ + ("magic-cli", "magic.cli", "main", "console"), + ("magic-gui", "magic.gui", "main", "gui"), + ], + id="cli-and-gui", + ), + ], + ) + def test_valid(self, script, expected): + iterable = parse_entrypoints(textwrap.dedent(script)) + assert list(iterable) == expected, expected diff --git a/tools/update_launchers.py b/tools/update_launchers.py new file mode 100644 index 0000000..9411f30 --- /dev/null +++ b/tools/update_launchers.py @@ -0,0 +1,48 @@ +import asyncio +import pathlib +import sys + +import httpx + +DOWNLOAD_URL = "https://bitbucket.org/vinay.sajip/simple_launcher/downloads/{}" +VENDOR_DIR = ( + pathlib.Path(__file__) + .parent.parent.joinpath("src", "installer", "_scripts") + .resolve() +) + +LAUNCHERS = [ + "t32.exe", + "t64.exe", + "t_arm.exe", + "t64-arm.exe", + "w32.exe", + "w64.exe", + "w_arm.exe", + "w64-arm.exe", +] + + +async def _download(client: httpx.AsyncClient, name): + url = DOWNLOAD_URL.format(name) + print(f" Fetching {url}") + resp = await client.get(url) + data = await resp.aread() + VENDOR_DIR.joinpath(name).write_bytes(data) + + +async def main(): + print(f"Downloading into {VENDOR_DIR} ...") + async with httpx.AsyncClient() as client: + await asyncio.gather(*(_download(client, name) for name in LAUNCHERS)) + + +def _patch_windows_38(): + # https://github.com/encode/httpx/issues/914 + if sys.version_info >= (3, 8) and sys.platform.startswith("win"): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + + +if __name__ == "__main__": + _patch_windows_38() + asyncio.run(main()) |