From 43a97878ce14b72f0981164f87f2e35e14151312 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 11:22:09 +0200 Subject: Adding upstream version 110.0.1. Signed-off-by: Daniel Baumann --- .../tests/tools/third_party/pytest/.coveragerc | 31 + .../tests/tools/third_party/pytest/.gitblameignore | 28 + .../tools/third_party/pytest/.github/FUNDING.yml | 5 + .../pytest/.github/ISSUE_TEMPLATE/1_bug_report.md | 16 + .../.github/ISSUE_TEMPLATE/2_feature_request.md | 25 + .../pytest/.github/ISSUE_TEMPLATE/config.yml | 5 + .../pytest/.github/PULL_REQUEST_TEMPLATE.md | 26 + .../tools/third_party/pytest/.github/config.yml | 2 + .../third_party/pytest/.github/dependabot.yml | 11 + .../tools/third_party/pytest/.github/labels.toml | 149 + .../third_party/pytest/.github/workflows/main.yml | 231 + .../.github/workflows/prepare-release-pr.yml | 52 + .../.github/workflows/update-plugin-list.yml | 49 + .../tests/tools/third_party/pytest/.gitignore | 58 + .../third_party/pytest/.pre-commit-config.yaml | 99 + .../tools/third_party/pytest/.readthedocs.yml | 19 + .../tests/tools/third_party/pytest/AUTHORS | 356 + .../tests/tools/third_party/pytest/CHANGELOG.rst | 7 + .../tests/tools/third_party/pytest/CITATION | 16 + .../tools/third_party/pytest/CODE_OF_CONDUCT.md | 83 + .../tools/third_party/pytest/CONTRIBUTING.rst | 481 ++ .../tests/tools/third_party/pytest/LICENSE | 21 + .../tools/third_party/pytest/OPENCOLLECTIVE.rst | 44 + .../tests/tools/third_party/pytest/README.rst | 167 + .../tests/tools/third_party/pytest/RELEASING.rst | 173 + .../tests/tools/third_party/pytest/TIDELIFT.rst | 60 + .../tests/tools/third_party/pytest/bench/bench.py | 13 + .../third_party/pytest/bench/bench_argcomplete.py | 19 + .../tests/tools/third_party/pytest/bench/empty.py | 2 + .../tools/third_party/pytest/bench/manyparam.py | 14 + .../tests/tools/third_party/pytest/bench/skip.py | 9 + .../tools/third_party/pytest/bench/unit_test.py | 13 + .../tests/tools/third_party/pytest/bench/xunit.py | 11 + .../tools/third_party/pytest/changelog/README.rst | 37 + .../third_party/pytest/changelog/_template.rst | 40 + .../tests/tools/third_party/pytest/codecov.yml | 6 + .../tests/tools/third_party/pytest/doc/en/Makefile | 43 + .../pytest/doc/en/_templates/globaltoc.html | 34 + .../pytest/doc/en/_templates/layout.html | 52 + .../pytest/doc/en/_templates/links.html | 7 + .../pytest/doc/en/_templates/relations.html | 19 + .../pytest/doc/en/_templates/sidebarintro.html | 5 + .../pytest/doc/en/_templates/slim_searchbox.html | 15 + .../tools/third_party/pytest/doc/en/adopt.rst | 78 + .../third_party/pytest/doc/en/announce/index.rst | 154 + .../pytest/doc/en/announce/release-2.0.0.rst | 129 + .../pytest/doc/en/announce/release-2.0.1.rst | 67 + .../pytest/doc/en/announce/release-2.0.2.rst | 73 + .../pytest/doc/en/announce/release-2.0.3.rst | 39 + .../pytest/doc/en/announce/release-2.1.0.rst | 47 + .../pytest/doc/en/announce/release-2.1.1.rst | 36 + .../pytest/doc/en/announce/release-2.1.2.rst | 32 + .../pytest/doc/en/announce/release-2.1.3.rst | 32 + .../pytest/doc/en/announce/release-2.2.0.rst | 95 + .../pytest/doc/en/announce/release-2.2.1.rst | 41 + .../pytest/doc/en/announce/release-2.2.2.rst | 43 + .../pytest/doc/en/announce/release-2.2.4.rst | 38 + .../pytest/doc/en/announce/release-2.3.0.rst | 133 + .../pytest/doc/en/announce/release-2.3.1.rst | 39 + .../pytest/doc/en/announce/release-2.3.2.rst | 57 + .../pytest/doc/en/announce/release-2.3.3.rst | 61 + .../pytest/doc/en/announce/release-2.3.4.rst | 39 + .../pytest/doc/en/announce/release-2.3.5.rst | 96 + .../pytest/doc/en/announce/release-2.4.0.rst | 223 + .../pytest/doc/en/announce/release-2.4.1.rst | 25 + .../pytest/doc/en/announce/release-2.4.2.rst | 39 + .../pytest/doc/en/announce/release-2.5.0.rst | 174 + .../pytest/doc/en/announce/release-2.5.1.rst | 46 + .../pytest/doc/en/announce/release-2.5.2.rst | 63 + .../pytest/doc/en/announce/release-2.6.0.rst | 153 + .../pytest/doc/en/announce/release-2.6.1.rst | 58 + .../pytest/doc/en/announce/release-2.6.2.rst | 51 + .../pytest/doc/en/announce/release-2.6.3.rst | 51 + .../pytest/doc/en/announce/release-2.7.0.rst | 100 + .../pytest/doc/en/announce/release-2.7.1.rst | 58 + .../pytest/doc/en/announce/release-2.7.2.rst | 57 + .../pytest/doc/en/announce/release-2.8.2.rst | 44 + .../pytest/doc/en/announce/release-2.8.3.rst | 58 + .../pytest/doc/en/announce/release-2.8.4.rst | 52 + .../pytest/doc/en/announce/release-2.8.5.rst | 39 + .../pytest/doc/en/announce/release-2.8.6.rst | 67 + .../pytest/doc/en/announce/release-2.8.7.rst | 31 + .../pytest/doc/en/announce/release-2.9.0.rst | 134 + .../pytest/doc/en/announce/release-2.9.1.rst | 57 + .../pytest/doc/en/announce/release-2.9.2.rst | 65 + .../pytest/doc/en/announce/release-3.0.0.rst | 82 + .../pytest/doc/en/announce/release-3.0.1.rst | 26 + .../pytest/doc/en/announce/release-3.0.2.rst | 24 + .../pytest/doc/en/announce/release-3.0.3.rst | 27 + .../pytest/doc/en/announce/release-3.0.4.rst | 29 + .../pytest/doc/en/announce/release-3.0.5.rst | 27 + .../pytest/doc/en/announce/release-3.0.6.rst | 33 + .../pytest/doc/en/announce/release-3.0.7.rst | 33 + .../pytest/doc/en/announce/release-3.1.0.rst | 61 + .../pytest/doc/en/announce/release-3.1.1.rst | 23 + .../pytest/doc/en/announce/release-3.1.2.rst | 23 + .../pytest/doc/en/announce/release-3.1.3.rst | 23 + .../pytest/doc/en/announce/release-3.10.0.rst | 43 + .../pytest/doc/en/announce/release-3.10.1.rst | 24 + .../pytest/doc/en/announce/release-3.2.0.rst | 48 + .../pytest/doc/en/announce/release-3.2.1.rst | 22 + .../pytest/doc/en/announce/release-3.2.2.rst | 28 + .../pytest/doc/en/announce/release-3.2.3.rst | 23 + .../pytest/doc/en/announce/release-3.2.4.rst | 36 + .../pytest/doc/en/announce/release-3.2.5.rst | 18 + .../pytest/doc/en/announce/release-3.3.0.rst | 50 + .../pytest/doc/en/announce/release-3.3.1.rst | 25 + .../pytest/doc/en/announce/release-3.3.2.rst | 28 + .../pytest/doc/en/announce/release-3.4.0.rst | 52 + .../pytest/doc/en/announce/release-3.4.1.rst | 27 + .../pytest/doc/en/announce/release-3.4.2.rst | 28 + .../pytest/doc/en/announce/release-3.5.0.rst | 51 + .../pytest/doc/en/announce/release-3.5.1.rst | 30 + .../pytest/doc/en/announce/release-3.6.0.rst | 41 + .../pytest/doc/en/announce/release-3.6.1.rst | 24 + .../pytest/doc/en/announce/release-3.6.2.rst | 29 + .../pytest/doc/en/announce/release-3.6.3.rst | 27 + .../pytest/doc/en/announce/release-3.6.4.rst | 24 + .../pytest/doc/en/announce/release-3.7.0.rst | 41 + .../pytest/doc/en/announce/release-3.7.1.rst | 21 + .../pytest/doc/en/announce/release-3.7.2.rst | 25 + .../pytest/doc/en/announce/release-3.7.3.rst | 32 + .../pytest/doc/en/announce/release-3.7.4.rst | 22 + .../pytest/doc/en/announce/release-3.8.0.rst | 38 + .../pytest/doc/en/announce/release-3.8.1.rst | 25 + .../pytest/doc/en/announce/release-3.8.2.rst | 28 + .../pytest/doc/en/announce/release-3.9.0.rst | 43 + .../pytest/doc/en/announce/release-3.9.1.rst | 20 + .../pytest/doc/en/announce/release-3.9.2.rst | 23 + .../pytest/doc/en/announce/release-3.9.3.rst | 24 + .../pytest/doc/en/announce/release-4.0.0.rst | 30 + .../pytest/doc/en/announce/release-4.0.1.rst | 23 + .../pytest/doc/en/announce/release-4.0.2.rst | 24 + .../pytest/doc/en/announce/release-4.1.0.rst | 44 + .../pytest/doc/en/announce/release-4.1.1.rst | 27 + .../pytest/doc/en/announce/release-4.2.0.rst | 37 + .../pytest/doc/en/announce/release-4.2.1.rst | 30 + .../pytest/doc/en/announce/release-4.3.0.rst | 36 + .../pytest/doc/en/announce/release-4.3.1.rst | 28 + .../pytest/doc/en/announce/release-4.4.0.rst | 39 + .../pytest/doc/en/announce/release-4.4.1.rst | 20 + .../pytest/doc/en/announce/release-4.4.2.rst | 33 + .../pytest/doc/en/announce/release-4.5.0.rst | 34 + .../pytest/doc/en/announce/release-4.6.0.rst | 43 + .../pytest/doc/en/announce/release-4.6.1.rst | 19 + .../pytest/doc/en/announce/release-4.6.2.rst | 18 + .../pytest/doc/en/announce/release-4.6.3.rst | 21 + .../pytest/doc/en/announce/release-4.6.4.rst | 22 + .../pytest/doc/en/announce/release-4.6.5.rst | 21 + .../pytest/doc/en/announce/release-4.6.6.rst | 20 + .../pytest/doc/en/announce/release-4.6.7.rst | 19 + .../pytest/doc/en/announce/release-4.6.8.rst | 20 + .../pytest/doc/en/announce/release-4.6.9.rst | 21 + .../pytest/doc/en/announce/release-5.0.0.rst | 46 + .../pytest/doc/en/announce/release-5.0.1.rst | 25 + .../pytest/doc/en/announce/release-5.1.0.rst | 56 + .../pytest/doc/en/announce/release-5.1.1.rst | 24 + .../pytest/doc/en/announce/release-5.1.2.rst | 23 + .../pytest/doc/en/announce/release-5.1.3.rst | 23 + .../pytest/doc/en/announce/release-5.2.0.rst | 35 + .../pytest/doc/en/announce/release-5.2.1.rst | 23 + .../pytest/doc/en/announce/release-5.2.2.rst | 29 + .../pytest/doc/en/announce/release-5.2.3.rst | 28 + .../pytest/doc/en/announce/release-5.2.4.rst | 22 + .../pytest/doc/en/announce/release-5.3.0.rst | 45 + .../pytest/doc/en/announce/release-5.3.1.rst | 26 + .../pytest/doc/en/announce/release-5.3.2.rst | 26 + .../pytest/doc/en/announce/release-5.3.3.rst | 30 + .../pytest/doc/en/announce/release-5.3.4.rst | 20 + .../pytest/doc/en/announce/release-5.3.5.rst | 19 + .../pytest/doc/en/announce/release-5.4.0.rst | 59 + .../pytest/doc/en/announce/release-5.4.1.rst | 18 + .../pytest/doc/en/announce/release-5.4.2.rst | 22 + .../pytest/doc/en/announce/release-5.4.3.rst | 21 + .../pytest/doc/en/announce/release-6.0.0.rst | 40 + .../pytest/doc/en/announce/release-6.0.0rc1.rst | 67 + .../pytest/doc/en/announce/release-6.0.1.rst | 21 + .../pytest/doc/en/announce/release-6.0.2.rst | 19 + .../pytest/doc/en/announce/release-6.1.0.rst | 44 + .../pytest/doc/en/announce/release-6.1.1.rst | 18 + .../pytest/doc/en/announce/release-6.1.2.rst | 22 + .../pytest/doc/en/announce/release-6.2.0.rst | 76 + .../pytest/doc/en/announce/release-6.2.1.rst | 20 + .../pytest/doc/en/announce/release-6.2.2.rst | 21 + .../pytest/doc/en/announce/release-6.2.3.rst | 19 + .../pytest/doc/en/announce/release-6.2.4.rst | 22 + .../pytest/doc/en/announce/release-6.2.5.rst | 30 + .../pytest/doc/en/announce/release-7.0.0.rst | 74 + .../pytest/doc/en/announce/release-7.0.0rc1.rst | 74 + .../pytest/doc/en/announce/release-7.0.1.rst | 20 + .../pytest/doc/en/announce/sprint2016.rst | 64 + .../pytest/doc/en/backwards-compatibility.rst | 79 + .../tools/third_party/pytest/doc/en/builtin.rst | 197 + .../tools/third_party/pytest/doc/en/changelog.rst | 9044 ++++++++++++++++++++ .../tests/tools/third_party/pytest/doc/en/conf.py | 478 ++ .../tools/third_party/pytest/doc/en/conftest.py | 1 + .../tools/third_party/pytest/doc/en/contact.rst | 54 + .../tools/third_party/pytest/doc/en/contents.rst | 116 + .../third_party/pytest/doc/en/contributing.rst | 3 + .../third_party/pytest/doc/en/deprecations.rst | 954 +++ .../pytest/doc/en/development_guide.rst | 7 + .../doc/en/example/assertion/failure_demo.py | 281 + .../assertion/global_testmodule_config/conftest.py | 14 + .../global_testmodule_config/test_hello_world.py | 5 + .../doc/en/example/assertion/test_failures.py | 13 + .../example/assertion/test_setup_flow_example.py | 44 + .../third_party/pytest/doc/en/example/attic.rst | 83 + .../third_party/pytest/doc/en/example/conftest.py | 1 + .../en/example/fixtures/fixture_availability.svg | 132 + .../fixtures/fixture_availability_plugins.svg | 142 + .../fixtures/test_fixtures_order_autouse.py | 45 + .../fixtures/test_fixtures_order_autouse.svg | 64 + .../fixtures/test_fixtures_order_autouse_flat.svg | 56 + .../test_fixtures_order_autouse_multiple_scopes.py | 31 + ...test_fixtures_order_autouse_multiple_scopes.svg | 76 + .../test_fixtures_order_autouse_temp_effects.py | 36 + .../test_fixtures_order_autouse_temp_effects.svg | 100 + .../fixtures/test_fixtures_order_dependencies.py | 45 + .../fixtures/test_fixtures_order_dependencies.svg | 60 + .../test_fixtures_order_dependencies_flat.svg | 51 + .../test_fixtures_order_dependencies_unclear.svg | 60 + .../example/fixtures/test_fixtures_order_scope.py | 36 + .../example/fixtures/test_fixtures_order_scope.svg | 55 + .../test_fixtures_request_different_scope.py | 29 + .../test_fixtures_request_different_scope.svg | 115 + .../third_party/pytest/doc/en/example/index.rst | 34 + .../third_party/pytest/doc/en/example/markers.rst | 734 ++ .../pytest/doc/en/example/multipython.py | 72 + .../pytest/doc/en/example/nonpython.rst | 102 + .../pytest/doc/en/example/nonpython/__init__.py | 0 .../pytest/doc/en/example/nonpython/conftest.py | 47 + .../doc/en/example/nonpython/test_simple.yaml | 7 + .../pytest/doc/en/example/parametrize.rst | 708 ++ .../pytest/doc/en/example/pythoncollection.py | 14 + .../pytest/doc/en/example/pythoncollection.rst | 321 + .../pytest/doc/en/example/reportingdemo.rst | 708 ++ .../third_party/pytest/doc/en/example/simple.rst | 1086 +++ .../third_party/pytest/doc/en/example/special.rst | 84 + .../pytest/doc/en/example/xfail_demo.py | 38 + .../pytest/doc/en/explanation/anatomy.rst | 46 + .../pytest/doc/en/explanation/fixtures.rst | 174 + .../pytest/doc/en/explanation/flaky.rst | 126 + .../pytest/doc/en/explanation/goodpractices.rst | 288 + .../pytest/doc/en/explanation/index.rst | 15 + .../pytest/doc/en/explanation/pythonpath.rst | 133 + .../third_party/pytest/doc/en/funcarg_compare.rst | 230 + .../tools/third_party/pytest/doc/en/funcargs.rst | 13 + .../third_party/pytest/doc/en/getting-started.rst | 257 + .../third_party/pytest/doc/en/historical-notes.rst | 312 + .../tools/third_party/pytest/doc/en/history.rst | 145 + .../third_party/pytest/doc/en/how-to/assert.rst | 336 + .../pytest/doc/en/how-to/bash-completion.rst | 33 + .../third_party/pytest/doc/en/how-to/cache.rst | 329 + .../pytest/doc/en/how-to/capture-stdout-stderr.rst | 170 + .../pytest/doc/en/how-to/capture-warnings.rst | 443 + .../third_party/pytest/doc/en/how-to/doctest.rst | 312 + .../pytest/doc/en/how-to/existingtestsuite.rst | 34 + .../third_party/pytest/doc/en/how-to/failures.rst | 160 + .../third_party/pytest/doc/en/how-to/fixtures.rst | 1887 ++++ .../third_party/pytest/doc/en/how-to/index.rst | 64 + .../third_party/pytest/doc/en/how-to/logging.rst | 292 + .../third_party/pytest/doc/en/how-to/mark.rst | 93 + .../pytest/doc/en/how-to/monkeypatch.rst | 444 + .../third_party/pytest/doc/en/how-to/nose.rst | 79 + .../third_party/pytest/doc/en/how-to/output.rst | 710 ++ .../pytest/doc/en/how-to/parametrize.rst | 298 + .../third_party/pytest/doc/en/how-to/plugins.rst | 136 + .../third_party/pytest/doc/en/how-to/skipping.rst | 430 + .../third_party/pytest/doc/en/how-to/tmp_path.rst | 139 + .../third_party/pytest/doc/en/how-to/unittest.rst | 251 + .../third_party/pytest/doc/en/how-to/usage.rst | 214 + .../doc/en/how-to/writing_hook_functions.rst | 352 + .../pytest/doc/en/how-to/writing_plugins.rst | 458 + .../pytest/doc/en/how-to/xunit_setup.rst | 117 + .../third_party/pytest/doc/en/img/cramer2.png | Bin 0 -> 25291 bytes .../third_party/pytest/doc/en/img/favicon.png | Bin 0 -> 1334 bytes .../third_party/pytest/doc/en/img/freiburg2.jpg | Bin 0 -> 104057 bytes .../third_party/pytest/doc/en/img/gaynor3.png | Bin 0 -> 23032 bytes .../third_party/pytest/doc/en/img/keleshev.png | Bin 0 -> 23246 bytes .../third_party/pytest/doc/en/img/pullrequest.png | Bin 0 -> 17035 bytes .../tools/third_party/pytest/doc/en/img/pylib.png | Bin 0 -> 8276 bytes .../third_party/pytest/doc/en/img/pytest1.png | Bin 0 -> 6010 bytes .../pytest/doc/en/img/pytest_logo_curves.svg | 29 + .../tools/third_party/pytest/doc/en/img/theuni.png | Bin 0 -> 31476 bytes .../tools/third_party/pytest/doc/en/index.rst | 148 + .../tools/third_party/pytest/doc/en/license.rst | 32 + .../tools/third_party/pytest/doc/en/naming20.rst | 20 + .../doc/en/proposals/parametrize_with_fixtures.rst | 164 + .../pytest/doc/en/py27-py34-deprecation.rst | 99 + .../tools/third_party/pytest/doc/en/pytest.ini | 2 + .../tools/third_party/pytest/doc/en/recwarn.rst | 3 + .../pytest/doc/en/reference/customize.rst | 248 + .../pytest/doc/en/reference/exit-codes.rst | 26 + .../pytest/doc/en/reference/fixtures.rst | 455 + .../third_party/pytest/doc/en/reference/index.rst | 15 + .../pytest/doc/en/reference/plugin_list.rst | 7728 +++++++++++++++++ .../pytest/doc/en/reference/reference.rst | 2101 +++++ .../third_party/pytest/doc/en/requirements.txt | 7 + .../tools/third_party/pytest/doc/en/sponsor.rst | 26 + .../tools/third_party/pytest/doc/en/talks.rst | 109 + .../tools/third_party/pytest/doc/en/tidelift.rst | 45 + .../third_party/pytest/doc/en/yieldfixture.rst | 18 + .../tools/third_party/pytest/extra/get_issues.py | 85 + .../pytest/extra/setup-py.test/setup.py | 11 + .../tests/tools/third_party/pytest/pyproject.toml | 116 + .../pytest/scripts/prepare-release-pr.py | 174 + .../pytest/scripts/publish-gh-release-notes.py | 102 + .../third_party/pytest/scripts/release.major.rst | 24 + .../third_party/pytest/scripts/release.minor.rst | 24 + .../third_party/pytest/scripts/release.patch.rst | 17 + .../third_party/pytest/scripts/release.pre.rst | 29 + .../tools/third_party/pytest/scripts/release.py | 131 + .../pytest/scripts/towncrier-draft-to-file.py | 15 + .../pytest/scripts/update-plugin-list.py | 140 + .../tests/tools/third_party/pytest/setup.cfg | 105 + .../tests/tools/third_party/pytest/setup.py | 4 + .../third_party/pytest/src/_pytest/__init__.py | 9 + .../third_party/pytest/src/_pytest/_argcomplete.py | 117 + .../pytest/src/_pytest/_code/__init__.py | 22 + .../third_party/pytest/src/_pytest/_code/code.py | 1274 +++ .../third_party/pytest/src/_pytest/_code/source.py | 217 + .../third_party/pytest/src/_pytest/_io/__init__.py | 8 + .../third_party/pytest/src/_pytest/_io/saferepr.py | 153 + .../pytest/src/_pytest/_io/terminalwriter.py | 233 + .../third_party/pytest/src/_pytest/_io/wcwidth.py | 55 + .../third_party/pytest/src/_pytest/_version.py | 5 + .../pytest/src/_pytest/assertion/__init__.py | 181 + .../pytest/src/_pytest/assertion/rewrite.py | 1136 +++ .../pytest/src/_pytest/assertion/truncate.py | 94 + .../pytest/src/_pytest/assertion/util.py | 498 ++ .../pytest/src/_pytest/cacheprovider.py | 580 ++ .../third_party/pytest/src/_pytest/capture.py | 942 ++ .../tools/third_party/pytest/src/_pytest/compat.py | 417 + .../pytest/src/_pytest/config/__init__.py | 1697 ++++ .../pytest/src/_pytest/config/argparsing.py | 535 ++ .../pytest/src/_pytest/config/compat.py | 71 + .../pytest/src/_pytest/config/exceptions.py | 11 + .../pytest/src/_pytest/config/findpaths.py | 213 + .../third_party/pytest/src/_pytest/debugging.py | 388 + .../third_party/pytest/src/_pytest/deprecated.py | 155 + .../third_party/pytest/src/_pytest/doctest.py | 734 ++ .../third_party/pytest/src/_pytest/faulthandler.py | 97 + .../third_party/pytest/src/_pytest/fixtures.py | 1686 ++++ .../pytest/src/_pytest/freeze_support.py | 44 + .../third_party/pytest/src/_pytest/helpconfig.py | 264 + .../third_party/pytest/src/_pytest/hookspec.py | 928 ++ .../third_party/pytest/src/_pytest/junitxml.py | 696 ++ .../third_party/pytest/src/_pytest/legacypath.py | 467 + .../third_party/pytest/src/_pytest/logging.py | 831 ++ .../tools/third_party/pytest/src/_pytest/main.py | 896 ++ .../pytest/src/_pytest/mark/__init__.py | 282 + .../pytest/src/_pytest/mark/expression.py | 225 + .../pytest/src/_pytest/mark/structures.py | 595 ++ .../third_party/pytest/src/_pytest/monkeypatch.py | 383 + .../tools/third_party/pytest/src/_pytest/nodes.py | 762 ++ .../tools/third_party/pytest/src/_pytest/nose.py | 42 + .../third_party/pytest/src/_pytest/outcomes.py | 307 + .../third_party/pytest/src/_pytest/pastebin.py | 110 + .../third_party/pytest/src/_pytest/pathlib.py | 724 ++ .../tools/third_party/pytest/src/_pytest/py.typed | 0 .../third_party/pytest/src/_pytest/pytester.py | 1748 ++++ .../pytest/src/_pytest/pytester_assertions.py | 75 + .../tools/third_party/pytest/src/_pytest/python.py | 1764 ++++ .../third_party/pytest/src/_pytest/python_api.py | 961 +++ .../third_party/pytest/src/_pytest/python_path.py | 24 + .../third_party/pytest/src/_pytest/recwarn.py | 296 + .../third_party/pytest/src/_pytest/reports.py | 598 ++ .../tools/third_party/pytest/src/_pytest/runner.py | 548 ++ .../tools/third_party/pytest/src/_pytest/scope.py | 91 + .../third_party/pytest/src/_pytest/setuponly.py | 97 + .../third_party/pytest/src/_pytest/setupplan.py | 40 + .../third_party/pytest/src/_pytest/skipping.py | 296 + .../tools/third_party/pytest/src/_pytest/stash.py | 112 + .../third_party/pytest/src/_pytest/stepwise.py | 122 + .../third_party/pytest/src/_pytest/terminal.py | 1394 +++ .../pytest/src/_pytest/threadexception.py | 88 + .../tools/third_party/pytest/src/_pytest/timing.py | 12 + .../tools/third_party/pytest/src/_pytest/tmpdir.py | 211 + .../third_party/pytest/src/_pytest/unittest.py | 414 + .../pytest/src/_pytest/unraisableexception.py | 93 + .../pytest/src/_pytest/warning_types.py | 145 + .../third_party/pytest/src/_pytest/warnings.py | 141 + .../third_party/pytest/src/pytest/__init__.py | 171 + .../third_party/pytest/src/pytest/__main__.py | 5 + .../tools/third_party/pytest/src/pytest/collect.py | 38 + .../tools/third_party/pytest/src/pytest/py.typed | 0 .../third_party/pytest/testing/acceptance_test.py | 1297 +++ .../third_party/pytest/testing/code/test_code.py | 212 + .../pytest/testing/code/test_excinfo.py | 1470 ++++ .../third_party/pytest/testing/code/test_source.py | 656 ++ .../tools/third_party/pytest/testing/conftest.py | 216 + .../third_party/pytest/testing/deprecated_test.py | 310 + .../pytest/testing/example_scripts/README.rst | 9 + .../pytest/testing/example_scripts/__init__.py | 0 .../acceptance/fixture_mock_integration.py | 16 + .../collect/collect_init_tests/pytest.ini | 2 + .../collect/collect_init_tests/tests/__init__.py | 2 + .../collect/collect_init_tests/tests/test_foo.py | 2 + .../package_infinite_recursion/__init__.pyi | 0 .../collect/package_infinite_recursion/conftest.py | 2 + .../package_infinite_recursion/tests/__init__.py | 0 .../package_infinite_recursion/tests/test_basic.py | 2 + .../package_init_given_as_arg/pkg/__init__.py | 0 .../package_init_given_as_arg/pkg/test_foo.py | 2 + .../config/collect_pytest_prefix/__init__.pyi | 0 .../config/collect_pytest_prefix/conftest.py | 2 + .../config/collect_pytest_prefix/test_foo.py | 2 + .../conftest_usageerror/__init__.pyi | 0 .../conftest_usageerror/conftest.py | 8 + .../dataclasses/test_compare_dataclasses.py | 14 + ...est_compare_dataclasses_field_comparison_off.py | 14 + .../test_compare_dataclasses_verbose.py | 14 + .../test_compare_recursive_dataclasses.py | 44 + .../test_compare_two_different_dataclasses.py | 19 + .../example_scripts/doctest/main_py/__main__.py | 2 + .../doctest/main_py/test_normal_module.py | 6 + .../fixtures/custom_item/__init__.pyi | 0 .../fixtures/custom_item/conftest.py | 15 + .../fixtures/custom_item/foo/__init__.py | 0 .../fixtures/custom_item/foo/test_foo.py | 2 + .../sub1/__init__.py | 0 .../sub1/conftest.py | 7 + .../sub1/test_in_sub1.py | 2 + .../sub2/__init__.py | 0 .../sub2/conftest.py | 6 + .../sub2/test_in_sub2.py | 2 + .../test_detect_recursive_dependency_error.py | 15 + .../__init__.pyi | 0 .../conftest.py | 6 + .../pkg/__init__.py | 0 .../pkg/conftest.py | 6 + .../pkg/test_spam.py | 2 + .../__init__.pyi | 0 .../conftest.py | 6 + .../test_extend_fixture_conftest_module.py | 10 + .../test_extend_fixture_module_class.py | 15 + .../fixtures/fill_fixtures/test_funcarg_basic.py | 15 + .../test_funcarg_lookup_classlevel.py | 10 + .../test_funcarg_lookup_modulelevel.py | 15 + .../fill_fixtures/test_funcarg_lookupfails.py | 10 + .../fixtures/test_fixture_named_request.py | 10 + .../fixtures/test_getfixturevalue_dynamic.py | 20 + .../issue88_initial_file_multinodes/__init__.pyi | 0 .../issue88_initial_file_multinodes/conftest.py | 14 + .../issue88_initial_file_multinodes/test_hello.py | 2 + .../pytest/testing/example_scripts/issue_519.py | 53 + .../pytest/testing/example_scripts/junit-10.xsd | 147 + .../marks/marks_considered_keywords/__init__.pyi | 0 .../marks/marks_considered_keywords/conftest.py | 0 .../test_marks_as_keywords.py | 6 + .../perf_examples/collect_stats/.gitignore | 1 + .../collect_stats/generate_folders.py | 27 + .../perf_examples/collect_stats/template_test.py | 2 + .../pytest/testing/example_scripts/pytest.ini | 2 + .../example_scripts/tmpdir/tmp_path_fixture.py | 7 + .../test_parametrized_fixture_error_message.py | 14 + .../example_scripts/unittest/test_setup_skip.py | 13 + .../unittest/test_setup_skip_class.py | 14 + .../unittest/test_setup_skip_module.py | 12 + .../unittest/test_unittest_asyncio.py | 25 + .../unittest/test_unittest_asynctest.py | 23 + .../unittest/test_unittest_plain_async.py | 6 + .../warnings/test_group_warnings_by_message.py | 21 + .../test_1.py | 21 + .../test_2.py | 5 + .../pytest/testing/examples/test_issue519.py | 7 + .../third_party/pytest/testing/freeze/.gitignore | 3 + .../pytest/testing/freeze/create_executable.py | 11 + .../pytest/testing/freeze/runtests_script.py | 10 + .../pytest/testing/freeze/tests/test_doctest.txt | 6 + .../pytest/testing/freeze/tests/test_trivial.py | 6 + .../third_party/pytest/testing/freeze/tox_run.py | 12 + .../third_party/pytest/testing/io/test_saferepr.py | 181 + .../pytest/testing/io/test_terminalwriter.py | 293 + .../third_party/pytest/testing/io/test_wcwidth.py | 38 + .../pytest/testing/logging/test_fixture.py | 310 + .../pytest/testing/logging/test_formatter.py | 173 + .../pytest/testing/logging/test_reporting.py | 1167 +++ .../pytest/testing/plugins_integration/.gitignore | 2 + .../pytest/testing/plugins_integration/README.rst | 13 + .../testing/plugins_integration/bdd_wallet.feature | 9 + .../testing/plugins_integration/bdd_wallet.py | 39 + .../testing/plugins_integration/django_settings.py | 1 + .../pytest/testing/plugins_integration/pytest.ini | 5 + .../pytest_anyio_integration.py | 8 + .../pytest_asyncio_integration.py | 8 + .../plugins_integration/pytest_mock_integration.py | 2 + .../plugins_integration/pytest_trio_integration.py | 8 + .../pytest_twisted_integration.py | 18 + .../testing/plugins_integration/requirements.txt | 15 + .../plugins_integration/simple_integration.py | 10 + .../third_party/pytest/testing/python/approx.py | 872 ++ .../third_party/pytest/testing/python/collect.py | 1493 ++++ .../third_party/pytest/testing/python/fixtures.py | 4474 ++++++++++ .../pytest/testing/python/integration.py | 503 ++ .../third_party/pytest/testing/python/metafunc.py | 1907 +++++ .../third_party/pytest/testing/python/raises.py | 298 + .../testing/python/show_fixtures_per_test.py | 254 + .../third_party/pytest/testing/test_argcomplete.py | 95 + .../third_party/pytest/testing/test_assertion.py | 1685 ++++ .../pytest/testing/test_assertrewrite.py | 1841 ++++ .../pytest/testing/test_cacheprovider.py | 1251 +++ .../third_party/pytest/testing/test_capture.py | 1666 ++++ .../third_party/pytest/testing/test_collection.py | 1506 ++++ .../third_party/pytest/testing/test_compat.py | 265 + .../third_party/pytest/testing/test_config.py | 2115 +++++ .../third_party/pytest/testing/test_conftest.py | 696 ++ .../third_party/pytest/testing/test_debugging.py | 1327 +++ .../third_party/pytest/testing/test_doctest.py | 1572 ++++ .../pytest/testing/test_entry_points.py | 7 + .../third_party/pytest/testing/test_error_diffs.py | 283 + .../pytest/testing/test_faulthandler.py | 172 + .../third_party/pytest/testing/test_findpaths.py | 135 + .../third_party/pytest/testing/test_helpconfig.py | 124 + .../third_party/pytest/testing/test_junitxml.py | 1703 ++++ .../third_party/pytest/testing/test_legacypath.py | 180 + .../pytest/testing/test_link_resolve.py | 80 + .../tools/third_party/pytest/testing/test_main.py | 264 + .../tools/third_party/pytest/testing/test_mark.py | 1130 +++ .../pytest/testing/test_mark_expression.py | 195 + .../tools/third_party/pytest/testing/test_meta.py | 32 + .../third_party/pytest/testing/test_monkeypatch.py | 455 + .../tools/third_party/pytest/testing/test_nodes.py | 167 + .../tools/third_party/pytest/testing/test_nose.py | 498 ++ .../third_party/pytest/testing/test_parseopt.py | 344 + .../third_party/pytest/testing/test_pastebin.py | 184 + .../third_party/pytest/testing/test_pathlib.py | 574 ++ .../pytest/testing/test_pluginmanager.py | 427 + .../third_party/pytest/testing/test_pytester.py | 855 ++ .../third_party/pytest/testing/test_python_path.py | 110 + .../third_party/pytest/testing/test_recwarn.py | 410 + .../third_party/pytest/testing/test_reports.py | 488 ++ .../third_party/pytest/testing/test_runner.py | 1061 +++ .../pytest/testing/test_runner_xunit.py | 297 + .../tools/third_party/pytest/testing/test_scope.py | 39 + .../third_party/pytest/testing/test_session.py | 369 + .../third_party/pytest/testing/test_setuponly.py | 318 + .../third_party/pytest/testing/test_setupplan.py | 120 + .../third_party/pytest/testing/test_skipping.py | 1533 ++++ .../tools/third_party/pytest/testing/test_stash.py | 67 + .../third_party/pytest/testing/test_stepwise.py | 280 + .../third_party/pytest/testing/test_terminal.py | 2486 ++++++ .../pytest/testing/test_threadexception.py | 137 + .../third_party/pytest/testing/test_tmpdir.py | 480 ++ .../third_party/pytest/testing/test_unittest.py | 1500 ++++ .../pytest/testing/test_unraisableexception.py | 133 + .../pytest/testing/test_warning_types.py | 38 + .../third_party/pytest/testing/test_warnings.py | 775 ++ .../third_party/pytest/testing/typing_checks.py | 24 + .../tests/tools/third_party/pytest/tox.ini | 184 + 550 files changed, 125405 insertions(+) create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.coveragerc create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.gitblameignore create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.github/FUNDING.yml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.github/ISSUE_TEMPLATE/1_bug_report.md create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.github/ISSUE_TEMPLATE/2_feature_request.md create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.github/ISSUE_TEMPLATE/config.yml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.github/PULL_REQUEST_TEMPLATE.md create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.github/config.yml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.github/dependabot.yml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.github/labels.toml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.github/workflows/main.yml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.github/workflows/prepare-release-pr.yml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.github/workflows/update-plugin-list.yml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.gitignore create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.pre-commit-config.yaml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/.readthedocs.yml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/AUTHORS create mode 100644 testing/web-platform/tests/tools/third_party/pytest/CHANGELOG.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/CITATION create mode 100644 testing/web-platform/tests/tools/third_party/pytest/CODE_OF_CONDUCT.md create mode 100644 testing/web-platform/tests/tools/third_party/pytest/CONTRIBUTING.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/LICENSE create mode 100644 testing/web-platform/tests/tools/third_party/pytest/OPENCOLLECTIVE.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/README.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/RELEASING.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/TIDELIFT.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/bench/bench.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/bench/bench_argcomplete.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/bench/empty.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/bench/manyparam.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/bench/skip.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/bench/unit_test.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/bench/xunit.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/changelog/README.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/changelog/_template.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/codecov.yml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/Makefile create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/globaltoc.html create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/layout.html create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/links.html create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/relations.html create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/sidebarintro.html create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/slim_searchbox.html create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/adopt.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/index.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.4.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.4.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.5.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.4.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.4.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.4.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.5.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.5.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.5.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.7.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.7.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.7.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.4.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.5.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.6.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.7.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.9.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.9.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.9.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.4.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.5.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.6.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.7.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.10.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.10.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.4.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.5.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.3.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.3.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.3.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.4.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.4.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.4.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.5.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.5.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.4.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.4.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.8.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.8.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.8.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.0.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.0.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.0.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.1.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.1.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.2.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.2.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.3.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.3.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.4.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.4.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.4.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.5.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.4.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.5.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.6.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.7.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.8.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.9.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.0.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.0.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.4.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.4.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.5.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.0rc1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.1.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.1.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.1.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.2.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.3.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.4.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.5.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-7.0.0.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-7.0.0rc1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-7.0.1.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/sprint2016.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/backwards-compatibility.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/builtin.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/changelog.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/conf.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/contact.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/contents.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/contributing.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/deprecations.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/development_guide.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/failure_demo.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/global_testmodule_config/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/global_testmodule_config/test_hello_world.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/test_failures.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/test_setup_flow_example.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/attic.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/fixture_availability.svg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/fixture_availability_plugins.svg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse.svg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_flat.svg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.svg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.svg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies.svg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies_flat.svg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies_unclear.svg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_scope.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_scope.svg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_request_different_scope.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_request_different_scope.svg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/index.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/markers.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/multipython.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython/test_simple.yaml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/parametrize.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/pythoncollection.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/pythoncollection.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/reportingdemo.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/simple.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/special.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/example/xfail_demo.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/anatomy.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/fixtures.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/flaky.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/goodpractices.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/index.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/pythonpath.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/funcarg_compare.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/funcargs.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/getting-started.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/historical-notes.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/history.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/assert.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/bash-completion.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/cache.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/capture-stdout-stderr.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/capture-warnings.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/doctest.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/existingtestsuite.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/failures.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/fixtures.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/index.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/logging.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/mark.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/monkeypatch.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/nose.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/output.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/parametrize.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/plugins.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/skipping.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/tmp_path.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/unittest.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/usage.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/writing_hook_functions.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/writing_plugins.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/xunit_setup.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/img/cramer2.png create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/img/favicon.png create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/img/freiburg2.jpg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/img/gaynor3.png create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/img/keleshev.png create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pullrequest.png create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pylib.png create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pytest1.png create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pytest_logo_curves.svg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/img/theuni.png create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/index.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/license.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/naming20.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/proposals/parametrize_with_fixtures.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/py27-py34-deprecation.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/pytest.ini create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/recwarn.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/customize.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/exit-codes.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/fixtures.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/index.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/plugin_list.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/reference.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/requirements.txt create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/sponsor.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/talks.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/tidelift.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/doc/en/yieldfixture.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/extra/get_issues.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/extra/setup-py.test/setup.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/pyproject.toml create mode 100644 testing/web-platform/tests/tools/third_party/pytest/scripts/prepare-release-pr.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/scripts/publish-gh-release-notes.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/scripts/release.major.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/scripts/release.minor.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/scripts/release.patch.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/scripts/release.pre.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/scripts/release.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/scripts/towncrier-draft-to-file.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/scripts/update-plugin-list.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/setup.cfg create mode 100644 testing/web-platform/tests/tools/third_party/pytest/setup.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_argcomplete.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_code/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_code/code.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_code/source.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/saferepr.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/terminalwriter.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/wcwidth.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_version.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/rewrite.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/truncate.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/util.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/cacheprovider.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/capture.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/compat.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/argparsing.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/compat.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/exceptions.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/findpaths.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/debugging.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/deprecated.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/doctest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/faulthandler.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/fixtures.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/freeze_support.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/helpconfig.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/hookspec.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/junitxml.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/legacypath.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/logging.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/main.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/mark/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/mark/expression.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/mark/structures.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/monkeypatch.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/nodes.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/nose.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/outcomes.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pastebin.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pathlib.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/py.typed create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pytester.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pytester_assertions.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/python.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/python_api.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/python_path.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/recwarn.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/reports.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/runner.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/scope.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/setuponly.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/setupplan.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/skipping.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/stash.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/stepwise.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/terminal.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/threadexception.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/timing.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/tmpdir.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/unittest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/unraisableexception.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/warning_types.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/_pytest/warnings.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/pytest/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/pytest/__main__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/pytest/collect.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/src/pytest/py.typed create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/acceptance_test.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/code/test_code.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/code/test_excinfo.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/code/test_source.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/deprecated_test.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/README.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/acceptance/fixture_mock_integration.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/collect_init_tests/pytest.ini create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/collect_init_tests/tests/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/collect_init_tests/tests/test_foo.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/__init__.pyi create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/tests/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/tests/test_basic.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_init_given_as_arg/pkg/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_init_given_as_arg/pkg/test_foo.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/config/collect_pytest_prefix/__init__.pyi create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/config/collect_pytest_prefix/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/config/collect_pytest_prefix/test_foo.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/conftest_usageerror/__init__.pyi create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/conftest_usageerror/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_dataclasses.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_dataclasses_field_comparison_off.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_dataclasses_verbose.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_recursive_dataclasses.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_two_different_dataclasses.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/doctest/main_py/__main__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/doctest/main_py/test_normal_module.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/__init__.pyi create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/foo/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/foo/test_foo.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub1/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub1/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub1/test_in_sub1.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/test_in_sub2.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_detect_recursive_dependency_error.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/__init__.pyi create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/pkg/__init__.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/pkg/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/pkg/test_spam.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_module/__init__.pyi create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_module/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_module/test_extend_fixture_conftest_module.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_module_class.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_basic.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_lookup_classlevel.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_lookup_modulelevel.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_lookupfails.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/test_fixture_named_request.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/test_getfixturevalue_dynamic.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue88_initial_file_multinodes/__init__.pyi create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue88_initial_file_multinodes/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue88_initial_file_multinodes/test_hello.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue_519.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/junit-10.xsd create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/marks/marks_considered_keywords/__init__.pyi create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/marks/marks_considered_keywords/conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/marks/marks_considered_keywords/test_marks_as_keywords.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/perf_examples/collect_stats/.gitignore create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/perf_examples/collect_stats/generate_folders.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/perf_examples/collect_stats/template_test.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/pytest.ini create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/tmpdir/tmp_path_fixture.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_parametrized_fixture_error_message.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_setup_skip.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_setup_skip_class.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_setup_skip_module.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_unittest_asyncio.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_unittest_asynctest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_unittest_plain_async.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/warnings/test_group_warnings_by_message.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/warnings/test_group_warnings_by_message_summary/test_1.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/warnings/test_group_warnings_by_message_summary/test_2.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/examples/test_issue519.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/freeze/.gitignore create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/freeze/create_executable.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/freeze/runtests_script.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/freeze/tests/test_doctest.txt create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/freeze/tests/test_trivial.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/freeze/tox_run.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/io/test_saferepr.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/io/test_terminalwriter.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/io/test_wcwidth.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/logging/test_fixture.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/logging/test_formatter.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/logging/test_reporting.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/.gitignore create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/README.rst create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/bdd_wallet.feature create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/bdd_wallet.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/django_settings.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest.ini create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_anyio_integration.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_asyncio_integration.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_mock_integration.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_trio_integration.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_twisted_integration.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/requirements.txt create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/simple_integration.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/python/approx.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/python/collect.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/python/fixtures.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/python/integration.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/python/metafunc.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/python/raises.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/python/show_fixtures_per_test.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_argcomplete.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_assertion.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_assertrewrite.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_cacheprovider.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_capture.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_collection.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_compat.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_config.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_conftest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_debugging.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_doctest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_entry_points.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_error_diffs.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_faulthandler.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_findpaths.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_helpconfig.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_junitxml.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_legacypath.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_link_resolve.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_main.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_mark.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_mark_expression.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_meta.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_monkeypatch.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_nodes.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_nose.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_parseopt.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_pastebin.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_pathlib.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_pluginmanager.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_pytester.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_python_path.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_recwarn.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_reports.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_runner.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_runner_xunit.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_scope.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_session.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_setuponly.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_setupplan.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_skipping.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_stash.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_stepwise.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_terminal.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_threadexception.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_tmpdir.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_unittest.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_unraisableexception.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_warning_types.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/test_warnings.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/testing/typing_checks.py create mode 100644 testing/web-platform/tests/tools/third_party/pytest/tox.ini (limited to 'testing/web-platform/tests/tools/third_party/pytest') diff --git a/testing/web-platform/tests/tools/third_party/pytest/.coveragerc b/testing/web-platform/tests/tools/third_party/pytest/.coveragerc new file mode 100644 index 0000000000..a335557d4f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.coveragerc @@ -0,0 +1,31 @@ +[run] +include = + src/* + testing/* + */lib/python*/site-packages/_pytest/* + */lib/python*/site-packages/pytest.py + */pypy*/site-packages/_pytest/* + */pypy*/site-packages/pytest.py + *\Lib\site-packages\_pytest\* + *\Lib\site-packages\pytest.py +parallel = 1 +branch = 1 + +[paths] +source = src/ + */lib/python*/site-packages/ + */pypy*/site-packages/ + *\Lib\site-packages\ + +[report] +skip_covered = True +show_missing = True +exclude_lines = + \#\s*pragma: no cover + ^\s*raise NotImplementedError\b + ^\s*return NotImplemented\b + ^\s*assert False(,|$) + ^\s*assert_never\( + + ^\s*if TYPE_CHECKING: + ^\s*@overload( |$) diff --git a/testing/web-platform/tests/tools/third_party/pytest/.gitblameignore b/testing/web-platform/tests/tools/third_party/pytest/.gitblameignore new file mode 100644 index 0000000000..0cb298b024 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.gitblameignore @@ -0,0 +1,28 @@ +# List of revisions that can be ignored with git-blame(1). +# +# See `blame.ignoreRevsFile` in git-config(1) to enable it by default, or +# use it with `--ignore-revs-file` manually with git-blame. +# +# To "install" it: +# +# git config --local blame.ignoreRevsFile .gitblameignore + +# run black +703e4b11ba76171eccd3f13e723c47b810ded7ef +# switched to src layout +eaa882f3d5340956beb176aa1753e07e3f3f2190 +# pre-commit run pyupgrade --all-files +a91fe1feddbded535a4322ab854429e3a3961fb4 +# move node base classes from main to nodes +afc607cfd81458d4e4f3b1f3cf8cc931b933907e +# [?] split most fixture related code into own plugin +8c49561470708761f7321504f5e8343811be87ac +# run pyupgrade +9aacb4635e81edd6ecf281d4f6c0cfc8e94ab301 +# run blacken-docs +5f95dce95602921a70bfbc7d8de2f7712c5e4505 +# ran pyupgrade-docs again +75d0b899bbb56d6849e9d69d83a9426ed3f43f8b + +# move argument parser to own file +c9df77cbd6a365dcb73c39618e4842711817e871 diff --git a/testing/web-platform/tests/tools/third_party/pytest/.github/FUNDING.yml b/testing/web-platform/tests/tools/third_party/pytest/.github/FUNDING.yml new file mode 100644 index 0000000000..5f2d1cf09c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.github/FUNDING.yml @@ -0,0 +1,5 @@ +# info: +# * https://help.github.com/en/articles/displaying-a-sponsor-button-in-your-repository +# * https://tidelift.com/subscription/how-to-connect-tidelift-with-github +tidelift: pypi/pytest +open_collective: pytest diff --git a/testing/web-platform/tests/tools/third_party/pytest/.github/ISSUE_TEMPLATE/1_bug_report.md b/testing/web-platform/tests/tools/third_party/pytest/.github/ISSUE_TEMPLATE/1_bug_report.md new file mode 100644 index 0000000000..0fc3e06cd2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.github/ISSUE_TEMPLATE/1_bug_report.md @@ -0,0 +1,16 @@ +--- +name: 🐛 Bug Report +about: Report errors and problems + +--- + + + +- [ ] a detailed description of the bug or problem you are having +- [ ] output of `pip list` from the virtual environment you are using +- [ ] pytest and operating system versions +- [ ] minimal example if possible diff --git a/testing/web-platform/tests/tools/third_party/pytest/.github/ISSUE_TEMPLATE/2_feature_request.md b/testing/web-platform/tests/tools/third_party/pytest/.github/ISSUE_TEMPLATE/2_feature_request.md new file mode 100644 index 0000000000..01fe96295e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.github/ISSUE_TEMPLATE/2_feature_request.md @@ -0,0 +1,25 @@ +--- +name: 🚀 Feature Request +about: Ideas for new features and improvements + +--- + + + +#### What's the problem this feature will solve? + + +#### Describe the solution you'd like + + + + +#### Alternative Solutions + + +#### Additional context + diff --git a/testing/web-platform/tests/tools/third_party/pytest/.github/ISSUE_TEMPLATE/config.yml b/testing/web-platform/tests/tools/third_party/pytest/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..742d2e4d66 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: ❓ Support Question + url: https://github.com/pytest-dev/pytest/discussions + about: Use GitHub's new Discussions feature for questions diff --git a/testing/web-platform/tests/tools/third_party/pytest/.github/PULL_REQUEST_TEMPLATE.md b/testing/web-platform/tests/tools/third_party/pytest/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..5e7282bfd7 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,26 @@ + diff --git a/testing/web-platform/tests/tools/third_party/pytest/.github/config.yml b/testing/web-platform/tests/tools/third_party/pytest/.github/config.yml new file mode 100644 index 0000000000..86a8a97e78 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.github/config.yml @@ -0,0 +1,2 @@ +rtd: + project: pytest diff --git a/testing/web-platform/tests/tools/third_party/pytest/.github/dependabot.yml b/testing/web-platform/tests/tools/third_party/pytest/.github/dependabot.yml new file mode 100644 index 0000000000..507789bf5a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: +- package-ecosystem: pip + directory: "/testing/plugins_integration" + schedule: + interval: weekly + time: "03:00" + open-pull-requests-limit: 10 + allow: + - dependency-type: direct + - dependency-type: indirect diff --git a/testing/web-platform/tests/tools/third_party/pytest/.github/labels.toml b/testing/web-platform/tests/tools/third_party/pytest/.github/labels.toml new file mode 100644 index 0000000000..aef1e913af --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.github/labels.toml @@ -0,0 +1,149 @@ +["os: cygwin"] +color = "006b75" +description = "cygwin platform-specific problem" +name = "os: cygwin" + +["os: linux"] +color = "1d76db" +description = "linux platform-specific problem" +name = "os: linux" + +["os: mac"] +color = "bfdadc" +description = "mac platform-specific problem" +name = "os: mac" + +["os: windows"] +color = "fbca04" +description = "windows platform-specific problem" +name = "os: windows" + +["plugin: argcomplete"] +color = "d4c5f9" +description = "related to the argcomplete builtin plugin" +name = "plugin: argcomplete" + +["plugin: cache"] +color = "c7def8" +description = "related to the cache builtin plugin" +name = "plugin: cache" + +["plugin: capture"] +color = "1d76db" +description = "related to the capture builtin plugin" +name = "plugin: capture" + +["plugin: debugging"] +color = "dd52a8" +description = "related to the debugging builtin plugin" +name = "plugin: debugging" + +["plugin: doctests"] +color = "fad8c7" +description = "related to the doctests builtin plugin" +name = "plugin: doctests" + +["plugin: junitxml"] +color = "c5def5" +description = "related to the junitxml builtin plugin" +name = "plugin: junitxml" + +["plugin: logging"] +color = "ff5432" +description = "related to the logging builtin plugin" +name = "plugin: logging" + +["plugin: monkeypatch"] +color = "0e8a16" +description = "related to the monkeypatch builtin plugin" +name = "plugin: monkeypatch" + +["plugin: nose"] +color = "bfdadc" +description = "related to the nose integration builtin plugin" +name = "plugin: nose" + +["plugin: pastebin"] +color = "bfd4f2" +description = "related to the pastebin builtin plugin" +name = "plugin: pastebin" + +["plugin: pytester"] +color = "c5def5" +description = "related to the pytester builtin plugin" +name = "plugin: pytester" + +["plugin: tmpdir"] +color = "bfd4f2" +description = "related to the tmpdir builtin plugin" +name = "plugin: tmpdir" + +["plugin: unittest"] +color = "006b75" +description = "related to the unittest integration builtin plugin" +name = "plugin: unittest" + +["plugin: warnings"] +color = "fef2c0" +description = "related to the warnings builtin plugin" +name = "plugin: warnings" + +["plugin: xdist"] +color = "5319e7" +description = "related to the xdist external plugin" +name = "plugin: xdist" + +["status: critical"] +color = "e11d21" +description = "grave problem or usability issue that affects lots of users" +name = "status: critical" + +["status: easy"] +color = "bfe5bf" +description = "easy issue that is friendly to new contributor" +name = "status: easy" + +["status: help wanted"] +color = "159818" +description = "developers would like help from experts on this topic" +name = "status: help wanted" + +["status: needs information"] +color = "5319e7" +description = "reporter needs to provide more information; can be closed after 2 or more weeks of inactivity" +name = "status: needs information" + +["topic: collection"] +color = "006b75" +description = "related to the collection phase" +name = "topic: collection" + +["topic: config"] +color = "006b75" +description = "related to config handling, argument parsing and config file" +name = "topic: config" + +["topic: fixtures"] +color = "5319e7" +description = "anything involving fixtures directly or indirectly" +name = "topic: fixtures" + +["topic: marks"] +color = "b60205" +description = "related to marks, either the general marks or builtin" +name = "topic: marks" + +["topic: parametrize"] +color = "fbca04" +description = "related to @pytest.mark.parametrize" +name = "topic: parametrize" + +["topic: reporting"] +color = "fef2c0" +description = "related to terminal output and user-facing messages and errors" +name = "topic: reporting" + +["topic: rewrite"] +color = "0e8a16" +description = "related to the assertion rewrite mechanism" +name = "topic: rewrite" diff --git a/testing/web-platform/tests/tools/third_party/pytest/.github/workflows/main.yml b/testing/web-platform/tests/tools/third_party/pytest/.github/workflows/main.yml new file mode 100644 index 0000000000..42759ce853 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.github/workflows/main.yml @@ -0,0 +1,231 @@ +name: main + +on: + push: + branches: + - main + - "[0-9]+.[0-9]+.x" + tags: + - "[0-9]+.[0-9]+.[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+" + + pull_request: + branches: + - main + - "[0-9]+.[0-9]+.x" + +env: + PYTEST_ADDOPTS: "--color=yes" + +# Set permissions at the job level. +permissions: {} + +jobs: + build: + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + permissions: + contents: read + + strategy: + fail-fast: false + matrix: + name: [ + "windows-py36", + "windows-py37", + "windows-py37-pluggy", + "windows-py38", + "windows-py39", + "windows-py310", + "windows-py311", + + "ubuntu-py36", + "ubuntu-py37", + "ubuntu-py37-pluggy", + "ubuntu-py37-freeze", + "ubuntu-py38", + "ubuntu-py39", + "ubuntu-py310", + "ubuntu-py311", + "ubuntu-pypy3", + + "macos-py37", + "macos-py38", + + "docs", + "doctesting", + "plugins", + ] + + include: + - name: "windows-py36" + python: "3.6" + os: windows-latest + tox_env: "py36-xdist" + - name: "windows-py37" + python: "3.7" + os: windows-latest + tox_env: "py37-numpy" + - name: "windows-py37-pluggy" + python: "3.7" + os: windows-latest + tox_env: "py37-pluggymain-xdist" + - name: "windows-py38" + python: "3.8" + os: windows-latest + tox_env: "py38-unittestextras" + use_coverage: true + - name: "windows-py39" + python: "3.9" + os: windows-latest + tox_env: "py39-xdist" + - name: "windows-py310" + python: "3.10" + os: windows-latest + tox_env: "py310-xdist" + - name: "windows-py311" + python: "3.11-dev" + os: windows-latest + tox_env: "py311" + + - name: "ubuntu-py36" + python: "3.6" + os: ubuntu-latest + tox_env: "py36-xdist" + - name: "ubuntu-py37" + python: "3.7" + os: ubuntu-latest + tox_env: "py37-lsof-numpy-pexpect" + use_coverage: true + - name: "ubuntu-py37-pluggy" + python: "3.7" + os: ubuntu-latest + tox_env: "py37-pluggymain-xdist" + - name: "ubuntu-py37-freeze" + python: "3.7" + os: ubuntu-latest + tox_env: "py37-freeze" + - name: "ubuntu-py38" + python: "3.8" + os: ubuntu-latest + tox_env: "py38-xdist" + - name: "ubuntu-py39" + python: "3.9" + os: ubuntu-latest + tox_env: "py39-xdist" + - name: "ubuntu-py310" + python: "3.10" + os: ubuntu-latest + tox_env: "py310-xdist" + - name: "ubuntu-py311" + python: "3.11-dev" + os: ubuntu-latest + tox_env: "py311" + - name: "ubuntu-pypy3" + python: "pypy-3.7" + os: ubuntu-latest + tox_env: "pypy3-xdist" + + - name: "macos-py37" + python: "3.7" + os: macos-latest + tox_env: "py37-xdist" + - name: "macos-py38" + python: "3.8" + os: macos-latest + tox_env: "py38-xdist" + use_coverage: true + + - name: "plugins" + python: "3.7" + os: ubuntu-latest + tox_env: "plugins" + + - name: "docs" + python: "3.7" + os: ubuntu-latest + tox_env: "docs" + - name: "doctesting" + python: "3.7" + os: ubuntu-latest + tox_env: "doctesting" + use_coverage: true + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox coverage + + - name: Test without coverage + if: "! matrix.use_coverage" + run: "tox -e ${{ matrix.tox_env }}" + + - name: Test with coverage + if: "matrix.use_coverage" + run: "tox -e ${{ matrix.tox_env }}-coverage" + + - name: Generate coverage report + if: "matrix.use_coverage" + run: python -m coverage xml + + - name: Upload coverage to Codecov + if: "matrix.use_coverage" + uses: codecov/codecov-action@v2 + with: + fail_ci_if_error: true + files: ./coverage.xml + verbose: true + + deploy: + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && github.repository == 'pytest-dev/pytest' + + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + + needs: [build] + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.7" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade build tox + + - name: Build package + run: | + python -m build + + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.pypi_token }} + + - name: Publish GitHub release notes + env: + GH_RELEASE_NOTES_TOKEN: ${{ github.token }} + run: | + sudo apt-get install pandoc + tox -e publish-gh-release-notes diff --git a/testing/web-platform/tests/tools/third_party/pytest/.github/workflows/prepare-release-pr.yml b/testing/web-platform/tests/tools/third_party/pytest/.github/workflows/prepare-release-pr.yml new file mode 100644 index 0000000000..429834b3f2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.github/workflows/prepare-release-pr.yml @@ -0,0 +1,52 @@ +name: prepare release pr + +on: + workflow_dispatch: + inputs: + branch: + description: 'Branch to base the release from' + required: true + default: '' + major: + description: 'Major release? (yes/no)' + required: true + default: 'no' + prerelease: + description: 'Prerelease (ex: rc1). Leave empty if not a pre-release.' + required: false + default: '' + +# Set permissions at the job level. +permissions: {} + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade setuptools tox + + - name: Prepare release PR (minor/patch release) + if: github.event.inputs.major == 'no' + run: | + tox -e prepare-release-pr -- ${{ github.event.inputs.branch }} ${{ github.token }} --prerelease='${{ github.event.inputs.prerelease }}' + + - name: Prepare release PR (major release) + if: github.event.inputs.major == 'yes' + run: | + tox -e prepare-release-pr -- ${{ github.event.inputs.branch }} ${{ github.token }} --major --prerelease='${{ github.event.inputs.prerelease }}' diff --git a/testing/web-platform/tests/tools/third_party/pytest/.github/workflows/update-plugin-list.yml b/testing/web-platform/tests/tools/third_party/pytest/.github/workflows/update-plugin-list.yml new file mode 100644 index 0000000000..193469072f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.github/workflows/update-plugin-list.yml @@ -0,0 +1,49 @@ +name: Update Plugin List + +on: + schedule: + # At 00:00 on Sunday. + # https://crontab.guru + - cron: '0 0 * * 0' + workflow_dispatch: + +# Set permissions at the job level. +permissions: {} + +jobs: + createPullRequest: + if: github.repository_owner == 'pytest-dev' + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install packaging requests tabulate[widechars] tqdm + + - name: Update Plugin List + run: python scripts/update-plugin-list.py + + - name: Create Pull Request + uses: peter-evans/create-pull-request@2455e1596942c2902952003bbb574afbbe2ab2e6 + with: + commit-message: '[automated] Update plugin list' + author: 'pytest bot ' + branch: update-plugin-list/patch + delete-branch: true + branch-suffix: short-commit-hash + title: '[automated] Update plugin list' + body: '[automated] Update plugin list' diff --git a/testing/web-platform/tests/tools/third_party/pytest/.gitignore b/testing/web-platform/tests/tools/third_party/pytest/.gitignore new file mode 100644 index 0000000000..935da3b9a2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.gitignore @@ -0,0 +1,58 @@ +# Automatically generated by `hgimportsvn` +.svn +.hgsvn + +# Ignore local virtualenvs +lib/ +bin/ +include/ +.Python/ + +# These lines are suggested according to the svn:ignore property +# Feel free to enable them by uncommenting them +*.pyc +*.pyo +*.swp +*.class +*.orig +*~ +.hypothesis/ + +# autogenerated +src/_pytest/_version.py +# setuptools +.eggs/ + +doc/*/_build +doc/*/.doctrees +doc/*/_changelog_towncrier_draft.rst +build/ +dist/ +*.egg-info +htmlcov/ +issue/ +env/ +.env/ +.venv/ +/pythonenv*/ +3rdparty/ +.tox +.cache +.pytest_cache +.mypy_cache +.coverage +.coverage.* +coverage.xml +.ropeproject +.idea +.hypothesis +.pydevproject +.project +.settings +.vscode + +# generated by pip +pip-wheel-metadata/ + +# pytest debug logs generated via --debug +pytestdebug.log diff --git a/testing/web-platform/tests/tools/third_party/pytest/.pre-commit-config.yaml b/testing/web-platform/tests/tools/third_party/pytest/.pre-commit-config.yaml new file mode 100644 index 0000000000..20cede3b7b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.pre-commit-config.yaml @@ -0,0 +1,99 @@ +repos: +- repo: https://github.com/psf/black + rev: 21.11b1 + hooks: + - id: black + args: [--safe, --quiet] +- repo: https://github.com/asottile/blacken-docs + rev: v1.12.0 + hooks: + - id: blacken-docs + additional_dependencies: [black==20.8b1] +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: fix-encoding-pragma + args: [--remove] + - id: check-yaml + - id: debug-statements + exclude: _pytest/(debugging|hookspec).py + language_version: python3 +- repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + language_version: python3 + additional_dependencies: + - flake8-typing-imports==1.9.0 + - flake8-docstrings==1.5.0 +- repo: https://github.com/asottile/reorder_python_imports + rev: v2.6.0 + hooks: + - id: reorder-python-imports + args: ['--application-directories=.:src', --py36-plus] +- repo: https://github.com/asottile/pyupgrade + rev: v2.29.1 + hooks: + - id: pyupgrade + args: [--py36-plus] +- repo: https://github.com/asottile/setup-cfg-fmt + rev: v1.20.0 + hooks: + - id: setup-cfg-fmt + args: [--max-py-version=3.10] +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.9.0 + hooks: + - id: python-use-type-annotations +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.910-1 + hooks: + - id: mypy + files: ^(src/|testing/) + args: [] + additional_dependencies: + - iniconfig>=1.1.0 + - py>=1.8.2 + - attrs>=19.2.0 + - packaging + - tomli + - types-atomicwrites + - types-pkg_resources +- repo: local + hooks: + - id: rst + name: rst + entry: rst-lint --encoding utf-8 + files: ^(RELEASING.rst|README.rst|TIDELIFT.rst)$ + language: python + additional_dependencies: [pygments, restructuredtext_lint] + - id: changelogs-rst + name: changelog filenames + language: fail + entry: 'changelog files must be named ####.(breaking|bugfix|deprecation|doc|feature|improvement|trivial|vendor).rst' + exclude: changelog/(\d+\.(breaking|bugfix|deprecation|doc|feature|improvement|trivial|vendor).rst|README.rst|_template.rst) + files: ^changelog/ + - id: py-deprecated + name: py library is deprecated + language: pygrep + entry: > + (?x)\bpy\.( + _code\.| + builtin\.| + code\.| + io\.| + path\.local\.sysfind| + process\.| + std\.| + error\.| + xml\. + ) + types: [python] + - id: py-path-deprecated + name: py.path usage is deprecated + exclude: docs|src/_pytest/deprecated.py|testing/deprecated_test.py + language: pygrep + entry: \bpy\.path\.local + types: [python] diff --git a/testing/web-platform/tests/tools/third_party/pytest/.readthedocs.yml b/testing/web-platform/tests/tools/third_party/pytest/.readthedocs.yml new file mode 100644 index 0000000000..bc44d38b4c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/.readthedocs.yml @@ -0,0 +1,19 @@ +version: 2 + +python: + install: + - requirements: doc/en/requirements.txt + - method: pip + path: . + +build: + os: ubuntu-20.04 + tools: + python: "3.9" + apt_packages: + - inkscape + +formats: + - epub + - pdf + - htmlzip diff --git a/testing/web-platform/tests/tools/third_party/pytest/AUTHORS b/testing/web-platform/tests/tools/third_party/pytest/AUTHORS new file mode 100644 index 0000000000..9413f9c2e7 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/AUTHORS @@ -0,0 +1,356 @@ +Holger Krekel, holger at merlinux eu +merlinux GmbH, Germany, office at merlinux eu + +Contributors include:: + +Aaron Coleman +Abdeali JK +Abdelrahman Elbehery +Abhijeet Kasurde +Adam Johnson +Adam Uhlir +Ahn Ki-Wook +Akiomi Kamakura +Alan Velasco +Alexander Johnson +Alexander King +Alexei Kozlenok +Allan Feldman +Aly Sivji +Amir Elkess +Anatoly Bubenkoff +Anders Hovmöller +Andras Mitzki +Andras Tim +Andrea Cimatoribus +Andreas Motl +Andreas Zeidler +Andrew Shapton +Andrey Paramonov +Andrzej Klajnert +Andrzej Ostrowski +Andy Freeland +Anthon van der Neut +Anthony Shaw +Anthony Sottile +Anton Grinevich +Anton Lodder +Antony Lee +Arel Cordero +Arias Emmanuel +Ariel Pillemer +Armin Rigo +Aron Coyle +Aron Curzon +Aviral Verma +Aviv Palivoda +Barney Gale +Ben Gartner +Ben Webb +Benjamin Peterson +Bernard Pratz +Bob Ippolito +Brian Dorsey +Brian Maissy +Brian Okken +Brianna Laugher +Bruno Oliveira +Cal Leeming +Carl Friedrich Bolz +Carlos Jenkins +Ceridwen +Charles Cloud +Charles Machalow +Charnjit SiNGH (CCSJ) +Chris Lamb +Chris NeJame +Chris Rose +Christian Boelsen +Christian Fetzer +Christian Neumüller +Christian Theunert +Christian Tismer +Christine Mecklenborg +Christoph Buelter +Christopher Dignam +Christopher Gilling +Claire Cecil +Claudio Madotto +CrazyMerlyn +Cristian Vera +Cyrus Maden +Damian Skrzypczak +Daniel Grana +Daniel Hahler +Daniel Nuri +Daniel Wandschneider +Daniele Procida +Danielle Jenkins +Daniil Galiev +Dave Hunt +David Díaz-Barquero +David Mohr +David Paul Röthlisberger +David Szotten +David Vierra +Daw-Ran Liou +Debi Mishra +Denis Kirisov +Denivy Braiam Rück +Dhiren Serai +Diego Russo +Dmitry Dygalo +Dmitry Pribysh +Dominic Mortlock +Duncan Betts +Edison Gustavo Muenz +Edoardo Batini +Edson Tadeu M. Manoel +Eduardo Schettino +Eli Boyarski +Elizaveta Shashkova +Éloi Rivard +Endre Galaczi +Eric Hunsberger +Eric Liu +Eric Siegerman +Erik Aronesty +Erik M. Bray +Evan Kepner +Fabien Zarifian +Fabio Zadrozny +Felix Nieuwenhuizen +Feng Ma +Florian Bruhin +Florian Dahlitz +Floris Bruynooghe +Gabriel Reis +Garvit Shubham +Gene Wood +George Kussumoto +Georgy Dyuldin +Gergely Kalmár +Gleb Nikonorov +Graeme Smecher +Graham Horler +Greg Price +Gregory Lee +Grig Gheorghiu +Grigorii Eremeev (budulianin) +Guido Wesdorp +Guoqiang Zhang +Harald Armin Massa +Harshna +Henk-Jaap Wagenaar +Holger Kohr +Hugo van Kemenade +Hui Wang (coldnight) +Ian Bicking +Ian Lesperance +Ilya Konstantinov +Ionuț Turturică +Iwan Briquemont +Jaap Broekhuizen +Jakob van Santen +Jakub Mitoraj +James Bourbeau +Jan Balster +Janne Vanhala +Jason R. Coombs +Javier Domingo Cansino +Javier Romero +Jeff Rackauckas +Jeff Widman +Jenni Rinker +John Eddie Ayson +John Towler +Jon Sonesen +Jonas Obrist +Jordan Guymon +Jordan Moldow +Jordan Speicher +Joseph Hunkeler +Josh Karpel +Joshua Bronson +Jurko Gospodnetić +Justyna Janczyszyn +Justice Ndou +Kale Kundert +Kamran Ahmad +Karl O. Pinc +Karthikeyan Singaravelan +Katarzyna Jachim +Katarzyna Król +Katerina Koukiou +Keri Volans +Kevin Cox +Kevin J. Foley +Kian-Meng Ang +Kodi B. Arfer +Kostis Anagnostopoulos +Kristoffer Nordström +Kyle Altendorf +Lawrence Mitchell +Lee Kamentsky +Lev Maximov +Lewis Cowles +Llandy Riveron Del Risco +Loic Esteve +Lukas Bednar +Luke Murphy +Maciek Fijalkowski +Maho +Maik Figura +Mandeep Bhutani +Manuel Krebber +Marc Schlaich +Marcelo Duarte Trevisani +Marcin Bachry +Marco Gorelli +Mark Abramowitz +Mark Dickinson +Markus Unterwaditzer +Martijn Faassen +Martin Altmayer +Martin K. Scherer +Martin Prusse +Mathieu Clabaut +Matt Bachmann +Matt Duck +Matt Williams +Matthias Hafner +Maxim Filipenko +Maximilian Cosmo Sitter +mbyt +Mickey Pashov +Michael Aquilina +Michael Birtwell +Michael Droettboom +Michael Goerz +Michael Krebs +Michael Seifert +Michal Wajszczuk +Michał Zięba +Mihai Capotă +Mike Hoyle (hoylemd) +Mike Lundy +Miro Hrončok +Nathaniel Compton +Nathaniel Waisbrot +Ned Batchelder +Neven Mundar +Nicholas Devenish +Nicholas Murphy +Niclas Olofsson +Nicolas Delaby +Nikolay Kondratyev +Olga Matoula +Oleg Pidsadnyi +Oleg Sushchenko +Oliver Bestwalter +Omar Kohl +Omer Hadari +Ondřej Súkup +Oscar Benjamin +Parth Patel +Patrick Hayes +Pauli Virtanen +Pavel Karateev +Paweł Adamczak +Pedro Algarvio +Petter Strandmark +Philipp Loose +Pieter Mulder +Piotr Banaszkiewicz +Piotr Helm +Prakhar Gurunani +Prashant Anand +Prashant Sharma +Pulkit Goyal +Punyashloka Biswal +Quentin Pradet +Ralf Schmitt +Ram Rachum +Ralph Giles +Ran Benita +Raphael Castaneda +Raphael Pierzina +Raquel Alegre +Ravi Chandra +Robert Holt +Roberto Polli +Roland Puntaier +Romain Dorgueil +Roman Bolshakov +Ronny Pfannschmidt +Ross Lawley +Ruaridh Williamson +Russel Winder +Ryan Wooden +Saiprasad Kale +Samuel Dion-Girardeau +Samuel Searles-Bryant +Samuele Pedroni +Sanket Duthade +Sankt Petersbug +Segev Finer +Serhii Mozghovyi +Seth Junot +Shantanu Jain +Shubham Adep +Simon Gomizelj +Simon Kerr +Skylar Downes +Srinivas Reddy Thatiparthy +Stefan Farmbauer +Stefan Scherfke +Stefan Zimmermann +Stefano Taschini +Steffen Allner +Stephan Obermann +Sven-Hendrik Haase +Sylvain Marié +Tadek Teleżyński +Takafumi Arakaki +Taneli Hukkinen +Tanvi Mehta +Tarcisio Fischer +Tareq Alayan +Ted Xiao +Terje Runde +Thomas Grainger +Thomas Hisch +Tim Hoffmann +Tim Strazny +Tom Dalton +Tom Viner +Tomáš Gavenčiak +Tomer Keren +Tor Colvin +Trevor Bekolay +Tyler Goodlet +Tzu-ping Chung +Vasily Kuznetsov +Victor Maryama +Victor Uriarte +Vidar T. Fauske +Virgil Dupras +Vitaly Lashmanov +Vlad Dragos +Vlad Radziuk +Vladyslav Rachek +Volodymyr Piskun +Wei Lin +Wil Cooley +William Lee +Wim Glenn +Wouter van Ackooy +Xixi Zhao +Xuan Luong +Xuecong Liao +Yoav Caspi +Yuval Shimon +Zac Hatfield-Dodds +Zachary Kneupper +Zoltán Máté +Zsolt Cserna diff --git a/testing/web-platform/tests/tools/third_party/pytest/CHANGELOG.rst b/testing/web-platform/tests/tools/third_party/pytest/CHANGELOG.rst new file mode 100644 index 0000000000..481f277813 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/CHANGELOG.rst @@ -0,0 +1,7 @@ +========= +Changelog +========= + +The pytest CHANGELOG is located `here `__. + +The source document can be found at: https://github.com/pytest-dev/pytest/blob/main/doc/en/changelog.rst diff --git a/testing/web-platform/tests/tools/third_party/pytest/CITATION b/testing/web-platform/tests/tools/third_party/pytest/CITATION new file mode 100644 index 0000000000..d4e9d8ec7a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/CITATION @@ -0,0 +1,16 @@ +NOTE: Change "x.y" by the version you use. If you are unsure about which version +you are using run: `pip show pytest`. + +Text: + +[pytest] pytest x.y, 2004 +Krekel et al., https://github.com/pytest-dev/pytest + +BibTeX: + +@misc{pytestx.y, + title = {pytest x.y}, + author = {Krekel, Holger and Oliveira, Bruno and Pfannschmidt, Ronny and Bruynooghe, Floris and Laugher, Brianna and Bruhin, Florian}, + year = {2004}, + url = {https://github.com/pytest-dev/pytest}, +} diff --git a/testing/web-platform/tests/tools/third_party/pytest/CODE_OF_CONDUCT.md b/testing/web-platform/tests/tools/third_party/pytest/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..f0ca304be4 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/CODE_OF_CONDUCT.md @@ -0,0 +1,83 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at coc@pytest.org. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +The coc@pytest.org address is routed to the following people who can also be +contacted individually: + +- Brianna Laugher ([@pfctdayelise](https://github.com/pfctdayelise)): brianna@laugher.id.au +- Bruno Oliveira ([@nicoddemus](https://github.com/nicoddemus)): nicoddemus@gmail.com +- Florian Bruhin ([@the-compiler](https://github.com/the-compiler)): pytest@the-compiler.org + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/testing/web-platform/tests/tools/third_party/pytest/CONTRIBUTING.rst b/testing/web-platform/tests/tools/third_party/pytest/CONTRIBUTING.rst new file mode 100644 index 0000000000..24bca723c8 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/CONTRIBUTING.rst @@ -0,0 +1,481 @@ +============================ +Contribution getting started +============================ + +Contributions are highly welcomed and appreciated. Every little bit of help counts, +so do not hesitate! + +.. contents:: + :depth: 2 + :backlinks: none + + +.. _submitfeedback: + +Feature requests and feedback +----------------------------- + +Do you like pytest? Share some love on Twitter or in your blog posts! + +We'd also like to hear about your propositions and suggestions. Feel free to +`submit them as issues `_ and: + +* Explain in detail how they should work. +* Keep the scope as narrow as possible. This will make it easier to implement. + + +.. _reportbugs: + +Report bugs +----------- + +Report bugs for pytest in the `issue tracker `_. + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting, + specifically the Python interpreter version, installed libraries, and pytest + version. +* Detailed steps to reproduce the bug. + +If you can write a demonstration test that currently fails but should pass +(xfail), that is a very useful commit to make as well, even if you cannot +fix the bug itself. + + +.. _fixbugs: + +Fix bugs +-------- + +Look through the `GitHub issues for bugs `_. + +:ref:`Talk ` to developers to find out how you can fix specific bugs. To indicate that you are going +to work on a particular issue, add a comment to that effect on the specific issue. + +Don't forget to check the issue trackers of your favourite plugins, too! + +.. _writeplugins: + +Implement features +------------------ + +Look through the `GitHub issues for enhancements `_. + +:ref:`Talk ` to developers to find out how you can implement specific +features. + +Write documentation +------------------- + +Pytest could always use more documentation. What exactly is needed? + +* More complementary documentation. Have you perhaps found something unclear? +* Documentation translations. We currently have only English. +* Docstrings. There can never be too many of them. +* Blog posts, articles and such -- they're all very appreciated. + +You can also edit documentation files directly in the GitHub web interface, +without using a local copy. This can be convenient for small fixes. + +.. note:: + Build the documentation locally with the following command: + + .. code:: bash + + $ tox -e docs + + The built documentation should be available in ``doc/en/_build/html``, + where 'en' refers to the documentation language. + +Pytest has an API reference which in large part is +`generated automatically `_ +from the docstrings of the documented items. Pytest uses the +`Sphinx docstring format `_. +For example: + +.. code-block:: python + + def my_function(arg: ArgType) -> Foo: + """Do important stuff. + + More detailed info here, in separate paragraphs from the subject line. + Use proper sentences -- start sentences with capital letters and end + with periods. + + Can include annotated documentation: + + :param short_arg: An argument which determines stuff. + :param long_arg: + A long explanation which spans multiple lines, overflows + like this. + :returns: The result. + :raises ValueError: + Detailed information when this can happen. + + .. versionadded:: 6.0 + + Including types into the annotations above is not necessary when + type-hinting is being used (as in this example). + """ + + +.. _submitplugin: + +Submitting Plugins to pytest-dev +-------------------------------- + +Pytest development of the core, some plugins and support code happens +in repositories living under the ``pytest-dev`` organisations: + +- `pytest-dev on GitHub `_ + +All pytest-dev Contributors team members have write access to all contained +repositories. Pytest core and plugins are generally developed +using `pull requests`_ to respective repositories. + +The objectives of the ``pytest-dev`` organisation are: + +* Having a central location for popular pytest plugins +* Sharing some of the maintenance responsibility (in case a maintainer no + longer wishes to maintain a plugin) + +You can submit your plugin by subscribing to the `pytest-dev mail list +`_ and writing a +mail pointing to your existing pytest plugin repository which must have +the following: + +- PyPI presence with packaging metadata that contains a ``pytest-`` + prefixed name, version number, authors, short and long description. + +- a `tox configuration `_ + for running tests using `tox `_. + +- a ``README`` describing how to use the plugin and on which + platforms it runs. + +- a ``LICENSE`` file containing the licensing information, with + matching info in its packaging metadata. + +- an issue tracker for bug reports and enhancement requests. + +- a `changelog `_. + +If no contributor strongly objects and two agree, the repository can then be +transferred to the ``pytest-dev`` organisation. + +Here's a rundown of how a repository transfer usually proceeds +(using a repository named ``joedoe/pytest-xyz`` as example): + +* ``joedoe`` transfers repository ownership to ``pytest-dev`` administrator ``calvin``. +* ``calvin`` creates ``pytest-xyz-admin`` and ``pytest-xyz-developers`` teams, inviting ``joedoe`` to both as **maintainer**. +* ``calvin`` transfers repository to ``pytest-dev`` and configures team access: + + - ``pytest-xyz-admin`` **admin** access; + - ``pytest-xyz-developers`` **write** access; + +The ``pytest-dev/Contributors`` team has write access to all projects, and +every project administrator is in it. We recommend that each plugin has at least three +people who have the right to release to PyPI. + +Repository owners can rest assured that no ``pytest-dev`` administrator will ever make +releases of your repository or take ownership in any way, except in rare cases +where someone becomes unresponsive after months of contact attempts. +As stated, the objective is to share maintenance and avoid "plugin-abandon". + + +.. _`pull requests`: +.. _pull-requests: + +Preparing Pull Requests +----------------------- + +Short version +~~~~~~~~~~~~~ + +#. Fork the repository. +#. Enable and install `pre-commit `_ to ensure style-guides and code checks are followed. +#. Follow **PEP-8** for naming and `black `_ for formatting. +#. Tests are run using ``tox``:: + + tox -e linting,py37 + + The test environments above are usually enough to cover most cases locally. + +#. Write a ``changelog`` entry: ``changelog/2574.bugfix.rst``, use issue id number + and one of ``feature``, ``improvement``, ``bugfix``, ``doc``, ``deprecation``, + ``breaking``, ``vendor`` or ``trivial`` for the issue type. + + +#. Unless your change is a trivial or a documentation fix (e.g., a typo or reword of a small section) please + add yourself to the ``AUTHORS`` file, in alphabetical order. + + +Long version +~~~~~~~~~~~~ + +What is a "pull request"? It informs the project's core developers about the +changes you want to review and merge. Pull requests are stored on +`GitHub servers `_. +Once you send a pull request, we can discuss its potential modifications and +even add more commits to it later on. There's an excellent tutorial on how Pull +Requests work in the +`GitHub Help Center `_. + +Here is a simple overview, with pytest-specific bits: + +#. Fork the + `pytest GitHub repository `__. It's + fine to use ``pytest`` as your fork repository name because it will live + under your user. + +#. Clone your fork locally using `git `_ and create a branch:: + + $ git clone git@github.com:YOUR_GITHUB_USERNAME/pytest.git + $ cd pytest + # now, create your own branch off "main": + + $ git checkout -b your-bugfix-branch-name main + + Given we have "major.minor.micro" version numbers, bug fixes will usually + be released in micro releases whereas features will be released in + minor releases and incompatible changes in major releases. + + If you need some help with Git, follow this quick start + guide: https://git.wiki.kernel.org/index.php/QuickStart + +#. Install `pre-commit `_ and its hook on the pytest repo:: + + $ pip install --user pre-commit + $ pre-commit install + + Afterwards ``pre-commit`` will run whenever you commit. + + https://pre-commit.com/ is a framework for managing and maintaining multi-language pre-commit hooks + to ensure code-style and code formatting is consistent. + +#. Install tox + + Tox is used to run all the tests and will automatically setup virtualenvs + to run the tests in. + (will implicitly use https://virtualenv.pypa.io/en/latest/):: + + $ pip install tox + +#. Run all the tests + + You need to have Python 3.7 available in your system. Now + running tests is as simple as issuing this command:: + + $ tox -e linting,py37 + + This command will run tests via the "tox" tool against Python 3.7 + and also perform "lint" coding-style checks. + +#. You can now edit your local working copy and run the tests again as necessary. Please follow PEP-8 for naming. + + You can pass different options to ``tox``. For example, to run tests on Python 3.7 and pass options to pytest + (e.g. enter pdb on failure) to pytest you can do:: + + $ tox -e py37 -- --pdb + + Or to only run tests in a particular test module on Python 3.7:: + + $ tox -e py37 -- testing/test_config.py + + + When committing, ``pre-commit`` will re-format the files if necessary. + +#. If instead of using ``tox`` you prefer to run the tests directly, then we suggest to create a virtual environment and use + an editable install with the ``testing`` extra:: + + $ python3 -m venv .venv + $ source .venv/bin/activate # Linux + $ .venv/Scripts/activate.bat # Windows + $ pip install -e ".[testing]" + + Afterwards, you can edit the files and run pytest normally:: + + $ pytest testing/test_config.py + +#. Create a new changelog entry in ``changelog``. The file should be named ``..rst``, + where *issueid* is the number of the issue related to the change and *type* is one of + ``feature``, ``improvement``, ``bugfix``, ``doc``, ``deprecation``, ``breaking``, ``vendor`` + or ``trivial``. You may skip creating the changelog entry if the change doesn't affect the + documented behaviour of pytest. + +#. Add yourself to ``AUTHORS`` file if not there yet, in alphabetical order. + +#. Commit and push once your tests pass and you are happy with your change(s):: + + $ git commit -a -m "" + $ git push -u + +#. Finally, submit a pull request through the GitHub website using this data:: + + head-fork: YOUR_GITHUB_USERNAME/pytest + compare: your-branch-name + + base-fork: pytest-dev/pytest + base: main + + +Writing Tests +~~~~~~~~~~~~~ + +Writing tests for plugins or for pytest itself is often done using the `pytester fixture `_, as a "black-box" test. + +For example, to ensure a simple test passes you can write: + +.. code-block:: python + + def test_true_assertion(pytester): + pytester.makepyfile( + """ + def test_foo(): + assert True + """ + ) + result = pytester.runpytest() + result.assert_outcomes(failed=0, passed=1) + + +Alternatively, it is possible to make checks based on the actual output of the termal using +*glob-like* expressions: + +.. code-block:: python + + def test_true_assertion(pytester): + pytester.makepyfile( + """ + def test_foo(): + assert False + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*assert False*", "*1 failed*"]) + +When choosing a file where to write a new test, take a look at the existing files and see if there's +one file which looks like a good fit. For example, a regression test about a bug in the ``--lf`` option +should go into ``test_cacheprovider.py``, given that this option is implemented in ``cacheprovider.py``. +If in doubt, go ahead and open a PR with your best guess and we can discuss this over the code. + +Joining the Development Team +---------------------------- + +Anyone who has successfully seen through a pull request which did not +require any extra work from the development team to merge will +themselves gain commit access if they so wish (if we forget to ask please send a friendly +reminder). This does not mean there is any change in your contribution workflow: +everyone goes through the same pull-request-and-review process and +no-one merges their own pull requests unless already approved. It does however mean you can +participate in the development process more fully since you can merge +pull requests from other contributors yourself after having reviewed +them. + + +Backporting bug fixes for the next patch release +------------------------------------------------ + +Pytest makes feature release every few weeks or months. In between, patch releases +are made to the previous feature release, containing bug fixes only. The bug fixes +usually fix regressions, but may be any change that should reach users before the +next feature release. + +Suppose for example that the latest release was 1.2.3, and you want to include +a bug fix in 1.2.4 (check https://github.com/pytest-dev/pytest/releases for the +actual latest release). The procedure for this is: + +#. First, make sure the bug is fixed the ``main`` branch, with a regular pull + request, as described above. An exception to this is if the bug fix is not + applicable to ``main`` anymore. + +#. ``git checkout origin/1.2.x -b backport-XXXX`` # use the main PR number here + +#. Locate the merge commit on the PR, in the *merged* message, for example: + + nicoddemus merged commit 0f8b462 into pytest-dev:main + +#. ``git cherry-pick -x -m1 REVISION`` # use the revision you found above (``0f8b462``). + +#. Open a PR targeting ``1.2.x``: + + * Prefix the message with ``[1.2.x]``. + * Delete the PR body, it usually contains a duplicate commit message. + + +Who does the backporting +~~~~~~~~~~~~~~~~~~~~~~~~ + +As mentioned above, bugs should first be fixed on ``main`` (except in rare occasions +that a bug only happens in a previous release). So, who should do the backport procedure described +above? + +1. If the bug was fixed by a core developer, it is the main responsibility of that core developer + to do the backport. +2. However, often the merge is done by another maintainer, in which case it is nice of them to + do the backport procedure if they have the time. +3. For bugs submitted by non-maintainers, it is expected that a core developer will to do + the backport, normally the one that merged the PR on ``main``. +4. If a non-maintainers notices a bug which is fixed on ``main`` but has not been backported + (due to maintainers forgetting to apply the *needs backport* label, or just plain missing it), + they are also welcome to open a PR with the backport. The procedure is simple and really + helps with the maintenance of the project. + +All the above are not rules, but merely some guidelines/suggestions on what we should expect +about backports. + +Handling stale issues/PRs +------------------------- + +Stale issues/PRs are those where pytest contributors have asked for questions/changes +and the authors didn't get around to answer/implement them yet after a somewhat long time, or +the discussion simply died because people seemed to lose interest. + +There are many reasons why people don't answer questions or implement requested changes: +they might get busy, lose interest, or just forget about it, +but the fact is that this is very common in open source software. + +The pytest team really appreciates every issue and pull request, but being a high-volume project +with many issues and pull requests being submitted daily, we try to reduce the number of stale +issues and PRs by regularly closing them. When an issue/pull request is closed in this manner, +it is by no means a dismissal of the topic being tackled by the issue/pull request, but it +is just a way for us to clear up the queue and make the maintainers' work more manageable. Submitters +can always reopen the issue/pull request in their own time later if it makes sense. + +When to close +~~~~~~~~~~~~~ + +Here are a few general rules the maintainers use deciding when to close issues/PRs because +of lack of inactivity: + +* Issues labeled ``question`` or ``needs information``: closed after 14 days inactive. +* Issues labeled ``proposal``: closed after six months inactive. +* Pull requests: after one month, consider pinging the author, update linked issue, or consider closing. For pull requests which are nearly finished, the team should consider finishing it up and merging it. + +The above are **not hard rules**, but merely **guidelines**, and can be (and often are!) reviewed on a case-by-case basis. + +Closing pull requests +~~~~~~~~~~~~~~~~~~~~~ + +When closing a Pull Request, it needs to be acknowledging the time, effort, and interest demonstrated by the person which submitted it. As mentioned previously, it is not the intent of the team to dismiss a stalled pull request entirely but to merely to clear up our queue, so a message like the one below is warranted when closing a pull request that went stale: + + Hi , + + First of all, we would like to thank you for your time and effort on working on this, the pytest team deeply appreciates it. + + We noticed it has been awhile since you have updated this PR, however. pytest is a high activity project, with many issues/PRs being opened daily, so it is hard for us maintainers to track which PRs are ready for merging, for review, or need more attention. + + So for those reasons we, think it is best to close the PR for now, but with the only intention to clean up our queue, it is by no means a rejection of your changes. We still encourage you to re-open this PR (it is just a click of a button away) when you are ready to get back to it. + + Again we appreciate your time for working on this, and hope you might get back to this at a later time! + + + +Closing Issues +-------------- + +When a pull request is submitted to fix an issue, add text like ``closes #XYZW`` to the PR description and/or commits (where ``XYZW`` is the issue number). See the `GitHub docs `_ for more information. + +When an issue is due to user error (e.g. misunderstanding of a functionality), please politely explain to the user why the issue raised is really a non-issue and ask them to close the issue if they have no further questions. If the original requestor is unresponsive, the issue will be handled as described in the section `Handling stale issues/PRs`_ above. diff --git a/testing/web-platform/tests/tools/third_party/pytest/LICENSE b/testing/web-platform/tests/tools/third_party/pytest/LICENSE new file mode 100644 index 0000000000..c3f1657fce --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2004 Holger Krekel and others + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/testing/web-platform/tests/tools/third_party/pytest/OPENCOLLECTIVE.rst b/testing/web-platform/tests/tools/third_party/pytest/OPENCOLLECTIVE.rst new file mode 100644 index 0000000000..8c1c90281e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/OPENCOLLECTIVE.rst @@ -0,0 +1,44 @@ +============== +OpenCollective +============== + +pytest has a collective setup at `OpenCollective`_. This document describes how the core team manages +OpenCollective-related activities. + +What is it +========== + +Open Collective is an online funding platform for open and transparent communities. +It provides tools to raise money and share your finances in full transparency. + +It is the platform of choice for individuals and companies that want to make one-time or +monthly donations directly to the project. + +Funds +===== + +The OpenCollective funds donated to pytest will be used to fund overall maintenance, +local sprints, merchandising (stickers to distribute in conferences for example), and future +gatherings of pytest developers (sprints). + +`Core contributors`_ which are contributing on a continuous basis are free to submit invoices +to bill maintenance hours using the platform. How much each contributor should request is still an +open question, but we should use common sense and trust in the contributors, most of which know +themselves in-person. A good rule of thumb is to bill the same amount as monthly payments +contributors which participate in the `Tidelift`_ subscription. If in doubt, just ask. + +Admins +====== + +A few people have admin access to the OpenCollective dashboard to make changes. Those people +are part of the `@pytest-dev/opencollective-admins`_ team. + +`Core contributors`_ interested in helping out with OpenCollective maintenance are welcome! We don't +expect much work here other than the occasional approval of expenses from other core contributors. +Just drop a line to one of the `@pytest-dev/opencollective-admins`_ or use the mailing list. + + +.. _`OpenCollective`: https://opencollective.com/pytest +.. _`Tidelift`: https://tidelift.com +.. _`core contributors`: https://github.com/orgs/pytest-dev/teams/core/members +.. _`@pytest-dev/opencollective-admins`: https://github.com/orgs/pytest-dev/teams/opencollective-admins/members diff --git a/testing/web-platform/tests/tools/third_party/pytest/README.rst b/testing/web-platform/tests/tools/third_party/pytest/README.rst new file mode 100644 index 0000000000..1473376517 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/README.rst @@ -0,0 +1,167 @@ +.. image:: https://github.com/pytest-dev/pytest/raw/main/doc/en/img/pytest_logo_curves.svg + :target: https://docs.pytest.org/en/stable/ + :align: center + :height: 200 + :alt: pytest + + +------ + +.. image:: https://img.shields.io/pypi/v/pytest.svg + :target: https://pypi.org/project/pytest/ + +.. image:: https://img.shields.io/conda/vn/conda-forge/pytest.svg + :target: https://anaconda.org/conda-forge/pytest + +.. image:: https://img.shields.io/pypi/pyversions/pytest.svg + :target: https://pypi.org/project/pytest/ + +.. image:: https://codecov.io/gh/pytest-dev/pytest/branch/main/graph/badge.svg + :target: https://codecov.io/gh/pytest-dev/pytest + :alt: Code coverage Status + +.. image:: https://github.com/pytest-dev/pytest/workflows/main/badge.svg + :target: https://github.com/pytest-dev/pytest/actions?query=workflow%3Amain + +.. image:: https://results.pre-commit.ci/badge/github/pytest-dev/pytest/main.svg + :target: https://results.pre-commit.ci/latest/github/pytest-dev/pytest/main + :alt: pre-commit.ci status + +.. image:: https://img.shields.io/badge/code%20style-black-000000.svg + :target: https://github.com/psf/black + +.. image:: https://www.codetriage.com/pytest-dev/pytest/badges/users.svg + :target: https://www.codetriage.com/pytest-dev/pytest + +.. image:: https://readthedocs.org/projects/pytest/badge/?version=latest + :target: https://pytest.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +.. image:: https://img.shields.io/badge/Discord-pytest--dev-blue + :target: https://discord.com/invite/pytest-dev + :alt: Discord + +.. image:: https://img.shields.io/badge/Libera%20chat-%23pytest-orange + :target: https://web.libera.chat/#pytest + :alt: Libera chat + + +The ``pytest`` framework makes it easy to write small tests, yet +scales to support complex functional testing for applications and libraries. + +An example of a simple test: + +.. code-block:: python + + # content of test_sample.py + def inc(x): + return x + 1 + + + def test_answer(): + assert inc(3) == 5 + + +To execute it:: + + $ pytest + ============================= test session starts ============================= + collected 1 items + + test_sample.py F + + ================================== FAILURES =================================== + _________________________________ test_answer _________________________________ + + def test_answer(): + > assert inc(3) == 5 + E assert 4 == 5 + E + where 4 = inc(3) + + test_sample.py:5: AssertionError + ========================== 1 failed in 0.04 seconds =========================== + + +Due to ``pytest``'s detailed assertion introspection, only plain ``assert`` statements are used. See `getting-started `_ for more examples. + + +Features +-------- + +- Detailed info on failing `assert statements `_ (no need to remember ``self.assert*`` names) + +- `Auto-discovery + `_ + of test modules and functions + +- `Modular fixtures `_ for + managing small or parametrized long-lived test resources + +- Can run `unittest `_ (or trial), + `nose `_ test suites out of the box + +- Python 3.6+ and PyPy3 + +- Rich plugin architecture, with over 850+ `external plugins `_ and thriving community + + +Documentation +------------- + +For full documentation, including installation, tutorials and PDF documents, please see https://docs.pytest.org/en/stable/. + + +Bugs/Requests +------------- + +Please use the `GitHub issue tracker `_ to submit bugs or request features. + + +Changelog +--------- + +Consult the `Changelog `__ page for fixes and enhancements of each version. + + +Support pytest +-------------- + +`Open Collective`_ is an online funding platform for open and transparent communities. +It provides tools to raise money and share your finances in full transparency. + +It is the platform of choice for individuals and companies that want to make one-time or +monthly donations directly to the project. + +See more details in the `pytest collective`_. + +.. _Open Collective: https://opencollective.com +.. _pytest collective: https://opencollective.com/pytest + + +pytest for enterprise +--------------------- + +Available as part of the Tidelift Subscription. + +The maintainers of pytest and thousands of other packages are working with Tidelift to deliver commercial support and +maintenance for the open source dependencies you use to build your applications. +Save time, reduce risk, and improve code health, while paying the maintainers of the exact dependencies you use. + +`Learn more. `_ + +Security +^^^^^^^^ + +pytest has never been associated with a security vulnerability, but in any case, to report a +security vulnerability please use the `Tidelift security contact `_. +Tidelift will coordinate the fix and disclosure. + + +License +------- + +Copyright Holger Krekel and others, 2004. + +Distributed under the terms of the `MIT`_ license, pytest is free and open source software. + +.. _`MIT`: https://github.com/pytest-dev/pytest/blob/main/LICENSE diff --git a/testing/web-platform/tests/tools/third_party/pytest/RELEASING.rst b/testing/web-platform/tests/tools/third_party/pytest/RELEASING.rst new file mode 100644 index 0000000000..25ce90d0f6 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/RELEASING.rst @@ -0,0 +1,173 @@ +Release Procedure +----------------- + +Our current policy for releasing is to aim for a bug-fix release every few weeks and a minor release every 2-3 months. The idea +is to get fixes and new features out instead of trying to cram a ton of features into a release and by consequence +taking a lot of time to make a new one. + +The git commands assume the following remotes are setup: + +* ``origin``: your own fork of the repository. +* ``upstream``: the ``pytest-dev/pytest`` official repository. + +Preparing: Automatic Method +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We have developed an automated workflow for releases, that uses GitHub workflows and is triggered +by `manually running `__ +the `prepare-release-pr workflow `__ +on GitHub Actions. + +The automation will decide the new version number based on the following criteria: + +- If the "major release" input is set to "yes", release a new major release + (e.g. 7.0.0 -> 8.0.0) +- If there are any ``.feature.rst`` or ``.breaking.rst`` files in the + ``changelog`` directory, release a new minor release (e.g. 7.0.0 -> 7.1.0) +- Otherwise, release a bugfix release (e.g. 7.0.0 -> 7.0.1) +- If the "prerelease" input is set, append the string to the version number + (e.g. 7.0.0 -> 8.0.0rc1), if "major" is set, and "prerelease" is set to `rc1`) + +Bug-fix and minor releases +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Bug-fix and minor releases are always done from a maintenance branch. First, +consider double-checking the ``changelog`` directory to see if there are any +breaking changes or new features. + +For a new minor release, first create a new maintenance branch from ``main``:: + + git fetch --all + git branch 7.1.x upstream/main + git push upstream 7.1.x + +Then, trigger the workflow with the following inputs: + +- branch: **7.1.x** +- major release: **no** +- prerelease: empty + +Or via the commandline using `GitHub's cli `__:: + + gh workflow run prepare-release-pr.yml -f branch=7.1.x -f major=no -f prerelease= + +Where ``7.1.x`` is the maintenance branch for the ``7.1`` series. The automated +workflow will publish a PR for a branch ``release-7.1.0``. + +Similarly, for a bug-fix release, use the existing maintenance branch and +trigger the workflow with e.g. ``branch: 7.0.x`` to get a new ``release-7.0.1`` +PR. + +Major releases +^^^^^^^^^^^^^^ + +1. Create a new maintenance branch from ``main``:: + + git fetch --all + git branch 8.0.x upstream/main + git push upstream 8.0.x + +2. Trigger the workflow with the following inputs: + + - branch: **8.0.x** + - major release: **yes** + - prerelease: empty + +Or via the commandline:: + + gh workflow run prepare-release-pr.yml -f branch=8.0.x -f major=yes -f prerelease= + +The automated workflow will publish a PR for a branch ``release-8.0.0``. + +At this point on, this follows the same workflow as other maintenance branches: bug-fixes are merged +into ``main`` and ported back to the maintenance branch, even for release candidates. + +Release candidates +^^^^^^^^^^^^^^^^^^ + +To release a release candidate, set the "prerelease" input to the version number +suffix to use. To release a ``8.0.0rc1``, proceed like under "major releases", but set: + +- branch: 8.0.x +- major release: yes +- prerelease: **rc1** + +Or via the commandline:: + + gh workflow run prepare-release-pr.yml -f branch=8.0.x -f major=yes -f prerelease=rc1 + +The automated workflow will publish a PR for a branch ``release-8.0.0rc1``. + +**A note about release candidates** + +During release candidates we can merge small improvements into +the maintenance branch before releasing the final major version, however we must take care +to avoid introducing big changes at this stage. + +Preparing: Manual Method +~~~~~~~~~~~~~~~~~~~~~~~~ + +**Important**: pytest releases must be prepared on **Linux** because the docs and examples expect +to be executed on that platform. + +To release a version ``MAJOR.MINOR.PATCH``, follow these steps: + +#. For major and minor releases, create a new branch ``MAJOR.MINOR.x`` from + ``upstream/main`` and push it to ``upstream``. + +#. Create a branch ``release-MAJOR.MINOR.PATCH`` from the ``MAJOR.MINOR.x`` branch. + + Ensure your are updated and in a clean working tree. + +#. Using ``tox``, generate docs, changelog, announcements:: + + $ tox -e release -- MAJOR.MINOR.PATCH + + This will generate a commit with all the changes ready for pushing. + +#. Open a PR for the ``release-MAJOR.MINOR.PATCH`` branch targeting ``MAJOR.MINOR.x``. + + +Releasing +~~~~~~~~~ + +Both automatic and manual processes described above follow the same steps from this point onward. + +#. After all tests pass and the PR has been approved, tag the release commit + in the ``release-MAJOR.MINOR.PATCH`` branch and push it. This will publish to PyPI:: + + git fetch --all + git tag MAJOR.MINOR.PATCH upstream/release-MAJOR.MINOR.PATCH + git push git@github.com:pytest-dev/pytest.git MAJOR.MINOR.PATCH + + Wait for the deploy to complete, then make sure it is `available on PyPI `_. + +#. Merge the PR. + +#. Cherry-pick the CHANGELOG / announce files to the ``main`` branch:: + + git fetch --all --prune + git checkout upstream/main -b cherry-pick-release + git cherry-pick -x -m1 upstream/MAJOR.MINOR.x + +#. Open a PR for ``cherry-pick-release`` and merge it once CI passes. No need to wait for approvals if there were no conflicts on the previous step. + +#. For major and minor releases, tag the release cherry-pick merge commit in main with + a dev tag for the next feature release:: + + git checkout main + git pull + git tag MAJOR.{MINOR+1}.0.dev0 + git push git@github.com:pytest-dev/pytest.git MAJOR.{MINOR+1}.0.dev0 + +#. Send an email announcement with the contents from:: + + doc/en/announce/release-.rst + + To the following mailing lists: + + * pytest-dev@python.org (all releases) + * python-announce-list@python.org (all releases) + * testing-in-python@lists.idyll.org (only major/minor releases) + + And announce it on `Twitter `_ with the ``#pytest`` hashtag. diff --git a/testing/web-platform/tests/tools/third_party/pytest/TIDELIFT.rst b/testing/web-platform/tests/tools/third_party/pytest/TIDELIFT.rst new file mode 100644 index 0000000000..2fe25841c3 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/TIDELIFT.rst @@ -0,0 +1,60 @@ +======== +Tidelift +======== + +pytest is a member of `Tidelift`_. This document describes how the core team manages +Tidelift-related activities. + +What is it +========== + +Tidelift aims to make Open Source sustainable by offering subscriptions to companies which rely +on Open Source packages. This subscription allows it to pay maintainers of those Open Source +packages to aid sustainability of the work. + +It is the perfect platform for companies that want to support Open Source packages and at the same +time obtain assurances regarding maintenance, quality and security. + +Funds +===== + +It was decided in the `mailing list`_ that the Tidelift contribution will be split evenly between +members of the `contributors team`_ interested in receiving funding. + +The current list of contributors receiving funding are: + +* `@asottile`_ +* `@nicoddemus`_ +* `@The-Compiler`_ + +Contributors interested in receiving a part of the funds just need to submit a PR adding their +name to the list. Contributors that want to stop receiving the funds should also submit a PR +in the same way. + +The PR should mention `@pytest-dev/tidelift-admins`_ so appropriate changes +can be made in the Tidelift platform. + +After the PR has been accepted and merged, the contributor should register in the `Tidelift`_ +platform and follow the instructions there, including signing an `agreement`_. + +Admins +====== + +A few people have admin access to the Tidelift dashboard to make changes. Those people +are part of the `@pytest-dev/tidelift-admins`_ team. + +`Core contributors`_ interested in helping out with Tidelift maintenance are welcome! We don't +expect much work here other than the occasional adding/removal of a contributor from receiving +funds. Just drop a line to one of the `@pytest-dev/tidelift-admins`_ or use the mailing list. + + +.. _`Tidelift`: https://tidelift.com +.. _`mailing list`: https://mail.python.org/pipermail/pytest-dev/2019-May/004716.html +.. _`contributors team`: https://github.com/orgs/pytest-dev/teams/contributors +.. _`core contributors`: https://github.com/orgs/pytest-dev/teams/core/members +.. _`@pytest-dev/tidelift-admins`: https://github.com/orgs/pytest-dev/teams/tidelift-admins/members +.. _`agreement`: https://tidelift.com/docs/lifting/agreement + +.. _`@asottile`: https://github.com/asottile +.. _`@nicoddemus`: https://github.com/nicoddemus +.. _`@The-Compiler`: https://github.com/The-Compiler diff --git a/testing/web-platform/tests/tools/third_party/pytest/bench/bench.py b/testing/web-platform/tests/tools/third_party/pytest/bench/bench.py new file mode 100644 index 0000000000..c40fc8636c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/bench/bench.py @@ -0,0 +1,13 @@ +import sys + +if __name__ == "__main__": + import cProfile + import pytest # NOQA + import pstats + + script = sys.argv[1:] if len(sys.argv) > 1 else ["empty.py"] + cProfile.run("pytest.cmdline.main(%r)" % script, "prof") + p = pstats.Stats("prof") + p.strip_dirs() + p.sort_stats("cumulative") + print(p.print_stats(500)) diff --git a/testing/web-platform/tests/tools/third_party/pytest/bench/bench_argcomplete.py b/testing/web-platform/tests/tools/third_party/pytest/bench/bench_argcomplete.py new file mode 100644 index 0000000000..335733df72 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/bench/bench_argcomplete.py @@ -0,0 +1,19 @@ +# 10000 iterations, just for relative comparison +# 2.7.5 3.3.2 +# FilesCompleter 75.1109 69.2116 +# FastFilesCompleter 0.7383 1.0760 +import timeit + +imports = [ + "from argcomplete.completers import FilesCompleter as completer", + "from _pytest._argcomplete import FastFilesCompleter as completer", +] + +count = 1000 # only a few seconds +setup = "%s\nfc = completer()" +run = 'fc("/d")' + + +if __name__ == "__main__": + print(timeit.timeit(run, setup=setup % imports[0], number=count)) + print(timeit.timeit(run, setup=setup % imports[1], number=count)) diff --git a/testing/web-platform/tests/tools/third_party/pytest/bench/empty.py b/testing/web-platform/tests/tools/third_party/pytest/bench/empty.py new file mode 100644 index 0000000000..4e7371b6f8 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/bench/empty.py @@ -0,0 +1,2 @@ +for i in range(1000): + exec("def test_func_%d(): pass" % i) diff --git a/testing/web-platform/tests/tools/third_party/pytest/bench/manyparam.py b/testing/web-platform/tests/tools/third_party/pytest/bench/manyparam.py new file mode 100644 index 0000000000..1226c73bd9 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/bench/manyparam.py @@ -0,0 +1,14 @@ +import pytest + + +@pytest.fixture(scope="module", params=range(966)) +def foo(request): + return request.param + + +def test_it(foo): + pass + + +def test_it2(foo): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/bench/skip.py b/testing/web-platform/tests/tools/third_party/pytest/bench/skip.py new file mode 100644 index 0000000000..f0c9d1ddbe --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/bench/skip.py @@ -0,0 +1,9 @@ +import pytest + +SKIP = True + + +@pytest.mark.parametrize("x", range(5000)) +def test_foo(x): + if SKIP: + pytest.skip("heh") diff --git a/testing/web-platform/tests/tools/third_party/pytest/bench/unit_test.py b/testing/web-platform/tests/tools/third_party/pytest/bench/unit_test.py new file mode 100644 index 0000000000..ad52069dbf --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/bench/unit_test.py @@ -0,0 +1,13 @@ +from unittest import TestCase # noqa: F401 + +for i in range(15000): + exec( + f""" +class Test{i}(TestCase): + @classmethod + def setUpClass(cls): pass + def test_1(self): pass + def test_2(self): pass + def test_3(self): pass +""" + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/bench/xunit.py b/testing/web-platform/tests/tools/third_party/pytest/bench/xunit.py new file mode 100644 index 0000000000..3a77dcdce4 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/bench/xunit.py @@ -0,0 +1,11 @@ +for i in range(5000): + exec( + f""" +class Test{i}: + @classmethod + def setup_class(cls): pass + def test_1(self): pass + def test_2(self): pass + def test_3(self): pass +""" + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/changelog/README.rst b/testing/web-platform/tests/tools/third_party/pytest/changelog/README.rst new file mode 100644 index 0000000000..6d026f57ef --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/changelog/README.rst @@ -0,0 +1,37 @@ +This directory contains "newsfragments" which are short files that contain a small **ReST**-formatted +text that will be added to the next ``CHANGELOG``. + +The ``CHANGELOG`` will be read by **users**, so this description should be aimed to pytest users +instead of describing internal changes which are only relevant to the developers. + +Make sure to use full sentences in the **past or present tense** and use punctuation, examples:: + + Improved verbose diff output with sequences. + + Terminal summary statistics now use multiple colors. + +Each file should be named like ``..rst``, where +```` is an issue number, and ```` is one of: + +* ``feature``: new user facing features, like new command-line options and new behavior. +* ``improvement``: improvement of existing functionality, usually without requiring user intervention (for example, new fields being written in ``--junitxml``, improved colors in terminal, etc). +* ``bugfix``: fixes a bug. +* ``doc``: documentation improvement, like rewording an entire session or adding missing docs. +* ``deprecation``: feature deprecation. +* ``breaking``: a change which may break existing suites, such as feature removal or behavior change. +* ``vendor``: changes in packages vendored in pytest. +* ``trivial``: fixing a small typo or internal change that might be noteworthy. + +So for example: ``123.feature.rst``, ``456.bugfix.rst``. + +If your PR fixes an issue, use that number here. If there is no issue, +then after you submit the PR and get the PR number you can add a +changelog using that instead. + +If you are not sure what issue type to use, don't hesitate to ask in your PR. + +``towncrier`` preserves multiple paragraphs and formatting (code blocks, lists, and so on), but for entries +other than ``features`` it is usually better to stick to a single paragraph to keep it concise. + +You can also run ``tox -e docs`` to build the documentation +with the draft changelog (``doc/en/_build/html/changelog.html``) if you want to get a preview of how your change will look in the final release notes. diff --git a/testing/web-platform/tests/tools/third_party/pytest/changelog/_template.rst b/testing/web-platform/tests/tools/third_party/pytest/changelog/_template.rst new file mode 100644 index 0000000000..5de4ae97ea --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/changelog/_template.rst @@ -0,0 +1,40 @@ +{% for section in sections %} +{% set underline = "-" %} +{% if section %} +{{section}} +{{ underline * section|length }}{% set underline = "~" %} + +{% endif %} +{% if sections[section] %} +{% for category, val in definitions.items() if category in sections[section] %} + +{{ definitions[category]['name'] }} +{{ underline * definitions[category]['name']|length }} + +{% if definitions[category]['showcontent'] %} +{% for text, values in sections[section][category]|dictsort(by='value') %} +{% set issue_joiner = joiner(', ') %} +- {% for value in values|sort %}{{ issue_joiner() }}`{{ value }} `_{% endfor %}: {{ text }} + + +{% endfor %} +{% else %} +- {{ sections[section][category]['']|sort|join(', ') }} + + +{% endif %} +{% if sections[section][category]|length == 0 %} + +No significant changes. + + +{% else %} +{% endif %} +{% endfor %} +{% else %} + +No significant changes. + + +{% endif %} +{% endfor %} diff --git a/testing/web-platform/tests/tools/third_party/pytest/codecov.yml b/testing/web-platform/tests/tools/third_party/pytest/codecov.yml new file mode 100644 index 0000000000..f1cc869733 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/codecov.yml @@ -0,0 +1,6 @@ +# reference: https://docs.codecov.io/docs/codecovyml-reference +coverage: + status: + patch: true + project: false +comment: false diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/Makefile b/testing/web-platform/tests/tools/third_party/pytest/doc/en/Makefile new file mode 100644 index 0000000000..f2db689121 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/Makefile @@ -0,0 +1,43 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + + +REGENDOC_ARGS := \ + --normalize "/[ \t]+\n/\n/" \ + --normalize "~\$$REGENDOC_TMPDIR~/home/sweet/project~" \ + --normalize "~/path/to/example~/home/sweet/project~" \ + --normalize "/in \d.\d\ds/in 0.12s/" \ + --normalize "@/tmp/pytest-of-.*/pytest-\d+@PYTEST_TMPDIR@" \ + --normalize "@pytest-(\d+)\\.[^ ,]+@pytest-\1.x.y@" \ + --normalize "@py-(\d+)\\.[^ ,]+@py-\1.x.y@" \ + --normalize "@pluggy-(\d+)\\.[.\d,]+@pluggy-\1.x.y@" \ + --normalize "@hypothesis-(\d+)\\.[.\d,]+@hypothesis-\1.x.y@" \ + --normalize "@Python (\d+)\\.[^ ,]+@Python \1.x.y@" + +regen: REGENDOC_FILES:=*.rst */*.rst +regen: +# need to reset cachedir to the non-tox default + PYTHONDONTWRITEBYTECODE=1 \ + PYTEST_ADDOPTS="-pno:hypothesis -p no:hypothesispytest -Wignore::pytest.PytestUnknownMarkWarning -o cache_dir=.pytest_cache" \ + COLUMNS=76 \ + regendoc --update ${REGENDOC_FILES} ${REGENDOC_ARGS} + +.PHONY: regen diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/globaltoc.html b/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/globaltoc.html new file mode 100644 index 0000000000..7c595e7ebf --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/globaltoc.html @@ -0,0 +1,34 @@ +

Contents

+ + + +

About the project

+ + + +{%- if display_toc %} +
+ {{ toc }} +{%- endif %} + +
+Index +
diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/layout.html b/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/layout.html new file mode 100644 index 0000000000..f7096eaaa5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/layout.html @@ -0,0 +1,52 @@ +{# + + Copied from: + + https://raw.githubusercontent.com/pallets/pallets-sphinx-themes/b0c6c41849b4e15cbf62cc1d95c05ef2b3e155c8/src/pallets_sphinx_themes/themes/pocoo/layout.html + + And removed the warning version (see #7331). + +#} + +{% extends "basic/layout.html" %} + +{% set metatags %} + {{- metatags }} + +{%- endset %} + +{% block extrahead %} + {%- if page_canonical_url %} + + {%- endif %} + + {{ super() }} +{%- endblock %} + +{% block sidebarlogo %} + {% if pagename != "index" or theme_index_sidebar_logo %} + {{ super() }} + {% endif %} +{% endblock %} + +{% block relbar2 %}{% endblock %} + +{% block sidebar2 %} + + {{- super() }} +{%- endblock %} + +{% block footer %} + {{ super() }} + {%- if READTHEDOCS and not readthedocs_docsearch %} + + {%- endif %} + {{ js_tag("_static/version_warning_offset.js") }} +{% endblock %} diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/links.html b/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/links.html new file mode 100644 index 0000000000..c253ecabfd --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/links.html @@ -0,0 +1,7 @@ +

Useful Links

+ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/relations.html b/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/relations.html new file mode 100644 index 0000000000..3bbcde85bb --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/relations.html @@ -0,0 +1,19 @@ +

Related Topics

+ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/sidebarintro.html b/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/sidebarintro.html new file mode 100644 index 0000000000..ae860c172f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/sidebarintro.html @@ -0,0 +1,5 @@ +

About pytest

+

+ pytest is a mature full-featured Python testing tool that helps + you write better programs. +

diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/slim_searchbox.html b/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/slim_searchbox.html new file mode 100644 index 0000000000..e98ad4ed90 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/_templates/slim_searchbox.html @@ -0,0 +1,15 @@ +{# + basic/searchbox.html with heading removed. +#} +{%- if pagename != "search" and builder != "singlehtml" %} + + +{%- endif %} diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/adopt.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/adopt.rst new file mode 100644 index 0000000000..13d82bf011 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/adopt.rst @@ -0,0 +1,78 @@ +:orphan: + +.. warnings about this file not being included in any toctree will be suppressed by :orphan: + + +April 2015 is "adopt pytest month" +============================================= + +Are you an enthusiastic pytest user, the local testing guru in your workplace? Or are you considering using pytest for your open source project, but not sure how to get started? Then you may be interested in "adopt pytest month"! + +We will pair experienced pytest users with open source projects, for a month's effort of getting new development teams started with pytest. + +In 2015 we are trying this for the first time. In February and March 2015 we will gather volunteers on both sides, in April we will do the work, and in May we will evaluate how it went. This effort is being coordinated by Brianna Laugher. If you have any questions or comments, you can raise them on the `@pytestdotorg twitter account `_\, the :issue:`issue tracker <676>` or the `pytest-dev mailing list`_. + + +.. _`pytest-dev mailing list`: https://mail.python.org/mailman/listinfo/pytest-dev + + +The ideal pytest helper +----------------------------------------- + + - will be able to commit 2-4 hours a week to working with their particular project (this might involve joining their mailing list, installing the software and exploring any existing tests, offering advice, writing some example tests) + - feels confident in using pytest (e.g. has explored command line options, knows how to write parametrized tests, has an idea about conftest contents) + - does not need to be an expert in every aspect! + +Pytest helpers, sign up here! (preferably in February, hard deadline 22 March) + + + +The ideal partner project +----------------------------------------- + + - is open source, and predominantly written in Python + - has an automated/documented install process for developers + - has more than one core developer + - has at least one official release (e.g. is available on pypi) + - has the support of the core development team, in trying out pytest adoption + - has no tests... or 100% test coverage... or somewhere in between! + +Partner projects, sign up here! (by 22 March) + + + +What does it mean to "adopt pytest"? +----------------------------------------- + +There can be many different definitions of "success". Pytest can run many nose_ and unittest_ tests by default, so using pytest as your testrunner may be possible from day 1. Job done, right? + +Progressive success might look like: + + - tests can be run (by pytest) without errors (there may be failures) + - tests can be run (by pytest) without failures + - test runner is integrated into CI server + - existing tests are rewritten to take advantage of pytest features - this can happen in several iterations, for example: + - changing to native assert_ statements (pycmd_ has a script to help with that, ``pyconvert_unittest.py``) + - changing `setUp/tearDown methods`_ to fixtures_ + - adding markers_ + - other changes to reduce boilerplate + - assess needs for future tests to be written, e.g. new fixtures, distributed_ testing tweaks + +"Success" should also include that the development team feels comfortable with their knowledge of how to use pytest. In fact this is probably more important than anything else. So spending a lot of time on communication, giving examples, etc will probably be important - both in running the tests, and in writing them. + +It may be after the month is up, the partner project decides that pytest is not right for it. That's okay - hopefully the pytest team will also learn something about its weaknesses or deficiencies. + +.. _nose: nose.html +.. _unittest: unittest.html +.. _assert: assert.html +.. _pycmd: https://bitbucket.org/hpk42/pycmd/overview +.. _`setUp/tearDown methods`: xunit_setup.html +.. _fixtures: fixture.html +.. _markers: mark.html +.. _distributed: xdist.html + + +Other ways to help +----------------------------------------- + +Promote! Do your favourite open source Python projects use pytest? If not, why not tell them about this page? diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/index.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/index.rst new file mode 100644 index 0000000000..9505b0b9e4 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/index.rst @@ -0,0 +1,154 @@ + +Release announcements +=========================================== + +.. toctree:: + :maxdepth: 2 + + + release-7.0.1 + release-7.0.0 + release-7.0.0rc1 + release-6.2.5 + release-6.2.4 + release-6.2.3 + release-6.2.2 + release-6.2.1 + release-6.2.0 + release-6.1.2 + release-6.1.1 + release-6.1.0 + release-6.0.2 + release-6.0.1 + release-6.0.0 + release-6.0.0rc1 + release-5.4.3 + release-5.4.2 + release-5.4.1 + release-5.4.0 + release-5.3.5 + release-5.3.4 + release-5.3.3 + release-5.3.2 + release-5.3.1 + release-5.3.0 + release-5.2.4 + release-5.2.3 + release-5.2.2 + release-5.2.1 + release-5.2.0 + release-5.1.3 + release-5.1.2 + release-5.1.1 + release-5.1.0 + release-5.0.1 + release-5.0.0 + release-4.6.9 + release-4.6.8 + release-4.6.7 + release-4.6.6 + release-4.6.5 + release-4.6.4 + release-4.6.3 + release-4.6.2 + release-4.6.1 + release-4.6.0 + release-4.5.0 + release-4.4.2 + release-4.4.1 + release-4.4.0 + release-4.3.1 + release-4.3.0 + release-4.2.1 + release-4.2.0 + release-4.1.1 + release-4.1.0 + release-4.0.2 + release-4.0.1 + release-4.0.0 + release-3.10.1 + release-3.10.0 + release-3.9.3 + release-3.9.2 + release-3.9.1 + release-3.9.0 + release-3.8.2 + release-3.8.1 + release-3.8.0 + release-3.7.4 + release-3.7.3 + release-3.7.2 + release-3.7.1 + release-3.7.0 + release-3.6.4 + release-3.6.3 + release-3.6.2 + release-3.6.1 + release-3.6.0 + release-3.5.1 + release-3.5.0 + release-3.4.2 + release-3.4.1 + release-3.4.0 + release-3.3.2 + release-3.3.1 + release-3.3.0 + release-3.2.5 + release-3.2.4 + release-3.2.3 + release-3.2.2 + release-3.2.1 + release-3.2.0 + release-3.1.3 + release-3.1.2 + release-3.1.1 + release-3.1.0 + release-3.0.7 + release-3.0.6 + release-3.0.5 + release-3.0.4 + release-3.0.3 + release-3.0.2 + release-3.0.1 + release-3.0.0 + sprint2016 + release-2.9.2 + release-2.9.1 + release-2.9.0 + release-2.8.7 + release-2.8.6 + release-2.8.5 + release-2.8.4 + release-2.8.3 + release-2.8.2 + release-2.7.2 + release-2.7.1 + release-2.7.0 + release-2.6.3 + release-2.6.2 + release-2.6.1 + release-2.6.0 + release-2.5.2 + release-2.5.1 + release-2.5.0 + release-2.4.2 + release-2.4.1 + release-2.4.0 + release-2.3.5 + release-2.3.4 + release-2.3.3 + release-2.3.2 + release-2.3.1 + release-2.3.0 + release-2.2.4 + release-2.2.2 + release-2.2.1 + release-2.2.0 + release-2.1.3 + release-2.1.2 + release-2.1.1 + release-2.1.0 + release-2.0.3 + release-2.0.2 + release-2.0.1 + release-2.0.0 diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.0.rst new file mode 100644 index 0000000000..ecb1a1db98 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.0.rst @@ -0,0 +1,129 @@ +py.test 2.0.0: asserts++, unittest++, reporting++, config++, docs++ +=========================================================================== + +Welcome to pytest-2.0.0, a major new release of "py.test", the rapid +easy Python testing tool. There are many new features and enhancements, +see below for summary and detailed lists. A lot of long-deprecated code +has been removed, resulting in a much smaller and cleaner +implementation. See the new docs with examples here: + + http://pytest.org/en/stable/index.html + +A note on packaging: pytest used to part of the "py" distribution up +until version py-1.3.4 but this has changed now: pytest-2.0.0 only +contains py.test related code and is expected to be backward-compatible +to existing test code. If you want to install pytest, just type one of:: + + pip install -U pytest + easy_install -U pytest + +Many thanks to all issue reporters and people asking questions or +complaining. Particular thanks to Floris Bruynooghe and Ronny Pfannschmidt +for their great coding contributions and many others for feedback and help. + +best, +holger krekel + + +New Features +----------------------- + +- new invocations through Python interpreter and from Python:: + + python -m pytest # on all pythons >= 2.5 + + or from a python program:: + + import pytest ; pytest.main(arglist, pluginlist) + + see http://pytest.org/en/stable/how-to/usage.html for details. + +- new and better reporting information in assert expressions + if comparing lists, sequences or strings. + + see http://pytest.org/en/stable/how-to/assert.html#newreport + +- new configuration through ini-files (setup.cfg or tox.ini recognized), + for example:: + + [pytest] + norecursedirs = .hg data* # don't ever recurse in such dirs + addopts = -x --pyargs # add these command line options by default + + see http://pytest.org/en/stable/reference/customize.html + +- improved standard unittest support. In general py.test should now + better be able to run custom unittest.TestCases like twisted trial + or Django based TestCases. Also you can now run the tests of an + installed 'unittest' package with py.test:: + + py.test --pyargs unittest + +- new "-q" option which decreases verbosity and prints a more + nose/unittest-style "dot" output. + +- many many more detailed improvements details + +Fixes +----------------------- + +- fix issue126 - introduce py.test.set_trace() to trace execution via + PDB during the running of tests even if capturing is ongoing. +- fix issue124 - make reporting more resilient against tests opening + files on filedescriptor 1 (stdout). +- fix issue109 - sibling conftest.py files will not be loaded. + (and Directory collectors cannot be customized anymore from a Directory's + conftest.py - this needs to happen at least one level up). +- fix issue88 (finding custom test nodes from command line arg) +- fix issue93 stdout/stderr is captured while importing conftest.py +- fix bug: unittest collected functions now also can have "pytestmark" + applied at class/module level + +Important Notes +-------------------- + +* The usual way in pre-2.0 times to use py.test in python code was + to import "py" and then e.g. use "py.test.raises" for the helper. + This remains valid and is not planned to be deprecated. However, + in most examples and internal code you'll find "import pytest" + and "pytest.raises" used as the recommended default way. + +* pytest now first performs collection of the complete test suite + before running any test. This changes for example the semantics of when + pytest_collectstart/pytest_collectreport are called. Some plugins may + need upgrading. + +* The pytest package consists of a 400 LOC core.py and about 20 builtin plugins, + summing up to roughly 5000 LOCs, including docstrings. To be fair, it also + uses generic code from the "pylib", and the new "py" package to help + with filesystem and introspection/code manipulation. + +(Incompatible) Removals +----------------------------- + +- py.test.config is now only available if you are in a test run. + +- the following (mostly already deprecated) functionality was removed: + + - removed support for Module/Class/... collection node definitions + in conftest.py files. They will cause nothing special. + - removed support for calling the pre-1.0 collection API of "run()" and "join" + - removed reading option values from conftest.py files or env variables. + This can now be done much much better and easier through the ini-file + mechanism and the "addopts" entry in particular. + - removed the "disabled" attribute in test classes. Use the skipping + and pytestmark mechanism to skip or xfail a test class. + +- py.test.collect.Directory does not exist anymore and it + is not possible to provide an own "Directory" object. + If you have used this and don't know what to do, get + in contact. We'll figure something out. + + Note that pytest_collect_directory() is still called but + any return value will be ignored. This allows to keep + old code working that performed for example "py.test.skip()" + in collect() to prevent recursion into directory trees + if a certain dependency or command line option is missing. + + +see :ref:`changelog` for more detailed changes. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.1.rst new file mode 100644 index 0000000000..4ff3e9f550 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.1.rst @@ -0,0 +1,67 @@ +py.test 2.0.1: bug fixes +=========================================================================== + +Welcome to pytest-2.0.1, a maintenance and bug fix release of pytest, +a mature testing tool for Python, supporting CPython 2.4-3.2, Jython +and latest PyPy interpreters. See extensive docs with tested examples here: + + http://pytest.org/ + +If you want to install or upgrade pytest, just type one of:: + + pip install -U pytest # or + easy_install -U pytest + +Many thanks to all issue reporters and people asking questions or +complaining. Particular thanks to Floris Bruynooghe and Ronny Pfannschmidt +for their great coding contributions and many others for feedback and help. + +best, +holger krekel + +Changes between 2.0.0 and 2.0.1 +---------------------------------------------- + +- refine and unify initial capturing so that it works nicely + even if the logging module is used on an early-loaded conftest.py + file or plugin. +- fix issue12 - show plugin versions with "--version" and + "--traceconfig" and also document how to add extra information + to reporting test header +- fix issue17 (import-* reporting issue on python3) by + requiring py>1.4.0 (1.4.1 is going to include it) +- fix issue10 (numpy arrays truth checking) by refining + assertion interpretation in py lib +- fix issue15: make nose compatibility tests compatible + with python3 (now that nose-1.0 supports python3) +- remove somewhat surprising "same-conftest" detection because + it ignores conftest.py when they appear in several subdirs. +- improve assertions ("not in"), thanks Floris Bruynooghe +- improve behaviour/warnings when running on top of "python -OO" + (assertions and docstrings are turned off, leading to potential + false positives) +- introduce a pytest_cmdline_processargs(args) hook + to allow dynamic computation of command line arguments. + This fixes a regression because py.test prior to 2.0 + allowed to set command line options from conftest.py + files which so far pytest-2.0 only allowed from ini-files now. +- fix issue7: assert failures in doctest modules. + unexpected failures in doctests will not generally + show nicer, i.e. within the doctest failing context. +- fix issue9: setup/teardown functions for an xfail-marked + test will report as xfail if they fail but report as normally + passing (not xpassing) if they succeed. This only is true + for "direct" setup/teardown invocations because teardown_class/ + teardown_module cannot closely relate to a single test. +- fix issue14: no logging errors at process exit +- refinements to "collecting" output on non-ttys +- refine internal plugin registration and --traceconfig output +- introduce a mechanism to prevent/unregister plugins from the + command line, see http://pytest.org/en/stable/how-to/plugins.html#cmdunregister +- activate resultlog plugin by default +- fix regression wrt yielded tests which due to the + collection-before-running semantics were not + setup as with pytest 1.3.4. Note, however, that + the recommended and much cleaner way to do test + parametrization remains the "pytest_generate_tests" + mechanism, see the docs. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.2.rst new file mode 100644 index 0000000000..f1f44f34f4 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.2.rst @@ -0,0 +1,73 @@ +py.test 2.0.2: bug fixes, improved xfail/skip expressions, speed ups +=========================================================================== + +Welcome to pytest-2.0.2, a maintenance and bug fix release of pytest, +a mature testing tool for Python, supporting CPython 2.4-3.2, Jython +and latest PyPy interpreters. See the extensive docs with tested examples here: + + http://pytest.org/ + +If you want to install or upgrade pytest, just type one of:: + + pip install -U pytest # or + easy_install -U pytest + +Many thanks to all issue reporters and people asking questions +or complaining, particularly Jurko for his insistence, +Laura, Victor and Brianna for helping with improving +and Ronny for his general advise. + +best, +holger krekel + +Changes between 2.0.1 and 2.0.2 +---------------------------------------------- + +- tackle issue32 - speed up test runs of very quick test functions + by reducing the relative overhead + +- fix issue30 - extended xfail/skipif handling and improved reporting. + If you have a syntax error in your skip/xfail + expressions you now get nice error reports. + + Also you can now access module globals from xfail/skipif + expressions so that this for example works now:: + + import pytest + import mymodule + @pytest.mark.skipif("mymodule.__version__[0] == "1") + def test_function(): + pass + + This will not run the test function if the module's version string + does not start with a "1". Note that specifying a string instead + of a boolean expressions allows py.test to report meaningful information + when summarizing a test run as to what conditions lead to skipping + (or xfail-ing) tests. + +- fix issue28 - setup_method and pytest_generate_tests work together + The setup_method fixture method now gets called also for + test function invocations generated from the pytest_generate_tests + hook. + +- fix issue27 - collectonly and keyword-selection (-k) now work together + Also, if you do "py.test --collectonly -q" you now get a flat list + of test ids that you can use to paste to the py.test commandline + in order to execute a particular test. + +- fix issue25 avoid reported problems with --pdb and python3.2/encodings output + +- fix issue23 - tmpdir argument now works on Python3.2 and WindowsXP + Starting with Python3.2 os.symlink may be supported. By requiring + a newer py lib version the py.path.local() implementation acknowledges + this. + +- fixed typos in the docs (thanks Victor Garcia, Brianna Laugher) and particular + thanks to Laura Creighton who also reviewed parts of the documentation. + +- fix slightly wrong output of verbose progress reporting for classes + (thanks Amaury) + +- more precise (avoiding of) deprecation warnings for node.Class|Function accesses + +- avoid std unittest assertion helper code in tracebacks (thanks Ronny) diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.3.rst new file mode 100644 index 0000000000..81d01eb99f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.0.3.rst @@ -0,0 +1,39 @@ +py.test 2.0.3: bug fixes and speed ups +=========================================================================== + +Welcome to pytest-2.0.3, a maintenance and bug fix release of pytest, +a mature testing tool for Python, supporting CPython 2.4-3.2, Jython +and latest PyPy interpreters. See the extensive docs with tested examples here: + + http://pytest.org/ + +If you want to install or upgrade pytest, just type one of:: + + pip install -U pytest # or + easy_install -U pytest + +There also is a bugfix release 1.6 of pytest-xdist, the plugin +that enables seamless distributed and "looponfail" testing for Python. + +best, +holger krekel + +Changes between 2.0.2 and 2.0.3 +---------------------------------------------- + +- fix issue38: nicer tracebacks on calls to hooks, particularly early + configure/sessionstart ones + +- fix missing skip reason/meta information in junitxml files, reported + via http://lists.idyll.org/pipermail/testing-in-python/2011-March/003928.html + +- fix issue34: avoid collection failure with "test" prefixed classes + deriving from object. + +- don't require zlib (and other libs) for genscript plugin without + --genscript actually being used. + +- speed up skips (by not doing a full traceback representation + internally) + +- fix issue37: avoid invalid characters in junitxml's output diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.0.rst new file mode 100644 index 0000000000..78247247e2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.0.rst @@ -0,0 +1,47 @@ +py.test 2.1.0: perfected assertions and bug fixes +=========================================================================== + +Welcome to the release of pytest-2.1, a mature testing tool for Python, +supporting CPython 2.4-3.2, Jython and latest PyPy interpreters. See +the improved extensive docs (now also as PDF!) with tested examples here: + + http://pytest.org/ + +The single biggest news about this release are **perfected assertions** +courtesy of Benjamin Peterson. You can now safely use ``assert`` +statements in test modules without having to worry about side effects +or python optimization ("-OO") options. This is achieved by rewriting +assert statements in test modules upon import, using a PEP302 hook. +See https://docs.pytest.org/en/stable/how-to/assert.html for +detailed information. The work has been partly sponsored by my company, +merlinux GmbH. + +For further details on bug fixes and smaller enhancements see below. + +If you want to install or upgrade pytest, just type one of:: + + pip install -U pytest # or + easy_install -U pytest + +best, +holger krekel / https://merlinux.eu/ + +Changes between 2.0.3 and 2.1.0 +---------------------------------------------- + +- fix issue53 call nosestyle setup functions with correct ordering +- fix issue58 and issue59: new assertion code fixes +- merge Benjamin's assertionrewrite branch: now assertions + for test modules on python 2.6 and above are done by rewriting + the AST and saving the pyc file before the test module is imported. + see doc/assert.txt for more info. +- fix issue43: improve doctests with better traceback reporting on + unexpected exceptions +- fix issue47: timing output in junitxml for test cases is now correct +- fix issue48: typo in MarkInfo repr leading to exception +- fix issue49: avoid confusing error when initialization partially fails +- fix issue44: env/username expansion for junitxml file path +- show releaselevel information in test runs for pypy +- reworked doc pages for better navigation and PDF generation +- report KeyboardInterrupt even if interrupted during session startup +- fix issue 35 - provide PDF doc version and download link from index page diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.1.rst new file mode 100644 index 0000000000..369428ed2e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.1.rst @@ -0,0 +1,36 @@ +py.test 2.1.1: assertion fixes and improved junitxml output +=========================================================================== + +pytest-2.1.1 is a backward compatible maintenance release of the +popular py.test testing tool. See extensive docs with examples here: + + http://pytest.org/ + +Most bug fixes address remaining issues with the perfected assertions +introduced with 2.1.0 - many thanks to the bug reporters and to Benjamin +Peterson for helping to fix them. Also, junitxml output now produces +system-out/err tags which lead to better displays of tracebacks with Jenkins. + +Also a quick note to package maintainers and others interested: there now +is a "pytest" man page which can be generated with "make man" in doc/. + +If you want to install or upgrade pytest, just type one of:: + + pip install -U pytest # or + easy_install -U pytest + +best, +holger krekel / https://merlinux.eu/ + +Changes between 2.1.0 and 2.1.1 +---------------------------------------------- + +- fix issue64 / pytest.set_trace now works within pytest_generate_tests hooks +- fix issue60 / fix error conditions involving the creation of __pycache__ +- fix issue63 / assertion rewriting on inserts involving strings containing '%' +- fix assertion rewriting on calls with a ** arg +- don't cache rewritten modules if bytecode generation is disabled +- fix assertion rewriting in read-only directories +- fix issue59: provide system-out/err tags for junitxml output +- fix issue61: assertion rewriting on boolean operations with 3 or more operands +- you can now build a man page with "cd doc ; make man" diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.2.rst new file mode 100644 index 0000000000..a3c0c1a38a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.2.rst @@ -0,0 +1,32 @@ +py.test 2.1.2: bug fixes and fixes for jython +=========================================================================== + +pytest-2.1.2 is a minor backward compatible maintenance release of the +popular py.test testing tool. pytest is commonly used for unit, +functional- and integration testing. See extensive docs with examples +here: + + http://pytest.org/ + +Most bug fixes address remaining issues with the perfected assertions +introduced in the 2.1 series - many thanks to the bug reporters and to Benjamin +Peterson for helping to fix them. pytest should also work better with +Jython-2.5.1 (and Jython trunk). + +If you want to install or upgrade pytest, just type one of:: + + pip install -U pytest # or + easy_install -U pytest + +best, +holger krekel / https://merlinux.eu/ + +Changes between 2.1.1 and 2.1.2 +---------------------------------------- + +- fix assertion rewriting on files with windows newlines on some Python versions +- refine test discovery by package/module name (--pyargs), thanks Florian Mayer +- fix issue69 / assertion rewriting fixed on some boolean operations +- fix issue68 / packages now work with assertion rewriting +- fix issue66: use different assertion rewriting caches when the -O option is passed +- don't try assertion rewriting on Jython, use reinterp diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.3.rst new file mode 100644 index 0000000000..a43bc058c1 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.1.3.rst @@ -0,0 +1,32 @@ +py.test 2.1.3: just some more fixes +=========================================================================== + +pytest-2.1.3 is a minor backward compatible maintenance release of the +popular py.test testing tool. It is commonly used for unit, functional- +and integration testing. See extensive docs with examples here: + + http://pytest.org/ + +The release contains another fix to the perfected assertions introduced +with the 2.1 series as well as the new possibility to customize reporting +for assertion expressions on a per-directory level. + +If you want to install or upgrade pytest, just type one of:: + + pip install -U pytest # or + easy_install -U pytest + +Thanks to the bug reporters and to Ronny Pfannschmidt, Benjamin Peterson +and Floris Bruynooghe who implemented the fixes. + +best, +holger krekel + +Changes between 2.1.2 and 2.1.3 +---------------------------------------- + +- fix issue79: assertion rewriting failed on some comparisons in boolops, +- correctly handle zero length arguments (a la pytest '') +- fix issue67 / junitxml now contains correct test durations +- fix issue75 / skipping test failure on jython +- fix issue77 / Allow assertrepr_compare hook to apply to a subset of tests diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.0.rst new file mode 100644 index 0000000000..7a32dca173 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.0.rst @@ -0,0 +1,95 @@ +py.test 2.2.0: test marking++, parametrization++ and duration profiling +=========================================================================== + +pytest-2.2.0 is a test-suite compatible release of the popular +py.test testing tool. Plugins might need upgrades. It comes +with these improvements: + +* easier and more powerful parametrization of tests: + + - new @pytest.mark.parametrize decorator to run tests with different arguments + - new metafunc.parametrize() API for parametrizing arguments independently + - see examples at http://pytest.org/en/stable/example/how-to/parametrize.html + - NOTE that parametrize() related APIs are still a bit experimental + and might change in future releases. + +* improved handling of test markers and refined marking mechanism: + + - "-m markexpr" option for selecting tests according to their mark + - a new "markers" ini-variable for registering test markers for your project + - the new "--strict" bails out with an error if using unregistered markers. + - see examples at http://pytest.org/en/stable/example/markers.html + +* duration profiling: new "--duration=N" option showing the N slowest test + execution or setup/teardown calls. This is most useful if you want to + find out where your slowest test code is. + +* also 2.2.0 performs more eager calling of teardown/finalizers functions + resulting in better and more accurate reporting when they fail + +Besides there is the usual set of bug fixes along with a cleanup of +pytest's own test suite allowing it to run on a wider range of environments. + +For general information, see extensive docs with examples here: + + http://pytest.org/ + +If you want to install or upgrade pytest you might just type:: + + pip install -U pytest # or + easy_install -U pytest + +Thanks to Ronny Pfannschmidt, David Burns, Jeff Donner, Daniel Nouri, Alfredo Deza and all who gave feedback or sent bug reports. + +best, +holger krekel + + +notes on incompatibility +------------------------------ + +While test suites should work unchanged you might need to upgrade plugins: + +* You need a new version of the pytest-xdist plugin (1.7) for distributing + test runs. + +* Other plugins might need an upgrade if they implement + the ``pytest_runtest_logreport`` hook which now is called unconditionally + for the setup/teardown fixture phases of a test. You may choose to + ignore setup/teardown failures by inserting "if rep.when != 'call': return" + or something similar. Note that most code probably "just" works because + the hook was already called for failing setup/teardown phases of a test + so a plugin should have been ready to grok such reports already. + + +Changes between 2.1.3 and 2.2.0 +---------------------------------------- + +- fix issue90: introduce eager tearing down of test items so that + teardown function are called earlier. +- add an all-powerful metafunc.parametrize function which allows to + parametrize test function arguments in multiple steps and therefore + from independent plugins and places. +- add a @pytest.mark.parametrize helper which allows to easily + call a test function with different argument values. +- Add examples to the "parametrize" example page, including a quick port + of Test scenarios and the new parametrize function and decorator. +- introduce registration for "pytest.mark.*" helpers via ini-files + or through plugin hooks. Also introduce a "--strict" option which + will treat unregistered markers as errors + allowing to avoid typos and maintain a well described set of markers + for your test suite. See examples at http://pytest.org/en/stable/how-to/mark.html + and its links. +- issue50: introduce "-m marker" option to select tests based on markers + (this is a stricter and more predictable version of "-k" in that "-m" + only matches complete markers and has more obvious rules for and/or + semantics. +- new feature to help optimizing the speed of your tests: + --durations=N option for displaying N slowest test calls + and setup/teardown methods. +- fix issue87: --pastebin now works with python3 +- fix issue89: --pdb with unexpected exceptions in doctest work more sensibly +- fix and cleanup pytest's own test suite to not leak FDs +- fix issue83: link to generated funcarg list +- fix issue74: pyarg module names are now checked against imp.find_module false positives +- fix compatibility with twisted/trial-11.1.0 use cases diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.1.rst new file mode 100644 index 0000000000..44281597ea --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.1.rst @@ -0,0 +1,41 @@ +pytest-2.2.1: bug fixes, perfect teardowns +=========================================================================== + + +pytest-2.2.1 is a minor backward-compatible release of the py.test +testing tool. It contains bug fixes and little improvements, including +documentation fixes. If you are using the distributed testing +pluginmake sure to upgrade it to pytest-xdist-1.8. + +For general information see here: + + http://pytest.org/ + +To install or upgrade pytest: + + pip install -U pytest # or + easy_install -U pytest + +Special thanks for helping on this release to Ronny Pfannschmidt, Jurko +Gospodnetic and Ralf Schmitt. + +best, +holger krekel + + +Changes between 2.2.0 and 2.2.1 +---------------------------------------- + +- fix issue99 (in pytest and py) internallerrors with resultlog now + produce better output - fixed by normalizing pytest_internalerror + input arguments. +- fix issue97 / traceback issues (in pytest and py) improve traceback output + in conjunction with jinja2 and cython which hack tracebacks +- fix issue93 (in pytest and pytest-xdist) avoid "delayed teardowns": + the final test in a test node will now run its teardown directly + instead of waiting for the end of the session. Thanks Dave Hunt for + the good reporting and feedback. The pytest_runtest_protocol as well + as the pytest_runtest_teardown hooks now have "nextitem" available + which will be None indicating the end of the test run. +- fix collection crash due to unknown-source collected items, thanks + to Ralf Schmitt (fixed by depending on a more recent pylib) diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.2.rst new file mode 100644 index 0000000000..22ef0bc7a1 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.2.rst @@ -0,0 +1,43 @@ +pytest-2.2.2: bug fixes +=========================================================================== + +pytest-2.2.2 (updated to 2.2.3 to fix packaging issues) is a minor +backward-compatible release of the versatile py.test testing tool. It +contains bug fixes and a few refinements particularly to reporting with +"--collectonly", see below for betails. + +For general information see here: + + http://pytest.org/ + +To install or upgrade pytest: + + pip install -U pytest # or + easy_install -U pytest + +Special thanks for helping on this release to Ronny Pfannschmidt +and Ralf Schmitt and the contributors of issues. + +best, +holger krekel + + +Changes between 2.2.1 and 2.2.2 +---------------------------------------- + +- fix issue101: wrong args to unittest.TestCase test function now + produce better output +- fix issue102: report more useful errors and hints for when a + test directory was renamed and some pyc/__pycache__ remain +- fix issue106: allow parametrize to be applied multiple times + e.g. from module, class and at function level. +- fix issue107: actually perform session scope finalization +- don't check in parametrize if indirect parameters are funcarg names +- add chdir method to monkeypatch funcarg +- fix crash resulting from calling monkeypatch undo a second time +- fix issue115: make --collectonly robust against early failure + (missing files/directories) +- "-qq --collectonly" now shows only files and the number of tests in them +- "-q --collectonly" now shows test ids +- allow adding of attributes to test reports such that it also works + with distributed testing (no upgrade of pytest-xdist needed) diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.4.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.4.rst new file mode 100644 index 0000000000..a8fb9b93c5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.2.4.rst @@ -0,0 +1,38 @@ +pytest-2.2.4: bug fixes, better junitxml/unittest/python3 compat +=========================================================================== + +pytest-2.2.4 is a minor backward-compatible release of the versatile +py.test testing tool. It contains bug fixes and a few refinements +to junitxml reporting, better unittest- and python3 compatibility. + +For general information see here: + + http://pytest.org/ + +To install or upgrade pytest: + + pip install -U pytest # or + easy_install -U pytest + +Special thanks for helping on this release to Ronny Pfannschmidt +and Benjamin Peterson and the contributors of issues. + +best, +holger krekel + +Changes between 2.2.3 and 2.2.4 +----------------------------------- + +- fix error message for rewritten assertions involving the % operator +- fix issue 126: correctly match all invalid xml characters for junitxml + binary escape +- fix issue with unittest: now @unittest.expectedFailure markers should + be processed correctly (you can also use @pytest.mark markers) +- document integration with the extended distribute/setuptools test commands +- fix issue 140: properly get the real functions + of bound classmethods for setup/teardown_class +- fix issue #141: switch from the deceased paste.pocoo.org to bpaste.net +- fix issue #143: call unconfigure/sessionfinish always when + configure/sessionstart where called +- fix issue #144: better mangle test ids to junitxml classnames +- upgrade distribute_setup.py to 0.6.27 diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.0.rst new file mode 100644 index 0000000000..6905b77b92 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.0.rst @@ -0,0 +1,133 @@ +pytest-2.3: improved fixtures / better unittest integration +============================================================================= + +pytest-2.3 comes with many major improvements for fixture/funcarg management +and parametrized testing in Python. It is now easier, more efficient and +more predictable to re-run the same tests with different fixture +instances. Also, you can directly declare the caching "scope" of +fixtures so that dependent tests throughout your whole test suite can +re-use database or other expensive fixture objects with ease. Lastly, +it's possible for fixture functions (formerly known as funcarg +factories) to use other fixtures, allowing for a completely modular and +re-usable fixture design. + +For detailed info and tutorial-style examples, see: + + http://pytest.org/en/stable/explanation/fixtures.html + +Moreover, there is now support for using pytest fixtures/funcargs with +unittest-style suites, see here for examples: + + http://pytest.org/en/stable/how-to/unittest.html + +Besides, more unittest-test suites are now expected to "simply work" +with pytest. + +All changes are backward compatible and you should be able to continue +to run your test suites and 3rd party plugins that worked with +pytest-2.2.4. + +If you are interested in the precise reasoning (including examples) of the +pytest-2.3 fixture evolution, please consult +http://pytest.org/en/stable/funcarg_compare.html + +For general info on installation and getting started: + + http://pytest.org/en/stable/getting-started.html + +Docs and PDF access as usual at: + + http://pytest.org + +and more details for those already in the knowing of pytest can be found +in the CHANGELOG below. + +Particular thanks for this release go to Floris Bruynooghe, Alex Okrushko +Carl Meyer, Ronny Pfannschmidt, Benjamin Peterson and Alex Gaynor for helping +to get the new features right and well integrated. Ronny and Floris +also helped to fix a number of bugs and yet more people helped by +providing bug reports. + +have fun, +holger krekel + + +Changes between 2.2.4 and 2.3.0 +----------------------------------- + +- fix issue202 - better automatic names for parametrized test functions +- fix issue139 - introduce @pytest.fixture which allows direct scoping + and parametrization of funcarg factories. Introduce new @pytest.setup + marker to allow the writing of setup functions which accept funcargs. +- fix issue198 - conftest fixtures were not found on windows32 in some + circumstances with nested directory structures due to path manipulation issues +- fix issue193 skip test functions with were parametrized with empty + parameter sets +- fix python3.3 compat, mostly reporting bits that previously depended + on dict ordering +- introduce re-ordering of tests by resource and parametrization setup + which takes precedence to the usual file-ordering +- fix issue185 monkeypatching time.time does not cause pytest to fail +- fix issue172 duplicate call of pytest.setup-decoratored setup_module + functions +- fix junitxml=path construction so that if tests change the + current working directory and the path is a relative path + it is constructed correctly from the original current working dir. +- fix "python setup.py test" example to cause a proper "errno" return +- fix issue165 - fix broken doc links and mention stackoverflow for FAQ +- catch unicode-issues when writing failure representations + to terminal to prevent the whole session from crashing +- fix xfail/skip confusion: a skip-mark or an imperative pytest.skip + will now take precedence before xfail-markers because we + can't determine xfail/xpass status in case of a skip. see also: + http://stackoverflow.com/questions/11105828/in-py-test-when-i-explicitly-skip-a-test-that-is-marked-as-xfail-how-can-i-get + +- always report installed 3rd party plugins in the header of a test run + +- fix issue160: a failing setup of an xfail-marked tests should + be reported as xfail (not xpass) + +- fix issue128: show captured output when capsys/capfd are used + +- fix issue179: properly show the dependency chain of factories + +- pluginmanager.register(...) now raises ValueError if the + plugin has been already registered or the name is taken + +- fix issue159: improve https://docs.pytest.org/en/6.0.1/faq.html + especially with respect to the "magic" history, also mention + pytest-django, trial and unittest integration. + +- make request.keywords and node.keywords writable. All descendant + collection nodes will see keyword values. Keywords are dictionaries + containing markers and other info. + +- fix issue 178: xml binary escapes are now wrapped in py.xml.raw + +- fix issue 176: correctly catch the builtin AssertionError + even when we replaced AssertionError with a subclass on the + python level + +- factory discovery no longer fails with magic global callables + that provide no sane __code__ object (mock.call for example) + +- fix issue 182: testdir.inprocess_run now considers passed plugins + +- fix issue 188: ensure sys.exc_info is clear on python2 + before calling into a test + +- fix issue 191: add unittest TestCase runTest method support +- fix issue 156: monkeypatch correctly handles class level descriptors + +- reporting refinements: + + - pytest_report_header now receives a "startdir" so that + you can use startdir.bestrelpath(yourpath) to show + nice relative path + + - allow plugins to implement both pytest_report_header and + pytest_sessionstart (sessionstart is invoked first). + + - don't show deselected reason line if there is none + + - py.test -vv will show all of assert comparisons instead of truncating diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.1.rst new file mode 100644 index 0000000000..6f8770b345 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.1.rst @@ -0,0 +1,39 @@ +pytest-2.3.1: fix regression with factory functions +=========================================================================== + +pytest-2.3.1 is a quick follow-up release: + +- fix issue202 - regression with fixture functions/funcarg factories: + using "self" is now safe again and works as in 2.2.4. Thanks + to Eduard Schettino for the quick bug report. + +- disable pexpect pytest self tests on Freebsd - thanks Koob for the + quick reporting + +- fix/improve interactive docs with --markers + +See + + http://pytest.org/ + +for general information. To install or upgrade pytest: + + pip install -U pytest # or + easy_install -U pytest + +best, +holger krekel + + +Changes between 2.3.0 and 2.3.1 +----------------------------------- + +- fix issue202 - fix regression: using "self" from fixture functions now + works as expected (it's the same "self" instance that a test method + which uses the fixture sees) + +- skip pexpect using tests (test_pdb.py mostly) on freebsd* systems + due to pexpect not supporting it properly (hanging) + +- link to web pages from --markers output which provides help for + pytest.mark.* usage. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.2.rst new file mode 100644 index 0000000000..484feaaa5a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.2.rst @@ -0,0 +1,57 @@ +pytest-2.3.2: some fixes and more traceback-printing speed +=========================================================================== + +pytest-2.3.2 is another stabilization release: + +- issue 205: fixes a regression with conftest detection +- issue 208/29: fixes traceback-printing speed in some bad cases +- fix teardown-ordering for parametrized setups +- fix unittest and trial compat behaviour with respect to runTest() methods +- issue 206 and others: some improvements to packaging +- fix issue127 and others: improve some docs + +See + + http://pytest.org/ + +for general information. To install or upgrade pytest: + + pip install -U pytest # or + easy_install -U pytest + +best, +holger krekel + + +Changes between 2.3.1 and 2.3.2 +----------------------------------- + +- fix issue208 and fix issue29 use new py version to avoid long pauses + when printing tracebacks in long modules + +- fix issue205 - conftests in subdirs customizing + pytest_pycollect_makemodule and pytest_pycollect_makeitem + now work properly + +- fix teardown-ordering for parametrized setups + +- fix issue127 - better documentation for pytest_addoption + and related objects. + +- fix unittest behaviour: TestCase.runtest only called if there are + test methods defined + +- improve trial support: don't collect its empty + unittest.TestCase.runTest() method + +- "python setup.py test" now works with pytest itself + +- fix/improve internal/packaging related bits: + + - exception message check of test_nose.py now passes on python33 as well + + - issue206 - fix test_assertrewrite.py to work when a global + PYTHONDONTWRITEBYTECODE=1 is present + + - add tox.ini to pytest distribution so that ignore-dirs and others config + bits are properly distributed for maintainers who run pytest-own tests diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.3.rst new file mode 100644 index 0000000000..0cb598a426 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.3.rst @@ -0,0 +1,61 @@ +pytest-2.3.3: integration fixes, py24 support, ``*/**`` shown in traceback +=========================================================================== + +pytest-2.3.3 is another stabilization release of the py.test tool +which offers uebersimple assertions, scalable fixture mechanisms +and deep customization for testing with Python. Particularly, +this release provides: + +- integration fixes and improvements related to flask, numpy, nose, + unittest, mock + +- makes pytest work on py24 again (yes, people sometimes still need to use it) + +- show ``*,**`` args in pytest tracebacks + +Thanks to Manuel Jacob, Thomas Waldmann, Ronny Pfannschmidt, Pavel Repin +and Andreas Taumoefolau for providing patches and all for the issues. + +See + + http://pytest.org/ + +for general information. To install or upgrade pytest: + + pip install -U pytest # or + easy_install -U pytest + +best, +holger krekel + +Changes between 2.3.2 and 2.3.3 +----------------------------------- + +- fix issue214 - parse modules that contain special objects like e. g. + flask's request object which blows up on getattr access if no request + is active. thanks Thomas Waldmann. + +- fix issue213 - allow to parametrize with values like numpy arrays that + do not support an __eq__ operator + +- fix issue215 - split test_python.org into multiple files + +- fix issue148 - @unittest.skip on classes is now recognized and avoids + calling setUpClass/tearDownClass, thanks Pavel Repin + +- fix issue209 - reintroduce python2.4 support by depending on newer + pylib which re-introduced statement-finding for pre-AST interpreters + +- nose support: only call setup if it's a callable, thanks Andrew + Taumoefolau + +- fix issue219 - add py2.4-3.3 classifiers to TROVE list + +- in tracebacks *,** arg values are now shown next to normal arguments + (thanks Manuel Jacob) + +- fix issue217 - support mock.patch with pytest's fixtures - note that + you need either mock-1.0.1 or the python3.3 builtin unittest.mock. + +- fix issue127 - improve documentation for pytest_addoption() and + add a ``config.getoption(name)`` helper function for consistency. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.4.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.4.rst new file mode 100644 index 0000000000..43bf03b02b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.4.rst @@ -0,0 +1,39 @@ +pytest-2.3.4: stabilization, more flexible selection via "-k expr" +=========================================================================== + +pytest-2.3.4 is a small stabilization release of the py.test tool +which offers uebersimple assertions, scalable fixture mechanisms +and deep customization for testing with Python. This release +comes with the following fixes and features: + +- make "-k" option accept an expressions the same as with "-m" so that one + can write: -k "name1 or name2" etc. This is a slight usage incompatibility + if you used special syntax like "TestClass.test_method" which you now + need to write as -k "TestClass and test_method" to match a certain + method in a certain test class. +- allow to dynamically define markers via + item.keywords[...]=assignment integrating with "-m" option +- yielded test functions will now have autouse-fixtures active but + cannot accept fixtures as funcargs - it's anyway recommended to + rather use the post-2.0 parametrize features instead of yield, see: + http://pytest.org/en/stable/example/how-to/parametrize.html +- fix autouse-issue where autouse-fixtures would not be discovered + if defined in an a/conftest.py file and tests in a/tests/test_some.py +- fix issue226 - LIFO ordering for fixture teardowns +- fix issue224 - invocations with >256 char arguments now work +- fix issue91 - add/discuss package/directory level setups in example +- fixes related to autouse discovery and calling + +Thanks in particular to Thomas Waldmann for spotting and reporting issues. + +See + + http://pytest.org/ + +for general information. To install or upgrade pytest: + + pip install -U pytest # or + easy_install -U pytest + +best, +holger krekel diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.5.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.5.rst new file mode 100644 index 0000000000..d68780a244 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.3.5.rst @@ -0,0 +1,96 @@ +pytest-2.3.5: bug fixes and little improvements +=========================================================================== + +pytest-2.3.5 is a maintenance release with many bug fixes and little +improvements. See the changelog below for details. No backward +compatibility issues are foreseen and all plugins which worked with the +prior version are expected to work unmodified. Speaking of which, a +few interesting new plugins saw the light last month: + +- pytest-instafail: show failure information while tests are running +- pytest-qt: testing of GUI applications written with QT/Pyside +- pytest-xprocess: managing external processes across test runs +- pytest-random: randomize test ordering + +And several others like pytest-django saw maintenance releases. +For a more complete list, check out +https://pypi.org/search/?q=pytest + +For general information see: + + http://pytest.org/ + +To install or upgrade pytest: + + pip install -U pytest # or + easy_install -U pytest + +Particular thanks to Floris, Ronny, Benjamin and the many bug reporters +and fix providers. + +may the fixtures be with you, +holger krekel + + +Changes between 2.3.4 and 2.3.5 +----------------------------------- + +- never consider a fixture function for test function collection + +- allow re-running of test items / helps to fix pytest-reruntests plugin + and also help to keep less fixture/resource references alive + +- put captured stdout/stderr into junitxml output even for passing tests + (thanks Adam Goucher) + +- Issue 265 - integrate nose setup/teardown with setupstate + so it doesn't try to teardown if it did not setup + +- issue 271 - don't write junitxml on worker nodes + +- Issue 274 - don't try to show full doctest example + when doctest does not know the example location + +- issue 280 - disable assertion rewriting on buggy CPython 2.6.0 + +- inject "getfixture()" helper to retrieve fixtures from doctests, + thanks Andreas Zeidler + +- issue 259 - when assertion rewriting, be consistent with the default + source encoding of ASCII on Python 2 + +- issue 251 - report a skip instead of ignoring classes with init + +- issue250 unicode/str mixes in parametrization names and values now works + +- issue257, assertion-triggered compilation of source ending in a + comment line doesn't blow up in python2.5 (fixed through py>=1.4.13.dev6) + +- fix --genscript option to generate standalone scripts that also + work with python3.3 (importer ordering) + +- issue171 - in assertion rewriting, show the repr of some + global variables + +- fix option help for "-k" + +- move long description of distribution into README.rst + +- improve docstring for metafunc.parametrize() + +- fix bug where using capsys with pytest.set_trace() in a test + function would break when looking at capsys.readouterr() + +- allow to specify prefixes starting with "_" when + customizing python_functions test discovery. (thanks Graham Horler) + +- improve PYTEST_DEBUG tracing output by putting + extra data on a new lines with additional indent + +- ensure OutcomeExceptions like skip/fail have initialized exception attributes + +- issue 260 - don't use nose special setup on plain unittest cases + +- fix issue134 - print the collect errors that prevent running specified test items + +- fix issue266 - accept unicode in MarkEvaluator expressions diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.4.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.4.0.rst new file mode 100644 index 0000000000..138cc89576 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.4.0.rst @@ -0,0 +1,223 @@ +pytest-2.4.0: new fixture features/hooks and bug fixes +=========================================================================== + +The just released pytest-2.4.0 brings many improvements and numerous +bug fixes while remaining plugin- and test-suite compatible apart +from a few supposedly very minor incompatibilities. See below for +a full list of details. A few feature highlights: + +- new yield-style fixtures `pytest.yield_fixture + `_, allowing to use + existing with-style context managers in fixture functions. + +- improved pdb support: ``import pdb ; pdb.set_trace()`` now works + without requiring prior disabling of stdout/stderr capturing. + Also the ``--pdb`` options works now on collection and internal errors + and we introduced a new experimental hook for IDEs/plugins to + intercept debugging: ``pytest_exception_interact(node, call, report)``. + +- shorter monkeypatch variant to allow specifying an import path as + a target, for example: ``monkeypatch.setattr("requests.get", myfunc)`` + +- better unittest/nose compatibility: all teardown methods are now only + called if the corresponding setup method succeeded. + +- integrate tab-completion on command line options if you + have :pypi:`argcomplete` configured. + +- allow boolean expression directly with skipif/xfail + if a "reason" is also specified. + +- a new hook ``pytest_load_initial_conftests`` allows plugins like + :pypi:`pytest-django` to + influence the environment before conftest files import ``django``. + +- reporting: color the last line red or green depending if + failures/errors occurred or everything passed. + +The documentation has been updated to accommodate the changes, +see `http://pytest.org `_ + +To install or upgrade pytest:: + + pip install -U pytest # or + easy_install -U pytest + + +**Many thanks to all who helped, including Floris Bruynooghe, +Brianna Laugher, Andreas Pelme, Anthon van der Neut, Anatoly Bubenkoff, +Vladimir Keleshev, Mathieu Agopian, Ronny Pfannschmidt, Christian +Theunert and many others.** + +may passing tests be with you, + +holger krekel + +Changes between 2.3.5 and 2.4 +----------------------------------- + +known incompatibilities: + +- if calling --genscript from python2.7 or above, you only get a + standalone script which works on python2.7 or above. Use Python2.6 + to also get a python2.5 compatible version. + +- all xunit-style teardown methods (nose-style, pytest-style, + unittest-style) will not be called if the corresponding setup method failed, + see issue322 below. + +- the pytest_plugin_unregister hook wasn't ever properly called + and there is no known implementation of the hook - so it got removed. + +- pytest.fixture-decorated functions cannot be generators (i.e. use + yield) anymore. This change might be reversed in 2.4.1 if it causes + unforeseen real-life issues. However, you can always write and return + an inner function/generator and change the fixture consumer to iterate + over the returned generator. This change was done in lieu of the new + ``pytest.yield_fixture`` decorator, see below. + +new features: + +- experimentally introduce a new ``pytest.yield_fixture`` decorator + which accepts exactly the same parameters as pytest.fixture but + mandates a ``yield`` statement instead of a ``return statement`` from + fixture functions. This allows direct integration with "with-style" + context managers in fixture functions and generally avoids registering + of finalization callbacks in favour of treating the "after-yield" as + teardown code. Thanks Andreas Pelme, Vladimir Keleshev, Floris + Bruynooghe, Ronny Pfannschmidt and many others for discussions. + +- allow boolean expression directly with skipif/xfail + if a "reason" is also specified. Rework skipping documentation + to recommend "condition as booleans" because it prevents surprises + when importing markers between modules. Specifying conditions + as strings will remain fully supported. + +- reporting: color the last line red or green depending if + failures/errors occurred or everything passed. thanks Christian + Theunert. + +- make "import pdb ; pdb.set_trace()" work natively wrt capturing (no + "-s" needed anymore), making ``pytest.set_trace()`` a mere shortcut. + +- fix issue181: --pdb now also works on collect errors (and + on internal errors) . This was implemented by a slight internal + refactoring and the introduction of a new hook + ``pytest_exception_interact`` hook (see next item). + +- fix issue341: introduce new experimental hook for IDEs/terminals to + intercept debugging: ``pytest_exception_interact(node, call, report)``. + +- new monkeypatch.setattr() variant to provide a shorter + invocation for patching out classes/functions from modules: + + monkeypatch.setattr("requests.get", myfunc) + + will replace the "get" function of the "requests" module with ``myfunc``. + +- fix issue322: tearDownClass is not run if setUpClass failed. Thanks + Mathieu Agopian for the initial fix. Also make all of pytest/nose + finalizer mimic the same generic behaviour: if a setupX exists and + fails, don't run teardownX. This internally introduces a new method + "node.addfinalizer()" helper which can only be called during the setup + phase of a node. + +- simplify pytest.mark.parametrize() signature: allow to pass a + CSV-separated string to specify argnames. For example: + ``pytest.mark.parametrize("input,expected", [(1,2), (2,3)])`` + works as well as the previous: + ``pytest.mark.parametrize(("input", "expected"), ...)``. + +- add support for setUpModule/tearDownModule detection, thanks Brian Okken. + +- integrate tab-completion on options through use of "argcomplete". + Thanks Anthon van der Neut for the PR. + +- change option names to be hyphen-separated long options but keep the + old spelling backward compatible. py.test -h will only show the + hyphenated version, for example "--collect-only" but "--collectonly" + will remain valid as well (for backward-compat reasons). Many thanks to + Anthon van der Neut for the implementation and to Hynek Schlawack for + pushing us. + +- fix issue 308 - allow to mark/xfail/skip individual parameter sets + when parametrizing. Thanks Brianna Laugher. + +- call new experimental pytest_load_initial_conftests hook to allow + 3rd party plugins to do something before a conftest is loaded. + +Bug fixes: + +- fix issue358 - capturing options are now parsed more properly + by using a new parser.parse_known_args method. + +- pytest now uses argparse instead of optparse (thanks Anthon) which + means that "argparse" is added as a dependency if installing into python2.6 + environments or below. + +- fix issue333: fix a case of bad unittest/pytest hook interaction. + +- PR27: correctly handle nose.SkipTest during collection. Thanks + Antonio Cuni, Ronny Pfannschmidt. + +- fix issue355: junitxml puts name="pytest" attribute to testsuite tag. + +- fix issue336: autouse fixture in plugins should work again. + +- fix issue279: improve object comparisons on assertion failure + for standard datatypes and recognise collections.abc. Thanks to + Brianna Laugher and Mathieu Agopian. + +- fix issue317: assertion rewriter support for the is_package method + +- fix issue335: document py.code.ExceptionInfo() object returned + from pytest.raises(), thanks Mathieu Agopian. + +- remove implicit distribute_setup support from setup.py. + +- fix issue305: ignore any problems when writing pyc files. + +- SO-17664702: call fixture finalizers even if the fixture function + partially failed (finalizers would not always be called before) + +- fix issue320 - fix class scope for fixtures when mixed with + module-level functions. Thanks Anatloy Bubenkoff. + +- you can specify "-q" or "-qq" to get different levels of "quieter" + reporting (thanks Katarzyna Jachim) + +- fix issue300 - Fix order of conftest loading when starting py.test + in a subdirectory. + +- fix issue323 - sorting of many module-scoped arg parametrizations + +- make sessionfinish hooks execute with the same cwd-context as at + session start (helps fix plugin behaviour which write output files + with relative path such as pytest-cov) + +- fix issue316 - properly reference collection hooks in docs + +- fix issue 306 - cleanup of -k/-m options to only match markers/test + names/keywords respectively. Thanks Wouter van Ackooy. + +- improved doctest counting for doctests in python modules -- + files without any doctest items will not show up anymore + and doctest examples are counted as separate test items. + thanks Danilo Bellini. + +- fix issue245 by depending on the released py-1.4.14 + which fixes py.io.dupfile to work with files with no + mode. Thanks Jason R. Coombs. + +- fix junitxml generation when test output contains control characters, + addressing issue267, thanks Jaap Broekhuizen + +- fix issue338: honor --tb style for setup/teardown errors as well. Thanks Maho. + +- fix issue307 - use yaml.safe_load in example, thanks Mark Eichin. + +- better parametrize error messages, thanks Brianna Laugher + +- pytest_terminal_summary(terminalreporter) hooks can now use + ".section(title)" and ".line(msg)" methods to print extra + information at the end of a test run. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.4.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.4.1.rst new file mode 100644 index 0000000000..308df6bdc4 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.4.1.rst @@ -0,0 +1,25 @@ +pytest-2.4.1: fixing three regressions compared to 2.3.5 +=========================================================================== + +pytest-2.4.1 is a quick follow up release to fix three regressions +compared to 2.3.5 before they hit more people: + +- When using parser.addoption() unicode arguments to the + "type" keyword should also be converted to the respective types. + thanks Floris Bruynooghe, @dnozay. (fixes issue360 and issue362) + +- fix dotted filename completion when using argcomplete + thanks Anthon van der Neuth. (fixes issue361) + +- fix regression when a 1-tuple ("arg",) is used for specifying + parametrization (the values of the parametrization were passed + nested in a tuple). Thanks Donald Stufft. + +- also merge doc typo fixes, thanks Andy Dirnberger + +as usual, docs at http://pytest.org and upgrades via:: + + pip install -U pytest + +have fun, +holger krekel diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.4.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.4.2.rst new file mode 100644 index 0000000000..ab08b72aaf --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.4.2.rst @@ -0,0 +1,39 @@ +pytest-2.4.2: colorama on windows, plugin/tmpdir fixes +=========================================================================== + +pytest-2.4.2 is another bug-fixing release: + +- on Windows require colorama and a newer py lib so that py.io.TerminalWriter() + now uses colorama instead of its own ctypes hacks. (fixes issue365) + thanks Paul Moore for bringing it up. + +- fix "-k" matching of tests where "repr" and "attr" and other names would + cause wrong matches because of an internal implementation quirk + (don't ask) which is now properly implemented. fixes issue345. + +- avoid tmpdir fixture to create too long filenames especially + when parametrization is used (issue354) + +- fix pytest-pep8 and pytest-flakes / pytest interactions + (collection names in mark plugin was assuming an item always + has a function which is not true for those plugins etc.) + Thanks Andi Zeidler. + +- introduce node.get_marker/node.add_marker API for plugins + like pytest-pep8 and pytest-flakes to avoid the messy + details of the node.keywords pseudo-dicts. Adapted + docs. + +- remove attempt to "dup" stdout at startup as it's icky. + the normal capturing should catch enough possibilities + of tests messing up standard FDs. + +- add pluginmanager.do_configure(config) as a link to + config.do_configure() for plugin-compatibility + +as usual, docs at http://pytest.org and upgrades via:: + + pip install -U pytest + +have fun, +holger krekel diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.5.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.5.0.rst new file mode 100644 index 0000000000..c6cdcdd8a8 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.5.0.rst @@ -0,0 +1,174 @@ +pytest-2.5.0: now down to ZERO reported bugs! +=========================================================================== + +pytest-2.5.0 is a big fixing release, the result of two community bug +fixing days plus numerous additional works from many people and +reporters. The release should be fully compatible to 2.4.2, existing +plugins and test suites. We aim at maintaining this level of ZERO reported +bugs because it's no fun if your testing tool has bugs, is it? Under a +condition, though: when submitting a bug report please provide +clear information about the circumstances and a simple example which +reproduces the problem. + +The issue tracker is of course not empty now. We have many remaining +"enhancement" issues which we'll hopefully can tackle in 2014 with your +help. + +For those who use older Python versions, please note that pytest is not +automatically tested on python2.5 due to virtualenv, setuptools and tox +not supporting it anymore. Manual verification shows that it mostly +works fine but it's not going to be part of the automated release +process and thus likely to break in the future. + +As usual, current docs are at + + http://pytest.org + +and you can upgrade from pypi via:: + + pip install -U pytest + +Particular thanks for helping with this release go to Anatoly Bubenkoff, +Floris Bruynooghe, Marc Abramowitz, Ralph Schmitt, Ronny Pfannschmidt, +Donald Stufft, James Lan, Rob Dennis, Jason R. Coombs, Mathieu Agopian, +Virgil Dupras, Bruno Oliveira, Alex Gaynor and others. + +have fun, +holger krekel + + +2.5.0 +----------------------------------- + +- dropped python2.5 from automated release testing of pytest itself + which means it's probably going to break soon (but still works + with this release we believe). + +- simplified and fixed implementation for calling finalizers when + parametrized fixtures or function arguments are involved. finalization + is now performed lazily at setup time instead of in the "teardown phase". + While this might sound odd at first, it helps to ensure that we are + correctly handling setup/teardown even in complex code. User-level code + should not be affected unless it's implementing the pytest_runtest_teardown + hook and expecting certain fixture instances are torn down within (very + unlikely and would have been unreliable anyway). + +- PR90: add --color=yes|no|auto option to force terminal coloring + mode ("auto" is default). Thanks Marc Abramowitz. + +- fix issue319 - correctly show unicode in assertion errors. Many + thanks to Floris Bruynooghe for the complete PR. Also means + we depend on py>=1.4.19 now. + +- fix issue396 - correctly sort and finalize class-scoped parametrized + tests independently from number of methods on the class. + +- refix issue323 in a better way -- parametrization should now never + cause Runtime Recursion errors because the underlying algorithm + for re-ordering tests per-scope/per-fixture is not recursive + anymore (it was tail-call recursive before which could lead + to problems for more than >966 non-function scoped parameters). + +- fix issue290 - there is preliminary support now for parametrizing + with repeated same values (sometimes useful to test if calling + a second time works as with the first time). + +- close issue240 - document precisely how pytest module importing + works, discuss the two common test directory layouts, and how it + interacts with PEP420-namespace packages. + +- fix issue246 fix finalizer order to be LIFO on independent fixtures + depending on a parametrized higher-than-function scoped fixture. + (was quite some effort so please bear with the complexity of this sentence :) + Thanks Ralph Schmitt for the precise failure example. + +- fix issue244 by implementing special index for parameters to only use + indices for paramentrized test ids + +- fix issue287 by running all finalizers but saving the exception + from the first failing finalizer and re-raising it so teardown will + still have failed. We reraise the first failing exception because + it might be the cause for other finalizers to fail. + +- fix ordering when mock.patch or other standard decorator-wrappings + are used with test methods. This fixes issue346 and should + help with random "xdist" collection failures. Thanks to + Ronny Pfannschmidt and Donald Stufft for helping to isolate it. + +- fix issue357 - special case "-k" expressions to allow for + filtering with simple strings that are not valid python expressions. + Examples: "-k 1.3" matches all tests parametrized with 1.3. + "-k None" filters all tests that have "None" in their name + and conversely "-k 'not None'". + Previously these examples would raise syntax errors. + +- fix issue384 by removing the trial support code + since the unittest compat enhancements allow + trial to handle it on its own + +- don't hide an ImportError when importing a plugin produces one. + fixes issue375. + +- fix issue275 - allow usefixtures and autouse fixtures + for running doctest text files. + +- fix issue380 by making --resultlog only rely on longrepr instead + of the "reprcrash" attribute which only exists sometimes. + +- address issue122: allow @pytest.fixture(params=iterator) by exploding + into a list early on. + +- fix pexpect-3.0 compatibility for pytest's own tests. + (fixes issue386) + +- allow nested parametrize-value markers, thanks James Lan for the PR. + +- fix unicode handling with new monkeypatch.setattr(import_path, value) + API. Thanks Rob Dennis. Fixes issue371. + +- fix unicode handling with junitxml, fixes issue368. + +- In assertion rewriting mode on Python 2, fix the detection of coding + cookies. See issue #330. + +- make "--runxfail" turn imperative pytest.xfail calls into no ops + (it already did neutralize pytest.mark.xfail markers) + +- refine pytest / pkg_resources interactions: The AssertionRewritingHook + PEP302 compliant loader now registers itself with setuptools/pkg_resources + properly so that the pkg_resources.resource_stream method works properly. + Fixes issue366. Thanks for the investigations and full PR to Jason R. Coombs. + +- pytestconfig fixture is now session-scoped as it is the same object during the + whole test run. Fixes issue370. + +- avoid one surprising case of marker malfunction/confusion:: + + @pytest.mark.some(lambda arg: ...) + def test_function(): + + would not work correctly because pytest assumes @pytest.mark.some + gets a function to be decorated already. We now at least detect if this + arg is a lambda and thus the example will work. Thanks Alex Gaynor + for bringing it up. + +- xfail a test on pypy that checks wrong encoding/ascii (pypy does + not error out). fixes issue385. + +- internally make varnames() deal with classes's __init__, + although it's not needed by pytest itself atm. Also + fix caching. Fixes issue376. + +- fix issue221 - handle importing of namespace-package with no + __init__.py properly. + +- refactor internal FixtureRequest handling to avoid monkeypatching. + One of the positive user-facing effects is that the "request" object + can now be used in closures. + +- fixed version comparison in pytest.importskip(modname, minverstring) + +- fix issue377 by clarifying in the nose-compat docs that pytest + does not duplicate the unittest-API into the "plain" namespace. + +- fix verbose reporting for @mock'd test functions diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.5.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.5.1.rst new file mode 100644 index 0000000000..ff39db2d52 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.5.1.rst @@ -0,0 +1,46 @@ +pytest-2.5.1: fixes and new home page styling +=========================================================================== + +pytest is a mature Python testing tool with more than 1000 tests +against itself, passing on many different interpreters and platforms. + +The 2.5.1 release maintains the "zero-reported-bugs" promise by fixing +the three bugs reported since the last release a few days ago. It also +features a new home page styling implemented by Tobias Bieniek, based on +the flask theme from Armin Ronacher: + + http://pytest.org + +If you have anything more to improve styling and docs, +we'd be very happy to merge further pull requests. + +On the coding side, the release also contains a little enhancement to +fixture decorators allowing to directly influence generation of test +ids, thanks to Floris Bruynooghe. Other thanks for helping with +this release go to Anatoly Bubenkoff and Ronny Pfannschmidt. + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +have fun and a nice remaining "bug-free" time of the year :) +holger krekel + +2.5.1 +----------------------------------- + +- merge new documentation styling PR from Tobias Bieniek. + +- fix issue403: allow parametrize of multiple same-name functions within + a collection node. Thanks Andreas Kloeckner and Alex Gaynor for reporting + and analysis. + +- Allow parameterized fixtures to specify the ID of the parameters by + adding an ids argument to pytest.fixture() and pytest.yield_fixture(). + Thanks Floris Bruynooghe. + +- fix issue404 by always using the binary xml escape in the junitxml + plugin. Thanks Ronny Pfannschmidt. + +- fix issue407: fix addoption docstring to point to argparse instead of + optparse. Thanks Daniel D. Wright. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.5.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.5.2.rst new file mode 100644 index 0000000000..edc4da6e19 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.5.2.rst @@ -0,0 +1,63 @@ +pytest-2.5.2: fixes +=========================================================================== + +pytest is a mature Python testing tool with more than 1000 tests +against itself, passing on many different interpreters and platforms. + +The 2.5.2 release fixes a few bugs with two maybe-bugs remaining and +actively being worked on (and waiting for the bug reporter's input). +We also have a new contribution guide thanks to Piotr Banaszkiewicz +and others. + +See docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to the following people who contributed to this release: + + Anatoly Bubenkov + Ronny Pfannschmidt + Floris Bruynooghe + Bruno Oliveira + Andreas Pelme + Jurko Gospodnetić + Piotr Banaszkiewicz + Simon Liedtke + lakka + Lukasz Balcerzak + Philippe Muller + Daniel Hahler + +have fun, +holger krekel + +2.5.2 +----------------------------------- + +- fix issue409 -- better interoperate with cx_freeze by not + trying to import from collections.abc which causes problems + for py27/cx_freeze. Thanks Wolfgang L. for reporting and tracking it down. + +- fixed docs and code to use "pytest" instead of "py.test" almost everywhere. + Thanks Jurko Gospodnetic for the complete PR. + +- fix issue425: mention at end of "py.test -h" that --markers + and --fixtures work according to specified test path (or current dir) + +- fix issue413: exceptions with unicode attributes are now printed + correctly also on python2 and with pytest-xdist runs. (the fix + requires py-1.4.20) + +- copy, cleanup and integrate py.io capture + from pylib 1.4.20.dev2 (rev 13d9af95547e) + +- address issue416: clarify docs as to conftest.py loading semantics + +- fix issue429: comparing byte strings with non-ascii chars in assert + expressions now work better. Thanks Floris Bruynooghe. + +- make capfd/capsys.capture private, its unused and shouldn't be exposed diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.0.rst new file mode 100644 index 0000000000..56fbd6cc1e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.0.rst @@ -0,0 +1,153 @@ +pytest-2.6.0: shorter tracebacks, new warning system, test runner compat +=========================================================================== + +pytest is a mature Python testing tool with more than 1000 tests +against itself, passing on many different interpreters and platforms. + +The 2.6.0 release should be drop-in backward compatible to 2.5.2 and +fixes a number of bugs and brings some new features, mainly: + +- shorter tracebacks by default: only the first (test function) entry + and the last (failure location) entry are shown, the ones between + only in "short" format. Use ``--tb=long`` to get back the old + behaviour of showing "long" entries everywhere. + +- a new warning system which reports oddities during collection + and execution. For example, ignoring collecting Test* classes with an + ``__init__`` now produces a warning. + +- various improvements to nose/mock/unittest integration + +Note also that 2.6.0 departs with the "zero reported bugs" policy +because it has been too hard to keep up with it, unfortunately. +Instead we are for now rather bound to work on "upvoted" issues in +the https://bitbucket.org/pytest-dev/pytest/issues?status=new&status=open&sort=-votes +issue tracker. + +See docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed, among them: + + Benjamin Peterson + Jurko Gospodnetić + Floris Bruynooghe + Marc Abramowitz + Marc Schlaich + Trevor Bekolay + Bruno Oliveira + Alex Groenholm + +have fun, +holger krekel + +2.6.0 +----------------------------------- + +- fix issue537: Avoid importing old assertion reinterpretation code by default. + Thanks Benjamin Peterson. + +- fix issue364: shorten and enhance tracebacks representation by default. + The new "--tb=auto" option (default) will only display long tracebacks + for the first and last entry. You can get the old behaviour of printing + all entries as long entries with "--tb=long". Also short entries by + default are now printed very similarly to "--tb=native" ones. + +- fix issue514: teach assertion reinterpretation about private class attributes + Thanks Benjamin Peterson. + +- change -v output to include full node IDs of tests. Users can copy + a node ID from a test run, including line number, and use it as a + positional argument in order to run only a single test. + +- fix issue 475: fail early and comprehensible if calling + pytest.raises with wrong exception type. + +- fix issue516: tell in getting-started about current dependencies. + +- cleanup setup.py a bit and specify supported versions. Thanks Jurko + Gospodnetic for the PR. + +- change XPASS colour to yellow rather then red when tests are run + with -v. + +- fix issue473: work around mock putting an unbound method into a class + dict when double-patching. + +- fix issue498: if a fixture finalizer fails, make sure that + the fixture is still invalidated. + +- fix issue453: the result of the pytest_assertrepr_compare hook now gets + it's newlines escaped so that format_exception does not blow up. + +- internal new warning system: pytest will now produce warnings when + it detects oddities in your test collection or execution. + Warnings are ultimately sent to a new pytest_logwarning hook which is + currently only implemented by the terminal plugin which displays + warnings in the summary line and shows more details when -rw (report on + warnings) is specified. + +- change skips into warnings for test classes with an __init__ and + callables in test modules which look like a test but are not functions. + +- fix issue436: improved finding of initial conftest files from command + line arguments by using the result of parse_known_args rather than + the previous flaky heuristics. Thanks Marc Abramowitz for tests + and initial fixing approaches in this area. + +- fix issue #479: properly handle nose/unittest(2) SkipTest exceptions + during collection/loading of test modules. Thanks to Marc Schlaich + for the complete PR. + +- fix issue490: include pytest_load_initial_conftests in documentation + and improve docstring. + +- fix issue472: clarify that ``pytest.config.getvalue()`` cannot work + if it's triggered ahead of command line parsing. + +- merge PR123: improved integration with mock.patch decorator on tests. + +- fix issue412: messing with stdout/stderr FD-level streams is now + captured without crashes. + +- fix issue483: trial/py33 works now properly. Thanks Daniel Grana for PR. + +- improve example for pytest integration with "python setup.py test" + which now has a generic "-a" or "--pytest-args" option where you + can pass additional options as a quoted string. Thanks Trevor Bekolay. + +- simplified internal capturing mechanism and made it more robust + against tests or setups changing FD1/FD2, also better integrated + now with pytest.pdb() in single tests. + +- improvements to pytest's own test-suite leakage detection, courtesy of PRs + from Marc Abramowitz + +- fix issue492: avoid leak in test_writeorg. Thanks Marc Abramowitz. + +- fix issue493: don't run tests in doc directory with ``python setup.py test`` + (use tox -e doctesting for that) + +- fix issue486: better reporting and handling of early conftest loading failures + +- some cleanup and simplification of internal conftest handling. + +- work a bit harder to break reference cycles when catching exceptions. + Thanks Jurko Gospodnetic. + +- fix issue443: fix skip examples to use proper comparison. Thanks Alex + Groenholm. + +- support nose-style ``__test__`` attribute on modules, classes and + functions, including unittest-style Classes. If set to False, the + test will not be collected. + +- fix issue512: show "" for arguments which might not be set + in monkeypatch plugin. Improves output in documentation. + +- avoid importing "py.test" (an old alias module for "pytest") diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.1.rst new file mode 100644 index 0000000000..7469c488e5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.1.rst @@ -0,0 +1,58 @@ +pytest-2.6.1: fixes and new xfail feature +=========================================================================== + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. +The 2.6.1 release is drop-in compatible to 2.5.2 and actually fixes some +regressions introduced with 2.6.0. It also brings a little feature +to the xfail marker which now recognizes expected exceptions, +see the CHANGELOG below. + +See docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed, among them: + + Floris Bruynooghe + Bruno Oliveira + Nicolas Delaby + +have fun, +holger krekel + +Changes 2.6.1 +================= + +- No longer show line numbers in the --verbose output, the output is now + purely the nodeid. The line number is still shown in failure reports. + Thanks Floris Bruynooghe. + +- fix issue437 where assertion rewriting could cause pytest-xdist worker nodes + to collect different tests. Thanks Bruno Oliveira. + +- fix issue555: add "errors" attribute to capture-streams to satisfy + some distutils and possibly other code accessing sys.stdout.errors. + +- fix issue547 capsys/capfd also work when output capturing ("-s") is disabled. + +- address issue170: allow pytest.mark.xfail(...) to specify expected exceptions via + an optional "raises=EXC" argument where EXC can be a single exception + or a tuple of exception classes. Thanks David Mohr for the complete + PR. + +- fix integration of pytest with unittest.mock.patch decorator when + it uses the "new" argument. Thanks Nicolas Delaby for test and PR. + +- fix issue with detecting conftest files if the arguments contain + "::" node id specifications (copy pasted from "-v" output) + +- fix issue544 by only removing "@NUM" at the end of "::" separated parts + and if the part has a ".py" extension + +- don't use py.std import helper, rather import things directly. + Thanks Bruno Oliveira. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.2.rst new file mode 100644 index 0000000000..9c3b7d96b0 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.2.rst @@ -0,0 +1,51 @@ +pytest-2.6.2: few fixes and cx_freeze support +=========================================================================== + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. +This release is drop-in compatible to 2.5.2 and 2.6.X. It also +brings support for including pytest with cx_freeze or similar +freezing tools into your single-file app distribution. For details +see the CHANGELOG below. + +See docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed, among them: + + Floris Bruynooghe + Benjamin Peterson + Bruno Oliveira + +have fun, +holger krekel + +2.6.2 +----------- + +- Added function pytest.freeze_includes(), which makes it easy to embed + pytest into executables using tools like cx_freeze. + See docs for examples and rationale. Thanks Bruno Oliveira. + +- Improve assertion rewriting cache invalidation precision. + +- fixed issue561: adapt autouse fixture example for python3. + +- fixed issue453: assertion rewriting issue with __repr__ containing + "\n{", "\n}" and "\n~". + +- fix issue560: correctly display code if an "else:" or "finally:" is + followed by statements on the same line. + +- Fix example in monkeypatch documentation, thanks t-8ch. + +- fix issue572: correct tmpdir doc example for python3. + +- Do not mark as universal wheel because Python 2.6 is different from + other builds due to the extra argparse dependency. Fixes issue566. + Thanks sontek. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.3.rst new file mode 100644 index 0000000000..56973a2b2f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.6.3.rst @@ -0,0 +1,51 @@ +pytest-2.6.3: fixes and little improvements +=========================================================================== + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. +This release is drop-in compatible to 2.5.2 and 2.6.X. +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed, among them: + + Floris Bruynooghe + Oleg Sinyavskiy + Uwe Schmitt + Charles Cloud + Wolfgang Schnerring + +have fun, +holger krekel + +Changes 2.6.3 +====================== + +- fix issue575: xunit-xml was reporting collection errors as failures + instead of errors, thanks Oleg Sinyavskiy. + +- fix issue582: fix setuptools example, thanks Laszlo Papp and Ronny + Pfannschmidt. + +- Fix infinite recursion bug when pickling capture.EncodedFile, thanks + Uwe Schmitt. + +- fix issue589: fix bad interaction with numpy and others when showing + exceptions. Check for precise "maximum recursion depth exceed" exception + instead of presuming any RuntimeError is that one (implemented in py + dep). Thanks Charles Cloud for analysing the issue. + +- fix conftest related fixture visibility issue: when running with a + CWD outside of a test package pytest would get fixture discovery wrong. + Thanks to Wolfgang Schnerring for figuring out a reproducible example. + +- Introduce pytest_enter_pdb hook (needed e.g. by pytest_timeout to cancel the + timeout when interactively entering pdb). Thanks Wolfgang Schnerring. + +- check xfail/skip also with non-python function test items. Thanks + Floris Bruynooghe. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.7.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.7.0.rst new file mode 100644 index 0000000000..2840178a07 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.7.0.rst @@ -0,0 +1,100 @@ +pytest-2.7.0: fixes, features, speed improvements +=========================================================================== + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. +This release is supposed to be drop-in compatible to 2.6.X. + +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed, among them: + + Anatoly Bubenkoff + Floris Bruynooghe + Brianna Laugher + Eric Siegerman + Daniel Hahler + Charles Cloud + Tom Viner + Holger Peters + Ldiary Translations + almarklein + +have fun, +holger krekel + +2.7.0 (compared to 2.6.4) +----------------------------- + +- fix issue435: make reload() work when assert rewriting is active. + Thanks Daniel Hahler. + +- fix issue616: conftest.py files and their contained fixtures are now + properly considered for visibility, independently from the exact + current working directory and test arguments that are used. + Many thanks to Eric Siegerman and his PR235 which contains + systematic tests for conftest visibility and now passes. + This change also introduces the concept of a ``rootdir`` which + is printed as a new pytest header and documented in the pytest + customize web page. + +- change reporting of "diverted" tests, i.e. tests that are collected + in one file but actually come from another (e.g. when tests in a test class + come from a base class in a different file). We now show the nodeid + and indicate via a postfix the other file. + +- add ability to set command line options by environment variable PYTEST_ADDOPTS. + +- added documentation on the new pytest-dev teams on bitbucket and + github. See https://pytest.org/en/stable/contributing.html . + Thanks to Anatoly for pushing and initial work on this. + +- fix issue650: new option ``--docttest-ignore-import-errors`` which + will turn import errors in doctests into skips. Thanks Charles Cloud + for the complete PR. + +- fix issue655: work around different ways that cause python2/3 + to leak sys.exc_info into fixtures/tests causing failures in 3rd party code + +- fix issue615: assertion rewriting did not correctly escape % signs + when formatting boolean operations, which tripped over mixing + booleans with modulo operators. Thanks to Tom Viner for the report, + triaging and fix. + +- implement issue351: add ability to specify parametrize ids as a callable + to generate custom test ids. Thanks Brianna Laugher for the idea and + implementation. + +- introduce and document new hookwrapper mechanism useful for plugins + which want to wrap the execution of certain hooks for their purposes. + This supersedes the undocumented ``__multicall__`` protocol which + pytest itself and some external plugins use. Note that pytest-2.8 + is scheduled to drop supporting the old ``__multicall__`` + and only support the hookwrapper protocol. + +- majorly speed up invocation of plugin hooks + +- use hookwrapper mechanism in builtin pytest plugins. + +- add a doctest ini option for doctest flags, thanks Holger Peters. + +- add note to docs that if you want to mark a parameter and the + parameter is a callable, you also need to pass in a reason to disambiguate + it from the "decorator" case. Thanks Tom Viner. + +- "python_classes" and "python_functions" options now support glob-patterns + for test discovery, as discussed in issue600. Thanks Ldiary Translations. + +- allow to override parametrized fixtures with non-parametrized ones and vice versa (bubenkoff). + +- fix issue463: raise specific error for 'parameterize' misspelling (pfctdayelise). + +- On failure, the ``sys.last_value``, ``sys.last_type`` and + ``sys.last_traceback`` are set, so that a user can inspect the error + via postmortem debugging (almarklein). diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.7.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.7.1.rst new file mode 100644 index 0000000000..5110c085e0 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.7.1.rst @@ -0,0 +1,58 @@ +pytest-2.7.1: bug fixes +======================= + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. +This release is supposed to be drop-in compatible to 2.7.0. + +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + + Bruno Oliveira + Holger Krekel + Ionel Maries Cristian + Floris Bruynooghe + +Happy testing, +The py.test Development Team + + +2.7.1 (compared to 2.7.0) +------------------------- + +- fix issue731: do not get confused by the braces which may be present + and unbalanced in an object's repr while collapsing False + explanations. Thanks Carl Meyer for the report and test case. + +- fix issue553: properly handling inspect.getsourcelines failures in + FixtureLookupError which would lead to an internal error, + obfuscating the original problem. Thanks talljosh for initial + diagnose/patch and Bruno Oliveira for final patch. + +- fix issue660: properly report scope-mismatch-access errors + independently from ordering of fixture arguments. Also + avoid the pytest internal traceback which does not provide + information to the user. Thanks Holger Krekel. + +- streamlined and documented release process. Also all versions + (in setup.py and documentation generation) are now read + from _pytest/__init__.py. Thanks Holger Krekel. + +- fixed docs to remove the notion that yield-fixtures are experimental. + They are here to stay :) Thanks Bruno Oliveira. + +- Support building wheels by using environment markers for the + requirements. Thanks Ionel Maries Cristian. + +- fixed regression to 2.6.4 which surfaced e.g. in lost stdout capture printing + when tests raised SystemExit. Thanks Holger Krekel. + +- reintroduced _pytest fixture of the pytester plugin which is used + at least by pytest-xdist. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.7.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.7.2.rst new file mode 100644 index 0000000000..93e5b64eee --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.7.2.rst @@ -0,0 +1,57 @@ +pytest-2.7.2: bug fixes +======================= + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. +This release is supposed to be drop-in compatible to 2.7.1. + +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + + Bruno Oliveira + Floris Bruynooghe + Punyashloka Biswal + Aron Curzon + Benjamin Peterson + Thomas De Schampheleire + Edison Gustavo Muenz + Holger Krekel + +Happy testing, +The py.test Development Team + + +2.7.2 (compared to 2.7.1) +----------------------------- + +- fix issue767: pytest.raises value attribute does not contain the exception + instance on Python 2.6. Thanks Eric Siegerman for providing the test + case and Bruno Oliveira for PR. + +- Automatically create directory for junitxml and results log. + Thanks Aron Curzon. + +- fix issue713: JUnit XML reports for doctest failures. + Thanks Punyashloka Biswal. + +- fix issue735: assertion failures on debug versions of Python 3.4+ + Thanks Benjamin Peterson. + +- fix issue114: skipif marker reports to internal skipping plugin; + Thanks Floris Bruynooghe for reporting and Bruno Oliveira for the PR. + +- fix issue748: unittest.SkipTest reports to internal pytest unittest plugin. + Thanks Thomas De Schampheleire for reporting and Bruno Oliveira for the PR. + +- fix issue718: failed to create representation of sets containing unsortable + elements in python 2. Thanks Edison Gustavo Muenz + +- fix issue756, fix issue752 (and similar issues): depend on py-1.4.29 + which has a refined algorithm for traceback generation. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.2.rst new file mode 100644 index 0000000000..e472633885 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.2.rst @@ -0,0 +1,44 @@ +pytest-2.8.2: bug fixes +======================= + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. +This release is supposed to be drop-in compatible to 2.8.1. + +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + + Bruno Oliveira + Demian Brecht + Florian Bruhin + Ionel Cristian Mărieș + Raphael Pierzina + Ronny Pfannschmidt + holger krekel + +Happy testing, +The py.test Development Team + + +2.8.2 (compared to 2.7.2) +----------------------------- + +- fix #1085: proper handling of encoding errors when passing encoded byte + strings to pytest.parametrize in Python 2. + Thanks Themanwithoutaplan for the report and Bruno Oliveira for the PR. + +- fix #1087: handling SystemError when passing empty byte strings to + pytest.parametrize in Python 3. + Thanks Paul Kehrer for the report and Bruno Oliveira for the PR. + +- fix #995: fixed internal error when filtering tracebacks where one entry + was generated by an exec() statement. + Thanks Daniel Hahler, Ashley C Straw, Philippe Gauthier and Pavel Savchenko + for contributing and Bruno Oliveira for the PR. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.3.rst new file mode 100644 index 0000000000..3f357252bb --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.3.rst @@ -0,0 +1,58 @@ +pytest-2.8.3: bug fixes +======================= + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. +This release is supposed to be drop-in compatible to 2.8.2. + +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + + Bruno Oliveira + Florian Bruhin + Gabe Hollombe + Gabriel Reis + Hartmut Goebel + John Vandenberg + Lee Kamentsky + Michael Birtwell + Raphael Pierzina + Ronny Pfannschmidt + William Martin Stewart + +Happy testing, +The py.test Development Team + + +2.8.3 (compared to 2.8.2) +----------------------------- + +- fix #1169: add __name__ attribute to testcases in TestCaseFunction to + support the @unittest.skip decorator on functions and methods. + Thanks Lee Kamentsky for the PR. + +- fix #1035: collecting tests if test module level obj has __getattr__(). + Thanks Suor for the report and Bruno Oliveira / Tom Viner for the PR. + +- fix #331: don't collect tests if their failure cannot be reported correctly + e.g. they are a callable instance of a class. + +- fix #1133: fixed internal error when filtering tracebacks where one entry + belongs to a file which is no longer available. + Thanks Bruno Oliveira for the PR. + +- enhancement made to highlight in red the name of the failing tests so + they stand out in the output. + Thanks Gabriel Reis for the PR. + +- add more talks to the documentation +- extend documentation on the --ignore cli option +- use pytest-runner for setuptools integration +- minor fixes for interaction with OS X El Capitan system integrity protection (thanks Florian) diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.4.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.4.rst new file mode 100644 index 0000000000..adbdecc87e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.4.rst @@ -0,0 +1,52 @@ +pytest-2.8.4 +============ + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. +This release is supposed to be drop-in compatible to 2.8.2. + +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + + Bruno Oliveira + Florian Bruhin + Jeff Widman + Mehdy Khoshnoody + Nicholas Chammas + Ronny Pfannschmidt + Tim Chan + + +Happy testing, +The py.test Development Team + + +2.8.4 (compared to 2.8.3) +----------------------------- + +- fix #1190: ``deprecated_call()`` now works when the deprecated + function has been already called by another test in the same + module. Thanks Mikhail Chernykh for the report and Bruno Oliveira for the + PR. + +- fix #1198: ``--pastebin`` option now works on Python 3. Thanks + Mehdy Khoshnoody for the PR. + +- fix #1219: ``--pastebin`` now works correctly when captured output contains + non-ascii characters. Thanks Bruno Oliveira for the PR. + +- fix #1204: another error when collecting with a nasty __getattr__(). + Thanks Florian Bruhin for the PR. + +- fix the summary printed when no tests did run. + Thanks Florian Bruhin for the PR. + +- a number of documentation modernizations wrt good practices. + Thanks Bruno Oliveira for the PR. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.5.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.5.rst new file mode 100644 index 0000000000..c5343d1ea7 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.5.rst @@ -0,0 +1,39 @@ +pytest-2.8.5 +============ + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. +This release is supposed to be drop-in compatible to 2.8.4. + +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + + Alex Gaynor + aselus-hub + Bruno Oliveira + Ronny Pfannschmidt + + +Happy testing, +The py.test Development Team + + +2.8.5 (compared to 2.8.4) +------------------------- + +- fix #1243: fixed issue where class attributes injected during collection could break pytest. + PR by Alexei Kozlenok, thanks Ronny Pfannschmidt and Bruno Oliveira for the review and help. + +- fix #1074: precompute junitxml chunks instead of storing the whole tree in objects + Thanks Bruno Oliveira for the report and Ronny Pfannschmidt for the PR + +- fix #1238: fix ``pytest.deprecated_call()`` receiving multiple arguments + (Regression introduced in 2.8.4). Thanks Alex Gaynor for the report and + Bruno Oliveira for the PR. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.6.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.6.rst new file mode 100644 index 0000000000..5d6565b16a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.6.rst @@ -0,0 +1,67 @@ +pytest-2.8.6 +============ + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. +This release is supposed to be drop-in compatible to 2.8.5. + +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + + AMiT Kumar + Bruno Oliveira + Erik M. Bray + Florian Bruhin + Georgy Dyuldin + Jeff Widman + Kartik Singhal + Loïc Estève + Manu Phatak + Peter Demin + Rick van Hattem + Ronny Pfannschmidt + Ulrich Petri + foxx + + +Happy testing, +The py.test Development Team + + +2.8.6 (compared to 2.8.5) +------------------------- + +- fix #1259: allow for double nodeids in junitxml, + this was a regression failing plugins combinations + like pytest-pep8 + pytest-flakes + +- Workaround for exception that occurs in pyreadline when using + ``--pdb`` with standard I/O capture enabled. + Thanks Erik M. Bray for the PR. + +- fix #900: Better error message in case the target of a ``monkeypatch`` call + raises an ``ImportError``. + +- fix #1292: monkeypatch calls (setattr, setenv, etc.) are now O(1). + Thanks David R. MacIver for the report and Bruno Oliveira for the PR. + +- fix #1223: captured stdout and stderr are now properly displayed before + entering pdb when ``--pdb`` is used instead of being thrown away. + Thanks Cal Leeming for the PR. + +- fix #1305: pytest warnings emitted during ``pytest_terminal_summary`` are now + properly displayed. + Thanks Ionel Maries Cristian for the report and Bruno Oliveira for the PR. + +- fix #628: fixed internal UnicodeDecodeError when doctests contain unicode. + Thanks Jason R. Coombs for the report and Bruno Oliveira for the PR. + +- fix #1334: Add captured stdout to jUnit XML report on setup error. + Thanks Georgy Dyuldin for the PR. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.7.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.7.rst new file mode 100644 index 0000000000..8236a09666 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.8.7.rst @@ -0,0 +1,31 @@ +pytest-2.8.7 +============ + +This is a hotfix release to solve a regression +in the builtin monkeypatch plugin that got introduced in 2.8.6. + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. +This release is supposed to be drop-in compatible to 2.8.5. + +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + + Ronny Pfannschmidt + + +Happy testing, +The py.test Development Team + + +2.8.7 (compared to 2.8.6) +------------------------- + +- fix #1338: use predictable object resolution for monkeypatch diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.9.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.9.0.rst new file mode 100644 index 0000000000..3aea08cb22 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.9.0.rst @@ -0,0 +1,134 @@ +pytest-2.9.0 +============ + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. + +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + + Anatoly Bubenkov + Bruno Oliveira + Buck Golemon + David Vierra + Florian Bruhin + Galaczi Endre + Georgy Dyuldin + Lukas Bednar + Luke Murphy + Marcin Biernat + Matt Williams + Michael Aquilina + Raphael Pierzina + Ronny Pfannschmidt + Ryan Wooden + Tiemo Kieft + TomV + holger krekel + jab + + +Happy testing, +The py.test Development Team + + +2.9.0 (compared to 2.8.7) +------------------------- + +**New Features** + +* New ``pytest.mark.skip`` mark, which unconditionally skips marked tests. + Thanks :user:`MichaelAquilina` for the complete PR (:pull:`1040`). + +* ``--doctest-glob`` may now be passed multiple times in the command-line. + Thanks :user:`jab` and :user:`nicoddemus` for the PR. + +* New ``-rp`` and ``-rP`` reporting options give the summary and full output + of passing tests, respectively. Thanks to :user:`codewarrior0` for the PR. + +* ``pytest.mark.xfail`` now has a ``strict`` option which makes ``XPASS`` + tests to fail the test suite, defaulting to ``False``. There's also a + ``xfail_strict`` ini option that can be used to configure it project-wise. + Thanks :user:`rabbbit` for the request and :user:`nicoddemus` for the PR (:issue:`1355`). + +* ``Parser.addini`` now supports options of type ``bool``. Thanks + :user:`nicoddemus` for the PR. + +* New ``ALLOW_BYTES`` doctest option strips ``b`` prefixes from byte strings + in doctest output (similar to ``ALLOW_UNICODE``). + Thanks :user:`jaraco` for the request and :user:`nicoddemus` for the PR (:issue:`1287`). + +* give a hint on KeyboardInterrupt to use the --fulltrace option to show the errors, + this fixes :issue:`1366`. + Thanks to :user:`hpk42` for the report and :user:`RonnyPfannschmidt` for the PR. + +* catch IndexError exceptions when getting exception source location. This fixes + pytest internal error for dynamically generated code (fixtures and tests) + where source lines are fake by intention + +**Changes** + +* **Important**: `py.code `_ has been + merged into the ``pytest`` repository as ``pytest._code``. This decision + was made because ``py.code`` had very few uses outside ``pytest`` and the + fact that it was in a different repository made it difficult to fix bugs on + its code in a timely manner. The team hopes with this to be able to better + refactor out and improve that code. + This change shouldn't affect users, but it is useful to let users aware + if they encounter any strange behavior. + + Keep in mind that the code for ``pytest._code`` is **private** and + **experimental**, so you definitely should not import it explicitly! + + Please note that the original ``py.code`` is still available in + `pylib `_. + +* ``pytest_enter_pdb`` now optionally receives the pytest config object. + Thanks :user:`nicoddemus` for the PR. + +* Removed code and documentation for Python 2.5 or lower versions, + including removal of the obsolete ``_pytest.assertion.oldinterpret`` module. + Thanks :user:`nicoddemus` for the PR (:issue:`1226`). + +* Comparisons now always show up in full when ``CI`` or ``BUILD_NUMBER`` is + found in the environment, even when -vv isn't used. + Thanks :user:`The-Compiler` for the PR. + +* ``--lf`` and ``--ff`` now support long names: ``--last-failed`` and + ``--failed-first`` respectively. + Thanks :user:`MichaelAquilina` for the PR. + +* Added expected exceptions to pytest.raises fail message + +* Collection only displays progress ("collecting X items") when in a terminal. + This avoids cluttering the output when using ``--color=yes`` to obtain + colors in CI integrations systems (:issue:`1397`). + +**Bug Fixes** + +* The ``-s`` and ``-c`` options should now work under ``xdist``; + ``Config.fromdictargs`` now represents its input much more faithfully. + Thanks to :user:`bukzor` for the complete PR (:issue:`680`). + +* Fix (:issue:`1290`): support Python 3.5's ``@`` operator in assertion rewriting. + Thanks :user:`Shinkenjoe` for report with test case and :user:`tomviner` for the PR. + +* Fix formatting utf-8 explanation messages (:issue:`1379`). + Thanks :user:`biern` for the PR. + +* Fix `traceback style docs`_ to describe all of the available options + (auto/long/short/line/native/no), with ``auto`` being the default since v2.6. + Thanks :user:`hackebrot` for the PR. + +* Fix (:issue:`1422`): junit record_xml_property doesn't allow multiple records + with same name. + + +.. _`traceback style docs`: https://pytest.org/en/stable/how-to/output.html#modifying-python-traceback-printing diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.9.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.9.1.rst new file mode 100644 index 0000000000..6a627ad3cd --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.9.1.rst @@ -0,0 +1,57 @@ +pytest-2.9.1 +============ + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. + +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + + Bruno Oliveira + Daniel Hahler + Dmitry Malinovsky + Florian Bruhin + Floris Bruynooghe + Matt Bachmann + Ronny Pfannschmidt + TomV + Vladimir Bolshakov + Zearin + palaviv + + +Happy testing, +The py.test Development Team + + +2.9.1 (compared to 2.9.0) +------------------------- + +**Bug Fixes** + +* Improve error message when a plugin fails to load. + Thanks :user:`nicoddemus` for the PR. + +* Fix (:issue:`1178`): + ``pytest.fail`` with non-ascii characters raises an internal pytest error. + Thanks :user:`nicoddemus` for the PR. + +* Fix (:issue:`469`): junit parses report.nodeid incorrectly, when params IDs + contain ``::``. Thanks :user:`tomviner` for the PR (:pull:`1431`). + +* Fix (:issue:`578`): SyntaxErrors + containing non-ascii lines at the point of failure generated an internal + py.test error. + Thanks :user:`asottile` for the report and :user:`nicoddemus` for the PR. + +* Fix (:issue:`1437`): When passing in a bytestring regex pattern to parameterize + attempt to decode it as utf-8 ignoring errors. + +* Fix (:issue:`649`): parametrized test nodes cannot be specified to run on the command line. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.9.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.9.2.rst new file mode 100644 index 0000000000..2dc82a1117 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-2.9.2.rst @@ -0,0 +1,65 @@ +pytest-2.9.2 +============ + +pytest is a mature Python testing tool with more than 1100 tests +against itself, passing on many different interpreters and platforms. + +See below for the changes and see docs at: + + http://pytest.org + +As usual, you can upgrade from pypi via:: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + + Adam Chainz + Benjamin Dopplinger + Bruno Oliveira + Florian Bruhin + John Towler + Martin Prusse + Meng Jue + MengJueM + Omar Kohl + Quentin Pradet + Ronny Pfannschmidt + Thomas Güttler + TomV + Tyler Goodlet + + +Happy testing, +The py.test Development Team + + +2.9.2 (compared to 2.9.1) +--------------------------- + +**Bug Fixes** + +* fix :issue:`510`: skip tests where one parameterize dimension was empty + thanks Alex Stapleton for the Report and :user:`RonnyPfannschmidt` for the PR + +* Fix Xfail does not work with condition keyword argument. + Thanks :user:`astraw38` for reporting the issue (:issue:`1496`) and :user:`tomviner` + for PR the (:pull:`1524`). + +* Fix win32 path issue when putting custom config file with absolute path + in ``pytest.main("-c your_absolute_path")``. + +* Fix maximum recursion depth detection when raised error class is not aware + of unicode/encoded bytes. + Thanks :user:`prusse-martin` for the PR (:pull:`1506`). + +* Fix ``pytest.mark.skip`` mark when used in strict mode. + Thanks :user:`pquentin` for the PR and :user:`RonnyPfannschmidt` for + showing how to fix the bug. + +* Minor improvements and fixes to the documentation. + Thanks :user:`omarkohl` for the PR. + +* Fix ``--fixtures`` to show all fixture definitions as opposed to just + one per fixture name. + Thanks to :user:`hackebrot` for the PR. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.0.rst new file mode 100644 index 0000000000..5de3891148 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.0.rst @@ -0,0 +1,82 @@ +pytest-3.0.0 +============ + +The pytest team is proud to announce the 3.0.0 release! + +pytest is a mature Python testing tool with more than 1600 tests +against itself, passing on many different interpreters and platforms. + +This release contains a lot of bugs fixes and improvements, and much of +the work done on it was possible because of the 2016 Sprint[1], which +was funded by an indiegogo campaign which raised over US$12,000 with +nearly 100 backers. + +There's a "What's new in pytest 3.0" [2] blog post highlighting the +major features in this release. + +To see the complete changelog and documentation, please visit: + + http://docs.pytest.org + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + + AbdealiJK + Ana Ribeiro + Antony Lee + Brandon W Maister + Brianna Laugher + Bruno Oliveira + Ceridwen + Christian Boelsen + Daniel Hahler + Danielle Jenkins + Dave Hunt + Diego Russo + Dmitry Dygalo + Edoardo Batini + Eli Boyarski + Florian Bruhin + Floris Bruynooghe + Greg Price + Guyzmo + HEAD KANGAROO + JJ + Javi Romero + Javier Domingo Cansino + Kale Kundert + Kalle Bronsen + Marius Gedminas + Matt Williams + Mike Lundy + Oliver Bestwalter + Omar Kohl + Raphael Pierzina + RedBeardCode + Roberto Polli + Romain Dorgueil + Roman Bolshakov + Ronny Pfannschmidt + Stefan Zimmermann + Steffen Allner + Tareq Alayan + Ted Xiao + Thomas Grainger + Tom Viner + TomV + Vasily Kuznetsov + aostr + marscher + palaviv + satoru + taschini + + +Happy testing, +The Pytest Development Team + +[1] http://blog.pytest.org/2016/pytest-development-sprint/ +[2] http://blog.pytest.org/2016/whats-new-in-pytest-30/ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.1.rst new file mode 100644 index 0000000000..8f5cfe411a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.1.rst @@ -0,0 +1,26 @@ +pytest-3.0.1 +============ + +pytest 3.0.1 has just been released to PyPI. + +This release fixes some regressions reported in version 3.0.0, being a +drop-in replacement. To upgrade: + + pip install --upgrade pytest + +The changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + + Adam Chainz + Andrew Svetlov + Bruno Oliveira + Daniel Hahler + Dmitry Dygalo + Florian Bruhin + Marcin Bachry + Ronny Pfannschmidt + matthiasha + +Happy testing, +The py.test Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.2.rst new file mode 100644 index 0000000000..86ba82ca6e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.2.rst @@ -0,0 +1,24 @@ +pytest-3.0.2 +============ + +pytest 3.0.2 has just been released to PyPI. + +This release fixes some regressions and bugs reported in version 3.0.1, being a +drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Ahn Ki-Wook +* Bruno Oliveira +* Florian Bruhin +* Jordan Guymon +* Raphael Pierzina +* Ronny Pfannschmidt +* mbyt + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.3.rst new file mode 100644 index 0000000000..89a2e0c744 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.3.rst @@ -0,0 +1,27 @@ +pytest-3.0.3 +============ + +pytest 3.0.3 has just been released to PyPI. + +This release fixes some regressions and bugs reported in the last version, +being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira +* Florian Bruhin +* Floris Bruynooghe +* Huayi Zhang +* Lev Maximov +* Raquel Alegre +* Ronny Pfannschmidt +* Roy Williams +* Tyler Goodlet +* mbyt + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.4.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.4.rst new file mode 100644 index 0000000000..72c2d29464 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.4.rst @@ -0,0 +1,29 @@ +pytest-3.0.4 +============ + +pytest 3.0.4 has just been released to PyPI. + +This release fixes some regressions and bugs reported in the last version, +being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira +* Dan Wandschneider +* Florian Bruhin +* Georgy Dyuldin +* Grigorii Eremeev +* Jason R. Coombs +* Manuel Jacob +* Mathieu Clabaut +* Michael Seifert +* Nikolaus Rath +* Ronny Pfannschmidt +* Tom V + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.5.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.5.rst new file mode 100644 index 0000000000..97edb7d462 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.5.rst @@ -0,0 +1,27 @@ +pytest-3.0.5 +============ + +pytest 3.0.5 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Ana Vojnovic +* Bruno Oliveira +* Daniel Hahler +* Duncan Betts +* Igor Starikov +* Ismail +* Luke Murphy +* Ned Batchelder +* Ronny Pfannschmidt +* Sebastian Ramacher +* nmundar + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.6.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.6.rst new file mode 100644 index 0000000000..9c072cedcc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.6.rst @@ -0,0 +1,33 @@ +pytest-3.0.6 +============ + +pytest 3.0.6 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + + +Thanks to all who contributed to this release, among them: + +* Andreas Pelme +* Bruno Oliveira +* Dmitry Malinovsky +* Eli Boyarski +* Jakub Wilk +* Jeff Widman +* Loïc Estève +* Luke Murphy +* Miro Hrončok +* Oscar Hellström +* Peter Heatwole +* Philippe Ombredanne +* Ronny Pfannschmidt +* Rutger Prins +* Stefan Scherfke + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.7.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.7.rst new file mode 100644 index 0000000000..4b7e075e76 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.0.7.rst @@ -0,0 +1,33 @@ +pytest-3.0.7 +============ + +pytest 3.0.7 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Barney Gale +* Bruno Oliveira +* Florian Bruhin +* Floris Bruynooghe +* Ionel Cristian Mărieș +* Katerina Koukiou +* NODA, Kai +* Omer Hadari +* Patrick Hayes +* Ran Benita +* Ronny Pfannschmidt +* Victor Uriarte +* Vidar Tonaas Fauske +* Ville Skyttä +* fbjorn +* mbyt + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.0.rst new file mode 100644 index 0000000000..5527706794 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.0.rst @@ -0,0 +1,61 @@ +pytest-3.1.0 +======================================= + +The pytest team is proud to announce the 3.1.0 release! + +pytest is a mature Python testing tool with more than 1600 tests +against itself, passing on many different interpreters and platforms. + +This release contains a bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + +http://doc.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + http://docs.pytest.org + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Ben Lloyd +* Bruno Oliveira +* David Giese +* David Szotten +* Dmitri Pribysh +* Florian Bruhin +* Florian Schulze +* Floris Bruynooghe +* John Towler +* Jonas Obrist +* Katerina Koukiou +* Kodi Arfer +* Krzysztof Szularz +* Lev Maximov +* Loïc Estève +* Luke Murphy +* Manuel Krebber +* Matthew Duck +* Matthias Bussonnier +* Michael Howitz +* Michal Wajszczuk +* Paweł Adamczak +* Rafael Bertoldi +* Ravi Chandra +* Ronny Pfannschmidt +* Skylar Downes +* Thomas Kriechbaumer +* Vitaly Lashmanov +* Vlad Dragos +* Wheerd +* Xander Johnson +* mandeep +* reut + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.1.rst new file mode 100644 index 0000000000..135b2fe844 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.1.rst @@ -0,0 +1,23 @@ +pytest-3.1.1 +======================================= + +pytest 3.1.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira +* Florian Bruhin +* Floris Bruynooghe +* Jason R. Coombs +* Ronny Pfannschmidt +* wanghui + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.2.rst new file mode 100644 index 0000000000..a9b85c4715 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.2.rst @@ -0,0 +1,23 @@ +pytest-3.1.2 +======================================= + +pytest 3.1.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Andreas Pelme +* ApaDoctor +* Bruno Oliveira +* Florian Bruhin +* Ronny Pfannschmidt +* Segev Finer + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.3.rst new file mode 100644 index 0000000000..bc2b85fcfd --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.1.3.rst @@ -0,0 +1,23 @@ +pytest-3.1.3 +======================================= + +pytest 3.1.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Antoine Legrand +* Bruno Oliveira +* Max Moroz +* Raphael Pierzina +* Ronny Pfannschmidt +* Ryan Fitzpatrick + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.10.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.10.0.rst new file mode 100644 index 0000000000..ff3c000b0e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.10.0.rst @@ -0,0 +1,43 @@ +pytest-3.10.0 +======================================= + +The pytest team is proud to announce the 3.10.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Anders Hovmöller +* Andreu Vallbona Plazas +* Ankit Goel +* Anthony Sottile +* Bernardo Gomes +* Brianna Laugher +* Bruno Oliveira +* Daniel Hahler +* David Szotten +* Mick Koch +* Niclas Olofsson +* Palash Chatterjee +* Ronny Pfannschmidt +* Sven-Hendrik Haase +* Ville Skyttä +* William Jamir Silva + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.10.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.10.1.rst new file mode 100644 index 0000000000..ad365f6347 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.10.1.rst @@ -0,0 +1,24 @@ +pytest-3.10.1 +======================================= + +pytest 3.10.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Boris Feld +* Bruno Oliveira +* Daniel Hahler +* Fabien ZARIFIAN +* Jon Dufresne +* Ronny Pfannschmidt + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.0.rst new file mode 100644 index 0000000000..edc66a28e7 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.0.rst @@ -0,0 +1,48 @@ +pytest-3.2.0 +======================================= + +The pytest team is proud to announce the 3.2.0 release! + +pytest is a mature Python testing tool with more than 1600 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + http://doc.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + http://docs.pytest.org + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Alex Hartoto +* Andras Tim +* Bruno Oliveira +* Daniel Hahler +* Florian Bruhin +* Floris Bruynooghe +* John Still +* Jordan Moldow +* Kale Kundert +* Lawrence Mitchell +* Llandy Riveron Del Risco +* Maik Figura +* Martin Altmayer +* Mihai Capotă +* Nathaniel Waisbrot +* Nguyễn Hồng Quân +* Pauli Virtanen +* Raphael Pierzina +* Ronny Pfannschmidt +* Segev Finer +* V.Kuznetsov + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.1.rst new file mode 100644 index 0000000000..c40217d311 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.1.rst @@ -0,0 +1,22 @@ +pytest-3.2.1 +======================================= + +pytest 3.2.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Alex Gaynor +* Bruno Oliveira +* Florian Bruhin +* Ronny Pfannschmidt +* Srinivas Reddy Thatiparthy + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.2.rst new file mode 100644 index 0000000000..5e6c43ab17 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.2.rst @@ -0,0 +1,28 @@ +pytest-3.2.2 +======================================= + +pytest 3.2.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Andreas Pelme +* Antonio Hidalgo +* Bruno Oliveira +* Felipe Dau +* Fernando Macedo +* Jesús Espino +* Joan Massich +* Joe Talbott +* Kirill Pinchuk +* Ronny Pfannschmidt +* Xuan Luong + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.3.rst new file mode 100644 index 0000000000..50dce29c1a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.3.rst @@ -0,0 +1,23 @@ +pytest-3.2.3 +======================================= + +pytest 3.2.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira +* Evan +* Joe Hamman +* Oliver Bestwalter +* Ronny Pfannschmidt +* Xuan Luong + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.4.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.4.rst new file mode 100644 index 0000000000..ff0b35781b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.4.rst @@ -0,0 +1,36 @@ +pytest-3.2.4 +======================================= + +pytest 3.2.4 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira +* Christian Boelsen +* Christoph Buchner +* Daw-Ran Liou +* Florian Bruhin +* Franck Michea +* Leonard Lausen +* Matty G +* Owen Tuz +* Pavel Karateev +* Pierre GIRAUD +* Ronny Pfannschmidt +* Stephen Finucane +* Sviatoslav Abakumov +* Thomas Hisch +* Tom Dalton +* Xuan Luong +* Yorgos Pagles +* Семён Марьясин + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.5.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.5.rst new file mode 100644 index 0000000000..68caccbdbc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.2.5.rst @@ -0,0 +1,18 @@ +pytest-3.2.5 +======================================= + +pytest 3.2.5 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.3.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.3.0.rst new file mode 100644 index 0000000000..1cbf2c448c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.3.0.rst @@ -0,0 +1,50 @@ +pytest-3.3.0 +======================================= + +The pytest team is proud to announce the 3.3.0 release! + +pytest is a mature Python testing tool with more than 1600 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + http://doc.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + http://docs.pytest.org + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Ceridwen +* Daniel Hahler +* Dirk Thomas +* Dmitry Malinovsky +* Florian Bruhin +* George Y. Kussumoto +* Hugo +* Jesús Espino +* Joan Massich +* Ofir +* OfirOshir +* Ronny Pfannschmidt +* Samuel Dion-Girardeau +* Srinivas Reddy Thatiparthy +* Sviatoslav Abakumov +* Tarcisio Fischer +* Thomas Hisch +* Tyler Goodlet +* hugovk +* je +* prokaktus + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.3.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.3.1.rst new file mode 100644 index 0000000000..98b6fa6c1b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.3.1.rst @@ -0,0 +1,25 @@ +pytest-3.3.1 +======================================= + +pytest 3.3.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira +* Daniel Hahler +* Eugene Prikazchikov +* Florian Bruhin +* Roland Puntaier +* Ronny Pfannschmidt +* Sebastian Rahlf +* Tom Viner + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.3.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.3.2.rst new file mode 100644 index 0000000000..7a2577d1ff --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.3.2.rst @@ -0,0 +1,28 @@ +pytest-3.3.2 +======================================= + +pytest 3.3.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Antony Lee +* Austin +* Bruno Oliveira +* Florian Bruhin +* Floris Bruynooghe +* Henk-Jaap Wagenaar +* Jurko Gospodnetić +* Ronny Pfannschmidt +* Srinivas Reddy Thatiparthy +* Thomas Hisch + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.4.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.4.0.rst new file mode 100644 index 0000000000..6ab5b124a2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.4.0.rst @@ -0,0 +1,52 @@ +pytest-3.4.0 +======================================= + +The pytest team is proud to announce the 3.4.0 release! + +pytest is a mature Python testing tool with more than 1600 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + http://doc.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + http://docs.pytest.org + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Aaron +* Alan Velasco +* Anders Hovmöller +* Andrew Toolan +* Anthony Sottile +* Aron Coyle +* Brian Maissy +* Bruno Oliveira +* Cyrus Maden +* Florian Bruhin +* Henk-Jaap Wagenaar +* Ian Lesperance +* Jon Dufresne +* Jurko Gospodnetić +* Kate +* Kimberly +* Per A. Brodtkorb +* Pierre-Alexandre Fonta +* Raphael Castaneda +* Ronny Pfannschmidt +* ST John +* Segev Finer +* Thomas Hisch +* Tzu-ping Chung +* feuillemorte + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.4.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.4.1.rst new file mode 100644 index 0000000000..d83949453a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.4.1.rst @@ -0,0 +1,27 @@ +pytest-3.4.1 +======================================= + +pytest 3.4.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Aaron +* Alan Velasco +* Andy Freeland +* Brian Maissy +* Bruno Oliveira +* Florian Bruhin +* Jason R. Coombs +* Marcin Bachry +* Pedro Algarvio +* Ronny Pfannschmidt + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.4.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.4.2.rst new file mode 100644 index 0000000000..07cd9d3a8b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.4.2.rst @@ -0,0 +1,28 @@ +pytest-3.4.2 +======================================= + +pytest 3.4.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Allan Feldman +* Bruno Oliveira +* Florian Bruhin +* Jason R. Coombs +* Kyle Altendorf +* Maik Figura +* Ronny Pfannschmidt +* codetriage-readme-bot +* feuillemorte +* joshm91 +* mike + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.5.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.5.0.rst new file mode 100644 index 0000000000..6bc2f3cd0c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.5.0.rst @@ -0,0 +1,51 @@ +pytest-3.5.0 +======================================= + +The pytest team is proud to announce the 3.5.0 release! + +pytest is a mature Python testing tool with more than 1600 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + http://doc.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + http://docs.pytest.org + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Allan Feldman +* Brian Maissy +* Bruno Oliveira +* Carlos Jenkins +* Daniel Hahler +* Florian Bruhin +* Jason R. Coombs +* Jeffrey Rackauckas +* Jordan Speicher +* Julien Palard +* Kale Kundert +* Kostis Anagnostopoulos +* Kyle Altendorf +* Maik Figura +* Pedro Algarvio +* Ronny Pfannschmidt +* Tadeu Manoel +* Tareq Alayan +* Thomas Hisch +* William Lee +* codetriage-readme-bot +* feuillemorte +* joshm91 +* mike + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.5.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.5.1.rst new file mode 100644 index 0000000000..802be03684 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.5.1.rst @@ -0,0 +1,30 @@ +pytest-3.5.1 +======================================= + +pytest 3.5.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Brian Maissy +* Bruno Oliveira +* Darren Burns +* David Chudzicki +* Floris Bruynooghe +* Holger Kohr +* Irmen de Jong +* Jeffrey Rackauckas +* Rachel Kogan +* Ronny Pfannschmidt +* Stefan Scherfke +* Tim Strazny +* Семён Марьясин + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.0.rst new file mode 100644 index 0000000000..44b178c169 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.0.rst @@ -0,0 +1,41 @@ +pytest-3.6.0 +======================================= + +The pytest team is proud to announce the 3.6.0 release! + +pytest is a mature Python testing tool with more than 1600 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + http://doc.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + http://docs.pytest.org + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Anthony Shaw +* ApaDoctor +* Brian Maissy +* Bruno Oliveira +* Jon Dufresne +* Katerina Koukiou +* Miro Hrončok +* Rachel Kogan +* Ronny Pfannschmidt +* Tim Hughes +* Tyler Goodlet +* Ville Skyttä +* aviral1701 +* feuillemorte + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.1.rst new file mode 100644 index 0000000000..d971a3d490 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.1.rst @@ -0,0 +1,24 @@ +pytest-3.6.1 +======================================= + +pytest 3.6.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Jeffrey Rackauckas +* Miro Hrončok +* Niklas Meinzer +* Oliver Bestwalter +* Ronny Pfannschmidt + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.2.rst new file mode 100644 index 0000000000..9d91995793 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.2.rst @@ -0,0 +1,29 @@ +pytest-3.6.2 +======================================= + +pytest 3.6.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Alan Velasco +* Alex Barbato +* Anthony Sottile +* Bartosz Cierocki +* Bruno Oliveira +* Daniel Hahler +* Guoqiang Zhang +* Hynek Schlawack +* John T. Wodder II +* Michael Käufl +* Ronny Pfannschmidt +* Samuel Dion-Girardeau + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.3.rst new file mode 100644 index 0000000000..4dda2460da --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.3.rst @@ -0,0 +1,27 @@ +pytest-3.6.3 +======================================= + +pytest 3.6.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* AdamEr8 +* Anthony Sottile +* Bruno Oliveira +* Jean-Paul Calderone +* Jon Dufresne +* Marcelo Duarte Trevisani +* Ondřej Súkup +* Ronny Pfannschmidt +* T.E.A de Souza +* Victor Maryama + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.4.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.4.rst new file mode 100644 index 0000000000..2c0f9efecc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.6.4.rst @@ -0,0 +1,24 @@ +pytest-3.6.4 +======================================= + +pytest 3.6.4 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bernhard M. Wiedemann +* Bruno Oliveira +* Drew +* E Hershey +* Hugo Martins +* Vlad Shcherbina + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.0.rst new file mode 100644 index 0000000000..89908a9101 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.0.rst @@ -0,0 +1,41 @@ +pytest-3.7.0 +======================================= + +The pytest team is proud to announce the 3.7.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + http://doc.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + http://docs.pytest.org + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Alan +* Alan Brammer +* Ammar Najjar +* Anthony Sottile +* Bruno Oliveira +* Jeffrey Rackauckas +* Kale Kundert +* Ronny Pfannschmidt +* Serhii Mozghovyi +* Tadek Teleżyński +* Wil Cooley +* abrammer +* avirlrma +* turturica + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.1.rst new file mode 100644 index 0000000000..7da5a3e1f7 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.1.rst @@ -0,0 +1,21 @@ +pytest-3.7.1 +======================================= + +pytest 3.7.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Kale Kundert +* Ronny Pfannschmidt + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.2.rst new file mode 100644 index 0000000000..fcc6121752 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.2.rst @@ -0,0 +1,25 @@ +pytest-3.7.2 +======================================= + +pytest 3.7.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Josh Holland +* Ronny Pfannschmidt +* Sankt Petersbug +* Wes Thomas +* turturica + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.3.rst new file mode 100644 index 0000000000..ee87da60d2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.3.rst @@ -0,0 +1,32 @@ +pytest-3.7.3 +======================================= + +pytest 3.7.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at http://doc.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Andrew Champion +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Gandalf Saxe +* Jennifer Rinker +* Natan Lao +* Ondřej Súkup +* Ronny Pfannschmidt +* Sankt Petersbug +* Tyler Richard +* Victor Maryama +* Vlad Shcherbina +* turturica +* wim glenn + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.4.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.4.rst new file mode 100644 index 0000000000..45be429388 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.7.4.rst @@ -0,0 +1,22 @@ +pytest-3.7.4 +======================================= + +pytest 3.7.4 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Jiri Kuncar +* Steve Piercy + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.8.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.8.0.rst new file mode 100644 index 0000000000..8c35a44f6d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.8.0.rst @@ -0,0 +1,38 @@ +pytest-3.8.0 +======================================= + +The pytest team is proud to announce the 3.8.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* CrazyMerlyn +* Daniel Hahler +* Fabio Zadrozny +* Jeffrey Rackauckas +* Ronny Pfannschmidt +* Virgil Dupras +* dhirensr +* hoefling +* wim glenn + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.8.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.8.1.rst new file mode 100644 index 0000000000..f8f8accc4c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.8.1.rst @@ -0,0 +1,25 @@ +pytest-3.8.1 +======================================= + +pytest 3.8.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Ankit Goel +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Maximilian Albert +* Ronny Pfannschmidt +* William Jamir Silva +* wim glenn + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.8.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.8.2.rst new file mode 100644 index 0000000000..9ea94c98a2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.8.2.rst @@ -0,0 +1,28 @@ +pytest-3.8.2 +======================================= + +pytest 3.8.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Ankit Goel +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Denis Otkidach +* Harry Percival +* Jeffrey Rackauckas +* Jose Carlos Menezes +* Ronny Pfannschmidt +* Zac Hatfield-Dodds +* iwanb + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.0.rst new file mode 100644 index 0000000000..0be6cf5be8 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.0.rst @@ -0,0 +1,43 @@ +pytest-3.9.0 +======================================= + +The pytest team is proud to announce the 3.9.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Andrea Cimatoribus +* Ankit Goel +* Anthony Sottile +* Ben Eyal +* Bruno Oliveira +* Daniel Hahler +* Jeffrey Rackauckas +* Jose Carlos Menezes +* Kyle Altendorf +* Niklas JQ +* Palash Chatterjee +* Ronny Pfannschmidt +* Thomas Hess +* Thomas Hisch +* Tomer Keren +* Victor Maryama + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.1.rst new file mode 100644 index 0000000000..e1afb3759d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.1.rst @@ -0,0 +1,20 @@ +pytest-3.9.1 +======================================= + +pytest 3.9.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira +* Ronny Pfannschmidt +* Thomas Hisch + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.2.rst new file mode 100644 index 0000000000..63e94e5aab --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.2.rst @@ -0,0 +1,23 @@ +pytest-3.9.2 +======================================= + +pytest 3.9.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Ankit Goel +* Anthony Sottile +* Bruno Oliveira +* Ronny Pfannschmidt +* Vincent Barbaresi +* ykantor + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.3.rst new file mode 100644 index 0000000000..661ddb5cb5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-3.9.3.rst @@ -0,0 +1,24 @@ +pytest-3.9.3 +======================================= + +pytest 3.9.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Andreas Profous +* Ankit Goel +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Jon Dufresne +* Ronny Pfannschmidt + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.0.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.0.0.rst new file mode 100644 index 0000000000..5eb0107758 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.0.0.rst @@ -0,0 +1,30 @@ +pytest-4.0.0 +======================================= + +The pytest team is proud to announce the 4.0.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira +* Daniel Hahler +* Ronny Pfannschmidt + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.0.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.0.1.rst new file mode 100644 index 0000000000..2902a6db9f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.0.1.rst @@ -0,0 +1,23 @@ +pytest-4.0.1 +======================================= + +pytest 4.0.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Michael D. Hoyle +* Ronny Pfannschmidt +* Slam + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.0.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.0.2.rst new file mode 100644 index 0000000000..f439b88fe2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.0.2.rst @@ -0,0 +1,24 @@ +pytest-4.0.2 +======================================= + +pytest 4.0.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Pedro Algarvio +* Ronny Pfannschmidt +* Tomer Keren +* Yash Todi + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.1.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.1.0.rst new file mode 100644 index 0000000000..314564eeb6 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.1.0.rst @@ -0,0 +1,44 @@ +pytest-4.1.0 +======================================= + +The pytest team is proud to announce the 4.1.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Adam Johnson +* Aly Sivji +* Andrey Paramonov +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* David Vo +* Hyunchel Kim +* Jeffrey Rackauckas +* Kanguros +* Nicholas Devenish +* Pedro Algarvio +* Randy Barlow +* Ronny Pfannschmidt +* Tomer Keren +* feuillemorte +* wim glenn + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.1.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.1.1.rst new file mode 100644 index 0000000000..1f45e082f8 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.1.1.rst @@ -0,0 +1,27 @@ +pytest-4.1.1 +======================================= + +pytest 4.1.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Anton Lodder +* Bruno Oliveira +* Daniel Hahler +* David Vo +* Oscar Benjamin +* Ronny Pfannschmidt +* Victor Maryama +* Yoav Caspi +* dmitry.dygalo + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.2.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.2.0.rst new file mode 100644 index 0000000000..bcd7f77547 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.2.0.rst @@ -0,0 +1,37 @@ +pytest-4.2.0 +======================================= + +The pytest team is proud to announce the 4.2.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Adam Uhlir +* Anthony Sottile +* Bruno Oliveira +* Christopher Dignam +* Daniel Hahler +* Joseph Hunkeler +* Kristoffer Nordstroem +* Ronny Pfannschmidt +* Thomas Hisch +* wim glenn + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.2.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.2.1.rst new file mode 100644 index 0000000000..36beafe11d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.2.1.rst @@ -0,0 +1,30 @@ +pytest-4.2.1 +======================================= + +pytest 4.2.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Arel Cordero +* Bruno Oliveira +* Daniel Hahler +* Holger Kohr +* Kevin J. Foley +* Nick Murphy +* Paweł Stradomski +* Raphael Pierzina +* Ronny Pfannschmidt +* Sam Brightman +* Thomas Hisch +* Zac Hatfield-Dodds + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.3.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.3.0.rst new file mode 100644 index 0000000000..3b0b428092 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.3.0.rst @@ -0,0 +1,36 @@ +pytest-4.3.0 +======================================= + +The pytest team is proud to announce the 4.3.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Andras Mitzki +* Anthony Sottile +* Bruno Oliveira +* Christian Fetzer +* Daniel Hahler +* Grygorii Iermolenko +* R. Alex Matevish +* Ronny Pfannschmidt +* cclauss + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.3.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.3.1.rst new file mode 100644 index 0000000000..4251c744e5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.3.1.rst @@ -0,0 +1,28 @@ +pytest-4.3.1 +======================================= + +pytest 4.3.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Andras Mitzki +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Danilo Horta +* Grygorii Iermolenko +* Jeff Hale +* Kyle Altendorf +* Stephan Hoyer +* Zac Hatfield-Dodds +* songbowen + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.4.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.4.0.rst new file mode 100644 index 0000000000..dc89739d0a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.4.0.rst @@ -0,0 +1,39 @@ +pytest-4.4.0 +======================================= + +The pytest team is proud to announce the 4.4.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* ApaDoctor +* Bernhard M. Wiedemann +* Brian Skinn +* Bruno Oliveira +* Daniel Hahler +* Gary Tyler +* Jeong YunWon +* Miro Hrončok +* Takafumi Arakaki +* henrykironde +* smheidrich + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.4.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.4.1.rst new file mode 100644 index 0000000000..1272cd8fde --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.4.1.rst @@ -0,0 +1,20 @@ +pytest-4.4.1 +======================================= + +pytest 4.4.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.4.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.4.2.rst new file mode 100644 index 0000000000..5876e83b3b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.4.2.rst @@ -0,0 +1,33 @@ +pytest-4.4.2 +======================================= + +pytest 4.4.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Allan Lewis +* Anthony Sottile +* Bruno Oliveira +* DamianSkrzypczak +* Daniel Hahler +* Don Kirkby +* Douglas Thor +* Hugo +* Ilya Konstantinov +* Jon Dufresne +* Matt Cooper +* Nikolay Kondratyev +* Ondřej Súkup +* Peter Schutt +* Romain Chossart +* Sitaktif + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.5.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.5.0.rst new file mode 100644 index 0000000000..d2a05d4f79 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.5.0.rst @@ -0,0 +1,34 @@ +pytest-4.5.0 +======================================= + +The pytest team is proud to announce the 4.5.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Floris Bruynooghe +* Pulkit Goyal +* Samuel Searles-Bryant +* Zac Hatfield-Dodds + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.0.rst new file mode 100644 index 0000000000..a82fdd47d6 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.0.rst @@ -0,0 +1,43 @@ +pytest-4.6.0 +======================================= + +The pytest team is proud to announce the 4.6.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Akiomi Kamakura +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* David Röthlisberger +* Evan Kepner +* Jeffrey Rackauckas +* MyComputer +* Nikita Krokosh +* Raul Tambre +* Thomas Hisch +* Tim Hoffmann +* Tomer Keren +* Victor Maryama +* danielx123 +* oleg-yegorov + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.1.rst new file mode 100644 index 0000000000..c79839b7b5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.1.rst @@ -0,0 +1,19 @@ +pytest-4.6.1 +======================================= + +pytest 4.6.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.2.rst new file mode 100644 index 0000000000..cfc595293a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.2.rst @@ -0,0 +1,18 @@ +pytest-4.6.2 +======================================= + +pytest 4.6.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.3.rst new file mode 100644 index 0000000000..f578464a7a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.3.rst @@ -0,0 +1,21 @@ +pytest-4.6.3 +======================================= + +pytest 4.6.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Dirk Thomas + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.4.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.4.rst new file mode 100644 index 0000000000..0eefcbeb1c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.4.rst @@ -0,0 +1,22 @@ +pytest-4.6.4 +======================================= + +pytest 4.6.4 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Thomas Grainger +* Zac Hatfield-Dodds + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.5.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.5.rst new file mode 100644 index 0000000000..1ebf361fdf --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.5.rst @@ -0,0 +1,21 @@ +pytest-4.6.5 +======================================= + +pytest 4.6.5 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Thomas Grainger + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.6.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.6.rst new file mode 100644 index 0000000000..b3bf1e431c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.6.rst @@ -0,0 +1,20 @@ +pytest-4.6.6 +======================================= + +pytest 4.6.6 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Michael Goerz + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.7.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.7.rst new file mode 100644 index 0000000000..f9d01845ec --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.7.rst @@ -0,0 +1,19 @@ +pytest-4.6.7 +======================================= + +pytest 4.6.7 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira +* Daniel Hahler + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.8.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.8.rst new file mode 100644 index 0000000000..5cabe7826e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.8.rst @@ -0,0 +1,20 @@ +pytest-4.6.8 +======================================= + +pytest 4.6.8 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Ryan Mast + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.9.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.9.rst new file mode 100644 index 0000000000..7f7bb5996e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-4.6.9.rst @@ -0,0 +1,21 @@ +pytest-4.6.9 +======================================= + +pytest 4.6.9 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Felix Yan +* Hugo + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.0.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.0.0.rst new file mode 100644 index 0000000000..f5e593e9d8 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.0.0.rst @@ -0,0 +1,46 @@ +pytest-5.0.0 +======================================= + +The pytest team is proud to announce the 5.0.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Dirk Thomas +* Evan Kepner +* Florian Bruhin +* Hugo +* Kevin J. Foley +* Pulkit Goyal +* Ralph Giles +* Ronny Pfannschmidt +* Thomas Grainger +* Thomas Hisch +* Tim Gates +* Victor Maryama +* Yuri Apollov +* Zac Hatfield-Dodds +* curiousjazz77 +* patriksevallius + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.0.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.0.1.rst new file mode 100644 index 0000000000..e16a8f716f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.0.1.rst @@ -0,0 +1,25 @@ +pytest-5.0.1 +======================================= + +pytest 5.0.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* AmirElkess +* Andreu Vallbona Plazas +* Anthony Sottile +* Bruno Oliveira +* Florian Bruhin +* Michael Moore +* Niklas Meinzer +* Thomas Grainger + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.0.rst new file mode 100644 index 0000000000..9ab54ff973 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.0.rst @@ -0,0 +1,56 @@ +pytest-5.1.0 +======================================= + +The pytest team is proud to announce the 5.1.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Albert Tugushev +* Alexey Zankevich +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* David Röthlisberger +* Florian Bruhin +* Ilya Stepin +* Jon Dufresne +* Kaiqi +* Max R +* Miro Hrončok +* Oliver Bestwalter +* Ran Benita +* Ronny Pfannschmidt +* Samuel Searles-Bryant +* Semen Zhydenko +* Steffen Schroeder +* Thomas Grainger +* Tim Hoffmann +* William Woodall +* Wojtek Erbetowski +* Xixi Zhao +* Yash Todi +* boris +* dmitry.dygalo +* helloocc +* martbln +* mei-li + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.1.rst new file mode 100644 index 0000000000..bb8de48014 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.1.rst @@ -0,0 +1,24 @@ +pytest-5.1.1 +======================================= + +pytest 5.1.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Florian Bruhin +* Hugo van Kemenade +* Ran Benita +* Ronny Pfannschmidt + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.2.rst new file mode 100644 index 0000000000..c4cb8e3fb4 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.2.rst @@ -0,0 +1,23 @@ +pytest-5.1.2 +======================================= + +pytest 5.1.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Andrzej Klajnert +* Anthony Sottile +* Bruno Oliveira +* Christian Neumüller +* Robert Holt +* linchiwei123 + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.3.rst new file mode 100644 index 0000000000..c4e88aed28 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.1.3.rst @@ -0,0 +1,23 @@ +pytest-5.1.3 +======================================= + +pytest 5.1.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Christian Neumüller +* Daniel Hahler +* Gene Wood +* Hugo + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.0.rst new file mode 100644 index 0000000000..f43767b750 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.0.rst @@ -0,0 +1,35 @@ +pytest-5.2.0 +======================================= + +The pytest team is proud to announce the 5.2.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Andrzej Klajnert +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* James Cooke +* Michael Goerz +* Ran Benita +* Tomáš Chvátal + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.1.rst new file mode 100644 index 0000000000..fe42b9bf15 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.1.rst @@ -0,0 +1,23 @@ +pytest-5.2.1 +======================================= + +pytest 5.2.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Florian Bruhin +* Hynek Schlawack +* Kevin J. Foley +* tadashigaki + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.2.rst new file mode 100644 index 0000000000..89fd6a534d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.2.rst @@ -0,0 +1,29 @@ +pytest-5.2.2 +======================================= + +pytest 5.2.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Albert Tugushev +* Andrzej Klajnert +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Florian Bruhin +* Nattaphoom Chaipreecha +* Oliver Bestwalter +* Philipp Loose +* Ran Benita +* Victor Maryama +* Yoav Caspi + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.3.rst new file mode 100644 index 0000000000..bab174495d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.3.rst @@ -0,0 +1,28 @@ +pytest-5.2.3 +======================================= + +pytest 5.2.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Brett Cannon +* Bruno Oliveira +* Daniel Hahler +* Daniil Galiev +* David Szotten +* Florian Bruhin +* Patrick Harmon +* Ran Benita +* Zac Hatfield-Dodds +* Zak Hassan + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.4.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.4.rst new file mode 100644 index 0000000000..5f51896797 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.2.4.rst @@ -0,0 +1,22 @@ +pytest-5.2.4 +======================================= + +pytest 5.2.4 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Hugo +* Michael Shields + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.0.rst new file mode 100644 index 0000000000..e13a71f09a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.0.rst @@ -0,0 +1,45 @@ +pytest-5.3.0 +======================================= + +The pytest team is proud to announce the 5.3.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bugs fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from pypi via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* AnjoMan +* Anthony Sottile +* Anton Lodder +* Bruno Oliveira +* Daniel Hahler +* Gregory Lee +* Josh Karpel +* JoshKarpel +* Joshua Storck +* Kale Kundert +* MarcoGorelli +* Michael Krebs +* NNRepos +* Ran Benita +* TH3CHARLie +* Tibor Arpas +* Zac Hatfield-Dodds +* 林玮 + + +Happy testing, +The Pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.1.rst new file mode 100644 index 0000000000..d575bb70e3 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.1.rst @@ -0,0 +1,26 @@ +pytest-5.3.1 +======================================= + +pytest 5.3.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Felix Yan +* Florian Bruhin +* Mark Dickinson +* Nikolay Kondratyev +* Steffen Schroeder +* Zac Hatfield-Dodds + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.2.rst new file mode 100644 index 0000000000..d562a33fb0 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.2.rst @@ -0,0 +1,26 @@ +pytest-5.3.2 +======================================= + +pytest 5.3.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Claudio Madotto +* Daniel Hahler +* Jared Vasquez +* Michael Rose +* Ran Benita +* Ronny Pfannschmidt +* Zac Hatfield-Dodds + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.3.rst new file mode 100644 index 0000000000..40a6fb5b56 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.3.rst @@ -0,0 +1,30 @@ +pytest-5.3.3 +======================================= + +pytest 5.3.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Adam Johnson +* Alexandre Mulatinho +* Anthony Sottile +* Bruno Oliveira +* Chris NeJame +* Daniel Hahler +* Hugo van Kemenade +* Marcelo Duarte Trevisani +* PaulC +* Ran Benita +* Ryan Barner +* Seth Junot +* marc + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.4.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.4.rst new file mode 100644 index 0000000000..0750a9d404 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.4.rst @@ -0,0 +1,20 @@ +pytest-5.3.4 +======================================= + +pytest 5.3.4 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira +* Daniel Hahler +* Ran Benita + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.5.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.5.rst new file mode 100644 index 0000000000..e632ce8538 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.3.5.rst @@ -0,0 +1,19 @@ +pytest-5.3.5 +======================================= + +pytest 5.3.5 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Daniel Hahler +* Ran Benita + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.0.rst new file mode 100644 index 0000000000..43dffc9290 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.0.rst @@ -0,0 +1,59 @@ +pytest-5.4.0 +======================================= + +The pytest team is proud to announce the 5.4.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bug fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from PyPI via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Christoph Buelter +* Christoph Bülter +* Daniel Arndt +* Daniel Hahler +* Holger Kohr +* Hugo +* Hugo van Kemenade +* Jakub Mitoraj +* Kyle Altendorf +* Minuddin Ahmed Rana +* Nathaniel Compton +* ParetoLife +* Pauli Virtanen +* Philipp Loose +* Ran Benita +* Ronny Pfannschmidt +* Stefan Scherfke +* Stefano Mazzucco +* TWood67 +* Tobias Schmidt +* Tomáš Gavenčiak +* Vinay Calastry +* Vladyslav Rachek +* Zac Hatfield-Dodds +* captainCapitalism +* cmachalo +* gftea +* kpinc +* rebecca-palmer +* sdementen + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.1.rst new file mode 100644 index 0000000000..f6a64efa49 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.1.rst @@ -0,0 +1,18 @@ +pytest-5.4.1 +======================================= + +pytest 5.4.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.2.rst new file mode 100644 index 0000000000..d742dd4aad --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.2.rst @@ -0,0 +1,22 @@ +pytest-5.4.2 +======================================= + +pytest 5.4.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Daniel Hahler +* Ran Benita +* Ronny Pfannschmidt + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.3.rst new file mode 100644 index 0000000000..6c995c1633 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-5.4.3.rst @@ -0,0 +1,21 @@ +pytest-5.4.3 +======================================= + +pytest 5.4.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Bruno Oliveira +* Ran Benita +* Tor Colvin + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.0.rst new file mode 100644 index 0000000000..9706fe59bc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.0.rst @@ -0,0 +1,40 @@ +pytest-6.0.0 +======================================= + +The pytest team is proud to announce the 6.0.0 release! + +pytest is a mature Python testing tool with more than 2000 tests +against itself, passing on many different interpreters and platforms. + +This release contains a number of bug fixes and improvements, so users are encouraged +to take a look at the CHANGELOG: + + https://docs.pytest.org/en/latest/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/latest/ + +As usual, you can upgrade from PyPI via: + + pip install -U pytest + +Thanks to all who contributed to this release, among them: + +* Anthony Sottile +* Arvin Firouzi +* Bruno Oliveira +* Debi Mishra +* Garrett Thomas +* Hugo van Kemenade +* Kelton Bassingthwaite +* Kostis Anagnostopoulos +* Lewis Cowles +* Miro Hrončok +* Ran Benita +* Simon K +* Zac Hatfield-Dodds + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.0rc1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.0rc1.rst new file mode 100644 index 0000000000..5690b514ba --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.0rc1.rst @@ -0,0 +1,67 @@ +pytest-6.0.0rc1 +======================================= + +pytest 6.0.0rc1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/latest/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Alfredo Deza +* Andreas Maier +* Andrew +* Anthony Sottile +* ArtyomKaltovich +* Bruno Oliveira +* Claire Cecil +* Curt J. Sampson +* Daniel +* Daniel Hahler +* Danny Sepler +* David Diaz Barquero +* Fabio Zadrozny +* Felix Nieuwenhuizen +* Florian Bruhin +* Florian Dahlitz +* Gleb Nikonorov +* Hugo van Kemenade +* Hunter Richards +* Katarzyna Król +* Katrin Leinweber +* Keri Volans +* Lewis Belcher +* Lukas Geiger +* Martin Michlmayr +* Mattwmaster58 +* Maximilian Cosmo Sitter +* Nikolay Kondratyev +* Pavel Karateev +* Paweł Wilczyński +* Prashant Anand +* Ram Rachum +* Ran Benita +* Ronny Pfannschmidt +* Ruaridh Williamson +* Simon K +* Tim Hoffmann +* Tor Colvin +* Vlad-Radz +* Xinbin Huang +* Zac Hatfield-Dodds +* earonesty +* gaurav dhameeja +* gdhameeja +* ibriquem +* mcsitter +* piotrhm +* smarie +* symonk +* xuiqzy + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.1.rst new file mode 100644 index 0000000000..33fdbed3f6 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.1.rst @@ -0,0 +1,21 @@ +pytest-6.0.1 +======================================= + +pytest 6.0.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/latest/changelog.html. + +Thanks to all who contributed to this release, among them: + +* Bruno Oliveira +* Mattreex +* Ran Benita +* hp310780 + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.2.rst new file mode 100644 index 0000000000..16eabc5863 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.0.2.rst @@ -0,0 +1,19 @@ +pytest-6.0.2 +======================================= + +pytest 6.0.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Bruno Oliveira +* Ran Benita + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.1.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.1.0.rst new file mode 100644 index 0000000000..f4b571ae84 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.1.0.rst @@ -0,0 +1,44 @@ +pytest-6.1.0 +======================================= + +The pytest team is proud to announce the 6.1.0 release! + +This release contains new features, improvements, bug fixes, and breaking changes, so users +are encouraged to take a look at the CHANGELOG carefully: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from PyPI via: + + pip install -U pytest + +Thanks to all of the contributors to this release: + +* Anthony Sottile +* Bruno Oliveira +* C. Titus Brown +* Drew Devereux +* Faris A Chugthai +* Florian Bruhin +* Hugo van Kemenade +* Hynek Schlawack +* Joseph Lucas +* Kamran Ahmad +* Mattreex +* Maximilian Cosmo Sitter +* Ran Benita +* Rüdiger Busche +* Sam Estep +* Sorin Sbarnea +* Thomas Grainger +* Vipul Kumar +* Yutaro Ikeda +* hp310780 + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.1.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.1.1.rst new file mode 100644 index 0000000000..e09408fdee --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.1.1.rst @@ -0,0 +1,18 @@ +pytest-6.1.1 +======================================= + +pytest 6.1.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Ran Benita + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.1.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.1.2.rst new file mode 100644 index 0000000000..aa2c809520 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.1.2.rst @@ -0,0 +1,22 @@ +pytest-6.1.2 +======================================= + +pytest 6.1.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Bruno Oliveira +* Manuel Mariñez +* Ran Benita +* Vasilis Gerakaris +* William Jamir Silva + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.0.rst new file mode 100644 index 0000000000..af16b830dd --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.0.rst @@ -0,0 +1,76 @@ +pytest-6.2.0 +======================================= + +The pytest team is proud to announce the 6.2.0 release! + +This release contains new features, improvements, bug fixes, and breaking changes, so users +are encouraged to take a look at the CHANGELOG carefully: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from PyPI via: + + pip install -U pytest + +Thanks to all of the contributors to this release: + +* Adam Johnson +* Albert Villanova del Moral +* Anthony Sottile +* Anton +* Ariel Pillemer +* Bruno Oliveira +* Charles Aracil +* Christine M +* Christine Mecklenborg +* Cserna Zsolt +* Dominic Mortlock +* Emiel van de Laar +* Florian Bruhin +* Garvit Shubham +* Gustavo Camargo +* Hugo Martins +* Hugo van Kemenade +* Jakob van Santen +* Josias Aurel +* Jürgen Gmach +* Karthikeyan Singaravelan +* Katarzyna +* Kyle Altendorf +* Manuel Mariñez +* Matthew Hughes +* Matthias Gabriel +* Max Voitko +* Maximilian Cosmo Sitter +* Mikhail Fesenko +* Nimesh Vashistha +* Pedro Algarvio +* Petter Strandmark +* Prakhar Gurunani +* Prashant Sharma +* Ran Benita +* Ronny Pfannschmidt +* Sanket Duthade +* Shubham Adep +* Simon K +* Tanvi Mehta +* Thomas Grainger +* Tim Hoffmann +* Vasilis Gerakaris +* William Jamir Silva +* Zac Hatfield-Dodds +* crricks +* dependabot[bot] +* duthades +* frankgerhardt +* kwgchi +* mickeypash +* symonk + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.1.rst new file mode 100644 index 0000000000..f9e7161835 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.1.rst @@ -0,0 +1,20 @@ +pytest-6.2.1 +======================================= + +pytest 6.2.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Bruno Oliveira +* Jakob van Santen +* Ran Benita + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.2.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.2.rst new file mode 100644 index 0000000000..c3999c5386 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.2.rst @@ -0,0 +1,21 @@ +pytest-6.2.2 +======================================= + +pytest 6.2.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Adam Johnson +* Bruno Oliveira +* Chris NeJame +* Ran Benita + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.3.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.3.rst new file mode 100644 index 0000000000..e45aa6a03e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.3.rst @@ -0,0 +1,19 @@ +pytest-6.2.3 +======================================= + +pytest 6.2.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Bruno Oliveira +* Ran Benita + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.4.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.4.rst new file mode 100644 index 0000000000..fa2e3e7813 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.4.rst @@ -0,0 +1,22 @@ +pytest-6.2.4 +======================================= + +pytest 6.2.4 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Anthony Sottile +* Bruno Oliveira +* Christian Maurer +* Florian Bruhin +* Ran Benita + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.5.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.5.rst new file mode 100644 index 0000000000..bc6b4cf422 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-6.2.5.rst @@ -0,0 +1,30 @@ +pytest-6.2.5 +======================================= + +pytest 6.2.5 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Anthony Sottile +* Bruno Oliveira +* Brylie Christopher Oxley +* Daniel Asztalos +* Florian Bruhin +* Jason Haugen +* MapleCCC +* Michał Górny +* Miro Hrončok +* Ran Benita +* Ronny Pfannschmidt +* Sylvain Bellemare +* Thomas Güttler + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-7.0.0.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-7.0.0.rst new file mode 100644 index 0000000000..3ce4335564 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-7.0.0.rst @@ -0,0 +1,74 @@ +pytest-7.0.0 +======================================= + +The pytest team is proud to announce the 7.0.0 release! + +This release contains new features, improvements, bug fixes, and breaking changes, so users +are encouraged to take a look at the CHANGELOG carefully: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from PyPI via: + + pip install -U pytest + +Thanks to all of the contributors to this release: + +* Adam J. Stewart +* Alexander King +* Amin Alaee +* Andrew Neitsch +* Anthony Sottile +* Ben Davies +* Bernát Gábor +* Brian Okken +* Bruno Oliveira +* Cristian Vera +* Dan Alvizu +* David Szotten +* Eddie +* Emmanuel Arias +* Emmanuel Meric de Bellefon +* Eric Liu +* Florian Bruhin +* GergelyKalmar +* Graeme Smecher +* Harshna +* Hugo van Kemenade +* Jakub Kulík +* James Myatt +* Jeff Rasley +* Kale Kundert +* Kian Meng, Ang +* Miro Hrončok +* Naveen-Pratap +* Oleg Höfling +* Olga Matoula +* Ran Benita +* Ronny Pfannschmidt +* Simon K +* Srip +* Sören Wegener +* Taneli Hukkinen +* Terje Runde +* Thomas Grainger +* Thomas Hisch +* William Jamir Silva +* Yuval Shimon +* Zac Hatfield-Dodds +* andrewdotn +* denivyruck +* ericluoliu +* oleg.hoefling +* symonk +* ziebam +* Éloi Rivard +* Éric + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-7.0.0rc1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-7.0.0rc1.rst new file mode 100644 index 0000000000..a5bf0ed3c4 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-7.0.0rc1.rst @@ -0,0 +1,74 @@ +pytest-7.0.0rc1 +======================================= + +The pytest team is proud to announce the 7.0.0rc1 prerelease! + +This is a prerelease, not intended for production use, but to test the upcoming features and improvements +in order to catch any major problems before the final version is released to the major public. + +We appreciate your help testing this out before the final release, making sure to report any +regressions to our issue tracker: + +https://github.com/pytest-dev/pytest/issues + +When doing so, please include the string ``[prerelease]`` in the title. + +You can upgrade from PyPI via: + + pip install pytest==7.0.0rc1 + +Users are encouraged to take a look at the CHANGELOG carefully: + + https://docs.pytest.org/en/7.0.x/changelog.html + +Thanks to all the contributors to this release: + +* Adam J. Stewart +* Alexander King +* Amin Alaee +* Andrew Neitsch +* Anthony Sottile +* Ben Davies +* Bernát Gábor +* Brian Okken +* Bruno Oliveira +* Cristian Vera +* David Szotten +* Eddie +* Emmanuel Arias +* Emmanuel Meric de Bellefon +* Eric Liu +* Florian Bruhin +* GergelyKalmar +* Graeme Smecher +* Harshna +* Hugo van Kemenade +* Jakub Kulík +* James Myatt +* Jeff Rasley +* Kale Kundert +* Miro Hrončok +* Naveen-Pratap +* Oleg Höfling +* Ran Benita +* Ronny Pfannschmidt +* Simon K +* Srip +* Sören Wegener +* Taneli Hukkinen +* Terje Runde +* Thomas Grainger +* Thomas Hisch +* William Jamir Silva +* Zac Hatfield-Dodds +* andrewdotn +* denivyruck +* ericluoliu +* oleg.hoefling +* symonk +* ziebam +* Éloi Rivard + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-7.0.1.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-7.0.1.rst new file mode 100644 index 0000000000..5accfbad0d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/release-7.0.1.rst @@ -0,0 +1,20 @@ +pytest-7.0.1 +======================================= + +pytest 7.0.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Anthony Sottile +* Bruno Oliveira +* Ran Benita + + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/sprint2016.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/sprint2016.rst new file mode 100644 index 0000000000..8e70658987 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/announce/sprint2016.rst @@ -0,0 +1,64 @@ +python testing sprint June 20th-26th 2016 +====================================================== + +.. image:: ../img/freiburg2.jpg + :width: 400 + +The pytest core group held the biggest sprint +in its history in June 2016, taking place in the black forest town Freiburg +in Germany. In February 2016 we started a `funding +campaign on Indiegogo to cover expenses +`_ The page also mentions +some preliminary topics: + +- improving pytest-xdist test scheduling to take into account + fixture setups and explicit user hints. + +- provide info on fixture dependencies during --collect-only + +- tying pytest-xdist to tox so that you can do "py.test -e py34" + to run tests in a particular tox-managed virtualenv. Also + look into making pytest-xdist use tox environments on + remote ssh-sides so that remote dependency management becomes + easier. + +- refactoring the fixture system so more people understand it :) + +- integrating PyUnit setup methods as autouse fixtures. + possibly adding ways to influence ordering of same-scoped + fixtures (so you can make a choice of which fixtures come + before others) + +- fixing bugs and issues from the tracker, really an endless source :) + + +Participants +-------------- + +Over 20 participants took part from 4 continents, including employees +from Splunk, Personalkollen, Cobe.io, FanDuel and Dolby. Some newcomers +mixed with developers who have worked on pytest since its beginning, and +of course everyone in between. + + +Sprint organisation, schedule +------------------------------- + +People arrived in Freiburg on the 19th, with sprint development taking +place on 20th, 21st, 22nd, 24th and 25th. On the 23rd we took a break +day for some hot hiking in the Black Forest. + +Sprint activity was organised heavily around pairing, with plenty of group +discusssions to take advantage of the high bandwidth, and lightning talks +as well. + + +Money / funding +--------------- + + +The Indiegogo campaign aimed for 11000 USD and in the end raised over +12000, to reimburse travel costs, pay for a sprint venue and catering. + +Excess money is reserved for further sprint/travel funding for pytest/tox +contributors. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/backwards-compatibility.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/backwards-compatibility.rst new file mode 100644 index 0000000000..3a0ff12616 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/backwards-compatibility.rst @@ -0,0 +1,79 @@ +.. _backwards-compatibility: + +Backwards Compatibility Policy +============================== + +.. versionadded: 6.0 + +pytest is actively evolving and is a project that has been decades in the making, +we keep learning about new and better structures to express different details about testing. + +While we implement those modifications we try to ensure an easy transition and don't want to impose unnecessary churn on our users and community/plugin authors. + +As of now, pytest considers multiple types of backward compatibility transitions: + +a) trivial: APIs which trivially translate to the new mechanism, + and do not cause problematic changes. + + We try to support those indefinitely while encouraging users to switch to newer/better mechanisms through documentation. + +b) transitional: the old and new API don't conflict + and we can help users transition by using warnings, while supporting both for a prolonged time. + + We will only start the removal of deprecated functionality in major releases (e.g. if we deprecate something in 3.0 we will start to remove it in 4.0), and keep it around for at least two minor releases (e.g. if we deprecate something in 3.9 and 4.0 is the next release, we start to remove it in 5.0, not in 4.0). + + A deprecated feature scheduled to be removed in major version X will use the warning class `PytestRemovedInXWarning` (a subclass of :class:`~pytest.PytestDeprecationwarning`). + + When the deprecation expires (e.g. 4.0 is released), we won't remove the deprecated functionality immediately, but will use the standard warning filters to turn `PytestRemovedInXWarning` (e.g. `PytestRemovedIn4Warning`) into **errors** by default. This approach makes it explicit that removal is imminent, and still gives you time to turn the deprecated feature into a warning instead of an error so it can be dealt with in your own time. In the next minor release (e.g. 4.1), the feature will be effectively removed. + + +c) true breakage: should only be considered when normal transition is unreasonably unsustainable and would offset important development/features by years. + In addition, they should be limited to APIs where the number of actual users is very small (for example only impacting some plugins), and can be coordinated with the community in advance. + + Examples for such upcoming changes: + + * removal of ``pytest_runtest_protocol/nextitem`` - :issue:`895` + * rearranging of the node tree to include ``FunctionDefinition`` + * rearranging of ``SetupState`` :issue:`895` + + True breakages must be announced first in an issue containing: + + * Detailed description of the change + * Rationale + * Expected impact on users and plugin authors (example in :issue:`895`) + + After there's no hard *-1* on the issue it should be followed up by an initial proof-of-concept Pull Request. + + This POC serves as both a coordination point to assess impact and potential inspiration to come up with a transitional solution after all. + + After a reasonable amount of time the PR can be merged to base a new major release. + + For the PR to mature from POC to acceptance, it must contain: + * Setup of deprecation errors/warnings that help users fix and port their code. If it is possible to introduce a deprecation period under the current series, before the true breakage, it should be introduced in a separate PR and be part of the current release stream. + * Detailed description of the rationale and examples on how to port code in ``doc/en/deprecations.rst``. + + +History +========= + + +Focus primary on smooth transition - stance (pre 6.0) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Keeping backwards compatibility has a very high priority in the pytest project. Although we have deprecated functionality over the years, most of it is still supported. All deprecations in pytest were done because simpler or more efficient ways of accomplishing the same tasks have emerged, making the old way of doing things unnecessary. + +With the pytest 3.0 release we introduced a clear communication scheme for when we will actually remove the old busted joint and politely ask you to use the new hotness instead, while giving you enough time to adjust your tests or raise concerns if there are valid reasons to keep deprecated functionality around. + +To communicate changes we issue deprecation warnings using a custom warning hierarchy (see :ref:`internal-warnings`). These warnings may be suppressed using the standard means: ``-W`` command-line flag or ``filterwarnings`` ini options (see :ref:`warnings`), but we suggest to use these sparingly and temporarily, and heed the warnings when possible. + +We will only start the removal of deprecated functionality in major releases (e.g. if we deprecate something in 3.0 we will start to remove it in 4.0), and keep it around for at least two minor releases (e.g. if we deprecate something in 3.9 and 4.0 is the next release, we start to remove it in 5.0, not in 4.0). + +When the deprecation expires (e.g. 4.0 is released), we won't remove the deprecated functionality immediately, but will use the standard warning filters to turn them into **errors** by default. This approach makes it explicit that removal is imminent, and still gives you time to turn the deprecated feature into a warning instead of an error so it can be dealt with in your own time. In the next minor release (e.g. 4.1), the feature will be effectively removed. + + +Deprecation Roadmap +------------------- + +Features currently deprecated and removed in previous releases can be found in :ref:`deprecations`. + +We track future deprecation and removal of features using milestones and the `deprecation `_ and `removal `_ labels on GitHub. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/builtin.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/builtin.rst new file mode 100644 index 0000000000..c7e7863b21 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/builtin.rst @@ -0,0 +1,197 @@ +:orphan: + +.. _`pytest helpers`: + +Pytest API and builtin fixtures +================================================ + + +Most of the information of this page has been moved over to :ref:`api-reference`. + +For information on plugin hooks and objects, see :ref:`plugins`. + +For information on the ``pytest.mark`` mechanism, see :ref:`mark`. + +For information about fixtures, see :ref:`fixtures`. To see a complete list of available fixtures (add ``-v`` to also see fixtures with leading ``_``), type : + +.. code-block:: pytest + + $ pytest --fixtures -v + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collected 0 items + cache -- .../_pytest/cacheprovider.py:510 + Return a cache object that can persist state between testing sessions. + + cache.get(key, default) + cache.set(key, value) + + Keys must be ``/`` separated strings, where the first part is usually the + name of your plugin or application to avoid clashes with other cache users. + + Values can be any object handled by the json stdlib module. + + capsys -- .../_pytest/capture.py:878 + Enable text capturing of writes to ``sys.stdout`` and ``sys.stderr``. + + The captured output is made available via ``capsys.readouterr()`` method + calls, which return a ``(out, err)`` namedtuple. + ``out`` and ``err`` will be ``text`` objects. + + capsysbinary -- .../_pytest/capture.py:895 + Enable bytes capturing of writes to ``sys.stdout`` and ``sys.stderr``. + + The captured output is made available via ``capsysbinary.readouterr()`` + method calls, which return a ``(out, err)`` namedtuple. + ``out`` and ``err`` will be ``bytes`` objects. + + capfd -- .../_pytest/capture.py:912 + Enable text capturing of writes to file descriptors ``1`` and ``2``. + + The captured output is made available via ``capfd.readouterr()`` method + calls, which return a ``(out, err)`` namedtuple. + ``out`` and ``err`` will be ``text`` objects. + + capfdbinary -- .../_pytest/capture.py:929 + Enable bytes capturing of writes to file descriptors ``1`` and ``2``. + + The captured output is made available via ``capfd.readouterr()`` method + calls, which return a ``(out, err)`` namedtuple. + ``out`` and ``err`` will be ``byte`` objects. + + doctest_namespace [session scope] -- .../_pytest/doctest.py:731 + Fixture that returns a :py:class:`dict` that will be injected into the + namespace of doctests. + + pytestconfig [session scope] -- .../_pytest/fixtures.py:1365 + Session-scoped fixture that returns the session's :class:`pytest.Config` + object. + + Example:: + + def test_foo(pytestconfig): + if pytestconfig.getoption("verbose") > 0: + ... + + record_property -- .../_pytest/junitxml.py:282 + Add extra properties to the calling test. + + User properties become part of the test report and are available to the + configured reporters, like JUnit XML. + + The fixture is callable with ``name, value``. The value is automatically + XML-encoded. + + Example:: + + def test_function(record_property): + record_property("example_key", 1) + + record_xml_attribute -- .../_pytest/junitxml.py:305 + Add extra xml attributes to the tag for the calling test. + + The fixture is callable with ``name, value``. The value is + automatically XML-encoded. + + record_testsuite_property [session scope] -- .../_pytest/junitxml.py:343 + Record a new ```` tag as child of the root ````. + + This is suitable to writing global information regarding the entire test + suite, and is compatible with ``xunit2`` JUnit family. + + This is a ``session``-scoped fixture which is called with ``(name, value)``. Example: + + .. code-block:: python + + def test_foo(record_testsuite_property): + record_testsuite_property("ARCH", "PPC") + record_testsuite_property("STORAGE_TYPE", "CEPH") + + ``name`` must be a string, ``value`` will be converted to a string and properly xml-escaped. + + .. warning:: + + Currently this fixture **does not work** with the + `pytest-xdist `__ plugin. See + :issue:`7767` for details. + + tmpdir_factory [session scope] -- .../_pytest/legacypath.py:295 + Return a :class:`pytest.TempdirFactory` instance for the test session. + + tmpdir -- .../_pytest/legacypath.py:302 + Return a temporary directory path object which is unique to each test + function invocation, created as a sub directory of the base temporary + directory. + + By default, a new base temporary directory is created each test session, + and old bases are removed after 3 sessions, to aid in debugging. If + ``--basetemp`` is used then it is cleared each session. See :ref:`base + temporary directory`. + + The returned object is a `legacy_path`_ object. + + .. _legacy_path: https://py.readthedocs.io/en/latest/path.html + + caplog -- .../_pytest/logging.py:483 + Access and control log capturing. + + Captured logs are available through the following properties/methods:: + + * caplog.messages -> list of format-interpolated log messages + * caplog.text -> string containing formatted log output + * caplog.records -> list of logging.LogRecord instances + * caplog.record_tuples -> list of (logger_name, level, message) tuples + * caplog.clear() -> clear captured records and formatted log output string + + monkeypatch -- .../_pytest/monkeypatch.py:29 + A convenient fixture for monkey-patching. + + The fixture provides these methods to modify objects, dictionaries or + os.environ:: + + monkeypatch.setattr(obj, name, value, raising=True) + monkeypatch.delattr(obj, name, raising=True) + monkeypatch.setitem(mapping, name, value) + monkeypatch.delitem(obj, name, raising=True) + monkeypatch.setenv(name, value, prepend=None) + monkeypatch.delenv(name, raising=True) + monkeypatch.syspath_prepend(path) + monkeypatch.chdir(path) + + All modifications will be undone after the requesting test function or + fixture has finished. The ``raising`` parameter determines if a KeyError + or AttributeError will be raised if the set/deletion operation has no target. + + recwarn -- .../_pytest/recwarn.py:29 + Return a :class:`WarningsRecorder` instance that records all warnings emitted by test functions. + + See https://docs.python.org/library/how-to/capture-warnings.html for information + on warning categories. + + tmp_path_factory [session scope] -- .../_pytest/tmpdir.py:183 + Return a :class:`pytest.TempPathFactory` instance for the test session. + + tmp_path -- .../_pytest/tmpdir.py:198 + Return a temporary directory path object which is unique to each test + function invocation, created as a sub directory of the base temporary + directory. + + By default, a new base temporary directory is created each test session, + and old bases are removed after 3 sessions, to aid in debugging. If + ``--basetemp`` is used then it is cleared each session. See :ref:`base + temporary directory`. + + The returned object is a :class:`pathlib.Path` object. + + + ========================== no tests ran in 0.12s =========================== + +You can also interactively ask for help, e.g. by typing on the Python interactive prompt something like: + +.. code-block:: python + + import pytest + + help(pytest) diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/changelog.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/changelog.rst new file mode 100644 index 0000000000..1acdad366d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/changelog.rst @@ -0,0 +1,9044 @@ +.. _`changelog`: + +========= +Changelog +========= + +Versions follow `Semantic Versioning `_ (``..``). + +Backward incompatible (breaking) changes will only be introduced in major versions +with advance notice in the **Deprecations** section of releases. + + +.. + You should *NOT* be adding new change log entries to this file, this + file is managed by towncrier. You *may* edit previous change logs to + fix problems like typo corrections or such. + To add a new change log entry, please see + https://pip.pypa.io/en/latest/development/contributing/#news-entries + we named the news folder changelog + + +.. only:: changelog_towncrier_draft + + .. The 'changelog_towncrier_draft' tag is included by our 'tox -e docs', + but not on readthedocs. + + .. include:: _changelog_towncrier_draft.rst + +.. towncrier release notes start + +pytest 7.0.1 (2022-02-11) +========================= + +Bug Fixes +--------- + +- `#9608 `_: Fix invalid importing of ``importlib.readers`` in Python 3.9. + + +- `#9610 `_: Restore `UnitTestFunction.obj` to return unbound rather than bound method. + Fixes a crash during a failed teardown in unittest TestCases with non-default `__init__`. + Regressed in pytest 7.0.0. + + +- `#9636 `_: The ``pythonpath`` plugin was renamed to ``python_path``. This avoids a conflict with the ``pytest-pythonpath`` plugin. + + +- `#9642 `_: Fix running tests by id with ``::`` in the parametrize portion. + + +- `#9643 `_: Delay issuing a :class:`~pytest.PytestWarning` about diamond inheritance involving :class:`~pytest.Item` and + :class:`~pytest.Collector` so it can be filtered using :ref:`standard warning filters `. + + +pytest 7.0.0 (2022-02-03) +========================= + +(**Please see the full set of changes for this release also in the 7.0.0rc1 notes below**) + +Deprecations +------------ + +- `#9488 `_: If custom subclasses of nodes like :class:`pytest.Item` override the + ``__init__`` method, they should take ``**kwargs``. See + :ref:`uncooperative-constructors-deprecated` for details. + + Note that a deprection warning is only emitted when there is a conflict in the + arguments pytest expected to pass. This deprecation was already part of pytest + 7.0.0rc1 but wasn't documented. + + + +Bug Fixes +--------- + +- `#9355 `_: Fixed error message prints function decorators when using assert in Python 3.8 and above. + + +- `#9396 `_: Ensure :attr:`pytest.Config.inifile` is available during the :func:`pytest_cmdline_main <_pytest.hookspec.pytest_cmdline_main>` hook (regression during ``7.0.0rc1``). + + + +Improved Documentation +---------------------- + +- `#9404 `_: Added extra documentation on alternatives to common misuses of `pytest.warns(None)` ahead of its deprecation. + + +- `#9505 `_: Clarify where the configuration files are located. To avoid confusions documentation mentions + that configuration file is located in the root of the repository. + + + +Trivial/Internal Changes +------------------------ + +- `#9521 `_: Add test coverage to assertion rewrite path. + + +pytest 7.0.0rc1 (2021-12-06) +============================ + +Breaking Changes +---------------- + +- `#7259 `_: The :ref:`Node.reportinfo() ` function first return value type has been expanded from `py.path.local | str` to `os.PathLike[str] | str`. + + Most plugins which refer to `reportinfo()` only define it as part of a custom :class:`pytest.Item` implementation. + Since `py.path.local` is a `os.PathLike[str]`, these plugins are unaffacted. + + Plugins and users which call `reportinfo()`, use the first return value and interact with it as a `py.path.local`, would need to adjust by calling `py.path.local(fspath)`. + Although preferably, avoid the legacy `py.path.local` and use `pathlib.Path`, or use `item.location` or `item.path`, instead. + + Note: pytest was not able to provide a deprecation period for this change. + + +- `#8246 `_: ``--version`` now writes version information to ``stdout`` rather than ``stderr``. + + +- `#8733 `_: Drop a workaround for `pyreadline `__ that made it work with ``--pdb``. + + The workaround was introduced in `#1281 `__ in 2015, however since then + `pyreadline seems to have gone unmaintained `__, is `generating + warnings `__, and will stop working on Python 3.10. + + +- `#9061 `_: Using :func:`pytest.approx` in a boolean context now raises an error hinting at the proper usage. + + It is apparently common for users to mistakenly use ``pytest.approx`` like this: + + .. code-block:: python + + assert pytest.approx(actual, expected) + + While the correct usage is: + + .. code-block:: python + + assert actual == pytest.approx(expected) + + The new error message helps catch those mistakes. + + +- `#9277 `_: The ``pytest.Instance`` collector type has been removed. + Importing ``pytest.Instance`` or ``_pytest.python.Instance`` returns a dummy type and emits a deprecation warning. + See :ref:`instance-collector-deprecation` for details. + + +- `#9308 `_: **PytestRemovedIn7Warning deprecation warnings are now errors by default.** + + Following our plan to remove deprecated features with as little disruption as + possible, all warnings of type ``PytestRemovedIn7Warning`` now generate errors + instead of warning messages by default. + + **The affected features will be effectively removed in pytest 7.1**, so please consult the + :ref:`deprecations` section in the docs for directions on how to update existing code. + + In the pytest ``7.0.X`` series, it is possible to change the errors back into warnings as a + stopgap measure by adding this to your ``pytest.ini`` file: + + .. code-block:: ini + + [pytest] + filterwarnings = + ignore::pytest.PytestRemovedIn7Warning + + But this will stop working when pytest ``7.1`` is released. + + **If you have concerns** about the removal of a specific feature, please add a + comment to :issue:`9308`. + + + +Deprecations +------------ + +- `#7259 `_: ``py.path.local`` arguments for hooks have been deprecated. See :ref:`the deprecation note ` for full details. + + ``py.path.local`` arguments to Node constructors have been deprecated. See :ref:`the deprecation note ` for full details. + + .. note:: + The name of the :class:`~_pytest.nodes.Node` arguments and attributes (the + new attribute being ``path``) is **the opposite** of the situation for hooks + (the old argument being ``path``). + + This is an unfortunate artifact due to historical reasons, which should be + resolved in future versions as we slowly get rid of the :pypi:`py` + dependency (see :issue:`9283` for a longer discussion). + + +- `#7469 `_: Directly constructing the following classes is now deprecated: + + - ``_pytest.mark.structures.Mark`` + - ``_pytest.mark.structures.MarkDecorator`` + - ``_pytest.mark.structures.MarkGenerator`` + - ``_pytest.python.Metafunc`` + - ``_pytest.runner.CallInfo`` + - ``_pytest._code.ExceptionInfo`` + - ``_pytest.config.argparsing.Parser`` + - ``_pytest.config.argparsing.OptionGroup`` + - ``_pytest.pytester.HookRecorder`` + + These constructors have always been considered private, but now issue a deprecation warning, which may become a hard error in pytest 8. + + +- `#8242 `_: Raising :class:`unittest.SkipTest` to skip collection of tests during the + pytest collection phase is deprecated. Use :func:`pytest.skip` instead. + + Note: This deprecation only relates to using :class:`unittest.SkipTest` during test + collection. You are probably not doing that. Ordinary usage of + :class:`unittest.SkipTest` / :meth:`unittest.TestCase.skipTest` / + :func:`unittest.skip` in unittest test cases is fully supported. + + +- `#8315 `_: Several behaviors of :meth:`Parser.addoption ` are now + scheduled for removal in pytest 8 (deprecated since pytest 2.4.0): + + - ``parser.addoption(..., help=".. %default ..")`` - use ``%(default)s`` instead. + - ``parser.addoption(..., type="int/string/float/complex")`` - use ``type=int`` etc. instead. + + +- `#8447 `_: Defining a custom pytest node type which is both an :class:`pytest.Item ` and a :class:`pytest.Collector ` (e.g. :class:`pytest.File `) now issues a warning. + It was never sanely supported and triggers hard to debug errors. + + See :ref:`the deprecation note ` for full details. + + +- `#8592 `_: :hook:`pytest_cmdline_preparse` has been officially deprecated. It will be removed in a future release. Use :hook:`pytest_load_initial_conftests` instead. + + See :ref:`the deprecation note ` for full details. + + +- `#8645 `_: :func:`pytest.warns(None) ` is now deprecated because many people used + it to mean "this code does not emit warnings", but it actually had the effect of + checking that the code emits at least one warning of any type - like ``pytest.warns()`` + or ``pytest.warns(Warning)``. + + +- `#8948 `_: :func:`pytest.skip(msg=...) `, :func:`pytest.fail(msg=...) ` and :func:`pytest.exit(msg=...) ` + signatures now accept a ``reason`` argument instead of ``msg``. Using ``msg`` still works, but is deprecated and will be removed in a future release. + + This was changed for consistency with :func:`pytest.mark.skip ` and :func:`pytest.mark.xfail ` which both accept + ``reason`` as an argument. + +- `#8174 `_: The following changes have been made to types reachable through :attr:`pytest.ExceptionInfo.traceback`: + + - The ``path`` property of ``_pytest.code.Code`` returns ``Path`` instead of ``py.path.local``. + - The ``path`` property of ``_pytest.code.TracebackEntry`` returns ``Path`` instead of ``py.path.local``. + + There was no deprecation period for this change (sorry!). + + +Features +-------- + +- `#5196 `_: Tests are now ordered by definition order in more cases. + + In a class hierarchy, tests from base classes are now consistently ordered before tests defined on their subclasses (reverse MRO order). + + +- `#7132 `_: Added two environment variables :envvar:`PYTEST_THEME` and :envvar:`PYTEST_THEME_MODE` to let the users customize the pygments theme used. + + +- `#7259 `_: Added :meth:`cache.mkdir() `, which is similar to the existing :meth:`cache.makedir() `, + but returns a :class:`pathlib.Path` instead of a legacy ``py.path.local``. + + Added a ``paths`` type to :meth:`parser.addini() `, + as in ``parser.addini("mypaths", "my paths", type="paths")``, + which is similar to the existing ``pathlist``, + but returns a list of :class:`pathlib.Path` instead of legacy ``py.path.local``. + + +- `#7469 `_: The types of objects used in pytest's API are now exported so they may be used in type annotations. + + The newly-exported types are: + + - ``pytest.Config`` for :class:`Config `. + - ``pytest.Mark`` for :class:`marks `. + - ``pytest.MarkDecorator`` for :class:`mark decorators `. + - ``pytest.MarkGenerator`` for the :class:`pytest.mark ` singleton. + - ``pytest.Metafunc`` for the :class:`metafunc ` argument to the :hook:`pytest_generate_tests` hook. + - ``pytest.CallInfo`` for the :class:`CallInfo ` type passed to various hooks. + - ``pytest.PytestPluginManager`` for :class:`PytestPluginManager `. + - ``pytest.ExceptionInfo`` for the :class:`ExceptionInfo ` type returned from :func:`pytest.raises` and passed to various hooks. + - ``pytest.Parser`` for the :class:`Parser ` type passed to the :hook:`pytest_addoption` hook. + - ``pytest.OptionGroup`` for the :class:`OptionGroup ` type returned from the :func:`parser.addgroup ` method. + - ``pytest.HookRecorder`` for the :class:`HookRecorder ` type returned from :class:`~pytest.Pytester`. + - ``pytest.RecordedHookCall`` for the :class:`RecordedHookCall ` type returned from :class:`~pytest.HookRecorder`. + - ``pytest.RunResult`` for the :class:`RunResult ` type returned from :class:`~pytest.Pytester`. + - ``pytest.LineMatcher`` for the :class:`LineMatcher ` type used in :class:`~pytest.RunResult` and others. + - ``pytest.TestReport`` for the :class:`TestReport ` type used in various hooks. + - ``pytest.CollectReport`` for the :class:`CollectReport ` type used in various hooks. + + Constructing most of them directly is not supported; they are only meant for use in type annotations. + Doing so will emit a deprecation warning, and may become a hard-error in pytest 8.0. + + Subclassing them is also not supported. This is not currently enforced at runtime, but is detected by type-checkers such as mypy. + + +- `#7856 `_: :ref:`--import-mode=importlib ` now works with features that + depend on modules being on :py:data:`sys.modules`, such as :mod:`pickle` and :mod:`dataclasses`. + + +- `#8144 `_: The following hooks now receive an additional ``pathlib.Path`` argument, equivalent to an existing ``py.path.local`` argument: + + - :hook:`pytest_ignore_collect` - The ``collection_path`` parameter (equivalent to existing ``path`` parameter). + - :hook:`pytest_collect_file` - The ``file_path`` parameter (equivalent to existing ``path`` parameter). + - :hook:`pytest_pycollect_makemodule` - The ``module_path`` parameter (equivalent to existing ``path`` parameter). + - :hook:`pytest_report_header` - The ``start_path`` parameter (equivalent to existing ``startdir`` parameter). + - :hook:`pytest_report_collectionfinish` - The ``start_path`` parameter (equivalent to existing ``startdir`` parameter). + + .. note:: + The name of the :class:`~_pytest.nodes.Node` arguments and attributes (the + new attribute being ``path``) is **the opposite** of the situation for hooks + (the old argument being ``path``). + + This is an unfortunate artifact due to historical reasons, which should be + resolved in future versions as we slowly get rid of the :pypi:`py` + dependency (see :issue:`9283` for a longer discussion). + + +- `#8251 `_: Implement ``Node.path`` as a ``pathlib.Path``. Both the old ``fspath`` and this new attribute gets set no matter whether ``path`` or ``fspath`` (deprecated) is passed to the constructor. It is a replacement for the ``fspath`` attribute (which represents the same path as ``py.path.local``). While ``fspath`` is not deprecated yet + due to the ongoing migration of methods like :meth:`~_pytest.Item.reportinfo`, we expect to deprecate it in a future release. + + .. note:: + The name of the :class:`~_pytest.nodes.Node` arguments and attributes (the + new attribute being ``path``) is **the opposite** of the situation for hooks + (the old argument being ``path``). + + This is an unfortunate artifact due to historical reasons, which should be + resolved in future versions as we slowly get rid of the :pypi:`py` + dependency (see :issue:`9283` for a longer discussion). + + +- `#8421 `_: :func:`pytest.approx` now works on :class:`~decimal.Decimal` within mappings/dicts and sequences/lists. + + +- `#8606 `_: pytest invocations with ``--fixtures-per-test`` and ``--fixtures`` have been enriched with: + + - Fixture location path printed with the fixture name. + - First section of the fixture's docstring printed under the fixture name. + - Whole of fixture's docstring printed under the fixture name using ``--verbose`` option. + + +- `#8761 `_: New :ref:`version-tuple` attribute, which makes it simpler for users to do something depending on the pytest version (such as declaring hooks which are introduced in later versions). + + +- `#8789 `_: Switch TOML parser from ``toml`` to ``tomli`` for TOML v1.0.0 support in ``pyproject.toml``. + + +- `#8920 `_: Added :class:`pytest.Stash`, a facility for plugins to store their data on :class:`~pytest.Config` and :class:`~_pytest.nodes.Node`\s in a type-safe and conflict-free manner. + See :ref:`plugin-stash` for details. + + +- `#8953 `_: :class:`RunResult <_pytest.pytester.RunResult>` method :meth:`assert_outcomes <_pytest.pytester.RunResult.assert_outcomes>` now accepts a + ``warnings`` argument to assert the total number of warnings captured. + + +- `#8954 `_: ``--debug`` flag now accepts a :class:`str` file to route debug logs into, remains defaulted to `pytestdebug.log`. + + +- `#9023 `_: Full diffs are now always shown for equality assertions of iterables when + `CI` or ``BUILD_NUMBER`` is found in the environment, even when ``-v`` isn't + used. + + +- `#9113 `_: :class:`RunResult <_pytest.pytester.RunResult>` method :meth:`assert_outcomes <_pytest.pytester.RunResult.assert_outcomes>` now accepts a + ``deselected`` argument to assert the total number of deselected tests. + + +- `#9114 `_: Added :confval:`pythonpath` setting that adds listed paths to :data:`sys.path` for the duration of the test session. If you currently use the pytest-pythonpath or pytest-srcpaths plugins, you should be able to replace them with built-in `pythonpath` setting. + + + +Improvements +------------ + +- `#7480 `_: A deprecation scheduled to be removed in a major version X (e.g. pytest 7, 8, 9, ...) now uses warning category `PytestRemovedInXWarning`, + a subclass of :class:`~pytest.PytestDeprecationWarning`, + instead of :class:`PytestDeprecationWarning` directly. + + See :ref:`backwards-compatibility` for more details. + + +- `#7864 `_: Improved error messages when parsing warning filters. + + Previously pytest would show an internal traceback, which besides being ugly sometimes would hide the cause + of the problem (for example an ``ImportError`` while importing a specific warning type). + + +- `#8335 `_: Improved :func:`pytest.approx` assertion messages for sequences of numbers. + + The assertion messages now dumps a table with the index and the error of each diff. + Example:: + + > assert [1, 2, 3, 4] == pytest.approx([1, 3, 3, 5]) + E assert comparison failed for 2 values: + E Index | Obtained | Expected + E 1 | 2 | 3 +- 3.0e-06 + E 3 | 4 | 5 +- 5.0e-06 + + +- `#8403 `_: By default, pytest will truncate long strings in assert errors so they don't clutter the output too much, + currently at ``240`` characters by default. + + However, in some cases the longer output helps, or is even crucial, to diagnose a failure. Using ``-v`` will + now increase the truncation threshold to ``2400`` characters, and ``-vv`` or higher will disable truncation entirely. + + +- `#8509 `_: Fixed issue where :meth:`unittest.TestCase.setUpClass` is not called when a test has `/` in its name since pytest 6.2.0. + + This refers to the path part in pytest node IDs, e.g. ``TestClass::test_it`` in the node ID ``tests/test_file.py::TestClass::test_it``. + + Now, instead of assuming that the test name does not contain ``/``, it is assumed that test path does not contain ``::``. We plan to hopefully make both of these work in the future. + + +- `#8803 `_: It is now possible to add colors to custom log levels on cli log. + + By using :func:`add_color_level <_pytest.logging.add_color_level>` from a ``pytest_configure`` hook, colors can be added:: + + logging_plugin = config.pluginmanager.get_plugin('logging-plugin') + logging_plugin.log_cli_handler.formatter.add_color_level(logging.INFO, 'cyan') + logging_plugin.log_cli_handler.formatter.add_color_level(logging.SPAM, 'blue') + + See :ref:`log_colors` for more information. + + +- `#8822 `_: When showing fixture paths in `--fixtures` or `--fixtures-by-test`, fixtures coming from pytest itself now display an elided path, rather than the full path to the file in the `site-packages` directory. + + +- `#8898 `_: Complex numbers are now treated like floats and integers when generating parameterization IDs. + + +- `#9062 `_: ``--stepwise-skip`` now implicitly enables ``--stepwise`` and can be used on its own. + + +- `#9205 `_: :meth:`pytest.Cache.set` now preserves key order when saving dicts. + + + +Bug Fixes +--------- + +- `#7124 `_: Fixed an issue where ``__main__.py`` would raise an ``ImportError`` when ``--doctest-modules`` was provided. + + +- `#8061 `_: Fixed failing ``staticmethod`` test cases if they are inherited from a parent test class. + + +- `#8192 `_: ``testdir.makefile`` now silently accepts values which don't start with ``.`` to maintain backward compatibility with older pytest versions. + + ``pytester.makefile`` now issues a clearer error if the ``.`` is missing in the ``ext`` argument. + + +- `#8258 `_: Fixed issue where pytest's ``faulthandler`` support would not dump traceback on crashes + if the :mod:`faulthandler` module was already enabled during pytest startup (using + ``python -X dev -m pytest`` for example). + + +- `#8317 `_: Fixed an issue where illegal directory characters derived from ``getpass.getuser()`` raised an ``OSError``. + + +- `#8367 `_: Fix ``Class.from_parent`` so it forwards extra keyword arguments to the constructor. + + +- `#8377 `_: The test selection options ``pytest -k`` and ``pytest -m`` now support matching + names containing forward slash (``/``) characters. + + +- `#8384 `_: The ``@pytest.mark.skip`` decorator now correctly handles its arguments. When the ``reason`` argument is accidentally given both positional and as a keyword (e.g. because it was confused with ``skipif``), a ``TypeError`` now occurs. Before, such tests were silently skipped, and the positional argument ignored. Additionally, ``reason`` is now documented correctly as positional or keyword (rather than keyword-only). + + +- `#8394 `_: Use private names for internal fixtures that handle classic setup/teardown so that they don't show up with the default ``--fixtures`` invocation (but they still show up with ``--fixtures -v``). + + +- `#8456 `_: The :confval:`required_plugins` config option now works correctly when pre-releases of plugins are installed, rather than falsely claiming that those plugins aren't installed at all. + + +- `#8464 `_: ``-c `` now also properly defines ``rootdir`` as the directory that contains ````. + + +- `#8503 `_: :meth:`pytest.MonkeyPatch.syspath_prepend` no longer fails when + ``setuptools`` is not installed. + It now only calls :func:`pkg_resources.fixup_namespace_packages` if + ``pkg_resources`` was previously imported, because it is not needed otherwise. + + +- `#8548 `_: Introduce fix to handle precision width in ``log-cli-format`` in turn to fix output coloring for certain formats. + + +- `#8796 `_: Fixed internal error when skipping doctests. + + +- `#8983 `_: The test selection options ``pytest -k`` and ``pytest -m`` now support matching names containing backslash (`\\`) characters. + Backslashes are treated literally, not as escape characters (the values being matched against are already escaped). + + +- `#8990 `_: Fix `pytest -vv` crashing with an internal exception `AttributeError: 'str' object has no attribute 'relative_to'` in some cases. + + +- `#9077 `_: Fixed confusing error message when ``request.fspath`` / ``request.path`` was accessed from a session-scoped fixture. + + +- `#9131 `_: Fixed the URL used by ``--pastebin`` to use `bpa.st `__. + + +- `#9163 `_: The end line number and end column offset are now properly set for rewritten assert statements. + + +- `#9169 `_: Support for the ``files`` API from ``importlib.resources`` within rewritten files. + + +- `#9272 `_: The nose compatibility module-level fixtures `setup()` and `teardown()` are now only called once per module, instead of for each test function. + They are now called even if object-level `setup`/`teardown` is defined. + + + +Improved Documentation +---------------------- + +- `#4320 `_: Improved docs for `pytester.copy_example`. + + +- `#5105 `_: Add automatically generated :ref:`plugin-list`. The list is updated on a periodic schedule. + + +- `#8337 `_: Recommend `numpy.testing `__ module on :func:`pytest.approx` documentation. + + +- `#8655 `_: Help text for ``--pdbcls`` more accurately reflects the option's behavior. + + +- `#9210 `_: Remove incorrect docs about ``confcutdir`` being a configuration option: it can only be set through the ``--confcutdir`` command-line option. + + +- `#9242 `_: Upgrade readthedocs configuration to use a `newer Ubuntu version `__` with better unicode support for PDF docs. + + +- `#9341 `_: Various methods commonly used for :ref:`non-python tests` are now correctly documented in the reference docs. They were undocumented previously. + + + +Trivial/Internal Changes +------------------------ + +- `#8133 `_: Migrate to ``setuptools_scm`` 6.x to use ``SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST`` for more robust release tooling. + + +- `#8174 `_: The following changes have been made to internal pytest types/functions: + + - The ``_pytest.code.getfslineno()`` function returns ``Path`` instead of ``py.path.local``. + - The ``_pytest.python.path_matches_patterns()`` function takes ``Path`` instead of ``py.path.local``. + - The ``_pytest._code.Traceback.cut()`` function accepts any ``os.PathLike[str]``, not just ``py.path.local``. + + +- `#8248 `_: Internal Restructure: let ``python.PyObjMixin`` inherit from ``nodes.Node`` to carry over typing information. + + +- `#8432 `_: Improve error message when :func:`pytest.skip` is used at module level without passing `allow_module_level=True`. + + +- `#8818 `_: Ensure ``regendoc`` opts out of ``TOX_ENV`` cachedir selection to ensure independent example test runs. + + +- `#8913 `_: The private ``CallSpec2._arg2scopenum`` attribute has been removed after an internal refactoring. + + +- `#8967 `_: :hook:`pytest_assertion_pass` is no longer considered experimental and + future changes to it will be considered more carefully. + + +- `#9202 `_: Add github action to upload coverage report to codecov instead of bash uploader. + + +- `#9225 `_: Changed the command used to create sdist and wheel artifacts: using the build package instead of setup.py. + + +- `#9351 `_: Correct minor typos in doc/en/example/special.rst. + + +pytest 6.2.5 (2021-08-29) +========================= + + +Trivial/Internal Changes +------------------------ + +- :issue:`8494`: Python 3.10 is now supported. + + +- :issue:`9040`: Enable compatibility with ``pluggy 1.0`` or later. + + +pytest 6.2.4 (2021-05-04) +========================= + +Bug Fixes +--------- + +- :issue:`8539`: Fixed assertion rewriting on Python 3.10. + + +pytest 6.2.3 (2021-04-03) +========================= + +Bug Fixes +--------- + +- :issue:`8414`: pytest used to create directories under ``/tmp`` with world-readable + permissions. This means that any user in the system was able to read + information written by tests in temporary directories (such as those created by + the ``tmp_path``/``tmpdir`` fixture). Now the directories are created with + private permissions. + + pytest used to silently use a pre-existing ``/tmp/pytest-of-`` directory, + even if owned by another user. This means another user could pre-create such a + directory and gain control of another user's temporary directory. Now such a + condition results in an error. + + +pytest 6.2.2 (2021-01-25) +========================= + +Bug Fixes +--------- + +- :issue:`8152`: Fixed "()" being shown as a skip reason in the verbose test summary line when the reason is empty. + + +- :issue:`8249`: Fix the ``faulthandler`` plugin for occasions when running with ``twisted.logger`` and using ``pytest --capture=no``. + + +pytest 6.2.1 (2020-12-15) +========================= + +Bug Fixes +--------- + +- :issue:`7678`: Fixed bug where ``ImportPathMismatchError`` would be raised for files compiled in + the host and loaded later from an UNC mounted path (Windows). + + +- :issue:`8132`: Fixed regression in ``approx``: in 6.2.0 ``approx`` no longer raises + ``TypeError`` when dealing with non-numeric types, falling back to normal comparison. + Before 6.2.0, array types like tf.DeviceArray fell through to the scalar case, + and happened to compare correctly to a scalar if they had only one element. + After 6.2.0, these types began failing, because they inherited neither from + standard Python number hierarchy nor from ``numpy.ndarray``. + + ``approx`` now converts arguments to ``numpy.ndarray`` if they expose the array + protocol and are not scalars. This treats array-like objects like numpy arrays, + regardless of size. + + +pytest 6.2.0 (2020-12-12) +========================= + +Breaking Changes +---------------- + +- :issue:`7808`: pytest now supports python3.6+ only. + + + +Deprecations +------------ + +- :issue:`7469`: Directly constructing/calling the following classes/functions is now deprecated: + + - ``_pytest.cacheprovider.Cache`` + - ``_pytest.cacheprovider.Cache.for_config()`` + - ``_pytest.cacheprovider.Cache.clear_cache()`` + - ``_pytest.cacheprovider.Cache.cache_dir_from_config()`` + - ``_pytest.capture.CaptureFixture`` + - ``_pytest.fixtures.FixtureRequest`` + - ``_pytest.fixtures.SubRequest`` + - ``_pytest.logging.LogCaptureFixture`` + - ``_pytest.pytester.Pytester`` + - ``_pytest.pytester.Testdir`` + - ``_pytest.recwarn.WarningsRecorder`` + - ``_pytest.recwarn.WarningsChecker`` + - ``_pytest.tmpdir.TempPathFactory`` + - ``_pytest.tmpdir.TempdirFactory`` + + These have always been considered private, but now issue a deprecation warning, which may become a hard error in pytest 8.0.0. + + +- :issue:`7530`: The ``--strict`` command-line option has been deprecated, use ``--strict-markers`` instead. + + We have plans to maybe in the future to reintroduce ``--strict`` and make it an encompassing flag for all strictness + related options (``--strict-markers`` and ``--strict-config`` at the moment, more might be introduced in the future). + + +- :issue:`7988`: The ``@pytest.yield_fixture`` decorator/function is now deprecated. Use :func:`pytest.fixture` instead. + + ``yield_fixture`` has been an alias for ``fixture`` for a very long time, so can be search/replaced safely. + + + +Features +-------- + +- :issue:`5299`: pytest now warns about unraisable exceptions and unhandled thread exceptions that occur in tests on Python>=3.8. + See :ref:`unraisable` for more information. + + +- :issue:`7425`: New :fixture:`pytester` fixture, which is identical to :fixture:`testdir` but its methods return :class:`pathlib.Path` when appropriate instead of ``py.path.local``. + + This is part of the movement to use :class:`pathlib.Path` objects internally, in order to remove the dependency to ``py`` in the future. + + Internally, the old :class:`Testdir <_pytest.pytester.Testdir>` is now a thin wrapper around :class:`Pytester <_pytest.pytester.Pytester>`, preserving the old interface. + + +- :issue:`7695`: A new hook was added, `pytest_markeval_namespace` which should return a dictionary. + This dictionary will be used to augment the "global" variables available to evaluate skipif/xfail/xpass markers. + + Pseudo example + + ``conftest.py``: + + .. code-block:: python + + def pytest_markeval_namespace(): + return {"color": "red"} + + ``test_func.py``: + + .. code-block:: python + + @pytest.mark.skipif("color == 'blue'", reason="Color is not red") + def test_func(): + assert False + + +- :issue:`8006`: It is now possible to construct a :class:`~pytest.MonkeyPatch` object directly as ``pytest.MonkeyPatch()``, + in cases when the :fixture:`monkeypatch` fixture cannot be used. Previously some users imported it + from the private `_pytest.monkeypatch.MonkeyPatch` namespace. + + Additionally, :meth:`MonkeyPatch.context ` is now a classmethod, + and can be used as ``with MonkeyPatch.context() as mp: ...``. This is the recommended way to use + ``MonkeyPatch`` directly, since unlike the ``monkeypatch`` fixture, an instance created directly + is not ``undo()``-ed automatically. + + + +Improvements +------------ + +- :issue:`1265`: Added an ``__str__`` implementation to the :class:`~pytest.pytester.LineMatcher` class which is returned from ``pytester.run_pytest().stdout`` and similar. It returns the entire output, like the existing ``str()`` method. + + +- :issue:`2044`: Verbose mode now shows the reason that a test was skipped in the test's terminal line after the "SKIPPED", "XFAIL" or "XPASS". + + +- :issue:`7469` The types of builtin pytest fixtures are now exported so they may be used in type annotations of test functions. + The newly-exported types are: + + - ``pytest.FixtureRequest`` for the :fixture:`request` fixture. + - ``pytest.Cache`` for the :fixture:`cache` fixture. + - ``pytest.CaptureFixture[str]`` for the :fixture:`capfd` and :fixture:`capsys` fixtures. + - ``pytest.CaptureFixture[bytes]`` for the :fixture:`capfdbinary` and :fixture:`capsysbinary` fixtures. + - ``pytest.LogCaptureFixture`` for the :fixture:`caplog` fixture. + - ``pytest.Pytester`` for the :fixture:`pytester` fixture. + - ``pytest.Testdir`` for the :fixture:`testdir` fixture. + - ``pytest.TempdirFactory`` for the :fixture:`tmpdir_factory` fixture. + - ``pytest.TempPathFactory`` for the :fixture:`tmp_path_factory` fixture. + - ``pytest.MonkeyPatch`` for the :fixture:`monkeypatch` fixture. + - ``pytest.WarningsRecorder`` for the :fixture:`recwarn` fixture. + + Constructing them is not supported (except for `MonkeyPatch`); they are only meant for use in type annotations. + Doing so will emit a deprecation warning, and may become a hard-error in pytest 8.0. + + Subclassing them is also not supported. This is not currently enforced at runtime, but is detected by type-checkers such as mypy. + + +- :issue:`7527`: When a comparison between :func:`namedtuple ` instances of the same type fails, pytest now shows the differing field names (possibly nested) instead of their indexes. + + +- :issue:`7615`: :meth:`Node.warn <_pytest.nodes.Node.warn>` now permits any subclass of :class:`Warning`, not just :class:`PytestWarning `. + + +- :issue:`7701`: Improved reporting when using ``--collected-only``. It will now show the number of collected tests in the summary stats. + + +- :issue:`7710`: Use strict equality comparison for non-numeric types in :func:`pytest.approx` instead of + raising :class:`TypeError`. + + This was the undocumented behavior before 3.7, but is now officially a supported feature. + + +- :issue:`7938`: New ``--sw-skip`` argument which is a shorthand for ``--stepwise-skip``. + + +- :issue:`8023`: Added ``'node_modules'`` to default value for :confval:`norecursedirs`. + + +- :issue:`8032`: :meth:`doClassCleanups ` (introduced in :mod:`unittest` in Python and 3.8) is now called appropriately. + + + +Bug Fixes +--------- + +- :issue:`4824`: Fixed quadratic behavior and improved performance of collection of items using autouse fixtures and xunit fixtures. + + +- :issue:`7758`: Fixed an issue where some files in packages are getting lost from ``--lf`` even though they contain tests that failed. Regressed in pytest 5.4.0. + + +- :issue:`7911`: Directories created by by :fixture:`tmp_path` and :fixture:`tmpdir` are now considered stale after 3 days without modification (previous value was 3 hours) to avoid deleting directories still in use in long running test suites. + + +- :issue:`7913`: Fixed a crash or hang in :meth:`pytester.spawn <_pytest.pytester.Pytester.spawn>` when the :mod:`readline` module is involved. + + +- :issue:`7951`: Fixed handling of recursive symlinks when collecting tests. + + +- :issue:`7981`: Fixed symlinked directories not being followed during collection. Regressed in pytest 6.1.0. + + +- :issue:`8016`: Fixed only one doctest being collected when using ``pytest --doctest-modules path/to/an/__init__.py``. + + + +Improved Documentation +---------------------- + +- :issue:`7429`: Add more information and use cases about skipping doctests. + + +- :issue:`7780`: Classes which should not be inherited from are now marked ``final class`` in the API reference. + + +- :issue:`7872`: ``_pytest.config.argparsing.Parser.addini()`` accepts explicit ``None`` and ``"string"``. + + +- :issue:`7878`: In pull request section, ask to commit after editing changelog and authors file. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`7802`: The ``attrs`` dependency requirement is now >=19.2.0 instead of >=17.4.0. + + +- :issue:`8014`: `.pyc` files created by pytest's assertion rewriting now conform to the newer :pep:`552` format on Python>=3.7. + (These files are internal and only interpreted by pytest itself.) + + +pytest 6.1.2 (2020-10-28) +========================= + +Bug Fixes +--------- + +- :issue:`7758`: Fixed an issue where some files in packages are getting lost from ``--lf`` even though they contain tests that failed. Regressed in pytest 5.4.0. + + +- :issue:`7911`: Directories created by `tmpdir` are now considered stale after 3 days without modification (previous value was 3 hours) to avoid deleting directories still in use in long running test suites. + + + +Improved Documentation +---------------------- + +- :issue:`7815`: Improve deprecation warning message for ``pytest._fillfuncargs()``. + + +pytest 6.1.1 (2020-10-03) +========================= + +Bug Fixes +--------- + +- :issue:`7807`: Fixed regression in pytest 6.1.0 causing incorrect rootdir to be determined in some non-trivial cases where parent directories have config files as well. + + +- :issue:`7814`: Fixed crash in header reporting when :confval:`testpaths` is used and contains absolute paths (regression in 6.1.0). + + +pytest 6.1.0 (2020-09-26) +========================= + +Breaking Changes +---------------- + +- :issue:`5585`: As per our policy, the following features which have been deprecated in the 5.X series are now + removed: + + * The ``funcargnames`` read-only property of ``FixtureRequest``, ``Metafunc``, and ``Function`` classes. Use ``fixturenames`` attribute. + + * ``@pytest.fixture`` no longer supports positional arguments, pass all arguments by keyword instead. + + * Direct construction of ``Node`` subclasses now raise an error, use ``from_parent`` instead. + + * The default value for ``junit_family`` has changed to ``xunit2``. If you require the old format, add ``junit_family=xunit1`` to your configuration file. + + * The ``TerminalReporter`` no longer has a ``writer`` attribute. Plugin authors may use the public functions of the ``TerminalReporter`` instead of accessing the ``TerminalWriter`` object directly. + + * The ``--result-log`` option has been removed. Users are recommended to use the `pytest-reportlog `__ plugin instead. + + + For more information consult :std:doc:`deprecations` in the docs. + + + +Deprecations +------------ + +- :issue:`6981`: The ``pytest.collect`` module is deprecated: all its names can be imported from ``pytest`` directly. + + +- :issue:`7097`: The ``pytest._fillfuncargs`` function is deprecated. This function was kept + for backward compatibility with an older plugin. + + It's functionality is not meant to be used directly, but if you must replace + it, use `function._request._fillfixtures()` instead, though note this is not + a public API and may break in the future. + + +- :issue:`7210`: The special ``-k '-expr'`` syntax to ``-k`` is deprecated. Use ``-k 'not expr'`` + instead. + + The special ``-k 'expr:'`` syntax to ``-k`` is deprecated. Please open an issue + if you use this and want a replacement. + + +- :issue:`7255`: The :hook:`pytest_warning_captured` hook is deprecated in favor + of :hook:`pytest_warning_recorded`, and will be removed in a future version. + + +- :issue:`7648`: The ``gethookproxy()`` and ``isinitpath()`` methods of ``FSCollector`` and ``Package`` are deprecated; + use ``self.session.gethookproxy()`` and ``self.session.isinitpath()`` instead. + This should work on all pytest versions. + + + +Features +-------- + +- :issue:`7667`: New ``--durations-min`` command-line flag controls the minimal duration for inclusion in the slowest list of tests shown by ``--durations``. Previously this was hard-coded to ``0.005s``. + + + +Improvements +------------ + +- :issue:`6681`: Internal pytest warnings issued during the early stages of initialization are now properly handled and can filtered through :confval:`filterwarnings` or ``--pythonwarnings/-W``. + + This also fixes a number of long standing issues: :issue:`2891`, :issue:`7620`, :issue:`7426`. + + +- :issue:`7572`: When a plugin listed in ``required_plugins`` is missing or an unknown config key is used with ``--strict-config``, a simple error message is now shown instead of a stacktrace. + + +- :issue:`7685`: Added two new attributes :attr:`rootpath <_pytest.config.Config.rootpath>` and :attr:`inipath <_pytest.config.Config.inipath>` to :class:`Config <_pytest.config.Config>`. + These attributes are :class:`pathlib.Path` versions of the existing :attr:`rootdir <_pytest.config.Config.rootdir>` and :attr:`inifile <_pytest.config.Config.inifile>` attributes, + and should be preferred over them when possible. + + +- :issue:`7780`: Public classes which are not designed to be inherited from are now marked :func:`@final `. + Code which inherits from these classes will trigger a type-checking (e.g. mypy) error, but will still work in runtime. + Currently the ``final`` designation does not appear in the API Reference but hopefully will in the future. + + + +Bug Fixes +--------- + +- :issue:`1953`: Fixed error when overwriting a parametrized fixture, while also reusing the super fixture value. + + .. code-block:: python + + # conftest.py + import pytest + + + @pytest.fixture(params=[1, 2]) + def foo(request): + return request.param + + + # test_foo.py + import pytest + + + @pytest.fixture + def foo(foo): + return foo * 2 + + +- :issue:`4984`: Fixed an internal error crash with ``IndexError: list index out of range`` when + collecting a module which starts with a decorated function, the decorator + raises, and assertion rewriting is enabled. + + +- :issue:`7591`: pylint shouldn't complain anymore about unimplemented abstract methods when inheriting from :ref:`File `. + + +- :issue:`7628`: Fixed test collection when a full path without a drive letter was passed to pytest on Windows (for example ``\projects\tests\test.py`` instead of ``c:\projects\tests\pytest.py``). + + +- :issue:`7638`: Fix handling of command-line options that appear as paths but trigger an OS-level syntax error on Windows, such as the options used internally by ``pytest-xdist``. + + +- :issue:`7742`: Fixed INTERNALERROR when accessing locals / globals with faulty ``exec``. + + + +Improved Documentation +---------------------- + +- :issue:`1477`: Removed faq.rst and its reference in contents.rst. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`7536`: The internal ``junitxml`` plugin has rewritten to use ``xml.etree.ElementTree``. + The order of attributes in XML elements might differ. Some unneeded escaping is + no longer performed. + + +- :issue:`7587`: The dependency on the ``more-itertools`` package has been removed. + + +- :issue:`7631`: The result type of :meth:`capfd.readouterr() <_pytest.capture.CaptureFixture.readouterr>` (and similar) is no longer a namedtuple, + but should behave like one in all respects. This was done for technical reasons. + + +- :issue:`7671`: When collecting tests, pytest finds test classes and functions by examining the + attributes of python objects (modules, classes and instances). To speed up this + process, pytest now ignores builtin attributes (like ``__class__``, + ``__delattr__`` and ``__new__``) without consulting the :confval:`python_classes` and + :confval:`python_functions` configuration options and without passing them to plugins + using the :hook:`pytest_pycollect_makeitem` hook. + + +pytest 6.0.2 (2020-09-04) +========================= + +Bug Fixes +--------- + +- :issue:`7148`: Fixed ``--log-cli`` potentially causing unrelated ``print`` output to be swallowed. + + +- :issue:`7672`: Fixed log-capturing level restored incorrectly if ``caplog.set_level`` is called more than once. + + +- :issue:`7686`: Fixed `NotSetType.token` being used as the parameter ID when the parametrization list is empty. + Regressed in pytest 6.0.0. + + +- :issue:`7707`: Fix internal error when handling some exceptions that contain multiple lines or the style uses multiple lines (``--tb=line`` for example). + + +pytest 6.0.1 (2020-07-30) +========================= + +Bug Fixes +--------- + +- :issue:`7394`: Passing an empty ``help`` value to ``Parser.add_option`` is now accepted instead of crashing when running ``pytest --help``. + Passing ``None`` raises a more informative ``TypeError``. + + +- :issue:`7558`: Fix pylint ``not-callable`` lint on ``pytest.mark.parametrize()`` and the other builtin marks: + ``skip``, ``skipif``, ``xfail``, ``usefixtures``, ``filterwarnings``. + + +- :issue:`7559`: Fix regression in plugins using ``TestReport.longreprtext`` (such as ``pytest-html``) when ``TestReport.longrepr`` is not a string. + + +- :issue:`7569`: Fix logging capture handler's level not reset on teardown after a call to ``caplog.set_level()``. + + +pytest 6.0.0 (2020-07-28) +========================= + +(**Please see the full set of changes for this release also in the 6.0.0rc1 notes below**) + +Breaking Changes +---------------- + +- :issue:`5584`: **PytestDeprecationWarning are now errors by default.** + + Following our plan to remove deprecated features with as little disruption as + possible, all warnings of type ``PytestDeprecationWarning`` now generate errors + instead of warning messages. + + **The affected features will be effectively removed in pytest 6.1**, so please consult the + :std:doc:`deprecations` section in the docs for directions on how to update existing code. + + In the pytest ``6.0.X`` series, it is possible to change the errors back into warnings as a + stopgap measure by adding this to your ``pytest.ini`` file: + + .. code-block:: ini + + [pytest] + filterwarnings = + ignore::pytest.PytestDeprecationWarning + + But this will stop working when pytest ``6.1`` is released. + + **If you have concerns** about the removal of a specific feature, please add a + comment to :issue:`5584`. + + +- :issue:`7472`: The ``exec_()`` and ``is_true()`` methods of ``_pytest._code.Frame`` have been removed. + + + +Features +-------- + +- :issue:`7464`: Added support for :envvar:`NO_COLOR` and :envvar:`FORCE_COLOR` environment variables to control colored output. + + + +Improvements +------------ + +- :issue:`7467`: ``--log-file`` CLI option and ``log_file`` ini marker now create subdirectories if needed. + + +- :issue:`7489`: The :func:`pytest.raises` function has a clearer error message when ``match`` equals the obtained string but is not a regex match. In this case it is suggested to escape the regex. + + + +Bug Fixes +--------- + +- :issue:`7392`: Fix the reported location of tests skipped with ``@pytest.mark.skip`` when ``--runxfail`` is used. + + +- :issue:`7491`: :fixture:`tmpdir` and :fixture:`tmp_path` no longer raise an error if the lock to check for + stale temporary directories is not accessible. + + +- :issue:`7517`: Preserve line endings when captured via ``capfd``. + + +- :issue:`7534`: Restored the previous formatting of ``TracebackEntry.__str__`` which was changed by accident. + + + +Improved Documentation +---------------------- + +- :issue:`7422`: Clarified when the ``usefixtures`` mark can apply fixtures to test. + + +- :issue:`7441`: Add a note about ``-q`` option used in getting started guide. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`7389`: Fixture scope ``package`` is no longer considered experimental. + + +pytest 6.0.0rc1 (2020-07-08) +============================ + +Breaking Changes +---------------- + +- :issue:`1316`: ``TestReport.longrepr`` is now always an instance of ``ReprExceptionInfo``. Previously it was a ``str`` when a test failed with ``pytest.fail(..., pytrace=False)``. + + +- :issue:`5965`: symlinks are no longer resolved during collection and matching `conftest.py` files with test file paths. + + Resolving symlinks for the current directory and during collection was introduced as a bugfix in 3.9.0, but it actually is a new feature which had unfortunate consequences in Windows and surprising results in other platforms. + + The team decided to step back on resolving symlinks at all, planning to review this in the future with a more solid solution (see discussion in + :pull:`6523` for details). + + This might break test suites which made use of this feature; the fix is to create a symlink + for the entire test tree, and not only to partial files/tress as it was possible previously. + + +- :issue:`6505`: ``Testdir.run().parseoutcomes()`` now always returns the parsed nouns in plural form. + + Originally ``parseoutcomes()`` would always returns the nouns in plural form, but a change + meant to improve the terminal summary by using singular form single items (``1 warning`` or ``1 error``) + caused an unintended regression by changing the keys returned by ``parseoutcomes()``. + + Now the API guarantees to always return the plural form, so calls like this: + + .. code-block:: python + + result = testdir.runpytest() + result.assert_outcomes(error=1) + + Need to be changed to: + + + .. code-block:: python + + result = testdir.runpytest() + result.assert_outcomes(errors=1) + + +- :issue:`6903`: The ``os.dup()`` function is now assumed to exist. We are not aware of any + supported Python 3 implementations which do not provide it. + + +- :issue:`7040`: ``-k`` no longer matches against the names of the directories outside the test session root. + + Also, ``pytest.Package.name`` is now just the name of the directory containing the package's + ``__init__.py`` file, instead of the full path. This is consistent with how the other nodes + are named, and also one of the reasons why ``-k`` would match against any directory containing + the test suite. + + +- :issue:`7122`: Expressions given to the ``-m`` and ``-k`` options are no longer evaluated using Python's :func:`eval`. + The format supports ``or``, ``and``, ``not``, parenthesis and general identifiers to match against. + Python constants, keywords or other operators are no longer evaluated differently. + + +- :issue:`7135`: Pytest now uses its own ``TerminalWriter`` class instead of using the one from the ``py`` library. + Plugins generally access this class through ``TerminalReporter.writer``, ``TerminalReporter.write()`` + (and similar methods), or ``_pytest.config.create_terminal_writer()``. + + The following breaking changes were made: + + - Output (``write()`` method and others) no longer flush implicitly; the flushing behavior + of the underlying file is respected. To flush explicitly (for example, if you + want output to be shown before an end-of-line is printed), use ``write(flush=True)`` or + ``terminal_writer.flush()``. + - Explicit Windows console support was removed, delegated to the colorama library. + - Support for writing ``bytes`` was removed. + - The ``reline`` method and ``chars_on_current_line`` property were removed. + - The ``stringio`` and ``encoding`` arguments was removed. + - Support for passing a callable instead of a file was removed. + + +- :issue:`7224`: The `item.catch_log_handler` and `item.catch_log_handlers` attributes, set by the + logging plugin and never meant to be public, are no longer available. + + The deprecated ``--no-print-logs`` option and ``log_print`` ini option are removed. Use ``--show-capture`` instead. + + +- :issue:`7226`: Removed the unused ``args`` parameter from ``pytest.Function.__init__``. + + +- :issue:`7418`: Removed the `pytest_doctest_prepare_content` hook specification. This hook + hasn't been triggered by pytest for at least 10 years. + + +- :issue:`7438`: Some changes were made to the internal ``_pytest._code.source``, listed here + for the benefit of plugin authors who may be using it: + + - The ``deindent`` argument to ``Source()`` has been removed, now it is always true. + - Support for zero or multiple arguments to ``Source()`` has been removed. + - Support for comparing ``Source`` with an ``str`` has been removed. + - The methods ``Source.isparseable()`` and ``Source.putaround()`` have been removed. + - The method ``Source.compile()`` and function ``_pytest._code.compile()`` have + been removed; use plain ``compile()`` instead. + - The function ``_pytest._code.source.getsource()`` has been removed; use + ``Source()`` directly instead. + + + +Deprecations +------------ + +- :issue:`7210`: The special ``-k '-expr'`` syntax to ``-k`` is deprecated. Use ``-k 'not expr'`` + instead. + + The special ``-k 'expr:'`` syntax to ``-k`` is deprecated. Please open an issue + if you use this and want a replacement. + +- :issue:`4049`: ``pytest_warning_captured`` is deprecated in favor of the ``pytest_warning_recorded`` hook. + + +Features +-------- + +- :issue:`1556`: pytest now supports ``pyproject.toml`` files for configuration. + + The configuration options is similar to the one available in other formats, but must be defined + in a ``[tool.pytest.ini_options]`` table to be picked up by pytest: + + .. code-block:: toml + + # pyproject.toml + [tool.pytest.ini_options] + minversion = "6.0" + addopts = "-ra -q" + testpaths = [ + "tests", + "integration", + ] + + More information can be found :ref:`in the docs `. + + +- :issue:`3342`: pytest now includes inline type annotations and exposes them to user programs. + Most of the user-facing API is covered, as well as internal code. + + If you are running a type checker such as mypy on your tests, you may start + noticing type errors indicating incorrect usage. If you run into an error that + you believe to be incorrect, please let us know in an issue. + + The types were developed against mypy version 0.780. Versions before 0.750 + are known not to work. We recommend using the latest version. Other type + checkers may work as well, but they are not officially verified to work by + pytest yet. + + +- :issue:`4049`: Introduced a new hook named `pytest_warning_recorded` to convey information about warnings captured by the internal `pytest` warnings plugin. + + This hook is meant to replace `pytest_warning_captured`, which is deprecated and will be removed in a future release. + + +- :issue:`6471`: New command-line flags: + + * `--no-header`: disables the initial header, including platform, version, and plugins. + * `--no-summary`: disables the final test summary, including warnings. + + +- :issue:`6856`: A warning is now shown when an unknown key is read from a config INI file. + + The `--strict-config` flag has been added to treat these warnings as errors. + + +- :issue:`6906`: Added `--code-highlight` command line option to enable/disable code highlighting in terminal output. + + +- :issue:`7245`: New ``--import-mode=importlib`` option that uses :mod:`importlib` to import test modules. + + Traditionally pytest used ``__import__`` while changing ``sys.path`` to import test modules (which + also changes ``sys.modules`` as a side-effect), which works but has a number of drawbacks, like requiring test modules + that don't live in packages to have unique names (as they need to reside under a unique name in ``sys.modules``). + + ``--import-mode=importlib`` uses more fine grained import mechanisms from ``importlib`` which don't + require pytest to change ``sys.path`` or ``sys.modules`` at all, eliminating much of the drawbacks + of the previous mode. + + We intend to make ``--import-mode=importlib`` the default in future versions, so users are encouraged + to try the new mode and provide feedback (both positive or negative) in issue :issue:`7245`. + + You can read more about this option in :std:ref:`the documentation `. + + +- :issue:`7305`: New ``required_plugins`` configuration option allows the user to specify a list of plugins, including version information, that are required for pytest to run. An error is raised if any required plugins are not found when running pytest. + + +Improvements +------------ + +- :issue:`4375`: The ``pytest`` command now suppresses the ``BrokenPipeError`` error message that + is printed to stderr when the output of ``pytest`` is piped and and the pipe is + closed by the piped-to program (common examples are ``less`` and ``head``). + + +- :issue:`4391`: Improved precision of test durations measurement. ``CallInfo`` items now have a new ``.duration`` attribute, created using ``time.perf_counter()``. This attribute is used to fill the ``.duration`` attribute, which is more accurate than the previous ``.stop - .start`` (as these are based on ``time.time()``). + + +- :issue:`4675`: Rich comparison for dataclasses and `attrs`-classes is now recursive. + + +- :issue:`6285`: Exposed the `pytest.FixtureLookupError` exception which is raised by `request.getfixturevalue()` + (where `request` is a `FixtureRequest` fixture) when a fixture with the given name cannot be returned. + + +- :issue:`6433`: If an error is encountered while formatting the message in a logging call, for + example ``logging.warning("oh no!: %s: %s", "first")`` (a second argument is + missing), pytest now propagates the error, likely causing the test to fail. + + Previously, such a mistake would cause an error to be printed to stderr, which + is not displayed by default for passing tests. This change makes the mistake + visible during testing. + + You may suppress this behavior temporarily or permanently by setting + ``logging.raiseExceptions = False``. + + +- :issue:`6817`: Explicit new-lines in help texts of command-line options are preserved, allowing plugins better control + of the help displayed to users. + + +- :issue:`6940`: When using the ``--duration`` option, the terminal message output is now more precise about the number and duration of hidden items. + + +- :issue:`6991`: Collected files are displayed after any reports from hooks, e.g. the status from ``--lf``. + + +- :issue:`7091`: When ``fd`` capturing is used, through ``--capture=fd`` or the ``capfd`` and + ``capfdbinary`` fixtures, and the file descriptor (0, 1, 2) cannot be + duplicated, FD capturing is still performed. Previously, direct writes to the + file descriptors would fail or be lost in this case. + + +- :issue:`7119`: Exit with an error if the ``--basetemp`` argument is empty, is the current working directory or is one of the parent directories. + This is done to protect against accidental data loss, as any directory passed to this argument is cleared. + + +- :issue:`7128`: `pytest --version` now displays just the pytest version, while `pytest --version --version` displays more verbose information including plugins. This is more consistent with how other tools show `--version`. + + +- :issue:`7133`: :meth:`caplog.set_level() <_pytest.logging.LogCaptureFixture.set_level>` will now override any :confval:`log_level` set via the CLI or configuration file. + + +- :issue:`7159`: :meth:`caplog.set_level() <_pytest.logging.LogCaptureFixture.set_level>` and :meth:`caplog.at_level() <_pytest.logging.LogCaptureFixture.at_level>` no longer affect + the level of logs that are shown in the *Captured log report* report section. + + +- :issue:`7348`: Improve recursive diff report for comparison asserts on dataclasses / attrs. + + +- :issue:`7385`: ``--junitxml`` now includes the exception cause in the ``message`` XML attribute for failures during setup and teardown. + + Previously: + + .. code-block:: xml + + + + Now: + + .. code-block:: xml + + + + + +Bug Fixes +--------- + +- :issue:`1120`: Fix issue where directories from :fixture:`tmpdir` are not removed properly when multiple instances of pytest are running in parallel. + + +- :issue:`4583`: Prevent crashing and provide a user-friendly error when a marker expression (`-m`) invoking of :func:`eval` raises any exception. + + +- :issue:`4677`: The path shown in the summary report for SKIPPED tests is now always relative. Previously it was sometimes absolute. + + +- :issue:`5456`: Fix a possible race condition when trying to remove lock files used to control access to folders + created by :fixture:`tmp_path` and :fixture:`tmpdir`. + + +- :issue:`6240`: Fixes an issue where logging during collection step caused duplication of log + messages to stderr. + + +- :issue:`6428`: Paths appearing in error messages are now correct in case the current working directory has + changed since the start of the session. + + +- :issue:`6755`: Support deleting paths longer than 260 characters on windows created inside :fixture:`tmpdir`. + + +- :issue:`6871`: Fix crash with captured output when using :fixture:`capsysbinary`. + + +- :issue:`6909`: Revert the change introduced by :pull:`6330`, which required all arguments to ``@pytest.mark.parametrize`` to be explicitly defined in the function signature. + + The intention of the original change was to remove what was expected to be an unintended/surprising behavior, but it turns out many people relied on it, so the restriction has been reverted. + + +- :issue:`6910`: Fix crash when plugins return an unknown stats while using the ``--reportlog`` option. + + +- :issue:`6924`: Ensure a ``unittest.IsolatedAsyncioTestCase`` is actually awaited. + + +- :issue:`6925`: Fix `TerminalRepr` instances to be hashable again. + + +- :issue:`6947`: Fix regression where functions registered with :meth:`unittest.TestCase.addCleanup` were not being called on test failures. + + +- :issue:`6951`: Allow users to still set the deprecated ``TerminalReporter.writer`` attribute. + + +- :issue:`6956`: Prevent pytest from printing `ConftestImportFailure` traceback to stdout. + + +- :issue:`6991`: Fix regressions with `--lf` filtering too much since pytest 5.4. + + +- :issue:`6992`: Revert "tmpdir: clean up indirection via config for factories" :issue:`6767` as it breaks pytest-xdist. + + +- :issue:`7061`: When a yielding fixture fails to yield a value, report a test setup error instead of crashing. + + +- :issue:`7076`: The path of file skipped by ``@pytest.mark.skip`` in the SKIPPED report is now relative to invocation directory. Previously it was relative to root directory. + + +- :issue:`7110`: Fixed regression: ``asyncbase.TestCase`` tests are executed correctly again. + + +- :issue:`7126`: ``--setup-show`` now doesn't raise an error when a bytes value is used as a ``parametrize`` + parameter when Python is called with the ``-bb`` flag. + + +- :issue:`7143`: Fix :meth:`pytest.File.from_parent` so it forwards extra keyword arguments to the constructor. + + +- :issue:`7145`: Classes with broken ``__getattribute__`` methods are displayed correctly during failures. + + +- :issue:`7150`: Prevent hiding the underlying exception when ``ConfTestImportFailure`` is raised. + + +- :issue:`7180`: Fix ``_is_setup_py`` for files encoded differently than locale. + + +- :issue:`7215`: Fix regression where running with ``--pdb`` would call :meth:`unittest.TestCase.tearDown` for skipped tests. + + +- :issue:`7253`: When using ``pytest.fixture`` on a function directly, as in ``pytest.fixture(func)``, + if the ``autouse`` or ``params`` arguments are also passed, the function is no longer + ignored, but is marked as a fixture. + + +- :issue:`7360`: Fix possibly incorrect evaluation of string expressions passed to ``pytest.mark.skipif`` and ``pytest.mark.xfail``, + in rare circumstances where the exact same string is used but refers to different global values. + + +- :issue:`7383`: Fixed exception causes all over the codebase, i.e. use `raise new_exception from old_exception` when wrapping an exception. + + + +Improved Documentation +---------------------- + +- :issue:`7202`: The development guide now links to the contributing section of the docs and `RELEASING.rst` on GitHub. + + +- :issue:`7233`: Add a note about ``--strict`` and ``--strict-markers`` and the preference for the latter one. + + +- :issue:`7345`: Explain indirect parametrization and markers for fixtures. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`7035`: The ``originalname`` attribute of ``_pytest.python.Function`` now defaults to ``name`` if not + provided explicitly, and is always set. + + +- :issue:`7264`: The dependency on the ``wcwidth`` package has been removed. + + +- :issue:`7291`: Replaced ``py.iniconfig`` with :pypi:`iniconfig`. + + +- :issue:`7295`: ``src/_pytest/config/__init__.py`` now uses the ``warnings`` module to report warnings instead of ``sys.stderr.write``. + + +- :issue:`7356`: Remove last internal uses of deprecated *slave* term from old ``pytest-xdist``. + + +- :issue:`7357`: ``py``>=1.8.2 is now required. + + +pytest 5.4.3 (2020-06-02) +========================= + +Bug Fixes +--------- + +- :issue:`6428`: Paths appearing in error messages are now correct in case the current working directory has + changed since the start of the session. + + +- :issue:`6755`: Support deleting paths longer than 260 characters on windows created inside tmpdir. + + +- :issue:`6956`: Prevent pytest from printing ConftestImportFailure traceback to stdout. + + +- :issue:`7150`: Prevent hiding the underlying exception when ``ConfTestImportFailure`` is raised. + + +- :issue:`7215`: Fix regression where running with ``--pdb`` would call the ``tearDown`` methods of ``unittest.TestCase`` + subclasses for skipped tests. + + +pytest 5.4.2 (2020-05-08) +========================= + +Bug Fixes +--------- + +- :issue:`6871`: Fix crash with captured output when using the :fixture:`capsysbinary fixture `. + + +- :issue:`6924`: Ensure a ``unittest.IsolatedAsyncioTestCase`` is actually awaited. + + +- :issue:`6925`: Fix TerminalRepr instances to be hashable again. + + +- :issue:`6947`: Fix regression where functions registered with ``TestCase.addCleanup`` were not being called on test failures. + + +- :issue:`6951`: Allow users to still set the deprecated ``TerminalReporter.writer`` attribute. + + +- :issue:`6992`: Revert "tmpdir: clean up indirection via config for factories" #6767 as it breaks pytest-xdist. + + +- :issue:`7110`: Fixed regression: ``asyncbase.TestCase`` tests are executed correctly again. + + +- :issue:`7143`: Fix ``File.from_parent`` so it forwards extra keyword arguments to the constructor. + + +- :issue:`7145`: Classes with broken ``__getattribute__`` methods are displayed correctly during failures. + + +- :issue:`7180`: Fix ``_is_setup_py`` for files encoded differently than locale. + + +pytest 5.4.1 (2020-03-13) +========================= + +Bug Fixes +--------- + +- :issue:`6909`: Revert the change introduced by :pull:`6330`, which required all arguments to ``@pytest.mark.parametrize`` to be explicitly defined in the function signature. + + The intention of the original change was to remove what was expected to be an unintended/surprising behavior, but it turns out many people relied on it, so the restriction has been reverted. + + +- :issue:`6910`: Fix crash when plugins return an unknown stats while using the ``--reportlog`` option. + + +pytest 5.4.0 (2020-03-12) +========================= + +Breaking Changes +---------------- + +- :issue:`6316`: Matching of ``-k EXPRESSION`` to test names is now case-insensitive. + + +- :issue:`6443`: Plugins specified with ``-p`` are now loaded after internal plugins, which results in their hooks being called *before* the internal ones. + + This makes the ``-p`` behavior consistent with ``PYTEST_PLUGINS``. + + +- :issue:`6637`: Removed the long-deprecated ``pytest_itemstart`` hook. + + This hook has been marked as deprecated and not been even called by pytest for over 10 years now. + + +- :issue:`6673`: Reversed / fix meaning of "+/-" in error diffs. "-" means that sth. expected is missing in the result and "+" means that there are unexpected extras in the result. + + +- :issue:`6737`: The ``cached_result`` attribute of ``FixtureDef`` is now set to ``None`` when + the result is unavailable, instead of being deleted. + + If your plugin performs checks like ``hasattr(fixturedef, 'cached_result')``, + for example in a ``pytest_fixture_post_finalizer`` hook implementation, replace + it with ``fixturedef.cached_result is not None``. If you ``del`` the attribute, + set it to ``None`` instead. + + + +Deprecations +------------ + +- :issue:`3238`: Option ``--no-print-logs`` is deprecated and meant to be removed in a future release. If you use ``--no-print-logs``, please try out ``--show-capture`` and + provide feedback. + + ``--show-capture`` command-line option was added in ``pytest 3.5.0`` and allows to specify how to + display captured output when tests fail: ``no``, ``stdout``, ``stderr``, ``log`` or ``all`` (the default). + + +- :issue:`571`: Deprecate the unused/broken `pytest_collect_directory` hook. + It was misaligned since the removal of the ``Directory`` collector in 2010 + and incorrect/unusable as soon as collection was split from test execution. + + +- :issue:`5975`: Deprecate using direct constructors for ``Nodes``. + + Instead they are now constructed via ``Node.from_parent``. + + This transitional mechanism enables us to untangle the very intensely + entangled ``Node`` relationships by enforcing more controlled creation/configuration patterns. + + As part of this change, session/config are already disallowed parameters and as we work on the details we might need disallow a few more as well. + + Subclasses are expected to use `super().from_parent` if they intend to expand the creation of `Nodes`. + + +- :issue:`6779`: The ``TerminalReporter.writer`` attribute has been deprecated and should no longer be used. This + was inadvertently exposed as part of the public API of that plugin and ties it too much + with ``py.io.TerminalWriter``. + + + +Features +-------- + +- :issue:`4597`: New :ref:`--capture=tee-sys ` option to allow both live printing and capturing of test output. + + +- :issue:`5712`: Now all arguments to ``@pytest.mark.parametrize`` need to be explicitly declared in the function signature or via ``indirect``. + Previously it was possible to omit an argument if a fixture with the same name existed, which was just an accident of implementation and was not meant to be a part of the API. + + +- :issue:`6454`: Changed default for `-r` to `fE`, which displays failures and errors in the :ref:`short test summary `. `-rN` can be used to disable it (the old behavior). + + +- :issue:`6469`: New options have been added to the :confval:`junit_logging` option: ``log``, ``out-err``, and ``all``. + + +- :issue:`6834`: Excess warning summaries are now collapsed per file to ensure readable display of warning summaries. + + + +Improvements +------------ + +- :issue:`1857`: ``pytest.mark.parametrize`` accepts integers for ``ids`` again, converting it to strings. + + +- :issue:`449`: Use "yellow" main color with any XPASSED tests. + + +- :issue:`4639`: Revert "A warning is now issued when assertions are made for ``None``". + + The warning proved to be less useful than initially expected and had quite a + few false positive cases. + + +- :issue:`5686`: ``tmpdir_factory.mktemp`` now fails when given absolute and non-normalized paths. + + +- :issue:`5984`: The ``pytest_warning_captured`` hook now receives a ``location`` parameter with the code location that generated the warning. + + +- :issue:`6213`: pytester: the ``testdir`` fixture respects environment settings from the ``monkeypatch`` fixture for inner runs. + + +- :issue:`6247`: ``--fulltrace`` is honored with collection errors. + + +- :issue:`6384`: Make `--showlocals` work also with `--tb=short`. + + +- :issue:`6653`: Add support for matching lines consecutively with :attr:`LineMatcher <_pytest.pytester.LineMatcher>`'s :func:`~_pytest.pytester.LineMatcher.fnmatch_lines` and :func:`~_pytest.pytester.LineMatcher.re_match_lines`. + + +- :issue:`6658`: Code is now highlighted in tracebacks when ``pygments`` is installed. + + Users are encouraged to install ``pygments`` into their environment and provide feedback, because + the plan is to make ``pygments`` a regular dependency in the future. + + +- :issue:`6795`: Import usage error message with invalid `-o` option. + + +- :issue:`759`: ``pytest.mark.parametrize`` supports iterators and generators for ``ids``. + + + +Bug Fixes +--------- + +- :issue:`310`: Add support for calling `pytest.xfail()` and `pytest.importorskip()` with doctests. + + +- :issue:`3823`: ``--trace`` now works with unittests. + + +- :issue:`4445`: Fixed some warning reports produced by pytest to point to the correct location of the warning in the user's code. + + +- :issue:`5301`: Fix ``--last-failed`` to collect new tests from files with known failures. + + +- :issue:`5928`: Report ``PytestUnknownMarkWarning`` at the level of the user's code, not ``pytest``'s. + + +- :issue:`5991`: Fix interaction with ``--pdb`` and unittests: do not use unittest's ``TestCase.debug()``. + + +- :issue:`6334`: Fix summary entries appearing twice when ``f/F`` and ``s/S`` report chars were used at the same time in the ``-r`` command-line option (for example ``-rFf``). + + The upper case variants were never documented and the preferred form should be the lower case. + + +- :issue:`6409`: Fallback to green (instead of yellow) for non-last items without previous passes with colored terminal progress indicator. + + +- :issue:`6454`: `--disable-warnings` is honored with `-ra` and `-rA`. + + +- :issue:`6497`: Fix bug in the comparison of request key with cached key in fixture. + + A construct ``if key == cached_key:`` can fail either because ``==`` is explicitly disallowed, or for, e.g., NumPy arrays, where the result of ``a == b`` cannot generally be converted to :class:`bool`. + The implemented fix replaces `==` with ``is``. + + +- :issue:`6557`: Make capture output streams ``.write()`` method return the same return value from original streams. + + +- :issue:`6566`: Fix ``EncodedFile.writelines`` to call the underlying buffer's ``writelines`` method. + + +- :issue:`6575`: Fix internal crash when ``faulthandler`` starts initialized + (for example with ``PYTHONFAULTHANDLER=1`` environment variable set) and ``faulthandler_timeout`` defined + in the configuration file. + + +- :issue:`6597`: Fix node ids which contain a parametrized empty-string variable. + + +- :issue:`6646`: Assertion rewriting hooks are (re)stored for the current item, which fixes them being still used after e.g. pytester's :func:`testdir.runpytest <_pytest.pytester.Testdir.runpytest>` etc. + + +- :issue:`6660`: :py:func:`pytest.exit` is handled when emitted from the :hook:`pytest_sessionfinish` hook. This includes quitting from a debugger. + + +- :issue:`6752`: When :py:func:`pytest.raises` is used as a function (as opposed to a context manager), + a `match` keyword argument is now passed through to the tested function. Previously + it was swallowed and ignored (regression in pytest 5.1.0). + + +- :issue:`6801`: Do not display empty lines in between traceback for unexpected exceptions with doctests. + + +- :issue:`6802`: The :fixture:`testdir fixture ` works within doctests now. + + + +Improved Documentation +---------------------- + +- :issue:`6696`: Add list of fixtures to start of fixture chapter. + + +- :issue:`6742`: Expand first sentence on fixtures into a paragraph. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`6404`: Remove usage of ``parser`` module, deprecated in Python 3.9. + + +pytest 5.3.5 (2020-01-29) +========================= + +Bug Fixes +--------- + +- :issue:`6517`: Fix regression in pytest 5.3.4 causing an INTERNALERROR due to a wrong assertion. + + +pytest 5.3.4 (2020-01-20) +========================= + +Bug Fixes +--------- + +- :issue:`6496`: Revert :issue:`6436`: unfortunately this change has caused a number of regressions in many suites, + so the team decided to revert this change and make a new release while we continue to look for a solution. + + +pytest 5.3.3 (2020-01-16) +========================= + +Bug Fixes +--------- + +- :issue:`2780`: Captured output during teardown is shown with ``-rP``. + + +- :issue:`5971`: Fix a ``pytest-xdist`` crash when dealing with exceptions raised in subprocesses created by the + ``multiprocessing`` module. + + +- :issue:`6436`: :class:`FixtureDef <_pytest.fixtures.FixtureDef>` objects now properly register their finalizers with autouse and + parameterized fixtures that execute before them in the fixture stack so they are torn + down at the right times, and in the right order. + + +- :issue:`6532`: Fix parsing of outcomes containing multiple errors with ``testdir`` results (regression in 5.3.0). + + + +Trivial/Internal Changes +------------------------ + +- :issue:`6350`: Optimized automatic renaming of test parameter IDs. + + +pytest 5.3.2 (2019-12-13) +========================= + +Improvements +------------ + +- :issue:`4639`: Revert "A warning is now issued when assertions are made for ``None``". + + The warning proved to be less useful than initially expected and had quite a + few false positive cases. + + + +Bug Fixes +--------- + +- :issue:`5430`: junitxml: Logs for failed test are now passed to junit report in case the test fails during call phase. + + +- :issue:`6290`: The supporting files in the ``.pytest_cache`` directory are kept with ``--cache-clear``, which only clears cached values now. + + +- :issue:`6301`: Fix assertion rewriting for egg-based distributions and ``editable`` installs (``pip install --editable``). + + +pytest 5.3.1 (2019-11-25) +========================= + +Improvements +------------ + +- :issue:`6231`: Improve check for misspelling of :ref:`pytest.mark.parametrize ref`. + + +- :issue:`6257`: Handle :func:`pytest.exit` being used via :hook:`pytest_internalerror`, e.g. when quitting pdb from post mortem. + + + +Bug Fixes +--------- + +- :issue:`5914`: pytester: fix :py:func:`~_pytest.pytester.LineMatcher.no_fnmatch_line` when used after positive matching. + + +- :issue:`6082`: Fix line detection for doctest samples inside :py:class:`python:property` docstrings, as a workaround to :bpo:`17446`. + + +- :issue:`6254`: Fix compatibility with pytest-parallel (regression in pytest 5.3.0). + + +- :issue:`6255`: Clear the :py:data:`sys.last_traceback`, :py:data:`sys.last_type` + and :py:data:`sys.last_value` attributes by deleting them instead + of setting them to ``None``. This better matches the behaviour of + the Python standard library. + + +pytest 5.3.0 (2019-11-19) +========================= + +Deprecations +------------ + +- :issue:`6179`: The default value of :confval:`junit_family` option will change to ``"xunit2"`` in pytest 6.0, given + that this is the version supported by default in modern tools that manipulate this type of file. + + In order to smooth the transition, pytest will issue a warning in case the ``--junitxml`` option + is given in the command line but :confval:`junit_family` is not explicitly configured in ``pytest.ini``. + + For more information, :ref:`see the docs `. + + + +Features +-------- + +- :issue:`4488`: The pytest team has created the `pytest-reportlog `__ + plugin, which provides a new ``--report-log=FILE`` option that writes *report logs* into a file as the test session executes. + + Each line of the report log contains a self contained JSON object corresponding to a testing event, + such as a collection or a test result report. The file is guaranteed to be flushed after writing + each line, so systems can read and process events in real-time. + + The plugin is meant to replace the ``--resultlog`` option, which is deprecated and meant to be removed + in a future release. If you use ``--resultlog``, please try out ``pytest-reportlog`` and + provide feedback. + + +- :issue:`4730`: When :py:data:`sys.pycache_prefix` (Python 3.8+) is set, it will be used by pytest to cache test files changed by the assertion rewriting mechanism. + + This makes it easier to benefit of cached ``.pyc`` files even on file systems without permissions. + + +- :issue:`5515`: Allow selective auto-indentation of multiline log messages. + + Adds command line option ``--log-auto-indent``, config option + :confval:`log_auto_indent` and support for per-entry configuration of + indentation behavior on calls to :py:func:`python:logging.log()`. + + Alters the default for auto-indention from ``"on"`` to ``"off"``. This + restores the older behavior that existed prior to v4.6.0. This + reversion to earlier behavior was done because it is better to + activate new features that may lead to broken tests explicitly + rather than implicitly. + + +- :issue:`5914`: :fixture:`testdir` learned two new functions, :py:func:`~_pytest.pytester.LineMatcher.no_fnmatch_line` and + :py:func:`~_pytest.pytester.LineMatcher.no_re_match_line`. + + The functions are used to ensure the captured text *does not* match the given + pattern. + + The previous idiom was to use :py:func:`python:re.match`: + + .. code-block:: python + + result = testdir.runpytest() + assert re.match(pat, result.stdout.str()) is None + + Or the ``in`` operator: + + .. code-block:: python + + result = testdir.runpytest() + assert text in result.stdout.str() + + But the new functions produce best output on failure. + + +- :issue:`6057`: Added tolerances to complex values when printing ``pytest.approx``. + + For example, ``repr(pytest.approx(3+4j))`` returns ``(3+4j) ± 5e-06 ∠ ±180°``. This is polar notation indicating a circle around the expected value, with a radius of 5e-06. For ``approx`` comparisons to return ``True``, the actual value should fall within this circle. + + +- :issue:`6061`: Added the pluginmanager as an argument to ``pytest_addoption`` + so that hooks can be invoked when setting up command line options. This is + useful for having one plugin communicate things to another plugin, + such as default values or which set of command line options to add. + + + +Improvements +------------ + +- :issue:`5061`: Use multiple colors with terminal summary statistics. + + +- :issue:`5630`: Quitting from debuggers is now properly handled in ``doctest`` items. + + +- :issue:`5924`: Improved verbose diff output with sequences. + + Before: + + :: + + E AssertionError: assert ['version', '...version_info'] == ['version', '...version', ...] + E Right contains 3 more items, first extra item: ' ' + E Full diff: + E - ['version', 'version_info', 'sys.version', 'sys.version_info'] + E + ['version', + E + 'version_info', + E + 'sys.version', + E + 'sys.version_info', + E + ' ', + E + 'sys.version', + E + 'sys.version_info'] + + After: + + :: + + E AssertionError: assert ['version', '...version_info'] == ['version', '...version', ...] + E Right contains 3 more items, first extra item: ' ' + E Full diff: + E [ + E 'version', + E 'version_info', + E 'sys.version', + E 'sys.version_info', + E + ' ', + E + 'sys.version', + E + 'sys.version_info', + E ] + + +- :issue:`5934`: ``repr`` of ``ExceptionInfo`` objects has been improved to honor the ``__repr__`` method of the underlying exception. + +- :issue:`5936`: Display untruncated assertion message with ``-vv``. + + +- :issue:`5990`: Fixed plurality mismatch in test summary (e.g. display "1 error" instead of "1 errors"). + + +- :issue:`6008`: ``Config.InvocationParams.args`` is now always a ``tuple`` to better convey that it should be + immutable and avoid accidental modifications. + + +- :issue:`6023`: ``pytest.main`` returns a ``pytest.ExitCode`` instance now, except for when custom exit codes are used (where it returns ``int`` then still). + + +- :issue:`6026`: Align prefixes in output of pytester's ``LineMatcher``. + + +- :issue:`6059`: Collection errors are reported as errors (and not failures like before) in the terminal's short test summary. + + +- :issue:`6069`: ``pytester.spawn`` does not skip/xfail tests on FreeBSD anymore unconditionally. + + +- :issue:`6097`: The "[...%]" indicator in the test summary is now colored according to the final (new) multi-colored line's main color. + + +- :issue:`6116`: Added ``--co`` as a synonym to ``--collect-only``. + + +- :issue:`6148`: ``atomicwrites`` is now only used on Windows, fixing a performance regression with assertion rewriting on Unix. + + +- :issue:`6152`: Now parametrization will use the ``__name__`` attribute of any object for the id, if present. Previously it would only use ``__name__`` for functions and classes. + + +- :issue:`6176`: Improved failure reporting with pytester's ``Hookrecorder.assertoutcome``. + + +- :issue:`6181`: The reason for a stopped session, e.g. with ``--maxfail`` / ``-x``, now gets reported in the test summary. + + +- :issue:`6206`: Improved ``cache.set`` robustness and performance. + + + +Bug Fixes +--------- + +- :issue:`2049`: Fixed ``--setup-plan`` showing inaccurate information about fixture lifetimes. + + +- :issue:`2548`: Fixed line offset mismatch of skipped tests in terminal summary. + + +- :issue:`6039`: The ``PytestDoctestRunner`` is now properly invalidated when unconfiguring the doctest plugin. + + This is important when used with ``pytester``'s ``runpytest_inprocess``. + + +- :issue:`6047`: BaseExceptions are now handled in ``saferepr``, which includes ``pytest.fail.Exception`` etc. + + +- :issue:`6074`: pytester: fixed order of arguments in ``rm_rf`` warning when cleaning up temporary directories, and do not emit warnings for errors with ``os.open``. + + +- :issue:`6189`: Fixed result of ``getmodpath`` method. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`4901`: ``RunResult`` from ``pytester`` now displays the mnemonic of the ``ret`` attribute when it is a + valid ``pytest.ExitCode`` value. + + +pytest 5.2.4 (2019-11-15) +========================= + +Bug Fixes +--------- + +- :issue:`6194`: Fix incorrect discovery of non-test ``__init__.py`` files. + + +- :issue:`6197`: Revert "The first test in a package (``__init__.py``) marked with ``@pytest.mark.skip`` is now correctly skipped.". + + +pytest 5.2.3 (2019-11-14) +========================= + +Bug Fixes +--------- + +- :issue:`5830`: The first test in a package (``__init__.py``) marked with ``@pytest.mark.skip`` is now correctly skipped. + + +- :issue:`6099`: Fix ``--trace`` when used with parametrized functions. + + +- :issue:`6183`: Using ``request`` as a parameter name in ``@pytest.mark.parametrize`` now produces a more + user-friendly error. + + +pytest 5.2.2 (2019-10-24) +========================= + +Bug Fixes +--------- + +- :issue:`5206`: Fix ``--nf`` to not forget about known nodeids with partial test selection. + + +- :issue:`5906`: Fix crash with ``KeyboardInterrupt`` during ``--setup-show``. + + +- :issue:`5946`: Fixed issue when parametrizing fixtures with numpy arrays (and possibly other sequence-like types). + + +- :issue:`6044`: Properly ignore ``FileNotFoundError`` exceptions when trying to remove old temporary directories, + for instance when multiple processes try to remove the same directory (common with ``pytest-xdist`` + for example). + + +pytest 5.2.1 (2019-10-06) +========================= + +Bug Fixes +--------- + +- :issue:`5902`: Fix warnings about deprecated ``cmp`` attribute in ``attrs>=19.2``. + + +pytest 5.2.0 (2019-09-28) +========================= + +Deprecations +------------ + +- :issue:`1682`: Passing arguments to pytest.fixture() as positional arguments is deprecated - pass them + as a keyword argument instead. + + + +Features +-------- + +- :issue:`1682`: The ``scope`` parameter of ``@pytest.fixture`` can now be a callable that receives + the fixture name and the ``config`` object as keyword-only parameters. + See :ref:`the docs ` for more information. + + +- :issue:`5764`: New behavior of the ``--pastebin`` option: failures to connect to the pastebin server are reported, without failing the pytest run + + + +Bug Fixes +--------- + +- :issue:`5806`: Fix "lexer" being used when uploading to bpaste.net from ``--pastebin`` to "text". + + +- :issue:`5884`: Fix ``--setup-only`` and ``--setup-show`` for custom pytest items. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`5056`: The HelpFormatter uses ``py.io.get_terminal_width`` for better width detection. + + +pytest 5.1.3 (2019-09-18) +========================= + +Bug Fixes +--------- + +- :issue:`5807`: Fix pypy3.6 (nightly) on windows. + + +- :issue:`5811`: Handle ``--fulltrace`` correctly with ``pytest.raises``. + + +- :issue:`5819`: Windows: Fix regression with conftest whose qualified name contains uppercase + characters (introduced by #5792). + + +pytest 5.1.2 (2019-08-30) +========================= + +Bug Fixes +--------- + +- :issue:`2270`: Fixed ``self`` reference in function-scoped fixtures defined plugin classes: previously ``self`` + would be a reference to a *test* class, not the *plugin* class. + + +- :issue:`570`: Fixed long standing issue where fixture scope was not respected when indirect fixtures were used during + parametrization. + + +- :issue:`5782`: Fix decoding error when printing an error response from ``--pastebin``. + + +- :issue:`5786`: Chained exceptions in test and collection reports are now correctly serialized, allowing plugins like + ``pytest-xdist`` to display them properly. + + +- :issue:`5792`: Windows: Fix error that occurs in certain circumstances when loading + ``conftest.py`` from a working directory that has casing other than the one stored + in the filesystem (e.g., ``c:\test`` instead of ``C:\test``). + + +pytest 5.1.1 (2019-08-20) +========================= + +Bug Fixes +--------- + +- :issue:`5751`: Fixed ``TypeError`` when importing pytest on Python 3.5.0 and 3.5.1. + + +pytest 5.1.0 (2019-08-15) +========================= + +Removals +-------- + +- :issue:`5180`: As per our policy, the following features have been deprecated in the 4.X series and are now + removed: + + * ``Request.getfuncargvalue``: use ``Request.getfixturevalue`` instead. + + * ``pytest.raises`` and ``pytest.warns`` no longer support strings as the second argument. + + * ``message`` parameter of ``pytest.raises``. + + * ``pytest.raises``, ``pytest.warns`` and ``ParameterSet.param`` now use native keyword-only + syntax. This might change the exception message from previous versions, but they still raise + ``TypeError`` on unknown keyword arguments as before. + + * ``pytest.config`` global variable. + + * ``tmpdir_factory.ensuretemp`` method. + + * ``pytest_logwarning`` hook. + + * ``RemovedInPytest4Warning`` warning type. + + * ``request`` is now a reserved name for fixtures. + + + For more information consult :std:doc:`deprecations` in the docs. + + +- :issue:`5565`: Removed unused support code for :pypi:`unittest2`. + + The ``unittest2`` backport module is no longer + necessary since Python 3.3+, and the small amount of code in pytest to support it also doesn't seem + to be used: after removed, all tests still pass unchanged. + + Although our policy is to introduce a deprecation period before removing any features or support + for third party libraries, because this code is apparently not used + at all (even if ``unittest2`` is used by a test suite executed by pytest), it was decided to + remove it in this release. + + If you experience a regression because of this, please :issue:`file an issue `. + + +- :issue:`5615`: ``pytest.fail``, ``pytest.xfail`` and ``pytest.skip`` no longer support bytes for the message argument. + + This was supported for Python 2 where it was tempting to use ``"message"`` + instead of ``u"message"``. + + Python 3 code is unlikely to pass ``bytes`` to these functions. If you do, + please decode it to an ``str`` beforehand. + + + +Features +-------- + +- :issue:`5564`: New ``Config.invocation_args`` attribute containing the unchanged arguments passed to ``pytest.main()``. + + +- :issue:`5576`: New :ref:`NUMBER ` + option for doctests to ignore irrelevant differences in floating-point numbers. + Inspired by Sébastien Boisgérault's `numtest `__ + extension for doctest. + + + +Improvements +------------ + +- :issue:`5471`: JUnit XML now includes a timestamp and hostname in the testsuite tag. + + +- :issue:`5707`: Time taken to run the test suite now includes a human-readable representation when it takes over + 60 seconds, for example:: + + ===== 2 failed in 102.70s (0:01:42) ===== + + + +Bug Fixes +--------- + +- :issue:`4344`: Fix RuntimeError/StopIteration when trying to collect package with "__init__.py" only. + + +- :issue:`5115`: Warnings issued during ``pytest_configure`` are explicitly not treated as errors, even if configured as such, because it otherwise completely breaks pytest. + + +- :issue:`5477`: The XML file produced by ``--junitxml`` now correctly contain a ```` root element. + + +- :issue:`5524`: Fix issue where ``tmp_path`` and ``tmpdir`` would not remove directories containing files marked as read-only, + which could lead to pytest crashing when executed a second time with the ``--basetemp`` option. + + +- :issue:`5537`: Replace ``importlib_metadata`` backport with ``importlib.metadata`` from the + standard library on Python 3.8+. + + +- :issue:`5578`: Improve type checking for some exception-raising functions (``pytest.xfail``, ``pytest.skip``, etc) + so they provide better error messages when users meant to use marks (for example ``@pytest.xfail`` + instead of ``@pytest.mark.xfail``). + + +- :issue:`5606`: Fixed internal error when test functions were patched with objects that cannot be compared + for truth values against others, like ``numpy`` arrays. + + +- :issue:`5634`: ``pytest.exit`` is now correctly handled in ``unittest`` cases. + This makes ``unittest`` cases handle ``quit`` from pytest's pdb correctly. + + +- :issue:`5650`: Improved output when parsing an ini configuration file fails. + + +- :issue:`5701`: Fix collection of ``staticmethod`` objects defined with ``functools.partial``. + + +- :issue:`5734`: Skip async generator test functions, and update the warning message to refer to ``async def`` functions. + + + +Improved Documentation +---------------------- + +- :issue:`5669`: Add docstring for ``Testdir.copy_example``. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`5095`: XML files of the ``xunit2`` family are now validated against the schema by pytest's own test suite + to avoid future regressions. + + +- :issue:`5516`: Cache node splitting function which can improve collection performance in very large test suites. + + +- :issue:`5603`: Simplified internal ``SafeRepr`` class and removed some dead code. + + +- :issue:`5664`: When invoking pytest's own testsuite with ``PYTHONDONTWRITEBYTECODE=1``, + the ``test_xfail_handling`` test no longer fails. + + +- :issue:`5684`: Replace manual handling of ``OSError.errno`` in the codebase by new ``OSError`` subclasses (``PermissionError``, ``FileNotFoundError``, etc.). + + +pytest 5.0.1 (2019-07-04) +========================= + +Bug Fixes +--------- + +- :issue:`5479`: Improve quoting in ``raises`` match failure message. + + +- :issue:`5523`: Fixed using multiple short options together in the command-line (for example ``-vs``) in Python 3.8+. + + +- :issue:`5547`: ``--step-wise`` now handles ``xfail(strict=True)`` markers properly. + + + +Improved Documentation +---------------------- + +- :issue:`5517`: Improve "Declaring new hooks" section in chapter "Writing Plugins" + + +pytest 5.0.0 (2019-06-28) +========================= + +Important +--------- + +This release is a Python3.5+ only release. + +For more details, see our :std:doc:`Python 2.7 and 3.4 support plan `. + +Removals +-------- + +- :issue:`1149`: Pytest no longer accepts prefixes of command-line arguments, for example + typing ``pytest --doctest-mod`` inplace of ``--doctest-modules``. + This was previously allowed where the ``ArgumentParser`` thought it was unambiguous, + but this could be incorrect due to delayed parsing of options for plugins. + See for example issues :issue:`1149`, + :issue:`3413`, and + :issue:`4009`. + + +- :issue:`5402`: **PytestDeprecationWarning are now errors by default.** + + Following our plan to remove deprecated features with as little disruption as + possible, all warnings of type ``PytestDeprecationWarning`` now generate errors + instead of warning messages. + + **The affected features will be effectively removed in pytest 5.1**, so please consult the + :std:doc:`deprecations` section in the docs for directions on how to update existing code. + + In the pytest ``5.0.X`` series, it is possible to change the errors back into warnings as a stop + gap measure by adding this to your ``pytest.ini`` file: + + .. code-block:: ini + + [pytest] + filterwarnings = + ignore::pytest.PytestDeprecationWarning + + But this will stop working when pytest ``5.1`` is released. + + **If you have concerns** about the removal of a specific feature, please add a + comment to :issue:`5402`. + + +- :issue:`5412`: ``ExceptionInfo`` objects (returned by ``pytest.raises``) now have the same ``str`` representation as ``repr``, which + avoids some confusion when users use ``print(e)`` to inspect the object. + + This means code like: + + .. code-block:: python + + with pytest.raises(SomeException) as e: + ... + assert "some message" in str(e) + + + Needs to be changed to: + + .. code-block:: python + + with pytest.raises(SomeException) as e: + ... + assert "some message" in str(e.value) + + + + +Deprecations +------------ + +- :issue:`4488`: The removal of the ``--result-log`` option and module has been postponed to (tentatively) pytest 6.0 as + the team has not yet got around to implement a good alternative for it. + + +- :issue:`466`: The ``funcargnames`` attribute has been an alias for ``fixturenames`` since + pytest 2.3, and is now deprecated in code too. + + + +Features +-------- + +- :issue:`3457`: New :hook:`pytest_assertion_pass` + hook, called with context information when an assertion *passes*. + + This hook is still **experimental** so use it with caution. + + +- :issue:`5440`: The :mod:`faulthandler` standard library + module is now enabled by default to help users diagnose crashes in C modules. + + This functionality was provided by integrating the external + `pytest-faulthandler `__ plugin into the core, + so users should remove that plugin from their requirements if used. + + For more information see the docs: :ref:`faulthandler`. + + +- :issue:`5452`: When warnings are configured as errors, pytest warnings now appear as originating from ``pytest.`` instead of the internal ``_pytest.warning_types.`` module. + + +- :issue:`5125`: ``Session.exitcode`` values are now coded in ``pytest.ExitCode``, an ``IntEnum``. This makes the exit code available for consumer code and are more explicit other than just documentation. User defined exit codes are still valid, but should be used with caution. + + The team doesn't expect this change to break test suites or plugins in general, except in esoteric/specific scenarios. + + **pytest-xdist** users should upgrade to ``1.29.0`` or later, as ``pytest-xdist`` required a compatibility fix because of this change. + + + +Bug Fixes +--------- + +- :issue:`1403`: Switch from ``imp`` to ``importlib``. + + +- :issue:`1671`: The name of the ``.pyc`` files cached by the assertion writer now includes the pytest version + to avoid stale caches. + + +- :issue:`2761`: Honor :pep:`235` on case-insensitive file systems. + + +- :issue:`5078`: Test module is no longer double-imported when using ``--pyargs``. + + +- :issue:`5260`: Improved comparison of byte strings. + + When comparing bytes, the assertion message used to show the byte numeric value when showing the differences:: + + def test(): + > assert b'spam' == b'eggs' + E AssertionError: assert b'spam' == b'eggs' + E At index 0 diff: 115 != 101 + E Use -v to get the full diff + + It now shows the actual ascii representation instead, which is often more useful:: + + def test(): + > assert b'spam' == b'eggs' + E AssertionError: assert b'spam' == b'eggs' + E At index 0 diff: b's' != b'e' + E Use -v to get the full diff + + +- :issue:`5335`: Colorize level names when the level in the logging format is formatted using + '%(levelname).Xs' (truncated fixed width alignment), where X is an integer. + + +- :issue:`5354`: Fix ``pytest.mark.parametrize`` when the argvalues is an iterator. + + +- :issue:`5370`: Revert unrolling of ``all()`` to fix ``NameError`` on nested comprehensions. + + +- :issue:`5371`: Revert unrolling of ``all()`` to fix incorrect handling of generators with ``if``. + + +- :issue:`5372`: Revert unrolling of ``all()`` to fix incorrect assertion when using ``all()`` in an expression. + + +- :issue:`5383`: ``-q`` has again an impact on the style of the collected items + (``--collect-only``) when ``--log-cli-level`` is used. + + +- :issue:`5389`: Fix regressions of :pull:`5063` for ``importlib_metadata.PathDistribution`` which have their ``files`` attribute being ``None``. + + +- :issue:`5390`: Fix regression where the ``obj`` attribute of ``TestCase`` items was no longer bound to methods. + + +- :issue:`5404`: Emit a warning when attempting to unwrap a broken object raises an exception, + for easier debugging (:issue:`5080`). + + +- :issue:`5432`: Prevent "already imported" warnings from assertion rewriter when invoking pytest in-process multiple times. + + +- :issue:`5433`: Fix assertion rewriting in packages (``__init__.py``). + + +- :issue:`5444`: Fix ``--stepwise`` mode when the first file passed on the command-line fails to collect. + + +- :issue:`5482`: Fix bug introduced in 4.6.0 causing collection errors when passing + more than 2 positional arguments to ``pytest.mark.parametrize``. + + +- :issue:`5505`: Fix crash when discovery fails while using ``-p no:terminal``. + + + +Improved Documentation +---------------------- + +- :issue:`5315`: Expand docs on mocking classes and dictionaries with ``monkeypatch``. + + +- :issue:`5416`: Fix PytestUnknownMarkWarning in run/skip example. + + +pytest 4.6.11 (2020-06-04) +========================== + +Bug Fixes +--------- + +- :issue:`6334`: Fix summary entries appearing twice when ``f/F`` and ``s/S`` report chars were used at the same time in the ``-r`` command-line option (for example ``-rFf``). + + The upper case variants were never documented and the preferred form should be the lower case. + + +- :issue:`7310`: Fix ``UnboundLocalError: local variable 'letter' referenced before + assignment`` in ``_pytest.terminal.pytest_report_teststatus()`` + when plugins return report objects in an unconventional state. + + This was making ``pytest_report_teststatus()`` skip + entering if-block branches that declare the ``letter`` variable. + + The fix was to set the initial value of the ``letter`` before + the if-block cascade so that it always has a value. + + +pytest 4.6.10 (2020-05-08) +========================== + +Features +-------- + +- :issue:`6870`: New ``Config.invocation_args`` attribute containing the unchanged arguments passed to ``pytest.main()``. + + Remark: while this is technically a new feature and according to our :ref:`policy ` it should not have been backported, we have opened an exception in this particular case because it fixes a serious interaction with ``pytest-xdist``, so it can also be considered a bugfix. + +Trivial/Internal Changes +------------------------ + +- :issue:`6404`: Remove usage of ``parser`` module, deprecated in Python 3.9. + + +pytest 4.6.9 (2020-01-04) +========================= + +Bug Fixes +--------- + +- :issue:`6301`: Fix assertion rewriting for egg-based distributions and ``editable`` installs (``pip install --editable``). + + +pytest 4.6.8 (2019-12-19) +========================= + +Features +-------- + +- :issue:`5471`: JUnit XML now includes a timestamp and hostname in the testsuite tag. + + + +Bug Fixes +--------- + +- :issue:`5430`: junitxml: Logs for failed test are now passed to junit report in case the test fails during call phase. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`6345`: Pin ``colorama`` to ``0.4.1`` only for Python 3.4 so newer Python versions can still receive colorama updates. + + +pytest 4.6.7 (2019-12-05) +========================= + +Bug Fixes +--------- + +- :issue:`5477`: The XML file produced by ``--junitxml`` now correctly contain a ```` root element. + + +- :issue:`6044`: Properly ignore ``FileNotFoundError`` (``OSError.errno == NOENT`` in Python 2) exceptions when trying to remove old temporary directories, + for instance when multiple processes try to remove the same directory (common with ``pytest-xdist`` + for example). + + +pytest 4.6.6 (2019-10-11) +========================= + +Bug Fixes +--------- + +- :issue:`5523`: Fixed using multiple short options together in the command-line (for example ``-vs``) in Python 3.8+. + + +- :issue:`5537`: Replace ``importlib_metadata`` backport with ``importlib.metadata`` from the + standard library on Python 3.8+. + + +- :issue:`5806`: Fix "lexer" being used when uploading to bpaste.net from ``--pastebin`` to "text". + + +- :issue:`5902`: Fix warnings about deprecated ``cmp`` attribute in ``attrs>=19.2``. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`5801`: Fixes python version checks (detected by ``flake8-2020``) in case python4 becomes a thing. + + +pytest 4.6.5 (2019-08-05) +========================= + +Bug Fixes +--------- + +- :issue:`4344`: Fix RuntimeError/StopIteration when trying to collect package with "__init__.py" only. + + +- :issue:`5478`: Fix encode error when using unicode strings in exceptions with ``pytest.raises``. + + +- :issue:`5524`: Fix issue where ``tmp_path`` and ``tmpdir`` would not remove directories containing files marked as read-only, + which could lead to pytest crashing when executed a second time with the ``--basetemp`` option. + + +- :issue:`5547`: ``--step-wise`` now handles ``xfail(strict=True)`` markers properly. + + +- :issue:`5650`: Improved output when parsing an ini configuration file fails. + +pytest 4.6.4 (2019-06-28) +========================= + +Bug Fixes +--------- + +- :issue:`5404`: Emit a warning when attempting to unwrap a broken object raises an exception, + for easier debugging (:issue:`5080`). + + +- :issue:`5444`: Fix ``--stepwise`` mode when the first file passed on the command-line fails to collect. + + +- :issue:`5482`: Fix bug introduced in 4.6.0 causing collection errors when passing + more than 2 positional arguments to ``pytest.mark.parametrize``. + + +- :issue:`5505`: Fix crash when discovery fails while using ``-p no:terminal``. + + +pytest 4.6.3 (2019-06-11) +========================= + +Bug Fixes +--------- + +- :issue:`5383`: ``-q`` has again an impact on the style of the collected items + (``--collect-only``) when ``--log-cli-level`` is used. + + +- :issue:`5389`: Fix regressions of :pull:`5063` for ``importlib_metadata.PathDistribution`` which have their ``files`` attribute being ``None``. + + +- :issue:`5390`: Fix regression where the ``obj`` attribute of ``TestCase`` items was no longer bound to methods. + + +pytest 4.6.2 (2019-06-03) +========================= + +Bug Fixes +--------- + +- :issue:`5370`: Revert unrolling of ``all()`` to fix ``NameError`` on nested comprehensions. + + +- :issue:`5371`: Revert unrolling of ``all()`` to fix incorrect handling of generators with ``if``. + + +- :issue:`5372`: Revert unrolling of ``all()`` to fix incorrect assertion when using ``all()`` in an expression. + + +pytest 4.6.1 (2019-06-02) +========================= + +Bug Fixes +--------- + +- :issue:`5354`: Fix ``pytest.mark.parametrize`` when the argvalues is an iterator. + + +- :issue:`5358`: Fix assertion rewriting of ``all()`` calls to deal with non-generators. + + +pytest 4.6.0 (2019-05-31) +========================= + +Important +--------- + +The ``4.6.X`` series will be the last series to support **Python 2 and Python 3.4**. + +For more details, see our :std:doc:`Python 2.7 and 3.4 support plan `. + + +Features +-------- + +- :issue:`4559`: Added the ``junit_log_passing_tests`` ini value which can be used to enable or disable logging of passing test output in the Junit XML file. + + +- :issue:`4956`: pytester's ``testdir.spawn`` uses ``tmpdir`` as HOME/USERPROFILE directory. + + +- :issue:`5062`: Unroll calls to ``all`` to full for-loops with assertion rewriting for better failure messages, especially when using Generator Expressions. + + +- :issue:`5063`: Switch from ``pkg_resources`` to ``importlib-metadata`` for entrypoint detection for improved performance and import time. + + +- :issue:`5091`: The output for ini options in ``--help`` has been improved. + + +- :issue:`5269`: ``pytest.importorskip`` includes the ``ImportError`` now in the default ``reason``. + + +- :issue:`5311`: Captured logs that are output for each failing test are formatted using the + ColoredLevelFormatter. + + +- :issue:`5312`: Improved formatting of multiline log messages in Python 3. + + + +Bug Fixes +--------- + +- :issue:`2064`: The debugging plugin imports the wrapped ``Pdb`` class (``--pdbcls``) on-demand now. + + +- :issue:`4908`: The ``pytest_enter_pdb`` hook gets called with post-mortem (``--pdb``). + + +- :issue:`5036`: Fix issue where fixtures dependent on other parametrized fixtures would be erroneously parametrized. + + +- :issue:`5256`: Handle internal error due to a lone surrogate unicode character not being representable in Jython. + + +- :issue:`5257`: Ensure that ``sys.stdout.mode`` does not include ``'b'`` as it is a text stream. + + +- :issue:`5278`: Pytest's internal python plugin can be disabled using ``-p no:python`` again. + + +- :issue:`5286`: Fix issue with ``disable_test_id_escaping_and_forfeit_all_rights_to_community_support`` option not working when using a list of test IDs in parametrized tests. + + +- :issue:`5330`: Show the test module being collected when emitting ``PytestCollectionWarning`` messages for + test classes with ``__init__`` and ``__new__`` methods to make it easier to pin down the problem. + + +- :issue:`5333`: Fix regression in 4.5.0 with ``--lf`` not re-running all tests with known failures from non-selected tests. + + + +Improved Documentation +---------------------- + +- :issue:`5250`: Expand docs on use of ``setenv`` and ``delenv`` with ``monkeypatch``. + + +pytest 4.5.0 (2019-05-11) +========================= + +Features +-------- + +- :issue:`4826`: A warning is now emitted when unknown marks are used as a decorator. + This is often due to a typo, which can lead to silently broken tests. + + +- :issue:`4907`: Show XFail reason as part of JUnitXML message field. + + +- :issue:`5013`: Messages from crash reports are displayed within test summaries now, truncated to the terminal width. + + +- :issue:`5023`: New flag ``--strict-markers`` that triggers an error when unknown markers (e.g. those not registered using the :confval:`markers` option in the configuration file) are used in the test suite. + + The existing ``--strict`` option has the same behavior currently, but can be augmented in the future for additional checks. + + +- :issue:`5026`: Assertion failure messages for sequences and dicts contain the number of different items now. + + +- :issue:`5034`: Improve reporting with ``--lf`` and ``--ff`` (run-last-failure). + + +- :issue:`5035`: The ``--cache-show`` option/action accepts an optional glob to show only matching cache entries. + + +- :issue:`5059`: Standard input (stdin) can be given to pytester's ``Testdir.run()`` and ``Testdir.popen()``. + + +- :issue:`5068`: The ``-r`` option learnt about ``A`` to display all reports (including passed ones) in the short test summary. + + +- :issue:`5108`: The short test summary is displayed after passes with output (``-rP``). + + +- :issue:`5172`: The ``--last-failed`` (``--lf``) option got smarter and will now skip entire files if all tests + of that test file have passed in previous runs, greatly speeding up collection. + + +- :issue:`5177`: Introduce new specific warning ``PytestWarning`` subclasses to make it easier to filter warnings based on the class, rather than on the message. The new subclasses are: + + + * ``PytestAssertRewriteWarning`` + + * ``PytestCacheWarning`` + + * ``PytestCollectionWarning`` + + * ``PytestConfigWarning`` + + * ``PytestUnhandledCoroutineWarning`` + + * ``PytestUnknownMarkWarning`` + + +- :issue:`5202`: New ``record_testsuite_property`` session-scoped fixture allows users to log ```` tags at the ``testsuite`` + level with the ``junitxml`` plugin. + + The generated XML is compatible with the latest xunit standard, contrary to + the properties recorded by ``record_property`` and ``record_xml_attribute``. + + +- :issue:`5214`: The default logging format has been changed to improve readability. Here is an + example of a previous logging message:: + + test_log_cli_enabled_disabled.py 3 CRITICAL critical message logged by test + + This has now become:: + + CRITICAL root:test_log_cli_enabled_disabled.py:3 critical message logged by test + + The formatting can be changed through the :confval:`log_format` configuration option. + + +- :issue:`5220`: ``--fixtures`` now also shows fixture scope for scopes other than ``"function"``. + + + +Bug Fixes +--------- + +- :issue:`5113`: Deselected items from plugins using ``pytest_collect_modifyitems`` as a hookwrapper are correctly reported now. + + +- :issue:`5144`: With usage errors ``exitstatus`` is set to ``EXIT_USAGEERROR`` in the ``pytest_sessionfinish`` hook now as expected. + + +- :issue:`5235`: ``outcome.exit`` is not used with ``EOF`` in the pdb wrapper anymore, but only with ``quit``. + + + +Improved Documentation +---------------------- + +- :issue:`4935`: Expand docs on registering marks and the effect of ``--strict``. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`4942`: ``logging.raiseExceptions`` is not set to ``False`` anymore. + + +- :issue:`5013`: pytest now depends on :pypi:`wcwidth` to properly track unicode character sizes for more precise terminal output. + + +- :issue:`5059`: pytester's ``Testdir.popen()`` uses ``stdout`` and ``stderr`` via keyword arguments with defaults now (``subprocess.PIPE``). + + +- :issue:`5069`: The code for the short test summary in the terminal was moved to the terminal plugin. + + +- :issue:`5082`: Improved validation of kwargs for various methods in the pytester plugin. + + +- :issue:`5202`: ``record_property`` now emits a ``PytestWarning`` when used with ``junit_family=xunit2``: the fixture generates + ``property`` tags as children of ``testcase``, which is not permitted according to the most + `recent schema `__. + + +- :issue:`5239`: Pin ``pluggy`` to ``< 1.0`` so we don't update to ``1.0`` automatically when + it gets released: there are planned breaking changes, and we want to ensure + pytest properly supports ``pluggy 1.0``. + + +pytest 4.4.2 (2019-05-08) +========================= + +Bug Fixes +--------- + +- :issue:`5089`: Fix crash caused by error in ``__repr__`` function with both ``showlocals`` and verbose output enabled. + + +- :issue:`5139`: Eliminate core dependency on 'terminal' plugin. + + +- :issue:`5229`: Require ``pluggy>=0.11.0`` which reverts a dependency to ``importlib-metadata`` added in ``0.10.0``. + The ``importlib-metadata`` package cannot be imported when installed as an egg and causes issues when relying on ``setup.py`` to install test dependencies. + + + +Improved Documentation +---------------------- + +- :issue:`5171`: Doc: ``pytest_ignore_collect``, ``pytest_collect_directory``, ``pytest_collect_file`` and ``pytest_pycollect_makemodule`` hooks's 'path' parameter documented type is now ``py.path.local`` + + +- :issue:`5188`: Improve help for ``--runxfail`` flag. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`5182`: Removed internal and unused ``_pytest.deprecated.MARK_INFO_ATTRIBUTE``. + + +pytest 4.4.1 (2019-04-15) +========================= + +Bug Fixes +--------- + +- :issue:`5031`: Environment variables are properly restored when using pytester's ``testdir`` fixture. + + +- :issue:`5039`: Fix regression with ``--pdbcls``, which stopped working with local modules in 4.0.0. + + +- :issue:`5092`: Produce a warning when unknown keywords are passed to ``pytest.param(...)``. + + +- :issue:`5098`: Invalidate import caches with ``monkeypatch.syspath_prepend``, which is required with namespace packages being used. + + +pytest 4.4.0 (2019-03-29) +========================= + +Features +-------- + +- :issue:`2224`: ``async`` test functions are skipped and a warning is emitted when a suitable + async plugin is not installed (such as ``pytest-asyncio`` or ``pytest-trio``). + + Previously ``async`` functions would not execute at all but still be marked as "passed". + + +- :issue:`2482`: Include new ``disable_test_id_escaping_and_forfeit_all_rights_to_community_support`` option to disable ascii-escaping in parametrized values. This may cause a series of problems and as the name makes clear, use at your own risk. + + +- :issue:`4718`: The ``-p`` option can now be used to early-load plugins also by entry-point name, instead of just + by module name. + + This makes it possible to early load external plugins like ``pytest-cov`` in the command-line:: + + pytest -p pytest_cov + + +- :issue:`4855`: The ``--pdbcls`` option handles classes via module attributes now (e.g. + ``pdb:pdb.Pdb`` with :pypi:`pdbpp`), and its validation was improved. + + +- :issue:`4875`: The :confval:`testpaths` configuration option is now displayed next + to the ``rootdir`` and ``inifile`` lines in the pytest header if the option is in effect, i.e., directories or file names were + not explicitly passed in the command line. + + Also, ``inifile`` is only displayed if there's a configuration file, instead of an empty ``inifile:`` string. + + +- :issue:`4911`: Doctests can be skipped now dynamically using ``pytest.skip()``. + + +- :issue:`4920`: Internal refactorings have been made in order to make the implementation of the + `pytest-subtests `__ plugin + possible, which adds unittest sub-test support and a new ``subtests`` fixture as discussed in + :issue:`1367`. + + For details on the internal refactorings, please see the details on the related PR. + + +- :issue:`4931`: pytester's ``LineMatcher`` asserts that the passed lines are a sequence. + + +- :issue:`4936`: Handle ``-p plug`` after ``-p no:plug``. + + This can be used to override a blocked plugin (e.g. in "addopts") from the + command line etc. + + +- :issue:`4951`: Output capturing is handled correctly when only capturing via fixtures (capsys, capfs) with ``pdb.set_trace()``. + + +- :issue:`4956`: ``pytester`` sets ``$HOME`` and ``$USERPROFILE`` to the temporary directory during test runs. + + This ensures to not load configuration files from the real user's home directory. + + +- :issue:`4980`: Namespace packages are handled better with ``monkeypatch.syspath_prepend`` and ``testdir.syspathinsert`` (via ``pkg_resources.fixup_namespace_packages``). + + +- :issue:`4993`: The stepwise plugin reports status information now. + + +- :issue:`5008`: If a ``setup.cfg`` file contains ``[tool:pytest]`` and also the no longer supported ``[pytest]`` section, pytest will use ``[tool:pytest]`` ignoring ``[pytest]``. Previously it would unconditionally error out. + + This makes it simpler for plugins to support old pytest versions. + + + +Bug Fixes +--------- + +- :issue:`1895`: Fix bug where fixtures requested dynamically via ``request.getfixturevalue()`` might be teardown + before the requesting fixture. + + +- :issue:`4851`: pytester unsets ``PYTEST_ADDOPTS`` now to not use outer options with ``testdir.runpytest()``. + + +- :issue:`4903`: Use the correct modified time for years after 2038 in rewritten ``.pyc`` files. + + +- :issue:`4928`: Fix line offsets with ``ScopeMismatch`` errors. + + +- :issue:`4957`: ``-p no:plugin`` is handled correctly for default (internal) plugins now, e.g. with ``-p no:capture``. + + Previously they were loaded (imported) always, making e.g. the ``capfd`` fixture available. + + +- :issue:`4968`: The pdb ``quit`` command is handled properly when used after the ``debug`` command with :pypi:`pdbpp`. + + +- :issue:`4975`: Fix the interpretation of ``-qq`` option where it was being considered as ``-v`` instead. + + +- :issue:`4978`: ``outcomes.Exit`` is not swallowed in ``assertrepr_compare`` anymore. + + +- :issue:`4988`: Close logging's file handler explicitly when the session finishes. + + +- :issue:`5003`: Fix line offset with mark collection error (off by one). + + + +Improved Documentation +---------------------- + +- :issue:`4974`: Update docs for ``pytest_cmdline_parse`` hook to note availability liminations + + + +Trivial/Internal Changes +------------------------ + +- :issue:`4718`: ``pluggy>=0.9`` is now required. + + +- :issue:`4815`: ``funcsigs>=1.0`` is now required for Python 2.7. + + +- :issue:`4829`: Some left-over internal code related to ``yield`` tests has been removed. + + +- :issue:`4890`: Remove internally unused ``anypython`` fixture from the pytester plugin. + + +- :issue:`4912`: Remove deprecated Sphinx directive, ``add_description_unit()``, + pin sphinx-removed-in to >= 0.2.0 to support Sphinx 2.0. + + +- :issue:`4913`: Fix pytest tests invocation with custom ``PYTHONPATH``. + + +- :issue:`4965`: New ``pytest_report_to_serializable`` and ``pytest_report_from_serializable`` **experimental** hooks. + + These hooks will be used by ``pytest-xdist``, ``pytest-subtests``, and the replacement for + resultlog to serialize and customize reports. + + They are experimental, meaning that their details might change or even be removed + completely in future patch releases without warning. + + Feedback is welcome from plugin authors and users alike. + + +- :issue:`4987`: ``Collector.repr_failure`` respects the ``--tb`` option, but only defaults to ``short`` now (with ``auto``). + + +pytest 4.3.1 (2019-03-11) +========================= + +Bug Fixes +--------- + +- :issue:`4810`: Logging messages inside ``pytest_runtest_logreport()`` are now properly captured and displayed. + + +- :issue:`4861`: Improve validation of contents written to captured output so it behaves the same as when capture is disabled. + + +- :issue:`4898`: Fix ``AttributeError: FixtureRequest has no 'confg' attribute`` bug in ``testdir.copy_example``. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`4768`: Avoid pkg_resources import at the top-level. + + +pytest 4.3.0 (2019-02-16) +========================= + +Deprecations +------------ + +- :issue:`4724`: ``pytest.warns()`` now emits a warning when it receives unknown keyword arguments. + + This will be changed into an error in the future. + + + +Features +-------- + +- :issue:`2753`: Usage errors from argparse are mapped to pytest's ``UsageError``. + + +- :issue:`3711`: Add the ``--ignore-glob`` parameter to exclude test-modules with Unix shell-style wildcards. + Add the :globalvar:`collect_ignore_glob` for ``conftest.py`` to exclude test-modules with Unix shell-style wildcards. + + +- :issue:`4698`: The warning about Python 2.7 and 3.4 not being supported in pytest 5.0 has been removed. + + In the end it was considered to be more + of a nuisance than actual utility and users of those Python versions shouldn't have problems as ``pip`` will not + install pytest 5.0 on those interpreters. + + +- :issue:`4707`: With the help of new ``set_log_path()`` method there is a way to set ``log_file`` paths from hooks. + + + +Bug Fixes +--------- + +- :issue:`4651`: ``--help`` and ``--version`` are handled with ``UsageError``. + + +- :issue:`4782`: Fix ``AssertionError`` with collection of broken symlinks with packages. + + +pytest 4.2.1 (2019-02-12) +========================= + +Bug Fixes +--------- + +- :issue:`2895`: The ``pytest_report_collectionfinish`` hook now is also called with ``--collect-only``. + + +- :issue:`3899`: Do not raise ``UsageError`` when an imported package has a ``pytest_plugins.py`` child module. + + +- :issue:`4347`: Fix output capturing when using pdb++ with recursive debugging. + + +- :issue:`4592`: Fix handling of ``collect_ignore`` via parent ``conftest.py``. + + +- :issue:`4700`: Fix regression where ``setUpClass`` would always be called in subclasses even if all tests + were skipped by a ``unittest.skip()`` decorator applied in the subclass. + + +- :issue:`4739`: Fix ``parametrize(... ids=)`` when the function returns non-strings. + + +- :issue:`4745`: Fix/improve collection of args when passing in ``__init__.py`` and a test file. + + +- :issue:`4770`: ``more_itertools`` is now constrained to <6.0.0 when required for Python 2.7 compatibility. + + +- :issue:`526`: Fix "ValueError: Plugin already registered" exceptions when running in build directories that symlink to actual source. + + + +Improved Documentation +---------------------- + +- :issue:`3899`: Add note to ``plugins.rst`` that ``pytest_plugins`` should not be used as a name for a user module containing plugins. + + +- :issue:`4324`: Document how to use ``raises`` and ``does_not_raise`` to write parametrized tests with conditional raises. + + +- :issue:`4709`: Document how to customize test failure messages when using + ``pytest.warns``. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`4741`: Some verbosity related attributes of the TerminalReporter plugin are now + read only properties. + + +pytest 4.2.0 (2019-01-30) +========================= + +Features +-------- + +- :issue:`3094`: :doc:`Classic xunit-style ` functions and methods + now obey the scope of *autouse* fixtures. + + This fixes a number of surprising issues like ``setup_method`` being called before session-scoped + autouse fixtures (see :issue:`517` for an example). + + +- :issue:`4627`: Display a message at the end of the test session when running under Python 2.7 and 3.4 that pytest 5.0 will no longer + support those Python versions. + + +- :issue:`4660`: The number of *selected* tests now are also displayed when the ``-k`` or ``-m`` flags are used. + + +- :issue:`4688`: ``pytest_report_teststatus`` hook now can also receive a ``config`` parameter. + + +- :issue:`4691`: ``pytest_terminal_summary`` hook now can also receive a ``config`` parameter. + + + +Bug Fixes +--------- + +- :issue:`3547`: ``--junitxml`` can emit XML compatible with Jenkins xUnit. + ``junit_family`` INI option accepts ``legacy|xunit1``, which produces old style output, and ``xunit2`` that conforms more strictly to https://github.com/jenkinsci/xunit-plugin/blob/xunit-2.3.2/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd + + +- :issue:`4280`: Improve quitting from pdb, especially with ``--trace``. + + Using ``q[quit]`` after ``pdb.set_trace()`` will quit pytest also. + + +- :issue:`4402`: Warning summary now groups warnings by message instead of by test id. + + This makes the output more compact and better conveys the general idea of how much code is + actually generating warnings, instead of how many tests call that code. + + +- :issue:`4536`: ``monkeypatch.delattr`` handles class descriptors like ``staticmethod``/``classmethod``. + + +- :issue:`4649`: Restore marks being considered keywords for keyword expressions. + + +- :issue:`4653`: ``tmp_path`` fixture and other related ones provides resolved path (a.k.a real path) + + +- :issue:`4667`: ``pytest_terminal_summary`` uses result from ``pytest_report_teststatus`` hook, rather than hardcoded strings. + + +- :issue:`4669`: Correctly handle ``unittest.SkipTest`` exception containing non-ascii characters on Python 2. + + +- :issue:`4680`: Ensure the ``tmpdir`` and the ``tmp_path`` fixtures are the same folder. + + +- :issue:`4681`: Ensure ``tmp_path`` is always a real path. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`4643`: Use ``a.item()`` instead of the deprecated ``np.asscalar(a)`` in ``pytest.approx``. + + ``np.asscalar`` has been :doc:`deprecated ` in ``numpy 1.16.``. + + +- :issue:`4657`: Copy saferepr from pylib + + +- :issue:`4668`: The verbose word for expected failures in the teststatus report changes from ``xfail`` to ``XFAIL`` to be consistent with other test outcomes. + + +pytest 4.1.1 (2019-01-12) +========================= + +Bug Fixes +--------- + +- :issue:`2256`: Show full repr with ``assert a==b`` and ``-vv``. + + +- :issue:`3456`: Extend Doctest-modules to ignore mock objects. + + +- :issue:`4617`: Fixed ``pytest.warns`` bug when context manager is reused (e.g. multiple parametrization). + + +- :issue:`4631`: Don't rewrite assertion when ``__getattr__`` is broken + + + +Improved Documentation +---------------------- + +- :issue:`3375`: Document that using ``setup.cfg`` may crash other tools or cause hard to track down problems because it uses a different parser than ``pytest.ini`` or ``tox.ini`` files. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`4602`: Uninstall ``hypothesis`` in regen tox env. + + +pytest 4.1.0 (2019-01-05) +========================= + +Removals +-------- + +- :issue:`2169`: ``pytest.mark.parametrize``: in previous versions, errors raised by id functions were suppressed and changed into warnings. Now the exceptions are propagated, along with a pytest message informing the node, parameter value and index where the exception occurred. + + +- :issue:`3078`: Remove legacy internal warnings system: ``config.warn``, ``Node.warn``. The ``pytest_logwarning`` now issues a warning when implemented. + + See our :ref:`docs ` on information on how to update your code. + + +- :issue:`3079`: Removed support for yield tests - they are fundamentally broken because they don't support fixtures properly since collection and test execution were separated. + + See our :ref:`docs ` on information on how to update your code. + + +- :issue:`3082`: Removed support for applying marks directly to values in ``@pytest.mark.parametrize``. Use ``pytest.param`` instead. + + See our :ref:`docs ` on information on how to update your code. + + +- :issue:`3083`: Removed ``Metafunc.addcall``. This was the predecessor mechanism to ``@pytest.mark.parametrize``. + + See our :ref:`docs ` on information on how to update your code. + + +- :issue:`3085`: Removed support for passing strings to ``pytest.main``. Now, always pass a list of strings instead. + + See our :ref:`docs ` on information on how to update your code. + + +- :issue:`3086`: ``[pytest]`` section in **setup.cfg** files is no longer supported, use ``[tool:pytest]`` instead. ``setup.cfg`` files + are meant for use with ``distutils``, and a section named ``pytest`` has notoriously been a source of conflicts and bugs. + + Note that for **pytest.ini** and **tox.ini** files the section remains ``[pytest]``. + + +- :issue:`3616`: Removed the deprecated compat properties for ``node.Class/Function/Module`` - use ``pytest.Class/Function/Module`` now. + + See our :ref:`docs ` on information on how to update your code. + + +- :issue:`4421`: Removed the implementation of the ``pytest_namespace`` hook. + + See our :ref:`docs ` on information on how to update your code. + + +- :issue:`4489`: Removed ``request.cached_setup``. This was the predecessor mechanism to modern fixtures. + + See our :ref:`docs ` on information on how to update your code. + + +- :issue:`4535`: Removed the deprecated ``PyCollector.makeitem`` method. This method was made public by mistake a long time ago. + + +- :issue:`4543`: Removed support to define fixtures using the ``pytest_funcarg__`` prefix. Use the ``@pytest.fixture`` decorator instead. + + See our :ref:`docs ` on information on how to update your code. + + +- :issue:`4545`: Calling fixtures directly is now always an error instead of a warning. + + See our :ref:`docs ` on information on how to update your code. + + +- :issue:`4546`: Remove ``Node.get_marker(name)`` the return value was not usable for more than a existence check. + + Use ``Node.get_closest_marker(name)`` as a replacement. + + +- :issue:`4547`: The deprecated ``record_xml_property`` fixture has been removed, use the more generic ``record_property`` instead. + + See our :ref:`docs ` for more information. + + +- :issue:`4548`: An error is now raised if the ``pytest_plugins`` variable is defined in a non-top-level ``conftest.py`` file (i.e., not residing in the ``rootdir``). + + See our :ref:`docs ` for more information. + + +- :issue:`891`: Remove ``testfunction.markername`` attributes - use ``Node.iter_markers(name=None)`` to iterate them. + + + +Deprecations +------------ + +- :issue:`3050`: Deprecated the ``pytest.config`` global. + + See :ref:`pytest.config global deprecated` for rationale. + + +- :issue:`3974`: Passing the ``message`` parameter of ``pytest.raises`` now issues a ``DeprecationWarning``. + + It is a common mistake to think this parameter will match the exception message, while in fact + it only serves to provide a custom message in case the ``pytest.raises`` check fails. To avoid this + mistake and because it is believed to be little used, pytest is deprecating it without providing + an alternative for the moment. + + If you have concerns about this, please comment on :issue:`3974`. + + +- :issue:`4435`: Deprecated ``raises(..., 'code(as_a_string)')`` and ``warns(..., 'code(as_a_string)')``. + + See :std:ref:`raises-warns-exec` for rationale and examples. + + + +Features +-------- + +- :issue:`3191`: A warning is now issued when assertions are made for ``None``. + + This is a common source of confusion among new users, which write: + + .. code-block:: python + + assert mocked_object.assert_called_with(3, 4, 5, key="value") + + When they should write: + + .. code-block:: python + + mocked_object.assert_called_with(3, 4, 5, key="value") + + Because the ``assert_called_with`` method of mock objects already executes an assertion. + + This warning will not be issued when ``None`` is explicitly checked. An assertion like: + + .. code-block:: python + + assert variable is None + + will not issue the warning. + + +- :issue:`3632`: Richer equality comparison introspection on ``AssertionError`` for objects created using `attrs `__ or :mod:`dataclasses` (Python 3.7+, :pypi:`backported to 3.6 `). + + +- :issue:`4278`: ``CACHEDIR.TAG`` files are now created inside cache directories. + + Those files are part of the `Cache Directory Tagging Standard `__, and can + be used by backup or synchronization programs to identify pytest's cache directory as such. + + +- :issue:`4292`: ``pytest.outcomes.Exit`` is derived from ``SystemExit`` instead of ``KeyboardInterrupt``. This allows us to better handle ``pdb`` exiting. + + +- :issue:`4371`: Updated the ``--collect-only`` option to display test descriptions when ran using ``--verbose``. + + +- :issue:`4386`: Restructured ``ExceptionInfo`` object construction and ensure incomplete instances have a ``repr``/``str``. + + +- :issue:`4416`: pdb: added support for keyword arguments with ``pdb.set_trace``. + + It handles ``header`` similar to Python 3.7 does it, and forwards any + other keyword arguments to the ``Pdb`` constructor. + + This allows for ``__import__("pdb").set_trace(skip=["foo.*"])``. + + +- :issue:`4483`: Added ini parameter ``junit_duration_report`` to optionally report test call durations, excluding setup and teardown times. + + The JUnit XML specification and the default pytest behavior is to include setup and teardown times in the test duration + report. You can include just the call durations instead (excluding setup and teardown) by adding this to your ``pytest.ini`` file: + + .. code-block:: ini + + [pytest] + junit_duration_report = call + + +- :issue:`4532`: ``-ra`` now will show errors and failures last, instead of as the first items in the summary. + + This makes it easier to obtain a list of errors and failures to run tests selectively. + + +- :issue:`4599`: ``pytest.importorskip`` now supports a ``reason`` parameter, which will be shown when the + requested module cannot be imported. + + + +Bug Fixes +--------- + +- :issue:`3532`: ``-p`` now accepts its argument without a space between the value, for example ``-pmyplugin``. + + +- :issue:`4327`: ``approx`` again works with more generic containers, more precisely instances of ``Iterable`` and ``Sized`` instead of more restrictive ``Sequence``. + + +- :issue:`4397`: Ensure that node ids are printable. + + +- :issue:`4435`: Fixed ``raises(..., 'code(string)')`` frame filename. + + +- :issue:`4458`: Display actual test ids in ``--collect-only``. + + + +Improved Documentation +---------------------- + +- :issue:`4557`: Markers example documentation page updated to support latest pytest version. + + +- :issue:`4558`: Update cache documentation example to correctly show cache hit and miss. + + +- :issue:`4580`: Improved detailed summary report documentation. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`4447`: Changed the deprecation type of ``--result-log`` to ``PytestDeprecationWarning``. + + It was decided to remove this feature at the next major revision. + + +pytest 4.0.2 (2018-12-13) +========================= + +Bug Fixes +--------- + +- :issue:`4265`: Validate arguments from the ``PYTEST_ADDOPTS`` environment variable and the ``addopts`` ini option separately. + + +- :issue:`4435`: Fix ``raises(..., 'code(string)')`` frame filename. + + +- :issue:`4500`: When a fixture yields and a log call is made after the test runs, and, if the test is interrupted, capture attributes are ``None``. + + +- :issue:`4538`: Raise ``TypeError`` for ``with raises(..., match=)``. + + + +Improved Documentation +---------------------- + +- :issue:`1495`: Document common doctest fixture directory tree structure pitfalls + + +pytest 4.0.1 (2018-11-23) +========================= + +Bug Fixes +--------- + +- :issue:`3952`: Display warnings before "short test summary info" again, but still later warnings in the end. + + +- :issue:`4386`: Handle uninitialized exceptioninfo in repr/str. + + +- :issue:`4393`: Do not create ``.gitignore``/``README.md`` files in existing cache directories. + + +- :issue:`4400`: Rearrange warning handling for the yield test errors so the opt-out in 4.0.x correctly works. + + +- :issue:`4405`: Fix collection of testpaths with ``--pyargs``. + + +- :issue:`4412`: Fix assertion rewriting involving ``Starred`` + side-effects. + + +- :issue:`4425`: Ensure we resolve the absolute path when the given ``--basetemp`` is a relative path. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`4315`: Use ``pkg_resources.parse_version`` instead of ``LooseVersion`` in minversion check. + + +- :issue:`4440`: Adjust the stack level of some internal pytest warnings. + + +pytest 4.0.0 (2018-11-13) +========================= + +Removals +-------- + +- :issue:`3737`: **RemovedInPytest4Warnings are now errors by default.** + + Following our plan to remove deprecated features with as little disruption as + possible, all warnings of type ``RemovedInPytest4Warnings`` now generate errors + instead of warning messages. + + **The affected features will be effectively removed in pytest 4.1**, so please consult the + :std:doc:`deprecations` section in the docs for directions on how to update existing code. + + In the pytest ``4.0.X`` series, it is possible to change the errors back into warnings as a stop + gap measure by adding this to your ``pytest.ini`` file: + + .. code-block:: ini + + [pytest] + filterwarnings = + ignore::pytest.RemovedInPytest4Warning + + But this will stop working when pytest ``4.1`` is released. + + **If you have concerns** about the removal of a specific feature, please add a + comment to :issue:`4348`. + + +- :issue:`4358`: Remove the ``::()`` notation to denote a test class instance in node ids. + + Previously, node ids that contain test instances would use ``::()`` to denote the instance like this:: + + test_foo.py::Test::()::test_bar + + The extra ``::()`` was puzzling to most users and has been removed, so that the test id becomes now:: + + test_foo.py::Test::test_bar + + This change could not accompany a deprecation period as is usual when user-facing functionality changes because + it was not really possible to detect when the functionality was being used explicitly. + + The extra ``::()`` might have been removed in some places internally already, + which then led to confusion in places where it was expected, e.g. with + ``--deselect`` (:issue:`4127`). + + Test class instances are also not listed with ``--collect-only`` anymore. + + + +Features +-------- + +- :issue:`4270`: The ``cache_dir`` option uses ``$TOX_ENV_DIR`` as prefix (if set in the environment). + + This uses a different cache per tox environment by default. + + + +Bug Fixes +--------- + +- :issue:`3554`: Fix ``CallInfo.__repr__`` for when the call is not finished yet. + + +pytest 3.10.1 (2018-11-11) +========================== + +Bug Fixes +--------- + +- :issue:`4287`: Fix nested usage of debugging plugin (pdb), e.g. with pytester's ``testdir.runpytest``. + + +- :issue:`4304`: Block the ``stepwise`` plugin if ``cacheprovider`` is also blocked, as one depends on the other. + + +- :issue:`4306`: Parse ``minversion`` as an actual version and not as dot-separated strings. + + +- :issue:`4310`: Fix duplicate collection due to multiple args matching the same packages. + + +- :issue:`4321`: Fix ``item.nodeid`` with resolved symlinks. + + +- :issue:`4325`: Fix collection of direct symlinked files, where the target does not match ``python_files``. + + +- :issue:`4329`: Fix TypeError in report_collect with _collect_report_last_write. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`4305`: Replace byte/unicode helpers in test_capture with python level syntax. + + +pytest 3.10.0 (2018-11-03) +========================== + +Features +-------- + +- :issue:`2619`: Resume capturing output after ``continue`` with ``__import__("pdb").set_trace()``. + + This also adds a new ``pytest_leave_pdb`` hook, and passes in ``pdb`` to the + existing ``pytest_enter_pdb`` hook. + + +- :issue:`4147`: Add ``--sw``, ``--stepwise`` as an alternative to ``--lf -x`` for stopping at the first failure, but starting the next test invocation from that test. See :ref:`the documentation ` for more info. + + +- :issue:`4188`: Make ``--color`` emit colorful dots when not running in verbose mode. Earlier, it would only colorize the test-by-test output if ``--verbose`` was also passed. + + +- :issue:`4225`: Improve performance with collection reporting in non-quiet mode with terminals. + + The "collecting …" message is only printed/updated every 0.5s. + + + +Bug Fixes +--------- + +- :issue:`2701`: Fix false ``RemovedInPytest4Warning: usage of Session... is deprecated, please use pytest`` warnings. + + +- :issue:`4046`: Fix problems with running tests in package ``__init__.py`` files. + + +- :issue:`4260`: Swallow warnings during anonymous compilation of source. + + +- :issue:`4262`: Fix access denied error when deleting stale directories created by ``tmpdir`` / ``tmp_path``. + + +- :issue:`611`: Naming a fixture ``request`` will now raise a warning: the ``request`` fixture is internal and + should not be overwritten as it will lead to internal errors. + +- :issue:`4266`: Handle (ignore) exceptions raised during collection, e.g. with Django's LazySettings proxy class. + + + +Improved Documentation +---------------------- + +- :issue:`4255`: Added missing documentation about the fact that module names passed to filter warnings are not regex-escaped. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`4272`: Display cachedir also in non-verbose mode if non-default. + + +- :issue:`4277`: pdb: improve message about output capturing with ``set_trace``. + + Do not display "IO-capturing turned off/on" when ``-s`` is used to avoid + confusion. + + +- :issue:`4279`: Improve message and stack level of warnings issued by ``monkeypatch.setenv`` when the value of the environment variable is not a ``str``. + + +pytest 3.9.3 (2018-10-27) +========================= + +Bug Fixes +--------- + +- :issue:`4174`: Fix "ValueError: Plugin already registered" with conftest plugins via symlink. + + +- :issue:`4181`: Handle race condition between creation and deletion of temporary folders. + + +- :issue:`4221`: Fix bug where the warning summary at the end of the test session was not showing the test where the warning was originated. + + +- :issue:`4243`: Fix regression when ``stacklevel`` for warnings was passed as positional argument on python2. + + + +Improved Documentation +---------------------- + +- :issue:`3851`: Add reference to ``empty_parameter_set_mark`` ini option in documentation of ``@pytest.mark.parametrize`` + + + +Trivial/Internal Changes +------------------------ + +- :issue:`4028`: Revert patching of ``sys.breakpointhook`` since it appears to do nothing. + + +- :issue:`4233`: Apply an import sorter (``reorder-python-imports``) to the codebase. + + +- :issue:`4248`: Remove use of unnecessary compat shim, six.binary_type + + +pytest 3.9.2 (2018-10-22) +========================= + +Bug Fixes +--------- + +- :issue:`2909`: Improve error message when a recursive dependency between fixtures is detected. + + +- :issue:`3340`: Fix logging messages not shown in hooks ``pytest_sessionstart()`` and ``pytest_sessionfinish()``. + + +- :issue:`3533`: Fix unescaped XML raw objects in JUnit report for skipped tests + + +- :issue:`3691`: Python 2: safely format warning message about passing unicode strings to ``warnings.warn``, which may cause + surprising ``MemoryError`` exception when monkey patching ``warnings.warn`` itself. + + +- :issue:`4026`: Improve error message when it is not possible to determine a function's signature. + + +- :issue:`4177`: Pin ``setuptools>=40.0`` to support ``py_modules`` in ``setup.cfg`` + + +- :issue:`4179`: Restore the tmpdir behaviour of symlinking the current test run. + + +- :issue:`4192`: Fix filename reported by ``warnings.warn`` when using ``recwarn`` under python2. + + +pytest 3.9.1 (2018-10-16) +========================= + +Features +-------- + +- :issue:`4159`: For test-suites containing test classes, the information about the subclassed + module is now output only if a higher verbosity level is specified (at least + "-vv"). + + +pytest 3.9.0 (2018-10-15 - not published due to a release automation bug) +========================================================================= + +Deprecations +------------ + +- :issue:`3616`: The following accesses have been documented as deprecated for years, but are now actually emitting deprecation warnings. + + * Access of ``Module``, ``Function``, ``Class``, ``Instance``, ``File`` and ``Item`` through ``Node`` instances. Now + users will this warning:: + + usage of Function.Module is deprecated, please use pytest.Module instead + + Users should just ``import pytest`` and access those objects using the ``pytest`` module. + + * ``request.cached_setup``, this was the precursor of the setup/teardown mechanism available to fixtures. You can + consult :std:doc:`funcarg comparison section in the docs `. + + * Using objects named ``"Class"`` as a way to customize the type of nodes that are collected in ``Collector`` + subclasses has been deprecated. Users instead should use ``pytest_collect_make_item`` to customize node types during + collection. + + This issue should affect only advanced plugins who create new collection types, so if you see this warning + message please contact the authors so they can change the code. + + * The warning that produces the message below has changed to ``RemovedInPytest4Warning``:: + + getfuncargvalue is deprecated, use getfixturevalue + + +- :issue:`3988`: Add a Deprecation warning for pytest.ensuretemp as it was deprecated since a while. + + + +Features +-------- + +- :issue:`2293`: Improve usage errors messages by hiding internal details which can be distracting and noisy. + + This has the side effect that some error conditions that previously raised generic errors (such as + ``ValueError`` for unregistered marks) are now raising ``Failed`` exceptions. + + +- :issue:`3332`: Improve the error displayed when a ``conftest.py`` file could not be imported. + + In order to implement this, a new ``chain`` parameter was added to ``ExceptionInfo.getrepr`` + to show or hide chained tracebacks in Python 3 (defaults to ``True``). + + +- :issue:`3849`: Add ``empty_parameter_set_mark=fail_at_collect`` ini option for raising an exception when parametrize collects an empty set. + + +- :issue:`3964`: Log messages generated in the collection phase are shown when + live-logging is enabled and/or when they are logged to a file. + + +- :issue:`3985`: Introduce ``tmp_path`` as a fixture providing a Path object. Also introduce ``tmp_path_factory`` as + a session-scoped fixture for creating arbitrary temporary directories from any other fixture or test. + + +- :issue:`4013`: Deprecation warnings are now shown even if you customize the warnings filters yourself. In the previous version + any customization would override pytest's filters and deprecation warnings would fall back to being hidden by default. + + +- :issue:`4073`: Allow specification of timeout for ``Testdir.runpytest_subprocess()`` and ``Testdir.run()``. + + +- :issue:`4098`: Add returncode argument to pytest.exit() to exit pytest with a specific return code. + + +- :issue:`4102`: Reimplement ``pytest.deprecated_call`` using ``pytest.warns`` so it supports the ``match='...'`` keyword argument. + + This has the side effect that ``pytest.deprecated_call`` now raises ``pytest.fail.Exception`` instead + of ``AssertionError``. + + +- :issue:`4149`: Require setuptools>=30.3 and move most of the metadata to ``setup.cfg``. + + + +Bug Fixes +--------- + +- :issue:`2535`: Improve error message when test functions of ``unittest.TestCase`` subclasses use a parametrized fixture. + + +- :issue:`3057`: ``request.fixturenames`` now correctly returns the name of fixtures created by ``request.getfixturevalue()``. + + +- :issue:`3946`: Warning filters passed as command line options using ``-W`` now take precedence over filters defined in ``ini`` + configuration files. + + +- :issue:`4066`: Fix source reindenting by using ``textwrap.dedent`` directly. + + +- :issue:`4102`: ``pytest.warn`` will capture previously-warned warnings in Python 2. Previously they were never raised. + + +- :issue:`4108`: Resolve symbolic links for args. + + This fixes running ``pytest tests/test_foo.py::test_bar``, where ``tests`` + is a symlink to ``project/app/tests``: + previously ``project/app/conftest.py`` would be ignored for fixtures then. + + +- :issue:`4132`: Fix duplicate printing of internal errors when using ``--pdb``. + + +- :issue:`4135`: pathlib based tmpdir cleanup now correctly handles symlinks in the folder. + + +- :issue:`4152`: Display the filename when encountering ``SyntaxWarning``. + + + +Improved Documentation +---------------------- + +- :issue:`3713`: Update usefixtures documentation to clarify that it can't be used with fixture functions. + + +- :issue:`4058`: Update fixture documentation to specify that a fixture can be invoked twice in the scope it's defined for. + + +- :issue:`4064`: According to unittest.rst, setUpModule and tearDownModule were not implemented, but it turns out they are. So updated the documentation for unittest. + + +- :issue:`4151`: Add tempir testing example to CONTRIBUTING.rst guide + + + +Trivial/Internal Changes +------------------------ + +- :issue:`2293`: The internal ``MarkerError`` exception has been removed. + + +- :issue:`3988`: Port the implementation of tmpdir to pathlib. + + +- :issue:`4063`: Exclude 0.00 second entries from ``--duration`` output unless ``-vv`` is passed on the command-line. + + +- :issue:`4093`: Fixed formatting of string literals in internal tests. + + +pytest 3.8.2 (2018-10-02) +========================= + +Deprecations and Removals +------------------------- + +- :issue:`4036`: The ``item`` parameter of ``pytest_warning_captured`` hook is now documented as deprecated. We realized only after + the ``3.8`` release that this parameter is incompatible with ``pytest-xdist``. + + Our policy is to not deprecate features during bug-fix releases, but in this case we believe it makes sense as we are + only documenting it as deprecated, without issuing warnings which might potentially break test suites. This will get + the word out that hook implementers should not use this parameter at all. + + In a future release ``item`` will always be ``None`` and will emit a proper warning when a hook implementation + makes use of it. + + + +Bug Fixes +--------- + +- :issue:`3539`: Fix reload on assertion rewritten modules. + + +- :issue:`4034`: The ``.user_properties`` attribute of ``TestReport`` objects is a list + of (name, value) tuples, but could sometimes be instantiated as a tuple + of tuples. It is now always a list. + + +- :issue:`4039`: No longer issue warnings about using ``pytest_plugins`` in non-top-level directories when using ``--pyargs``: the + current ``--pyargs`` mechanism is not reliable and might give false negatives. + + +- :issue:`4040`: Exclude empty reports for passed tests when ``-rP`` option is used. + + +- :issue:`4051`: Improve error message when an invalid Python expression is passed to the ``-m`` option. + + +- :issue:`4056`: ``MonkeyPatch.setenv`` and ``MonkeyPatch.delenv`` issue a warning if the environment variable name is not ``str`` on Python 2. + + In Python 2, adding ``unicode`` keys to ``os.environ`` causes problems with ``subprocess`` (and possible other modules), + making this a subtle bug specially susceptible when used with ``from __future__ import unicode_literals``. + + + +Improved Documentation +---------------------- + +- :issue:`3928`: Add possible values for fixture scope to docs. + + +pytest 3.8.1 (2018-09-22) +========================= + +Bug Fixes +--------- + +- :issue:`3286`: ``.pytest_cache`` directory is now automatically ignored by Git. Users who would like to contribute a solution for other SCMs please consult/comment on this issue. + + +- :issue:`3749`: Fix the following error during collection of tests inside packages:: + + TypeError: object of type 'Package' has no len() + + +- :issue:`3941`: Fix bug where indirect parametrization would consider the scope of all fixtures used by the test function to determine the parametrization scope, and not only the scope of the fixtures being parametrized. + + +- :issue:`3973`: Fix crash of the assertion rewriter if a test changed the current working directory without restoring it afterwards. + + +- :issue:`3998`: Fix issue that prevented some caplog properties (for example ``record_tuples``) from being available when entering the debugger with ``--pdb``. + + +- :issue:`3999`: Fix ``UnicodeDecodeError`` in python2.x when a class returns a non-ascii binary ``__repr__`` in an assertion which also contains non-ascii text. + + + +Improved Documentation +---------------------- + +- :issue:`3996`: New :std:doc:`deprecations` page shows all currently + deprecated features, the rationale to do so, and alternatives to update your code. It also list features removed + from pytest in past major releases to help those with ancient pytest versions to upgrade. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`3955`: Improve pre-commit detection for changelog filenames + + +- :issue:`3975`: Remove legacy code around im_func as that was python2 only + + +pytest 3.8.0 (2018-09-05) +========================= + +Deprecations and Removals +------------------------- + +- :issue:`2452`: ``Config.warn`` and ``Node.warn`` have been + deprecated, see :ref:`config.warn and node.warn deprecated` for rationale and + examples. + +- :issue:`3936`: ``@pytest.mark.filterwarnings`` second parameter is no longer regex-escaped, + making it possible to actually use regular expressions to check the warning message. + + **Note**: regex-escaping the match string was an implementation oversight that might break test suites which depend + on the old behavior. + + + +Features +-------- + +- :issue:`2452`: Internal pytest warnings are now issued using the standard ``warnings`` module, making it possible to use + the standard warnings filters to manage those warnings. This introduces ``PytestWarning``, + ``PytestDeprecationWarning`` and ``RemovedInPytest4Warning`` warning types as part of the public API. + + Consult :ref:`the documentation ` for more info. + + +- :issue:`2908`: ``DeprecationWarning`` and ``PendingDeprecationWarning`` are now shown by default if no other warning filter is + configured. This makes pytest more compliant with + :pep:`506#recommended-filter-settings-for-test-runners`. See + :ref:`the docs ` for + more info. + + +- :issue:`3251`: Warnings are now captured and displayed during test collection. + + +- :issue:`3784`: ``PYTEST_DISABLE_PLUGIN_AUTOLOAD`` environment variable disables plugin auto-loading when set. + + +- :issue:`3829`: Added the ``count`` option to ``console_output_style`` to enable displaying the progress as a count instead of a percentage. + + +- :issue:`3837`: Added support for 'xfailed' and 'xpassed' outcomes to the ``pytester.RunResult.assert_outcomes`` signature. + + + +Bug Fixes +--------- + +- :issue:`3911`: Terminal writer now takes into account unicode character width when writing out progress. + + +- :issue:`3913`: Pytest now returns with correct exit code (EXIT_USAGEERROR, 4) when called with unknown arguments. + + +- :issue:`3918`: Improve performance of assertion rewriting. + + + +Improved Documentation +---------------------- + +- :issue:`3566`: Added a blurb in usage.rst for the usage of -r flag which is used to show an extra test summary info. + + +- :issue:`3907`: Corrected type of the exceptions collection passed to ``xfail``: ``raises`` argument accepts a ``tuple`` instead of ``list``. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`3853`: Removed ``"run all (no recorded failures)"`` message printed with ``--failed-first`` and ``--last-failed`` when there are no failed tests. + + +pytest 3.7.4 (2018-08-29) +========================= + +Bug Fixes +--------- + +- :issue:`3506`: Fix possible infinite recursion when writing ``.pyc`` files. + + +- :issue:`3853`: Cache plugin now obeys the ``-q`` flag when ``--last-failed`` and ``--failed-first`` flags are used. + + +- :issue:`3883`: Fix bad console output when using ``console_output_style=classic``. + + +- :issue:`3888`: Fix macOS specific code using ``capturemanager`` plugin in doctests. + + + +Improved Documentation +---------------------- + +- :issue:`3902`: Fix pytest.org links + + +pytest 3.7.3 (2018-08-26) +========================= + +Bug Fixes +--------- + +- :issue:`3033`: Fixtures during teardown can again use ``capsys`` and ``capfd`` to inspect output captured during tests. + + +- :issue:`3773`: Fix collection of tests from ``__init__.py`` files if they match the ``python_files`` configuration option. + + +- :issue:`3796`: Fix issue where teardown of fixtures of consecutive sub-packages were executed once, at the end of the outer + package. + + +- :issue:`3816`: Fix bug where ``--show-capture=no`` option would still show logs printed during fixture teardown. + + +- :issue:`3819`: Fix ``stdout/stderr`` not getting captured when real-time cli logging is active. + + +- :issue:`3843`: Fix collection error when specifying test functions directly in the command line using ``test.py::test`` syntax together with ``--doctest-modules``. + + +- :issue:`3848`: Fix bugs where unicode arguments could not be passed to ``testdir.runpytest`` on Python 2. + + +- :issue:`3854`: Fix double collection of tests within packages when the filename starts with a capital letter. + + + +Improved Documentation +---------------------- + +- :issue:`3824`: Added example for multiple glob pattern matches in ``python_files``. + + +- :issue:`3833`: Added missing docs for ``pytester.Testdir``. + + +- :issue:`3870`: Correct documentation for setuptools integration. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`3826`: Replace broken type annotations with type comments. + + +- :issue:`3845`: Remove a reference to issue :issue:`568` from the documentation, which has since been + fixed. + + +pytest 3.7.2 (2018-08-16) +========================= + +Bug Fixes +--------- + +- :issue:`3671`: Fix ``filterwarnings`` not being registered as a builtin mark. + + +- :issue:`3768`, :issue:`3789`: Fix test collection from packages mixed with normal directories. + + +- :issue:`3771`: Fix infinite recursion during collection if a ``pytest_ignore_collect`` hook returns ``False`` instead of ``None``. + + +- :issue:`3774`: Fix bug where decorated fixtures would lose functionality (for example ``@mock.patch``). + + +- :issue:`3775`: Fix bug where importing modules or other objects with prefix ``pytest_`` prefix would raise a ``PluginValidationError``. + + +- :issue:`3788`: Fix ``AttributeError`` during teardown of ``TestCase`` subclasses which raise an exception during ``__init__``. + + +- :issue:`3804`: Fix traceback reporting for exceptions with ``__cause__`` cycles. + + + +Improved Documentation +---------------------- + +- :issue:`3746`: Add documentation for ``metafunc.config`` that had been mistakenly hidden. + + +pytest 3.7.1 (2018-08-02) +========================= + +Bug Fixes +--------- + +- :issue:`3473`: Raise immediately if ``approx()`` is given an expected value of a type it doesn't understand (e.g. strings, nested dicts, etc.). + + +- :issue:`3712`: Correctly represent the dimensions of a numpy array when calling ``repr()`` on ``approx()``. + +- :issue:`3742`: Fix incompatibility with third party plugins during collection, which produced the error ``object has no attribute '_collectfile'``. + +- :issue:`3745`: Display the absolute path if ``cache_dir`` is not relative to the ``rootdir`` instead of failing. + + +- :issue:`3747`: Fix compatibility problem with plugins and the warning code issued by fixture functions when they are called directly. + + +- :issue:`3748`: Fix infinite recursion in ``pytest.approx`` with arrays in ``numpy<1.13``. + + +- :issue:`3757`: Pin pathlib2 to ``>=2.2.0`` as we require ``__fspath__`` support. + + +- :issue:`3763`: Fix ``TypeError`` when the assertion message is ``bytes`` in python 3. + + +pytest 3.7.0 (2018-07-30) +========================= + +Deprecations and Removals +------------------------- + +- :issue:`2639`: ``pytest_namespace`` has been :ref:`deprecated `. + + +- :issue:`3661`: Calling a fixture function directly, as opposed to request them in a test function, now issues a ``RemovedInPytest4Warning``. See :ref:`the documentation for rationale and examples `. + + + +Features +-------- + +- :issue:`2283`: New ``package`` fixture scope: fixtures are finalized when the last test of a *package* finishes. This feature is considered **experimental**, so use it sparingly. + + +- :issue:`3576`: ``Node.add_marker`` now supports an ``append=True/False`` parameter to determine whether the mark comes last (default) or first. + + +- :issue:`3579`: Fixture ``caplog`` now has a ``messages`` property, providing convenient access to the format-interpolated log messages without the extra data provided by the formatter/handler. + + +- :issue:`3610`: New ``--trace`` option to enter the debugger at the start of a test. + + +- :issue:`3623`: Introduce ``pytester.copy_example`` as helper to do acceptance tests against examples from the project. + + + +Bug Fixes +--------- + +- :issue:`2220`: Fix a bug where fixtures overridden by direct parameters (for example parametrization) were being instantiated even if they were not being used by a test. + + +- :issue:`3695`: Fix ``ApproxNumpy`` initialisation argument mixup, ``abs`` and ``rel`` tolerances were flipped causing strange comparison results. + Add tests to check ``abs`` and ``rel`` tolerances for ``np.array`` and test for expecting ``nan`` with ``np.array()`` + + +- :issue:`980`: Fix truncated locals output in verbose mode. + + + +Improved Documentation +---------------------- + +- :issue:`3295`: Correct the usage documentation of ``--last-failed-no-failures`` by adding the missing ``--last-failed`` argument in the presented examples, because they are misleading and lead to think that the missing argument is not needed. + + + +Trivial/Internal Changes +------------------------ + +- :issue:`3519`: Now a ``README.md`` file is created in ``.pytest_cache`` to make it clear why the directory exists. + + +pytest 3.6.4 (2018-07-28) +========================= + +Bug Fixes +--------- + +- Invoke pytest using ``-mpytest`` so ``sys.path`` does not get polluted by packages installed in ``site-packages``. (:issue:`742`) + + +Improved Documentation +---------------------- + +- Use ``smtp_connection`` instead of ``smtp`` in fixtures documentation to avoid possible confusion. (:issue:`3592`) + + +Trivial/Internal Changes +------------------------ + +- Remove obsolete ``__future__`` imports. (:issue:`2319`) + +- Add CITATION to provide information on how to formally cite pytest. (:issue:`3402`) + +- Replace broken type annotations with type comments. (:issue:`3635`) + +- Pin ``pluggy`` to ``<0.8``. (:issue:`3727`) + + +pytest 3.6.3 (2018-07-04) +========================= + +Bug Fixes +--------- + +- Fix ``ImportWarning`` triggered by explicit relative imports in + assertion-rewritten package modules. (:issue:`3061`) + +- Fix error in ``pytest.approx`` when dealing with 0-dimension numpy + arrays. (:issue:`3593`) + +- No longer raise ``ValueError`` when using the ``get_marker`` API. (:issue:`3605`) + +- Fix problem where log messages with non-ascii characters would not + appear in the output log file. + (:issue:`3630`) + +- No longer raise ``AttributeError`` when legacy marks can't be stored in + functions. (:issue:`3631`) + + +Improved Documentation +---------------------- + +- The description above the example for ``@pytest.mark.skipif`` now better + matches the code. (:issue:`3611`) + + +Trivial/Internal Changes +------------------------ + +- Internal refactoring: removed unused ``CallSpec2tox ._globalid_args`` + attribute and ``metafunc`` parameter from ``CallSpec2.copy()``. (:issue:`3598`) + +- Silence usage of ``reduce`` warning in Python 2 (:issue:`3609`) + +- Fix usage of ``attr.ib`` deprecated ``convert`` parameter. (:issue:`3653`) + + +pytest 3.6.2 (2018-06-20) +========================= + +Bug Fixes +--------- + +- Fix regression in ``Node.add_marker`` by extracting the mark object of a + ``MarkDecorator``. (:issue:`3555`) + +- Warnings without ``location`` were reported as ``None``. This is corrected to + now report ````. (:issue:`3563`) + +- Continue to call finalizers in the stack when a finalizer in a former scope + raises an exception. (:issue:`3569`) + +- Fix encoding error with ``print`` statements in doctests (:issue:`3583`) + + +Improved Documentation +---------------------- + +- Add documentation for the ``--strict`` flag. (:issue:`3549`) + + +Trivial/Internal Changes +------------------------ + +- Update old quotation style to parens in fixture.rst documentation. (:issue:`3525`) + +- Improve display of hint about ``--fulltrace`` with ``KeyboardInterrupt``. + (:issue:`3545`) + +- pytest's testsuite is no longer runnable through ``python setup.py test`` -- + instead invoke ``pytest`` or ``tox`` directly. (:issue:`3552`) + +- Fix typo in documentation (:issue:`3567`) + + +pytest 3.6.1 (2018-06-05) +========================= + +Bug Fixes +--------- + +- Fixed a bug where stdout and stderr were logged twice by junitxml when a test + was marked xfail. (:issue:`3491`) + +- Fix ``usefixtures`` mark applied to unittest tests by correctly instantiating + ``FixtureInfo``. (:issue:`3498`) + +- Fix assertion rewriter compatibility with libraries that monkey patch + ``file`` objects. (:issue:`3503`) + + +Improved Documentation +---------------------- + +- Added a section on how to use fixtures as factories to the fixture + documentation. (:issue:`3461`) + + +Trivial/Internal Changes +------------------------ + +- Enable caching for pip/pre-commit in order to reduce build time on + travis/appveyor. (:issue:`3502`) + +- Switch pytest to the src/ layout as we already suggested it for good practice + - now we implement it as well. (:issue:`3513`) + +- Fix if in tests to support 3.7.0b5, where a docstring handling in AST got + reverted. (:issue:`3530`) + +- Remove some python2.5 compatibility code. (:issue:`3529`) + + +pytest 3.6.0 (2018-05-23) +========================= + +Features +-------- + +- Revamp the internals of the ``pytest.mark`` implementation with correct per + node handling which fixes a number of long standing bugs caused by the old + design. This introduces new ``Node.iter_markers(name)`` and + ``Node.get_closest_marker(name)`` APIs. Users are **strongly encouraged** to + read the :ref:`reasons for the revamp in the docs `, + or jump over to details about :ref:`updating existing code to use the new APIs + `. + (:issue:`3317`) + +- Now when ``@pytest.fixture`` is applied more than once to the same function a + ``ValueError`` is raised. This buggy behavior would cause surprising problems + and if was working for a test suite it was mostly by accident. (:issue:`2334`) + +- Support for Python 3.7's builtin ``breakpoint()`` method, see + :ref:`Using the builtin breakpoint function ` for + details. (:issue:`3180`) + +- ``monkeypatch`` now supports a ``context()`` function which acts as a context + manager which undoes all patching done within the ``with`` block. (:issue:`3290`) + +- The ``--pdb`` option now causes KeyboardInterrupt to enter the debugger, + instead of stopping the test session. On python 2.7, hitting CTRL+C again + exits the debugger. On python 3.2 and higher, use CTRL+D. (:issue:`3299`) + +- pytest no longer changes the log level of the root logger when the + ``log-level`` parameter has greater numeric value than that of the level of + the root logger, which makes it play better with custom logging configuration + in user code. (:issue:`3307`) + + +Bug Fixes +--------- + +- A rare race-condition which might result in corrupted ``.pyc`` files on + Windows has been hopefully solved. (:issue:`3008`) + +- Also use iter_marker for discovering the marks applying for marker + expressions from the cli to avoid the bad data from the legacy mark storage. + (:issue:`3441`) + +- When showing diffs of failed assertions where the contents contain only + whitespace, escape them using ``repr()`` first to make it easy to spot the + differences. (:issue:`3443`) + + +Improved Documentation +---------------------- + +- Change documentation copyright year to a range which auto-updates itself each + time it is published. (:issue:`3303`) + + +Trivial/Internal Changes +------------------------ + +- ``pytest`` now depends on the `python-atomicwrites + `_ library. (:issue:`3008`) + +- Update all pypi.python.org URLs to pypi.org. (:issue:`3431`) + +- Detect `pytest_` prefixed hooks using the internal plugin manager since + ``pluggy`` is deprecating the ``implprefix`` argument to ``PluginManager``. + (:issue:`3487`) + +- Import ``Mapping`` and ``Sequence`` from ``_pytest.compat`` instead of + directly from ``collections`` in ``python_api.py::approx``. Add ``Mapping`` + to ``_pytest.compat``, import it from ``collections`` on python 2, but from + ``collections.abc`` on Python 3 to avoid a ``DeprecationWarning`` on Python + 3.7 or newer. (:issue:`3497`) + + +pytest 3.5.1 (2018-04-23) +========================= + + +Bug Fixes +--------- + +- Reset ``sys.last_type``, ``sys.last_value`` and ``sys.last_traceback`` before + each test executes. Those attributes are added by pytest during the test run + to aid debugging, but were never reset so they would create a leaking + reference to the last failing test's frame which in turn could never be + reclaimed by the garbage collector. (:issue:`2798`) + +- ``pytest.raises`` now raises ``TypeError`` when receiving an unknown keyword + argument. (:issue:`3348`) + +- ``pytest.raises`` now works with exception classes that look like iterables. + (:issue:`3372`) + + +Improved Documentation +---------------------- + +- Fix typo in ``caplog`` fixture documentation, which incorrectly identified + certain attributes as methods. (:issue:`3406`) + + +Trivial/Internal Changes +------------------------ + +- Added a more indicative error message when parametrizing a function whose + argument takes a default value. (:issue:`3221`) + +- Remove internal ``_pytest.terminal.flatten`` function in favor of + ``more_itertools.collapse``. (:issue:`3330`) + +- Import some modules from ``collections.abc`` instead of ``collections`` as + the former modules trigger ``DeprecationWarning`` in Python 3.7. (:issue:`3339`) + +- record_property is no longer experimental, removing the warnings was + forgotten. (:issue:`3360`) + +- Mention in documentation and CLI help that fixtures with leading ``_`` are + printed by ``pytest --fixtures`` only if the ``-v`` option is added. (:issue:`3398`) + + +pytest 3.5.0 (2018-03-21) +========================= + +Deprecations and Removals +------------------------- + +- ``record_xml_property`` fixture is now deprecated in favor of the more + generic ``record_property``. (:issue:`2770`) + +- Defining ``pytest_plugins`` is now deprecated in non-top-level conftest.py + files, because they "leak" to the entire directory tree. + :ref:`See the docs ` + for the rationale behind this decision (:issue:`3084`) + + +Features +-------- + +- New ``--show-capture`` command-line option that allows to specify how to + display captured output when tests fail: ``no``, ``stdout``, ``stderr``, + ``log`` or ``all`` (the default). (:issue:`1478`) + +- New ``--rootdir`` command-line option to override the rules for discovering + the root directory. See :doc:`customize ` in the documentation for + details. (:issue:`1642`) + +- Fixtures are now instantiated based on their scopes, with higher-scoped + fixtures (such as ``session``) being instantiated first than lower-scoped + fixtures (such as ``function``). The relative order of fixtures of the same + scope is kept unchanged, based in their declaration order and their + dependencies. (:issue:`2405`) + +- ``record_xml_property`` renamed to ``record_property`` and is now compatible + with xdist, markers and any reporter. ``record_xml_property`` name is now + deprecated. (:issue:`2770`) + +- New ``--nf``, ``--new-first`` options: run new tests first followed by the + rest of the tests, in both cases tests are also sorted by the file modified + time, with more recent files coming first. (:issue:`3034`) + +- New ``--last-failed-no-failures`` command-line option that allows to specify + the behavior of the cache plugin's ```--last-failed`` feature when no tests + failed in the last run (or no cache was found): ``none`` or ``all`` (the + default). (:issue:`3139`) + +- New ``--doctest-continue-on-failure`` command-line option to enable doctests + to show multiple failures for each snippet, instead of stopping at the first + failure. (:issue:`3149`) + +- Captured log messages are added to the ```` tag in the generated + junit xml file if the ``junit_logging`` ini option is set to ``system-out``. + If the value of this ini option is ``system-err``, the logs are written to + ````. The default value for ``junit_logging`` is ``no``, meaning + captured logs are not written to the output file. (:issue:`3156`) + +- Allow the logging plugin to handle ``pytest_runtest_logstart`` and + ``pytest_runtest_logfinish`` hooks when live logs are enabled. (:issue:`3189`) + +- Passing ``--log-cli-level`` in the command-line now automatically activates + live logging. (:issue:`3190`) + +- Add command line option ``--deselect`` to allow deselection of individual + tests at collection time. (:issue:`3198`) + +- Captured logs are printed before entering pdb. (:issue:`3204`) + +- Deselected item count is now shown before tests are run, e.g. ``collected X + items / Y deselected``. (:issue:`3213`) + +- The builtin module ``platform`` is now available for use in expressions in + ``pytest.mark``. (:issue:`3236`) + +- The *short test summary info* section now is displayed after tracebacks and + warnings in the terminal. (:issue:`3255`) + +- New ``--verbosity`` flag to set verbosity level explicitly. (:issue:`3296`) + +- ``pytest.approx`` now accepts comparing a numpy array with a scalar. (:issue:`3312`) + + +Bug Fixes +--------- + +- Suppress ``IOError`` when closing the temporary file used for capturing + streams in Python 2.7. (:issue:`2370`) + +- Fixed ``clear()`` method on ``caplog`` fixture which cleared ``records``, but + not the ``text`` property. (:issue:`3297`) + +- During test collection, when stdin is not allowed to be read, the + ``DontReadFromStdin`` object still allow itself to be iterable and resolved + to an iterator without crashing. (:issue:`3314`) + + +Improved Documentation +---------------------- + +- Added a :doc:`reference ` page + to the docs. (:issue:`1713`) + + +Trivial/Internal Changes +------------------------ + +- Change minimum requirement of ``attrs`` to ``17.4.0``. (:issue:`3228`) + +- Renamed example directories so all tests pass when ran from the base + directory. (:issue:`3245`) + +- Internal ``mark.py`` module has been turned into a package. (:issue:`3250`) + +- ``pytest`` now depends on the `more-itertools + `_ package. (:issue:`3265`) + +- Added warning when ``[pytest]`` section is used in a ``.cfg`` file passed + with ``-c`` (:issue:`3268`) + +- ``nodeids`` can now be passed explicitly to ``FSCollector`` and ``Node`` + constructors. (:issue:`3291`) + +- Internal refactoring of ``FormattedExcinfo`` to use ``attrs`` facilities and + remove old support code for legacy Python versions. (:issue:`3292`) + +- Refactoring to unify how verbosity is handled internally. (:issue:`3296`) + +- Internal refactoring to better integrate with argparse. (:issue:`3304`) + +- Fix a python example when calling a fixture in doc/en/usage.rst (:issue:`3308`) + + +pytest 3.4.2 (2018-03-04) +========================= + +Bug Fixes +--------- + +- Removed progress information when capture option is ``no``. (:issue:`3203`) + +- Refactor check of bindir from ``exists`` to ``isdir``. (:issue:`3241`) + +- Fix ``TypeError`` issue when using ``approx`` with a ``Decimal`` value. + (:issue:`3247`) + +- Fix reference cycle generated when using the ``request`` fixture. (:issue:`3249`) + +- ``[tool:pytest]`` sections in ``*.cfg`` files passed by the ``-c`` option are + now properly recognized. (:issue:`3260`) + + +Improved Documentation +---------------------- + +- Add logging plugin to plugins list. (:issue:`3209`) + + +Trivial/Internal Changes +------------------------ + +- Fix minor typo in fixture.rst (:issue:`3259`) + + +pytest 3.4.1 (2018-02-20) +========================= + +Bug Fixes +--------- + +- Move import of ``doctest.UnexpectedException`` to top-level to avoid possible + errors when using ``--pdb``. (:issue:`1810`) + +- Added printing of captured stdout/stderr before entering pdb, and improved a + test which was giving false negatives about output capturing. (:issue:`3052`) + +- Fix ordering of tests using parametrized fixtures which can lead to fixtures + being created more than necessary. (:issue:`3161`) + +- Fix bug where logging happening at hooks outside of "test run" hooks would + cause an internal error. (:issue:`3184`) + +- Detect arguments injected by ``unittest.mock.patch`` decorator correctly when + pypi ``mock.patch`` is installed and imported. (:issue:`3206`) + +- Errors shown when a ``pytest.raises()`` with ``match=`` fails are now cleaner + on what happened: When no exception was raised, the "matching '...'" part got + removed as it falsely implies that an exception was raised but it didn't + match. When a wrong exception was raised, it's now thrown (like + ``pytest.raised()`` without ``match=`` would) instead of complaining about + the unmatched text. (:issue:`3222`) + +- Fixed output capture handling in doctests on macOS. (:issue:`985`) + + +Improved Documentation +---------------------- + +- Add Sphinx parameter docs for ``match`` and ``message`` args to + ``pytest.raises``. (:issue:`3202`) + + +Trivial/Internal Changes +------------------------ + +- pytest has changed the publication procedure and is now being published to + PyPI directly from Travis. (:issue:`3060`) + +- Rename ``ParameterSet._for_parameterize()`` to ``_for_parametrize()`` in + order to comply with the naming convention. (:issue:`3166`) + +- Skip failing pdb/doctest test on mac. (:issue:`985`) + + +pytest 3.4.0 (2018-01-30) +========================= + +Deprecations and Removals +------------------------- + +- All pytest classes now subclass ``object`` for better Python 2/3 compatibility. + This should not affect user code except in very rare edge cases. (:issue:`2147`) + + +Features +-------- + +- Introduce ``empty_parameter_set_mark`` ini option to select which mark to + apply when ``@pytest.mark.parametrize`` is given an empty set of parameters. + Valid options are ``skip`` (default) and ``xfail``. Note that it is planned + to change the default to ``xfail`` in future releases as this is considered + less error prone. (:issue:`2527`) + +- **Incompatible change**: after community feedback the :doc:`logging ` functionality has + undergone some changes. Please consult the :ref:`logging documentation ` + for details. (:issue:`3013`) + +- Console output falls back to "classic" mode when capturing is disabled (``-s``), + otherwise the output gets garbled to the point of being useless. (:issue:`3038`) + +- New :hook:`pytest_runtest_logfinish` + hook which is called when a test item has finished executing, analogous to + :hook:`pytest_runtest_logstart`. + (:issue:`3101`) + +- Improve performance when collecting tests using many fixtures. (:issue:`3107`) + +- New ``caplog.get_records(when)`` method which provides access to the captured + records for the ``"setup"``, ``"call"`` and ``"teardown"`` + testing stages. (:issue:`3117`) + +- New fixture ``record_xml_attribute`` that allows modifying and inserting + attributes on the ```` xml node in JUnit reports. (:issue:`3130`) + +- The default cache directory has been renamed from ``.cache`` to + ``.pytest_cache`` after community feedback that the name ``.cache`` did not + make it clear that it was used by pytest. (:issue:`3138`) + +- Colorize the levelname column in the live-log output. (:issue:`3142`) + + +Bug Fixes +--------- + +- Fix hanging pexpect test on MacOS by using flush() instead of wait(). + (:issue:`2022`) + +- Fix restoring Python state after in-process pytest runs with the + ``pytester`` plugin; this may break tests using multiple inprocess + pytest runs if later ones depend on earlier ones leaking global interpreter + changes. (:issue:`3016`) + +- Fix skipping plugin reporting hook when test aborted before plugin setup + hook. (:issue:`3074`) + +- Fix progress percentage reported when tests fail during teardown. (:issue:`3088`) + +- **Incompatible change**: ``-o/--override`` option no longer eats all the + remaining options, which can lead to surprising behavior: for example, + ``pytest -o foo=1 /path/to/test.py`` would fail because ``/path/to/test.py`` + would be considered as part of the ``-o`` command-line argument. One + consequence of this is that now multiple configuration overrides need + multiple ``-o`` flags: ``pytest -o foo=1 -o bar=2``. (:issue:`3103`) + + +Improved Documentation +---------------------- + +- Document hooks (defined with ``historic=True``) which cannot be used with + ``hookwrapper=True``. (:issue:`2423`) + +- Clarify that warning capturing doesn't change the warning filter by default. + (:issue:`2457`) + +- Clarify a possible confusion when using pytest_fixture_setup with fixture + functions that return None. (:issue:`2698`) + +- Fix the wording of a sentence on doctest flags used in pytest. (:issue:`3076`) + +- Prefer ``https://*.readthedocs.io`` over ``http://*.rtfd.org`` for links in + the documentation. (:issue:`3092`) + +- Improve readability (wording, grammar) of Getting Started guide (:issue:`3131`) + +- Added note that calling pytest.main multiple times from the same process is + not recommended because of import caching. (:issue:`3143`) + + +Trivial/Internal Changes +------------------------ + +- Show a simple and easy error when keyword expressions trigger a syntax error + (for example, ``"-k foo and import"`` will show an error that you can not use + the ``import`` keyword in expressions). (:issue:`2953`) + +- Change parametrized automatic test id generation to use the ``__name__`` + attribute of functions instead of the fallback argument name plus counter. + (:issue:`2976`) + +- Replace py.std with stdlib imports. (:issue:`3067`) + +- Corrected 'you' to 'your' in logging docs. (:issue:`3129`) + + +pytest 3.3.2 (2017-12-25) +========================= + +Bug Fixes +--------- + +- pytester: ignore files used to obtain current user metadata in the fd leak + detector. (:issue:`2784`) + +- Fix **memory leak** where objects returned by fixtures were never destructed + by the garbage collector. (:issue:`2981`) + +- Fix conversion of pyargs to filename to not convert symlinks on Python 2. (:issue:`2985`) + +- ``PYTEST_DONT_REWRITE`` is now checked for plugins too rather than only for + test modules. (:issue:`2995`) + + +Improved Documentation +---------------------- + +- Add clarifying note about behavior of multiple parametrized arguments (:issue:`3001`) + + +Trivial/Internal Changes +------------------------ + +- Code cleanup. (:issue:`3015`, + :issue:`3021`) + +- Clean up code by replacing imports and references of ``_ast`` to ``ast``. + (:issue:`3018`) + + +pytest 3.3.1 (2017-12-05) +========================= + +Bug Fixes +--------- + +- Fix issue about ``-p no:`` having no effect. (:issue:`2920`) + +- Fix regression with warnings that contained non-strings in their arguments in + Python 2. (:issue:`2956`) + +- Always escape null bytes when setting ``PYTEST_CURRENT_TEST``. (:issue:`2957`) + +- Fix ``ZeroDivisionError`` when using the ``testmon`` plugin when no tests + were actually collected. (:issue:`2971`) + +- Bring back ``TerminalReporter.writer`` as an alias to + ``TerminalReporter._tw``. This alias was removed by accident in the ``3.3.0`` + release. (:issue:`2984`) + +- The ``pytest-capturelog`` plugin is now also blacklisted, avoiding errors when + running pytest with it still installed. (:issue:`3004`) + + +Improved Documentation +---------------------- + +- Fix broken link to plugin ``pytest-localserver``. (:issue:`2963`) + + +Trivial/Internal Changes +------------------------ + +- Update github "bugs" link in ``CONTRIBUTING.rst`` (:issue:`2949`) + + +pytest 3.3.0 (2017-11-23) +========================= + +Deprecations and Removals +------------------------- + +- pytest no longer supports Python **2.6** and **3.3**. Those Python versions + are EOL for some time now and incur maintenance and compatibility costs on + the pytest core team, and following up with the rest of the community we + decided that they will no longer be supported starting on this version. Users + which still require those versions should pin pytest to ``<3.3``. (:issue:`2812`) + +- Remove internal ``_preloadplugins()`` function. This removal is part of the + ``pytest_namespace()`` hook deprecation. (:issue:`2636`) + +- Internally change ``CallSpec2`` to have a list of marks instead of a broken + mapping of keywords. This removes the keywords attribute of the internal + ``CallSpec2`` class. (:issue:`2672`) + +- Remove ParameterSet.deprecated_arg_dict - its not a public api and the lack + of the underscore was a naming error. (:issue:`2675`) + +- Remove the internal multi-typed attribute ``Node._evalskip`` and replace it + with the boolean ``Node._skipped_by_mark``. (:issue:`2767`) + +- The ``params`` list passed to ``pytest.fixture`` is now for + all effects considered immutable and frozen at the moment of the ``pytest.fixture`` + call. Previously the list could be changed before the first invocation of the fixture + allowing for a form of dynamic parametrization (for example, updated from command-line options), + but this was an unwanted implementation detail which complicated the internals and prevented + some internal cleanup. See issue :issue:`2959` + for details and a recommended workaround. + +Features +-------- + +- ``pytest_fixture_post_finalizer`` hook can now receive a ``request`` + argument. (:issue:`2124`) + +- Replace the old introspection code in compat.py that determines the available + arguments of fixtures with inspect.signature on Python 3 and + funcsigs.signature on Python 2. This should respect ``__signature__`` + declarations on functions. (:issue:`2267`) + +- Report tests with global ``pytestmark`` variable only once. (:issue:`2549`) + +- Now pytest displays the total progress percentage while running tests. The + previous output style can be set by configuring the ``console_output_style`` + setting to ``classic``. (:issue:`2657`) + +- Match ``warns`` signature to ``raises`` by adding ``match`` keyword. (:issue:`2708`) + +- pytest now captures and displays output from the standard ``logging`` module. + The user can control the logging level to be captured by specifying options + in ``pytest.ini``, the command line and also during individual tests using + markers. Also, a ``caplog`` fixture is available that enables users to test + the captured log during specific tests (similar to ``capsys`` for example). + For more information, please see the :doc:`logging docs `. This feature was + introduced by merging the popular :pypi:`pytest-catchlog` plugin, thanks to :user:`thisch`. + Be advised that during the merging the + backward compatibility interface with the defunct ``pytest-capturelog`` has + been dropped. (:issue:`2794`) + +- Add ``allow_module_level`` kwarg to ``pytest.skip()``, enabling to skip the + whole module. (:issue:`2808`) + +- Allow setting ``file_or_dir``, ``-c``, and ``-o`` in PYTEST_ADDOPTS. (:issue:`2824`) + +- Return stdout/stderr capture results as a ``namedtuple``, so ``out`` and + ``err`` can be accessed by attribute. (:issue:`2879`) + +- Add ``capfdbinary``, a version of ``capfd`` which returns bytes from + ``readouterr()``. (:issue:`2923`) + +- Add ``capsysbinary`` a version of ``capsys`` which returns bytes from + ``readouterr()``. (:issue:`2934`) + +- Implement feature to skip ``setup.py`` files when run with + ``--doctest-modules``. (:issue:`502`) + + +Bug Fixes +--------- + +- Resume output capturing after ``capsys/capfd.disabled()`` context manager. + (:issue:`1993`) + +- ``pytest_fixture_setup`` and ``pytest_fixture_post_finalizer`` hooks are now + called for all ``conftest.py`` files. (:issue:`2124`) + +- If an exception happens while loading a plugin, pytest no longer hides the + original traceback. In Python 2 it will show the original traceback with a new + message that explains in which plugin. In Python 3 it will show 2 canonized + exceptions, the original exception while loading the plugin in addition to an + exception that pytest throws about loading a plugin. (:issue:`2491`) + +- ``capsys`` and ``capfd`` can now be used by other fixtures. (:issue:`2709`) + +- Internal ``pytester`` plugin properly encodes ``bytes`` arguments to + ``utf-8``. (:issue:`2738`) + +- ``testdir`` now uses use the same method used by ``tmpdir`` to create its + temporary directory. This changes the final structure of the ``testdir`` + directory slightly, but should not affect usage in normal scenarios and + avoids a number of potential problems. (:issue:`2751`) + +- pytest no longer complains about warnings with unicode messages being + non-ascii compatible even for ascii-compatible messages. As a result of this, + warnings with unicode messages are converted first to an ascii representation + for safety. (:issue:`2809`) + +- Change return value of pytest command when ``--maxfail`` is reached from + ``2`` (interrupted) to ``1`` (failed). (:issue:`2845`) + +- Fix issue in assertion rewriting which could lead it to rewrite modules which + should not be rewritten. (:issue:`2939`) + +- Handle marks without description in ``pytest.ini``. (:issue:`2942`) + + +Trivial/Internal Changes +------------------------ + +- pytest now depends on :pypi:`attrs` for internal + structures to ease code maintainability. (:issue:`2641`) + +- Refactored internal Python 2/3 compatibility code to use ``six``. (:issue:`2642`) + +- Stop vendoring ``pluggy`` - we're missing out on its latest changes for not + much benefit (:issue:`2719`) + +- Internal refactor: simplify ascii string escaping by using the + backslashreplace error handler in newer Python 3 versions. (:issue:`2734`) + +- Remove unnecessary mark evaluator in unittest plugin (:issue:`2767`) + +- Calls to ``Metafunc.addcall`` now emit a deprecation warning. This function + is scheduled to be removed in ``pytest-4.0``. (:issue:`2876`) + +- Internal move of the parameterset extraction to a more maintainable place. + (:issue:`2877`) + +- Internal refactoring to simplify scope node lookup. (:issue:`2910`) + +- Configure ``pytest`` to prevent pip from installing pytest in unsupported + Python versions. (:issue:`2922`) + + +pytest 3.2.5 (2017-11-15) +========================= + +Bug Fixes +--------- + +- Remove ``py<1.5`` restriction from ``pytest`` as this can cause version + conflicts in some installations. (:issue:`2926`) + + +pytest 3.2.4 (2017-11-13) +========================= + +Bug Fixes +--------- + +- Fix the bug where running with ``--pyargs`` will result in items with + empty ``parent.nodeid`` if run from a different root directory. (:issue:`2775`) + +- Fix issue with ``@pytest.parametrize`` if argnames was specified as keyword arguments. + (:issue:`2819`) + +- Strip whitespace from marker names when reading them from INI config. (:issue:`2856`) + +- Show full context of doctest source in the pytest output, if the line number of + failed example in the docstring is < 9. (:issue:`2882`) + +- Match fixture paths against actual path segments in order to avoid matching folders which share a prefix. + (:issue:`2836`) + +Improved Documentation +---------------------- + +- Introduce a dedicated section about conftest.py. (:issue:`1505`) + +- Explicitly mention ``xpass`` in the documentation of ``xfail``. (:issue:`1997`) + +- Append example for pytest.param in the example/parametrize document. (:issue:`2658`) + +- Clarify language of proposal for fixtures parameters (:issue:`2893`) + +- List python 3.6 in the documented supported versions in the getting started + document. (:issue:`2903`) + +- Clarify the documentation of available fixture scopes. (:issue:`538`) + +- Add documentation about the ``python -m pytest`` invocation adding the + current directory to sys.path. (:issue:`911`) + + +pytest 3.2.3 (2017-10-03) +========================= + +Bug Fixes +--------- + +- Fix crash in tab completion when no prefix is given. (:issue:`2748`) + +- The equality checking function (``__eq__``) of ``MarkDecorator`` returns + ``False`` if one object is not an instance of ``MarkDecorator``. (:issue:`2758`) + +- When running ``pytest --fixtures-per-test``: don't crash if an item has no + _fixtureinfo attribute (e.g. doctests) (:issue:`2788`) + + +Improved Documentation +---------------------- + +- In help text of ``-k`` option, add example of using ``not`` to not select + certain tests whose names match the provided expression. (:issue:`1442`) + +- Add note in ``parametrize.rst`` about calling ``metafunc.parametrize`` + multiple times. (:issue:`1548`) + + +Trivial/Internal Changes +------------------------ + +- Set ``xfail_strict=True`` in pytest's own test suite to catch expected + failures as soon as they start to pass. (:issue:`2722`) + +- Fix typo in example of passing a callable to markers (in example/markers.rst) + (:issue:`2765`) + + +pytest 3.2.2 (2017-09-06) +========================= + +Bug Fixes +--------- + +- Calling the deprecated ``request.getfuncargvalue()`` now shows the source of + the call. (:issue:`2681`) + +- Allow tests declared as ``@staticmethod`` to use fixtures. (:issue:`2699`) + +- Fixed edge-case during collection: attributes which raised ``pytest.fail`` + when accessed would abort the entire collection. (:issue:`2707`) + +- Fix ``ReprFuncArgs`` with mixed unicode and UTF-8 args. (:issue:`2731`) + + +Improved Documentation +---------------------- + +- In examples on working with custom markers, add examples demonstrating the + usage of ``pytest.mark.MARKER_NAME.with_args`` in comparison with + ``pytest.mark.MARKER_NAME.__call__`` (:issue:`2604`) + +- In one of the simple examples, use ``pytest_collection_modifyitems()`` to skip + tests based on a command-line option, allowing its sharing while preventing a + user error when accessing ``pytest.config`` before the argument parsing. + (:issue:`2653`) + + +Trivial/Internal Changes +------------------------ + +- Fixed minor error in 'Good Practices/Manual Integration' code snippet. + (:issue:`2691`) + +- Fixed typo in goodpractices.rst. (:issue:`2721`) + +- Improve user guidance regarding ``--resultlog`` deprecation. (:issue:`2739`) + + +pytest 3.2.1 (2017-08-08) +========================= + +Bug Fixes +--------- + +- Fixed small terminal glitch when collecting a single test item. (:issue:`2579`) + +- Correctly consider ``/`` as the file separator to automatically mark plugin + files for rewrite on Windows. (:issue:`2591`) + +- Properly escape test names when setting ``PYTEST_CURRENT_TEST`` environment + variable. (:issue:`2644`) + +- Fix error on Windows and Python 3.6+ when ``sys.stdout`` has been replaced + with a stream-like object which does not implement the full ``io`` module + buffer protocol. In particular this affects ``pytest-xdist`` users on the + aforementioned platform. (:issue:`2666`) + + +Improved Documentation +---------------------- + +- Explicitly document which pytest features work with ``unittest``. (:issue:`2626`) + + +pytest 3.2.0 (2017-07-30) +========================= + +Deprecations and Removals +------------------------- + +- ``pytest.approx`` no longer supports ``>``, ``>=``, ``<`` and ``<=`` + operators to avoid surprising/inconsistent behavior. See the :func:`~pytest.approx` docs for more + information. (:issue:`2003`) + +- All old-style specific behavior in current classes in the pytest's API is + considered deprecated at this point and will be removed in a future release. + This affects Python 2 users only and in rare situations. (:issue:`2147`) + +- A deprecation warning is now raised when using marks for parameters + in ``pytest.mark.parametrize``. Use ``pytest.param`` to apply marks to + parameters instead. (:issue:`2427`) + + +Features +-------- + +- Add support for numpy arrays (and dicts) to approx. (:issue:`1994`) + +- Now test function objects have a ``pytestmark`` attribute containing a list + of marks applied directly to the test function, as opposed to marks inherited + from parent classes or modules. (:issue:`2516`) + +- Collection ignores local virtualenvs by default; ``--collect-in-virtualenv`` + overrides this behavior. (:issue:`2518`) + +- Allow class methods decorated as ``@staticmethod`` to be candidates for + collection as a test function. (Only for Python 2.7 and above. Python 2.6 + will still ignore static methods.) (:issue:`2528`) + +- Introduce ``mark.with_args`` in order to allow passing functions/classes as + sole argument to marks. (:issue:`2540`) + +- New ``cache_dir`` ini option: sets the directory where the contents of the + cache plugin are stored. Directory may be relative or absolute path: if relative path, then + directory is created relative to ``rootdir``, otherwise it is used as is. + Additionally path may contain environment variables which are expanded during + runtime. (:issue:`2543`) + +- Introduce the ``PYTEST_CURRENT_TEST`` environment variable that is set with + the ``nodeid`` and stage (``setup``, ``call`` and ``teardown``) of the test + being currently executed. See the :ref:`documentation ` + for more info. (:issue:`2583`) + +- Introduced ``@pytest.mark.filterwarnings`` mark which allows overwriting the + warnings filter on a per test, class or module level. See the :ref:`docs ` + for more information. (:issue:`2598`) + +- ``--last-failed`` now remembers forever when a test has failed and only + forgets it if it passes again. This makes it easy to fix a test suite by + selectively running files and fixing tests incrementally. (:issue:`2621`) + +- New ``pytest_report_collectionfinish`` hook which allows plugins to add + messages to the terminal reporting after collection has been finished + successfully. (:issue:`2622`) + +- Added support for :pep:`415`\'s + ``Exception.__suppress_context__``. Now if a ``raise exception from None`` is + caught by pytest, pytest will no longer chain the context in the test report. + The behavior now matches Python's traceback behavior. (:issue:`2631`) + +- Exceptions raised by ``pytest.fail``, ``pytest.skip`` and ``pytest.xfail`` + now subclass BaseException, making them harder to be caught unintentionally + by normal code. (:issue:`580`) + + +Bug Fixes +--------- + +- Set ``stdin`` to a closed ``PIPE`` in ``pytester.py.Testdir.popen()`` for + avoid unwanted interactive ``pdb`` (:issue:`2023`) + +- Add missing ``encoding`` attribute to ``sys.std*`` streams when using + ``capsys`` capture mode. (:issue:`2375`) + +- Fix terminal color changing to black on Windows if ``colorama`` is imported + in a ``conftest.py`` file. (:issue:`2510`) + +- Fix line number when reporting summary of skipped tests. (:issue:`2548`) + +- capture: ensure that EncodedFile.name is a string. (:issue:`2555`) + +- The options ``--fixtures`` and ``--fixtures-per-test`` will now keep + indentation within docstrings. (:issue:`2574`) + +- doctests line numbers are now reported correctly, fixing `pytest-sugar#122 + `_. (:issue:`2610`) + +- Fix non-determinism in order of fixture collection. Adds new dependency + (ordereddict) for Python 2.6. (:issue:`920`) + + +Improved Documentation +---------------------- + +- Clarify ``pytest_configure`` hook call order. (:issue:`2539`) + +- Extend documentation for testing plugin code with the ``pytester`` plugin. + (:issue:`971`) + + +Trivial/Internal Changes +------------------------ + +- Update help message for ``--strict`` to make it clear it only deals with + unregistered markers, not warnings. (:issue:`2444`) + +- Internal code move: move code for pytest.approx/pytest.raises to own files in + order to cut down the size of python.py (:issue:`2489`) + +- Renamed the utility function ``_pytest.compat._escape_strings`` to + ``_ascii_escaped`` to better communicate the function's purpose. (:issue:`2533`) + +- Improve error message for CollectError with skip/skipif. (:issue:`2546`) + +- Emit warning about ``yield`` tests being deprecated only once per generator. + (:issue:`2562`) + +- Ensure final collected line doesn't include artifacts of previous write. + (:issue:`2571`) + +- Fixed all flake8 errors and warnings. (:issue:`2581`) + +- Added ``fix-lint`` tox environment to run automatic pep8 fixes on the code. + (:issue:`2582`) + +- Turn warnings into errors in pytest's own test suite in order to catch + regressions due to deprecations more promptly. (:issue:`2588`) + +- Show multiple issue links in CHANGELOG entries. (:issue:`2620`) + + +pytest 3.1.3 (2017-07-03) +========================= + +Bug Fixes +--------- + +- Fix decode error in Python 2 for doctests in docstrings. (:issue:`2434`) + +- Exceptions raised during teardown by finalizers are now suppressed until all + finalizers are called, with the initial exception reraised. (:issue:`2440`) + +- Fix incorrect "collected items" report when specifying tests on the command- + line. (:issue:`2464`) + +- ``deprecated_call`` in context-manager form now captures deprecation warnings + even if the same warning has already been raised. Also, ``deprecated_call`` + will always produce the same error message (previously it would produce + different messages in context-manager vs. function-call mode). (:issue:`2469`) + +- Fix issue where paths collected by pytest could have triple leading ``/`` + characters. (:issue:`2475`) + +- Fix internal error when trying to detect the start of a recursive traceback. + (:issue:`2486`) + + +Improved Documentation +---------------------- + +- Explicitly state for which hooks the calls stop after the first non-None + result. (:issue:`2493`) + + +Trivial/Internal Changes +------------------------ + +- Create invoke tasks for updating the vendored packages. (:issue:`2474`) + +- Update copyright dates in LICENSE, README.rst and in the documentation. + (:issue:`2499`) + + +pytest 3.1.2 (2017-06-08) +========================= + +Bug Fixes +--------- + +- Required options added via ``pytest_addoption`` will no longer prevent using + --help without passing them. (#1999) + +- Respect ``python_files`` in assertion rewriting. (#2121) + +- Fix recursion error detection when frames in the traceback contain objects + that can't be compared (like ``numpy`` arrays). (#2459) + +- ``UnicodeWarning`` is issued from the internal pytest warnings plugin only + when the message contains non-ascii unicode (Python 2 only). (#2463) + +- Added a workaround for Python 3.6 ``WindowsConsoleIO`` breaking due to Pytests's + ``FDCapture``. Other code using console handles might still be affected by the + very same issue and might require further workarounds/fixes, i.e. ``colorama``. + (#2467) + + +Improved Documentation +---------------------- + +- Fix internal API links to ``pluggy`` objects. (#2331) + +- Make it clear that ``pytest.xfail`` stops test execution at the calling point + and improve overall flow of the ``skipping`` docs. (#810) + + +pytest 3.1.1 (2017-05-30) +========================= + +Bug Fixes +--------- + +- pytest warning capture no longer overrides existing warning filters. The + previous behaviour would override all filters and caused regressions in test + suites which configure warning filters to match their needs. Note that as a + side-effect of this is that ``DeprecationWarning`` and + ``PendingDeprecationWarning`` are no longer shown by default. (#2430) + +- Fix issue with non-ascii contents in doctest text files. (#2434) + +- Fix encoding errors for unicode warnings in Python 2. (#2436) + +- ``pytest.deprecated_call`` now captures ``PendingDeprecationWarning`` in + context manager form. (#2441) + + +Improved Documentation +---------------------- + +- Addition of towncrier for changelog management. (#2390) + + +3.1.0 (2017-05-22) +================== + + +New Features +------------ + +* The ``pytest-warnings`` plugin has been integrated into the core and now ``pytest`` automatically + captures and displays warnings at the end of the test session. + + .. warning:: + + This feature may disrupt test suites which apply and treat warnings themselves, and can be + disabled in your ``pytest.ini``: + + .. code-block:: ini + + [pytest] + addopts = -p no:warnings + + See the :doc:`warnings documentation page ` for more + information. + + Thanks :user:`nicoddemus` for the PR. + +* Added ``junit_suite_name`` ini option to specify root ```` name for JUnit XML reports (:issue:`533`). + +* Added an ini option ``doctest_encoding`` to specify which encoding to use for doctest files. + Thanks :user:`wheerd` for the PR (:pull:`2101`). + +* ``pytest.warns`` now checks for subclass relationship rather than + class equality. Thanks :user:`lesteve` for the PR (:pull:`2166`) + +* ``pytest.raises`` now asserts that the error message matches a text or regex + with the ``match`` keyword argument. Thanks :user:`Kriechi` for the PR. + +* ``pytest.param`` can be used to declare test parameter sets with marks and test ids. + Thanks :user:`RonnyPfannschmidt` for the PR. + + +Changes +------- + +* remove all internal uses of pytest_namespace hooks, + this is to prepare the removal of preloadconfig in pytest 4.0 + Thanks to :user:`RonnyPfannschmidt` for the PR. + +* pytest now warns when a callable ids raises in a parametrized test. Thanks :user:`fogo` for the PR. + +* It is now possible to skip test classes from being collected by setting a + ``__test__`` attribute to ``False`` in the class body (:issue:`2007`). Thanks + to :user:`syre` for the report and :user:`lwm` for the PR. + +* Change junitxml.py to produce reports that comply with Junitxml schema. + If the same test fails with failure in call and then errors in teardown + we split testcase element into two, one containing the error and the other + the failure. (:issue:`2228`) Thanks to :user:`kkoukiou` for the PR. + +* Testcase reports with a ``url`` attribute will now properly write this to junitxml. + Thanks :user:`fushi` for the PR (:pull:`1874`). + +* Remove common items from dict comparison output when verbosity=1. Also update + the truncation message to make it clearer that pytest truncates all + assertion messages if verbosity < 2 (:issue:`1512`). + Thanks :user:`mattduck` for the PR + +* ``--pdbcls`` no longer implies ``--pdb``. This makes it possible to use + ``addopts=--pdbcls=module.SomeClass`` on ``pytest.ini``. Thanks :user:`davidszotten` for + the PR (:pull:`1952`). + +* fix :issue:`2013`: turn RecordedWarning into ``namedtuple``, + to give it a comprehensible repr while preventing unwarranted modification. + +* fix :issue:`2208`: ensure an iteration limit for _pytest.compat.get_real_func. + Thanks :user:`RonnyPfannschmidt` for the report and PR. + +* Hooks are now verified after collection is complete, rather than right after loading installed plugins. This + makes it easy to write hooks for plugins which will be loaded during collection, for example using the + ``pytest_plugins`` special variable (:issue:`1821`). + Thanks :user:`nicoddemus` for the PR. + +* Modify ``pytest_make_parametrize_id()`` hook to accept ``argname`` as an + additional parameter. + Thanks :user:`unsignedint` for the PR. + +* Add ``venv`` to the default ``norecursedirs`` setting. + Thanks :user:`The-Compiler` for the PR. + +* ``PluginManager.import_plugin`` now accepts unicode plugin names in Python 2. + Thanks :user:`reutsharabani` for the PR. + +* fix :issue:`2308`: When using both ``--lf`` and ``--ff``, only the last failed tests are run. + Thanks :user:`ojii` for the PR. + +* Replace minor/patch level version numbers in the documentation with placeholders. + This significantly reduces change-noise as different contributors regenerate + the documentation on different platforms. + Thanks :user:`RonnyPfannschmidt` for the PR. + +* fix :issue:`2391`: consider pytest_plugins on all plugin modules + Thanks :user:`RonnyPfannschmidt` for the PR. + + +Bug Fixes +--------- + +* Fix ``AttributeError`` on ``sys.stdout.buffer`` / ``sys.stderr.buffer`` + while using ``capsys`` fixture in python 3. (:issue:`1407`). + Thanks to :user:`asottile`. + +* Change capture.py's ``DontReadFromInput`` class to throw ``io.UnsupportedOperation`` errors rather + than ValueErrors in the ``fileno`` method (:issue:`2276`). + Thanks :user:`metasyn` and :user:`vlad-dragos` for the PR. + +* Fix exception formatting while importing modules when the exception message + contains non-ascii characters (:issue:`2336`). + Thanks :user:`fabioz` for the report and :user:`nicoddemus` for the PR. + +* Added documentation related to issue (:issue:`1937`) + Thanks :user:`skylarjhdownes` for the PR. + +* Allow collecting files with any file extension as Python modules (:issue:`2369`). + Thanks :user:`Kodiologist` for the PR. + +* Show the correct error message when collect "parametrize" func with wrong args (:issue:`2383`). + Thanks :user:`The-Compiler` for the report and :user:`robin0371` for the PR. + + +3.0.7 (2017-03-14) +================== + + +* Fix issue in assertion rewriting breaking due to modules silently discarding + other modules when importing fails + Notably, importing the ``anydbm`` module is fixed. (:issue:`2248`). + Thanks :user:`pfhayes` for the PR. + +* junitxml: Fix problematic case where system-out tag occurred twice per testcase + element in the XML report. Thanks :user:`kkoukiou` for the PR. + +* Fix regression, pytest now skips unittest correctly if run with ``--pdb`` + (:issue:`2137`). Thanks to :user:`gst` for the report and :user:`mbyt` for the PR. + +* Ignore exceptions raised from descriptors (e.g. properties) during Python test collection (:issue:`2234`). + Thanks to :user:`bluetech`. + +* ``--override-ini`` now correctly overrides some fundamental options like ``python_files`` (:issue:`2238`). + Thanks :user:`sirex` for the report and :user:`nicoddemus` for the PR. + +* Replace ``raise StopIteration`` usages in the code by simple ``returns`` to finish generators, in accordance to :pep:`479` (:issue:`2160`). + Thanks to :user:`nicoddemus` for the PR. + +* Fix internal errors when an unprintable ``AssertionError`` is raised inside a test. + Thanks :user:`omerhadari` for the PR. + +* Skipping plugin now also works with test items generated by custom collectors (:issue:`2231`). + Thanks to :user:`vidartf`. + +* Fix trailing whitespace in console output if no .ini file presented (:issue:`2281`). Thanks :user:`fbjorn` for the PR. + +* Conditionless ``xfail`` markers no longer rely on the underlying test item + being an instance of ``PyobjMixin``, and can therefore apply to tests not + collected by the built-in python test collector. Thanks :user:`barneygale` for the + PR. + + +3.0.6 (2017-01-22) +================== + +* pytest no longer generates ``PendingDeprecationWarning`` from its own operations, which was introduced by mistake in version ``3.0.5`` (:issue:`2118`). + Thanks to :user:`nicoddemus` for the report and :user:`RonnyPfannschmidt` for the PR. + + +* pytest no longer recognizes coroutine functions as yield tests (:issue:`2129`). + Thanks to :user:`malinoff` for the PR. + +* Plugins loaded by the ``PYTEST_PLUGINS`` environment variable are now automatically + considered for assertion rewriting (:issue:`2185`). + Thanks :user:`nicoddemus` for the PR. + +* Improve error message when pytest.warns fails (:issue:`2150`). The type(s) of the + expected warnings and the list of caught warnings is added to the + error message. Thanks :user:`lesteve` for the PR. + +* Fix ``pytester`` internal plugin to work correctly with latest versions of + ``zope.interface`` (:issue:`1989`). Thanks :user:`nicoddemus` for the PR. + +* Assert statements of the ``pytester`` plugin again benefit from assertion rewriting (:issue:`1920`). + Thanks :user:`RonnyPfannschmidt` for the report and :user:`nicoddemus` for the PR. + +* Specifying tests with colons like ``test_foo.py::test_bar`` for tests in + subdirectories with ini configuration files now uses the correct ini file + (:issue:`2148`). Thanks :user:`pelme`. + +* Fail ``testdir.runpytest().assert_outcomes()`` explicitly if the pytest + terminal output it relies on is missing. Thanks to :user:`eli-b` for the PR. + + +3.0.5 (2016-12-05) +================== + +* Add warning when not passing ``option=value`` correctly to ``-o/--override-ini`` (:issue:`2105`). + Also improved the help documentation. Thanks to :user:`mbukatov` for the report and + :user:`lwm` for the PR. + +* Now ``--confcutdir`` and ``--junit-xml`` are properly validated if they are directories + and filenames, respectively (:issue:`2089` and :issue:`2078`). Thanks to :user:`lwm` for the PR. + +* Add hint to error message hinting possible missing ``__init__.py`` (:issue:`478`). Thanks :user:`DuncanBetts`. + +* More accurately describe when fixture finalization occurs in documentation (:issue:`687`). Thanks :user:`DuncanBetts`. + +* Provide ``:ref:`` targets for ``recwarn.rst`` so we can use intersphinx referencing. + Thanks to :user:`dupuy` for the report and :user:`lwm` for the PR. + +* In Python 2, use a simple ``+-`` ASCII string in the string representation of ``pytest.approx`` (for example ``"4 +- 4.0e-06"``) + because it is brittle to handle that in different contexts and representations internally in pytest + which can result in bugs such as :issue:`2111`. In Python 3, the representation still uses ``±`` (for example ``4 ± 4.0e-06``). + Thanks :user:`kerrick-lyft` for the report and :user:`nicoddemus` for the PR. + +* Using ``item.Function``, ``item.Module``, etc., is now issuing deprecation warnings, prefer + ``pytest.Function``, ``pytest.Module``, etc., instead (:issue:`2034`). + Thanks :user:`nmundar` for the PR. + +* Fix error message using ``approx`` with complex numbers (:issue:`2082`). + Thanks :user:`adler-j` for the report and :user:`nicoddemus` for the PR. + +* Fixed false-positives warnings from assertion rewrite hook for modules imported more than + once by the ``pytest_plugins`` mechanism. + Thanks :user:`nicoddemus` for the PR. + +* Remove an internal cache which could cause hooks from ``conftest.py`` files in + sub-directories to be called in other directories incorrectly (:issue:`2016`). + Thanks :user:`d-b-w` for the report and :user:`nicoddemus` for the PR. + +* Remove internal code meant to support earlier Python 3 versions that produced the side effect + of leaving ``None`` in ``sys.modules`` when expressions were evaluated by pytest (for example passing a condition + as a string to ``pytest.mark.skipif``)(:issue:`2103`). + Thanks :user:`jaraco` for the report and :user:`nicoddemus` for the PR. + +* Cope gracefully with a .pyc file with no matching .py file (:issue:`2038`). Thanks + :user:`nedbat`. + + +3.0.4 (2016-11-09) +================== + +* Import errors when collecting test modules now display the full traceback (:issue:`1976`). + Thanks :user:`cwitty` for the report and :user:`nicoddemus` for the PR. + +* Fix confusing command-line help message for custom options with two or more ``metavar`` properties (:issue:`2004`). + Thanks :user:`okulynyak` and :user:`davehunt` for the report and :user:`nicoddemus` for the PR. + +* When loading plugins, import errors which contain non-ascii messages are now properly handled in Python 2 (:issue:`1998`). + Thanks :user:`nicoddemus` for the PR. + +* Fixed cyclic reference when ``pytest.raises`` is used in context-manager form (:issue:`1965`). Also as a + result of this fix, ``sys.exc_info()`` is left empty in both context-manager and function call usages. + Previously, ``sys.exc_info`` would contain the exception caught by the context manager, + even when the expected exception occurred. + Thanks :user:`MSeifert04` for the report and the PR. + +* Fixed false-positives warnings from assertion rewrite hook for modules that were rewritten but + were later marked explicitly by ``pytest.register_assert_rewrite`` + or implicitly as a plugin (:issue:`2005`). + Thanks :user:`RonnyPfannschmidt` for the report and :user:`nicoddemus` for the PR. + +* Report teardown output on test failure (:issue:`442`). + Thanks :user:`matclab` for the PR. + +* Fix teardown error message in generated xUnit XML. + Thanks :user:`gdyuldin` for the PR. + +* Properly handle exceptions in ``multiprocessing`` tasks (:issue:`1984`). + Thanks :user:`adborden` for the report and :user:`nicoddemus` for the PR. + +* Clean up unittest TestCase objects after tests are complete (:issue:`1649`). + Thanks :user:`d_b_w` for the report and PR. + + +3.0.3 (2016-09-28) +================== + +* The ``ids`` argument to ``parametrize`` again accepts ``unicode`` strings + in Python 2 (:issue:`1905`). + Thanks :user:`philpep` for the report and :user:`nicoddemus` for the PR. + +* Assertions are now being rewritten for plugins in development mode + (``pip install -e``) (:issue:`1934`). + Thanks :user:`nicoddemus` for the PR. + +* Fix pkg_resources import error in Jython projects (:issue:`1853`). + Thanks :user:`raquel-ucl` for the PR. + +* Got rid of ``AttributeError: 'Module' object has no attribute '_obj'`` exception + in Python 3 (:issue:`1944`). + Thanks :user:`axil` for the PR. + +* Explain a bad scope value passed to ``@fixture`` declarations or + a ``MetaFunc.parametrize()`` call. + +* This version includes ``pluggy-0.4.0``, which correctly handles + ``VersionConflict`` errors in plugins (:issue:`704`). + Thanks :user:`nicoddemus` for the PR. + + +3.0.2 (2016-09-01) +================== + +* Improve error message when passing non-string ids to ``pytest.mark.parametrize`` (:issue:`1857`). + Thanks :user:`okken` for the report and :user:`nicoddemus` for the PR. + +* Add ``buffer`` attribute to stdin stub class ``pytest.capture.DontReadFromInput`` + Thanks :user:`joguSD` for the PR. + +* Fix ``UnicodeEncodeError`` when string comparison with unicode has failed. (:issue:`1864`) + Thanks :user:`AiOO` for the PR. + +* ``pytest_plugins`` is now handled correctly if defined as a string (as opposed as + a sequence of strings) when modules are considered for assertion rewriting. + Due to this bug, much more modules were being rewritten than necessary + if a test suite uses ``pytest_plugins`` to load internal plugins (:issue:`1888`). + Thanks :user:`jaraco` for the report and :user:`nicoddemus` for the PR (:pull:`1891`). + +* Do not call tearDown and cleanups when running tests from + ``unittest.TestCase`` subclasses with ``--pdb`` + enabled. This allows proper post mortem debugging for all applications + which have significant logic in their tearDown machinery (:issue:`1890`). Thanks + :user:`mbyt` for the PR. + +* Fix use of deprecated ``getfuncargvalue`` method in the internal doctest plugin. + Thanks :user:`ViviCoder` for the report (:issue:`1898`). + + +3.0.1 (2016-08-23) +================== + +* Fix regression when ``importorskip`` is used at module level (:issue:`1822`). + Thanks :user:`jaraco` and :user:`The-Compiler` for the report and :user:`nicoddemus` for the PR. + +* Fix parametrization scope when session fixtures are used in conjunction + with normal parameters in the same call (:issue:`1832`). + Thanks :user:`The-Compiler` for the report, :user:`Kingdread` and :user:`nicoddemus` for the PR. + +* Fix internal error when parametrizing tests or fixtures using an empty ``ids`` argument (:issue:`1849`). + Thanks :user:`OPpuolitaival` for the report and :user:`nicoddemus` for the PR. + +* Fix loader error when running ``pytest`` embedded in a zipfile. + Thanks :user:`mbachry` for the PR. + + +.. _release-3.0.0: + +3.0.0 (2016-08-18) +================== + +**Incompatible changes** + + +A number of incompatible changes were made in this release, with the intent of removing features deprecated for a long +time or change existing behaviors in order to make them less surprising/more useful. + +* Reinterpretation mode has now been removed. Only plain and rewrite + mode are available, consequently the ``--assert=reinterp`` option is + no longer available. This also means files imported from plugins or + ``conftest.py`` will not benefit from improved assertions by + default, you should use ``pytest.register_assert_rewrite()`` to + explicitly turn on assertion rewriting for those files. Thanks + :user:`flub` for the PR. + +* The following deprecated commandline options were removed: + + * ``--genscript``: no longer supported; + * ``--no-assert``: use ``--assert=plain`` instead; + * ``--nomagic``: use ``--assert=plain`` instead; + * ``--report``: use ``-r`` instead; + + Thanks to :user:`RedBeardCode` for the PR (:pull:`1664`). + +* ImportErrors in plugins now are a fatal error instead of issuing a + pytest warning (:issue:`1479`). Thanks to :user:`The-Compiler` for the PR. + +* Removed support code for Python 3 versions < 3.3 (:pull:`1627`). + +* Removed all ``py.test-X*`` entry points. The versioned, suffixed entry points + were never documented and a leftover from a pre-virtualenv era. These entry + points also created broken entry points in wheels, so removing them also + removes a source of confusion for users (:issue:`1632`). + Thanks :user:`obestwalter` for the PR. + +* ``pytest.skip()`` now raises an error when used to decorate a test function, + as opposed to its original intent (to imperatively skip a test inside a test function). Previously + this usage would cause the entire module to be skipped (:issue:`607`). + Thanks :user:`omarkohl` for the complete PR (:pull:`1519`). + +* Exit tests if a collection error occurs. A poll indicated most users will hit CTRL-C + anyway as soon as they see collection errors, so pytest might as well make that the default behavior (:issue:`1421`). + A ``--continue-on-collection-errors`` option has been added to restore the previous behaviour. + Thanks :user:`olegpidsadnyi` and :user:`omarkohl` for the complete PR (:pull:`1628`). + +* Renamed the pytest ``pdb`` module (plugin) into ``debugging`` to avoid clashes with the builtin ``pdb`` module. + +* Raise a helpful failure message when requesting a parametrized fixture at runtime, + e.g. with ``request.getfixturevalue``. Previously these parameters were simply + never defined, so a fixture decorated like ``@pytest.fixture(params=[0, 1, 2])`` + only ran once (:pull:`460`). + Thanks to :user:`nikratio` for the bug report, :user:`RedBeardCode` and :user:`tomviner` for the PR. + +* ``_pytest.monkeypatch.monkeypatch`` class has been renamed to ``_pytest.monkeypatch.MonkeyPatch`` + so it doesn't conflict with the ``monkeypatch`` fixture. + +* ``--exitfirst / -x`` can now be overridden by a following ``--maxfail=N`` + and is just a synonym for ``--maxfail=1``. + + +**New Features** + +* Support nose-style ``__test__`` attribute on methods of classes, + including unittest-style Classes. If set to ``False``, the test will not be + collected. + +* New ``doctest_namespace`` fixture for injecting names into the + namespace in which doctests run. + Thanks :user:`milliams` for the complete PR (:pull:`1428`). + +* New ``--doctest-report`` option available to change the output format of diffs + when running (failing) doctests (implements :issue:`1749`). + Thanks :user:`hartym` for the PR. + +* New ``name`` argument to ``pytest.fixture`` decorator which allows a custom name + for a fixture (to solve the funcarg-shadowing-fixture problem). + Thanks :user:`novas0x2a` for the complete PR (:pull:`1444`). + +* New ``approx()`` function for easily comparing floating-point numbers in + tests. + Thanks :user:`kalekundert` for the complete PR (:pull:`1441`). + +* Ability to add global properties in the final xunit output file by accessing + the internal ``junitxml`` plugin (experimental). + Thanks :user:`tareqalayan` for the complete PR :pull:`1454`). + +* New ``ExceptionInfo.match()`` method to match a regular expression on the + string representation of an exception (:issue:`372`). + Thanks :user:`omarkohl` for the complete PR (:pull:`1502`). + +* ``__tracebackhide__`` can now also be set to a callable which then can decide + whether to filter the traceback based on the ``ExceptionInfo`` object passed + to it. Thanks :user:`The-Compiler` for the complete PR (:pull:`1526`). + +* New ``pytest_make_parametrize_id(config, val)`` hook which can be used by plugins to provide + friendly strings for custom types. + Thanks :user:`palaviv` for the PR. + +* ``capsys`` and ``capfd`` now have a ``disabled()`` context-manager method, which + can be used to temporarily disable capture within a test. + Thanks :user:`nicoddemus` for the PR. + +* New cli flag ``--fixtures-per-test``: shows which fixtures are being used + for each selected test item. Features doc strings of fixtures by default. + Can also show where fixtures are defined if combined with ``-v``. + Thanks :user:`hackebrot` for the PR. + +* Introduce ``pytest`` command as recommended entry point. Note that ``py.test`` + still works and is not scheduled for removal. Closes proposal + :issue:`1629`. Thanks :user:`obestwalter` and :user:`davehunt` for the complete PR + (:pull:`1633`). + +* New cli flags: + + + ``--setup-plan``: performs normal collection and reports + the potential setup and teardown and does not execute any fixtures and tests; + + ``--setup-only``: performs normal collection, executes setup and teardown of + fixtures and reports them; + + ``--setup-show``: performs normal test execution and additionally shows + setup and teardown of fixtures; + + ``--keep-duplicates``: py.test now ignores duplicated paths given in the command + line. To retain the previous behavior where the same test could be run multiple + times by specifying it in the command-line multiple times, pass the ``--keep-duplicates`` + argument (:issue:`1609`); + + Thanks :user:`d6e`, :user:`kvas-it`, :user:`sallner`, :user:`ioggstream` and :user:`omarkohl` for the PRs. + +* New CLI flag ``--override-ini``/``-o``: overrides values from the ini file. + For example: ``"-o xfail_strict=True"``'. + Thanks :user:`blueyed` and :user:`fengxx` for the PR. + +* New hooks: + + + ``pytest_fixture_setup(fixturedef, request)``: executes fixture setup; + + ``pytest_fixture_post_finalizer(fixturedef)``: called after the fixture's + finalizer and has access to the fixture's result cache. + + Thanks :user:`d6e`, :user:`sallner`. + +* Issue warnings for asserts whose test is a tuple literal. Such asserts will + never fail because tuples are always truthy and are usually a mistake + (see :issue:`1562`). Thanks :user:`kvas-it`, for the PR. + +* Allow passing a custom debugger class (e.g. ``--pdbcls=IPython.core.debugger:Pdb``). + Thanks to :user:`anntzer` for the PR. + + +**Changes** + +* Plugins now benefit from assertion rewriting. Thanks + :user:`sober7`, :user:`nicoddemus` and :user:`flub` for the PR. + +* Change ``report.outcome`` for ``xpassed`` tests to ``"passed"`` in non-strict + mode and ``"failed"`` in strict mode. Thanks to :user:`hackebrot` for the PR + (:pull:`1795`) and :user:`gprasad84` for report (:issue:`1546`). + +* Tests marked with ``xfail(strict=False)`` (the default) now appear in + JUnitXML reports as passing tests instead of skipped. + Thanks to :user:`hackebrot` for the PR (:pull:`1795`). + +* Highlight path of the file location in the error report to make it easier to copy/paste. + Thanks :user:`suzaku` for the PR (:pull:`1778`). + +* Fixtures marked with ``@pytest.fixture`` can now use ``yield`` statements exactly like + those marked with the ``@pytest.yield_fixture`` decorator. This change renders + ``@pytest.yield_fixture`` deprecated and makes ``@pytest.fixture`` with ``yield`` statements + the preferred way to write teardown code (:pull:`1461`). + Thanks :user:`csaftoiu` for bringing this to attention and :user:`nicoddemus` for the PR. + +* Explicitly passed parametrize ids do not get escaped to ascii (:issue:`1351`). + Thanks :user:`ceridwen` for the PR. + +* Fixtures are now sorted in the error message displayed when an unknown + fixture is declared in a test function. + Thanks :user:`nicoddemus` for the PR. + +* ``pytest_terminal_summary`` hook now receives the ``exitstatus`` + of the test session as argument. Thanks :user:`blueyed` for the PR (:pull:`1809`). + +* Parametrize ids can accept ``None`` as specific test id, in which case the + automatically generated id for that argument will be used. + Thanks :user:`palaviv` for the complete PR (:pull:`1468`). + +* The parameter to xunit-style setup/teardown methods (``setup_method``, + ``setup_module``, etc.) is now optional and may be omitted. + Thanks :user:`okken` for bringing this to attention and :user:`nicoddemus` for the PR. + +* Improved automatic id generation selection in case of duplicate ids in + parametrize. + Thanks :user:`palaviv` for the complete PR (:pull:`1474`). + +* Now pytest warnings summary is shown up by default. Added a new flag + ``--disable-pytest-warnings`` to explicitly disable the warnings summary (:issue:`1668`). + +* Make ImportError during collection more explicit by reminding + the user to check the name of the test module/package(s) (:issue:`1426`). + Thanks :user:`omarkohl` for the complete PR (:pull:`1520`). + +* Add ``build/`` and ``dist/`` to the default ``--norecursedirs`` list. Thanks + :user:`mikofski` for the report and :user:`tomviner` for the PR (:issue:`1544`). + +* ``pytest.raises`` in the context manager form accepts a custom + ``message`` to raise when no exception occurred. + Thanks :user:`palaviv` for the complete PR (:pull:`1616`). + +* ``conftest.py`` files now benefit from assertion rewriting; previously it + was only available for test modules. Thanks :user:`flub`, :user:`sober7` and + :user:`nicoddemus` for the PR (:issue:`1619`). + +* Text documents without any doctests no longer appear as "skipped". + Thanks :user:`graingert` for reporting and providing a full PR (:pull:`1580`). + +* Ensure that a module within a namespace package can be found when it + is specified on the command line together with the ``--pyargs`` + option. Thanks to :user:`taschini` for the PR (:pull:`1597`). + +* Always include full assertion explanation during assertion rewriting. The previous behaviour was hiding + sub-expressions that happened to be ``False``, assuming this was redundant information. + Thanks :user:`bagerard` for reporting (:issue:`1503`). Thanks to :user:`davehunt` and + :user:`tomviner` for the PR. + +* ``OptionGroup.addoption()`` now checks if option names were already + added before, to make it easier to track down issues like :issue:`1618`. + Before, you only got exceptions later from ``argparse`` library, + giving no clue about the actual reason for double-added options. + +* ``yield``-based tests are considered deprecated and will be removed in pytest-4.0. + Thanks :user:`nicoddemus` for the PR. + +* ``[pytest]`` sections in ``setup.cfg`` files should now be named ``[tool:pytest]`` + to avoid conflicts with other distutils commands (see :pull:`567`). ``[pytest]`` sections in + ``pytest.ini`` or ``tox.ini`` files are supported and unchanged. + Thanks :user:`nicoddemus` for the PR. + +* Using ``pytest_funcarg__`` prefix to declare fixtures is considered deprecated and will be + removed in pytest-4.0 (:pull:`1684`). + Thanks :user:`nicoddemus` for the PR. + +* Passing a command-line string to ``pytest.main()`` is considered deprecated and scheduled + for removal in pytest-4.0. It is recommended to pass a list of arguments instead (:pull:`1723`). + +* Rename ``getfuncargvalue`` to ``getfixturevalue``. ``getfuncargvalue`` is + still present but is now considered deprecated. Thanks to :user:`RedBeardCode` and :user:`tomviner` + for the PR (:pull:`1626`). + +* ``optparse`` type usage now triggers DeprecationWarnings (:issue:`1740`). + + +* ``optparse`` backward compatibility supports float/complex types (:issue:`457`). + +* Refined logic for determining the ``rootdir``, considering only valid + paths which fixes a number of issues: :issue:`1594`, :issue:`1435` and :issue:`1471`. + Updated the documentation according to current behavior. Thanks to + :user:`blueyed`, :user:`davehunt` and :user:`matthiasha` for the PR. + +* Always include full assertion explanation. The previous behaviour was hiding + sub-expressions that happened to be False, assuming this was redundant information. + Thanks :user:`bagerard` for reporting (:issue:`1503`). Thanks to :user:`davehunt` and + :user:`tomviner` for PR. + +* Better message in case of not using parametrized variable (see :issue:`1539`). + Thanks to :user:`tramwaj29` for the PR. + +* Updated docstrings with a more uniform style. + +* Add stderr write for ``pytest.exit(msg)`` during startup. Previously the message was never shown. + Thanks :user:`BeyondEvil` for reporting :issue:`1210`. Thanks to @jgsonesen and + :user:`tomviner` for the PR. + +* No longer display the incorrect test deselection reason (:issue:`1372`). + Thanks :user:`ronnypfannschmidt` for the PR. + +* The ``--resultlog`` command line option has been deprecated: it is little used + and there are more modern and better alternatives (see :issue:`830`). + Thanks :user:`nicoddemus` for the PR. + +* Improve error message with fixture lookup errors: add an 'E' to the first + line and '>' to the rest. Fixes :issue:`717`. Thanks :user:`blueyed` for reporting and + a PR, :user:`eolo999` for the initial PR and :user:`tomviner` for his guidance during + EuroPython2016 sprint. + + +**Bug Fixes** + +* Parametrize now correctly handles duplicated test ids. + +* Fix internal error issue when the ``method`` argument is missing for + ``teardown_method()`` (:issue:`1605`). + +* Fix exception visualization in case the current working directory (CWD) gets + deleted during testing (:issue:`1235`). Thanks :user:`bukzor` for reporting. PR by + :user:`marscher`. + +* Improve test output for logical expression with brackets (:issue:`925`). + Thanks :user:`DRMacIver` for reporting and :user:`RedBeardCode` for the PR. + +* Create correct diff for strings ending with newlines (:issue:`1553`). + Thanks :user:`Vogtinator` for reporting and :user:`RedBeardCode` and + :user:`tomviner` for the PR. + +* ``ConftestImportFailure`` now shows the traceback making it easier to + identify bugs in ``conftest.py`` files (:pull:`1516`). Thanks :user:`txomon` for + the PR. + +* Text documents without any doctests no longer appear as "skipped". + Thanks :user:`graingert` for reporting and providing a full PR (:pull:`1580`). + +* Fixed collection of classes with custom ``__new__`` method. + Fixes :issue:`1579`. Thanks to :user:`Stranger6667` for the PR. + +* Fixed scope overriding inside metafunc.parametrize (:issue:`634`). + Thanks to :user:`Stranger6667` for the PR. + +* Fixed the total tests tally in junit xml output (:pull:`1798`). + Thanks to :user:`cboelsen` for the PR. + +* Fixed off-by-one error with lines from ``request.node.warn``. + Thanks to :user:`blueyed` for the PR. + + +2.9.2 (2016-05-31) +================== + +**Bug Fixes** + +* fix :issue:`510`: skip tests where one parameterize dimension was empty + thanks Alex Stapleton for the Report and :user:`RonnyPfannschmidt` for the PR + +* Fix Xfail does not work with condition keyword argument. + Thanks :user:`astraw38` for reporting the issue (:issue:`1496`) and :user:`tomviner` + for PR the (:pull:`1524`). + +* Fix win32 path issue when putting custom config file with absolute path + in ``pytest.main("-c your_absolute_path")``. + +* Fix maximum recursion depth detection when raised error class is not aware + of unicode/encoded bytes. + Thanks :user:`prusse-martin` for the PR (:pull:`1506`). + +* Fix ``pytest.mark.skip`` mark when used in strict mode. + Thanks :user:`pquentin` for the PR and :user:`RonnyPfannschmidt` for + showing how to fix the bug. + +* Minor improvements and fixes to the documentation. + Thanks :user:`omarkohl` for the PR. + +* Fix ``--fixtures`` to show all fixture definitions as opposed to just + one per fixture name. + Thanks to :user:`hackebrot` for the PR. + + +2.9.1 (2016-03-17) +================== + +**Bug Fixes** + +* Improve error message when a plugin fails to load. + Thanks :user:`nicoddemus` for the PR. + +* Fix (:issue:`1178`): + ``pytest.fail`` with non-ascii characters raises an internal pytest error. + Thanks :user:`nicoddemus` for the PR. + +* Fix (:issue:`469`): junit parses report.nodeid incorrectly, when params IDs + contain ``::``. Thanks :user:`tomviner` for the PR (:pull:`1431`). + +* Fix (:issue:`578`): SyntaxErrors + containing non-ascii lines at the point of failure generated an internal + py.test error. + Thanks :user:`asottile` for the report and :user:`nicoddemus` for the PR. + +* Fix (:issue:`1437`): When passing in a bytestring regex pattern to parameterize + attempt to decode it as utf-8 ignoring errors. + +* Fix (:issue:`649`): parametrized test nodes cannot be specified to run on the command line. + +* Fix (:issue:`138`): better reporting for python 3.3+ chained exceptions + + +2.9.0 (2016-02-29) +================== + +**New Features** + +* New ``pytest.mark.skip`` mark, which unconditionally skips marked tests. + Thanks :user:`MichaelAquilina` for the complete PR (:pull:`1040`). + +* ``--doctest-glob`` may now be passed multiple times in the command-line. + Thanks :user:`jab` and :user:`nicoddemus` for the PR. + +* New ``-rp`` and ``-rP`` reporting options give the summary and full output + of passing tests, respectively. Thanks to :user:`codewarrior0` for the PR. + +* ``pytest.mark.xfail`` now has a ``strict`` option, which makes ``XPASS`` + tests to fail the test suite (defaulting to ``False``). There's also a + ``xfail_strict`` ini option that can be used to configure it project-wise. + Thanks :user:`rabbbit` for the request and :user:`nicoddemus` for the PR (:pull:`1355`). + +* ``Parser.addini`` now supports options of type ``bool``. + Thanks :user:`nicoddemus` for the PR. + +* New ``ALLOW_BYTES`` doctest option. This strips ``b`` prefixes from byte strings + in doctest output (similar to ``ALLOW_UNICODE``). + Thanks :user:`jaraco` for the request and :user:`nicoddemus` for the PR (:pull:`1287`). + +* Give a hint on ``KeyboardInterrupt`` to use the ``--fulltrace`` option to show the errors. + Fixes :issue:`1366`. + Thanks to :user:`hpk42` for the report and :user:`RonnyPfannschmidt` for the PR. + +* Catch ``IndexError`` exceptions when getting exception source location. + Fixes a pytest internal error for dynamically generated code (fixtures and tests) + where source lines are fake by intention. + +**Changes** + +* **Important**: `py.code `_ has been + merged into the ``pytest`` repository as ``pytest._code``. This decision + was made because ``py.code`` had very few uses outside ``pytest`` and the + fact that it was in a different repository made it difficult to fix bugs on + its code in a timely manner. The team hopes with this to be able to better + refactor out and improve that code. + This change shouldn't affect users, but it is useful to let users aware + if they encounter any strange behavior. + + Keep in mind that the code for ``pytest._code`` is **private** and + **experimental**, so you definitely should not import it explicitly! + + Please note that the original ``py.code`` is still available in + `pylib `_. + +* ``pytest_enter_pdb`` now optionally receives the pytest config object. + Thanks :user:`nicoddemus` for the PR. + +* Removed code and documentation for Python 2.5 or lower versions, + including removal of the obsolete ``_pytest.assertion.oldinterpret`` module. + Thanks :user:`nicoddemus` for the PR (:pull:`1226`). + +* Comparisons now always show up in full when ``CI`` or ``BUILD_NUMBER`` is + found in the environment, even when ``-vv`` isn't used. + Thanks :user:`The-Compiler` for the PR. + +* ``--lf`` and ``--ff`` now support long names: ``--last-failed`` and + ``--failed-first`` respectively. + Thanks :user:`MichaelAquilina` for the PR. + +* Added expected exceptions to ``pytest.raises`` fail message. + +* Collection only displays progress ("collecting X items") when in a terminal. + This avoids cluttering the output when using ``--color=yes`` to obtain + colors in CI integrations systems (:issue:`1397`). + +**Bug Fixes** + +* The ``-s`` and ``-c`` options should now work under ``xdist``; + ``Config.fromdictargs`` now represents its input much more faithfully. + Thanks to :user:`bukzor` for the complete PR (:issue:`680`). + +* Fix (:issue:`1290`): support Python 3.5's ``@`` operator in assertion rewriting. + Thanks :user:`Shinkenjoe` for report with test case and :user:`tomviner` for the PR. + +* Fix formatting utf-8 explanation messages (:issue:`1379`). + Thanks :user:`biern` for the PR. + +* Fix :ref:`traceback style docs ` to describe all of the available options + (auto/long/short/line/native/no), with ``auto`` being the default since v2.6. + Thanks :user:`hackebrot` for the PR. + +* Fix (:issue:`1422`): junit record_xml_property doesn't allow multiple records + with same name. + + +2.8.7 (2016-01-24) +================== + +- fix #1338: use predictable object resolution for monkeypatch + +2.8.6 (2016-01-21) +================== + +- fix #1259: allow for double nodeids in junitxml, + this was a regression failing plugins combinations + like pytest-pep8 + pytest-flakes + +- Workaround for exception that occurs in pyreadline when using + ``--pdb`` with standard I/O capture enabled. + Thanks Erik M. Bray for the PR. + +- fix #900: Better error message in case the target of a ``monkeypatch`` call + raises an ``ImportError``. + +- fix #1292: monkeypatch calls (setattr, setenv, etc.) are now O(1). + Thanks David R. MacIver for the report and Bruno Oliveira for the PR. + +- fix #1223: captured stdout and stderr are now properly displayed before + entering pdb when ``--pdb`` is used instead of being thrown away. + Thanks Cal Leeming for the PR. + +- fix #1305: pytest warnings emitted during ``pytest_terminal_summary`` are now + properly displayed. + Thanks Ionel Maries Cristian for the report and Bruno Oliveira for the PR. + +- fix #628: fixed internal UnicodeDecodeError when doctests contain unicode. + Thanks Jason R. Coombs for the report and Bruno Oliveira for the PR. + +- fix #1334: Add captured stdout to jUnit XML report on setup error. + Thanks Georgy Dyuldin for the PR. + + +2.8.5 (2015-12-11) +================== + +- fix #1243: fixed issue where class attributes injected during collection could break pytest. + PR by Alexei Kozlenok, thanks Ronny Pfannschmidt and Bruno Oliveira for the review and help. + +- fix #1074: precompute junitxml chunks instead of storing the whole tree in objects + Thanks Bruno Oliveira for the report and Ronny Pfannschmidt for the PR + +- fix #1238: fix ``pytest.deprecated_call()`` receiving multiple arguments + (Regression introduced in 2.8.4). Thanks Alex Gaynor for the report and + Bruno Oliveira for the PR. + + +2.8.4 (2015-12-06) +================== + +- fix #1190: ``deprecated_call()`` now works when the deprecated + function has been already called by another test in the same + module. Thanks Mikhail Chernykh for the report and Bruno Oliveira for the + PR. + +- fix #1198: ``--pastebin`` option now works on Python 3. Thanks + Mehdy Khoshnoody for the PR. + +- fix #1219: ``--pastebin`` now works correctly when captured output contains + non-ascii characters. Thanks Bruno Oliveira for the PR. + +- fix #1204: another error when collecting with a nasty __getattr__(). + Thanks Florian Bruhin for the PR. + +- fix the summary printed when no tests did run. + Thanks Florian Bruhin for the PR. +- fix #1185 - ensure MANIFEST.in exactly matches what should go to a sdist + +- a number of documentation modernizations wrt good practices. + Thanks Bruno Oliveira for the PR. + +2.8.3 (2015-11-18) +================== + +- fix #1169: add __name__ attribute to testcases in TestCaseFunction to + support the @unittest.skip decorator on functions and methods. + Thanks Lee Kamentsky for the PR. + +- fix #1035: collecting tests if test module level obj has __getattr__(). + Thanks Suor for the report and Bruno Oliveira / Tom Viner for the PR. + +- fix #331: don't collect tests if their failure cannot be reported correctly + e.g. they are a callable instance of a class. + +- fix #1133: fixed internal error when filtering tracebacks where one entry + belongs to a file which is no longer available. + Thanks Bruno Oliveira for the PR. + +- enhancement made to highlight in red the name of the failing tests so + they stand out in the output. + Thanks Gabriel Reis for the PR. + +- add more talks to the documentation +- extend documentation on the --ignore cli option +- use pytest-runner for setuptools integration +- minor fixes for interaction with OS X El Capitan + system integrity protection (thanks Florian) + + +2.8.2 (2015-10-07) +================== + +- fix #1085: proper handling of encoding errors when passing encoded byte + strings to pytest.parametrize in Python 2. + Thanks Themanwithoutaplan for the report and Bruno Oliveira for the PR. + +- fix #1087: handling SystemError when passing empty byte strings to + pytest.parametrize in Python 3. + Thanks Paul Kehrer for the report and Bruno Oliveira for the PR. + +- fix #995: fixed internal error when filtering tracebacks where one entry + was generated by an exec() statement. + Thanks Daniel Hahler, Ashley C Straw, Philippe Gauthier and Pavel Savchenko + for contributing and Bruno Oliveira for the PR. + +- fix #1100 and #1057: errors when using autouse fixtures and doctest modules. + Thanks Sergey B Kirpichev and Vital Kudzelka for contributing and Bruno + Oliveira for the PR. + +2.8.1 (2015-09-29) +================== + +- fix #1034: Add missing nodeid on pytest_logwarning call in + addhook. Thanks Simon Gomizelj for the PR. + +- 'deprecated_call' is now only satisfied with a DeprecationWarning or + PendingDeprecationWarning. Before 2.8.0, it accepted any warning, and 2.8.0 + made it accept only DeprecationWarning (but not PendingDeprecationWarning). + Thanks Alex Gaynor for the issue and Eric Hunsberger for the PR. + +- fix issue #1073: avoid calling __getattr__ on potential plugin objects. + This fixes an incompatibility with pytest-django. Thanks Andreas Pelme, + Bruno Oliveira and Ronny Pfannschmidt for contributing and Holger Krekel + for the fix. + +- Fix issue #704: handle versionconflict during plugin loading more + gracefully. Thanks Bruno Oliveira for the PR. + +- Fix issue #1064: ""--junitxml" regression when used with the + "pytest-xdist" plugin, with test reports being assigned to the wrong tests. + Thanks Daniel Grunwald for the report and Bruno Oliveira for the PR. + +- (experimental) adapt more SEMVER style versioning and change meaning of + master branch in git repo: "master" branch now keeps the bug fixes, changes + aimed for micro releases. "features" branch will only be released + with minor or major pytest releases. + +- Fix issue #766 by removing documentation references to distutils. + Thanks Russel Winder. + +- Fix issue #1030: now byte-strings are escaped to produce item node ids + to make them always serializable. + Thanks Andy Freeland for the report and Bruno Oliveira for the PR. + +- Python 2: if unicode parametrized values are convertible to ascii, their + ascii representation is used for the node id. + +- Fix issue #411: Add __eq__ method to assertion comparison example. + Thanks Ben Webb. +- Fix issue #653: deprecated_call can be used as context manager. + +- fix issue 877: properly handle assertion explanations with non-ascii repr + Thanks Mathieu Agopian for the report and Ronny Pfannschmidt for the PR. + +- fix issue 1029: transform errors when writing cache values into pytest-warnings + +2.8.0 (2015-09-18) +================== + +- new ``--lf`` and ``-ff`` options to run only the last failing tests or + "failing tests first" from the last run. This functionality is provided + through porting the formerly external pytest-cache plugin into pytest core. + BACKWARD INCOMPAT: if you used pytest-cache's functionality to persist + data between test runs be aware that we don't serialize sets anymore. + Thanks Ronny Pfannschmidt for most of the merging work. + +- "-r" option now accepts "a" to include all possible reports, similar + to passing "fEsxXw" explicitly (issue960). + Thanks Abhijeet Kasurde for the PR. + +- avoid python3.5 deprecation warnings by introducing version + specific inspection helpers, thanks Michael Droettboom. + +- fix issue562: @nose.tools.istest now fully respected. + +- fix issue934: when string comparison fails and a diff is too large to display + without passing -vv, still show a few lines of the diff. + Thanks Florian Bruhin for the report and Bruno Oliveira for the PR. + +- fix issue736: Fix a bug where fixture params would be discarded when combined + with parametrization markers. + Thanks to Markus Unterwaditzer for the PR. + +- fix issue710: introduce ALLOW_UNICODE doctest option: when enabled, the + ``u`` prefix is stripped from unicode strings in expected doctest output. This + allows doctests which use unicode to run in Python 2 and 3 unchanged. + Thanks Jason R. Coombs for the report and Bruno Oliveira for the PR. + +- parametrize now also generates meaningful test IDs for enum, regex and class + objects (as opposed to class instances). + Thanks to Florian Bruhin for the PR. + +- Add 'warns' to assert that warnings are thrown (like 'raises'). + Thanks to Eric Hunsberger for the PR. + +- Fix issue683: Do not apply an already applied mark. Thanks ojake for the PR. + +- Deal with capturing failures better so fewer exceptions get lost to + /dev/null. Thanks David Szotten for the PR. + +- fix issue730: deprecate and warn about the --genscript option. + Thanks Ronny Pfannschmidt for the report and Christian Pommranz for the PR. + +- fix issue751: multiple parametrize with ids bug if it parametrizes class with + two or more test methods. Thanks Sergey Chipiga for reporting and Jan + Bednarik for PR. + +- fix issue82: avoid loading conftest files from setup.cfg/pytest.ini/tox.ini + files and upwards by default (--confcutdir can still be set to override this). + Thanks Bruno Oliveira for the PR. + +- fix issue768: docstrings found in python modules were not setting up session + fixtures. Thanks Jason R. Coombs for reporting and Bruno Oliveira for the PR. + +- added ``tmpdir_factory``, a session-scoped fixture that can be used to create + directories under the base temporary directory. Previously this object was + installed as a ``_tmpdirhandler`` attribute of the ``config`` object, but now it + is part of the official API and using ``config._tmpdirhandler`` is + deprecated. + Thanks Bruno Oliveira for the PR. + +- fix issue808: pytest's internal assertion rewrite hook now implements the + optional :pep:`302` get_data API so tests can access data files next to them. + Thanks xmo-odoo for request and example and Bruno Oliveira for + the PR. + +- rootdir and inifile are now displayed during usage errors to help + users diagnose problems such as unexpected ini files which add + unknown options being picked up by pytest. Thanks to Pavel Savchenko for + bringing the problem to attention in #821 and Bruno Oliveira for the PR. + +- Summary bar now is colored yellow for warning + situations such as: all tests either were skipped or xpass/xfailed, + or no tests were run at all (this is a partial fix for issue500). + +- fix issue812: pytest now exits with status code 5 in situations where no + tests were run at all, such as the directory given in the command line does + not contain any tests or as result of a command line option filters + all out all tests (-k for example). + Thanks Eric Siegerman (issue812) and Bruno Oliveira for the PR. + +- Summary bar now is colored yellow for warning + situations such as: all tests either were skipped or xpass/xfailed, + or no tests were run at all (related to issue500). + Thanks Eric Siegerman. + +- New ``testpaths`` ini option: list of directories to search for tests + when executing pytest from the root directory. This can be used + to speed up test collection when a project has well specified directories + for tests, being usually more practical than configuring norecursedirs for + all directories that do not contain tests. + Thanks to Adrian for idea (#694) and Bruno Oliveira for the PR. + +- fix issue713: JUnit XML reports for doctest failures. + Thanks Punyashloka Biswal. + +- fix issue970: internal pytest warnings now appear as "pytest-warnings" in + the terminal instead of "warnings", so it is clear for users that those + warnings are from pytest and not from the builtin "warnings" module. + Thanks Bruno Oliveira. + +- Include setup and teardown in junitxml test durations. + Thanks Janne Vanhala. + +- fix issue735: assertion failures on debug versions of Python 3.4+ + +- new option ``--import-mode`` to allow to change test module importing + behaviour to append to sys.path instead of prepending. This better allows + to run test modules against installed versions of a package even if the + package under test has the same import root. In this example:: + + testing/__init__.py + testing/test_pkg_under_test.py + pkg_under_test/ + + the tests will run against the installed version + of pkg_under_test when ``--import-mode=append`` is used whereas + by default they would always pick up the local version. Thanks Holger Krekel. + +- pytester: add method ``TmpTestdir.delete_loaded_modules()``, and call it + from ``inline_run()`` to allow temporary modules to be reloaded. + Thanks Eduardo Schettino. + +- internally refactor pluginmanager API and code so that there + is a clear distinction between a pytest-agnostic rather simple + pluginmanager and the PytestPluginManager which adds a lot of + behaviour, among it handling of the local conftest files. + In terms of documented methods this is a backward compatible + change but it might still break 3rd party plugins which relied on + details like especially the pluginmanager.add_shutdown() API. + Thanks Holger Krekel. + +- pluginmanagement: introduce ``pytest.hookimpl`` and + ``pytest.hookspec`` decorators for setting impl/spec + specific parameters. This substitutes the previous + now deprecated use of ``pytest.mark`` which is meant to + contain markers for test functions only. + +- write/refine docs for "writing plugins" which now have their + own page and are separate from the "using/installing plugins`` page. + +- fix issue732: properly unregister plugins from any hook calling + sites allowing to have temporary plugins during test execution. + +- deprecate and warn about ``__multicall__`` argument in hook + implementations. Use the ``hookwrapper`` mechanism instead already + introduced with pytest-2.7. + +- speed up pytest's own test suite considerably by using inprocess + tests by default (testrun can be modified with --runpytest=subprocess + to create subprocesses in many places instead). The main + APIs to run pytest in a test is "runpytest()" or "runpytest_subprocess" + and "runpytest_inprocess" if you need a particular way of running + the test. In all cases you get back a RunResult but the inprocess + one will also have a "reprec" attribute with the recorded events/reports. + +- fix monkeypatch.setattr("x.y", raising=False) to actually not raise + if "y" is not a pre-existing attribute. Thanks Florian Bruhin. + +- fix issue741: make running output from testdir.run copy/pasteable + Thanks Bruno Oliveira. + +- add a new ``--noconftest`` argument which ignores all ``conftest.py`` files. + +- add ``file`` and ``line`` attributes to JUnit-XML output. + +- fix issue890: changed extension of all documentation files from ``txt`` to + ``rst``. Thanks to Abhijeet for the PR. + +- fix issue714: add ability to apply indirect=True parameter on particular argnames. + Thanks Elizaveta239. + +- fix issue890: changed extension of all documentation files from ``txt`` to + ``rst``. Thanks to Abhijeet for the PR. + +- fix issue957: "# doctest: SKIP" option will now register doctests as SKIPPED + rather than PASSED. + Thanks Thomas Grainger for the report and Bruno Oliveira for the PR. + +- issue951: add new record_xml_property fixture, that supports logging + additional information on xml output. Thanks David Diaz for the PR. + +- issue949: paths after normal options (for example ``-s``, ``-v``, etc) are now + properly used to discover ``rootdir`` and ``ini`` files. + Thanks Peter Lauri for the report and Bruno Oliveira for the PR. + +2.7.3 (2015-09-15) +================== + +- Allow 'dev', 'rc', or other non-integer version strings in ``importorskip``. + Thanks to Eric Hunsberger for the PR. + +- fix issue856: consider --color parameter in all outputs (for example + --fixtures). Thanks Barney Gale for the report and Bruno Oliveira for the PR. + +- fix issue855: passing str objects as ``plugins`` argument to pytest.main + is now interpreted as a module name to be imported and registered as a + plugin, instead of silently having no effect. + Thanks xmo-odoo for the report and Bruno Oliveira for the PR. + +- fix issue744: fix for ast.Call changes in Python 3.5+. Thanks + Guido van Rossum, Matthias Bussonnier, Stefan Zimmermann and + Thomas Kluyver. + +- fix issue842: applying markers in classes no longer propagate this markers + to superclasses which also have markers. + Thanks xmo-odoo for the report and Bruno Oliveira for the PR. + +- preserve warning functions after call to pytest.deprecated_call. Thanks + Pieter Mulder for PR. + +- fix issue854: autouse yield_fixtures defined as class members of + unittest.TestCase subclasses now work as expected. + Thannks xmo-odoo for the report and Bruno Oliveira for the PR. + +- fix issue833: --fixtures now shows all fixtures of collected test files, instead of just the + fixtures declared on the first one. + Thanks Florian Bruhin for reporting and Bruno Oliveira for the PR. + +- fix issue863: skipped tests now report the correct reason when a skip/xfail + condition is met when using multiple markers. + Thanks Raphael Pierzina for reporting and Bruno Oliveira for the PR. + +- optimized tmpdir fixture initialization, which should make test sessions + faster (specially when using pytest-xdist). The only visible effect + is that now pytest uses a subdirectory in the $TEMP directory for all + directories created by this fixture (defaults to $TEMP/pytest-$USER). + Thanks Bruno Oliveira for the PR. + +2.7.2 (2015-06-23) +================== + +- fix issue767: pytest.raises value attribute does not contain the exception + instance on Python 2.6. Thanks Eric Siegerman for providing the test + case and Bruno Oliveira for PR. + +- Automatically create directory for junitxml and results log. + Thanks Aron Curzon. + +- fix issue713: JUnit XML reports for doctest failures. + Thanks Punyashloka Biswal. + +- fix issue735: assertion failures on debug versions of Python 3.4+ + Thanks Benjamin Peterson. + +- fix issue114: skipif marker reports to internal skipping plugin; + Thanks Floris Bruynooghe for reporting and Bruno Oliveira for the PR. + +- fix issue748: unittest.SkipTest reports to internal pytest unittest plugin. + Thanks Thomas De Schampheleire for reporting and Bruno Oliveira for the PR. + +- fix issue718: failed to create representation of sets containing unsortable + elements in python 2. Thanks Edison Gustavo Muenz. + +- fix issue756, fix issue752 (and similar issues): depend on py-1.4.29 + which has a refined algorithm for traceback generation. + + +2.7.1 (2015-05-19) +================== + +- fix issue731: do not get confused by the braces which may be present + and unbalanced in an object's repr while collapsing False + explanations. Thanks Carl Meyer for the report and test case. + +- fix issue553: properly handling inspect.getsourcelines failures in + FixtureLookupError which would lead to an internal error, + obfuscating the original problem. Thanks talljosh for initial + diagnose/patch and Bruno Oliveira for final patch. + +- fix issue660: properly report scope-mismatch-access errors + independently from ordering of fixture arguments. Also + avoid the pytest internal traceback which does not provide + information to the user. Thanks Holger Krekel. + +- streamlined and documented release process. Also all versions + (in setup.py and documentation generation) are now read + from _pytest/__init__.py. Thanks Holger Krekel. + +- fixed docs to remove the notion that yield-fixtures are experimental. + They are here to stay :) Thanks Bruno Oliveira. + +- Support building wheels by using environment markers for the + requirements. Thanks Ionel Maries Cristian. + +- fixed regression to 2.6.4 which surfaced e.g. in lost stdout capture printing + when tests raised SystemExit. Thanks Holger Krekel. + +- reintroduced _pytest fixture of the pytester plugin which is used + at least by pytest-xdist. + +2.7.0 (2015-03-26) +================== + +- fix issue435: make reload() work when assert rewriting is active. + Thanks Daniel Hahler. + +- fix issue616: conftest.py files and their contained fixtures are now + properly considered for visibility, independently from the exact + current working directory and test arguments that are used. + Many thanks to Eric Siegerman and his PR235 which contains + systematic tests for conftest visibility and now passes. + This change also introduces the concept of a ``rootdir`` which + is printed as a new pytest header and documented in the pytest + customize web page. + +- change reporting of "diverted" tests, i.e. tests that are collected + in one file but actually come from another (e.g. when tests in a test class + come from a base class in a different file). We now show the nodeid + and indicate via a postfix the other file. + +- add ability to set command line options by environment variable PYTEST_ADDOPTS. + +- added documentation on the new pytest-dev teams on bitbucket and + github. See https://pytest.org/en/stable/contributing.html . + Thanks to Anatoly for pushing and initial work on this. + +- fix issue650: new option ``--docttest-ignore-import-errors`` which + will turn import errors in doctests into skips. Thanks Charles Cloud + for the complete PR. + +- fix issue655: work around different ways that cause python2/3 + to leak sys.exc_info into fixtures/tests causing failures in 3rd party code + +- fix issue615: assertion rewriting did not correctly escape % signs + when formatting boolean operations, which tripped over mixing + booleans with modulo operators. Thanks to Tom Viner for the report, + triaging and fix. + +- implement issue351: add ability to specify parametrize ids as a callable + to generate custom test ids. Thanks Brianna Laugher for the idea and + implementation. + +- introduce and document new hookwrapper mechanism useful for plugins + which want to wrap the execution of certain hooks for their purposes. + This supersedes the undocumented ``__multicall__`` protocol which + pytest itself and some external plugins use. Note that pytest-2.8 + is scheduled to drop supporting the old ``__multicall__`` + and only support the hookwrapper protocol. + +- majorly speed up invocation of plugin hooks + +- use hookwrapper mechanism in builtin pytest plugins. + +- add a doctest ini option for doctest flags, thanks Holger Peters. + +- add note to docs that if you want to mark a parameter and the + parameter is a callable, you also need to pass in a reason to disambiguate + it from the "decorator" case. Thanks Tom Viner. + +- "python_classes" and "python_functions" options now support glob-patterns + for test discovery, as discussed in issue600. Thanks Ldiary Translations. + +- allow to override parametrized fixtures with non-parametrized ones and vice versa (bubenkoff). + +- fix issue463: raise specific error for 'parameterize' misspelling (pfctdayelise). + +- On failure, the ``sys.last_value``, ``sys.last_type`` and + ``sys.last_traceback`` are set, so that a user can inspect the error + via postmortem debugging (almarklein). + +2.6.4 (2014-10-24) +================== + +- Improve assertion failure reporting on iterables, by using ndiff and + pprint. + +- removed outdated japanese docs from source tree. + +- docs for "pytest_addhooks" hook. Thanks Bruno Oliveira. + +- updated plugin index docs. Thanks Bruno Oliveira. + +- fix issue557: with "-k" we only allow the old style "-" for negation + at the beginning of strings and even that is deprecated. Use "not" instead. + This should allow to pick parametrized tests where "-" appeared in the parameter. + +- fix issue604: Escape % character in the assertion message. + +- fix issue620: add explanation in the --genscript target about what + the binary blob means. Thanks Dinu Gherman. + +- fix issue614: fixed pastebin support. + + +- fix issue620: add explanation in the --genscript target about what + the binary blob means. Thanks Dinu Gherman. + +- fix issue614: fixed pastebin support. + +2.6.3 (2014-09-24) +================== + +- fix issue575: xunit-xml was reporting collection errors as failures + instead of errors, thanks Oleg Sinyavskiy. + +- fix issue582: fix setuptools example, thanks Laszlo Papp and Ronny + Pfannschmidt. + +- Fix infinite recursion bug when pickling capture.EncodedFile, thanks + Uwe Schmitt. + +- fix issue589: fix bad interaction with numpy and others when showing + exceptions. Check for precise "maximum recursion depth exceed" exception + instead of presuming any RuntimeError is that one (implemented in py + dep). Thanks Charles Cloud for analysing the issue. + +- fix conftest related fixture visibility issue: when running with a + CWD outside of a test package pytest would get fixture discovery wrong. + Thanks to Wolfgang Schnerring for figuring out a reproducible example. + +- Introduce pytest_enter_pdb hook (needed e.g. by pytest_timeout to cancel the + timeout when interactively entering pdb). Thanks Wolfgang Schnerring. + +- check xfail/skip also with non-python function test items. Thanks + Floris Bruynooghe. + +2.6.2 (2014-09-05) +================== + +- Added function pytest.freeze_includes(), which makes it easy to embed + pytest into executables using tools like cx_freeze. + See docs for examples and rationale. Thanks Bruno Oliveira. + +- Improve assertion rewriting cache invalidation precision. + +- fixed issue561: adapt autouse fixture example for python3. + +- fixed issue453: assertion rewriting issue with __repr__ containing + "\n{", "\n}" and "\n~". + +- fix issue560: correctly display code if an "else:" or "finally:" is + followed by statements on the same line. + +- Fix example in monkeypatch documentation, thanks t-8ch. + +- fix issue572: correct tmpdir doc example for python3. + +- Do not mark as universal wheel because Python 2.6 is different from + other builds due to the extra argparse dependency. Fixes issue566. + Thanks sontek. + +- Implement issue549: user-provided assertion messages now no longer + replace the py.test introspection message but are shown in addition + to them. + +2.6.1 (2014-08-07) +================== + +- No longer show line numbers in the --verbose output, the output is now + purely the nodeid. The line number is still shown in failure reports. + Thanks Floris Bruynooghe. + +- fix issue437 where assertion rewriting could cause pytest-xdist worker nodes + to collect different tests. Thanks Bruno Oliveira. + +- fix issue555: add "errors" attribute to capture-streams to satisfy + some distutils and possibly other code accessing sys.stdout.errors. + +- fix issue547 capsys/capfd also work when output capturing ("-s") is disabled. + +- address issue170: allow pytest.mark.xfail(...) to specify expected exceptions via + an optional "raises=EXC" argument where EXC can be a single exception + or a tuple of exception classes. Thanks David Mohr for the complete + PR. + +- fix integration of pytest with unittest.mock.patch decorator when + it uses the "new" argument. Thanks Nicolas Delaby for test and PR. + +- fix issue with detecting conftest files if the arguments contain + "::" node id specifications (copy pasted from "-v" output) + +- fix issue544 by only removing "@NUM" at the end of "::" separated parts + and if the part has a ".py" extension + +- don't use py.std import helper, rather import things directly. + Thanks Bruno Oliveira. + +2.6 +=== + +- Cache exceptions from fixtures according to their scope (issue 467). + +- fix issue537: Avoid importing old assertion reinterpretation code by default. + +- fix issue364: shorten and enhance tracebacks representation by default. + The new "--tb=auto" option (default) will only display long tracebacks + for the first and last entry. You can get the old behaviour of printing + all entries as long entries with "--tb=long". Also short entries by + default are now printed very similarly to "--tb=native" ones. + +- fix issue514: teach assertion reinterpretation about private class attributes + +- change -v output to include full node IDs of tests. Users can copy + a node ID from a test run, including line number, and use it as a + positional argument in order to run only a single test. + +- fix issue 475: fail early and comprehensible if calling + pytest.raises with wrong exception type. + +- fix issue516: tell in getting-started about current dependencies. + +- cleanup setup.py a bit and specify supported versions. Thanks Jurko + Gospodnetic for the PR. + +- change XPASS colour to yellow rather then red when tests are run + with -v. + +- fix issue473: work around mock putting an unbound method into a class + dict when double-patching. + +- fix issue498: if a fixture finalizer fails, make sure that + the fixture is still invalidated. + +- fix issue453: the result of the pytest_assertrepr_compare hook now gets + it's newlines escaped so that format_exception does not blow up. + +- internal new warning system: pytest will now produce warnings when + it detects oddities in your test collection or execution. + Warnings are ultimately sent to a new pytest_logwarning hook which is + currently only implemented by the terminal plugin which displays + warnings in the summary line and shows more details when -rw (report on + warnings) is specified. + +- change skips into warnings for test classes with an __init__ and + callables in test modules which look like a test but are not functions. + +- fix issue436: improved finding of initial conftest files from command + line arguments by using the result of parse_known_args rather than + the previous flaky heuristics. Thanks Marc Abramowitz for tests + and initial fixing approaches in this area. + +- fix issue #479: properly handle nose/unittest(2) SkipTest exceptions + during collection/loading of test modules. Thanks to Marc Schlaich + for the complete PR. + +- fix issue490: include pytest_load_initial_conftests in documentation + and improve docstring. + +- fix issue472: clarify that ``pytest.config.getvalue()`` cannot work + if it's triggered ahead of command line parsing. + +- merge PR123: improved integration with mock.patch decorator on tests. + +- fix issue412: messing with stdout/stderr FD-level streams is now + captured without crashes. + +- fix issue483: trial/py33 works now properly. Thanks Daniel Grana for PR. + +- improve example for pytest integration with "python setup.py test" + which now has a generic "-a" or "--pytest-args" option where you + can pass additional options as a quoted string. Thanks Trevor Bekolay. + +- simplified internal capturing mechanism and made it more robust + against tests or setups changing FD1/FD2, also better integrated + now with pytest.pdb() in single tests. + +- improvements to pytest's own test-suite leakage detection, courtesy of PRs + from Marc Abramowitz + +- fix issue492: avoid leak in test_writeorg. Thanks Marc Abramowitz. + +- fix issue493: don't run tests in doc directory with ``python setup.py test`` + (use tox -e doctesting for that) + +- fix issue486: better reporting and handling of early conftest loading failures + +- some cleanup and simplification of internal conftest handling. + +- work a bit harder to break reference cycles when catching exceptions. + Thanks Jurko Gospodnetic. + +- fix issue443: fix skip examples to use proper comparison. Thanks Alex + Groenholm. + +- support nose-style ``__test__`` attribute on modules, classes and + functions, including unittest-style Classes. If set to False, the + test will not be collected. + +- fix issue512: show "" for arguments which might not be set + in monkeypatch plugin. Improves output in documentation. + + +2.5.2 (2014-01-29) +================== + +- fix issue409 -- better interoperate with cx_freeze by not + trying to import from collections.abc which causes problems + for py27/cx_freeze. Thanks Wolfgang L. for reporting and tracking it down. + +- fixed docs and code to use "pytest" instead of "py.test" almost everywhere. + Thanks Jurko Gospodnetic for the complete PR. + +- fix issue425: mention at end of "py.test -h" that --markers + and --fixtures work according to specified test path (or current dir) + +- fix issue413: exceptions with unicode attributes are now printed + correctly also on python2 and with pytest-xdist runs. (the fix + requires py-1.4.20) + +- copy, cleanup and integrate py.io capture + from pylib 1.4.20.dev2 (rev 13d9af95547e) + +- address issue416: clarify docs as to conftest.py loading semantics + +- fix issue429: comparing byte strings with non-ascii chars in assert + expressions now work better. Thanks Floris Bruynooghe. + +- make capfd/capsys.capture private, its unused and shouldn't be exposed + + +2.5.1 (2013-12-17) +================== + +- merge new documentation styling PR from Tobias Bieniek. + +- fix issue403: allow parametrize of multiple same-name functions within + a collection node. Thanks Andreas Kloeckner and Alex Gaynor for reporting + and analysis. + +- Allow parameterized fixtures to specify the ID of the parameters by + adding an ids argument to pytest.fixture() and pytest.yield_fixture(). + Thanks Floris Bruynooghe. + +- fix issue404 by always using the binary xml escape in the junitxml + plugin. Thanks Ronny Pfannschmidt. + +- fix issue407: fix addoption docstring to point to argparse instead of + optparse. Thanks Daniel D. Wright. + + + +2.5.0 (2013-12-12) +================== + +- dropped python2.5 from automated release testing of pytest itself + which means it's probably going to break soon (but still works + with this release we believe). + +- simplified and fixed implementation for calling finalizers when + parametrized fixtures or function arguments are involved. finalization + is now performed lazily at setup time instead of in the "teardown phase". + While this might sound odd at first, it helps to ensure that we are + correctly handling setup/teardown even in complex code. User-level code + should not be affected unless it's implementing the pytest_runtest_teardown + hook and expecting certain fixture instances are torn down within (very + unlikely and would have been unreliable anyway). + +- PR90: add --color=yes|no|auto option to force terminal coloring + mode ("auto" is default). Thanks Marc Abramowitz. + +- fix issue319 - correctly show unicode in assertion errors. Many + thanks to Floris Bruynooghe for the complete PR. Also means + we depend on py>=1.4.19 now. + +- fix issue396 - correctly sort and finalize class-scoped parametrized + tests independently from number of methods on the class. + +- refix issue323 in a better way -- parametrization should now never + cause Runtime Recursion errors because the underlying algorithm + for re-ordering tests per-scope/per-fixture is not recursive + anymore (it was tail-call recursive before which could lead + to problems for more than >966 non-function scoped parameters). + +- fix issue290 - there is preliminary support now for parametrizing + with repeated same values (sometimes useful to test if calling + a second time works as with the first time). + +- close issue240 - document precisely how pytest module importing + works, discuss the two common test directory layouts, and how it + interacts with :pep:`420`\-namespace packages. + +- fix issue246 fix finalizer order to be LIFO on independent fixtures + depending on a parametrized higher-than-function scoped fixture. + (was quite some effort so please bear with the complexity of this sentence :) + Thanks Ralph Schmitt for the precise failure example. + +- fix issue244 by implementing special index for parameters to only use + indices for paramentrized test ids + +- fix issue287 by running all finalizers but saving the exception + from the first failing finalizer and re-raising it so teardown will + still have failed. We reraise the first failing exception because + it might be the cause for other finalizers to fail. + +- fix ordering when mock.patch or other standard decorator-wrappings + are used with test methods. This fixues issue346 and should + help with random "xdist" collection failures. Thanks to + Ronny Pfannschmidt and Donald Stufft for helping to isolate it. + +- fix issue357 - special case "-k" expressions to allow for + filtering with simple strings that are not valid python expressions. + Examples: "-k 1.3" matches all tests parametrized with 1.3. + "-k None" filters all tests that have "None" in their name + and conversely "-k 'not None'". + Previously these examples would raise syntax errors. + +- fix issue384 by removing the trial support code + since the unittest compat enhancements allow + trial to handle it on its own + +- don't hide an ImportError when importing a plugin produces one. + fixes issue375. + +- fix issue275 - allow usefixtures and autouse fixtures + for running doctest text files. + +- fix issue380 by making --resultlog only rely on longrepr instead + of the "reprcrash" attribute which only exists sometimes. + +- address issue122: allow @pytest.fixture(params=iterator) by exploding + into a list early on. + +- fix pexpect-3.0 compatibility for pytest's own tests. + (fixes issue386) + +- allow nested parametrize-value markers, thanks James Lan for the PR. + +- fix unicode handling with new monkeypatch.setattr(import_path, value) + API. Thanks Rob Dennis. Fixes issue371. + +- fix unicode handling with junitxml, fixes issue368. + +- In assertion rewriting mode on Python 2, fix the detection of coding + cookies. See issue #330. + +- make "--runxfail" turn imperative pytest.xfail calls into no ops + (it already did neutralize pytest.mark.xfail markers) + +- refine pytest / pkg_resources interactions: The AssertionRewritingHook + :pep:`302` compliant loader now registers itself with setuptools/pkg_resources + properly so that the pkg_resources.resource_stream method works properly. + Fixes issue366. Thanks for the investigations and full PR to Jason R. Coombs. + +- pytestconfig fixture is now session-scoped as it is the same object during the + whole test run. Fixes issue370. + +- avoid one surprising case of marker malfunction/confusion:: + + @pytest.mark.some(lambda arg: ...) + def test_function(): + + would not work correctly because pytest assumes @pytest.mark.some + gets a function to be decorated already. We now at least detect if this + arg is a lambda and thus the example will work. Thanks Alex Gaynor + for bringing it up. + +- xfail a test on pypy that checks wrong encoding/ascii (pypy does + not error out). fixes issue385. + +- internally make varnames() deal with classes's __init__, + although it's not needed by pytest itself atm. Also + fix caching. Fixes issue376. + +- fix issue221 - handle importing of namespace-package with no + __init__.py properly. + +- refactor internal FixtureRequest handling to avoid monkeypatching. + One of the positive user-facing effects is that the "request" object + can now be used in closures. + +- fixed version comparison in pytest.importskip(modname, minverstring) + +- fix issue377 by clarifying in the nose-compat docs that pytest + does not duplicate the unittest-API into the "plain" namespace. + +- fix verbose reporting for @mock'd test functions + +2.4.2 (2013-10-04) +================== + +- on Windows require colorama and a newer py lib so that py.io.TerminalWriter() + now uses colorama instead of its own ctypes hacks. (fixes issue365) + thanks Paul Moore for bringing it up. + +- fix "-k" matching of tests where "repr" and "attr" and other names would + cause wrong matches because of an internal implementation quirk + (don't ask) which is now properly implemented. fixes issue345. + +- avoid tmpdir fixture to create too long filenames especially + when parametrization is used (issue354) + +- fix pytest-pep8 and pytest-flakes / pytest interactions + (collection names in mark plugin was assuming an item always + has a function which is not true for those plugins etc.) + Thanks Andi Zeidler. + +- introduce node.get_marker/node.add_marker API for plugins + like pytest-pep8 and pytest-flakes to avoid the messy + details of the node.keywords pseudo-dicts. Adapted + docs. + +- remove attempt to "dup" stdout at startup as it's icky. + the normal capturing should catch enough possibilities + of tests messing up standard FDs. + +- add pluginmanager.do_configure(config) as a link to + config.do_configure() for plugin-compatibility + +2.4.1 (2013-10-02) +================== + +- When using parser.addoption() unicode arguments to the + "type" keyword should also be converted to the respective types. + thanks Floris Bruynooghe, @dnozay. (fixes issue360 and issue362) + +- fix dotted filename completion when using argcomplete + thanks Anthon van der Neuth. (fixes issue361) + +- fix regression when a 1-tuple ("arg",) is used for specifying + parametrization (the values of the parametrization were passed + nested in a tuple). Thanks Donald Stufft. + +- merge doc typo fixes, thanks Andy Dirnberger + +2.4 +=== + +known incompatibilities: + +- if calling --genscript from python2.7 or above, you only get a + standalone script which works on python2.7 or above. Use Python2.6 + to also get a python2.5 compatible version. + +- all xunit-style teardown methods (nose-style, pytest-style, + unittest-style) will not be called if the corresponding setup method failed, + see issue322 below. + +- the pytest_plugin_unregister hook wasn't ever properly called + and there is no known implementation of the hook - so it got removed. + +- pytest.fixture-decorated functions cannot be generators (i.e. use + yield) anymore. This change might be reversed in 2.4.1 if it causes + unforeseen real-life issues. However, you can always write and return + an inner function/generator and change the fixture consumer to iterate + over the returned generator. This change was done in lieu of the new + ``pytest.yield_fixture`` decorator, see below. + +new features: + +- experimentally introduce a new ``pytest.yield_fixture`` decorator + which accepts exactly the same parameters as pytest.fixture but + mandates a ``yield`` statement instead of a ``return statement`` from + fixture functions. This allows direct integration with "with-style" + context managers in fixture functions and generally avoids registering + of finalization callbacks in favour of treating the "after-yield" as + teardown code. Thanks Andreas Pelme, Vladimir Keleshev, Floris + Bruynooghe, Ronny Pfannschmidt and many others for discussions. + +- allow boolean expression directly with skipif/xfail + if a "reason" is also specified. Rework skipping documentation + to recommend "condition as booleans" because it prevents surprises + when importing markers between modules. Specifying conditions + as strings will remain fully supported. + +- reporting: color the last line red or green depending if + failures/errors occurred or everything passed. thanks Christian + Theunert. + +- make "import pdb ; pdb.set_trace()" work natively wrt capturing (no + "-s" needed anymore), making ``pytest.set_trace()`` a mere shortcut. + +- fix issue181: --pdb now also works on collect errors (and + on internal errors) . This was implemented by a slight internal + refactoring and the introduction of a new hook + ``pytest_exception_interact`` hook (see next item). + +- fix issue341: introduce new experimental hook for IDEs/terminals to + intercept debugging: ``pytest_exception_interact(node, call, report)``. + +- new monkeypatch.setattr() variant to provide a shorter + invocation for patching out classes/functions from modules: + + monkeypatch.setattr("requests.get", myfunc) + + will replace the "get" function of the "requests" module with ``myfunc``. + +- fix issue322: tearDownClass is not run if setUpClass failed. Thanks + Mathieu Agopian for the initial fix. Also make all of pytest/nose + finalizer mimic the same generic behaviour: if a setupX exists and + fails, don't run teardownX. This internally introduces a new method + "node.addfinalizer()" helper which can only be called during the setup + phase of a node. + +- simplify pytest.mark.parametrize() signature: allow to pass a + CSV-separated string to specify argnames. For example: + ``pytest.mark.parametrize("input,expected", [(1,2), (2,3)])`` + works as well as the previous: + ``pytest.mark.parametrize(("input", "expected"), ...)``. + +- add support for setUpModule/tearDownModule detection, thanks Brian Okken. + +- integrate tab-completion on options through use of "argcomplete". + Thanks Anthon van der Neut for the PR. + +- change option names to be hyphen-separated long options but keep the + old spelling backward compatible. py.test -h will only show the + hyphenated version, for example "--collect-only" but "--collectonly" + will remain valid as well (for backward-compat reasons). Many thanks to + Anthon van der Neut for the implementation and to Hynek Schlawack for + pushing us. + +- fix issue 308 - allow to mark/xfail/skip individual parameter sets + when parametrizing. Thanks Brianna Laugher. + +- call new experimental pytest_load_initial_conftests hook to allow + 3rd party plugins to do something before a conftest is loaded. + +Bug fixes: + +- fix issue358 - capturing options are now parsed more properly + by using a new parser.parse_known_args method. + +- pytest now uses argparse instead of optparse (thanks Anthon) which + means that "argparse" is added as a dependency if installing into python2.6 + environments or below. + +- fix issue333: fix a case of bad unittest/pytest hook interaction. + +- PR27: correctly handle nose.SkipTest during collection. Thanks + Antonio Cuni, Ronny Pfannschmidt. + +- fix issue355: junitxml puts name="pytest" attribute to testsuite tag. + +- fix issue336: autouse fixture in plugins should work again. + +- fix issue279: improve object comparisons on assertion failure + for standard datatypes and recognise collections.abc. Thanks to + Brianna Laugher and Mathieu Agopian. + +- fix issue317: assertion rewriter support for the is_package method + +- fix issue335: document py.code.ExceptionInfo() object returned + from pytest.raises(), thanks Mathieu Agopian. + +- remove implicit distribute_setup support from setup.py. + +- fix issue305: ignore any problems when writing pyc files. + +- SO-17664702: call fixture finalizers even if the fixture function + partially failed (finalizers would not always be called before) + +- fix issue320 - fix class scope for fixtures when mixed with + module-level functions. Thanks Anatloy Bubenkoff. + +- you can specify "-q" or "-qq" to get different levels of "quieter" + reporting (thanks Katarzyna Jachim) + +- fix issue300 - Fix order of conftest loading when starting py.test + in a subdirectory. + +- fix issue323 - sorting of many module-scoped arg parametrizations + +- make sessionfinish hooks execute with the same cwd-context as at + session start (helps fix plugin behaviour which write output files + with relative path such as pytest-cov) + +- fix issue316 - properly reference collection hooks in docs + +- fix issue 306 - cleanup of -k/-m options to only match markers/test + names/keywords respectively. Thanks Wouter van Ackooy. + +- improved doctest counting for doctests in python modules -- + files without any doctest items will not show up anymore + and doctest examples are counted as separate test items. + thanks Danilo Bellini. + +- fix issue245 by depending on the released py-1.4.14 + which fixes py.io.dupfile to work with files with no + mode. Thanks Jason R. Coombs. + +- fix junitxml generation when test output contains control characters, + addressing issue267, thanks Jaap Broekhuizen + +- fix issue338: honor --tb style for setup/teardown errors as well. Thanks Maho. + +- fix issue307 - use yaml.safe_load in example, thanks Mark Eichin. + +- better parametrize error messages, thanks Brianna Laugher + +- pytest_terminal_summary(terminalreporter) hooks can now use + ".section(title)" and ".line(msg)" methods to print extra + information at the end of a test run. + +2.3.5 (2013-04-30) +================== + +- fix issue169: respect --tb=style with setup/teardown errors as well. + +- never consider a fixture function for test function collection + +- allow re-running of test items / helps to fix pytest-reruntests plugin + and also help to keep less fixture/resource references alive + +- put captured stdout/stderr into junitxml output even for passing tests + (thanks Adam Goucher) + +- Issue 265 - integrate nose setup/teardown with setupstate + so it doesn't try to teardown if it did not setup + +- issue 271 - don't write junitxml on worker nodes + +- Issue 274 - don't try to show full doctest example + when doctest does not know the example location + +- issue 280 - disable assertion rewriting on buggy CPython 2.6.0 + +- inject "getfixture()" helper to retrieve fixtures from doctests, + thanks Andreas Zeidler + +- issue 259 - when assertion rewriting, be consistent with the default + source encoding of ASCII on Python 2 + +- issue 251 - report a skip instead of ignoring classes with init + +- issue250 unicode/str mixes in parametrization names and values now works + +- issue257, assertion-triggered compilation of source ending in a + comment line doesn't blow up in python2.5 (fixed through py>=1.4.13.dev6) + +- fix --genscript option to generate standalone scripts that also + work with python3.3 (importer ordering) + +- issue171 - in assertion rewriting, show the repr of some + global variables + +- fix option help for "-k" + +- move long description of distribution into README.rst + +- improve docstring for metafunc.parametrize() + +- fix bug where using capsys with pytest.set_trace() in a test + function would break when looking at capsys.readouterr() + +- allow to specify prefixes starting with "_" when + customizing python_functions test discovery. (thanks Graham Horler) + +- improve PYTEST_DEBUG tracing output by putting + extra data on a new lines with additional indent + +- ensure OutcomeExceptions like skip/fail have initialized exception attributes + +- issue 260 - don't use nose special setup on plain unittest cases + +- fix issue134 - print the collect errors that prevent running specified test items + +- fix issue266 - accept unicode in MarkEvaluator expressions + +2.3.4 (2012-11-20) +================== + +- yielded test functions will now have autouse-fixtures active but + cannot accept fixtures as funcargs - it's anyway recommended to + rather use the post-2.0 parametrize features instead of yield, see: + http://pytest.org/en/stable/example/how-to/parametrize.html +- fix autouse-issue where autouse-fixtures would not be discovered + if defined in an a/conftest.py file and tests in a/tests/test_some.py +- fix issue226 - LIFO ordering for fixture teardowns +- fix issue224 - invocations with >256 char arguments now work +- fix issue91 - add/discuss package/directory level setups in example +- allow to dynamically define markers via + item.keywords[...]=assignment integrating with "-m" option +- make "-k" accept an expressions the same as with "-m" so that one + can write: -k "name1 or name2" etc. This is a slight incompatibility + if you used special syntax like "TestClass.test_method" which you now + need to write as -k "TestClass and test_method" to match a certain + method in a certain test class. + +2.3.3 (2012-11-06) +================== + +- fix issue214 - parse modules that contain special objects like e. g. + flask's request object which blows up on getattr access if no request + is active. thanks Thomas Waldmann. + +- fix issue213 - allow to parametrize with values like numpy arrays that + do not support an __eq__ operator + +- fix issue215 - split test_python.org into multiple files + +- fix issue148 - @unittest.skip on classes is now recognized and avoids + calling setUpClass/tearDownClass, thanks Pavel Repin + +- fix issue209 - reintroduce python2.4 support by depending on newer + pylib which re-introduced statement-finding for pre-AST interpreters + +- nose support: only call setup if it's a callable, thanks Andrew + Taumoefolau + +- fix issue219 - add py2.4-3.3 classifiers to TROVE list + +- in tracebacks *,** arg values are now shown next to normal arguments + (thanks Manuel Jacob) + +- fix issue217 - support mock.patch with pytest's fixtures - note that + you need either mock-1.0.1 or the python3.3 builtin unittest.mock. + +- fix issue127 - improve documentation for pytest_addoption() and + add a ``config.getoption(name)`` helper function for consistency. + +2.3.2 (2012-10-25) +================== + +- fix issue208 and fix issue29 use new py version to avoid long pauses + when printing tracebacks in long modules + +- fix issue205 - conftests in subdirs customizing + pytest_pycollect_makemodule and pytest_pycollect_makeitem + now work properly + +- fix teardown-ordering for parametrized setups + +- fix issue127 - better documentation for pytest_addoption + and related objects. + +- fix unittest behaviour: TestCase.runtest only called if there are + test methods defined + +- improve trial support: don't collect its empty + unittest.TestCase.runTest() method + +- "python setup.py test" now works with pytest itself + +- fix/improve internal/packaging related bits: + + - exception message check of test_nose.py now passes on python33 as well + + - issue206 - fix test_assertrewrite.py to work when a global + PYTHONDONTWRITEBYTECODE=1 is present + + - add tox.ini to pytest distribution so that ignore-dirs and others config + bits are properly distributed for maintainers who run pytest-own tests + +2.3.1 (2012-10-20) +================== + +- fix issue202 - fix regression: using "self" from fixture functions now + works as expected (it's the same "self" instance that a test method + which uses the fixture sees) + +- skip pexpect using tests (test_pdb.py mostly) on freebsd* systems + due to pexpect not supporting it properly (hanging) + +- link to web pages from --markers output which provides help for + pytest.mark.* usage. + +2.3.0 (2012-10-19) +================== + +- fix issue202 - better automatic names for parametrized test functions +- fix issue139 - introduce @pytest.fixture which allows direct scoping + and parametrization of funcarg factories. +- fix issue198 - conftest fixtures were not found on windows32 in some + circumstances with nested directory structures due to path manipulation issues +- fix issue193 skip test functions with were parametrized with empty + parameter sets +- fix python3.3 compat, mostly reporting bits that previously depended + on dict ordering +- introduce re-ordering of tests by resource and parametrization setup + which takes precedence to the usual file-ordering +- fix issue185 monkeypatching time.time does not cause pytest to fail +- fix issue172 duplicate call of pytest.fixture decoratored setup_module + functions +- fix junitxml=path construction so that if tests change the + current working directory and the path is a relative path + it is constructed correctly from the original current working dir. +- fix "python setup.py test" example to cause a proper "errno" return +- fix issue165 - fix broken doc links and mention stackoverflow for FAQ +- catch unicode-issues when writing failure representations + to terminal to prevent the whole session from crashing +- fix xfail/skip confusion: a skip-mark or an imperative pytest.skip + will now take precedence before xfail-markers because we + can't determine xfail/xpass status in case of a skip. see also: + http://stackoverflow.com/questions/11105828/in-py-test-when-i-explicitly-skip-a-test-that-is-marked-as-xfail-how-can-i-get + +- always report installed 3rd party plugins in the header of a test run + +- fix issue160: a failing setup of an xfail-marked tests should + be reported as xfail (not xpass) + +- fix issue128: show captured output when capsys/capfd are used + +- fix issue179: properly show the dependency chain of factories + +- pluginmanager.register(...) now raises ValueError if the + plugin has been already registered or the name is taken + +- fix issue159: improve https://docs.pytest.org/en/6.0.1/faq.html + especially with respect to the "magic" history, also mention + pytest-django, trial and unittest integration. + +- make request.keywords and node.keywords writable. All descendant + collection nodes will see keyword values. Keywords are dictionaries + containing markers and other info. + +- fix issue 178: xml binary escapes are now wrapped in py.xml.raw + +- fix issue 176: correctly catch the builtin AssertionError + even when we replaced AssertionError with a subclass on the + python level + +- factory discovery no longer fails with magic global callables + that provide no sane __code__ object (mock.call for example) + +- fix issue 182: testdir.inprocess_run now considers passed plugins + +- fix issue 188: ensure sys.exc_info is clear on python2 + before calling into a test + +- fix issue 191: add unittest TestCase runTest method support +- fix issue 156: monkeypatch correctly handles class level descriptors + +- reporting refinements: + + - pytest_report_header now receives a "startdir" so that + you can use startdir.bestrelpath(yourpath) to show + nice relative path + + - allow plugins to implement both pytest_report_header and + pytest_sessionstart (sessionstart is invoked first). + + - don't show deselected reason line if there is none + + - py.test -vv will show all of assert comparisons instead of truncating + +2.2.4 (2012-05-22) +================== + +- fix error message for rewritten assertions involving the % operator +- fix issue 126: correctly match all invalid xml characters for junitxml + binary escape +- fix issue with unittest: now @unittest.expectedFailure markers should + be processed correctly (you can also use @pytest.mark markers) +- document integration with the extended distribute/setuptools test commands +- fix issue 140: properly get the real functions + of bound classmethods for setup/teardown_class +- fix issue #141: switch from the deceased paste.pocoo.org to bpaste.net +- fix issue #143: call unconfigure/sessionfinish always when + configure/sessionstart where called +- fix issue #144: better mangle test ids to junitxml classnames +- upgrade distribute_setup.py to 0.6.27 + +2.2.3 (2012-02-05) +================== + +- fix uploaded package to only include necessary files + +2.2.2 (2012-02-05) +================== + +- fix issue101: wrong args to unittest.TestCase test function now + produce better output +- fix issue102: report more useful errors and hints for when a + test directory was renamed and some pyc/__pycache__ remain +- fix issue106: allow parametrize to be applied multiple times + e.g. from module, class and at function level. +- fix issue107: actually perform session scope finalization +- don't check in parametrize if indirect parameters are funcarg names +- add chdir method to monkeypatch funcarg +- fix crash resulting from calling monkeypatch undo a second time +- fix issue115: make --collectonly robust against early failure + (missing files/directories) +- "-qq --collectonly" now shows only files and the number of tests in them +- "-q --collectonly" now shows test ids +- allow adding of attributes to test reports such that it also works + with distributed testing (no upgrade of pytest-xdist needed) + +2.2.1 (2011-12-16) +================== + +- fix issue99 (in pytest and py) internallerrors with resultlog now + produce better output - fixed by normalizing pytest_internalerror + input arguments. +- fix issue97 / traceback issues (in pytest and py) improve traceback output + in conjunction with jinja2 and cython which hack tracebacks +- fix issue93 (in pytest and pytest-xdist) avoid "delayed teardowns": + the final test in a test node will now run its teardown directly + instead of waiting for the end of the session. Thanks Dave Hunt for + the good reporting and feedback. The pytest_runtest_protocol as well + as the pytest_runtest_teardown hooks now have "nextitem" available + which will be None indicating the end of the test run. +- fix collection crash due to unknown-source collected items, thanks + to Ralf Schmitt (fixed by depending on a more recent pylib) + +2.2.0 (2011-11-18) +================== + +- fix issue90: introduce eager tearing down of test items so that + teardown function are called earlier. +- add an all-powerful metafunc.parametrize function which allows to + parametrize test function arguments in multiple steps and therefore + from independent plugins and places. +- add a @pytest.mark.parametrize helper which allows to easily + call a test function with different argument values +- Add examples to the "parametrize" example page, including a quick port + of Test scenarios and the new parametrize function and decorator. +- introduce registration for "pytest.mark.*" helpers via ini-files + or through plugin hooks. Also introduce a "--strict" option which + will treat unregistered markers as errors + allowing to avoid typos and maintain a well described set of markers + for your test suite. See examples at http://pytest.org/en/stable/how-to/mark.html + and its links. +- issue50: introduce "-m marker" option to select tests based on markers + (this is a stricter and more predictable version of '-k' in that "-m" + only matches complete markers and has more obvious rules for and/or + semantics. +- new feature to help optimizing the speed of your tests: + --durations=N option for displaying N slowest test calls + and setup/teardown methods. +- fix issue87: --pastebin now works with python3 +- fix issue89: --pdb with unexpected exceptions in doctest work more sensibly +- fix and cleanup pytest's own test suite to not leak FDs +- fix issue83: link to generated funcarg list +- fix issue74: pyarg module names are now checked against imp.find_module false positives +- fix compatibility with twisted/trial-11.1.0 use cases +- simplify Node.listchain +- simplify junitxml output code by relying on py.xml +- add support for skip properties on unittest classes and functions + +2.1.3 (2011-10-18) +================== + +- fix issue79: assertion rewriting failed on some comparisons in boolops +- correctly handle zero length arguments (a la pytest '') +- fix issue67 / junitxml now contains correct test durations, thanks ronny +- fix issue75 / skipping test failure on jython +- fix issue77 / Allow assertrepr_compare hook to apply to a subset of tests + +2.1.2 (2011-09-24) +================== + +- fix assertion rewriting on files with windows newlines on some Python versions +- refine test discovery by package/module name (--pyargs), thanks Florian Mayer +- fix issue69 / assertion rewriting fixed on some boolean operations +- fix issue68 / packages now work with assertion rewriting +- fix issue66: use different assertion rewriting caches when the -O option is passed +- don't try assertion rewriting on Jython, use reinterp + +2.1.1 +===== + +- fix issue64 / pytest.set_trace now works within pytest_generate_tests hooks +- fix issue60 / fix error conditions involving the creation of __pycache__ +- fix issue63 / assertion rewriting on inserts involving strings containing '%' +- fix assertion rewriting on calls with a ** arg +- don't cache rewritten modules if bytecode generation is disabled +- fix assertion rewriting in read-only directories +- fix issue59: provide system-out/err tags for junitxml output +- fix issue61: assertion rewriting on boolean operations with 3 or more operands +- you can now build a man page with "cd doc ; make man" + +2.1.0 (2011-07-09) +================== + +- fix issue53 call nosestyle setup functions with correct ordering +- fix issue58 and issue59: new assertion code fixes +- merge Benjamin's assertionrewrite branch: now assertions + for test modules on python 2.6 and above are done by rewriting + the AST and saving the pyc file before the test module is imported. + see doc/assert.txt for more info. +- fix issue43: improve doctests with better traceback reporting on + unexpected exceptions +- fix issue47: timing output in junitxml for test cases is now correct +- fix issue48: typo in MarkInfo repr leading to exception +- fix issue49: avoid confusing error when initizaliation partially fails +- fix issue44: env/username expansion for junitxml file path +- show releaselevel information in test runs for pypy +- reworked doc pages for better navigation and PDF generation +- report KeyboardInterrupt even if interrupted during session startup +- fix issue 35 - provide PDF doc version and download link from index page + +2.0.3 (2011-05-11) +================== + +- fix issue38: nicer tracebacks on calls to hooks, particularly early + configure/sessionstart ones + +- fix missing skip reason/meta information in junitxml files, reported + via http://lists.idyll.org/pipermail/testing-in-python/2011-March/003928.html + +- fix issue34: avoid collection failure with "test" prefixed classes + deriving from object. + +- don't require zlib (and other libs) for genscript plugin without + --genscript actually being used. + +- speed up skips (by not doing a full traceback representation + internally) + +- fix issue37: avoid invalid characters in junitxml's output + +2.0.2 (2011-03-09) +================== + +- tackle issue32 - speed up test runs of very quick test functions + by reducing the relative overhead + +- fix issue30 - extended xfail/skipif handling and improved reporting. + If you have a syntax error in your skip/xfail + expressions you now get nice error reports. + + Also you can now access module globals from xfail/skipif + expressions so that this for example works now:: + + import pytest + import mymodule + @pytest.mark.skipif("mymodule.__version__[0] == "1") + def test_function(): + pass + + This will not run the test function if the module's version string + does not start with a "1". Note that specifying a string instead + of a boolean expressions allows py.test to report meaningful information + when summarizing a test run as to what conditions lead to skipping + (or xfail-ing) tests. + +- fix issue28 - setup_method and pytest_generate_tests work together + The setup_method fixture method now gets called also for + test function invocations generated from the pytest_generate_tests + hook. + +- fix issue27 - collectonly and keyword-selection (-k) now work together + Also, if you do "py.test --collectonly -q" you now get a flat list + of test ids that you can use to paste to the py.test commandline + in order to execute a particular test. + +- fix issue25 avoid reported problems with --pdb and python3.2/encodings output + +- fix issue23 - tmpdir argument now works on Python3.2 and WindowsXP + Starting with Python3.2 os.symlink may be supported. By requiring + a newer py lib version the py.path.local() implementation acknowledges + this. + +- fixed typos in the docs (thanks Victor Garcia, Brianna Laugher) and particular + thanks to Laura Creighton who also reviewed parts of the documentation. + +- fix slightly wrong output of verbose progress reporting for classes + (thanks Amaury) + +- more precise (avoiding of) deprecation warnings for node.Class|Function accesses + +- avoid std unittest assertion helper code in tracebacks (thanks Ronny) + +2.0.1 (2011-02-07) +================== + +- refine and unify initial capturing so that it works nicely + even if the logging module is used on an early-loaded conftest.py + file or plugin. +- allow to omit "()" in test ids to allow for uniform test ids + as produced by Alfredo's nice pytest.vim plugin. +- fix issue12 - show plugin versions with "--version" and + "--traceconfig" and also document how to add extra information + to reporting test header +- fix issue17 (import-* reporting issue on python3) by + requiring py>1.4.0 (1.4.1 is going to include it) +- fix issue10 (numpy arrays truth checking) by refining + assertion interpretation in py lib +- fix issue15: make nose compatibility tests compatible + with python3 (now that nose-1.0 supports python3) +- remove somewhat surprising "same-conftest" detection because + it ignores conftest.py when they appear in several subdirs. +- improve assertions ("not in"), thanks Floris Bruynooghe +- improve behaviour/warnings when running on top of "python -OO" + (assertions and docstrings are turned off, leading to potential + false positives) +- introduce a pytest_cmdline_processargs(args) hook + to allow dynamic computation of command line arguments. + This fixes a regression because py.test prior to 2.0 + allowed to set command line options from conftest.py + files which so far pytest-2.0 only allowed from ini-files now. +- fix issue7: assert failures in doctest modules. + unexpected failures in doctests will not generally + show nicer, i.e. within the doctest failing context. +- fix issue9: setup/teardown functions for an xfail-marked + test will report as xfail if they fail but report as normally + passing (not xpassing) if they succeed. This only is true + for "direct" setup/teardown invocations because teardown_class/ + teardown_module cannot closely relate to a single test. +- fix issue14: no logging errors at process exit +- refinements to "collecting" output on non-ttys +- refine internal plugin registration and --traceconfig output +- introduce a mechanism to prevent/unregister plugins from the + command line, see http://pytest.org/en/stable/how-to/plugins.html#cmdunregister +- activate resultlog plugin by default +- fix regression wrt yielded tests which due to the + collection-before-running semantics were not + setup as with pytest 1.3.4. Note, however, that + the recommended and much cleaner way to do test + parametraization remains the "pytest_generate_tests" + mechanism, see the docs. + +2.0.0 (2010-11-25) +================== + +- pytest-2.0 is now its own package and depends on pylib-2.0 +- new ability: python -m pytest / python -m pytest.main ability +- new python invocation: pytest.main(args, plugins) to load + some custom plugins early. +- try harder to run unittest test suites in a more compatible manner + by deferring setup/teardown semantics to the unittest package. + also work harder to run twisted/trial and Django tests which + should now basically work by default. +- introduce a new way to set config options via ini-style files, + by default setup.cfg and tox.ini files are searched. The old + ways (certain environment variables, dynamic conftest.py reading + is removed). +- add a new "-q" option which decreases verbosity and prints a more + nose/unittest-style "dot" output. +- fix issue135 - marks now work with unittest test cases as well +- fix issue126 - introduce py.test.set_trace() to trace execution via + PDB during the running of tests even if capturing is ongoing. +- fix issue123 - new "python -m py.test" invocation for py.test + (requires Python 2.5 or above) +- fix issue124 - make reporting more resilient against tests opening + files on filedescriptor 1 (stdout). +- fix issue109 - sibling conftest.py files will not be loaded. + (and Directory collectors cannot be customized anymore from a Directory's + conftest.py - this needs to happen at least one level up). +- introduce (customizable) assertion failure representations and enhance + output on assertion failures for comparisons and other cases (Floris Bruynooghe) +- nose-plugin: pass through type-signature failures in setup/teardown + functions instead of not calling them (Ed Singleton) +- remove py.test.collect.Directory (follows from a major refactoring + and simplification of the collection process) +- majorly reduce py.test core code, shift function/python testing to own plugin +- fix issue88 (finding custom test nodes from command line arg) +- refine 'tmpdir' creation, will now create basenames better associated + with test names (thanks Ronny) +- "xpass" (unexpected pass) tests don't cause exitcode!=0 +- fix issue131 / issue60 - importing doctests in __init__ files used as namespace packages +- fix issue93 stdout/stderr is captured while importing conftest.py +- fix bug: unittest collected functions now also can have "pytestmark" + applied at class/module level +- add ability to use "class" level for cached_setup helper +- fix strangeness: mark.* objects are now immutable, create new instances + +1.3.4 (2010-09-14) +================== + +- fix issue111: improve install documentation for windows +- fix issue119: fix custom collectability of __init__.py as a module +- fix issue116: --doctestmodules work with __init__.py files as well +- fix issue115: unify internal exception passthrough/catching/GeneratorExit +- fix issue118: new --tb=native for presenting cpython-standard exceptions + +1.3.3 (2010-07-30) +================== + +- fix issue113: assertion representation problem with triple-quoted strings + (and possibly other cases) +- make conftest loading detect that a conftest file with the same + content was already loaded, avoids surprises in nested directory structures + which can be produced e.g. by Hudson. It probably removes the need to use + --confcutdir in most cases. +- fix terminal coloring for win32 + (thanks Michael Foord for reporting) +- fix weirdness: make terminal width detection work on stdout instead of stdin + (thanks Armin Ronacher for reporting) +- remove trailing whitespace in all py/text distribution files + +1.3.2 (2010-07-08) +================== + +**New features** + +- fix issue103: introduce py.test.raises as context manager, examples:: + + with py.test.raises(ZeroDivisionError): + x = 0 + 1 / x + + with py.test.raises(RuntimeError) as excinfo: + call_something() + + # you may do extra checks on excinfo.value|type|traceback here + + (thanks Ronny Pfannschmidt) + +- Funcarg factories can now dynamically apply a marker to a + test invocation. This is for example useful if a factory + provides parameters to a test which are expected-to-fail:: + + def pytest_funcarg__arg(request): + request.applymarker(py.test.mark.xfail(reason="flaky config")) + ... + + def test_function(arg): + ... + +- improved error reporting on collection and import errors. This makes + use of a more general mechanism, namely that for custom test item/collect + nodes ``node.repr_failure(excinfo)`` is now uniformly called so that you can + override it to return a string error representation of your choice + which is going to be reported as a (red) string. + +- introduce '--junitprefix=STR' option to prepend a prefix + to all reports in the junitxml file. + +**Bug fixes** + +- make tests and the ``pytest_recwarn`` plugin in particular fully compatible + to Python2.7 (if you use the ``recwarn`` funcarg warnings will be enabled so that + you can properly check for their existence in a cross-python manner). +- refine --pdb: ignore xfailed tests, unify its TB-reporting and + don't display failures again at the end. +- fix assertion interpretation with the ** operator (thanks Benjamin Peterson) +- fix issue105 assignment on the same line as a failing assertion (thanks Benjamin Peterson) +- fix issue104 proper escaping for test names in junitxml plugin (thanks anonymous) +- fix issue57 -f|--looponfail to work with xpassing tests (thanks Ronny) +- fix issue92 collectonly reporter and --pastebin (thanks Benjamin Peterson) +- fix py.code.compile(source) to generate unique filenames +- fix assertion re-interp problems on PyPy, by deferring code + compilation to the (overridable) Frame.eval class. (thanks Amaury Forgeot) +- fix py.path.local.pyimport() to work with directories +- streamline py.path.local.mkdtemp implementation and usage +- don't print empty lines when showing junitxml-filename +- add optional boolean ignore_errors parameter to py.path.local.remove +- fix terminal writing on win32/python2.4 +- py.process.cmdexec() now tries harder to return properly encoded unicode objects + on all python versions +- install plain py.test/py.which scripts also for Jython, this helps to + get canonical script paths in virtualenv situations +- make path.bestrelpath(path) return ".", note that when calling + X.bestrelpath the assumption is that X is a directory. +- make initial conftest discovery ignore "--" prefixed arguments +- fix resultlog plugin when used in a multicpu/multihost xdist situation + (thanks Jakub Gustak) +- perform distributed testing related reporting in the xdist-plugin + rather than having dist-related code in the generic py.test + distribution +- fix homedir detection on Windows +- ship distribute_setup.py version 0.6.13 + +1.3.1 (2010-05-25) +================== + +**New features** + +- issue91: introduce new py.test.xfail(reason) helper + to imperatively mark a test as expected to fail. Can + be used from within setup and test functions. This is + useful especially for parametrized tests when certain + configurations are expected-to-fail. In this case the + declarative approach with the @py.test.mark.xfail cannot + be used as it would mark all configurations as xfail. + +- issue102: introduce new --maxfail=NUM option to stop + test runs after NUM failures. This is a generalization + of the '-x' or '--exitfirst' option which is now equivalent + to '--maxfail=1'. Both '-x' and '--maxfail' will + now also print a line near the end indicating the Interruption. + +- issue89: allow py.test.mark decorators to be used on classes + (class decorators were introduced with python2.6) and + also allow to have multiple markers applied at class/module level + by specifying a list. + +- improve and refine letter reporting in the progress bar: + . pass + f failed test + s skipped tests (reminder: use for dependency/platform mismatch only) + x xfailed test (test that was expected to fail) + X xpassed test (test that was expected to fail but passed) + + You can use any combination of 'fsxX' with the '-r' extended + reporting option. The xfail/xpass results will show up as + skipped tests in the junitxml output - which also fixes + issue99. + +- make py.test.cmdline.main() return the exitstatus instead of raising + SystemExit and also allow it to be called multiple times. This of + course requires that your application and tests are properly teared + down and don't have global state. + +**Bug Fixes** + +- improved traceback presentation: + - improved and unified reporting for "--tb=short" option + - Errors during test module imports are much shorter, (using --tb=short style) + - raises shows shorter more relevant tracebacks + - --fulltrace now more systematically makes traces longer / inhibits cutting + +- improve support for raises and other dynamically compiled code by + manipulating python's linecache.cache instead of the previous + rather hacky way of creating custom code objects. This makes + it seamlessly work on Jython and PyPy where it previously didn't. + +- fix issue96: make capturing more resilient against Control-C + interruptions (involved somewhat substantial refactoring + to the underlying capturing functionality to avoid race + conditions). + +- fix chaining of conditional skipif/xfail decorators - so it works now + as expected to use multiple @py.test.mark.skipif(condition) decorators, + including specific reporting which of the conditions lead to skipping. + +- fix issue95: late-import zlib so that it's not required + for general py.test startup. + +- fix issue94: make reporting more robust against bogus source code + (and internally be more careful when presenting unexpected byte sequences) + + +1.3.0 (2010-05-05) +================== + +- deprecate --report option in favour of a new shorter and easier to + remember -r option: it takes a string argument consisting of any + combination of 'xfsX' characters. They relate to the single chars + you see during the dotted progress printing and will print an extra line + per test at the end of the test run. This extra line indicates the exact + position or test ID that you directly paste to the py.test cmdline in order + to re-run a particular test. + +- allow external plugins to register new hooks via the new + pytest_addhooks(pluginmanager) hook. The new release of + the pytest-xdist plugin for distributed and looponfailing + testing requires this feature. + +- add a new pytest_ignore_collect(path, config) hook to allow projects and + plugins to define exclusion behaviour for their directory structure - + for example you may define in a conftest.py this method:: + + def pytest_ignore_collect(path): + return path.check(link=1) + + to prevent even a collection try of any tests in symlinked dirs. + +- new pytest_pycollect_makemodule(path, parent) hook for + allowing customization of the Module collection object for a + matching test module. + +- extend and refine xfail mechanism: + ``@py.test.mark.xfail(run=False)`` do not run the decorated test + ``@py.test.mark.xfail(reason="...")`` prints the reason string in xfail summaries + specifying ``--runxfail`` on command line virtually ignores xfail markers + +- expose (previously internal) commonly useful methods: + py.io.get_terminal_with() -> return terminal width + py.io.ansi_print(...) -> print colored/bold text on linux/win32 + py.io.saferepr(obj) -> return limited representation string + +- expose test outcome related exceptions as py.test.skip.Exception, + py.test.raises.Exception etc., useful mostly for plugins + doing special outcome interpretation/tweaking + +- (issue85) fix junitxml plugin to handle tests with non-ascii output + +- fix/refine python3 compatibility (thanks Benjamin Peterson) + +- fixes for making the jython/win32 combination work, note however: + jython2.5.1/win32 does not provide a command line launcher, see + https://bugs.jython.org/issue1491 . See pylib install documentation + for how to work around. + +- fixes for handling of unicode exception values and unprintable objects + +- (issue87) fix unboundlocal error in assertionold code + +- (issue86) improve documentation for looponfailing + +- refine IO capturing: stdin-redirect pseudo-file now has a NOP close() method + +- ship distribute_setup.py version 0.6.10 + +- added links to the new capturelog and coverage plugins + + +1.2.0 (2010-01-18) +================== + +- refined usage and options for "py.cleanup":: + + py.cleanup # remove "*.pyc" and "*$py.class" (jython) files + py.cleanup -e .swp -e .cache # also remove files with these extensions + py.cleanup -s # remove "build" and "dist" directory next to setup.py files + py.cleanup -d # also remove empty directories + py.cleanup -a # synonym for "-s -d -e 'pip-log.txt'" + py.cleanup -n # dry run, only show what would be removed + +- add a new option "py.test --funcargs" which shows available funcargs + and their help strings (docstrings on their respective factory function) + for a given test path + +- display a short and concise traceback if a funcarg lookup fails + +- early-load "conftest.py" files in non-dot first-level sub directories. + allows to conveniently keep and access test-related options in a ``test`` + subdir and still add command line options. + +- fix issue67: new super-short traceback-printing option: "--tb=line" will print a single line for each failing (python) test indicating its filename, lineno and the failure value + +- fix issue78: always call python-level teardown functions even if the + according setup failed. This includes refinements for calling setup_module/class functions + which will now only be called once instead of the previous behaviour where they'd be called + multiple times if they raise an exception (including a Skipped exception). Any exception + will be re-corded and associated with all tests in the according module/class scope. + +- fix issue63: assume <40 columns to be a bogus terminal width, default to 80 + +- fix pdb debugging to be in the correct frame on raises-related errors + +- update apipkg.py to fix an issue where recursive imports might + unnecessarily break importing + +- fix plugin links + +1.1.1 (2009-11-24) +================== + +- moved dist/looponfailing from py.test core into a new + separately released pytest-xdist plugin. + +- new junitxml plugin: --junitxml=path will generate a junit style xml file + which is processable e.g. by the Hudson CI system. + +- new option: --genscript=path will generate a standalone py.test script + which will not need any libraries installed. thanks to Ralf Schmitt. + +- new option: --ignore will prevent specified path from collection. + Can be specified multiple times. + +- new option: --confcutdir=dir will make py.test only consider conftest + files that are relative to the specified dir. + +- new funcarg: "pytestconfig" is the pytest config object for access + to command line args and can now be easily used in a test. + +- install ``py.test`` and ``py.which`` with a ``-$VERSION`` suffix to + disambiguate between Python3, python2.X, Jython and PyPy installed versions. + +- new "pytestconfig" funcarg allows access to test config object + +- new "pytest_report_header" hook can return additional lines + to be displayed at the header of a test run. + +- (experimental) allow "py.test path::name1::name2::..." for pointing + to a test within a test collection directly. This might eventually + evolve as a full substitute to "-k" specifications. + +- streamlined plugin loading: order is now as documented in + customize.html: setuptools, ENV, commandline, conftest. + also setuptools entry point names are turned to canonical names ("pytest_*") + +- automatically skip tests that need 'capfd' but have no os.dup + +- allow pytest_generate_tests to be defined in classes as well + +- deprecate usage of 'disabled' attribute in favour of pytestmark +- deprecate definition of Directory, Module, Class and Function nodes + in conftest.py files. Use pytest collect hooks instead. + +- collection/item node specific runtest/collect hooks are only called exactly + on matching conftest.py files, i.e. ones which are exactly below + the filesystem path of an item + +- change: the first pytest_collect_directory hook to return something + will now prevent further hooks to be called. + +- change: figleaf plugin now requires --figleaf to run. Also + change its long command line options to be a bit shorter (see py.test -h). + +- change: pytest doctest plugin is now enabled by default and has a + new option --doctest-glob to set a pattern for file matches. + +- change: remove internal py._* helper vars, only keep py._pydir + +- robustify capturing to survive if custom pytest_runtest_setup + code failed and prevented the capturing setup code from running. + +- make py.test.* helpers provided by default plugins visible early - + works transparently both for pydoc and for interactive sessions + which will regularly see e.g. py.test.mark and py.test.importorskip. + +- simplify internal plugin manager machinery +- simplify internal collection tree by introducing a RootCollector node + +- fix assert reinterpreation that sees a call containing "keyword=..." + +- fix issue66: invoke pytest_sessionstart and pytest_sessionfinish + hooks on worker nodes during dist-testing, report module/session teardown + hooks correctly. + +- fix issue65: properly handle dist-testing if no + execnet/py lib installed remotely. + +- skip some install-tests if no execnet is available + +- fix docs, fix internal bin/ script generation + + +1.1.0 (2009-11-05) +================== + +- introduce automatic plugin registration via 'pytest11' + entrypoints via setuptools' pkg_resources.iter_entry_points + +- fix py.test dist-testing to work with execnet >= 1.0.0b4 + +- re-introduce py.test.cmdline.main() for better backward compatibility + +- svn paths: fix a bug with path.check(versioned=True) for svn paths, + allow '%' in svn paths, make svnwc.update() default to interactive mode + like in 1.0.x and add svnwc.update(interactive=False) to inhibit interaction. + +- refine distributed tarball to contain test and no pyc files + +- try harder to have deprecation warnings for py.compat.* accesses + report a correct location + +1.0.3 +===== + +* adjust and improve docs + +* remove py.rest tool and internal namespace - it was + never really advertised and can still be used with + the old release if needed. If there is interest + it could be revived into its own tool i guess. + +* fix issue48 and issue59: raise an Error if the module + from an imported test file does not seem to come from + the filepath - avoids "same-name" confusion that has + been reported repeatedly + +* merged Ronny's nose-compatibility hacks: now + nose-style setup_module() and setup() functions are + supported + +* introduce generalized py.test.mark function marking + +* reshuffle / refine command line grouping + +* deprecate parser.addgroup in favour of getgroup which creates option group + +* add --report command line option that allows to control showing of skipped/xfailed sections + +* generalized skipping: a new way to mark python functions with skipif or xfail + at function, class and modules level based on platform or sys-module attributes. + +* extend py.test.mark decorator to allow for positional args + +* introduce and test "py.cleanup -d" to remove empty directories + +* fix issue #59 - robustify unittest test collection + +* make bpython/help interaction work by adding an __all__ attribute + to ApiModule, cleanup initpkg + +* use MIT license for pylib, add some contributors + +* remove py.execnet code and substitute all usages with 'execnet' proper + +* fix issue50 - cached_setup now caches more to expectations + for test functions with multiple arguments. + +* merge Jarko's fixes, issue #45 and #46 + +* add the ability to specify a path for py.lookup to search in + +* fix a funcarg cached_setup bug probably only occurring + in distributed testing and "module" scope with teardown. + +* many fixes and changes for making the code base python3 compatible, + many thanks to Benjamin Peterson for helping with this. + +* consolidate builtins implementation to be compatible with >=2.3, + add helpers to ease keeping 2 and 3k compatible code + +* deprecate py.compat.doctest|subprocess|textwrap|optparse + +* deprecate py.magic.autopath, remove py/magic directory + +* move pytest assertion handling to py/code and a pytest_assertion + plugin, add "--no-assert" option, deprecate py.magic namespaces + in favour of (less) py.code ones. + +* consolidate and cleanup py/code classes and files + +* cleanup py/misc, move tests to bin-for-dist + +* introduce delattr/delitem/delenv methods to py.test's monkeypatch funcarg + +* consolidate py.log implementation, remove old approach. + +* introduce py.io.TextIO and py.io.BytesIO for distinguishing between + text/unicode and byte-streams (uses underlying standard lib io.* + if available) + +* make py.unittest_convert helper script available which converts "unittest.py" + style files into the simpler assert/direct-test-classes py.test/nosetests + style. The script was written by Laura Creighton. + +* simplified internal localpath implementation + +1.0.2 (2009-08-27) +================== + +* fixing packaging issues, triggered by fedora redhat packaging, + also added doc, examples and contrib dirs to the tarball. + +* added a documentation link to the new django plugin. + +1.0.1 (2009-08-19) +================== + +* added a 'pytest_nose' plugin which handles nose.SkipTest, + nose-style function/method/generator setup/teardown and + tries to report functions correctly. + +* capturing of unicode writes or encoded strings to sys.stdout/err + work better, also terminalwriting was adapted and somewhat + unified between windows and linux. + +* improved documentation layout and content a lot + +* added a "--help-config" option to show conftest.py / ENV-var names for + all longopt cmdline options, and some special conftest.py variables. + renamed 'conf_capture' conftest setting to 'option_capture' accordingly. + +* fix issue #27: better reporting on non-collectable items given on commandline + (e.g. pyc files) + +* fix issue #33: added --version flag (thanks Benjamin Peterson) + +* fix issue #32: adding support for "incomplete" paths to wcpath.status() + +* "Test" prefixed classes are *not* collected by default anymore if they + have an __init__ method + +* monkeypatch setenv() now accepts a "prepend" parameter + +* improved reporting of collection error tracebacks + +* simplified multicall mechanism and plugin architecture, + renamed some internal methods and argnames + +1.0.0 (2009-08-04) +================== + +* more terse reporting try to show filesystem path relatively to current dir +* improve xfail output a bit + +1.0.0b9 (2009-07-31) +==================== + +* cleanly handle and report final teardown of test setup + +* fix svn-1.6 compat issue with py.path.svnwc().versioned() + (thanks Wouter Vanden Hove) + +* setup/teardown or collection problems now show as ERRORs + or with big "E"'s in the progress lines. they are reported + and counted separately. + +* dist-testing: properly handle test items that get locally + collected but cannot be collected on the remote side - often + due to platform/dependency reasons + +* simplified py.test.mark API - see keyword plugin documentation + +* integrate better with logging: capturing now by default captures + test functions and their immediate setup/teardown in a single stream + +* capsys and capfd funcargs now have a readouterr() and a close() method + (underlyingly py.io.StdCapture/FD objects are used which grew a + readouterr() method as well to return snapshots of captured out/err) + +* make assert-reinterpretation work better with comparisons not + returning bools (reported with numpy from thanks maciej fijalkowski) + +* reworked per-test output capturing into the pytest_iocapture.py plugin + and thus removed capturing code from config object + +* item.repr_failure(excinfo) instead of item.repr_failure(excinfo, outerr) + + +1.0.0b8 (2009-07-22) +==================== + +* pytest_unittest-plugin is now enabled by default + +* introduced pytest_keyboardinterrupt hook and + refined pytest_sessionfinish hooked, added tests. + +* workaround a buggy logging module interaction ("closing already closed + files"). Thanks to Sridhar Ratnakumar for triggering. + +* if plugins use "py.test.importorskip" for importing + a dependency only a warning will be issued instead + of exiting the testing process. + +* many improvements to docs: + - refined funcargs doc , use the term "factory" instead of "provider" + - added a new talk/tutorial doc page + - better download page + - better plugin docstrings + - added new plugins page and automatic doc generation script + +* fixed teardown problem related to partially failing funcarg setups + (thanks MrTopf for reporting), "pytest_runtest_teardown" is now + always invoked even if the "pytest_runtest_setup" failed. + +* tweaked doctest output for docstrings in py modules, + thanks Radomir. + +1.0.0b7 +======= + +* renamed py.test.xfail back to py.test.mark.xfail to avoid + two ways to decorate for xfail + +* re-added py.test.mark decorator for setting keywords on functions + (it was actually documented so removing it was not nice) + +* remove scope-argument from request.addfinalizer() because + request.cached_setup has the scope arg. TOOWTDI. + +* perform setup finalization before reporting failures + +* apply modified patches from Andreas Kloeckner to allow + test functions to have no func_code (#22) and to make + "-k" and function keywords work (#20) + +* apply patch from Daniel Peolzleithner (issue #23) + +* resolve issue #18, multiprocessing.Manager() and + redirection clash + +* make __name__ == "__channelexec__" for remote_exec code + +1.0.0b3 (2009-06-19) +==================== + +* plugin classes are removed: one now defines + hooks directly in conftest.py or global pytest_*.py + files. + +* added new pytest_namespace(config) hook that allows + to inject helpers directly to the py.test.* namespace. + +* documented and refined many hooks + +* added new style of generative tests via + pytest_generate_tests hook that integrates + well with function arguments. + + +1.0.0b1 +======= + +* introduced new "funcarg" setup method, + see doc/test/funcarg.txt + +* introduced plugin architecture and many + new py.test plugins, see + doc/test/plugins.txt + +* teardown_method is now guaranteed to get + called after a test method has run. + +* new method: py.test.importorskip(mod,minversion) + will either import or call py.test.skip() + +* completely revised internal py.test architecture + +* new py.process.ForkedFunc object allowing to + fork execution of a function to a sub process + and getting a result back. + +XXX lots of things missing here XXX + +0.9.2 +===== + +* refined installation and metadata, created new setup.py, + now based on setuptools/ez_setup (thanks to Ralf Schmitt + for his support). + +* improved the way of making py.* scripts available in + windows environments, they are now added to the + Scripts directory as ".cmd" files. + +* py.path.svnwc.status() now is more complete and + uses xml output from the 'svn' command if available + (Guido Wesdorp) + +* fix for py.path.svn* to work with svn 1.5 + (Chris Lamb) + +* fix path.relto(otherpath) method on windows to + use normcase for checking if a path is relative. + +* py.test's traceback is better parseable from editors + (follows the filenames:LINENO: MSG convention) + (thanks to Osmo Salomaa) + +* fix to javascript-generation, "py.test --runbrowser" + should work more reliably now + +* removed previously accidentally added + py.test.broken and py.test.notimplemented helpers. + +* there now is a py.__version__ attribute + +0.9.1 +===== + +This is a fairly complete list of v0.9.1, which can +serve as a reference for developers. + +* allowing + signs in py.path.svn urls [39106] +* fixed support for Failed exceptions without excinfo in py.test [39340] +* added support for killing processes for Windows (as well as platforms that + support os.kill) in py.misc.killproc [39655] +* added setup/teardown for generative tests to py.test [40702] +* added detection of FAILED TO LOAD MODULE to py.test [40703, 40738, 40739] +* fixed problem with calling .remove() on wcpaths of non-versioned files in + py.path [44248] +* fixed some import and inheritance issues in py.test [41480, 44648, 44655] +* fail to run greenlet tests when pypy is available, but without stackless + [45294] +* small fixes in rsession tests [45295] +* fixed issue with 2.5 type representations in py.test [45483, 45484] +* made that internal reporting issues displaying is done atomically in py.test + [45518] +* made that non-existing files are ignored by the py.lookup script [45519] +* improved exception name creation in py.test [45535] +* made that less threads are used in execnet [merge in 45539] +* removed lock required for atomic reporting issue displaying in py.test + [45545] +* removed globals from execnet [45541, 45547] +* refactored cleanup mechanics, made that setDaemon is set to 1 to make atexit + get called in 2.5 (py.execnet) [45548] +* fixed bug in joining threads in py.execnet's servemain [45549] +* refactored py.test.rsession tests to not rely on exact output format anymore + [45646] +* using repr() on test outcome [45647] +* added 'Reason' classes for py.test.skip() [45648, 45649] +* killed some unnecessary sanity check in py.test.collect [45655] +* avoid using os.tmpfile() in py.io.fdcapture because on Windows it's only + usable by Administrators [45901] +* added support for locking and non-recursive commits to py.path.svnwc [45994] +* locking files in py.execnet to prevent CPython from segfaulting [46010] +* added export() method to py.path.svnurl +* fixed -d -x in py.test [47277] +* fixed argument concatenation problem in py.path.svnwc [49423] +* restore py.test behaviour that it exits with code 1 when there are failures + [49974] +* don't fail on html files that don't have an accompanying .txt file [50606] +* fixed 'utestconvert.py < input' [50645] +* small fix for code indentation in py.code.source [50755] +* fix _docgen.py documentation building [51285] +* improved checks for source representation of code blocks in py.test [51292] +* added support for passing authentication to py.path.svn* objects [52000, + 52001] +* removed sorted() call for py.apigen tests in favour of [].sort() to support + Python 2.3 [52481] diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/conf.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/conf.py new file mode 100644 index 0000000000..b316163532 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/conf.py @@ -0,0 +1,478 @@ +# +# pytest documentation build configuration file, created by +# sphinx-quickstart on Fri Oct 8 17:54:28 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The full version, including alpha/beta/rc tags. +# The short X.Y version. +import ast +import os +import shutil +import sys +from textwrap import dedent +from typing import List +from typing import TYPE_CHECKING + +from _pytest import __version__ as version + +if TYPE_CHECKING: + import sphinx.application + + +release = ".".join(version.split(".")[:2]) + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# sys.path.insert(0, os.path.abspath('.')) + +autodoc_member_order = "bysource" +autodoc_typehints = "description" +todo_include_todos = 1 + +latex_engine = "lualatex" + +latex_elements = { + "preamble": dedent( + r""" + \directlua{ + luaotfload.add_fallback("fallbacks", { + "Noto Serif CJK SC:style=Regular;", + "Symbola:Style=Regular;" + }) + } + + \setmainfont{FreeSerif}[RawFeature={fallback=fallbacks}] + """ + ) +} + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = [ + "pallets_sphinx_themes", + "pygments_pytest", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.extlinks", + "sphinx.ext.intersphinx", + "sphinx.ext.todo", + "sphinx.ext.viewcode", + "sphinx_removed_in", + "sphinxcontrib_trio", +] + +# Building PDF docs on readthedocs requires inkscape for svg to pdf +# conversion. The relevant plugin is not useful for normal HTML builds, but +# it still raises warnings and fails CI if inkscape is not available. So +# only use the plugin if inkscape is actually available. +if shutil.which("inkscape"): + extensions.append("sphinxcontrib.inkscapeconverter") + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix of source filenames. +source_suffix = ".rst" + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = "contents" + +# General information about the project. +project = "pytest" +copyright = "2015, holger krekel and pytest-dev team" + + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = [ + "_build", + "naming20.rst", + "test/*", + "old_*", + "*attic*", + "*/attic*", + "funcargs.rst", + "setup.rst", + "example/remoteinterp.rst", +] + + +# The reST default role (used for this markup: `text`) to use for all documents. +default_role = "literal" + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +add_module_names = False + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# A list of regular expressions that match URIs that should not be checked when +# doing a linkcheck. +linkcheck_ignore = [ + "https://blogs.msdn.microsoft.com/bharry/2017/06/28/testing-in-a-cloud-delivery-cadence/", + "http://pythontesting.net/framework/pytest-introduction/", + r"https://github.com/pytest-dev/pytest/issues/\d+", + r"https://github.com/pytest-dev/pytest/pull/\d+", +] + +# The number of worker threads to use when checking links (default=5). +linkcheck_workers = 5 + + +_repo = "https://github.com/pytest-dev/pytest" +extlinks = { + "bpo": ("https://bugs.python.org/issue%s", "bpo-"), + "pypi": ("https://pypi.org/project/%s/", ""), + "issue": (f"{_repo}/issues/%s", "issue #"), + "pull": (f"{_repo}/pull/%s", "pull request #"), + "user": ("https://github.com/%s", "@"), +} + + +# -- Options for HTML output --------------------------------------------------- + +sys.path.append(os.path.abspath("_themes")) +html_theme_path = ["_themes"] + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "flask" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {"index_logo": None} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +html_title = "pytest documentation" + +# A shorter title for the navigation bar. Default is the same as html_title. +html_short_title = "pytest-%s" % release + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +html_logo = "img/pytest_logo_curves.svg" + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +html_favicon = "img/favicon.png" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} +# html_sidebars = {'index': 'indexsidebar.html'} + +html_sidebars = { + "index": [ + "slim_searchbox.html", + "sidebarintro.html", + "globaltoc.html", + "links.html", + "sourcelink.html", + ], + "**": [ + "slim_searchbox.html", + "globaltoc.html", + "relations.html", + "links.html", + "sourcelink.html", + ], +} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} +# html_additional_pages = {'index': 'index.html'} + + +# If false, no module index is generated. +html_domain_indices = True + +# If false, no index is generated. +html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +html_show_sourcelink = False + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = "pytestdoc" + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +# latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +# latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ( + "contents", + "pytest.tex", + "pytest Documentation", + "holger krekel, trainer and consultant, https://merlinux.eu/", + "manual", + ) +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +latex_logo = "img/pytest1.png" + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Additional stuff for the LaTeX preamble. +# latex_preamble = '' + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +latex_domain_indices = False + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [("usage", "pytest", "pytest usage", ["holger krekel at merlinux eu"], 1)] + + +# -- Options for Epub output --------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = "pytest" +epub_author = "holger krekel at merlinux eu" +epub_publisher = "holger krekel at merlinux eu" +epub_copyright = "2013, holger krekel et alii" + +# The language of the text. It defaults to the language option +# or en if the language is not set. +# epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +# epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# epub_identifier = '' + +# A unique identification for the text. +# epub_uid = '' + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +# epub_pre_files = [] + +# HTML files shat should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +# epub_post_files = [] + +# A list of files that should not be packed into the epub file. +# epub_exclude_files = [] + +# The depth of the table of contents in toc.ncx. +# epub_tocdepth = 3 + +# Allow duplicate toc entries. +# epub_tocdup = True + + +# -- Options for texinfo output ------------------------------------------------ + +texinfo_documents = [ + ( + master_doc, + "pytest", + "pytest Documentation", + ( + "Holger Krekel@*Benjamin Peterson@*Ronny Pfannschmidt@*" + "Floris Bruynooghe@*others" + ), + "pytest", + "simple powerful testing with Python", + "Programming", + 1, + ) +] + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + "pluggy": ("https://pluggy.readthedocs.io/en/stable", None), + "python": ("https://docs.python.org/3", None), + "numpy": ("https://numpy.org/doc/stable", None), + "pip": ("https://pip.pypa.io/en/stable", None), + "tox": ("https://tox.wiki/en/stable", None), + "virtualenv": ("https://virtualenv.pypa.io/en/stable", None), + "django": ( + "http://docs.djangoproject.com/en/stable", + "http://docs.djangoproject.com/en/stable/_objects", + ), + "setuptools": ("https://setuptools.pypa.io/en/stable", None), +} + + +def configure_logging(app: "sphinx.application.Sphinx") -> None: + """Configure Sphinx's WarningHandler to handle (expected) missing include.""" + import sphinx.util.logging + import logging + + class WarnLogFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + """Ignore warnings about missing include with "only" directive. + + Ref: https://github.com/sphinx-doc/sphinx/issues/2150.""" + if ( + record.msg.startswith('Problems with "include" directive path:') + and "_changelog_towncrier_draft.rst" in record.msg + ): + return False + return True + + logger = logging.getLogger(sphinx.util.logging.NAMESPACE) + warn_handler = [x for x in logger.handlers if x.level == logging.WARNING] + assert len(warn_handler) == 1, warn_handler + warn_handler[0].filters.insert(0, WarnLogFilter()) + + +def setup(app: "sphinx.application.Sphinx") -> None: + # from sphinx.ext.autodoc import cut_lines + # app.connect('autodoc-process-docstring', cut_lines(4, what=['module'])) + app.add_crossref_type( + "fixture", + "fixture", + objname="built-in fixture", + indextemplate="pair: %s; fixture", + ) + + app.add_object_type( + "confval", + "confval", + objname="configuration value", + indextemplate="pair: %s; configuration value", + ) + + app.add_object_type( + "globalvar", + "globalvar", + objname="global variable interpreted by pytest", + indextemplate="pair: %s; global variable interpreted by pytest", + ) + + app.add_crossref_type( + directivename="hook", + rolename="hook", + objname="pytest hook", + indextemplate="pair: %s; hook", + ) + + configure_logging(app) + + # Make Sphinx mark classes with "final" when decorated with @final. + # We need this because we import final from pytest._compat, not from + # typing (for Python < 3.8 compat), so Sphinx doesn't detect it. + # To keep things simple we accept any `@final` decorator. + # Ref: https://github.com/pytest-dev/pytest/pull/7780 + import sphinx.pycode.ast + import sphinx.pycode.parser + + original_is_final = sphinx.pycode.parser.VariableCommentPicker.is_final + + def patched_is_final(self, decorators: List[ast.expr]) -> bool: + if original_is_final(self, decorators): + return True + return any( + sphinx.pycode.ast.unparse(decorator) == "final" for decorator in decorators + ) + + sphinx.pycode.parser.VariableCommentPicker.is_final = patched_is_final + + # legacypath.py monkey-patches pytest.Testdir in. Import the file so + # that autodoc can discover references to it. + import _pytest.legacypath # noqa: F401 diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/conftest.py new file mode 100644 index 0000000000..1a62e1b5df --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/conftest.py @@ -0,0 +1 @@ +collect_ignore = ["conf.py"] diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/contact.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/contact.rst new file mode 100644 index 0000000000..beed10d7f2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/contact.rst @@ -0,0 +1,54 @@ + +.. _`contact channels`: +.. _`contact`: + +Contact channels +=================================== + +- `pytest issue tracker`_ to report bugs or suggest features (for version + 2.0 and above). +- `pytest discussions`_ at github for general questions. +- `pytest discord server `_ + for pytest development visibility and general assistance. +- `pytest on stackoverflow.com `_ + to post precise questions with the tag ``pytest``. New Questions will usually + be seen by pytest users or developers and answered quickly. + +- `Testing In Python`_: a mailing list for Python testing tools and discussion. + +- `pytest-dev at python.org (mailing list)`_ pytest specific announcements and discussions. + +- :doc:`contribution guide ` for help on submitting pull + requests to GitHub. + +- ``#pytest`` `on irc.libera.chat `_ IRC + channel for random questions (using an IRC client, `via webchat + `_, or `via Matrix + `_). + +- private mail to Holger.Krekel at gmail com if you want to communicate sensitive issues + + +- `merlinux.eu`_ offers pytest and tox-related professional teaching and + consulting. + +.. _`pytest issue tracker`: https://github.com/pytest-dev/pytest/issues +.. _`old issue tracker`: https://bitbucket.org/hpk42/py-trunk/issues/ + +.. _`pytest discussions`: https://github.com/pytest-dev/pytest/discussions + +.. _`merlinux.eu`: https://merlinux.eu/ + +.. _`get an account`: + +.. _tetamap: https://tetamap.wordpress.com/ + +.. _`@pylibcommit`: https://twitter.com/pylibcommit + + +.. _`Testing in Python`: http://lists.idyll.org/listinfo/testing-in-python +.. _FOAF: https://en.wikipedia.org/wiki/FOAF +.. _`py-dev`: +.. _`development mailing list`: +.. _`pytest-dev at python.org (mailing list)`: http://mail.python.org/mailman/listinfo/pytest-dev +.. _`pytest-commit at python.org (mailing list)`: http://mail.python.org/mailman/listinfo/pytest-commit diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/contents.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/contents.rst new file mode 100644 index 0000000000..049d44ba9d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/contents.rst @@ -0,0 +1,116 @@ +.. _toc: + +Full pytest documentation +=========================== + +`Download latest version as PDF `_ + +.. `Download latest version as EPUB `_ + + +Start here +----------- + +.. toctree:: + :maxdepth: 2 + + getting-started + + +How-to guides +------------- + +.. toctree:: + :maxdepth: 2 + + how-to/usage + how-to/assert + how-to/fixtures + how-to/mark + how-to/parametrize + how-to/tmp_path + how-to/monkeypatch + how-to/doctest + how-to/cache + + how-to/logging + how-to/capture-stdout-stderr + how-to/capture-warnings + how-to/skipping + + how-to/plugins + how-to/writing_plugins + how-to/writing_hook_functions + + how-to/existingtestsuite + how-to/unittest + how-to/nose + how-to/xunit_setup + + how-to/bash-completion + + +Reference guides +----------------- + +.. toctree:: + :maxdepth: 2 + + reference/fixtures + reference/plugin_list + reference/customize + reference/reference + + +Explanation +----------------- + +.. toctree:: + :maxdepth: 2 + + explanation/anatomy + explanation/fixtures + explanation/goodpractices + explanation/flaky + explanation/pythonpath + + +Further topics +----------------- + +.. toctree:: + :maxdepth: 2 + + example/index + + backwards-compatibility + deprecations + py27-py34-deprecation + + contributing + development_guide + + sponsor + tidelift + license + contact + + history + historical-notes + talks + + +.. only:: html + + .. toctree:: + :maxdepth: 1 + + announce/index + +.. only:: html + + .. toctree:: + :hidden: + :maxdepth: 1 + + changelog diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/contributing.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/contributing.rst new file mode 100644 index 0000000000..2b6578f6b9 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/contributing.rst @@ -0,0 +1,3 @@ +.. _contributing: + +.. include:: ../../CONTRIBUTING.rst diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/deprecations.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/deprecations.rst new file mode 100644 index 0000000000..0f19744ade --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/deprecations.rst @@ -0,0 +1,954 @@ +.. _deprecations: + +Deprecations and Removals +========================= + +This page lists all pytest features that are currently deprecated or have been removed in past major releases. +The objective is to give users a clear rationale why a certain feature has been removed, and what alternatives +should be used instead. + +.. contents:: + :depth: 3 + :local: + + +Deprecated Features +------------------- + +Below is a complete list of all pytest features which are considered deprecated. Using those features will issue +:class:`PytestWarning` or subclasses, which can be filtered using :ref:`standard warning filters `. + +.. _instance-collector-deprecation: + +The ``pytest.Instance`` collector +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 7.0 + +The ``pytest.Instance`` collector type has been removed. + +Previously, Python test methods were collected as :class:`~pytest.Class` -> ``Instance`` -> :class:`~pytest.Function`. +Now :class:`~pytest.Class` collects the test methods directly. + +Most plugins which reference ``Instance`` do so in order to ignore or skip it, +using a check such as ``if isinstance(node, Instance): return``. +Such plugins should simply remove consideration of ``Instance`` on pytest>=7. +However, to keep such uses working, a dummy type has been instanted in ``pytest.Instance`` and ``_pytest.python.Instance``, +and importing it emits a deprecation warning. This will be removed in pytest 8. + + +.. _node-ctor-fspath-deprecation: + +``fspath`` argument for Node constructors replaced with ``pathlib.Path`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 7.0 + +In order to support the transition from ``py.path.local`` to :mod:`pathlib`, +the ``fspath`` argument to :class:`~_pytest.nodes.Node` constructors like +:func:`pytest.Function.from_parent()` and :func:`pytest.Class.from_parent()` +is now deprecated. + +Plugins which construct nodes should pass the ``path`` argument, of type +:class:`pathlib.Path`, instead of the ``fspath`` argument. + +Plugins which implement custom items and collectors are encouraged to replace +``fspath`` parameters (``py.path.local``) with ``path`` parameters +(``pathlib.Path``), and drop any other usage of the ``py`` library if possible. + +If possible, plugins with custom items should use :ref:`cooperative +constructors ` to avoid hardcoding +arguments they only pass on to the superclass. + +.. note:: + The name of the :class:`~_pytest.nodes.Node` arguments and attributes (the + new attribute being ``path``) is **the opposite** of the situation for + hooks, :ref:`outlined below ` (the old + argument being ``path``). + + This is an unfortunate artifact due to historical reasons, which should be + resolved in future versions as we slowly get rid of the :pypi:`py` + dependency (see :issue:`9283` for a longer discussion). + +Due to the ongoing migration of methods like :meth:`~_pytest.Item.reportinfo` +which still is expected to return a ``py.path.local`` object, nodes still have +both ``fspath`` (``py.path.local``) and ``path`` (``pathlib.Path``) attributes, +no matter what argument was used in the constructor. We expect to deprecate the +``fspath`` attribute in a future release. + +.. _legacy-path-hooks-deprecated: + +``py.path.local`` arguments for hooks replaced with ``pathlib.Path`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 7.0 + +In order to support the transition from ``py.path.local`` to :mod:`pathlib`, the following hooks now receive additional arguments: + +* :hook:`pytest_ignore_collect(collection_path: pathlib.Path) ` as equivalent to ``path`` +* :hook:`pytest_collect_file(file_path: pathlib.Path) ` as equivalent to ``path`` +* :hook:`pytest_pycollect_makemodule(module_path: pathlib.Path) ` as equivalent to ``path`` +* :hook:`pytest_report_header(start_path: pathlib.Path) ` as equivalent to ``startdir`` +* :hook:`pytest_report_collectionfinish(start_path: pathlib.Path) ` as equivalent to ``startdir`` + +The accompanying ``py.path.local`` based paths have been deprecated: plugins which manually invoke those hooks should only pass the new ``pathlib.Path`` arguments, and users should change their hook implementations to use the new ``pathlib.Path`` arguments. + +.. note:: + The name of the :class:`~_pytest.nodes.Node` arguments and attributes, + :ref:`outlined above ` (the new attribute + being ``path``) is **the opposite** of the situation for hooks (the old + argument being ``path``). + + This is an unfortunate artifact due to historical reasons, which should be + resolved in future versions as we slowly get rid of the :pypi:`py` + dependency (see :issue:`9283` for a longer discussion). + +Directly constructing internal classes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 7.0 + +Directly constructing the following classes is now deprecated: + +- ``_pytest.mark.structures.Mark`` +- ``_pytest.mark.structures.MarkDecorator`` +- ``_pytest.mark.structures.MarkGenerator`` +- ``_pytest.python.Metafunc`` +- ``_pytest.runner.CallInfo`` +- ``_pytest._code.ExceptionInfo`` +- ``_pytest.config.argparsing.Parser`` +- ``_pytest.config.argparsing.OptionGroup`` +- ``_pytest.pytester.HookRecorder`` + +These constructors have always been considered private, but now issue a deprecation warning, which may become a hard error in pytest 8. + +.. _cmdline-preparse-deprecated: + +Passing ``msg=`` to ``pytest.skip``, ``pytest.fail`` or ``pytest.exit`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 7.0 + +Passing the keyword argument ``msg`` to :func:`pytest.skip`, :func:`pytest.fail` or :func:`pytest.exit` +is now deprecated and ``reason`` should be used instead. This change is to bring consistency between these +functions and the ``@pytest.mark.skip`` and ``@pytest.mark.xfail`` markers which already accept a ``reason`` argument. + +.. code-block:: python + + def test_fail_example(): + # old + pytest.fail(msg="foo") + # new + pytest.fail(reason="bar") + + + def test_skip_example(): + # old + pytest.skip(msg="foo") + # new + pytest.skip(reason="bar") + + + def test_exit_example(): + # old + pytest.exit(msg="foo") + # new + pytest.exit(reason="bar") + + +Implementing the ``pytest_cmdline_preparse`` hook +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 7.0 + +Implementing the :hook:`pytest_cmdline_preparse` hook has been officially deprecated. +Implement the :hook:`pytest_load_initial_conftests` hook instead. + +.. code-block:: python + + def pytest_cmdline_preparse(config: Config, args: List[str]) -> None: + ... + + + # becomes: + + + def pytest_load_initial_conftests( + early_config: Config, parser: Parser, args: List[str] + ) -> None: + ... + +.. _diamond-inheritance-deprecated: + +Diamond inheritance between :class:`pytest.Collector` and :class:`pytest.Item` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 7.0 + +Defining a custom pytest node type which is both an :class:`pytest.Item ` and a :class:`pytest.Collector ` (e.g. :class:`pytest.File `) now issues a warning. +It was never sanely supported and triggers hard to debug errors. + +Some plugins providing linting/code analysis have been using this as a hack. +Instead, a separate collector node should be used, which collects the item. See +:ref:`non-python tests` for an example, as well as an `example pr fixing inheritance`_. + +.. _example pr fixing inheritance: https://github.com/asmeurer/pytest-flakes/pull/40/files + + +.. _uncooperative-constructors-deprecated: + +Constructors of custom :class:`pytest.Node` subclasses should take ``**kwargs`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 7.0 + +If custom subclasses of nodes like :class:`pytest.Item` override the +``__init__`` method, they should take ``**kwargs``. Thus, + +.. code-block:: python + + class CustomItem(pytest.Item): + def __init__(self, name, parent, additional_arg): + super().__init__(name, parent) + self.additional_arg = additional_arg + +should be turned into: + +.. code-block:: python + + class CustomItem(pytest.Item): + def __init__(self, *, additional_arg, **kwargs): + super().__init__(**kwargs) + self.additional_arg = additional_arg + +to avoid hard-coding the arguments pytest can pass to the superclass. +See :ref:`non-python tests` for a full example. + +For cases without conflicts, no deprecation warning is emitted. For cases with +conflicts (such as :class:`pytest.File` now taking ``path`` instead of +``fspath``, as :ref:`outlined above `), a +deprecation warning is now raised. + +Backward compatibilities in ``Parser.addoption`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 2.4 + +Several behaviors of :meth:`Parser.addoption ` are now +scheduled for removal in pytest 8 (deprecated since pytest 2.4.0): + +- ``parser.addoption(..., help=".. %default ..")`` - use ``%(default)s`` instead. +- ``parser.addoption(..., type="int/string/float/complex")`` - use ``type=int`` etc. instead. + + +Raising ``unittest.SkipTest`` during collection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 7.0 + +Raising :class:`unittest.SkipTest` to skip collection of tests during the +pytest collection phase is deprecated. Use :func:`pytest.skip` instead. + +Note: This deprecation only relates to using `unittest.SkipTest` during test +collection. You are probably not doing that. Ordinary usage of +:class:`unittest.SkipTest` / :meth:`unittest.TestCase.skipTest` / +:func:`unittest.skip` in unittest test cases is fully supported. + +Using ``pytest.warns(None)`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 7.0 + +:func:`pytest.warns(None) ` is now deprecated because it was frequently misused. +Its correct usage was checking that the code emits at least one warning of any type - like ``pytest.warns()`` +or ``pytest.warns(Warning)``. + +See :ref:`warns use cases` for examples. + +The ``--strict`` command-line option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 6.2 + +The ``--strict`` command-line option has been deprecated in favor of ``--strict-markers``, which +better conveys what the option does. + +We have plans to maybe in the future to reintroduce ``--strict`` and make it an encompassing +flag for all strictness related options (``--strict-markers`` and ``--strict-config`` +at the moment, more might be introduced in the future). + + +The ``yield_fixture`` function/decorator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 6.2 + +``pytest.yield_fixture`` is a deprecated alias for :func:`pytest.fixture`. + +It has been so for a very long time, so can be search/replaced safely. + + +The ``pytest_warning_captured`` hook +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 6.0 + +This hook has an `item` parameter which cannot be serialized by ``pytest-xdist``. + +Use the ``pytest_warning_recored`` hook instead, which replaces the ``item`` parameter +by a ``nodeid`` parameter. + +The ``pytest.collect`` module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 6.0 + +The ``pytest.collect`` module is no longer part of the public API, all its names +should now be imported from ``pytest`` directly instead. + + +The ``pytest._fillfuncargs`` function +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 6.0 + +This function was kept for backward compatibility with an older plugin. + +It's functionality is not meant to be used directly, but if you must replace +it, use `function._request._fillfixtures()` instead, though note this is not +a public API and may break in the future. + + +Removed Features +---------------- + +As stated in our :ref:`backwards-compatibility` policy, deprecated features are removed only in major releases after +an appropriate period of deprecation has passed. + +``--no-print-logs`` command-line option +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 5.4 +.. versionremoved:: 6.0 + + +The ``--no-print-logs`` option and ``log_print`` ini setting are removed. If +you used them, please use ``--show-capture`` instead. + +A ``--show-capture`` command-line option was added in ``pytest 3.5.0`` which allows to specify how to +display captured output when tests fail: ``no``, ``stdout``, ``stderr``, ``log`` or ``all`` (the default). + + +.. _resultlog deprecated: + +Result log (``--result-log``) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 4.0 +.. versionremoved:: 6.0 + +The ``--result-log`` option produces a stream of test reports which can be +analysed at runtime, but it uses a custom format which requires users to implement their own +parser. + +The `pytest-reportlog `__ plugin provides a ``--report-log`` option, a more standard and extensible alternative, producing +one JSON object per-line, and should cover the same use cases. Please try it out and provide feedback. + +The ``pytest-reportlog`` plugin might even be merged into the core +at some point, depending on the plans for the plugins and number of users using it. + +``pytest_collect_directory`` hook +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 6.0 + +The ``pytest_collect_directory`` hook has not worked properly for years (it was called +but the results were ignored). Users may consider using :hook:`pytest_collection_modifyitems` instead. + +TerminalReporter.writer +~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 6.0 + +The ``TerminalReporter.writer`` attribute has been deprecated and should no longer be used. This +was inadvertently exposed as part of the public API of that plugin and ties it too much +with ``py.io.TerminalWriter``. + +Plugins that used ``TerminalReporter.writer`` directly should instead use ``TerminalReporter`` +methods that provide the same functionality. + +.. _junit-family changed default value: + +``junit_family`` default value change to "xunit2" +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionchanged:: 6.0 + +The default value of ``junit_family`` option will change to ``xunit2`` in pytest 6.0, which +is an update of the old ``xunit1`` format and is supported by default in modern tools +that manipulate this type of file (for example, Jenkins, Azure Pipelines, etc.). + +Users are recommended to try the new ``xunit2`` format and see if their tooling that consumes the JUnit +XML file supports it. + +To use the new format, update your ``pytest.ini``: + +.. code-block:: ini + + [pytest] + junit_family=xunit2 + +If you discover that your tooling does not support the new format, and want to keep using the +legacy version, set the option to ``legacy`` instead: + +.. code-block:: ini + + [pytest] + junit_family=legacy + +By using ``legacy`` you will keep using the legacy/xunit1 format when upgrading to +pytest 6.0, where the default format will be ``xunit2``. + +In order to let users know about the transition, pytest will issue a warning in case +the ``--junitxml`` option is given in the command line but ``junit_family`` is not explicitly +configured in ``pytest.ini``. + +Services known to support the ``xunit2`` format: + +* `Jenkins `__ with the `JUnit `__ plugin. +* `Azure Pipelines `__. + +Node Construction changed to ``Node.from_parent`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionchanged:: 6.0 + +The construction of nodes now should use the named constructor ``from_parent``. +This limitation in api surface intends to enable better/simpler refactoring of the collection tree. + +This means that instead of :code:`MyItem(name="foo", parent=collector, obj=42)` +one now has to invoke :code:`MyItem.from_parent(collector, name="foo")`. + +Plugins that wish to support older versions of pytest and suppress the warning can use +`hasattr` to check if `from_parent` exists in that version: + +.. code-block:: python + + def pytest_pycollect_makeitem(collector, name, obj): + if hasattr(MyItem, "from_parent"): + item = MyItem.from_parent(collector, name="foo") + item.obj = 42 + return item + else: + return MyItem(name="foo", parent=collector, obj=42) + +Note that ``from_parent`` should only be called with keyword arguments for the parameters. + + +``pytest.fixture`` arguments are keyword only +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 6.0 + +Passing arguments to pytest.fixture() as positional arguments has been removed - pass them by keyword instead. + +``funcargnames`` alias for ``fixturenames`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 6.0 + +The ``FixtureRequest``, ``Metafunc``, and ``Function`` classes track the names of +their associated fixtures, with the aptly-named ``fixturenames`` attribute. + +Prior to pytest 2.3, this attribute was named ``funcargnames``, and we have kept +that as an alias since. It is finally due for removal, as it is often confusing +in places where we or plugin authors must distinguish between fixture names and +names supplied by non-fixture things such as ``pytest.mark.parametrize``. + + +.. _pytest.config global deprecated: + +``pytest.config`` global +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 5.0 + +The ``pytest.config`` global object is deprecated. Instead use +``request.config`` (via the ``request`` fixture) or if you are a plugin author +use the ``pytest_configure(config)`` hook. Note that many hooks can also access +the ``config`` object indirectly, through ``session.config`` or ``item.config`` for example. + + +.. _`raises message deprecated`: + +``"message"`` parameter of ``pytest.raises`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 5.0 + +It is a common mistake to think this parameter will match the exception message, while in fact +it only serves to provide a custom message in case the ``pytest.raises`` check fails. To prevent +users from making this mistake, and because it is believed to be little used, pytest is +deprecating it without providing an alternative for the moment. + +If you have a valid use case for this parameter, consider that to obtain the same results +you can just call ``pytest.fail`` manually at the end of the ``with`` statement. + +For example: + +.. code-block:: python + + with pytest.raises(TimeoutError, message="Client got unexpected message"): + wait_for(websocket.recv(), 0.5) + + +Becomes: + +.. code-block:: python + + with pytest.raises(TimeoutError): + wait_for(websocket.recv(), 0.5) + pytest.fail("Client got unexpected message") + + +If you still have concerns about this deprecation and future removal, please comment on +:issue:`3974`. + + +.. _raises-warns-exec: + +``raises`` / ``warns`` with a string as the second argument +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 5.0 + +Use the context manager form of these instead. When necessary, invoke ``exec`` +directly. + +Example: + +.. code-block:: python + + pytest.raises(ZeroDivisionError, "1 / 0") + pytest.raises(SyntaxError, "a $ b") + + pytest.warns(DeprecationWarning, "my_function()") + pytest.warns(SyntaxWarning, "assert(1, 2)") + +Becomes: + +.. code-block:: python + + with pytest.raises(ZeroDivisionError): + 1 / 0 + with pytest.raises(SyntaxError): + exec("a $ b") # exec is required for invalid syntax + + with pytest.warns(DeprecationWarning): + my_function() + with pytest.warns(SyntaxWarning): + exec("assert(1, 2)") # exec is used to avoid a top-level warning + + + + +Using ``Class`` in custom Collectors +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +Using objects named ``"Class"`` as a way to customize the type of nodes that are collected in ``Collector`` +subclasses has been deprecated. Users instead should use ``pytest_pycollect_makeitem`` to customize node types during +collection. + +This issue should affect only advanced plugins who create new collection types, so if you see this warning +message please contact the authors so they can change the code. + + +.. _marks in pytest.parametrize deprecated: + +marks in ``pytest.mark.parametrize`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +Applying marks to values of a ``pytest.mark.parametrize`` call is now deprecated. For example: + +.. code-block:: python + + @pytest.mark.parametrize( + "a, b", + [ + (3, 9), + pytest.mark.xfail(reason="flaky")(6, 36), + (10, 100), + (20, 200), + (40, 400), + (50, 500), + ], + ) + def test_foo(a, b): + ... + +This code applies the ``pytest.mark.xfail(reason="flaky")`` mark to the ``(6, 36)`` value of the above parametrization +call. + +This was considered hard to read and understand, and also its implementation presented problems to the code preventing +further internal improvements in the marks architecture. + +To update the code, use ``pytest.param``: + +.. code-block:: python + + @pytest.mark.parametrize( + "a, b", + [ + (3, 9), + pytest.param(6, 36, marks=pytest.mark.xfail(reason="flaky")), + (10, 100), + (20, 200), + (40, 400), + (50, 500), + ], + ) + def test_foo(a, b): + ... + + +.. _pytest_funcarg__ prefix deprecated: + +``pytest_funcarg__`` prefix +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +In very early pytest versions fixtures could be defined using the ``pytest_funcarg__`` prefix: + +.. code-block:: python + + def pytest_funcarg__data(): + return SomeData() + +Switch over to the ``@pytest.fixture`` decorator: + +.. code-block:: python + + @pytest.fixture + def data(): + return SomeData() + + + +[pytest] section in setup.cfg files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +``[pytest]`` sections in ``setup.cfg`` files should now be named ``[tool:pytest]`` +to avoid conflicts with other distutils commands. + + +.. _metafunc.addcall deprecated: + +Metafunc.addcall +~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +``Metafunc.addcall`` was a precursor to the current parametrized mechanism. Users should use +:meth:`pytest.Metafunc.parametrize` instead. + +Example: + +.. code-block:: python + + def pytest_generate_tests(metafunc): + metafunc.addcall({"i": 1}, id="1") + metafunc.addcall({"i": 2}, id="2") + +Becomes: + +.. code-block:: python + + def pytest_generate_tests(metafunc): + metafunc.parametrize("i", [1, 2], ids=["1", "2"]) + + +.. _cached_setup deprecated: + +``cached_setup`` +~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +``request.cached_setup`` was the precursor of the setup/teardown mechanism available to fixtures. + +Example: + +.. code-block:: python + + @pytest.fixture + def db_session(): + return request.cached_setup( + setup=Session.create, teardown=lambda session: session.close(), scope="module" + ) + +This should be updated to make use of standard fixture mechanisms: + +.. code-block:: python + + @pytest.fixture(scope="module") + def db_session(): + session = Session.create() + yield session + session.close() + + +You can consult :std:doc:`funcarg comparison section in the docs ` for +more information. + + +.. _pytest_plugins in non-top-level conftest files deprecated: + +pytest_plugins in non-top-level conftest files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +Defining :globalvar:`pytest_plugins` is now deprecated in non-top-level conftest.py +files because they will activate referenced plugins *globally*, which is surprising because for all other pytest +features ``conftest.py`` files are only *active* for tests at or below it. + + +.. _config.warn and node.warn deprecated: + +``Config.warn`` and ``Node.warn`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +Those methods were part of the internal pytest warnings system, but since ``3.8`` pytest is using the builtin warning +system for its own warnings, so those two functions are now deprecated. + +``Config.warn`` should be replaced by calls to the standard ``warnings.warn``, example: + +.. code-block:: python + + config.warn("C1", "some warning") + +Becomes: + +.. code-block:: python + + warnings.warn(pytest.PytestWarning("some warning")) + +``Node.warn`` now supports two signatures: + +* ``node.warn(PytestWarning("some message"))``: is now the **recommended** way to call this function. + The warning instance must be a PytestWarning or subclass. + +* ``node.warn("CI", "some message")``: this code/message form has been **removed** and should be converted to the warning instance form above. + +.. _record_xml_property deprecated: + +record_xml_property +~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +The ``record_xml_property`` fixture is now deprecated in favor of the more generic ``record_property``, which +can be used by other consumers (for example ``pytest-html``) to obtain custom information about the test run. + +This is just a matter of renaming the fixture as the API is the same: + +.. code-block:: python + + def test_foo(record_xml_property): + ... + +Change to: + +.. code-block:: python + + def test_foo(record_property): + ... + + +.. _passing command-line string to pytest.main deprecated: + +Passing command-line string to ``pytest.main()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +Passing a command-line string to ``pytest.main()`` is deprecated: + +.. code-block:: python + + pytest.main("-v -s") + +Pass a list instead: + +.. code-block:: python + + pytest.main(["-v", "-s"]) + + +By passing a string, users expect that pytest will interpret that command-line using the shell rules they are working +on (for example ``bash`` or ``Powershell``), but this is very hard/impossible to do in a portable way. + + +.. _calling fixtures directly deprecated: + +Calling fixtures directly +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +Calling a fixture function directly, as opposed to request them in a test function, is deprecated. + +For example: + +.. code-block:: python + + @pytest.fixture + def cell(): + return ... + + + @pytest.fixture + def full_cell(): + cell = cell() + cell.make_full() + return cell + +This is a great source of confusion to new users, which will often call the fixture functions and request them from test functions interchangeably, which breaks the fixture resolution model. + +In those cases just request the function directly in the dependent fixture: + +.. code-block:: python + + @pytest.fixture + def cell(): + return ... + + + @pytest.fixture + def full_cell(cell): + cell.make_full() + return cell + +Alternatively if the fixture function is called multiple times inside a test (making it hard to apply the above pattern) or +if you would like to make minimal changes to the code, you can create a fixture which calls the original function together +with the ``name`` parameter: + +.. code-block:: python + + def cell(): + return ... + + + @pytest.fixture(name="cell") + def cell_fixture(): + return cell() + + +.. _yield tests deprecated: + +``yield`` tests +~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +pytest supported ``yield``-style tests, where a test function actually ``yield`` functions and values +that are then turned into proper test methods. Example: + +.. code-block:: python + + def check(x, y): + assert x ** x == y + + + def test_squared(): + yield check, 2, 4 + yield check, 3, 9 + +This would result into two actual test functions being generated. + +This form of test function doesn't support fixtures properly, and users should switch to ``pytest.mark.parametrize``: + +.. code-block:: python + + @pytest.mark.parametrize("x, y", [(2, 4), (3, 9)]) + def test_squared(x, y): + assert x ** x == y + +.. _internal classes accessed through node deprecated: + +Internal classes accessed through ``Node`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +Access of ``Module``, ``Function``, ``Class``, ``Instance``, ``File`` and ``Item`` through ``Node`` instances now issue +this warning: + +.. code-block:: text + + usage of Function.Module is deprecated, please use pytest.Module instead + +Users should just ``import pytest`` and access those objects using the ``pytest`` module. + +This has been documented as deprecated for years, but only now we are actually emitting deprecation warnings. + +``Node.get_marker`` +~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +As part of a large :ref:`marker-revamp`, ``_pytest.nodes.Node.get_marker`` is removed. See +:ref:`the documentation ` on tips on how to update your code. + + +``somefunction.markname`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +As part of a large :ref:`marker-revamp` we already deprecated using ``MarkInfo`` +the only correct way to get markers of an element is via ``node.iter_markers(name)``. + + +.. _pytest.namespace deprecated: + +``pytest_namespace`` +~~~~~~~~~~~~~~~~~~~~ + +.. versionremoved:: 4.0 + +This hook is deprecated because it greatly complicates the pytest internals regarding configuration and initialization, making some +bug fixes and refactorings impossible. + +Example of usage: + +.. code-block:: python + + class MySymbol: + ... + + + def pytest_namespace(): + return {"my_symbol": MySymbol()} + + +Plugin authors relying on this hook should instead require that users now import the plugin modules directly (with an appropriate public API). + +As a stopgap measure, plugin authors may still inject their names into pytest's namespace, usually during ``pytest_configure``: + +.. code-block:: python + + import pytest + + + def pytest_configure(): + pytest.my_symbol = MySymbol() diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/development_guide.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/development_guide.rst new file mode 100644 index 0000000000..3ee0ebbc23 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/development_guide.rst @@ -0,0 +1,7 @@ +================= +Development Guide +================= + +The contributing guidelines are to be found :ref:`here `. +The release procedure for pytest is documented on +`GitHub `_. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/failure_demo.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/failure_demo.py new file mode 100644 index 0000000000..abb9bce509 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/failure_demo.py @@ -0,0 +1,281 @@ +import pytest +from pytest import raises + + +def otherfunc(a, b): + assert a == b + + +def somefunc(x, y): + otherfunc(x, y) + + +def otherfunc_multi(a, b): + assert a == b + + +@pytest.mark.parametrize("param1, param2", [(3, 6)]) +def test_generative(param1, param2): + assert param1 * 2 < param2 + + +class TestFailing: + def test_simple(self): + def f(): + return 42 + + def g(): + return 43 + + assert f() == g() + + def test_simple_multiline(self): + otherfunc_multi(42, 6 * 9) + + def test_not(self): + def f(): + return 42 + + assert not f() + + +class TestSpecialisedExplanations: + def test_eq_text(self): + assert "spam" == "eggs" + + def test_eq_similar_text(self): + assert "foo 1 bar" == "foo 2 bar" + + def test_eq_multiline_text(self): + assert "foo\nspam\nbar" == "foo\neggs\nbar" + + def test_eq_long_text(self): + a = "1" * 100 + "a" + "2" * 100 + b = "1" * 100 + "b" + "2" * 100 + assert a == b + + def test_eq_long_text_multiline(self): + a = "1\n" * 100 + "a" + "2\n" * 100 + b = "1\n" * 100 + "b" + "2\n" * 100 + assert a == b + + def test_eq_list(self): + assert [0, 1, 2] == [0, 1, 3] + + def test_eq_list_long(self): + a = [0] * 100 + [1] + [3] * 100 + b = [0] * 100 + [2] + [3] * 100 + assert a == b + + def test_eq_dict(self): + assert {"a": 0, "b": 1, "c": 0} == {"a": 0, "b": 2, "d": 0} + + def test_eq_set(self): + assert {0, 10, 11, 12} == {0, 20, 21} + + def test_eq_longer_list(self): + assert [1, 2] == [1, 2, 3] + + def test_in_list(self): + assert 1 in [0, 2, 3, 4, 5] + + def test_not_in_text_multiline(self): + text = "some multiline\ntext\nwhich\nincludes foo\nand a\ntail" + assert "foo" not in text + + def test_not_in_text_single(self): + text = "single foo line" + assert "foo" not in text + + def test_not_in_text_single_long(self): + text = "head " * 50 + "foo " + "tail " * 20 + assert "foo" not in text + + def test_not_in_text_single_long_term(self): + text = "head " * 50 + "f" * 70 + "tail " * 20 + assert "f" * 70 not in text + + def test_eq_dataclass(self): + from dataclasses import dataclass + + @dataclass + class Foo: + a: int + b: str + + left = Foo(1, "b") + right = Foo(1, "c") + assert left == right + + def test_eq_attrs(self): + import attr + + @attr.s + class Foo: + a = attr.ib() + b = attr.ib() + + left = Foo(1, "b") + right = Foo(1, "c") + assert left == right + + +def test_attribute(): + class Foo: + b = 1 + + i = Foo() + assert i.b == 2 + + +def test_attribute_instance(): + class Foo: + b = 1 + + assert Foo().b == 2 + + +def test_attribute_failure(): + class Foo: + def _get_b(self): + raise Exception("Failed to get attrib") + + b = property(_get_b) + + i = Foo() + assert i.b == 2 + + +def test_attribute_multiple(): + class Foo: + b = 1 + + class Bar: + b = 2 + + assert Foo().b == Bar().b + + +def globf(x): + return x + 1 + + +class TestRaises: + def test_raises(self): + s = "qwe" + raises(TypeError, int, s) + + def test_raises_doesnt(self): + raises(OSError, int, "3") + + def test_raise(self): + raise ValueError("demo error") + + def test_tupleerror(self): + a, b = [1] # NOQA + + def test_reinterpret_fails_with_print_for_the_fun_of_it(self): + items = [1, 2, 3] + print(f"items is {items!r}") + a, b = items.pop() + + def test_some_error(self): + if namenotexi: # NOQA + pass + + def func1(self): + assert 41 == 42 + + +# thanks to Matthew Scott for this test +def test_dynamic_compile_shows_nicely(): + import importlib.util + import sys + + src = "def foo():\n assert 1 == 0\n" + name = "abc-123" + spec = importlib.util.spec_from_loader(name, loader=None) + module = importlib.util.module_from_spec(spec) + code = compile(src, name, "exec") + exec(code, module.__dict__) + sys.modules[name] = module + module.foo() + + +class TestMoreErrors: + def test_complex_error(self): + def f(): + return 44 + + def g(): + return 43 + + somefunc(f(), g()) + + def test_z1_unpack_error(self): + items = [] + a, b = items + + def test_z2_type_error(self): + items = 3 + a, b = items + + def test_startswith(self): + s = "123" + g = "456" + assert s.startswith(g) + + def test_startswith_nested(self): + def f(): + return "123" + + def g(): + return "456" + + assert f().startswith(g()) + + def test_global_func(self): + assert isinstance(globf(42), float) + + def test_instance(self): + self.x = 6 * 7 + assert self.x != 42 + + def test_compare(self): + assert globf(10) < 5 + + def test_try_finally(self): + x = 1 + try: + assert x == 0 + finally: + x = 0 + + +class TestCustomAssertMsg: + def test_single_line(self): + class A: + a = 1 + + b = 2 + assert A.a == b, "A.a appears not to be b" + + def test_multiline(self): + class A: + a = 1 + + b = 2 + assert ( + A.a == b + ), "A.a appears not to be b\nor does not appear to be b\none of those" + + def test_custom_repr(self): + class JSON: + a = 1 + + def __repr__(self): + return "This is JSON\n{\n 'foo': 'bar'\n}" + + a = JSON() + b = 2 + assert a.a == b, a diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/global_testmodule_config/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/global_testmodule_config/conftest.py new file mode 100644 index 0000000000..7cdf18cdbc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/global_testmodule_config/conftest.py @@ -0,0 +1,14 @@ +import os.path + +import pytest + +mydir = os.path.dirname(__file__) + + +def pytest_runtest_setup(item): + if isinstance(item, pytest.Function): + if not item.fspath.relto(mydir): + return + mod = item.getparent(pytest.Module).obj + if hasattr(mod, "hello"): + print(f"mod.hello {mod.hello!r}") diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/global_testmodule_config/test_hello_world.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/global_testmodule_config/test_hello_world.py new file mode 100644 index 0000000000..a31a601a1c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/global_testmodule_config/test_hello_world.py @@ -0,0 +1,5 @@ +hello = "world" + + +def test_func(): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/test_failures.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/test_failures.py new file mode 100644 index 0000000000..350518b43c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/test_failures.py @@ -0,0 +1,13 @@ +import os.path +import shutil + +failure_demo = os.path.join(os.path.dirname(__file__), "failure_demo.py") +pytest_plugins = ("pytester",) + + +def test_failure_demo_fails_properly(pytester): + target = pytester.path.joinpath(os.path.basename(failure_demo)) + shutil.copy(failure_demo, target) + result = pytester.runpytest(target, syspathinsert=True) + result.stdout.fnmatch_lines(["*44 failed*"]) + assert result.ret != 0 diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/test_setup_flow_example.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/test_setup_flow_example.py new file mode 100644 index 0000000000..0e7eded06b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/assertion/test_setup_flow_example.py @@ -0,0 +1,44 @@ +def setup_module(module): + module.TestStateFullThing.classcount = 0 + + +class TestStateFullThing: + def setup_class(cls): + cls.classcount += 1 + + def teardown_class(cls): + cls.classcount -= 1 + + def setup_method(self, method): + self.id = eval(method.__name__[5:]) + + def test_42(self): + assert self.classcount == 1 + assert self.id == 42 + + def test_23(self): + assert self.classcount == 1 + assert self.id == 23 + + +def teardown_module(module): + assert module.TestStateFullThing.classcount == 0 + + +""" For this example the control flow happens as follows:: + import test_setup_flow_example + setup_module(test_setup_flow_example) + setup_class(TestStateFullThing) + instance = TestStateFullThing() + setup_method(instance, instance.test_42) + instance.test_42() + setup_method(instance, instance.test_23) + instance.test_23() + teardown_class(TestStateFullThing) + teardown_module(test_setup_flow_example) + +Note that ``setup_class(TestStateFullThing)`` is called and not +``TestStateFullThing.setup_class()`` which would require you +to insert ``setup_class = classmethod(setup_class)`` to make +your setup function callable. +""" diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/attic.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/attic.rst new file mode 100644 index 0000000000..2ea8700620 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/attic.rst @@ -0,0 +1,83 @@ + +.. _`accept example`: + +example: specifying and selecting acceptance tests +-------------------------------------------------------------- + +.. sourcecode:: python + + # ./conftest.py + def pytest_option(parser): + group = parser.getgroup("myproject") + group.addoption( + "-A", dest="acceptance", action="store_true", help="run (slow) acceptance tests" + ) + + + def pytest_funcarg__accept(request): + return AcceptFixture(request) + + + class AcceptFixture: + def __init__(self, request): + if not request.config.getoption("acceptance"): + pytest.skip("specify -A to run acceptance tests") + self.tmpdir = request.config.mktemp(request.function.__name__, numbered=True) + + def run(self, *cmd): + """ called by test code to execute an acceptance test. """ + self.tmpdir.chdir() + return subprocess.check_output(cmd).decode() + + +and the actual test function example: + +.. sourcecode:: python + + def test_some_acceptance_aspect(accept): + accept.tmpdir.mkdir("somesub") + result = accept.run("ls", "-la") + assert "somesub" in result + +If you run this test without specifying a command line option +the test will get skipped with an appropriate message. Otherwise +you can start to add convenience and test support methods +to your AcceptFixture and drive running of tools or +applications and provide ways to do assertions about +the output. + +.. _`decorate a funcarg`: + +example: decorating a funcarg in a test module +-------------------------------------------------------------- + +For larger scale setups it's sometimes useful to decorate +a funcarg just for a particular test module. We can +extend the `accept example`_ by putting this in our test module: + +.. sourcecode:: python + + def pytest_funcarg__accept(request): + # call the next factory (living in our conftest.py) + arg = request.getfuncargvalue("accept") + # create a special layout in our tempdir + arg.tmpdir.mkdir("special") + return arg + + + class TestSpecialAcceptance: + def test_sometest(self, accept): + assert accept.tmpdir.join("special").check() + +Our module level factory will be invoked first and it can +ask its request object to call the next factory and then +decorate its result. This mechanism allows us to stay +ignorant of how/where the function argument is provided - +in our example from a `conftest plugin`_. + +sidenote: the temporary directory used here are instances of +the `py.path.local`_ class which provides many of the os.path +methods in a convenient way. + +.. _`py.path.local`: ../path.html#local +.. _`conftest plugin`: customize.html#conftestplugin diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/conftest.py new file mode 100644 index 0000000000..f905738c4f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/conftest.py @@ -0,0 +1 @@ +collect_ignore = ["nonpython"] diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/fixture_availability.svg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/fixture_availability.svg new file mode 100644 index 0000000000..066caac344 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/fixture_availability.svg @@ -0,0 +1,132 @@ + + + + + + + + + + + tests + + + + + + + + + + subpackage + + + + + + + + + + test_subpackage.py + + + + + + + + + + + innermost + + test_order + + mid + + + + + + + 1 + + + + + + + 2 + + + + + + + 3 + + + + + + + + + test_top.py + + + + + + + + + innermost + + test_order + + + + + + + 1 + + + 2 + + + top + + order + diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/fixture_availability_plugins.svg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/fixture_availability_plugins.svg new file mode 100644 index 0000000000..36e3005507 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/fixture_availability_plugins.svg @@ -0,0 +1,142 @@ + + + + + + + + + + + plugin_a + + + + + + + + 4 + + + + + + + + + plugin_b + + + + + + + + 4 + + + + + + + + + tests + + + + + + + + 3 + + + + + + + + + subpackage + + + + + + + + 2 + + + + + + + + + test_subpackage.py + + + + + + + + 1 + + + + + + + + + + + + + inner + + test_order + + mid + + order + + + a_fix + + b_fix + diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse.py new file mode 100644 index 0000000000..ec282ab4b2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse.py @@ -0,0 +1,45 @@ +import pytest + + +@pytest.fixture +def order(): + return [] + + +@pytest.fixture +def a(order): + order.append("a") + + +@pytest.fixture +def b(a, order): + order.append("b") + + +@pytest.fixture(autouse=True) +def c(b, order): + order.append("c") + + +@pytest.fixture +def d(b, order): + order.append("d") + + +@pytest.fixture +def e(d, order): + order.append("e") + + +@pytest.fixture +def f(e, order): + order.append("f") + + +@pytest.fixture +def g(f, c, order): + order.append("g") + + +def test_order_and_g(g, order): + assert order == ["a", "b", "c", "d", "e", "f", "g"] diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse.svg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse.svg new file mode 100644 index 0000000000..36362e4fb0 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse.svg @@ -0,0 +1,64 @@ + + + + + + + + + + + + + autouse + + order + + a + + b + + c + + d + + e + + f + + g + + test_order + diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_flat.svg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_flat.svg new file mode 100644 index 0000000000..03c4598272 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_flat.svg @@ -0,0 +1,56 @@ + + + + + order + + a + + b + + c + + autouse + + d + + e + + f + + g + + test_order + diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.py new file mode 100644 index 0000000000..de0c264279 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.py @@ -0,0 +1,31 @@ +import pytest + + +@pytest.fixture(scope="class") +def order(): + return [] + + +@pytest.fixture(scope="class", autouse=True) +def c1(order): + order.append("c1") + + +@pytest.fixture(scope="class") +def c2(order): + order.append("c2") + + +@pytest.fixture(scope="class") +def c3(order, c1): + order.append("c3") + + +class TestClassWithC1Request: + def test_order(self, order, c1, c3): + assert order == ["c1", "c3"] + + +class TestClassWithoutC1Request: + def test_order(self, order, c2): + assert order == ["c1", "c2"] diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.svg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.svg new file mode 100644 index 0000000000..fe5772993e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_multiple_scopes.svg @@ -0,0 +1,76 @@ + + + + + + + + order + + c1 + + c3 + + test_order + + + + + + TestWithC1Request + + + + + + + order + + c1 + + c2 + + test_order + + + + + + TestWithoutC1Request + + + + + autouse + diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.py new file mode 100644 index 0000000000..ba01ad32f5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.py @@ -0,0 +1,36 @@ +import pytest + + +@pytest.fixture +def order(): + return [] + + +@pytest.fixture +def c1(order): + order.append("c1") + + +@pytest.fixture +def c2(order): + order.append("c2") + + +class TestClassWithAutouse: + @pytest.fixture(autouse=True) + def c3(self, order, c2): + order.append("c3") + + def test_req(self, order, c1): + assert order == ["c2", "c3", "c1"] + + def test_no_req(self, order): + assert order == ["c2", "c3"] + + +class TestClassWithoutAutouse: + def test_req(self, order, c1): + assert order == ["c1"] + + def test_no_req(self, order): + assert order == [] diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.svg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.svg new file mode 100644 index 0000000000..2a9f51673f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_autouse_temp_effects.svg @@ -0,0 +1,100 @@ + + + + + + + + + + + TestWithAutouse + + + + + + + + + order + + c2 + + c3 + + c1 + + test_req + + + + + order + + c2 + + c3 + + test_no_req + + + autouse + + + + + + + + + TestWithoutAutouse + + + + + + order + + c1 + + test_req + + + + + order + + test_no_req + diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies.py new file mode 100644 index 0000000000..b3512c2a64 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies.py @@ -0,0 +1,45 @@ +import pytest + + +@pytest.fixture +def order(): + return [] + + +@pytest.fixture +def a(order): + order.append("a") + + +@pytest.fixture +def b(a, order): + order.append("b") + + +@pytest.fixture +def c(a, b, order): + order.append("c") + + +@pytest.fixture +def d(c, b, order): + order.append("d") + + +@pytest.fixture +def e(d, b, order): + order.append("e") + + +@pytest.fixture +def f(e, order): + order.append("f") + + +@pytest.fixture +def g(f, c, order): + order.append("g") + + +def test_order(g, order): + assert order == ["a", "b", "c", "d", "e", "f", "g"] diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies.svg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies.svg new file mode 100644 index 0000000000..24418e63c9 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies.svg @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + order + + a + + b + + c + + d + + e + + f + + g + + test_order + diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies_flat.svg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies_flat.svg new file mode 100644 index 0000000000..bbe7ad2833 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies_flat.svg @@ -0,0 +1,51 @@ + + + + + order + + a + + b + + c + + d + + e + + f + + g + + test_order + diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies_unclear.svg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies_unclear.svg new file mode 100644 index 0000000000..150724f80a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_dependencies_unclear.svg @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + order + + a + + b + + c + + d + + e + + f + + g + + test_order + diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_scope.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_scope.py new file mode 100644 index 0000000000..5d9487cab3 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_scope.py @@ -0,0 +1,36 @@ +import pytest + + +@pytest.fixture(scope="session") +def order(): + return [] + + +@pytest.fixture +def func(order): + order.append("function") + + +@pytest.fixture(scope="class") +def cls(order): + order.append("class") + + +@pytest.fixture(scope="module") +def mod(order): + order.append("module") + + +@pytest.fixture(scope="package") +def pack(order): + order.append("package") + + +@pytest.fixture(scope="session") +def sess(order): + order.append("session") + + +class TestClass: + def test_order(self, func, cls, mod, pack, sess, order): + assert order == ["session", "package", "module", "class", "function"] diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_scope.svg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_scope.svg new file mode 100644 index 0000000000..f38ee60f1f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_order_scope.svg @@ -0,0 +1,55 @@ + + + + + + + order + + sess + + pack + + mod + + cls + + func + + test_order + + + + + + TestClass + + diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_request_different_scope.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_request_different_scope.py new file mode 100644 index 0000000000..00e2e46d84 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_request_different_scope.py @@ -0,0 +1,29 @@ +import pytest + + +@pytest.fixture +def order(): + return [] + + +@pytest.fixture +def outer(order, inner): + order.append("outer") + + +class TestOne: + @pytest.fixture + def inner(self, order): + order.append("one") + + def test_order(self, order, outer): + assert order == ["one", "outer"] + + +class TestTwo: + @pytest.fixture + def inner(self, order): + order.append("two") + + def test_order(self, order, outer): + assert order == ["two", "outer"] diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_request_different_scope.svg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_request_different_scope.svg new file mode 100644 index 0000000000..0a78a889fd --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/fixtures/test_fixtures_request_different_scope.svg @@ -0,0 +1,115 @@ + + + + + + + + + + test_fixtures_request_different_scope.py + + + + + + + + + + + + inner + + test_order + + + + + + TestOne + + + + + + + + 1 + + + + + + + 2 + + + + + + + + + + + inner + + test_order + + + + + + TestTwo + + + + + + + + 1 + + + 2 + + + outer + + order + diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/index.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/index.rst new file mode 100644 index 0000000000..71e855534f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/index.rst @@ -0,0 +1,34 @@ + +.. _examples: + +Examples and customization tricks +================================= + +Here is a (growing) list of examples. :ref:`Contact ` us if you +need more examples or have questions. Also take a look at the +:ref:`comprehensive documentation ` which contains many example +snippets as well. Also, `pytest on stackoverflow.com +`_ often comes with example +answers. + +For basic examples, see + +- :ref:`get-started` for basic introductory examples +- :ref:`assert` for basic assertion examples +- :ref:`Fixtures ` for basic fixture/setup examples +- :ref:`parametrize` for basic test function parametrization +- :ref:`unittest` for basic unittest integration +- :ref:`noseintegration` for basic nosetests integration + +The following examples aim at various use cases you might encounter. + +.. toctree:: + :maxdepth: 2 + + reportingdemo + simple + parametrize + markers + special + pythoncollection + nonpython diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/markers.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/markers.rst new file mode 100644 index 0000000000..3226c0871e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/markers.rst @@ -0,0 +1,734 @@ + +.. _`mark examples`: + +Working with custom markers +================================================= + +Here are some examples using the :ref:`mark` mechanism. + +.. _`mark run`: + +Marking test functions and selecting them for a run +---------------------------------------------------- + +You can "mark" a test function with custom metadata like this: + +.. code-block:: python + + # content of test_server.py + + import pytest + + + @pytest.mark.webtest + def test_send_http(): + pass # perform some webtest test for your app + + + def test_something_quick(): + pass + + + def test_another(): + pass + + + class TestClass: + def test_method(self): + pass + + + +You can then restrict a test run to only run tests marked with ``webtest``: + +.. code-block:: pytest + + $ pytest -v -m webtest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 4 items / 3 deselected / 1 selected + + test_server.py::test_send_http PASSED [100%] + + ===================== 1 passed, 3 deselected in 0.12s ====================== + +Or the inverse, running all tests except the webtest ones: + +.. code-block:: pytest + + $ pytest -v -m "not webtest" + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 4 items / 1 deselected / 3 selected + + test_server.py::test_something_quick PASSED [ 33%] + test_server.py::test_another PASSED [ 66%] + test_server.py::TestClass::test_method PASSED [100%] + + ===================== 3 passed, 1 deselected in 0.12s ====================== + +Selecting tests based on their node ID +-------------------------------------- + +You can provide one or more :ref:`node IDs ` as positional +arguments to select only specified tests. This makes it easy to select +tests based on their module, class, method, or function name: + +.. code-block:: pytest + + $ pytest -v test_server.py::TestClass::test_method + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 1 item + + test_server.py::TestClass::test_method PASSED [100%] + + ============================ 1 passed in 0.12s ============================= + +You can also select on the class: + +.. code-block:: pytest + + $ pytest -v test_server.py::TestClass + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 1 item + + test_server.py::TestClass::test_method PASSED [100%] + + ============================ 1 passed in 0.12s ============================= + +Or select multiple nodes: + +.. code-block:: pytest + + $ pytest -v test_server.py::TestClass test_server.py::test_send_http + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 2 items + + test_server.py::TestClass::test_method PASSED [ 50%] + test_server.py::test_send_http PASSED [100%] + + ============================ 2 passed in 0.12s ============================= + +.. _node-id: + +.. note:: + + Node IDs are of the form ``module.py::class::method`` or + ``module.py::function``. Node IDs control which tests are + collected, so ``module.py::class`` will select all test methods + on the class. Nodes are also created for each parameter of a + parametrized fixture or test, so selecting a parametrized test + must include the parameter value, e.g. + ``module.py::function[param]``. + + Node IDs for failing tests are displayed in the test summary info + when running pytest with the ``-rf`` option. You can also + construct Node IDs from the output of ``pytest --collectonly``. + +Using ``-k expr`` to select tests based on their name +------------------------------------------------------- + +.. versionadded:: 2.0/2.3.4 + +You can use the ``-k`` command line option to specify an expression +which implements a substring match on the test names instead of the +exact match on markers that ``-m`` provides. This makes it easy to +select tests based on their names: + +.. versionchanged:: 5.4 + +The expression matching is now case-insensitive. + +.. code-block:: pytest + + $ pytest -v -k http # running with the above defined example module + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 4 items / 3 deselected / 1 selected + + test_server.py::test_send_http PASSED [100%] + + ===================== 1 passed, 3 deselected in 0.12s ====================== + +And you can also run all tests except the ones that match the keyword: + +.. code-block:: pytest + + $ pytest -k "not send_http" -v + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 4 items / 1 deselected / 3 selected + + test_server.py::test_something_quick PASSED [ 33%] + test_server.py::test_another PASSED [ 66%] + test_server.py::TestClass::test_method PASSED [100%] + + ===================== 3 passed, 1 deselected in 0.12s ====================== + +Or to select "http" and "quick" tests: + +.. code-block:: pytest + + $ pytest -k "http or quick" -v + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 4 items / 2 deselected / 2 selected + + test_server.py::test_send_http PASSED [ 50%] + test_server.py::test_something_quick PASSED [100%] + + ===================== 2 passed, 2 deselected in 0.12s ====================== + +You can use ``and``, ``or``, ``not`` and parentheses. + + +In addition to the test's name, ``-k`` also matches the names of the test's parents (usually, the name of the file and class it's in), +attributes set on the test function, markers applied to it or its parents and any :attr:`extra keywords <_pytest.nodes.Node.extra_keyword_matches>` +explicitly added to it or its parents. + + +Registering markers +------------------------------------- + + + +.. ini-syntax for custom markers: + +Registering markers for your test suite is simple: + +.. code-block:: ini + + # content of pytest.ini + [pytest] + markers = + webtest: mark a test as a webtest. + slow: mark test as slow. + +Multiple custom markers can be registered, by defining each one in its own line, as shown in above example. + +You can ask which markers exist for your test suite - the list includes our just defined ``webtest`` and ``slow`` markers: + +.. code-block:: pytest + + $ pytest --markers + @pytest.mark.webtest: mark a test as a webtest. + + @pytest.mark.slow: mark test as slow. + + @pytest.mark.filterwarnings(warning): add a warning filter to the given test. see https://docs.pytest.org/en/stable/how-to/capture-warnings.html#pytest-mark-filterwarnings + + @pytest.mark.skip(reason=None): skip the given test function with an optional reason. Example: skip(reason="no way of currently testing this") skips the test. + + @pytest.mark.skipif(condition, ..., *, reason=...): skip the given test function if any of the conditions evaluate to True. Example: skipif(sys.platform == 'win32') skips the test if we are on the win32 platform. See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-skipif + + @pytest.mark.xfail(condition, ..., *, reason=..., run=True, raises=None, strict=xfail_strict): mark the test function as an expected failure if any of the conditions evaluate to True. Optionally specify a reason for better reporting and run=False if you don't even want to execute the test function. If only specific exception(s) are expected, you can list them in raises, and if the test fails in other ways, it will be reported as a true failure. See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-xfail + + @pytest.mark.parametrize(argnames, argvalues): call a test function multiple times passing in different arguments in turn. argvalues generally needs to be a list of values if argnames specifies only one name or a list of tuples of values if argnames specifies multiple names. Example: @parametrize('arg1', [1,2]) would lead to two calls of the decorated test function, one with arg1=1 and another with arg1=2.see https://docs.pytest.org/en/stable/how-to/parametrize.html for more info and examples. + + @pytest.mark.usefixtures(fixturename1, fixturename2, ...): mark tests as needing all of the specified fixtures. see https://docs.pytest.org/en/stable/explanation/fixtures.html#usefixtures + + @pytest.mark.tryfirst: mark a hook implementation function such that the plugin machinery will try to call it first/as early as possible. + + @pytest.mark.trylast: mark a hook implementation function such that the plugin machinery will try to call it last/as late as possible. + + +For an example on how to add and work with markers from a plugin, see +:ref:`adding a custom marker from a plugin`. + +.. note:: + + It is recommended to explicitly register markers so that: + + * There is one place in your test suite defining your markers + + * Asking for existing markers via ``pytest --markers`` gives good output + + * Typos in function markers are treated as an error if you use + the ``--strict-markers`` option. + +.. _`scoped-marking`: + +Marking whole classes or modules +---------------------------------------------------- + +You may use ``pytest.mark`` decorators with classes to apply markers to all of +its test methods: + +.. code-block:: python + + # content of test_mark_classlevel.py + import pytest + + + @pytest.mark.webtest + class TestClass: + def test_startup(self): + pass + + def test_startup_and_more(self): + pass + +This is equivalent to directly applying the decorator to the +two test functions. + +To apply marks at the module level, use the :globalvar:`pytestmark` global variable:: + + import pytest + pytestmark = pytest.mark.webtest + +or multiple markers:: + + pytestmark = [pytest.mark.webtest, pytest.mark.slowtest] + + +Due to legacy reasons, before class decorators were introduced, it is possible to set the +:globalvar:`pytestmark` attribute on a test class like this: + +.. code-block:: python + + import pytest + + + class TestClass: + pytestmark = pytest.mark.webtest + +.. _`marking individual tests when using parametrize`: + +Marking individual tests when using parametrize +----------------------------------------------- + +When using parametrize, applying a mark will make it apply +to each individual test. However it is also possible to +apply a marker to an individual test instance: + +.. code-block:: python + + import pytest + + + @pytest.mark.foo + @pytest.mark.parametrize( + ("n", "expected"), [(1, 2), pytest.param(1, 3, marks=pytest.mark.bar), (2, 3)] + ) + def test_increment(n, expected): + assert n + 1 == expected + +In this example the mark "foo" will apply to each of the three +tests, whereas the "bar" mark is only applied to the second test. +Skip and xfail marks can also be applied in this way, see :ref:`skip/xfail with parametrize`. + +.. _`adding a custom marker from a plugin`: + +Custom marker and command line option to control test runs +---------------------------------------------------------- + +.. regendoc:wipe + +Plugins can provide custom markers and implement specific behaviour +based on it. This is a self-contained example which adds a command +line option and a parametrized test function marker to run tests +specifies via named environments: + +.. code-block:: python + + # content of conftest.py + + import pytest + + + def pytest_addoption(parser): + parser.addoption( + "-E", + action="store", + metavar="NAME", + help="only run tests matching the environment NAME.", + ) + + + def pytest_configure(config): + # register an additional marker + config.addinivalue_line( + "markers", "env(name): mark test to run only on named environment" + ) + + + def pytest_runtest_setup(item): + envnames = [mark.args[0] for mark in item.iter_markers(name="env")] + if envnames: + if item.config.getoption("-E") not in envnames: + pytest.skip("test requires env in {!r}".format(envnames)) + +A test file using this local plugin: + +.. code-block:: python + + # content of test_someenv.py + + import pytest + + + @pytest.mark.env("stage1") + def test_basic_db_operation(): + pass + +and an example invocations specifying a different environment than what +the test needs: + +.. code-block:: pytest + + $ pytest -E stage2 + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 1 item + + test_someenv.py s [100%] + + ============================ 1 skipped in 0.12s ============================ + +and here is one that specifies exactly the environment needed: + +.. code-block:: pytest + + $ pytest -E stage1 + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 1 item + + test_someenv.py . [100%] + + ============================ 1 passed in 0.12s ============================= + +The ``--markers`` option always gives you a list of available markers: + +.. code-block:: pytest + + $ pytest --markers + @pytest.mark.env(name): mark test to run only on named environment + + @pytest.mark.filterwarnings(warning): add a warning filter to the given test. see https://docs.pytest.org/en/stable/how-to/capture-warnings.html#pytest-mark-filterwarnings + + @pytest.mark.skip(reason=None): skip the given test function with an optional reason. Example: skip(reason="no way of currently testing this") skips the test. + + @pytest.mark.skipif(condition, ..., *, reason=...): skip the given test function if any of the conditions evaluate to True. Example: skipif(sys.platform == 'win32') skips the test if we are on the win32 platform. See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-skipif + + @pytest.mark.xfail(condition, ..., *, reason=..., run=True, raises=None, strict=xfail_strict): mark the test function as an expected failure if any of the conditions evaluate to True. Optionally specify a reason for better reporting and run=False if you don't even want to execute the test function. If only specific exception(s) are expected, you can list them in raises, and if the test fails in other ways, it will be reported as a true failure. See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-xfail + + @pytest.mark.parametrize(argnames, argvalues): call a test function multiple times passing in different arguments in turn. argvalues generally needs to be a list of values if argnames specifies only one name or a list of tuples of values if argnames specifies multiple names. Example: @parametrize('arg1', [1,2]) would lead to two calls of the decorated test function, one with arg1=1 and another with arg1=2.see https://docs.pytest.org/en/stable/how-to/parametrize.html for more info and examples. + + @pytest.mark.usefixtures(fixturename1, fixturename2, ...): mark tests as needing all of the specified fixtures. see https://docs.pytest.org/en/stable/explanation/fixtures.html#usefixtures + + @pytest.mark.tryfirst: mark a hook implementation function such that the plugin machinery will try to call it first/as early as possible. + + @pytest.mark.trylast: mark a hook implementation function such that the plugin machinery will try to call it last/as late as possible. + + +.. _`passing callables to custom markers`: + +Passing a callable to custom markers +-------------------------------------------- + +.. regendoc:wipe + +Below is the config file that will be used in the next examples: + +.. code-block:: python + + # content of conftest.py + import sys + + + def pytest_runtest_setup(item): + for marker in item.iter_markers(name="my_marker"): + print(marker) + sys.stdout.flush() + +A custom marker can have its argument set, i.e. ``args`` and ``kwargs`` properties, defined by either invoking it as a callable or using ``pytest.mark.MARKER_NAME.with_args``. These two methods achieve the same effect most of the time. + +However, if there is a callable as the single positional argument with no keyword arguments, using the ``pytest.mark.MARKER_NAME(c)`` will not pass ``c`` as a positional argument but decorate ``c`` with the custom marker (see :ref:`MarkDecorator `). Fortunately, ``pytest.mark.MARKER_NAME.with_args`` comes to the rescue: + +.. code-block:: python + + # content of test_custom_marker.py + import pytest + + + def hello_world(*args, **kwargs): + return "Hello World" + + + @pytest.mark.my_marker.with_args(hello_world) + def test_with_args(): + pass + +The output is as follows: + +.. code-block:: pytest + + $ pytest -q -s + Mark(name='my_marker', args=(,), kwargs={}) + . + 1 passed in 0.12s + +We can see that the custom marker has its argument set extended with the function ``hello_world``. This is the key difference between creating a custom marker as a callable, which invokes ``__call__`` behind the scenes, and using ``with_args``. + + +Reading markers which were set from multiple places +---------------------------------------------------- + +.. versionadded: 2.2.2 + +.. regendoc:wipe + +If you are heavily using markers in your test suite you may encounter the case where a marker is applied several times to a test function. From plugin +code you can read over all such settings. Example: + +.. code-block:: python + + # content of test_mark_three_times.py + import pytest + + pytestmark = pytest.mark.glob("module", x=1) + + + @pytest.mark.glob("class", x=2) + class TestClass: + @pytest.mark.glob("function", x=3) + def test_something(self): + pass + +Here we have the marker "glob" applied three times to the same +test function. From a conftest file we can read it like this: + +.. code-block:: python + + # content of conftest.py + import sys + + + def pytest_runtest_setup(item): + for mark in item.iter_markers(name="glob"): + print("glob args={} kwargs={}".format(mark.args, mark.kwargs)) + sys.stdout.flush() + +Let's run this without capturing output and see what we get: + +.. code-block:: pytest + + $ pytest -q -s + glob args=('function',) kwargs={'x': 3} + glob args=('class',) kwargs={'x': 2} + glob args=('module',) kwargs={'x': 1} + . + 1 passed in 0.12s + +Marking platform specific tests with pytest +-------------------------------------------------------------- + +.. regendoc:wipe + +Consider you have a test suite which marks tests for particular platforms, +namely ``pytest.mark.darwin``, ``pytest.mark.win32`` etc. and you +also have tests that run on all platforms and have no specific +marker. If you now want to have a way to only run the tests +for your particular platform, you could use the following plugin: + +.. code-block:: python + + # content of conftest.py + # + import sys + import pytest + + ALL = set("darwin linux win32".split()) + + + def pytest_runtest_setup(item): + supported_platforms = ALL.intersection(mark.name for mark in item.iter_markers()) + plat = sys.platform + if supported_platforms and plat not in supported_platforms: + pytest.skip("cannot run on platform {}".format(plat)) + +then tests will be skipped if they were specified for a different platform. +Let's do a little test file to show how this looks like: + +.. code-block:: python + + # content of test_plat.py + + import pytest + + + @pytest.mark.darwin + def test_if_apple_is_evil(): + pass + + + @pytest.mark.linux + def test_if_linux_works(): + pass + + + @pytest.mark.win32 + def test_if_win32_crashes(): + pass + + + def test_runs_everywhere(): + pass + +then you will see two tests skipped and two executed tests as expected: + +.. code-block:: pytest + + $ pytest -rs # this option reports skip reasons + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 4 items + + test_plat.py s.s. [100%] + + ========================= short test summary info ========================== + SKIPPED [2] conftest.py:12: cannot run on platform linux + ======================= 2 passed, 2 skipped in 0.12s ======================= + +Note that if you specify a platform via the marker-command line option like this: + +.. code-block:: pytest + + $ pytest -m linux + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 4 items / 3 deselected / 1 selected + + test_plat.py . [100%] + + ===================== 1 passed, 3 deselected in 0.12s ====================== + +then the unmarked-tests will not be run. It is thus a way to restrict the run to the specific tests. + +Automatically adding markers based on test names +-------------------------------------------------------- + +.. regendoc:wipe + +If you have a test suite where test function names indicate a certain +type of test, you can implement a hook that automatically defines +markers so that you can use the ``-m`` option with it. Let's look +at this test module: + +.. code-block:: python + + # content of test_module.py + + + def test_interface_simple(): + assert 0 + + + def test_interface_complex(): + assert 0 + + + def test_event_simple(): + assert 0 + + + def test_something_else(): + assert 0 + +We want to dynamically define two markers and can do it in a +``conftest.py`` plugin: + +.. code-block:: python + + # content of conftest.py + + import pytest + + + def pytest_collection_modifyitems(items): + for item in items: + if "interface" in item.nodeid: + item.add_marker(pytest.mark.interface) + elif "event" in item.nodeid: + item.add_marker(pytest.mark.event) + +We can now use the ``-m option`` to select one set: + +.. code-block:: pytest + + $ pytest -m interface --tb=short + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 4 items / 2 deselected / 2 selected + + test_module.py FF [100%] + + ================================= FAILURES ================================= + __________________________ test_interface_simple ___________________________ + test_module.py:4: in test_interface_simple + assert 0 + E assert 0 + __________________________ test_interface_complex __________________________ + test_module.py:8: in test_interface_complex + assert 0 + E assert 0 + ========================= short test summary info ========================== + FAILED test_module.py::test_interface_simple - assert 0 + FAILED test_module.py::test_interface_complex - assert 0 + ===================== 2 failed, 2 deselected in 0.12s ====================== + +or to select both "event" and "interface" tests: + +.. code-block:: pytest + + $ pytest -m "interface or event" --tb=short + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 4 items / 1 deselected / 3 selected + + test_module.py FFF [100%] + + ================================= FAILURES ================================= + __________________________ test_interface_simple ___________________________ + test_module.py:4: in test_interface_simple + assert 0 + E assert 0 + __________________________ test_interface_complex __________________________ + test_module.py:8: in test_interface_complex + assert 0 + E assert 0 + ____________________________ test_event_simple _____________________________ + test_module.py:12: in test_event_simple + assert 0 + E assert 0 + ========================= short test summary info ========================== + FAILED test_module.py::test_interface_simple - assert 0 + FAILED test_module.py::test_interface_complex - assert 0 + FAILED test_module.py::test_event_simple - assert 0 + ===================== 3 failed, 1 deselected in 0.12s ====================== diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/multipython.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/multipython.py new file mode 100644 index 0000000000..9005d31add --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/multipython.py @@ -0,0 +1,72 @@ +""" +module containing a parametrized tests testing cross-python +serialization via the pickle module. +""" +import shutil +import subprocess +import textwrap + +import pytest + +pythonlist = ["python3.5", "python3.6", "python3.7"] + + +@pytest.fixture(params=pythonlist) +def python1(request, tmp_path): + picklefile = tmp_path / "data.pickle" + return Python(request.param, picklefile) + + +@pytest.fixture(params=pythonlist) +def python2(request, python1): + return Python(request.param, python1.picklefile) + + +class Python: + def __init__(self, version, picklefile): + self.pythonpath = shutil.which(version) + if not self.pythonpath: + pytest.skip(f"{version!r} not found") + self.picklefile = picklefile + + def dumps(self, obj): + dumpfile = self.picklefile.with_name("dump.py") + dumpfile.write_text( + textwrap.dedent( + r""" + import pickle + f = open({!r}, 'wb') + s = pickle.dump({!r}, f, protocol=2) + f.close() + """.format( + str(self.picklefile), obj + ) + ) + ) + subprocess.check_call((self.pythonpath, str(dumpfile))) + + def load_and_is_true(self, expression): + loadfile = self.picklefile.with_name("load.py") + loadfile.write_text( + textwrap.dedent( + r""" + import pickle + f = open({!r}, 'rb') + obj = pickle.load(f) + f.close() + res = eval({!r}) + if not res: + raise SystemExit(1) + """.format( + str(self.picklefile), expression + ) + ) + ) + print(loadfile) + subprocess.check_call((self.pythonpath, str(loadfile))) + + +@pytest.mark.parametrize("obj", [42, {}, {1: 3}]) +def test_basic_objects(python1, python2, obj): + python1.dumps(obj) + python2.load_and_is_true(f"obj == {obj}") diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython.rst new file mode 100644 index 0000000000..f79f15b4f7 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython.rst @@ -0,0 +1,102 @@ + +.. _`non-python tests`: + +Working with non-python tests +==================================================== + +.. _`yaml plugin`: + +A basic example for specifying tests in Yaml files +-------------------------------------------------------------- + +.. _`pytest-yamlwsgi`: http://bitbucket.org/aafshar/pytest-yamlwsgi/src/tip/pytest_yamlwsgi.py + +Here is an example ``conftest.py`` (extracted from Ali Afshar's special purpose `pytest-yamlwsgi`_ plugin). This ``conftest.py`` will collect ``test*.yaml`` files and will execute the yaml-formatted content as custom tests: + +.. include:: nonpython/conftest.py + :literal: + +You can create a simple example file: + +.. include:: nonpython/test_simple.yaml + :literal: + +and if you installed :pypi:`PyYAML` or a compatible YAML-parser you can +now execute the test specification: + +.. code-block:: pytest + + nonpython $ pytest test_simple.yaml + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project/nonpython + collected 2 items + + test_simple.yaml F. [100%] + + ================================= FAILURES ================================= + ______________________________ usecase: hello ______________________________ + usecase execution failed + spec failed: 'some': 'other' + no further details known at this point. + ========================= short test summary info ========================== + FAILED test_simple.yaml::hello + ======================= 1 failed, 1 passed in 0.12s ======================== + +.. regendoc:wipe + +You get one dot for the passing ``sub1: sub1`` check and one failure. +Obviously in the above ``conftest.py`` you'll want to implement a more +interesting interpretation of the yaml-values. You can easily write +your own domain specific testing language this way. + +.. note:: + + ``repr_failure(excinfo)`` is called for representing test failures. + If you create custom collection nodes you can return an error + representation string of your choice. It + will be reported as a (red) string. + +``reportinfo()`` is used for representing the test location and is also +consulted when reporting in ``verbose`` mode: + +.. code-block:: pytest + + nonpython $ pytest -v + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project/nonpython + collecting ... collected 2 items + + test_simple.yaml::hello FAILED [ 50%] + test_simple.yaml::ok PASSED [100%] + + ================================= FAILURES ================================= + ______________________________ usecase: hello ______________________________ + usecase execution failed + spec failed: 'some': 'other' + no further details known at this point. + ========================= short test summary info ========================== + FAILED test_simple.yaml::hello + ======================= 1 failed, 1 passed in 0.12s ======================== + +.. regendoc:wipe + +While developing your custom test collection and execution it's also +interesting to just look at the collection tree: + +.. code-block:: pytest + + nonpython $ pytest --collect-only + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project/nonpython + collected 2 items + + + + + + + ======================== 2 tests collected in 0.12s ======================== diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython/conftest.py new file mode 100644 index 0000000000..bc39a1f6b2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython/conftest.py @@ -0,0 +1,47 @@ +# content of conftest.py +import pytest + + +def pytest_collect_file(parent, file_path): + if file_path.suffix == ".yaml" and file_path.name.startswith("test"): + return YamlFile.from_parent(parent, path=file_path) + + +class YamlFile(pytest.File): + def collect(self): + # We need a yaml parser, e.g. PyYAML. + import yaml + + raw = yaml.safe_load(self.path.open()) + for name, spec in sorted(raw.items()): + yield YamlItem.from_parent(self, name=name, spec=spec) + + +class YamlItem(pytest.Item): + def __init__(self, *, spec, **kwargs): + super().__init__(**kwargs) + self.spec = spec + + def runtest(self): + for name, value in sorted(self.spec.items()): + # Some custom test execution (dumb example follows). + if name != value: + raise YamlException(self, name, value) + + def repr_failure(self, excinfo): + """Called when self.runtest() raises an exception.""" + if isinstance(excinfo.value, YamlException): + return "\n".join( + [ + "usecase execution failed", + " spec failed: {1!r}: {2!r}".format(*excinfo.value.args), + " no further details known at this point.", + ] + ) + + def reportinfo(self): + return self.path, 0, f"usecase: {self.name}" + + +class YamlException(Exception): + """Custom exception for error reporting.""" diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython/test_simple.yaml b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython/test_simple.yaml new file mode 100644 index 0000000000..8e3e7a4bbc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/nonpython/test_simple.yaml @@ -0,0 +1,7 @@ +# test_simple.yaml +ok: + sub1: sub1 + +hello: + world: world + some: other diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/parametrize.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/parametrize.rst new file mode 100644 index 0000000000..66d72f3cc0 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/parametrize.rst @@ -0,0 +1,708 @@ + +.. _paramexamples: + +Parametrizing tests +================================================= + +.. currentmodule:: _pytest.python + +``pytest`` allows to easily parametrize test functions. +For basic docs, see :ref:`parametrize-basics`. + +In the following we provide some examples using +the builtin mechanisms. + +Generating parameters combinations, depending on command line +---------------------------------------------------------------------------- + +.. regendoc:wipe + +Let's say we want to execute a test with different computation +parameters and the parameter range shall be determined by a command +line argument. Let's first write a simple (do-nothing) computation test: + +.. code-block:: python + + # content of test_compute.py + + + def test_compute(param1): + assert param1 < 4 + +Now we add a test configuration like this: + +.. code-block:: python + + # content of conftest.py + + + def pytest_addoption(parser): + parser.addoption("--all", action="store_true", help="run all combinations") + + + def pytest_generate_tests(metafunc): + if "param1" in metafunc.fixturenames: + if metafunc.config.getoption("all"): + end = 5 + else: + end = 2 + metafunc.parametrize("param1", range(end)) + +This means that we only run 2 tests if we do not pass ``--all``: + +.. code-block:: pytest + + $ pytest -q test_compute.py + .. [100%] + 2 passed in 0.12s + +We run only two computations, so we see two dots. +let's run the full monty: + +.. code-block:: pytest + + $ pytest -q --all + ....F [100%] + ================================= FAILURES ================================= + _____________________________ test_compute[4] ______________________________ + + param1 = 4 + + def test_compute(param1): + > assert param1 < 4 + E assert 4 < 4 + + test_compute.py:4: AssertionError + ========================= short test summary info ========================== + FAILED test_compute.py::test_compute[4] - assert 4 < 4 + 1 failed, 4 passed in 0.12s + +As expected when running the full range of ``param1`` values +we'll get an error on the last one. + + +Different options for test IDs +------------------------------------ + +pytest will build a string that is the test ID for each set of values in a +parametrized test. These IDs can be used with ``-k`` to select specific cases +to run, and they will also identify the specific case when one is failing. +Running pytest with ``--collect-only`` will show the generated IDs. + +Numbers, strings, booleans and None will have their usual string representation +used in the test ID. For other objects, pytest will make a string based on +the argument name: + +.. code-block:: python + + # content of test_time.py + + from datetime import datetime, timedelta + + import pytest + + testdata = [ + (datetime(2001, 12, 12), datetime(2001, 12, 11), timedelta(1)), + (datetime(2001, 12, 11), datetime(2001, 12, 12), timedelta(-1)), + ] + + + @pytest.mark.parametrize("a,b,expected", testdata) + def test_timedistance_v0(a, b, expected): + diff = a - b + assert diff == expected + + + @pytest.mark.parametrize("a,b,expected", testdata, ids=["forward", "backward"]) + def test_timedistance_v1(a, b, expected): + diff = a - b + assert diff == expected + + + def idfn(val): + if isinstance(val, (datetime,)): + # note this wouldn't show any hours/minutes/seconds + return val.strftime("%Y%m%d") + + + @pytest.mark.parametrize("a,b,expected", testdata, ids=idfn) + def test_timedistance_v2(a, b, expected): + diff = a - b + assert diff == expected + + + @pytest.mark.parametrize( + "a,b,expected", + [ + pytest.param( + datetime(2001, 12, 12), datetime(2001, 12, 11), timedelta(1), id="forward" + ), + pytest.param( + datetime(2001, 12, 11), datetime(2001, 12, 12), timedelta(-1), id="backward" + ), + ], + ) + def test_timedistance_v3(a, b, expected): + diff = a - b + assert diff == expected + +In ``test_timedistance_v0``, we let pytest generate the test IDs. + +In ``test_timedistance_v1``, we specified ``ids`` as a list of strings which were +used as the test IDs. These are succinct, but can be a pain to maintain. + +In ``test_timedistance_v2``, we specified ``ids`` as a function that can generate a +string representation to make part of the test ID. So our ``datetime`` values use the +label generated by ``idfn``, but because we didn't generate a label for ``timedelta`` +objects, they are still using the default pytest representation: + +.. code-block:: pytest + + $ pytest test_time.py --collect-only + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 8 items + + + + + + + + + + + + ======================== 8 tests collected in 0.12s ======================== + +In ``test_timedistance_v3``, we used ``pytest.param`` to specify the test IDs +together with the actual data, instead of listing them separately. + +A quick port of "testscenarios" +------------------------------------ + +Here is a quick port to run tests configured with :pypi:`testscenarios`, +an add-on from Robert Collins for the standard unittest framework. We +only have to work a bit to construct the correct arguments for pytest's +:py:func:`Metafunc.parametrize`: + +.. code-block:: python + + # content of test_scenarios.py + + + def pytest_generate_tests(metafunc): + idlist = [] + argvalues = [] + for scenario in metafunc.cls.scenarios: + idlist.append(scenario[0]) + items = scenario[1].items() + argnames = [x[0] for x in items] + argvalues.append([x[1] for x in items]) + metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class") + + + scenario1 = ("basic", {"attribute": "value"}) + scenario2 = ("advanced", {"attribute": "value2"}) + + + class TestSampleWithScenarios: + scenarios = [scenario1, scenario2] + + def test_demo1(self, attribute): + assert isinstance(attribute, str) + + def test_demo2(self, attribute): + assert isinstance(attribute, str) + +this is a fully self-contained example which you can run with: + +.. code-block:: pytest + + $ pytest test_scenarios.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 4 items + + test_scenarios.py .... [100%] + + ============================ 4 passed in 0.12s ============================= + +If you just collect tests you'll also nicely see 'advanced' and 'basic' as variants for the test function: + +.. code-block:: pytest + + $ pytest --collect-only test_scenarios.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 4 items + + + + + + + + + ======================== 4 tests collected in 0.12s ======================== + +Note that we told ``metafunc.parametrize()`` that your scenario values +should be considered class-scoped. With pytest-2.3 this leads to a +resource-based ordering. + +Deferring the setup of parametrized resources +--------------------------------------------------- + +.. regendoc:wipe + +The parametrization of test functions happens at collection +time. It is a good idea to setup expensive resources like DB +connections or subprocess only when the actual test is run. +Here is a simple example how you can achieve that. This test +requires a ``db`` object fixture: + +.. code-block:: python + + # content of test_backends.py + + import pytest + + + def test_db_initialized(db): + # a dummy test + if db.__class__.__name__ == "DB2": + pytest.fail("deliberately failing for demo purposes") + +We can now add a test configuration that generates two invocations of +the ``test_db_initialized`` function and also implements a factory that +creates a database object for the actual test invocations: + +.. code-block:: python + + # content of conftest.py + import pytest + + + def pytest_generate_tests(metafunc): + if "db" in metafunc.fixturenames: + metafunc.parametrize("db", ["d1", "d2"], indirect=True) + + + class DB1: + "one database object" + + + class DB2: + "alternative database object" + + + @pytest.fixture + def db(request): + if request.param == "d1": + return DB1() + elif request.param == "d2": + return DB2() + else: + raise ValueError("invalid internal test config") + +Let's first see how it looks like at collection time: + +.. code-block:: pytest + + $ pytest test_backends.py --collect-only + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 2 items + + + + + + ======================== 2 tests collected in 0.12s ======================== + +And then when we run the test: + +.. code-block:: pytest + + $ pytest -q test_backends.py + .F [100%] + ================================= FAILURES ================================= + _________________________ test_db_initialized[d2] __________________________ + + db = + + def test_db_initialized(db): + # a dummy test + if db.__class__.__name__ == "DB2": + > pytest.fail("deliberately failing for demo purposes") + E Failed: deliberately failing for demo purposes + + test_backends.py:8: Failed + ========================= short test summary info ========================== + FAILED test_backends.py::test_db_initialized[d2] - Failed: deliberately f... + 1 failed, 1 passed in 0.12s + +The first invocation with ``db == "DB1"`` passed while the second with ``db == "DB2"`` failed. Our ``db`` fixture function has instantiated each of the DB values during the setup phase while the ``pytest_generate_tests`` generated two according calls to the ``test_db_initialized`` during the collection phase. + +Indirect parametrization +--------------------------------------------------- + +Using the ``indirect=True`` parameter when parametrizing a test allows to +parametrize a test with a fixture receiving the values before passing them to a +test: + +.. code-block:: python + + import pytest + + + @pytest.fixture + def fixt(request): + return request.param * 3 + + + @pytest.mark.parametrize("fixt", ["a", "b"], indirect=True) + def test_indirect(fixt): + assert len(fixt) == 3 + +This can be used, for example, to do more expensive setup at test run time in +the fixture, rather than having to run those setup steps at collection time. + +.. regendoc:wipe + +Apply indirect on particular arguments +--------------------------------------------------- + +Very often parametrization uses more than one argument name. There is opportunity to apply ``indirect`` +parameter on particular arguments. It can be done by passing list or tuple of +arguments' names to ``indirect``. In the example below there is a function ``test_indirect`` which uses +two fixtures: ``x`` and ``y``. Here we give to indirect the list, which contains the name of the +fixture ``x``. The indirect parameter will be applied to this argument only, and the value ``a`` +will be passed to respective fixture function: + +.. code-block:: python + + # content of test_indirect_list.py + + import pytest + + + @pytest.fixture(scope="function") + def x(request): + return request.param * 3 + + + @pytest.fixture(scope="function") + def y(request): + return request.param * 2 + + + @pytest.mark.parametrize("x, y", [("a", "b")], indirect=["x"]) + def test_indirect(x, y): + assert x == "aaa" + assert y == "b" + +The result of this test will be successful: + +.. code-block:: pytest + + $ pytest -v test_indirect_list.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 1 item + + test_indirect_list.py::test_indirect[a-b] PASSED [100%] + + ============================ 1 passed in 0.12s ============================= + +.. regendoc:wipe + +Parametrizing test methods through per-class configuration +-------------------------------------------------------------- + +.. _`unittest parametrizer`: https://github.com/testing-cabal/unittest-ext/blob/master/params.py + + +Here is an example ``pytest_generate_tests`` function implementing a +parametrization scheme similar to Michael Foord's `unittest +parametrizer`_ but in a lot less code: + +.. code-block:: python + + # content of ./test_parametrize.py + import pytest + + + def pytest_generate_tests(metafunc): + # called once per each test function + funcarglist = metafunc.cls.params[metafunc.function.__name__] + argnames = sorted(funcarglist[0]) + metafunc.parametrize( + argnames, [[funcargs[name] for name in argnames] for funcargs in funcarglist] + ) + + + class TestClass: + # a map specifying multiple argument sets for a test method + params = { + "test_equals": [dict(a=1, b=2), dict(a=3, b=3)], + "test_zerodivision": [dict(a=1, b=0)], + } + + def test_equals(self, a, b): + assert a == b + + def test_zerodivision(self, a, b): + with pytest.raises(ZeroDivisionError): + a / b + +Our test generator looks up a class-level definition which specifies which +argument sets to use for each test function. Let's run it: + +.. code-block:: pytest + + $ pytest -q + F.. [100%] + ================================= FAILURES ================================= + ________________________ TestClass.test_equals[1-2] ________________________ + + self = , a = 1, b = 2 + + def test_equals(self, a, b): + > assert a == b + E assert 1 == 2 + + test_parametrize.py:21: AssertionError + ========================= short test summary info ========================== + FAILED test_parametrize.py::TestClass::test_equals[1-2] - assert 1 == 2 + 1 failed, 2 passed in 0.12s + +Indirect parametrization with multiple fixtures +-------------------------------------------------------------- + +Here is a stripped down real-life example of using parametrized +testing for testing serialization of objects between different python +interpreters. We define a ``test_basic_objects`` function which +is to be run with different sets of arguments for its three arguments: + +* ``python1``: first python interpreter, run to pickle-dump an object to a file +* ``python2``: second interpreter, run to pickle-load an object from a file +* ``obj``: object to be dumped/loaded + +.. literalinclude:: multipython.py + +Running it results in some skips if we don't have all the python interpreters installed and otherwise runs all combinations (3 interpreters times 3 interpreters times 3 objects to serialize/deserialize): + +.. code-block:: pytest + + . $ pytest -rs -q multipython.py + sssssssssssssssssssssssssss [100%] + ========================= short test summary info ========================== + SKIPPED [9] multipython.py:29: 'python3.5' not found + SKIPPED [9] multipython.py:29: 'python3.6' not found + SKIPPED [9] multipython.py:29: 'python3.7' not found + 27 skipped in 0.12s + +Indirect parametrization of optional implementations/imports +-------------------------------------------------------------------- + +If you want to compare the outcomes of several implementations of a given +API, you can write test functions that receive the already imported implementations +and get skipped in case the implementation is not importable/available. Let's +say we have a "base" implementation and the other (possibly optimized ones) +need to provide similar results: + +.. code-block:: python + + # content of conftest.py + + import pytest + + + @pytest.fixture(scope="session") + def basemod(request): + return pytest.importorskip("base") + + + @pytest.fixture(scope="session", params=["opt1", "opt2"]) + def optmod(request): + return pytest.importorskip(request.param) + +And then a base implementation of a simple function: + +.. code-block:: python + + # content of base.py + def func1(): + return 1 + +And an optimized version: + +.. code-block:: python + + # content of opt1.py + def func1(): + return 1.0001 + +And finally a little test module: + +.. code-block:: python + + # content of test_module.py + + + def test_func1(basemod, optmod): + assert round(basemod.func1(), 3) == round(optmod.func1(), 3) + + +If you run this with reporting for skips enabled: + +.. code-block:: pytest + + $ pytest -rs test_module.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 2 items + + test_module.py .s [100%] + + ========================= short test summary info ========================== + SKIPPED [1] conftest.py:12: could not import 'opt2': No module named 'opt2' + ======================= 1 passed, 1 skipped in 0.12s ======================= + +You'll see that we don't have an ``opt2`` module and thus the second test run +of our ``test_func1`` was skipped. A few notes: + +- the fixture functions in the ``conftest.py`` file are "session-scoped" because we + don't need to import more than once + +- if you have multiple test functions and a skipped import, you will see + the ``[1]`` count increasing in the report + +- you can put :ref:`@pytest.mark.parametrize <@pytest.mark.parametrize>` style + parametrization on the test functions to parametrize input/output + values as well. + + +Set marks or test ID for individual parametrized test +-------------------------------------------------------------------- + +Use ``pytest.param`` to apply marks or set test ID to individual parametrized test. +For example: + +.. code-block:: python + + # content of test_pytest_param_example.py + import pytest + + + @pytest.mark.parametrize( + "test_input,expected", + [ + ("3+5", 8), + pytest.param("1+7", 8, marks=pytest.mark.basic), + pytest.param("2+4", 6, marks=pytest.mark.basic, id="basic_2+4"), + pytest.param( + "6*9", 42, marks=[pytest.mark.basic, pytest.mark.xfail], id="basic_6*9" + ), + ], + ) + def test_eval(test_input, expected): + assert eval(test_input) == expected + +In this example, we have 4 parametrized tests. Except for the first test, +we mark the rest three parametrized tests with the custom marker ``basic``, +and for the fourth test we also use the built-in mark ``xfail`` to indicate this +test is expected to fail. For explicitness, we set test ids for some tests. + +Then run ``pytest`` with verbose mode and with only the ``basic`` marker: + +.. code-block:: pytest + + $ pytest -v -m basic + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 24 items / 21 deselected / 3 selected + + test_pytest_param_example.py::test_eval[1+7-8] PASSED [ 33%] + test_pytest_param_example.py::test_eval[basic_2+4] PASSED [ 66%] + test_pytest_param_example.py::test_eval[basic_6*9] XFAIL [100%] + + =============== 2 passed, 21 deselected, 1 xfailed in 0.12s ================ + +As the result: + +- Four tests were collected +- One test was deselected because it doesn't have the ``basic`` mark. +- Three tests with the ``basic`` mark was selected. +- The test ``test_eval[1+7-8]`` passed, but the name is autogenerated and confusing. +- The test ``test_eval[basic_2+4]`` passed. +- The test ``test_eval[basic_6*9]`` was expected to fail and did fail. + +.. _`parametrizing_conditional_raising`: + +Parametrizing conditional raising +-------------------------------------------------------------------- + +Use :func:`pytest.raises` with the +:ref:`pytest.mark.parametrize ref` decorator to write parametrized tests +in which some tests raise exceptions and others do not. + +It is helpful to define a no-op context manager ``does_not_raise`` to serve +as a complement to ``raises``. For example: + +.. code-block:: python + + from contextlib import contextmanager + import pytest + + + @contextmanager + def does_not_raise(): + yield + + + @pytest.mark.parametrize( + "example_input,expectation", + [ + (3, does_not_raise()), + (2, does_not_raise()), + (1, does_not_raise()), + (0, pytest.raises(ZeroDivisionError)), + ], + ) + def test_division(example_input, expectation): + """Test how much I know division.""" + with expectation: + assert (6 / example_input) is not None + +In the example above, the first three test cases should run unexceptionally, +while the fourth should raise ``ZeroDivisionError``. + +If you're only supporting Python 3.7+, you can simply use ``nullcontext`` +to define ``does_not_raise``: + +.. code-block:: python + + from contextlib import nullcontext as does_not_raise + +Or, if you're supporting Python 3.3+ you can use: + +.. code-block:: python + + from contextlib import ExitStack as does_not_raise + +Or, if desired, you can ``pip install contextlib2`` and use: + +.. code-block:: python + + from contextlib2 import nullcontext as does_not_raise diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/pythoncollection.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/pythoncollection.py new file mode 100644 index 0000000000..8742526a19 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/pythoncollection.py @@ -0,0 +1,14 @@ +# run this with $ pytest --collect-only test_collectonly.py +# + + +def test_function(): + pass + + +class TestClass: + def test_method(self): + pass + + def test_anothermethod(self): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/pythoncollection.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/pythoncollection.rst new file mode 100644 index 0000000000..b9c2386ac5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/pythoncollection.rst @@ -0,0 +1,321 @@ +Changing standard (Python) test discovery +=============================================== + +Ignore paths during test collection +----------------------------------- + +You can easily ignore certain test directories and modules during collection +by passing the ``--ignore=path`` option on the cli. ``pytest`` allows multiple +``--ignore`` options. Example: + +.. code-block:: text + + tests/ + |-- example + | |-- test_example_01.py + | |-- test_example_02.py + | '-- test_example_03.py + |-- foobar + | |-- test_foobar_01.py + | |-- test_foobar_02.py + | '-- test_foobar_03.py + '-- hello + '-- world + |-- test_world_01.py + |-- test_world_02.py + '-- test_world_03.py + +Now if you invoke ``pytest`` with ``--ignore=tests/foobar/test_foobar_03.py --ignore=tests/hello/``, +you will see that ``pytest`` only collects test-modules, which do not match the patterns specified: + +.. code-block:: pytest + + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-5.x.y, py-1.x.y, pluggy-0.x.y + rootdir: $REGENDOC_TMPDIR, inifile: + collected 5 items + + tests/example/test_example_01.py . [ 20%] + tests/example/test_example_02.py . [ 40%] + tests/example/test_example_03.py . [ 60%] + tests/foobar/test_foobar_01.py . [ 80%] + tests/foobar/test_foobar_02.py . [100%] + + ========================= 5 passed in 0.02 seconds ========================= + +The ``--ignore-glob`` option allows to ignore test file paths based on Unix shell-style wildcards. +If you want to exclude test-modules that end with ``_01.py``, execute ``pytest`` with ``--ignore-glob='*_01.py'``. + +Deselect tests during test collection +------------------------------------- + +Tests can individually be deselected during collection by passing the ``--deselect=item`` option. +For example, say ``tests/foobar/test_foobar_01.py`` contains ``test_a`` and ``test_b``. +You can run all of the tests within ``tests/`` *except* for ``tests/foobar/test_foobar_01.py::test_a`` +by invoking ``pytest`` with ``--deselect tests/foobar/test_foobar_01.py::test_a``. +``pytest`` allows multiple ``--deselect`` options. + +Keeping duplicate paths specified from command line +---------------------------------------------------- + +Default behavior of ``pytest`` is to ignore duplicate paths specified from the command line. +Example: + +.. code-block:: pytest + + pytest path_a path_a + + ... + collected 1 item + ... + +Just collect tests once. + +To collect duplicate tests, use the ``--keep-duplicates`` option on the cli. +Example: + +.. code-block:: pytest + + pytest --keep-duplicates path_a path_a + + ... + collected 2 items + ... + +As the collector just works on directories, if you specify twice a single test file, ``pytest`` will +still collect it twice, no matter if the ``--keep-duplicates`` is not specified. +Example: + +.. code-block:: pytest + + pytest test_a.py test_a.py + + ... + collected 2 items + ... + + +Changing directory recursion +----------------------------------------------------- + +You can set the :confval:`norecursedirs` option in an ini-file, for example your ``pytest.ini`` in the project root directory: + +.. code-block:: ini + + # content of pytest.ini + [pytest] + norecursedirs = .svn _build tmp* + +This would tell ``pytest`` to not recurse into typical subversion or sphinx-build directories or into any ``tmp`` prefixed directory. + +.. _`change naming conventions`: + +Changing naming conventions +----------------------------------------------------- + +You can configure different naming conventions by setting +the :confval:`python_files`, :confval:`python_classes` and +:confval:`python_functions` in your :ref:`configuration file `. +Here is an example: + +.. code-block:: ini + + # content of pytest.ini + # Example 1: have pytest look for "check" instead of "test" + [pytest] + python_files = check_*.py + python_classes = Check + python_functions = *_check + +This would make ``pytest`` look for tests in files that match the ``check_* +.py`` glob-pattern, ``Check`` prefixes in classes, and functions and methods +that match ``*_check``. For example, if we have: + +.. code-block:: python + + # content of check_myapp.py + class CheckMyApp: + def simple_check(self): + pass + + def complex_check(self): + pass + +The test collection would look like this: + +.. code-block:: pytest + + $ pytest --collect-only + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project, configfile: pytest.ini + collected 2 items + + + + + + + ======================== 2 tests collected in 0.12s ======================== + +You can check for multiple glob patterns by adding a space between the patterns: + +.. code-block:: ini + + # Example 2: have pytest look for files with "test" and "example" + # content of pytest.ini + [pytest] + python_files = test_*.py example_*.py + +.. note:: + + the ``python_functions`` and ``python_classes`` options has no effect + for ``unittest.TestCase`` test discovery because pytest delegates + discovery of test case methods to unittest code. + +Interpreting cmdline arguments as Python packages +----------------------------------------------------- + +You can use the ``--pyargs`` option to make ``pytest`` try +interpreting arguments as python package names, deriving +their file system path and then running the test. For +example if you have unittest2 installed you can type: + +.. code-block:: bash + + pytest --pyargs unittest2.test.test_skipping -q + +which would run the respective test module. Like with +other options, through an ini-file and the :confval:`addopts` option you +can make this change more permanently: + +.. code-block:: ini + + # content of pytest.ini + [pytest] + addopts = --pyargs + +Now a simple invocation of ``pytest NAME`` will check +if NAME exists as an importable package/module and otherwise +treat it as a filesystem path. + +Finding out what is collected +----------------------------------------------- + +You can always peek at the collection tree without running tests like this: + +.. code-block:: pytest + + . $ pytest --collect-only pythoncollection.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project, configfile: pytest.ini + collected 3 items + + + + + + + + ======================== 3 tests collected in 0.12s ======================== + +.. _customizing-test-collection: + +Customizing test collection +--------------------------- + +.. regendoc:wipe + +You can easily instruct ``pytest`` to discover tests from every Python file: + +.. code-block:: ini + + # content of pytest.ini + [pytest] + python_files = *.py + +However, many projects will have a ``setup.py`` which they don't want to be +imported. Moreover, there may files only importable by a specific python +version. For such cases you can dynamically define files to be ignored by +listing them in a ``conftest.py`` file: + +.. code-block:: python + + # content of conftest.py + import sys + + collect_ignore = ["setup.py"] + if sys.version_info[0] > 2: + collect_ignore.append("pkg/module_py2.py") + +and then if you have a module file like this: + +.. code-block:: python + + # content of pkg/module_py2.py + def test_only_on_python2(): + try: + assert 0 + except Exception, e: + pass + +and a ``setup.py`` dummy file like this: + +.. code-block:: python + + # content of setup.py + 0 / 0 # will raise exception if imported + +If you run with a Python 2 interpreter then you will find the one test and will +leave out the ``setup.py`` file: + +.. code-block:: pytest + + #$ pytest --collect-only + ====== test session starts ====== + platform linux2 -- Python 2.7.10, pytest-2.9.1, py-1.4.31, pluggy-0.3.1 + rootdir: $REGENDOC_TMPDIR, inifile: pytest.ini + collected 1 items + + + + ====== 1 tests found in 0.04 seconds ====== + +If you run with a Python 3 interpreter both the one test and the ``setup.py`` +file will be left out: + +.. code-block:: pytest + + $ pytest --collect-only + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project, configfile: pytest.ini + collected 0 items + + ======================= no tests collected in 0.12s ======================== + +It's also possible to ignore files based on Unix shell-style wildcards by adding +patterns to :globalvar:`collect_ignore_glob`. + +The following example ``conftest.py`` ignores the file ``setup.py`` and in +addition all files that end with ``*_py2.py`` when executed with a Python 3 +interpreter: + +.. code-block:: python + + # content of conftest.py + import sys + + collect_ignore = ["setup.py"] + if sys.version_info[0] > 2: + collect_ignore_glob = ["*_py2.py"] + +Since Pytest 2.6, users can prevent pytest from discovering classes that start +with ``Test`` by setting a boolean ``__test__`` attribute to ``False``. + +.. code-block:: python + + # Will not be discovered as a test + class TestClass: + __test__ = False diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/reportingdemo.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/reportingdemo.rst new file mode 100644 index 0000000000..cab9314361 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/reportingdemo.rst @@ -0,0 +1,708 @@ +.. _`tbreportdemo`: + +Demo of Python failure reports with pytest +========================================== + +Here is a nice run of several failures and how ``pytest`` presents things: + +.. code-block:: pytest + + assertion $ pytest failure_demo.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project/assertion + collected 44 items + + failure_demo.py FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF [100%] + + ================================= FAILURES ================================= + ___________________________ test_generative[3-6] ___________________________ + + param1 = 3, param2 = 6 + + @pytest.mark.parametrize("param1, param2", [(3, 6)]) + def test_generative(param1, param2): + > assert param1 * 2 < param2 + E assert (3 * 2) < 6 + + failure_demo.py:19: AssertionError + _________________________ TestFailing.test_simple __________________________ + + self = + + def test_simple(self): + def f(): + return 42 + + def g(): + return 43 + + > assert f() == g() + E assert 42 == 43 + E + where 42 = .f at 0xdeadbeef0002>() + E + and 43 = .g at 0xdeadbeef0003>() + + failure_demo.py:30: AssertionError + ____________________ TestFailing.test_simple_multiline _____________________ + + self = + + def test_simple_multiline(self): + > otherfunc_multi(42, 6 * 9) + + failure_demo.py:33: + _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + a = 42, b = 54 + + def otherfunc_multi(a, b): + > assert a == b + E assert 42 == 54 + + failure_demo.py:14: AssertionError + ___________________________ TestFailing.test_not ___________________________ + + self = + + def test_not(self): + def f(): + return 42 + + > assert not f() + E assert not 42 + E + where 42 = .f at 0xdeadbeef0006>() + + failure_demo.py:39: AssertionError + _________________ TestSpecialisedExplanations.test_eq_text _________________ + + self = + + def test_eq_text(self): + > assert "spam" == "eggs" + E AssertionError: assert 'spam' == 'eggs' + E - eggs + E + spam + + failure_demo.py:44: AssertionError + _____________ TestSpecialisedExplanations.test_eq_similar_text _____________ + + self = + + def test_eq_similar_text(self): + > assert "foo 1 bar" == "foo 2 bar" + E AssertionError: assert 'foo 1 bar' == 'foo 2 bar' + E - foo 2 bar + E ? ^ + E + foo 1 bar + E ? ^ + + failure_demo.py:47: AssertionError + ____________ TestSpecialisedExplanations.test_eq_multiline_text ____________ + + self = + + def test_eq_multiline_text(self): + > assert "foo\nspam\nbar" == "foo\neggs\nbar" + E AssertionError: assert 'foo\nspam\nbar' == 'foo\neggs\nbar' + E foo + E - eggs + E + spam + E bar + + failure_demo.py:50: AssertionError + ______________ TestSpecialisedExplanations.test_eq_long_text _______________ + + self = + + def test_eq_long_text(self): + a = "1" * 100 + "a" + "2" * 100 + b = "1" * 100 + "b" + "2" * 100 + > assert a == b + E AssertionError: assert '111111111111...2222222222222' == '111111111111...2222222222222' + E Skipping 90 identical leading characters in diff, use -v to show + E Skipping 91 identical trailing characters in diff, use -v to show + E - 1111111111b222222222 + E ? ^ + E + 1111111111a222222222 + E ? ^ + + failure_demo.py:55: AssertionError + _________ TestSpecialisedExplanations.test_eq_long_text_multiline __________ + + self = + + def test_eq_long_text_multiline(self): + a = "1\n" * 100 + "a" + "2\n" * 100 + b = "1\n" * 100 + "b" + "2\n" * 100 + > assert a == b + E AssertionError: assert '1\n1\n1\n1\n...n2\n2\n2\n2\n' == '1\n1\n1\n1\n...n2\n2\n2\n2\n' + E Skipping 190 identical leading characters in diff, use -v to show + E Skipping 191 identical trailing characters in diff, use -v to show + E 1 + E 1 + E 1 + E 1 + E 1... + E + E ...Full output truncated (7 lines hidden), use '-vv' to show + + failure_demo.py:60: AssertionError + _________________ TestSpecialisedExplanations.test_eq_list _________________ + + self = + + def test_eq_list(self): + > assert [0, 1, 2] == [0, 1, 3] + E assert [0, 1, 2] == [0, 1, 3] + E At index 2 diff: 2 != 3 + E Use -v to get the full diff + + failure_demo.py:63: AssertionError + ______________ TestSpecialisedExplanations.test_eq_list_long _______________ + + self = + + def test_eq_list_long(self): + a = [0] * 100 + [1] + [3] * 100 + b = [0] * 100 + [2] + [3] * 100 + > assert a == b + E assert [0, 0, 0, 0, 0, 0, ...] == [0, 0, 0, 0, 0, 0, ...] + E At index 100 diff: 1 != 2 + E Use -v to get the full diff + + failure_demo.py:68: AssertionError + _________________ TestSpecialisedExplanations.test_eq_dict _________________ + + self = + + def test_eq_dict(self): + > assert {"a": 0, "b": 1, "c": 0} == {"a": 0, "b": 2, "d": 0} + E AssertionError: assert {'a': 0, 'b': 1, 'c': 0} == {'a': 0, 'b': 2, 'd': 0} + E Omitting 1 identical items, use -vv to show + E Differing items: + E {'b': 1} != {'b': 2} + E Left contains 1 more item: + E {'c': 0} + E Right contains 1 more item: + E {'d': 0}... + E + E ...Full output truncated (2 lines hidden), use '-vv' to show + + failure_demo.py:71: AssertionError + _________________ TestSpecialisedExplanations.test_eq_set __________________ + + self = + + def test_eq_set(self): + > assert {0, 10, 11, 12} == {0, 20, 21} + E AssertionError: assert {0, 10, 11, 12} == {0, 20, 21} + E Extra items in the left set: + E 10 + E 11 + E 12 + E Extra items in the right set: + E 20 + E 21... + E + E ...Full output truncated (2 lines hidden), use '-vv' to show + + failure_demo.py:74: AssertionError + _____________ TestSpecialisedExplanations.test_eq_longer_list ______________ + + self = + + def test_eq_longer_list(self): + > assert [1, 2] == [1, 2, 3] + E assert [1, 2] == [1, 2, 3] + E Right contains one more item: 3 + E Use -v to get the full diff + + failure_demo.py:77: AssertionError + _________________ TestSpecialisedExplanations.test_in_list _________________ + + self = + + def test_in_list(self): + > assert 1 in [0, 2, 3, 4, 5] + E assert 1 in [0, 2, 3, 4, 5] + + failure_demo.py:80: AssertionError + __________ TestSpecialisedExplanations.test_not_in_text_multiline __________ + + self = + + def test_not_in_text_multiline(self): + text = "some multiline\ntext\nwhich\nincludes foo\nand a\ntail" + > assert "foo" not in text + E AssertionError: assert 'foo' not in 'some multil...nand a\ntail' + E 'foo' is contained here: + E some multiline + E text + E which + E includes foo + E ? +++ + E and a... + E + E ...Full output truncated (2 lines hidden), use '-vv' to show + + failure_demo.py:84: AssertionError + ___________ TestSpecialisedExplanations.test_not_in_text_single ____________ + + self = + + def test_not_in_text_single(self): + text = "single foo line" + > assert "foo" not in text + E AssertionError: assert 'foo' not in 'single foo line' + E 'foo' is contained here: + E single foo line + E ? +++ + + failure_demo.py:88: AssertionError + _________ TestSpecialisedExplanations.test_not_in_text_single_long _________ + + self = + + def test_not_in_text_single_long(self): + text = "head " * 50 + "foo " + "tail " * 20 + > assert "foo" not in text + E AssertionError: assert 'foo' not in 'head head h...l tail tail ' + E 'foo' is contained here: + E head head foo tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail + E ? +++ + + failure_demo.py:92: AssertionError + ______ TestSpecialisedExplanations.test_not_in_text_single_long_term _______ + + self = + + def test_not_in_text_single_long_term(self): + text = "head " * 50 + "f" * 70 + "tail " * 20 + > assert "f" * 70 not in text + E AssertionError: assert 'fffffffffff...ffffffffffff' not in 'head head h...l tail tail ' + E 'ffffffffffffffffff...fffffffffffffffffff' is contained here: + E head head fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffftail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail + E ? ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + + failure_demo.py:96: AssertionError + ______________ TestSpecialisedExplanations.test_eq_dataclass _______________ + + self = + + def test_eq_dataclass(self): + from dataclasses import dataclass + + @dataclass + class Foo: + a: int + b: str + + left = Foo(1, "b") + right = Foo(1, "c") + > assert left == right + E AssertionError: assert TestSpecialis...oo(a=1, b='b') == TestSpecialis...oo(a=1, b='c') + E + E Omitting 1 identical items, use -vv to show + E Differing attributes: + E ['b'] + E + E Drill down into differing attribute b: + E b: 'b' != 'c'... + E + E ...Full output truncated (3 lines hidden), use '-vv' to show + + failure_demo.py:108: AssertionError + ________________ TestSpecialisedExplanations.test_eq_attrs _________________ + + self = + + def test_eq_attrs(self): + import attr + + @attr.s + class Foo: + a = attr.ib() + b = attr.ib() + + left = Foo(1, "b") + right = Foo(1, "c") + > assert left == right + E AssertionError: assert Foo(a=1, b='b') == Foo(a=1, b='c') + E + E Omitting 1 identical items, use -vv to show + E Differing attributes: + E ['b'] + E + E Drill down into differing attribute b: + E b: 'b' != 'c'... + E + E ...Full output truncated (3 lines hidden), use '-vv' to show + + failure_demo.py:120: AssertionError + ______________________________ test_attribute ______________________________ + + def test_attribute(): + class Foo: + b = 1 + + i = Foo() + > assert i.b == 2 + E assert 1 == 2 + E + where 1 = .Foo object at 0xdeadbeef0018>.b + + failure_demo.py:128: AssertionError + _________________________ test_attribute_instance __________________________ + + def test_attribute_instance(): + class Foo: + b = 1 + + > assert Foo().b == 2 + E AssertionError: assert 1 == 2 + E + where 1 = .Foo object at 0xdeadbeef0019>.b + E + where .Foo object at 0xdeadbeef0019> = .Foo'>() + + failure_demo.py:135: AssertionError + __________________________ test_attribute_failure __________________________ + + def test_attribute_failure(): + class Foo: + def _get_b(self): + raise Exception("Failed to get attrib") + + b = property(_get_b) + + i = Foo() + > assert i.b == 2 + + failure_demo.py:146: + _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + self = .Foo object at 0xdeadbeef001a> + + def _get_b(self): + > raise Exception("Failed to get attrib") + E Exception: Failed to get attrib + + failure_demo.py:141: Exception + _________________________ test_attribute_multiple __________________________ + + def test_attribute_multiple(): + class Foo: + b = 1 + + class Bar: + b = 2 + + > assert Foo().b == Bar().b + E AssertionError: assert 1 == 2 + E + where 1 = .Foo object at 0xdeadbeef001b>.b + E + where .Foo object at 0xdeadbeef001b> = .Foo'>() + E + and 2 = .Bar object at 0xdeadbeef001c>.b + E + where .Bar object at 0xdeadbeef001c> = .Bar'>() + + failure_demo.py:156: AssertionError + __________________________ TestRaises.test_raises __________________________ + + self = + + def test_raises(self): + s = "qwe" + > raises(TypeError, int, s) + E ValueError: invalid literal for int() with base 10: 'qwe' + + failure_demo.py:166: ValueError + ______________________ TestRaises.test_raises_doesnt _______________________ + + self = + + def test_raises_doesnt(self): + > raises(OSError, int, "3") + E Failed: DID NOT RAISE + + failure_demo.py:169: Failed + __________________________ TestRaises.test_raise ___________________________ + + self = + + def test_raise(self): + > raise ValueError("demo error") + E ValueError: demo error + + failure_demo.py:172: ValueError + ________________________ TestRaises.test_tupleerror ________________________ + + self = + + def test_tupleerror(self): + > a, b = [1] # NOQA + E ValueError: not enough values to unpack (expected 2, got 1) + + failure_demo.py:175: ValueError + ______ TestRaises.test_reinterpret_fails_with_print_for_the_fun_of_it ______ + + self = + + def test_reinterpret_fails_with_print_for_the_fun_of_it(self): + items = [1, 2, 3] + print(f"items is {items!r}") + > a, b = items.pop() + E TypeError: cannot unpack non-iterable int object + + failure_demo.py:180: TypeError + --------------------------- Captured stdout call --------------------------- + items is [1, 2, 3] + ________________________ TestRaises.test_some_error ________________________ + + self = + + def test_some_error(self): + > if namenotexi: # NOQA + E NameError: name 'namenotexi' is not defined + + failure_demo.py:183: NameError + ____________________ test_dynamic_compile_shows_nicely _____________________ + + def test_dynamic_compile_shows_nicely(): + import importlib.util + import sys + + src = "def foo():\n assert 1 == 0\n" + name = "abc-123" + spec = importlib.util.spec_from_loader(name, loader=None) + module = importlib.util.module_from_spec(spec) + code = compile(src, name, "exec") + exec(code, module.__dict__) + sys.modules[name] = module + > module.foo() + + failure_demo.py:202: + _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + > ??? + E AssertionError + + abc-123:2: AssertionError + ____________________ TestMoreErrors.test_complex_error _____________________ + + self = + + def test_complex_error(self): + def f(): + return 44 + + def g(): + return 43 + + > somefunc(f(), g()) + + failure_demo.py:213: + _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + failure_demo.py:10: in somefunc + otherfunc(x, y) + _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + a = 44, b = 43 + + def otherfunc(a, b): + > assert a == b + E assert 44 == 43 + + failure_demo.py:6: AssertionError + ___________________ TestMoreErrors.test_z1_unpack_error ____________________ + + self = + + def test_z1_unpack_error(self): + items = [] + > a, b = items + E ValueError: not enough values to unpack (expected 2, got 0) + + failure_demo.py:217: ValueError + ____________________ TestMoreErrors.test_z2_type_error _____________________ + + self = + + def test_z2_type_error(self): + items = 3 + > a, b = items + E TypeError: cannot unpack non-iterable int object + + failure_demo.py:221: TypeError + ______________________ TestMoreErrors.test_startswith ______________________ + + self = + + def test_startswith(self): + s = "123" + g = "456" + > assert s.startswith(g) + E AssertionError: assert False + E + where False = ('456') + E + where = '123'.startswith + + failure_demo.py:226: AssertionError + __________________ TestMoreErrors.test_startswith_nested ___________________ + + self = + + def test_startswith_nested(self): + def f(): + return "123" + + def g(): + return "456" + + > assert f().startswith(g()) + E AssertionError: assert False + E + where False = ('456') + E + where = '123'.startswith + E + where '123' = .f at 0xdeadbeef0029>() + E + and '456' = .g at 0xdeadbeef002a>() + + failure_demo.py:235: AssertionError + _____________________ TestMoreErrors.test_global_func ______________________ + + self = + + def test_global_func(self): + > assert isinstance(globf(42), float) + E assert False + E + where False = isinstance(43, float) + E + where 43 = globf(42) + + failure_demo.py:238: AssertionError + _______________________ TestMoreErrors.test_instance _______________________ + + self = + + def test_instance(self): + self.x = 6 * 7 + > assert self.x != 42 + E assert 42 != 42 + E + where 42 = .x + + failure_demo.py:242: AssertionError + _______________________ TestMoreErrors.test_compare ________________________ + + self = + + def test_compare(self): + > assert globf(10) < 5 + E assert 11 < 5 + E + where 11 = globf(10) + + failure_demo.py:245: AssertionError + _____________________ TestMoreErrors.test_try_finally ______________________ + + self = + + def test_try_finally(self): + x = 1 + try: + > assert x == 0 + E assert 1 == 0 + + failure_demo.py:250: AssertionError + ___________________ TestCustomAssertMsg.test_single_line ___________________ + + self = + + def test_single_line(self): + class A: + a = 1 + + b = 2 + > assert A.a == b, "A.a appears not to be b" + E AssertionError: A.a appears not to be b + E assert 1 == 2 + E + where 1 = .A'>.a + + failure_demo.py:261: AssertionError + ____________________ TestCustomAssertMsg.test_multiline ____________________ + + self = + + def test_multiline(self): + class A: + a = 1 + + b = 2 + > assert ( + A.a == b + ), "A.a appears not to be b\nor does not appear to be b\none of those" + E AssertionError: A.a appears not to be b + E or does not appear to be b + E one of those + E assert 1 == 2 + E + where 1 = .A'>.a + + failure_demo.py:268: AssertionError + ___________________ TestCustomAssertMsg.test_custom_repr ___________________ + + self = + + def test_custom_repr(self): + class JSON: + a = 1 + + def __repr__(self): + return "This is JSON\n{\n 'foo': 'bar'\n}" + + a = JSON() + b = 2 + > assert a.a == b, a + E AssertionError: This is JSON + E { + E 'foo': 'bar' + E } + E assert 1 == 2 + E + where 1 = This is JSON\n{\n 'foo': 'bar'\n}.a + + failure_demo.py:281: AssertionError + ========================= short test summary info ========================== + FAILED failure_demo.py::test_generative[3-6] - assert (3 * 2) < 6 + FAILED failure_demo.py::TestFailing::test_simple - assert 42 == 43 + FAILED failure_demo.py::TestFailing::test_simple_multiline - assert 42 == 54 + FAILED failure_demo.py::TestFailing::test_not - assert not 42 + FAILED failure_demo.py::TestSpecialisedExplanations::test_eq_text - Asser... + FAILED failure_demo.py::TestSpecialisedExplanations::test_eq_similar_text + FAILED failure_demo.py::TestSpecialisedExplanations::test_eq_multiline_text + FAILED failure_demo.py::TestSpecialisedExplanations::test_eq_long_text - ... + FAILED failure_demo.py::TestSpecialisedExplanations::test_eq_long_text_multiline + FAILED failure_demo.py::TestSpecialisedExplanations::test_eq_list - asser... + FAILED failure_demo.py::TestSpecialisedExplanations::test_eq_list_long - ... + FAILED failure_demo.py::TestSpecialisedExplanations::test_eq_dict - Asser... + FAILED failure_demo.py::TestSpecialisedExplanations::test_eq_set - Assert... + FAILED failure_demo.py::TestSpecialisedExplanations::test_eq_longer_list + FAILED failure_demo.py::TestSpecialisedExplanations::test_in_list - asser... + FAILED failure_demo.py::TestSpecialisedExplanations::test_not_in_text_multiline + FAILED failure_demo.py::TestSpecialisedExplanations::test_not_in_text_single + FAILED failure_demo.py::TestSpecialisedExplanations::test_not_in_text_single_long + FAILED failure_demo.py::TestSpecialisedExplanations::test_not_in_text_single_long_term + FAILED failure_demo.py::TestSpecialisedExplanations::test_eq_dataclass - ... + FAILED failure_demo.py::TestSpecialisedExplanations::test_eq_attrs - Asse... + FAILED failure_demo.py::test_attribute - assert 1 == 2 + FAILED failure_demo.py::test_attribute_instance - AssertionError: assert ... + FAILED failure_demo.py::test_attribute_failure - Exception: Failed to get... + FAILED failure_demo.py::test_attribute_multiple - AssertionError: assert ... + FAILED failure_demo.py::TestRaises::test_raises - ValueError: invalid lit... + FAILED failure_demo.py::TestRaises::test_raises_doesnt - Failed: DID NOT ... + FAILED failure_demo.py::TestRaises::test_raise - ValueError: demo error + FAILED failure_demo.py::TestRaises::test_tupleerror - ValueError: not eno... + FAILED failure_demo.py::TestRaises::test_reinterpret_fails_with_print_for_the_fun_of_it + FAILED failure_demo.py::TestRaises::test_some_error - NameError: name 'na... + FAILED failure_demo.py::test_dynamic_compile_shows_nicely - AssertionError + FAILED failure_demo.py::TestMoreErrors::test_complex_error - assert 44 == 43 + FAILED failure_demo.py::TestMoreErrors::test_z1_unpack_error - ValueError... + FAILED failure_demo.py::TestMoreErrors::test_z2_type_error - TypeError: c... + FAILED failure_demo.py::TestMoreErrors::test_startswith - AssertionError:... + FAILED failure_demo.py::TestMoreErrors::test_startswith_nested - Assertio... + FAILED failure_demo.py::TestMoreErrors::test_global_func - assert False + FAILED failure_demo.py::TestMoreErrors::test_instance - assert 42 != 42 + FAILED failure_demo.py::TestMoreErrors::test_compare - assert 11 < 5 + FAILED failure_demo.py::TestMoreErrors::test_try_finally - assert 1 == 0 + FAILED failure_demo.py::TestCustomAssertMsg::test_single_line - Assertion... + FAILED failure_demo.py::TestCustomAssertMsg::test_multiline - AssertionEr... + FAILED failure_demo.py::TestCustomAssertMsg::test_custom_repr - Assertion... + ============================ 44 failed in 0.12s ============================ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/simple.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/simple.rst new file mode 100644 index 0000000000..a70f340499 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/simple.rst @@ -0,0 +1,1086 @@ + + +Basic patterns and examples +========================================================== + +How to change command line options defaults +------------------------------------------- + +It can be tedious to type the same series of command line options +every time you use ``pytest``. For example, if you always want to see +detailed info on skipped and xfailed tests, as well as have terser "dot" +progress output, you can write it into a configuration file: + +.. code-block:: ini + + # content of pytest.ini + [pytest] + addopts = -ra -q + + +Alternatively, you can set a ``PYTEST_ADDOPTS`` environment variable to add command +line options while the environment is in use: + +.. code-block:: bash + + export PYTEST_ADDOPTS="-v" + +Here's how the command-line is built in the presence of ``addopts`` or the environment variable: + +.. code-block:: text + + $PYTEST_ADDOPTS + +So if the user executes in the command-line: + +.. code-block:: bash + + pytest -m slow + +The actual command line executed is: + +.. code-block:: bash + + pytest -ra -q -v -m slow + +Note that as usual for other command-line applications, in case of conflicting options the last one wins, so the example +above will show verbose output because ``-v`` overwrites ``-q``. + + +.. _request example: + +Pass different values to a test function, depending on command line options +---------------------------------------------------------------------------- + +.. regendoc:wipe + +Suppose we want to write a test that depends on a command line option. +Here is a basic pattern to achieve this: + +.. code-block:: python + + # content of test_sample.py + def test_answer(cmdopt): + if cmdopt == "type1": + print("first") + elif cmdopt == "type2": + print("second") + assert 0 # to see what was printed + + +For this to work we need to add a command line option and +provide the ``cmdopt`` through a :ref:`fixture function `: + +.. code-block:: python + + # content of conftest.py + import pytest + + + def pytest_addoption(parser): + parser.addoption( + "--cmdopt", action="store", default="type1", help="my option: type1 or type2" + ) + + + @pytest.fixture + def cmdopt(request): + return request.config.getoption("--cmdopt") + +Let's run this without supplying our new option: + +.. code-block:: pytest + + $ pytest -q test_sample.py + F [100%] + ================================= FAILURES ================================= + _______________________________ test_answer ________________________________ + + cmdopt = 'type1' + + def test_answer(cmdopt): + if cmdopt == "type1": + print("first") + elif cmdopt == "type2": + print("second") + > assert 0 # to see what was printed + E assert 0 + + test_sample.py:6: AssertionError + --------------------------- Captured stdout call --------------------------- + first + ========================= short test summary info ========================== + FAILED test_sample.py::test_answer - assert 0 + 1 failed in 0.12s + +And now with supplying a command line option: + +.. code-block:: pytest + + $ pytest -q --cmdopt=type2 + F [100%] + ================================= FAILURES ================================= + _______________________________ test_answer ________________________________ + + cmdopt = 'type2' + + def test_answer(cmdopt): + if cmdopt == "type1": + print("first") + elif cmdopt == "type2": + print("second") + > assert 0 # to see what was printed + E assert 0 + + test_sample.py:6: AssertionError + --------------------------- Captured stdout call --------------------------- + second + ========================= short test summary info ========================== + FAILED test_sample.py::test_answer - assert 0 + 1 failed in 0.12s + +You can see that the command line option arrived in our test. + +We could add simple validation for the input by listing the choices: + +.. code-block:: python + + # content of conftest.py + import pytest + + + def pytest_addoption(parser): + parser.addoption( + "--cmdopt", + action="store", + default="type1", + help="my option: type1 or type2", + choices=("type1", "type2"), + ) + +Now we'll get feedback on a bad argument: + +.. code-block:: pytest + + $ pytest -q --cmdopt=type3 + ERROR: usage: pytest [options] [file_or_dir] [file_or_dir] [...] + pytest: error: argument --cmdopt: invalid choice: 'type3' (choose from 'type1', 'type2') + + +If you need to provide more detailed error messages, you can use the +``type`` parameter and raise ``pytest.UsageError``: + +.. code-block:: python + + # content of conftest.py + import pytest + + + def type_checker(value): + msg = "cmdopt must specify a numeric type as typeNNN" + if not value.startswith("type"): + raise pytest.UsageError(msg) + try: + int(value[4:]) + except ValueError: + raise pytest.UsageError(msg) + + return value + + + def pytest_addoption(parser): + parser.addoption( + "--cmdopt", + action="store", + default="type1", + help="my option: type1 or type2", + type=type_checker, + ) + +This completes the basic pattern. However, one often rather wants to +process command line options outside of the test and rather pass in +different or more complex objects. + +Dynamically adding command line options +-------------------------------------------------------------- + +.. regendoc:wipe + +Through :confval:`addopts` you can statically add command line +options for your project. You can also dynamically modify +the command line arguments before they get processed: + +.. code-block:: python + + # setuptools plugin + import sys + + + def pytest_load_initial_conftests(args): + if "xdist" in sys.modules: # pytest-xdist plugin + import multiprocessing + + num = max(multiprocessing.cpu_count() / 2, 1) + args[:] = ["-n", str(num)] + args + +If you have the :pypi:`xdist plugin ` installed +you will now always perform test runs using a number +of subprocesses close to your CPU. Running in an empty +directory with the above conftest.py: + +.. code-block:: pytest + + $ pytest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 0 items + + ========================== no tests ran in 0.12s =========================== + +.. _`excontrolskip`: + +Control skipping of tests according to command line option +-------------------------------------------------------------- + +.. regendoc:wipe + +Here is a ``conftest.py`` file adding a ``--runslow`` command +line option to control skipping of ``pytest.mark.slow`` marked tests: + +.. code-block:: python + + # content of conftest.py + + import pytest + + + def pytest_addoption(parser): + parser.addoption( + "--runslow", action="store_true", default=False, help="run slow tests" + ) + + + def pytest_configure(config): + config.addinivalue_line("markers", "slow: mark test as slow to run") + + + def pytest_collection_modifyitems(config, items): + if config.getoption("--runslow"): + # --runslow given in cli: do not skip slow tests + return + skip_slow = pytest.mark.skip(reason="need --runslow option to run") + for item in items: + if "slow" in item.keywords: + item.add_marker(skip_slow) + +We can now write a test module like this: + +.. code-block:: python + + # content of test_module.py + import pytest + + + def test_func_fast(): + pass + + + @pytest.mark.slow + def test_func_slow(): + pass + +and when running it will see a skipped "slow" test: + +.. code-block:: pytest + + $ pytest -rs # "-rs" means report details on the little 's' + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 2 items + + test_module.py .s [100%] + + ========================= short test summary info ========================== + SKIPPED [1] test_module.py:8: need --runslow option to run + ======================= 1 passed, 1 skipped in 0.12s ======================= + +Or run it including the ``slow`` marked test: + +.. code-block:: pytest + + $ pytest --runslow + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 2 items + + test_module.py .. [100%] + + ============================ 2 passed in 0.12s ============================= + +.. _`__tracebackhide__`: + +Writing well integrated assertion helpers +----------------------------------------- + +.. regendoc:wipe + +If you have a test helper function called from a test you can +use the ``pytest.fail`` marker to fail a test with a certain message. +The test support function will not show up in the traceback if you +set the ``__tracebackhide__`` option somewhere in the helper function. +Example: + +.. code-block:: python + + # content of test_checkconfig.py + import pytest + + + def checkconfig(x): + __tracebackhide__ = True + if not hasattr(x, "config"): + pytest.fail("not configured: {}".format(x)) + + + def test_something(): + checkconfig(42) + +The ``__tracebackhide__`` setting influences ``pytest`` showing +of tracebacks: the ``checkconfig`` function will not be shown +unless the ``--full-trace`` command line option is specified. +Let's run our little function: + +.. code-block:: pytest + + $ pytest -q test_checkconfig.py + F [100%] + ================================= FAILURES ================================= + ______________________________ test_something ______________________________ + + def test_something(): + > checkconfig(42) + E Failed: not configured: 42 + + test_checkconfig.py:11: Failed + ========================= short test summary info ========================== + FAILED test_checkconfig.py::test_something - Failed: not configured: 42 + 1 failed in 0.12s + +If you only want to hide certain exceptions, you can set ``__tracebackhide__`` +to a callable which gets the ``ExceptionInfo`` object. You can for example use +this to make sure unexpected exception types aren't hidden: + +.. code-block:: python + + import operator + import pytest + + + class ConfigException(Exception): + pass + + + def checkconfig(x): + __tracebackhide__ = operator.methodcaller("errisinstance", ConfigException) + if not hasattr(x, "config"): + raise ConfigException("not configured: {}".format(x)) + + + def test_something(): + checkconfig(42) + +This will avoid hiding the exception traceback on unrelated exceptions (i.e. +bugs in assertion helpers). + + +Detect if running from within a pytest run +-------------------------------------------------------------- + +.. regendoc:wipe + +Usually it is a bad idea to make application code +behave differently if called from a test. But if you +absolutely must find out if your application code is +running from a test you can do something like this: + +.. code-block:: python + + # content of your_module.py + + + _called_from_test = False + +.. code-block:: python + + # content of conftest.py + + + def pytest_configure(config): + your_module._called_from_test = True + +and then check for the ``your_module._called_from_test`` flag: + +.. code-block:: python + + if your_module._called_from_test: + # called from within a test run + ... + else: + # called "normally" + ... + +accordingly in your application. + +Adding info to test report header +-------------------------------------------------------------- + +.. regendoc:wipe + +It's easy to present extra information in a ``pytest`` run: + +.. code-block:: python + + # content of conftest.py + + + def pytest_report_header(config): + return "project deps: mylib-1.1" + +which will add the string to the test header accordingly: + +.. code-block:: pytest + + $ pytest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + project deps: mylib-1.1 + rootdir: /home/sweet/project + collected 0 items + + ========================== no tests ran in 0.12s =========================== + +.. regendoc:wipe + +It is also possible to return a list of strings which will be considered as several +lines of information. You may consider ``config.getoption('verbose')`` in order to +display more information if applicable: + +.. code-block:: python + + # content of conftest.py + + + def pytest_report_header(config): + if config.getoption("verbose") > 0: + return ["info1: did you know that ...", "did you?"] + +which will add info only when run with "--v": + +.. code-block:: pytest + + $ pytest -v + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + info1: did you know that ... + did you? + rootdir: /home/sweet/project + collecting ... collected 0 items + + ========================== no tests ran in 0.12s =========================== + +and nothing when run plainly: + +.. code-block:: pytest + + $ pytest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 0 items + + ========================== no tests ran in 0.12s =========================== + +Profiling test duration +-------------------------- + +.. regendoc:wipe + +.. versionadded: 2.2 + +If you have a slow running large test suite you might want to find +out which tests are the slowest. Let's make an artificial test suite: + +.. code-block:: python + + # content of test_some_are_slow.py + import time + + + def test_funcfast(): + time.sleep(0.1) + + + def test_funcslow1(): + time.sleep(0.2) + + + def test_funcslow2(): + time.sleep(0.3) + +Now we can profile which test functions execute the slowest: + +.. code-block:: pytest + + $ pytest --durations=3 + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 3 items + + test_some_are_slow.py ... [100%] + + =========================== slowest 3 durations ============================ + 0.30s call test_some_are_slow.py::test_funcslow2 + 0.20s call test_some_are_slow.py::test_funcslow1 + 0.10s call test_some_are_slow.py::test_funcfast + ============================ 3 passed in 0.12s ============================= + +Incremental testing - test steps +--------------------------------------------------- + +.. regendoc:wipe + +Sometimes you may have a testing situation which consists of a series +of test steps. If one step fails it makes no sense to execute further +steps as they are all expected to fail anyway and their tracebacks +add no insight. Here is a simple ``conftest.py`` file which introduces +an ``incremental`` marker which is to be used on classes: + +.. code-block:: python + + # content of conftest.py + + from typing import Dict, Tuple + import pytest + + # store history of failures per test class name and per index in parametrize (if parametrize used) + _test_failed_incremental: Dict[str, Dict[Tuple[int, ...], str]] = {} + + + def pytest_runtest_makereport(item, call): + if "incremental" in item.keywords: + # incremental marker is used + if call.excinfo is not None: + # the test has failed + # retrieve the class name of the test + cls_name = str(item.cls) + # retrieve the index of the test (if parametrize is used in combination with incremental) + parametrize_index = ( + tuple(item.callspec.indices.values()) + if hasattr(item, "callspec") + else () + ) + # retrieve the name of the test function + test_name = item.originalname or item.name + # store in _test_failed_incremental the original name of the failed test + _test_failed_incremental.setdefault(cls_name, {}).setdefault( + parametrize_index, test_name + ) + + + def pytest_runtest_setup(item): + if "incremental" in item.keywords: + # retrieve the class name of the test + cls_name = str(item.cls) + # check if a previous test has failed for this class + if cls_name in _test_failed_incremental: + # retrieve the index of the test (if parametrize is used in combination with incremental) + parametrize_index = ( + tuple(item.callspec.indices.values()) + if hasattr(item, "callspec") + else () + ) + # retrieve the name of the first test function to fail for this class name and index + test_name = _test_failed_incremental[cls_name].get(parametrize_index, None) + # if name found, test has failed for the combination of class name & test name + if test_name is not None: + pytest.xfail("previous test failed ({})".format(test_name)) + + +These two hook implementations work together to abort incremental-marked +tests in a class. Here is a test module example: + +.. code-block:: python + + # content of test_step.py + + import pytest + + + @pytest.mark.incremental + class TestUserHandling: + def test_login(self): + pass + + def test_modification(self): + assert 0 + + def test_deletion(self): + pass + + + def test_normal(): + pass + +If we run this: + +.. code-block:: pytest + + $ pytest -rx + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 4 items + + test_step.py .Fx. [100%] + + ================================= FAILURES ================================= + ____________________ TestUserHandling.test_modification ____________________ + + self = + + def test_modification(self): + > assert 0 + E assert 0 + + test_step.py:11: AssertionError + ========================= short test summary info ========================== + XFAIL test_step.py::TestUserHandling::test_deletion + reason: previous test failed (test_modification) + ================== 1 failed, 2 passed, 1 xfailed in 0.12s ================== + +We'll see that ``test_deletion`` was not executed because ``test_modification`` +failed. It is reported as an "expected failure". + + +Package/Directory-level fixtures (setups) +------------------------------------------------------- + +If you have nested test directories, you can have per-directory fixture scopes +by placing fixture functions in a ``conftest.py`` file in that directory. +You can use all types of fixtures including :ref:`autouse fixtures +` which are the equivalent of xUnit's setup/teardown +concept. It's however recommended to have explicit fixture references in your +tests or test classes rather than relying on implicitly executing +setup/teardown functions, especially if they are far away from the actual tests. + +Here is an example for making a ``db`` fixture available in a directory: + +.. code-block:: python + + # content of a/conftest.py + import pytest + + + class DB: + pass + + + @pytest.fixture(scope="session") + def db(): + return DB() + +and then a test module in that directory: + +.. code-block:: python + + # content of a/test_db.py + def test_a1(db): + assert 0, db # to show value + +another test module: + +.. code-block:: python + + # content of a/test_db2.py + def test_a2(db): + assert 0, db # to show value + +and then a module in a sister directory which will not see +the ``db`` fixture: + +.. code-block:: python + + # content of b/test_error.py + def test_root(db): # no db here, will error out + pass + +We can run this: + +.. code-block:: pytest + + $ pytest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 7 items + + test_step.py .Fx. [ 57%] + a/test_db.py F [ 71%] + a/test_db2.py F [ 85%] + b/test_error.py E [100%] + + ================================== ERRORS ================================== + _______________________ ERROR at setup of test_root ________________________ + file /home/sweet/project/b/test_error.py, line 1 + def test_root(db): # no db here, will error out + E fixture 'db' not found + > available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, doctest_namespace, monkeypatch, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory + > use 'pytest --fixtures [testpath]' for help on them. + + /home/sweet/project/b/test_error.py:1 + ================================= FAILURES ================================= + ____________________ TestUserHandling.test_modification ____________________ + + self = + + def test_modification(self): + > assert 0 + E assert 0 + + test_step.py:11: AssertionError + _________________________________ test_a1 __________________________________ + + db = + + def test_a1(db): + > assert 0, db # to show value + E AssertionError: + E assert 0 + + a/test_db.py:2: AssertionError + _________________________________ test_a2 __________________________________ + + db = + + def test_a2(db): + > assert 0, db # to show value + E AssertionError: + E assert 0 + + a/test_db2.py:2: AssertionError + ========================= short test summary info ========================== + FAILED test_step.py::TestUserHandling::test_modification - assert 0 + FAILED a/test_db.py::test_a1 - AssertionError: assert 0 + E assert 0 + + test_module.py:2: AssertionError + ________________________________ test_fail2 ________________________________ + + def test_fail2(): + > assert 0 + E assert 0 + + test_module.py:6: AssertionError + ========================= short test summary info ========================== + FAILED test_module.py::test_fail1 - assert 0 + FAILED test_module.py::test_fail2 - assert 0 + ============================ 2 failed in 0.12s ============================= + +you will have a "failures" file which contains the failing test ids: + +.. code-block:: bash + + $ cat failures + test_module.py::test_fail1 (PYTEST_TMPDIR/test_fail10) + test_module.py::test_fail2 + +Making test result information available in fixtures +----------------------------------------------------------- + +.. regendoc:wipe + +If you want to make test result reports available in fixture finalizers +here is a little example implemented via a local plugin: + +.. code-block:: python + + # content of conftest.py + + import pytest + + + @pytest.hookimpl(tryfirst=True, hookwrapper=True) + def pytest_runtest_makereport(item, call): + # execute all other hooks to obtain the report object + outcome = yield + rep = outcome.get_result() + + # set a report attribute for each phase of a call, which can + # be "setup", "call", "teardown" + + setattr(item, "rep_" + rep.when, rep) + + + @pytest.fixture + def something(request): + yield + # request.node is an "item" because we use the default + # "function" scope + if request.node.rep_setup.failed: + print("setting up a test failed!", request.node.nodeid) + elif request.node.rep_setup.passed: + if request.node.rep_call.failed: + print("executing test failed", request.node.nodeid) + + +if you then have failing tests: + +.. code-block:: python + + # content of test_module.py + + import pytest + + + @pytest.fixture + def other(): + assert 0 + + + def test_setup_fails(something, other): + pass + + + def test_call_fails(something): + assert 0 + + + def test_fail2(): + assert 0 + +and run it: + +.. code-block:: pytest + + $ pytest -s test_module.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 3 items + + test_module.py Esetting up a test failed! test_module.py::test_setup_fails + Fexecuting test failed test_module.py::test_call_fails + F + + ================================== ERRORS ================================== + ____________________ ERROR at setup of test_setup_fails ____________________ + + @pytest.fixture + def other(): + > assert 0 + E assert 0 + + test_module.py:7: AssertionError + ================================= FAILURES ================================= + _____________________________ test_call_fails ______________________________ + + something = None + + def test_call_fails(something): + > assert 0 + E assert 0 + + test_module.py:15: AssertionError + ________________________________ test_fail2 ________________________________ + + def test_fail2(): + > assert 0 + E assert 0 + + test_module.py:19: AssertionError + ========================= short test summary info ========================== + FAILED test_module.py::test_call_fails - assert 0 + FAILED test_module.py::test_fail2 - assert 0 + ERROR test_module.py::test_setup_fails - assert 0 + ======================== 2 failed, 1 error in 0.12s ======================== + +You'll see that the fixture finalizers could use the precise reporting +information. + +.. _pytest current test env: + +``PYTEST_CURRENT_TEST`` environment variable +-------------------------------------------- + + + +Sometimes a test session might get stuck and there might be no easy way to figure out +which test got stuck, for example if pytest was run in quiet mode (``-q``) or you don't have access to the console +output. This is particularly a problem if the problem happens only sporadically, the famous "flaky" kind of tests. + +``pytest`` sets the :envvar:`PYTEST_CURRENT_TEST` environment variable when running tests, which can be inspected +by process monitoring utilities or libraries like :pypi:`psutil` to discover which test got stuck if necessary: + +.. code-block:: python + + import psutil + + for pid in psutil.pids(): + environ = psutil.Process(pid).environ() + if "PYTEST_CURRENT_TEST" in environ: + print(f'pytest process {pid} running: {environ["PYTEST_CURRENT_TEST"]}') + +During the test session pytest will set ``PYTEST_CURRENT_TEST`` to the current test +:ref:`nodeid ` and the current stage, which can be ``setup``, ``call``, +or ``teardown``. + +For example, when running a single test function named ``test_foo`` from ``foo_module.py``, +``PYTEST_CURRENT_TEST`` will be set to: + +#. ``foo_module.py::test_foo (setup)`` +#. ``foo_module.py::test_foo (call)`` +#. ``foo_module.py::test_foo (teardown)`` + +In that order. + +.. note:: + + The contents of ``PYTEST_CURRENT_TEST`` is meant to be human readable and the actual format + can be changed between releases (even bug fixes) so it shouldn't be relied on for scripting + or automation. + +.. _freezing-pytest: + +Freezing pytest +--------------- + +If you freeze your application using a tool like +`PyInstaller `_ +in order to distribute it to your end-users, it is a good idea to also package +your test runner and run your tests using the frozen application. This way packaging +errors such as dependencies not being included into the executable can be detected early +while also allowing you to send test files to users so they can run them in their +machines, which can be useful to obtain more information about a hard to reproduce bug. + +Fortunately recent ``PyInstaller`` releases already have a custom hook +for pytest, but if you are using another tool to freeze executables +such as ``cx_freeze`` or ``py2exe``, you can use ``pytest.freeze_includes()`` +to obtain the full list of internal pytest modules. How to configure the tools +to find the internal modules varies from tool to tool, however. + +Instead of freezing the pytest runner as a separate executable, you can make +your frozen program work as the pytest runner by some clever +argument handling during program startup. This allows you to +have a single executable, which is usually more convenient. +Please note that the mechanism for plugin discovery used by pytest +(setuptools entry points) doesn't work with frozen executables so pytest +can't find any third party plugins automatically. To include third party plugins +like ``pytest-timeout`` they must be imported explicitly and passed on to pytest.main. + +.. code-block:: python + + # contents of app_main.py + import sys + import pytest_timeout # Third party plugin + + if len(sys.argv) > 1 and sys.argv[1] == "--pytest": + import pytest + + sys.exit(pytest.main(sys.argv[2:], plugins=[pytest_timeout])) + else: + # normal application execution: at this point argv can be parsed + # by your argument-parsing library of choice as usual + ... + + +This allows you to execute tests using the frozen +application with standard ``pytest`` command-line options: + +.. code-block:: bash + + ./app_main --pytest --verbose --tb=long --junitxml=results.xml test-suite/ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/special.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/special.rst new file mode 100644 index 0000000000..ace37c7278 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/special.rst @@ -0,0 +1,84 @@ +A session-fixture which can look at all collected tests +---------------------------------------------------------------- + +A session-scoped fixture effectively has access to all +collected test items. Here is an example of a fixture +function which walks all collected tests and looks +if their test class defines a ``callme`` method and +calls it: + +.. code-block:: python + + # content of conftest.py + + import pytest + + + @pytest.fixture(scope="session", autouse=True) + def callattr_ahead_of_alltests(request): + print("callattr_ahead_of_alltests called") + seen = {None} + session = request.node + for item in session.items: + cls = item.getparent(pytest.Class) + if cls not in seen: + if hasattr(cls.obj, "callme"): + cls.obj.callme() + seen.add(cls) + +test classes may now define a ``callme`` method which +will be called ahead of running any tests: + +.. code-block:: python + + # content of test_module.py + + + class TestHello: + @classmethod + def callme(cls): + print("callme called!") + + def test_method1(self): + print("test_method1 called") + + def test_method2(self): + print("test_method2 called") + + + class TestOther: + @classmethod + def callme(cls): + print("callme other called") + + def test_other(self): + print("test other") + + + # works with unittest as well ... + import unittest + + + class SomeTest(unittest.TestCase): + @classmethod + def callme(self): + print("SomeTest callme called") + + def test_unit1(self): + print("test_unit1 method called") + +If you run this without output capturing: + +.. code-block:: pytest + + $ pytest -q -s test_module.py + callattr_ahead_of_alltests called + callme called! + callme other called + SomeTest callme called + test_method1 called + .test_method2 called + .test other + .test_unit1 method called + . + 4 passed in 0.12s diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/xfail_demo.py b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/xfail_demo.py new file mode 100644 index 0000000000..01e6da1ad2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/example/xfail_demo.py @@ -0,0 +1,38 @@ +import pytest + +xfail = pytest.mark.xfail + + +@xfail +def test_hello(): + assert 0 + + +@xfail(run=False) +def test_hello2(): + assert 0 + + +@xfail("hasattr(os, 'sep')") +def test_hello3(): + assert 0 + + +@xfail(reason="bug 110") +def test_hello4(): + assert 0 + + +@xfail('pytest.__version__[0] != "17"') +def test_hello5(): + assert 0 + + +def test_hello6(): + pytest.xfail("reason") + + +@xfail(raises=IndexError) +def test_hello7(): + x = [] + x[1] = 1 diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/anatomy.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/anatomy.rst new file mode 100644 index 0000000000..e86dd74251 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/anatomy.rst @@ -0,0 +1,46 @@ +.. _test-anatomy: + +Anatomy of a test +================= + +In the simplest terms, a test is meant to look at the result of a particular +behavior, and make sure that result aligns with what you would expect. +Behavior is not something that can be empirically measured, which is why writing +tests can be challenging. + +"Behavior" is the way in which some system **acts in response** to a particular +situation and/or stimuli. But exactly *how* or *why* something is done is not +quite as important as *what* was done. + +You can think of a test as being broken down into four steps: + +1. **Arrange** +2. **Act** +3. **Assert** +4. **Cleanup** + +**Arrange** is where we prepare everything for our test. This means pretty +much everything except for the "**act**". It's lining up the dominoes so that +the **act** can do its thing in one, state-changing step. This can mean +preparing objects, starting/killing services, entering records into a database, +or even things like defining a URL to query, generating some credentials for a +user that doesn't exist yet, or just waiting for some process to finish. + +**Act** is the singular, state-changing action that kicks off the **behavior** +we want to test. This behavior is what carries out the changing of the state of +the system under test (SUT), and it's the resulting changed state that we can +look at to make a judgement about the behavior. This typically takes the form of +a function/method call. + +**Assert** is where we look at that resulting state and check if it looks how +we'd expect after the dust has settled. It's where we gather evidence to say the +behavior does or does not aligns with what we expect. The ``assert`` in our test +is where we take that measurement/observation and apply our judgement to it. If +something should be green, we'd say ``assert thing == "green"``. + +**Cleanup** is where the test picks up after itself, so other tests aren't being +accidentally influenced by it. + +At its core, the test is ultimately the **act** and **assert** steps, with the +**arrange** step only providing the context. **Behavior** exists between **act** +and **assert**. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/fixtures.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/fixtures.rst new file mode 100644 index 0000000000..194e576493 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/fixtures.rst @@ -0,0 +1,174 @@ +.. _about-fixtures: + +About fixtures +=============== + +.. seealso:: :ref:`how-to-fixtures` +.. seealso:: :ref:`Fixtures reference ` + +pytest fixtures are designed to be explicit, modular and scalable. + +What fixtures are +----------------- + +In testing, a `fixture `_ +provides a defined, reliable and consistent context for the tests. This could +include environment (for example a database configured with known parameters) +or content (such as a dataset). + +Fixtures define the steps and data that constitute the *arrange* phase of a +test (see :ref:`test-anatomy`). In pytest, they are functions you define that +serve this purpose. They can also be used to define a test's *act* phase; this +is a powerful technique for designing more complex tests. + +The services, state, or other operating environments set up by fixtures are +accessed by test functions through arguments. For each fixture used by a test +function there is typically a parameter (named after the fixture) in the test +function's definition. + +We can tell pytest that a particular function is a fixture by decorating it with +:py:func:`@pytest.fixture `. Here's a simple example of +what a fixture in pytest might look like: + +.. code-block:: python + + import pytest + + + class Fruit: + def __init__(self, name): + self.name = name + + def __eq__(self, other): + return self.name == other.name + + + @pytest.fixture + def my_fruit(): + return Fruit("apple") + + + @pytest.fixture + def fruit_basket(my_fruit): + return [Fruit("banana"), my_fruit] + + + def test_my_fruit_in_basket(my_fruit, fruit_basket): + assert my_fruit in fruit_basket + +Tests don't have to be limited to a single fixture, either. They can depend on +as many fixtures as you want, and fixtures can use other fixtures, as well. This +is where pytest's fixture system really shines. + + +Improvements over xUnit-style setup/teardown functions +----------------------------------------------------------- + +pytest fixtures offer dramatic improvements over the classic xUnit +style of setup/teardown functions: + +* fixtures have explicit names and are activated by declaring their use + from test functions, modules, classes or whole projects. + +* fixtures are implemented in a modular manner, as each fixture name + triggers a *fixture function* which can itself use other fixtures. + +* fixture management scales from simple unit to complex + functional testing, allowing to parametrize fixtures and tests according + to configuration and component options, or to re-use fixtures + across function, class, module or whole test session scopes. + +* teardown logic can be easily, and safely managed, no matter how many fixtures + are used, without the need to carefully handle errors by hand or micromanage + the order that cleanup steps are added. + +In addition, pytest continues to support :ref:`xunitsetup`. You can mix +both styles, moving incrementally from classic to new style, as you +prefer. You can also start out from existing :ref:`unittest.TestCase +style ` or :ref:`nose based ` projects. + + + +Fixture errors +-------------- + +pytest does its best to put all the fixtures for a given test in a linear order +so that it can see which fixture happens first, second, third, and so on. If an +earlier fixture has a problem, though, and raises an exception, pytest will stop +executing fixtures for that test and mark the test as having an error. + +When a test is marked as having an error, it doesn't mean the test failed, +though. It just means the test couldn't even be attempted because one of the +things it depends on had a problem. + +This is one reason why it's a good idea to cut out as many unnecessary +dependencies as possible for a given test. That way a problem in something +unrelated isn't causing us to have an incomplete picture of what may or may not +have issues. + +Here's a quick example to help explain: + +.. code-block:: python + + import pytest + + + @pytest.fixture + def order(): + return [] + + + @pytest.fixture + def append_first(order): + order.append(1) + + + @pytest.fixture + def append_second(order, append_first): + order.extend([2]) + + + @pytest.fixture(autouse=True) + def append_third(order, append_second): + order += [3] + + + def test_order(order): + assert order == [1, 2, 3] + + +If, for whatever reason, ``order.append(1)`` had a bug and it raises an exception, +we wouldn't be able to know if ``order.extend([2])`` or ``order += [3]`` would +also have problems. After ``append_first`` throws an exception, pytest won't run +any more fixtures for ``test_order``, and it won't even try to run +``test_order`` itself. The only things that would've run would be ``order`` and +``append_first``. + + +Sharing test data +----------------- + +If you want to make test data from files available to your tests, a good way +to do this is by loading these data in a fixture for use by your tests. +This makes use of the automatic caching mechanisms of pytest. + +Another good approach is by adding the data files in the ``tests`` folder. +There are also community plugins available to help to manage this aspect of +testing, e.g. :pypi:`pytest-datadir` and :pypi:`pytest-datafiles`. + +.. _fixtures-signal-cleanup: + +A note about fixture cleanup +---------------------------- + +pytest does not do any special processing for :data:`SIGTERM ` and +:data:`SIGQUIT ` signals (:data:`SIGINT ` is handled naturally +by the Python runtime via :class:`KeyboardInterrupt`), so fixtures that manage external resources which are important +to be cleared when the Python process is terminated (by those signals) might leak resources. + +The reason pytest does not handle those signals to perform fixture cleanup is that signal handlers are global, +and changing them might interfere with the code under execution. + +If fixtures in your suite need special care regarding termination in those scenarios, +see :issue:`this comment <5243#issuecomment-491522595>` in the issue +tracker for a possible workaround. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/flaky.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/flaky.rst new file mode 100644 index 0000000000..50121c7a76 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/flaky.rst @@ -0,0 +1,126 @@ + +Flaky tests +----------- + +A "flaky" test is one that exhibits intermittent or sporadic failure, that seems to have non-deterministic behaviour. Sometimes it passes, sometimes it fails, and it's not clear why. This page discusses pytest features that can help and other general strategies for identifying, fixing or mitigating them. + +Why flaky tests are a problem +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Flaky tests are particularly troublesome when a continuous integration (CI) server is being used, so that all tests must pass before a new code change can be merged. If the test result is not a reliable signal -- that a test failure means the code change broke the test -- developers can become mistrustful of the test results, which can lead to overlooking genuine failures. It is also a source of wasted time as developers must re-run test suites and investigate spurious failures. + + +Potential root causes +^^^^^^^^^^^^^^^^^^^^^ + +System state +~~~~~~~~~~~~ + +Broadly speaking, a flaky test indicates that the test relies on some system state that is not being appropriately controlled - the test environment is not sufficiently isolated. Higher level tests are more likely to be flaky as they rely on more state. + +Flaky tests sometimes appear when a test suite is run in parallel (such as use of pytest-xdist). This can indicate a test is reliant on test ordering. + +- Perhaps a different test is failing to clean up after itself and leaving behind data which causes the flaky test to fail. +- The flaky test is reliant on data from a previous test that doesn't clean up after itself, and in parallel runs that previous test is not always present +- Tests that modify global state typically cannot be run in parallel. + + +Overly strict assertion +~~~~~~~~~~~~~~~~~~~~~~~ + +Overly strict assertions can cause problems with floating point comparison as well as timing issues. :func:`pytest.approx` is useful here. + + +Pytest features +^^^^^^^^^^^^^^^ + +Xfail strict +~~~~~~~~~~~~ + +:ref:`pytest.mark.xfail ref` with ``strict=False`` can be used to mark a test so that its failure does not cause the whole build to break. This could be considered like a manual quarantine, and is rather dangerous to use permanently. + + +PYTEST_CURRENT_TEST +~~~~~~~~~~~~~~~~~~~ + +:envvar:`PYTEST_CURRENT_TEST` may be useful for figuring out "which test got stuck". +See :ref:`pytest current test env` for more details. + + +Plugins +~~~~~~~ + +Rerunning any failed tests can mitigate the negative effects of flaky tests by giving them additional chances to pass, so that the overall build does not fail. Several pytest plugins support this: + +* `flaky `_ +* `pytest-flakefinder `_ - `blog post `_ +* `pytest-rerunfailures `_ +* `pytest-replay `_: This plugin helps to reproduce locally crashes or flaky tests observed during CI runs. + +Plugins to deliberately randomize tests can help expose tests with state problems: + +* `pytest-random-order `_ +* `pytest-randomly `_ + + +Other general strategies +^^^^^^^^^^^^^^^^^^^^^^^^ + +Split up test suites +~~~~~~~~~~~~~~~~~~~~ + +It can be common to split a single test suite into two, such as unit vs integration, and only use the unit test suite as a CI gate. This also helps keep build times manageable as high level tests tend to be slower. However, it means it does become possible for code that breaks the build to be merged, so extra vigilance is needed for monitoring the integration test results. + + +Video/screenshot on failure +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For UI tests these are important for understanding what the state of the UI was when the test failed. pytest-splinter can be used with plugins like pytest-bdd and can `save a screenshot on test failure `_, which can help to isolate the cause. + + +Delete or rewrite the test +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If the functionality is covered by other tests, perhaps the test can be removed. If not, perhaps it can be rewritten at a lower level which will remove the flakiness or make its source more apparent. + + +Quarantine +~~~~~~~~~~ + +Mark Lapierre discusses the `Pros and Cons of Quarantined Tests `_ in a post from 2018. + + + +CI tools that rerun on failure +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Azure Pipelines (the Azure cloud CI/CD tool, formerly Visual Studio Team Services or VSTS) has a feature to `identify flaky tests `_ and rerun failed tests. + + + +Research +^^^^^^^^ + +This is a limited list, please submit an issue or pull request to expand it! + +* Gao, Zebao, Yalan Liang, Myra B. Cohen, Atif M. Memon, and Zhen Wang. "Making system user interactive tests repeatable: When and what should we control?." In *Software Engineering (ICSE), 2015 IEEE/ACM 37th IEEE International Conference on*, vol. 1, pp. 55-65. IEEE, 2015. `PDF `__ +* Palomba, Fabio, and Andy Zaidman. "Does refactoring of test smells induce fixing flaky tests?." In *Software Maintenance and Evolution (ICSME), 2017 IEEE International Conference on*, pp. 1-12. IEEE, 2017. `PDF in Google Drive `__ +* Bell, Jonathan, Owolabi Legunsen, Michael Hilton, Lamyaa Eloussi, Tifany Yung, and Darko Marinov. "DeFlaker: Automatically detecting flaky tests." In *Proceedings of the 2018 International Conference on Software Engineering*. 2018. `PDF `__ + + +Resources +^^^^^^^^^ + +* `Eradicating Non-Determinism in Tests `_ by Martin Fowler, 2011 +* `No more flaky tests on the Go team `_ by Pavan Sudarshan, 2012 +* `The Build That Cried Broken: Building Trust in your Continuous Integration Tests `_ talk (video) by `Angie Jones `_ at SeleniumConf Austin 2017 +* `Test and Code Podcast: Flaky Tests and How to Deal with Them `_ by Brian Okken and Anthony Shaw, 2018 +* Microsoft: + + * `How we approach testing VSTS to enable continuous delivery `_ by Brian Harry MS, 2017 + * `Eliminating Flaky Tests `_ blog and talk (video) by Munil Shah, 2017 + +* Google: + + * `Flaky Tests at Google and How We Mitigate Them `_ by John Micco, 2016 + * `Where do Google's flaky tests come from? `_ by Jeff Listfield, 2017 diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/goodpractices.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/goodpractices.rst new file mode 100644 index 0000000000..32a14991ae --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/goodpractices.rst @@ -0,0 +1,288 @@ +.. highlight:: python +.. _`goodpractices`: + +Good Integration Practices +================================================= + +Install package with pip +------------------------------------------------- + +For development, we recommend you use :mod:`venv` for virtual environments and +:doc:`pip:index` for installing your application and any dependencies, +as well as the ``pytest`` package itself. +This ensures your code and dependencies are isolated from your system Python installation. + +Next, place a ``pyproject.toml`` file in the root of your package: + +.. code-block:: toml + + [build-system] + requires = ["setuptools>=42", "wheel"] + build-backend = "setuptools.build_meta" + +and a ``setup.cfg`` file containing your package's metadata with the following minimum content: + +.. code-block:: ini + + [metadata] + name = PACKAGENAME + + [options] + packages = find: + +where ``PACKAGENAME`` is the name of your package. + +.. note:: + + If your pip version is older than ``21.3``, you'll also need a ``setup.py`` file: + + .. code-block:: python + + from setuptools import setup + + setup() + +You can then install your package in "editable" mode by running from the same directory: + +.. code-block:: bash + + pip install -e . + +which lets you change your source code (both tests and application) and rerun tests at will. + +.. _`test discovery`: +.. _`Python test discovery`: + +Conventions for Python test discovery +------------------------------------------------- + +``pytest`` implements the following standard test discovery: + +* If no arguments are specified then collection starts from :confval:`testpaths` + (if configured) or the current directory. Alternatively, command line arguments + can be used in any combination of directories, file names or node ids. +* Recurse into directories, unless they match :confval:`norecursedirs`. +* In those directories, search for ``test_*.py`` or ``*_test.py`` files, imported by their `test package name`_. +* From those files, collect test items: + + * ``test`` prefixed test functions or methods outside of class + * ``test`` prefixed test functions or methods inside ``Test`` prefixed test classes (without an ``__init__`` method) + +For examples of how to customize your test discovery :doc:`/example/pythoncollection`. + +Within Python modules, ``pytest`` also discovers tests using the standard +:ref:`unittest.TestCase ` subclassing technique. + + +Choosing a test layout / import rules +------------------------------------- + +``pytest`` supports two common test layouts: + +Tests outside application code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Putting tests into an extra directory outside your actual application code +might be useful if you have many functional tests or for other reasons want +to keep tests separate from actual application code (often a good idea): + +.. code-block:: text + + pyproject.toml + setup.cfg + mypkg/ + __init__.py + app.py + view.py + tests/ + test_app.py + test_view.py + ... + +This has the following benefits: + +* Your tests can run against an installed version after executing ``pip install .``. +* Your tests can run against the local copy with an editable install after executing ``pip install --editable .``. +* If you don't use an editable install and are relying on the fact that Python by default puts the current + directory in ``sys.path`` to import your package, you can execute ``python -m pytest`` to execute the tests against the + local copy directly, without using ``pip``. + +.. note:: + + See :ref:`pytest vs python -m pytest` for more information about the difference between calling ``pytest`` and + ``python -m pytest``. + +Note that this scheme has a drawback if you are using ``prepend`` :ref:`import mode ` +(which is the default): your test files must have **unique names**, because +``pytest`` will import them as *top-level* modules since there are no packages +to derive a full package name from. In other words, the test files in the example above will +be imported as ``test_app`` and ``test_view`` top-level modules by adding ``tests/`` to +``sys.path``. + +If you need to have test modules with the same name, you might add ``__init__.py`` files to your +``tests`` folder and subfolders, changing them to packages: + +.. code-block:: text + + pyproject.toml + setup.cfg + mypkg/ + ... + tests/ + __init__.py + foo/ + __init__.py + test_view.py + bar/ + __init__.py + test_view.py + +Now pytest will load the modules as ``tests.foo.test_view`` and ``tests.bar.test_view``, allowing +you to have modules with the same name. But now this introduces a subtle problem: in order to load +the test modules from the ``tests`` directory, pytest prepends the root of the repository to +``sys.path``, which adds the side-effect that now ``mypkg`` is also importable. + +This is problematic if you are using a tool like `tox`_ to test your package in a virtual environment, +because you want to test the *installed* version of your package, not the local code from the repository. + +.. _`src-layout`: + +In this situation, it is **strongly** suggested to use a ``src`` layout where application root package resides in a +sub-directory of your root: + +.. code-block:: text + + pyproject.toml + setup.cfg + src/ + mypkg/ + __init__.py + app.py + view.py + tests/ + __init__.py + foo/ + __init__.py + test_view.py + bar/ + __init__.py + test_view.py + + +This layout prevents a lot of common pitfalls and has many benefits, which are better explained in this excellent +`blog post by Ionel Cristian Mărieș `_. + +.. note:: + The new ``--import-mode=importlib`` (see :ref:`import-modes`) doesn't have + any of the drawbacks above because ``sys.path`` is not changed when importing + test modules, so users that run + into this issue are strongly encouraged to try it and report if the new option works well for them. + + The ``src`` directory layout is still strongly recommended however. + + +Tests as part of application code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Inlining test directories into your application package +is useful if you have direct relation between tests and application modules and +want to distribute them along with your application: + +.. code-block:: text + + pyproject.toml + setup.cfg + mypkg/ + __init__.py + app.py + view.py + test/ + __init__.py + test_app.py + test_view.py + ... + +In this scheme, it is easy to run your tests using the ``--pyargs`` option: + +.. code-block:: bash + + pytest --pyargs mypkg + +``pytest`` will discover where ``mypkg`` is installed and collect tests from there. + +Note that this layout also works in conjunction with the ``src`` layout mentioned in the previous section. + + +.. note:: + + You can use namespace packages (PEP420) for your application + but pytest will still perform `test package name`_ discovery based on the + presence of ``__init__.py`` files. If you use one of the + two recommended file system layouts above but leave away the ``__init__.py`` + files from your directories, it should just work. From + "inlined tests", however, you will need to use absolute imports for + getting at your application code. + +.. _`test package name`: + +.. note:: + + In ``prepend`` and ``append`` import-modes, if pytest finds a ``"a/b/test_module.py"`` + test file while recursing into the filesystem it determines the import name + as follows: + + * determine ``basedir``: this is the first "upward" (towards the root) + directory not containing an ``__init__.py``. If e.g. both ``a`` + and ``b`` contain an ``__init__.py`` file then the parent directory + of ``a`` will become the ``basedir``. + + * perform ``sys.path.insert(0, basedir)`` to make the test module + importable under the fully qualified import name. + + * ``import a.b.test_module`` where the path is determined + by converting path separators ``/`` into "." characters. This means + you must follow the convention of having directory and file + names map directly to the import names. + + The reason for this somewhat evolved importing technique is + that in larger projects multiple test modules might import + from each other and thus deriving a canonical import name helps + to avoid surprises such as a test module getting imported twice. + + With ``--import-mode=importlib`` things are less convoluted because + pytest doesn't need to change ``sys.path`` or ``sys.modules``, making things + much less surprising. + + +.. _`buildout`: http://www.buildout.org/en/latest/ + +.. _`use tox`: + +tox +--- + +Once you are done with your work and want to make sure that your actual +package passes all tests you may want to look into :doc:`tox `, the +virtualenv test automation tool and its :doc:`pytest support `. +tox helps you to setup virtualenv environments with pre-defined +dependencies and then executing a pre-configured test command with +options. It will run tests against the installed package and not +against your source code checkout, helping to detect packaging +glitches. + +Do not run via setuptools +------------------------- + +Integration with setuptools is **not recommended**, +i.e. you should not be using ``python setup.py test`` or ``pytest-runner``, +and may stop working in the future. + +This is deprecated since it depends on deprecated features of setuptools +and relies on features that break security mechanisms in pip. +For example 'setup_requires' and 'tests_require' bypass ``pip --require-hashes``. +For more information and migration instructions, +see the `pytest-runner notice `_. +See also `pypa/setuptools#1684 `_. + +setuptools intends to +`remove the test command `_. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/index.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/index.rst new file mode 100644 index 0000000000..53910f1eb7 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/index.rst @@ -0,0 +1,15 @@ +:orphan: + +.. _explanation: + +Explanation +================ + +.. toctree:: + :maxdepth: 1 + + anatomy + fixtures + goodpractices + flaky + pythonpath diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/pythonpath.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/pythonpath.rst new file mode 100644 index 0000000000..2330356b86 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/explanation/pythonpath.rst @@ -0,0 +1,133 @@ +.. _pythonpath: + +pytest import mechanisms and ``sys.path``/``PYTHONPATH`` +======================================================== + +.. _`import-modes`: + +Import modes +------------ + +pytest as a testing framework needs to import test modules and ``conftest.py`` files for execution. + +Importing files in Python (at least until recently) is a non-trivial processes, often requiring +changing :data:`sys.path`. Some aspects of the +import process can be controlled through the ``--import-mode`` command-line flag, which can assume +these values: + +* ``prepend`` (default): the directory path containing each module will be inserted into the *beginning* + of :py:data:`sys.path` if not already there, and then imported with the :func:`__import__ <__import__>` builtin. + + This requires test module names to be unique when the test directory tree is not arranged in + packages, because the modules will put in :py:data:`sys.modules` after importing. + + This is the classic mechanism, dating back from the time Python 2 was still supported. + +* ``append``: the directory containing each module is appended to the end of :py:data:`sys.path` if not already + there, and imported with ``__import__``. + + This better allows to run test modules against installed versions of a package even if the + package under test has the same import root. For example: + + :: + + testing/__init__.py + testing/test_pkg_under_test.py + pkg_under_test/ + + the tests will run against the installed version + of ``pkg_under_test`` when ``--import-mode=append`` is used whereas + with ``prepend`` they would pick up the local version. This kind of confusion is why + we advocate for using :ref:`src ` layouts. + + Same as ``prepend``, requires test module names to be unique when the test directory tree is + not arranged in packages, because the modules will put in :py:data:`sys.modules` after importing. + +* ``importlib``: new in pytest-6.0, this mode uses :mod:`importlib` to import test modules. This gives full control over the import process, and doesn't require changing :py:data:`sys.path`. + + For this reason this doesn't require test module names to be unique, but also makes test + modules non-importable by each other. + + We intend to make ``importlib`` the default in future releases, depending on feedback. + +``prepend`` and ``append`` import modes scenarios +------------------------------------------------- + +Here's a list of scenarios when using ``prepend`` or ``append`` import modes where pytest needs to +change ``sys.path`` in order to import test modules or ``conftest.py`` files, and the issues users +might encounter because of that. + +Test modules / ``conftest.py`` files inside packages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Consider this file and directory layout:: + + root/ + |- foo/ + |- __init__.py + |- conftest.py + |- bar/ + |- __init__.py + |- tests/ + |- __init__.py + |- test_foo.py + + +When executing: + +.. code-block:: bash + + pytest root/ + +pytest will find ``foo/bar/tests/test_foo.py`` and realize it is part of a package given that +there's an ``__init__.py`` file in the same folder. It will then search upwards until it can find the +last folder which still contains an ``__init__.py`` file in order to find the package *root* (in +this case ``foo/``). To load the module, it will insert ``root/`` to the front of +``sys.path`` (if not there already) in order to load +``test_foo.py`` as the *module* ``foo.bar.tests.test_foo``. + +The same logic applies to the ``conftest.py`` file: it will be imported as ``foo.conftest`` module. + +Preserving the full package name is important when tests live in a package to avoid problems +and allow test modules to have duplicated names. This is also discussed in details in +:ref:`test discovery`. + +Standalone test modules / ``conftest.py`` files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Consider this file and directory layout:: + + root/ + |- foo/ + |- conftest.py + |- bar/ + |- tests/ + |- test_foo.py + + +When executing: + +.. code-block:: bash + + pytest root/ + +pytest will find ``foo/bar/tests/test_foo.py`` and realize it is NOT part of a package given that +there's no ``__init__.py`` file in the same folder. It will then add ``root/foo/bar/tests`` to +``sys.path`` in order to import ``test_foo.py`` as the *module* ``test_foo``. The same is done +with the ``conftest.py`` file by adding ``root/foo`` to ``sys.path`` to import it as ``conftest``. + +For this reason this layout cannot have test modules with the same name, as they all will be +imported in the global import namespace. + +This is also discussed in details in :ref:`test discovery`. + +.. _`pytest vs python -m pytest`: + +Invoking ``pytest`` versus ``python -m pytest`` +----------------------------------------------- + +Running pytest with ``pytest [...]`` instead of ``python -m pytest [...]`` yields nearly +equivalent behaviour, except that the latter will add the current directory to ``sys.path``, which +is standard ``python`` behavior. + +See also :ref:`invoke-python`. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/funcarg_compare.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/funcarg_compare.rst new file mode 100644 index 0000000000..3bf4527cfb --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/funcarg_compare.rst @@ -0,0 +1,230 @@ +:orphan: + +.. _`funcargcompare`: + +pytest-2.3: reasoning for fixture/funcarg evolution +============================================================= + +**Target audience**: Reading this document requires basic knowledge of +python testing, xUnit setup methods and the (previous) basic pytest +funcarg mechanism, see :ref:`historical funcargs and pytest.funcargs`. +If you are new to pytest, then you can simply ignore this +section and read the other sections. + +.. currentmodule:: _pytest + +Shortcomings of the previous ``pytest_funcarg__`` mechanism +-------------------------------------------------------------- + +The pre pytest-2.3 funcarg mechanism calls a factory each time a +funcarg for a test function is required. If a factory wants to +re-use a resource across different scopes, it often used +the ``request.cached_setup()`` helper to manage caching of +resources. Here is a basic example how we could implement +a per-session Database object: + +.. code-block:: python + + # content of conftest.py + class Database: + def __init__(self): + print("database instance created") + + def destroy(self): + print("database instance destroyed") + + + def pytest_funcarg__db(request): + return request.cached_setup( + setup=DataBase, teardown=lambda db: db.destroy, scope="session" + ) + +There are several limitations and difficulties with this approach: + +1. Scoping funcarg resource creation is not straight forward, instead one must + understand the intricate cached_setup() method mechanics. + +2. parametrizing the "db" resource is not straight forward: + you need to apply a "parametrize" decorator or implement a + :py:func:`~hookspec.pytest_generate_tests` hook + calling :py:func:`~pytest.Metafunc.parametrize` which + performs parametrization at the places where the resource + is used. Moreover, you need to modify the factory to use an + ``extrakey`` parameter containing ``request.param`` to the + ``Request.cached_setup`` call. + +3. Multiple parametrized session-scoped resources will be active + at the same time, making it hard for them to affect global state + of the application under test. + +4. there is no way how you can make use of funcarg factories + in xUnit setup methods. + +5. A non-parametrized fixture function cannot use a parametrized + funcarg resource if it isn't stated in the test function signature. + +All of these limitations are addressed with pytest-2.3 and its +improved :ref:`fixture mechanism `. + + +Direct scoping of fixture/funcarg factories +-------------------------------------------------------- + +Instead of calling cached_setup() with a cache scope, you can use the +:ref:`@pytest.fixture ` decorator and directly state +the scope: + +.. code-block:: python + + @pytest.fixture(scope="session") + def db(request): + # factory will only be invoked once per session - + db = DataBase() + request.addfinalizer(db.destroy) # destroy when session is finished + return db + +This factory implementation does not need to call ``cached_setup()`` anymore +because it will only be invoked once per session. Moreover, the +``request.addfinalizer()`` registers a finalizer according to the specified +resource scope on which the factory function is operating. + + +Direct parametrization of funcarg resource factories +---------------------------------------------------------- + +Previously, funcarg factories could not directly cause parametrization. +You needed to specify a ``@parametrize`` decorator on your test function +or implement a ``pytest_generate_tests`` hook to perform +parametrization, i.e. calling a test multiple times with different value +sets. pytest-2.3 introduces a decorator for use on the factory itself: + +.. code-block:: python + + @pytest.fixture(params=["mysql", "pg"]) + def db(request): + ... # use request.param + +Here the factory will be invoked twice (with the respective "mysql" +and "pg" values set as ``request.param`` attributes) and all of +the tests requiring "db" will run twice as well. The "mysql" and +"pg" values will also be used for reporting the test-invocation variants. + +This new way of parametrizing funcarg factories should in many cases +allow to re-use already written factories because effectively +``request.param`` was already used when test functions/classes were +parametrized via +:py:func:`metafunc.parametrize(indirect=True) ` calls. + +Of course it's perfectly fine to combine parametrization and scoping: + +.. code-block:: python + + @pytest.fixture(scope="session", params=["mysql", "pg"]) + def db(request): + if request.param == "mysql": + db = MySQL() + elif request.param == "pg": + db = PG() + request.addfinalizer(db.destroy) # destroy when session is finished + return db + +This would execute all tests requiring the per-session "db" resource twice, +receiving the values created by the two respective invocations to the +factory function. + + +No ``pytest_funcarg__`` prefix when using @fixture decorator +------------------------------------------------------------------- + +When using the ``@fixture`` decorator the name of the function +denotes the name under which the resource can be accessed as a function +argument: + +.. code-block:: python + + @pytest.fixture() + def db(request): + ... + +The name under which the funcarg resource can be requested is ``db``. + +You can still use the "old" non-decorator way of specifying funcarg factories +aka: + +.. code-block:: python + + def pytest_funcarg__db(request): + ... + + +But it is then not possible to define scoping and parametrization. +It is thus recommended to use the factory decorator. + + +solving per-session setup / autouse fixtures +-------------------------------------------------------------- + +pytest for a long time offered a pytest_configure and a pytest_sessionstart +hook which are often used to setup global resources. This suffers from +several problems: + +1. in distributed testing the managing process would setup test resources + that are never needed because it only co-ordinates the test run + activities of the worker processes. + +2. if you only perform a collection (with "--collect-only") + resource-setup will still be executed. + +3. If a pytest_sessionstart is contained in some subdirectories + conftest.py file, it will not be called. This stems from the + fact that this hook is actually used for reporting, in particular + the test-header with platform/custom information. + +Moreover, it was not easy to define a scoped setup from plugins or +conftest files other than to implement a ``pytest_runtest_setup()`` hook +and caring for scoping/caching yourself. And it's virtually impossible +to do this with parametrization as ``pytest_runtest_setup()`` is called +during test execution and parametrization happens at collection time. + +It follows that pytest_configure/session/runtest_setup are often not +appropriate for implementing common fixture needs. Therefore, +pytest-2.3 introduces :ref:`autouse fixtures` which fully +integrate with the generic :ref:`fixture mechanism ` +and obsolete many prior uses of pytest hooks. + +funcargs/fixture discovery now happens at collection time +--------------------------------------------------------------------- + +Since pytest-2.3, discovery of fixture/funcarg factories are taken care of +at collection time. This is more efficient especially for large test suites. +Moreover, a call to "pytest --collect-only" should be able to in the future +show a lot of setup-information and thus presents a nice method to get an +overview of fixture management in your project. + +.. _`compatibility notes`: + +.. _`funcargscompat`: + +Conclusion and compatibility notes +--------------------------------------------------------- + +**funcargs** were originally introduced to pytest-2.0. In pytest-2.3 +the mechanism was extended and refined and is now described as +fixtures: + +* previously funcarg factories were specified with a special + ``pytest_funcarg__NAME`` prefix instead of using the + ``@pytest.fixture`` decorator. + +* Factories received a ``request`` object which managed caching through + ``request.cached_setup()`` calls and allowed using other funcargs via + ``request.getfuncargvalue()`` calls. These intricate APIs made it hard + to do proper parametrization and implement resource caching. The + new :py:func:`pytest.fixture` decorator allows to declare the scope + and let pytest figure things out for you. + +* if you used parametrization and funcarg factories which made use of + ``request.cached_setup()`` it is recommended to invest a few minutes + and simplify your fixture function code to use the :ref:`@pytest.fixture` + decorator instead. This will also allow to take advantage of + the automatic per-resource grouping of tests. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/funcargs.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/funcargs.rst new file mode 100644 index 0000000000..4173675cdd --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/funcargs.rst @@ -0,0 +1,13 @@ + +======================================================= +funcargs: resource injection and parametrization +======================================================= + +pytest-2.3 introduces major refinements to fixture management +of which the funcarg mechanism introduced with pytest-2.0 remains +a core part. The documentation has been refactored as well +and you can read on here: + +- :ref:`fixtures` +- :ref:`parametrize` +- :ref:`funcargcompare` diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/getting-started.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/getting-started.rst new file mode 100644 index 0000000000..5d13a76806 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/getting-started.rst @@ -0,0 +1,257 @@ +.. _get-started: + +Get Started +=================================== + +.. _`getstarted`: +.. _`installation`: + +Install ``pytest`` +---------------------------------------- + +``pytest`` requires: Python 3.6, 3.7, 3.8, 3.9, or PyPy3. + +1. Run the following command in your command line: + +.. code-block:: bash + + pip install -U pytest + +2. Check that you installed the correct version: + +.. code-block:: bash + + $ pytest --version + pytest 7.0.1 + +.. _`simpletest`: + +Create your first test +---------------------------------------------------------- + +Create a new file called ``test_sample.py``, containing a function, and a test: + +.. code-block:: python + + # content of test_sample.py + def func(x): + return x + 1 + + + def test_answer(): + assert func(3) == 5 + +The test + +.. code-block:: pytest + + $ pytest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 1 item + + test_sample.py F [100%] + + ================================= FAILURES ================================= + _______________________________ test_answer ________________________________ + + def test_answer(): + > assert func(3) == 5 + E assert 4 == 5 + E + where 4 = func(3) + + test_sample.py:6: AssertionError + ========================= short test summary info ========================== + FAILED test_sample.py::test_answer - assert 4 == 5 + ============================ 1 failed in 0.12s ============================= + +The ``[100%]`` refers to the overall progress of running all test cases. After it finishes, pytest then shows a failure report because ``func(3)`` does not return ``5``. + +.. note:: + + You can use the ``assert`` statement to verify test expectations. pytest’s :ref:`Advanced assertion introspection ` will intelligently report intermediate values of the assert expression so you can avoid the many names :ref:`of JUnit legacy methods `. + +Run multiple tests +---------------------------------------------------------- + +``pytest`` will run all files of the form test_*.py or \*_test.py in the current directory and its subdirectories. More generally, it follows :ref:`standard test discovery rules `. + + +Assert that a certain exception is raised +-------------------------------------------------------------- + +Use the :ref:`raises ` helper to assert that some code raises an exception: + +.. code-block:: python + + # content of test_sysexit.py + import pytest + + + def f(): + raise SystemExit(1) + + + def test_mytest(): + with pytest.raises(SystemExit): + f() + +Execute the test function with “quiet” reporting mode: + +.. code-block:: pytest + + $ pytest -q test_sysexit.py + . [100%] + 1 passed in 0.12s + +.. note:: + + The ``-q/--quiet`` flag keeps the output brief in this and following examples. + +Group multiple tests in a class +-------------------------------------------------------------- + +.. regendoc:wipe + +Once you develop multiple tests, you may want to group them into a class. pytest makes it easy to create a class containing more than one test: + +.. code-block:: python + + # content of test_class.py + class TestClass: + def test_one(self): + x = "this" + assert "h" in x + + def test_two(self): + x = "hello" + assert hasattr(x, "check") + +``pytest`` discovers all tests following its :ref:`Conventions for Python test discovery `, so it finds both ``test_`` prefixed functions. There is no need to subclass anything, but make sure to prefix your class with ``Test`` otherwise the class will be skipped. We can simply run the module by passing its filename: + +.. code-block:: pytest + + $ pytest -q test_class.py + .F [100%] + ================================= FAILURES ================================= + ____________________________ TestClass.test_two ____________________________ + + self = + + def test_two(self): + x = "hello" + > assert hasattr(x, "check") + E AssertionError: assert False + E + where False = hasattr('hello', 'check') + + test_class.py:8: AssertionError + ========================= short test summary info ========================== + FAILED test_class.py::TestClass::test_two - AssertionError: assert False + 1 failed, 1 passed in 0.12s + +The first test passed and the second failed. You can easily see the intermediate values in the assertion to help you understand the reason for the failure. + +Grouping tests in classes can be beneficial for the following reasons: + + * Test organization + * Sharing fixtures for tests only in that particular class + * Applying marks at the class level and having them implicitly apply to all tests + +Something to be aware of when grouping tests inside classes is that each test has a unique instance of the class. +Having each test share the same class instance would be very detrimental to test isolation and would promote poor test practices. +This is outlined below: + +.. regendoc:wipe + +.. code-block:: python + + # content of test_class_demo.py + class TestClassDemoInstance: + value = 0 + + def test_one(self): + self.value = 1 + assert self.value == 1 + + def test_two(self): + assert self.value == 1 + + +.. code-block:: pytest + + $ pytest -k TestClassDemoInstance -q + .F [100%] + ================================= FAILURES ================================= + ______________________ TestClassDemoInstance.test_two ______________________ + + self = + + def test_two(self): + > assert self.value == 1 + E assert 0 == 1 + E + where 0 = .value + + test_class_demo.py:9: AssertionError + ========================= short test summary info ========================== + FAILED test_class_demo.py::TestClassDemoInstance::test_two - assert 0 == 1 + 1 failed, 1 passed in 0.12s + +Note that attributes added at class level are *class attributes*, so they will be shared between tests. + +Request a unique temporary directory for functional tests +-------------------------------------------------------------- + +``pytest`` provides :std:doc:`Builtin fixtures/function arguments ` to request arbitrary resources, like a unique temporary directory: + +.. code-block:: python + + # content of test_tmp_path.py + def test_needsfiles(tmp_path): + print(tmp_path) + assert 0 + +List the name ``tmp_path`` in the test function signature and ``pytest`` will lookup and call a fixture factory to create the resource before performing the test function call. Before the test runs, ``pytest`` creates a unique-per-test-invocation temporary directory: + +.. code-block:: pytest + + $ pytest -q test_tmp_path.py + F [100%] + ================================= FAILURES ================================= + _____________________________ test_needsfiles ______________________________ + + tmp_path = PosixPath('PYTEST_TMPDIR/test_needsfiles0') + + def test_needsfiles(tmp_path): + print(tmp_path) + > assert 0 + E assert 0 + + test_tmp_path.py:3: AssertionError + --------------------------- Captured stdout call --------------------------- + PYTEST_TMPDIR/test_needsfiles0 + ========================= short test summary info ========================== + FAILED test_tmp_path.py::test_needsfiles - assert 0 + 1 failed in 0.12s + +More info on temporary directory handling is available at :ref:`Temporary directories and files `. + +Find out what kind of builtin :ref:`pytest fixtures ` exist with the command: + +.. code-block:: bash + + pytest --fixtures # shows builtin and custom fixtures + +Note that this command omits fixtures with leading ``_`` unless the ``-v`` option is added. + +Continue reading +------------------------------------- + +Check out additional pytest resources to help you customize tests for your unique workflow: + +* ":ref:`usage`" for command line invocation examples +* ":ref:`existingtestsuite`" for working with pre-existing tests +* ":ref:`mark`" for information on the ``pytest.mark`` mechanism +* ":ref:`fixtures`" for providing a functional baseline to your tests +* ":ref:`plugins`" for managing and writing plugins +* ":ref:`goodpractices`" for virtualenv and test layouts diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/historical-notes.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/historical-notes.rst new file mode 100644 index 0000000000..29ebbd5d19 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/historical-notes.rst @@ -0,0 +1,312 @@ +Historical Notes +================ + +This page lists features or behavior from previous versions of pytest which have changed over the years. They are +kept here as a historical note so users looking at old code can find documentation related to them. + + +.. _marker-revamp: + +Marker revamp and iteration +--------------------------- + +.. versionchanged:: 3.6 + +pytest's marker implementation traditionally worked by simply updating the ``__dict__`` attribute of functions to cumulatively add markers. As a result, markers would unintentionally be passed along class hierarchies in surprising ways. Further, the API for retrieving them was inconsistent, as markers from parameterization would be stored differently than markers applied using the ``@pytest.mark`` decorator and markers added via ``node.add_marker``. + +This state of things made it technically next to impossible to use data from markers correctly without having a deep understanding of the internals, leading to subtle and hard to understand bugs in more advanced usages. + +Depending on how a marker got declared/changed one would get either a ``MarkerInfo`` which might contain markers from sibling classes, +``MarkDecorators`` when marks came from parameterization or from a ``node.add_marker`` call, discarding prior marks. Also ``MarkerInfo`` acts like a single mark, when it in fact represents a merged view on multiple marks with the same name. + +On top of that markers were not accessible in the same way for modules, classes, and functions/methods. +In fact, markers were only accessible in functions, even if they were declared on classes/modules. + +A new API to access markers has been introduced in pytest 3.6 in order to solve the problems with +the initial design, providing the :func:`_pytest.nodes.Node.iter_markers` method to iterate over +markers in a consistent manner and reworking the internals, which solved a great deal of problems +with the initial design. + + +.. _update marker code: + +Updating code +~~~~~~~~~~~~~ + +The old ``Node.get_marker(name)`` function is considered deprecated because it returns an internal ``MarkerInfo`` object +which contains the merged name, ``*args`` and ``**kwargs`` of all the markers which apply to that node. + +In general there are two scenarios on how markers should be handled: + +1. Marks overwrite each other. Order matters but you only want to think of your mark as a single item. E.g. +``log_level('info')`` at a module level can be overwritten by ``log_level('debug')`` for a specific test. + + In this case, use ``Node.get_closest_marker(name)``: + + .. code-block:: python + + # replace this: + marker = item.get_marker("log_level") + if marker: + level = marker.args[0] + + # by this: + marker = item.get_closest_marker("log_level") + if marker: + level = marker.args[0] + +2. Marks compose in an additive manner. E.g. ``skipif(condition)`` marks mean you just want to evaluate all of them, +order doesn't even matter. You probably want to think of your marks as a set here. + + In this case iterate over each mark and handle their ``*args`` and ``**kwargs`` individually. + + .. code-block:: python + + # replace this + skipif = item.get_marker("skipif") + if skipif: + for condition in skipif.args: + # eval condition + ... + + # by this: + for skipif in item.iter_markers("skipif"): + condition = skipif.args[0] + # eval condition + + +If you are unsure or have any questions, please consider opening +:issue:`an issue `. + +Related issues +~~~~~~~~~~~~~~ + +Here is a non-exhaustive list of issues fixed by the new implementation: + +* Marks don't pick up nested classes (:issue:`199`). + +* Markers stain on all related classes (:issue:`568`). + +* Combining marks - args and kwargs calculation (:issue:`2897`). + +* ``request.node.get_marker('name')`` returns ``None`` for markers applied in classes (:issue:`902`). + +* Marks applied in parametrize are stored as markdecorator (:issue:`2400`). + +* Fix marker interaction in a backward incompatible way (:issue:`1670`). + +* Refactor marks to get rid of the current "marks transfer" mechanism (:issue:`2363`). + +* Introduce FunctionDefinition node, use it in generate_tests (:issue:`2522`). + +* Remove named marker attributes and collect markers in items (:issue:`891`). + +* skipif mark from parametrize hides module level skipif mark (:issue:`1540`). + +* skipif + parametrize not skipping tests (:issue:`1296`). + +* Marker transfer incompatible with inheritance (:issue:`535`). + +More details can be found in the :pull:`original PR <3317>`. + +.. note:: + + in a future major release of pytest we will introduce class based markers, + at which point markers will no longer be limited to instances of :py:class:`~_pytest.mark.Mark`. + + +cache plugin integrated into the core +------------------------------------- + + + +The functionality of the :ref:`core cache ` plugin was previously distributed +as a third party plugin named ``pytest-cache``. The core plugin +is compatible regarding command line options and API usage except that you +can only store/receive data between test runs that is json-serializable. + +.. _historical funcargs and pytest.funcargs: + +funcargs and ``pytest_funcarg__`` +--------------------------------- + + + +In versions prior to 2.3 there was no ``@pytest.fixture`` marker +and you had to use a magic ``pytest_funcarg__NAME`` prefix +for the fixture factory. This remains and will remain supported +but is not anymore advertised as the primary means of declaring fixture +functions. + + +``@pytest.yield_fixture`` decorator +----------------------------------- + + + +Prior to version 2.10, in order to use a ``yield`` statement to execute teardown code one +had to mark a fixture using the ``yield_fixture`` marker. From 2.10 onward, normal +fixtures can use ``yield`` directly so the ``yield_fixture`` decorator is no longer needed +and considered deprecated. + + +``[pytest]`` header in ``setup.cfg`` +------------------------------------ + + + +Prior to 3.0, the supported section name was ``[pytest]``. Due to how +this may collide with some distutils commands, the recommended +section name for ``setup.cfg`` files is now ``[tool:pytest]``. + +Note that for ``pytest.ini`` and ``tox.ini`` files the section +name is ``[pytest]``. + + +Applying marks to ``@pytest.mark.parametrize`` parameters +--------------------------------------------------------- + + + +Prior to version 3.1 the supported mechanism for marking values +used the syntax: + +.. code-block:: python + + import pytest + + + @pytest.mark.parametrize( + "test_input,expected", [("3+5", 8), ("2+4", 6), pytest.mark.xfail(("6*9", 42))] + ) + def test_eval(test_input, expected): + assert eval(test_input) == expected + + +This was an initial hack to support the feature but soon was demonstrated to be incomplete, +broken for passing functions or applying multiple marks with the same name but different parameters. + +The old syntax is planned to be removed in pytest-4.0. + + +``@pytest.mark.parametrize`` argument names as a tuple +------------------------------------------------------ + + + +In versions prior to 2.4 one needed to specify the argument +names as a tuple. This remains valid but the simpler ``"name1,name2,..."`` +comma-separated-string syntax is now advertised first because +it's easier to write and produces less line noise. + + +setup: is now an "autouse fixture" +---------------------------------- + + + +During development prior to the pytest-2.3 release the name +``pytest.setup`` was used but before the release it was renamed +and moved to become part of the general fixture mechanism, +namely :ref:`autouse fixtures` + + +.. _string conditions: + +Conditions as strings instead of booleans +----------------------------------------- + + + +Prior to pytest-2.4 the only way to specify skipif/xfail conditions was +to use strings: + +.. code-block:: python + + import sys + + + @pytest.mark.skipif("sys.version_info >= (3,3)") + def test_function(): + ... + +During test function setup the skipif condition is evaluated by calling +``eval('sys.version_info >= (3,0)', namespace)``. The namespace contains +all the module globals, and ``os`` and ``sys`` as a minimum. + +Since pytest-2.4 :ref:`boolean conditions ` are considered preferable +because markers can then be freely imported between test modules. +With strings you need to import not only the marker but all variables +used by the marker, which violates encapsulation. + +The reason for specifying the condition as a string was that ``pytest`` can +report a summary of skip conditions based purely on the condition string. +With conditions as booleans you are required to specify a ``reason`` string. + +Note that string conditions will remain fully supported and you are free +to use them if you have no need for cross-importing markers. + +The evaluation of a condition string in ``pytest.mark.skipif(conditionstring)`` +or ``pytest.mark.xfail(conditionstring)`` takes place in a namespace +dictionary which is constructed as follows: + +* the namespace is initialized by putting the ``sys`` and ``os`` modules + and the pytest ``config`` object into it. + +* updated with the module globals of the test function for which the + expression is applied. + +The pytest ``config`` object allows you to skip based on a test +configuration value which you might have added: + +.. code-block:: python + + @pytest.mark.skipif("not config.getvalue('db')") + def test_function(): + ... + +The equivalent with "boolean conditions" is: + +.. code-block:: python + + @pytest.mark.skipif(not pytest.config.getvalue("db"), reason="--db was not specified") + def test_function(): + pass + +.. note:: + + You cannot use ``pytest.config.getvalue()`` in code + imported before pytest's argument parsing takes place. For example, + ``conftest.py`` files are imported before command line parsing and thus + ``config.getvalue()`` will not execute correctly. + +``pytest.set_trace()`` +---------------------- + + + +Previous to version 2.4 to set a break point in code one needed to use ``pytest.set_trace()``: + +.. code-block:: python + + import pytest + + + def test_function(): + ... + pytest.set_trace() # invoke PDB debugger and tracing + + +This is no longer needed and one can use the native ``import pdb;pdb.set_trace()`` call directly. + +For more details see :ref:`breakpoints`. + +"compat" properties +------------------- + + + +Access of ``Module``, ``Function``, ``Class``, ``Instance``, ``File`` and ``Item`` through ``Node`` instances have long +been documented as deprecated, but started to emit warnings from pytest ``3.9`` and onward. + +Users should just ``import pytest`` and access those objects using the ``pytest`` module. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/history.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/history.rst new file mode 100644 index 0000000000..bb5aa49302 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/history.rst @@ -0,0 +1,145 @@ +History +======= + +pytest has a long and interesting history. The `first commit +`__ +in this repository is from January 2007, and even that commit alone already +tells a lot: The repository originally was from the :pypi:`py` +library (later split off to pytest), and it +originally was a SVN revision, migrated to Mercurial, and finally migrated to +git. + +However, the commit says “create the new development trunk” and is +already quite big: *435 files changed, 58640 insertions(+)*. This is because +pytest originally was born as part of `PyPy `__, to make +it easier to write tests for it. Here's how it evolved from there to its own +project: + + +- Late 2002 / early 2003, `PyPy was + born `__. +- Like that blog post mentioned, from very early on, there was a big + focus on testing. There were various ``testsupport`` files on top of + unittest.py, and as early as June 2003, Holger Krekel (:user:`hpk42`) + `refactored `__ + its test framework to clean things up (``pypy.tool.test``, but still + on top of ``unittest.py``, with nothing pytest-like yet). +- In December 2003, there was `another + iteration `__ + at improving their testing situation, by Stefan Schwarzer, called + ``pypy.tool.newtest``. +- However, it didn’t seem to be around for long, as around June/July + 2004, efforts started on a thing called ``utest``, offering plain + assertions. This seems like the start of something pytest-like, but + unfortunately, it's unclear where the test runner's code was at the time. + The closest thing still around is `this + file `__, + but that doesn’t seem like a complete test runner at all. What can be seen + is that there were `various + efforts `__ + by Laura Creighton and Samuele Pedroni (:user:`pedronis`) at automatically + converting existing tests to the new ``utest`` framework. +- Around the same time, for Europython 2004, @hpk42 `started a + project `__ + originally called “std”, intended to be a “complementary standard + library” - already laying out the principles behind what later became + pytest: + + - current “batteries included” are very useful, but + + - some of them are written in a pretty much java-like style, + especially the unittest-framework + - […] + - the best API is one that doesn’t exist + + […] + + - a testing package should require as few boilerplate code as + possible and offer much flexibility + - it should provide premium quality tracebacks and debugging aid + + […] + + - first of all … forget about limited “assertXYZ APIs” and use the + real thing, e.g.:: + + assert x == y + + - this works with plain python but you get unhelpful “assertion + failed” errors with no information + + - std.utest (magic!) actually reinterprets the assertion expression + and offers detailed information about underlying values + +- In September 2004, the ``py-dev`` mailinglist gets born, which `is + now `__ ``pytest-dev``, + but thankfully with all the original archives still intact. + +- Around September/October 2004, the ``std`` project `was renamed + `__ to + ``py`` and ``std.utest`` became ``py.test``. This is also the first time the + `entire source + code `__, + seems to be available, with much of the API still being around today: + + - ``py.path.local``, which is being phased out of pytest (in favour of + pathlib) some 16-17 years later + - The idea of the collection tree, including ``Collector``, + ``FSCollector``, ``Directory``, ``PyCollector``, ``Module``, + ``Class`` + - Arguments like ``-x`` / ``--exitfirst``, ``-l`` / + ``--showlocals``, ``--fulltrace``, ``--pdb``, ``-S`` / + ``--nocapture`` (``-s`` / ``--capture=off`` today), + ``--collectonly`` (``--collect-only`` today) + +- In the same month, the ``py`` library `gets split off + `__ + from ``PyPy`` + +- It seemed to get rather quiet for a while, and little seemed to happen + between October 2004 (removing ``py`` from PyPy) and January + 2007 (first commit in the now-pytest repository). However, there were + various discussions about features/ideas on the mailinglist, and + :pypi:`a couple of releases ` every + couple of months: + + - March 2006: py 0.8.0-alpha2 + - May 2007: py 0.9.0 + - March 2008: py 0.9.1 (first release to be found `in the pytest + changelog `__!) + - August 2008: py 0.9.2 + +- In August 2009, py 1.0.0 was released, `introducing a lot of + fundamental + features `__: + + - funcargs/fixtures + - A `plugin + architecture `__ + which still looks very much the same today! + - Various `default + plugins `__, + including + `monkeypatch `__ + +- Even back there, the + `FAQ `__ + said: + + Clearly, [a second standard library] was ambitious and the naming has + maybe haunted the project rather than helping it. There may be a + project name change and possibly a split up into different projects + sometime. + + and that finally happened in November 2010, when pytest 2.0.0 `was + released `__ + as a package separate from ``py`` (but still called ``py.test``). + +- In August 2016, pytest 3.0.0 :std:ref:`was released `, + which adds ``pytest`` (rather than ``py.test``) as the recommended + command-line entry point + +Due to this history, it's difficult to answer the question when pytest was started. +It depends what point should really be seen as the start of it all. One +possible interpretation is to pick Europython 2004, i.e. around June/July +2004. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/assert.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/assert.rst new file mode 100644 index 0000000000..cb70db6b8e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/assert.rst @@ -0,0 +1,336 @@ +.. _`assert`: + +How to write and report assertions in tests +================================================== + +.. _`assert with the assert statement`: + +Asserting with the ``assert`` statement +--------------------------------------------------------- + +``pytest`` allows you to use the standard Python ``assert`` for verifying +expectations and values in Python tests. For example, you can write the +following: + +.. code-block:: python + + # content of test_assert1.py + def f(): + return 3 + + + def test_function(): + assert f() == 4 + +to assert that your function returns a certain value. If this assertion fails +you will see the return value of the function call: + +.. code-block:: pytest + + $ pytest test_assert1.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 1 item + + test_assert1.py F [100%] + + ================================= FAILURES ================================= + ______________________________ test_function _______________________________ + + def test_function(): + > assert f() == 4 + E assert 3 == 4 + E + where 3 = f() + + test_assert1.py:6: AssertionError + ========================= short test summary info ========================== + FAILED test_assert1.py::test_function - assert 3 == 4 + ============================ 1 failed in 0.12s ============================= + +``pytest`` has support for showing the values of the most common subexpressions +including calls, attributes, comparisons, and binary and unary +operators. (See :ref:`tbreportdemo`). This allows you to use the +idiomatic python constructs without boilerplate code while not losing +introspection information. + +However, if you specify a message with the assertion like this: + +.. code-block:: python + + assert a % 2 == 0, "value was odd, should be even" + +then no assertion introspection takes places at all and the message +will be simply shown in the traceback. + +See :ref:`assert-details` for more information on assertion introspection. + +.. _`assertraises`: + +Assertions about expected exceptions +------------------------------------------ + +In order to write assertions about raised exceptions, you can use +:func:`pytest.raises` as a context manager like this: + +.. code-block:: python + + import pytest + + + def test_zero_division(): + with pytest.raises(ZeroDivisionError): + 1 / 0 + +and if you need to have access to the actual exception info you may use: + +.. code-block:: python + + def test_recursion_depth(): + with pytest.raises(RuntimeError) as excinfo: + + def f(): + f() + + f() + assert "maximum recursion" in str(excinfo.value) + +``excinfo`` is an :class:`~pytest.ExceptionInfo` instance, which is a wrapper around +the actual exception raised. The main attributes of interest are +``.type``, ``.value`` and ``.traceback``. + +You can pass a ``match`` keyword parameter to the context-manager to test +that a regular expression matches on the string representation of an exception +(similar to the ``TestCase.assertRaisesRegex`` method from ``unittest``): + +.. code-block:: python + + import pytest + + + def myfunc(): + raise ValueError("Exception 123 raised") + + + def test_match(): + with pytest.raises(ValueError, match=r".* 123 .*"): + myfunc() + +The regexp parameter of the ``match`` method is matched with the ``re.search`` +function, so in the above example ``match='123'`` would have worked as +well. + +There's an alternate form of the :func:`pytest.raises` function where you pass +a function that will be executed with the given ``*args`` and ``**kwargs`` and +assert that the given exception is raised: + +.. code-block:: python + + pytest.raises(ExpectedException, func, *args, **kwargs) + +The reporter will provide you with helpful output in case of failures such as *no +exception* or *wrong exception*. + +Note that it is also possible to specify a "raises" argument to +``pytest.mark.xfail``, which checks that the test is failing in a more +specific way than just having any exception raised: + +.. code-block:: python + + @pytest.mark.xfail(raises=IndexError) + def test_f(): + f() + +Using :func:`pytest.raises` is likely to be better for cases where you are +testing exceptions your own code is deliberately raising, whereas using +``@pytest.mark.xfail`` with a check function is probably better for something +like documenting unfixed bugs (where the test describes what "should" happen) +or bugs in dependencies. + + +.. _`assertwarns`: + +Assertions about expected warnings +----------------------------------------- + + + +You can check that code raises a particular warning using +:ref:`pytest.warns `. + + +.. _newreport: + +Making use of context-sensitive comparisons +------------------------------------------------- + + + +``pytest`` has rich support for providing context-sensitive information +when it encounters comparisons. For example: + +.. code-block:: python + + # content of test_assert2.py + def test_set_comparison(): + set1 = set("1308") + set2 = set("8035") + assert set1 == set2 + +if you run this module: + +.. code-block:: pytest + + $ pytest test_assert2.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 1 item + + test_assert2.py F [100%] + + ================================= FAILURES ================================= + ___________________________ test_set_comparison ____________________________ + + def test_set_comparison(): + set1 = set("1308") + set2 = set("8035") + > assert set1 == set2 + E AssertionError: assert {'0', '1', '3', '8'} == {'0', '3', '5', '8'} + E Extra items in the left set: + E '1' + E Extra items in the right set: + E '5' + E Use -v to get the full diff + + test_assert2.py:4: AssertionError + ========================= short test summary info ========================== + FAILED test_assert2.py::test_set_comparison - AssertionError: assert {'0'... + ============================ 1 failed in 0.12s ============================= + +Special comparisons are done for a number of cases: + +* comparing long strings: a context diff is shown +* comparing long sequences: first failing indices +* comparing dicts: different entries + +See the :ref:`reporting demo ` for many more examples. + +Defining your own explanation for failed assertions +--------------------------------------------------- + +It is possible to add your own detailed explanations by implementing +the ``pytest_assertrepr_compare`` hook. + +.. autofunction:: _pytest.hookspec.pytest_assertrepr_compare + :noindex: + +As an example consider adding the following hook in a :ref:`conftest.py ` +file which provides an alternative explanation for ``Foo`` objects: + +.. code-block:: python + + # content of conftest.py + from test_foocompare import Foo + + + def pytest_assertrepr_compare(op, left, right): + if isinstance(left, Foo) and isinstance(right, Foo) and op == "==": + return [ + "Comparing Foo instances:", + " vals: {} != {}".format(left.val, right.val), + ] + +now, given this test module: + +.. code-block:: python + + # content of test_foocompare.py + class Foo: + def __init__(self, val): + self.val = val + + def __eq__(self, other): + return self.val == other.val + + + def test_compare(): + f1 = Foo(1) + f2 = Foo(2) + assert f1 == f2 + +you can run the test module and get the custom output defined in +the conftest file: + +.. code-block:: pytest + + $ pytest -q test_foocompare.py + F [100%] + ================================= FAILURES ================================= + _______________________________ test_compare _______________________________ + + def test_compare(): + f1 = Foo(1) + f2 = Foo(2) + > assert f1 == f2 + E assert Comparing Foo instances: + E vals: 1 != 2 + + test_foocompare.py:12: AssertionError + ========================= short test summary info ========================== + FAILED test_foocompare.py::test_compare - assert Comparing Foo instances: + 1 failed in 0.12s + +.. _assert-details: +.. _`assert introspection`: + +Assertion introspection details +------------------------------- + + +Reporting details about a failing assertion is achieved by rewriting assert +statements before they are run. Rewritten assert statements put introspection +information into the assertion failure message. ``pytest`` only rewrites test +modules directly discovered by its test collection process, so **asserts in +supporting modules which are not themselves test modules will not be rewritten**. + +You can manually enable assertion rewriting for an imported module by calling +:ref:`register_assert_rewrite ` +before you import it (a good place to do that is in your root ``conftest.py``). + +For further information, Benjamin Peterson wrote up `Behind the scenes of pytest's new assertion rewriting `_. + +Assertion rewriting caches files on disk +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``pytest`` will write back the rewritten modules to disk for caching. You can disable +this behavior (for example to avoid leaving stale ``.pyc`` files around in projects that +move files around a lot) by adding this to the top of your ``conftest.py`` file: + +.. code-block:: python + + import sys + + sys.dont_write_bytecode = True + +Note that you still get the benefits of assertion introspection, the only change is that +the ``.pyc`` files won't be cached on disk. + +Additionally, rewriting will silently skip caching if it cannot write new ``.pyc`` files, +i.e. in a read-only filesystem or a zipfile. + + +Disabling assert rewriting +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``pytest`` rewrites test modules on import by using an import +hook to write new ``pyc`` files. Most of the time this works transparently. +However, if you are working with the import machinery yourself, the import hook may +interfere. + +If this is the case you have two options: + +* Disable rewriting for a specific module by adding the string + ``PYTEST_DONT_REWRITE`` to its docstring. + +* Disable rewriting for all modules by using ``--assert=plain``. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/bash-completion.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/bash-completion.rst new file mode 100644 index 0000000000..245dfd6d9a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/bash-completion.rst @@ -0,0 +1,33 @@ + +.. _bash_completion: + +How to set up bash completion +============================= + +When using bash as your shell, ``pytest`` can use argcomplete +(https://argcomplete.readthedocs.io/) for auto-completion. +For this ``argcomplete`` needs to be installed **and** enabled. + +Install argcomplete using: + +.. code-block:: bash + + sudo pip install 'argcomplete>=0.5.7' + +For global activation of all argcomplete enabled python applications run: + +.. code-block:: bash + + sudo activate-global-python-argcomplete + +For permanent (but not global) ``pytest`` activation, use: + +.. code-block:: bash + + register-python-argcomplete pytest >> ~/.bashrc + +For one-time activation of argcomplete for ``pytest`` only, use: + +.. code-block:: bash + + eval "$(register-python-argcomplete pytest)" diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/cache.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/cache.rst new file mode 100644 index 0000000000..e7994645dd --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/cache.rst @@ -0,0 +1,329 @@ +.. _`cache_provider`: +.. _cache: + + +How to re-run failed tests and maintain state between test runs +=============================================================== + + + +Usage +--------- + +The plugin provides two command line options to rerun failures from the +last ``pytest`` invocation: + +* ``--lf``, ``--last-failed`` - to only re-run the failures. +* ``--ff``, ``--failed-first`` - to run the failures first and then the rest of + the tests. + +For cleanup (usually not needed), a ``--cache-clear`` option allows to remove +all cross-session cache contents ahead of a test run. + +Other plugins may access the `config.cache`_ object to set/get +**json encodable** values between ``pytest`` invocations. + +.. note:: + + This plugin is enabled by default, but can be disabled if needed: see + :ref:`cmdunregister` (the internal name for this plugin is + ``cacheprovider``). + + +Rerunning only failures or failures first +----------------------------------------------- + +First, let's create 50 test invocation of which only 2 fail: + +.. code-block:: python + + # content of test_50.py + import pytest + + + @pytest.mark.parametrize("i", range(50)) + def test_num(i): + if i in (17, 25): + pytest.fail("bad luck") + +If you run this for the first time you will see two failures: + +.. code-block:: pytest + + $ pytest -q + .................F.......F........................ [100%] + ================================= FAILURES ================================= + _______________________________ test_num[17] _______________________________ + + i = 17 + + @pytest.mark.parametrize("i", range(50)) + def test_num(i): + if i in (17, 25): + > pytest.fail("bad luck") + E Failed: bad luck + + test_50.py:7: Failed + _______________________________ test_num[25] _______________________________ + + i = 25 + + @pytest.mark.parametrize("i", range(50)) + def test_num(i): + if i in (17, 25): + > pytest.fail("bad luck") + E Failed: bad luck + + test_50.py:7: Failed + ========================= short test summary info ========================== + FAILED test_50.py::test_num[17] - Failed: bad luck + FAILED test_50.py::test_num[25] - Failed: bad luck + 2 failed, 48 passed in 0.12s + +If you then run it with ``--lf``: + +.. code-block:: pytest + + $ pytest --lf + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 2 items + run-last-failure: rerun previous 2 failures + + test_50.py FF [100%] + + ================================= FAILURES ================================= + _______________________________ test_num[17] _______________________________ + + i = 17 + + @pytest.mark.parametrize("i", range(50)) + def test_num(i): + if i in (17, 25): + > pytest.fail("bad luck") + E Failed: bad luck + + test_50.py:7: Failed + _______________________________ test_num[25] _______________________________ + + i = 25 + + @pytest.mark.parametrize("i", range(50)) + def test_num(i): + if i in (17, 25): + > pytest.fail("bad luck") + E Failed: bad luck + + test_50.py:7: Failed + ========================= short test summary info ========================== + FAILED test_50.py::test_num[17] - Failed: bad luck + FAILED test_50.py::test_num[25] - Failed: bad luck + ============================ 2 failed in 0.12s ============================= + +You have run only the two failing tests from the last run, while the 48 passing +tests have not been run ("deselected"). + +Now, if you run with the ``--ff`` option, all tests will be run but the first +previous failures will be executed first (as can be seen from the series +of ``FF`` and dots): + +.. code-block:: pytest + + $ pytest --ff + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 50 items + run-last-failure: rerun previous 2 failures first + + test_50.py FF................................................ [100%] + + ================================= FAILURES ================================= + _______________________________ test_num[17] _______________________________ + + i = 17 + + @pytest.mark.parametrize("i", range(50)) + def test_num(i): + if i in (17, 25): + > pytest.fail("bad luck") + E Failed: bad luck + + test_50.py:7: Failed + _______________________________ test_num[25] _______________________________ + + i = 25 + + @pytest.mark.parametrize("i", range(50)) + def test_num(i): + if i in (17, 25): + > pytest.fail("bad luck") + E Failed: bad luck + + test_50.py:7: Failed + ========================= short test summary info ========================== + FAILED test_50.py::test_num[17] - Failed: bad luck + FAILED test_50.py::test_num[25] - Failed: bad luck + ======================= 2 failed, 48 passed in 0.12s ======================= + +.. _`config.cache`: + +New ``--nf``, ``--new-first`` options: run new tests first followed by the rest +of the tests, in both cases tests are also sorted by the file modified time, +with more recent files coming first. + +Behavior when no tests failed in the last run +--------------------------------------------- + +When no tests failed in the last run, or when no cached ``lastfailed`` data was +found, ``pytest`` can be configured either to run all of the tests or no tests, +using the ``--last-failed-no-failures`` option, which takes one of the following values: + +.. code-block:: bash + + pytest --last-failed --last-failed-no-failures all # run all tests (default behavior) + pytest --last-failed --last-failed-no-failures none # run no tests and exit + +The new config.cache object +-------------------------------- + +.. regendoc:wipe + +Plugins or conftest.py support code can get a cached value using the +pytest ``config`` object. Here is a basic example plugin which +implements a :ref:`fixture ` which re-uses previously created state +across pytest invocations: + +.. code-block:: python + + # content of test_caching.py + import pytest + import time + + + def expensive_computation(): + print("running expensive computation...") + + + @pytest.fixture + def mydata(request): + val = request.config.cache.get("example/value", None) + if val is None: + expensive_computation() + val = 42 + request.config.cache.set("example/value", val) + return val + + + def test_function(mydata): + assert mydata == 23 + +If you run this command for the first time, you can see the print statement: + +.. code-block:: pytest + + $ pytest -q + F [100%] + ================================= FAILURES ================================= + ______________________________ test_function _______________________________ + + mydata = 42 + + def test_function(mydata): + > assert mydata == 23 + E assert 42 == 23 + + test_caching.py:20: AssertionError + -------------------------- Captured stdout setup --------------------------- + running expensive computation... + ========================= short test summary info ========================== + FAILED test_caching.py::test_function - assert 42 == 23 + 1 failed in 0.12s + +If you run it a second time, the value will be retrieved from +the cache and nothing will be printed: + +.. code-block:: pytest + + $ pytest -q + F [100%] + ================================= FAILURES ================================= + ______________________________ test_function _______________________________ + + mydata = 42 + + def test_function(mydata): + > assert mydata == 23 + E assert 42 == 23 + + test_caching.py:20: AssertionError + ========================= short test summary info ========================== + FAILED test_caching.py::test_function - assert 42 == 23 + 1 failed in 0.12s + +See the :fixture:`config.cache fixture ` for more details. + + +Inspecting Cache content +------------------------ + +You can always peek at the content of the cache using the +``--cache-show`` command line option: + +.. code-block:: pytest + + $ pytest --cache-show + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + cachedir: /home/sweet/project/.pytest_cache + --------------------------- cache values for '*' --------------------------- + cache/lastfailed contains: + {'test_caching.py::test_function': True} + cache/nodeids contains: + ['test_caching.py::test_function'] + cache/stepwise contains: + [] + example/value contains: + 42 + + ========================== no tests ran in 0.12s =========================== + +``--cache-show`` takes an optional argument to specify a glob pattern for +filtering: + +.. code-block:: pytest + + $ pytest --cache-show example/* + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + cachedir: /home/sweet/project/.pytest_cache + ----------------------- cache values for 'example/*' ----------------------- + example/value contains: + 42 + + ========================== no tests ran in 0.12s =========================== + +Clearing Cache content +---------------------- + +You can instruct pytest to clear all cache files and values +by adding the ``--cache-clear`` option like this: + +.. code-block:: bash + + pytest --cache-clear + +This is recommended for invocations from Continuous Integration +servers where isolation and correctness is more important +than speed. + + +.. _cache stepwise: + +Stepwise +-------- + +As an alternative to ``--lf -x``, especially for cases where you expect a large part of the test suite will fail, ``--sw``, ``--stepwise`` allows you to fix them one at a time. The test suite will run until the first failure and then stop. At the next invocation, tests will continue from the last failing test and then run until the next failing test. You may use the ``--stepwise-skip`` option to ignore one failing test and stop the test execution on the second failing test instead. This is useful if you get stuck on a failing test and just want to ignore it until later. Providing ``--stepwise-skip`` will also enable ``--stepwise`` implicitly. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/capture-stdout-stderr.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/capture-stdout-stderr.rst new file mode 100644 index 0000000000..9ccea719b6 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/capture-stdout-stderr.rst @@ -0,0 +1,170 @@ + +.. _`captures`: + +How to capture stdout/stderr output +========================================================= + +Default stdout/stderr/stdin capturing behaviour +--------------------------------------------------------- + +During test execution any output sent to ``stdout`` and ``stderr`` is +captured. If a test or a setup method fails its according captured +output will usually be shown along with the failure traceback. (this +behavior can be configured by the ``--show-capture`` command-line option). + +In addition, ``stdin`` is set to a "null" object which will +fail on attempts to read from it because it is rarely desired +to wait for interactive input when running automated tests. + +By default capturing is done by intercepting writes to low level +file descriptors. This allows to capture output from simple +print statements as well as output from a subprocess started by +a test. + +.. _capture-method: + +Setting capturing methods or disabling capturing +------------------------------------------------- + +There are three ways in which ``pytest`` can perform capturing: + +* ``fd`` (file descriptor) level capturing (default): All writes going to the + operating system file descriptors 1 and 2 will be captured. + +* ``sys`` level capturing: Only writes to Python files ``sys.stdout`` + and ``sys.stderr`` will be captured. No capturing of writes to + filedescriptors is performed. + +* ``tee-sys`` capturing: Python writes to ``sys.stdout`` and ``sys.stderr`` + will be captured, however the writes will also be passed-through to + the actual ``sys.stdout`` and ``sys.stderr``. This allows output to be + 'live printed' and captured for plugin use, such as junitxml (new in pytest 5.4). + +.. _`disable capturing`: + +You can influence output capturing mechanisms from the command line: + +.. code-block:: bash + + pytest -s # disable all capturing + pytest --capture=sys # replace sys.stdout/stderr with in-mem files + pytest --capture=fd # also point filedescriptors 1 and 2 to temp file + pytest --capture=tee-sys # combines 'sys' and '-s', capturing sys.stdout/stderr + # and passing it along to the actual sys.stdout/stderr + +.. _printdebugging: + +Using print statements for debugging +--------------------------------------------------- + +One primary benefit of the default capturing of stdout/stderr output +is that you can use print statements for debugging: + +.. code-block:: python + + # content of test_module.py + + + def setup_function(function): + print("setting up", function) + + + def test_func1(): + assert True + + + def test_func2(): + assert False + +and running this module will show you precisely the output +of the failing function and hide the other one: + +.. code-block:: pytest + + $ pytest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 2 items + + test_module.py .F [100%] + + ================================= FAILURES ================================= + ________________________________ test_func2 ________________________________ + + def test_func2(): + > assert False + E assert False + + test_module.py:12: AssertionError + -------------------------- Captured stdout setup --------------------------- + setting up + ========================= short test summary info ========================== + FAILED test_module.py::test_func2 - assert False + ======================= 1 failed, 1 passed in 0.12s ======================== + +Accessing captured output from a test function +--------------------------------------------------- + +The ``capsys``, ``capsysbinary``, ``capfd``, and ``capfdbinary`` fixtures +allow access to stdout/stderr output created during test execution. Here is +an example test function that performs some output related checks: + +.. code-block:: python + + def test_myoutput(capsys): # or use "capfd" for fd-level + print("hello") + sys.stderr.write("world\n") + captured = capsys.readouterr() + assert captured.out == "hello\n" + assert captured.err == "world\n" + print("next") + captured = capsys.readouterr() + assert captured.out == "next\n" + +The ``readouterr()`` call snapshots the output so far - +and capturing will be continued. After the test +function finishes the original streams will +be restored. Using ``capsys`` this way frees your +test from having to care about setting/resetting +output streams and also interacts well with pytest's +own per-test capturing. + +If you want to capture on filedescriptor level you can use +the ``capfd`` fixture which offers the exact +same interface but allows to also capture output from +libraries or subprocesses that directly write to operating +system level output streams (FD1 and FD2). + + + +The return value from ``readouterr`` changed to a ``namedtuple`` with two attributes, ``out`` and ``err``. + + + +If the code under test writes non-textual data, you can capture this using +the ``capsysbinary`` fixture which instead returns ``bytes`` from +the ``readouterr`` method. + + + + +If the code under test writes non-textual data, you can capture this using +the ``capfdbinary`` fixture which instead returns ``bytes`` from +the ``readouterr`` method. The ``capfdbinary`` fixture operates on the +filedescriptor level. + + + + +To temporarily disable capture within a test, both ``capsys`` +and ``capfd`` have a ``disabled()`` method that can be used +as a context manager, disabling capture inside the ``with`` block: + +.. code-block:: python + + def test_disabling_capturing(capsys): + print("this output is captured") + with capsys.disabled(): + print("output not captured, going directly to sys.stdout") + print("this output is also captured") diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/capture-warnings.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/capture-warnings.rst new file mode 100644 index 0000000000..065c11e610 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/capture-warnings.rst @@ -0,0 +1,443 @@ +.. _`warnings`: + +How to capture warnings +======================= + + + +Starting from version ``3.1``, pytest now automatically catches warnings during test execution +and displays them at the end of the session: + +.. code-block:: python + + # content of test_show_warnings.py + import warnings + + + def api_v1(): + warnings.warn(UserWarning("api v1, should use functions from v2")) + return 1 + + + def test_one(): + assert api_v1() == 1 + +Running pytest now produces this output: + +.. code-block:: pytest + + $ pytest test_show_warnings.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 1 item + + test_show_warnings.py . [100%] + + ============================= warnings summary ============================= + test_show_warnings.py::test_one + /home/sweet/project/test_show_warnings.py:5: UserWarning: api v1, should use functions from v2 + warnings.warn(UserWarning("api v1, should use functions from v2")) + + -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html + ======================= 1 passed, 1 warning in 0.12s ======================= + +Controlling warnings +-------------------- + +Similar to Python's `warning filter`_ and :option:`-W option ` flag, pytest provides +its own ``-W`` flag to control which warnings are ignored, displayed, or turned into +errors. See the `warning filter`_ documentation for more +advanced use-cases. + +.. _`warning filter`: https://docs.python.org/3/library/warnings.html#warning-filter + +This code sample shows how to treat any ``UserWarning`` category class of warning +as an error: + +.. code-block:: pytest + + $ pytest -q test_show_warnings.py -W error::UserWarning + F [100%] + ================================= FAILURES ================================= + _________________________________ test_one _________________________________ + + def test_one(): + > assert api_v1() == 1 + + test_show_warnings.py:10: + _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + + def api_v1(): + > warnings.warn(UserWarning("api v1, should use functions from v2")) + E UserWarning: api v1, should use functions from v2 + + test_show_warnings.py:5: UserWarning + ========================= short test summary info ========================== + FAILED test_show_warnings.py::test_one - UserWarning: api v1, should use ... + 1 failed in 0.12s + +The same option can be set in the ``pytest.ini`` or ``pyproject.toml`` file using the +``filterwarnings`` ini option. For example, the configuration below will ignore all +user warnings and specific deprecation warnings matching a regex, but will transform +all other warnings into errors. + +.. code-block:: ini + + # pytest.ini + [pytest] + filterwarnings = + error + ignore::UserWarning + ignore:function ham\(\) is deprecated:DeprecationWarning + +.. code-block:: toml + + # pyproject.toml + [tool.pytest.ini_options] + filterwarnings = [ + "error", + "ignore::UserWarning", + # note the use of single quote below to denote "raw" strings in TOML + 'ignore:function ham\(\) is deprecated:DeprecationWarning', + ] + + +When a warning matches more than one option in the list, the action for the last matching option +is performed. + + +.. _`filterwarnings`: + +``@pytest.mark.filterwarnings`` +------------------------------- + + + +You can use the ``@pytest.mark.filterwarnings`` to add warning filters to specific test items, +allowing you to have finer control of which warnings should be captured at test, class or +even module level: + +.. code-block:: python + + import warnings + + + def api_v1(): + warnings.warn(UserWarning("api v1, should use functions from v2")) + return 1 + + + @pytest.mark.filterwarnings("ignore:api v1") + def test_one(): + assert api_v1() == 1 + + +Filters applied using a mark take precedence over filters passed on the command line or configured +by the ``filterwarnings`` ini option. + +You may apply a filter to all tests of a class by using the ``filterwarnings`` mark as a class +decorator or to all tests in a module by setting the :globalvar:`pytestmark` variable: + +.. code-block:: python + + # turns all warnings into errors for this module + pytestmark = pytest.mark.filterwarnings("error") + + + +*Credits go to Florian Schulze for the reference implementation in the* `pytest-warnings`_ +*plugin.* + +.. _`pytest-warnings`: https://github.com/fschulze/pytest-warnings + +Disabling warnings summary +-------------------------- + +Although not recommended, you can use the ``--disable-warnings`` command-line option to suppress the +warning summary entirely from the test run output. + +Disabling warning capture entirely +---------------------------------- + +This plugin is enabled by default but can be disabled entirely in your ``pytest.ini`` file with: + + .. code-block:: ini + + [pytest] + addopts = -p no:warnings + +Or passing ``-p no:warnings`` in the command-line. This might be useful if your test suites handles warnings +using an external system. + + +.. _`deprecation-warnings`: + +DeprecationWarning and PendingDeprecationWarning +------------------------------------------------ + + +By default pytest will display ``DeprecationWarning`` and ``PendingDeprecationWarning`` warnings from +user code and third-party libraries, as recommended by :pep:`565`. +This helps users keep their code modern and avoid breakages when deprecated warnings are effectively removed. + +Sometimes it is useful to hide some specific deprecation warnings that happen in code that you have no control over +(such as third-party libraries), in which case you might use the warning filters options (ini or marks) to ignore +those warnings. + +For example: + +.. code-block:: ini + + [pytest] + filterwarnings = + ignore:.*U.*mode is deprecated:DeprecationWarning + + +This will ignore all warnings of type ``DeprecationWarning`` where the start of the message matches +the regular expression ``".*U.*mode is deprecated"``. + +.. note:: + + If warnings are configured at the interpreter level, using + the :envvar:`python:PYTHONWARNINGS` environment variable or the + ``-W`` command-line option, pytest will not configure any filters by default. + + Also pytest doesn't follow :pep:`506` suggestion of resetting all warning filters because + it might break test suites that configure warning filters themselves + by calling :func:`warnings.simplefilter` (see :issue:`2430` for an example of that). + + +.. _`ensuring a function triggers a deprecation warning`: + +.. _ensuring_function_triggers: + +Ensuring code triggers a deprecation warning +-------------------------------------------- + +You can also use :func:`pytest.deprecated_call` for checking +that a certain function call triggers a ``DeprecationWarning`` or +``PendingDeprecationWarning``: + +.. code-block:: python + + import pytest + + + def test_myfunction_deprecated(): + with pytest.deprecated_call(): + myfunction(17) + +This test will fail if ``myfunction`` does not issue a deprecation warning +when called with a ``17`` argument. + + + + +.. _`asserting warnings`: + +.. _assertwarnings: + +.. _`asserting warnings with the warns function`: + +.. _warns: + +Asserting warnings with the warns function +------------------------------------------ + + + +You can check that code raises a particular warning using :func:`pytest.warns`, +which works in a similar manner to :ref:`raises `: + +.. code-block:: python + + import warnings + import pytest + + + def test_warning(): + with pytest.warns(UserWarning): + warnings.warn("my warning", UserWarning) + +The test will fail if the warning in question is not raised. The keyword +argument ``match`` to assert that the exception matches a text or regex:: + + >>> with warns(UserWarning, match='must be 0 or None'): + ... warnings.warn("value must be 0 or None", UserWarning) + + >>> with warns(UserWarning, match=r'must be \d+$'): + ... warnings.warn("value must be 42", UserWarning) + + >>> with warns(UserWarning, match=r'must be \d+$'): + ... warnings.warn("this is not here", UserWarning) + Traceback (most recent call last): + ... + Failed: DID NOT WARN. No warnings of type ...UserWarning... were emitted... + +You can also call :func:`pytest.warns` on a function or code string: + +.. code-block:: python + + pytest.warns(expected_warning, func, *args, **kwargs) + pytest.warns(expected_warning, "func(*args, **kwargs)") + +The function also returns a list of all raised warnings (as +``warnings.WarningMessage`` objects), which you can query for +additional information: + +.. code-block:: python + + with pytest.warns(RuntimeWarning) as record: + warnings.warn("another warning", RuntimeWarning) + + # check that only one warning was raised + assert len(record) == 1 + # check that the message matches + assert record[0].message.args[0] == "another warning" + +Alternatively, you can examine raised warnings in detail using the +:ref:`recwarn ` fixture (see below). + + +The :ref:`recwarn ` fixture automatically ensures to reset the warnings +filter at the end of the test, so no global state is leaked. + +.. _`recording warnings`: + +.. _recwarn: + +Recording warnings +------------------ + +You can record raised warnings either using :func:`pytest.warns` or with +the ``recwarn`` fixture. + +To record with :func:`pytest.warns` without asserting anything about the warnings, +pass no arguments as the expected warning type and it will default to a generic Warning: + +.. code-block:: python + + with pytest.warns() as record: + warnings.warn("user", UserWarning) + warnings.warn("runtime", RuntimeWarning) + + assert len(record) == 2 + assert str(record[0].message) == "user" + assert str(record[1].message) == "runtime" + +The ``recwarn`` fixture will record warnings for the whole function: + +.. code-block:: python + + import warnings + + + def test_hello(recwarn): + warnings.warn("hello", UserWarning) + assert len(recwarn) == 1 + w = recwarn.pop(UserWarning) + assert issubclass(w.category, UserWarning) + assert str(w.message) == "hello" + assert w.filename + assert w.lineno + +Both ``recwarn`` and :func:`pytest.warns` return the same interface for recorded +warnings: a WarningsRecorder instance. To view the recorded warnings, you can +iterate over this instance, call ``len`` on it to get the number of recorded +warnings, or index into it to get a particular recorded warning. + +.. currentmodule:: _pytest.warnings + +Full API: :class:`~_pytest.recwarn.WarningsRecorder`. + +.. _`warns use cases`: + +Additional use cases of warnings in tests +----------------------------------------- + +Here are some use cases involving warnings that often come up in tests, and suggestions on how to deal with them: + +- To ensure that **any** warning is emitted, use: + +.. code-block:: python + + with pytest.warns(): + ... + +- To ensure that **no** warnings are emitted, use: + +.. code-block:: python + + with warnings.catch_warnings(): + warnings.simplefilter("error") + ... + +- To suppress warnings, use: + +.. code-block:: python + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + ... + + +.. _custom_failure_messages: + +Custom failure messages +----------------------- + +Recording warnings provides an opportunity to produce custom test +failure messages for when no warnings are issued or other conditions +are met. + +.. code-block:: python + + def test(): + with pytest.warns(Warning) as record: + f() + if not record: + pytest.fail("Expected a warning!") + +If no warnings are issued when calling ``f``, then ``not record`` will +evaluate to ``True``. You can then call :func:`pytest.fail` with a +custom error message. + +.. _internal-warnings: + +Internal pytest warnings +------------------------ + +pytest may generate its own warnings in some situations, such as improper usage or deprecated features. + +For example, pytest will emit a warning if it encounters a class that matches :confval:`python_classes` but also +defines an ``__init__`` constructor, as this prevents the class from being instantiated: + +.. code-block:: python + + # content of test_pytest_warnings.py + class Test: + def __init__(self): + pass + + def test_foo(self): + assert 1 == 1 + +.. code-block:: pytest + + $ pytest test_pytest_warnings.py -q + + ============================= warnings summary ============================= + test_pytest_warnings.py:1 + /home/sweet/project/test_pytest_warnings.py:1: PytestCollectionWarning: cannot collect test class 'Test' because it has a __init__ constructor (from: test_pytest_warnings.py) + class Test: + + -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html + 1 warning in 0.12s + +These warnings might be filtered using the same builtin mechanisms used to filter other types of warnings. + +Please read our :ref:`backwards-compatibility` to learn how we proceed about deprecating and eventually removing +features. + +The full list of warnings is listed in :ref:`the reference documentation `. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/doctest.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/doctest.rst new file mode 100644 index 0000000000..ce0b5a5f64 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/doctest.rst @@ -0,0 +1,312 @@ +.. _doctest: + +How to run doctests +========================================================= + +By default, all files matching the ``test*.txt`` pattern will +be run through the python standard :mod:`doctest` module. You +can change the pattern by issuing: + +.. code-block:: bash + + pytest --doctest-glob="*.rst" + +on the command line. ``--doctest-glob`` can be given multiple times in the command-line. + +If you then have a text file like this: + +.. code-block:: text + + # content of test_example.txt + + hello this is a doctest + >>> x = 3 + >>> x + 3 + +then you can just invoke ``pytest`` directly: + +.. code-block:: pytest + + $ pytest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 1 item + + test_example.txt . [100%] + + ============================ 1 passed in 0.12s ============================= + +By default, pytest will collect ``test*.txt`` files looking for doctest directives, but you +can pass additional globs using the ``--doctest-glob`` option (multi-allowed). + +In addition to text files, you can also execute doctests directly from docstrings of your classes +and functions, including from test modules: + +.. code-block:: python + + # content of mymodule.py + def something(): + """a doctest in a docstring + >>> something() + 42 + """ + return 42 + +.. code-block:: bash + + $ pytest --doctest-modules + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 2 items + + mymodule.py . [ 50%] + test_example.txt . [100%] + + ============================ 2 passed in 0.12s ============================= + +You can make these changes permanent in your project by +putting them into a pytest.ini file like this: + +.. code-block:: ini + + # content of pytest.ini + [pytest] + addopts = --doctest-modules + + +Encoding +-------- + +The default encoding is **UTF-8**, but you can specify the encoding +that will be used for those doctest files using the +``doctest_encoding`` ini option: + +.. code-block:: ini + + # content of pytest.ini + [pytest] + doctest_encoding = latin1 + +.. _using doctest options: + +Using 'doctest' options +----------------------- + +Python's standard :mod:`doctest` module provides some :ref:`options ` +to configure the strictness of doctest tests. In pytest, you can enable those flags using the +configuration file. + +For example, to make pytest ignore trailing whitespaces and ignore +lengthy exception stack traces you can just write: + +.. code-block:: ini + + [pytest] + doctest_optionflags = NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL + +Alternatively, options can be enabled by an inline comment in the doc test +itself: + +.. code-block:: rst + + >>> something_that_raises() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: ... + +pytest also introduces new options: + +* ``ALLOW_UNICODE``: when enabled, the ``u`` prefix is stripped from unicode + strings in expected doctest output. This allows doctests to run in Python 2 + and Python 3 unchanged. + +* ``ALLOW_BYTES``: similarly, the ``b`` prefix is stripped from byte strings + in expected doctest output. + +* ``NUMBER``: when enabled, floating-point numbers only need to match as far as + the precision you have written in the expected doctest output. For example, + the following output would only need to match to 2 decimal places:: + + >>> math.pi + 3.14 + + If you wrote ``3.1416`` then the actual output would need to match to 4 + decimal places; and so on. + + This avoids false positives caused by limited floating-point precision, like + this:: + + Expected: + 0.233 + Got: + 0.23300000000000001 + + ``NUMBER`` also supports lists of floating-point numbers -- in fact, it + matches floating-point numbers appearing anywhere in the output, even inside + a string! This means that it may not be appropriate to enable globally in + ``doctest_optionflags`` in your configuration file. + + .. versionadded:: 5.1 + + +Continue on failure +------------------- + +By default, pytest would report only the first failure for a given doctest. If +you want to continue the test even when you have failures, do: + +.. code-block:: bash + + pytest --doctest-modules --doctest-continue-on-failure + + +Output format +------------- + +You can change the diff output format on failure for your doctests +by using one of standard doctest modules format in options +(see :data:`python:doctest.REPORT_UDIFF`, :data:`python:doctest.REPORT_CDIFF`, +:data:`python:doctest.REPORT_NDIFF`, :data:`python:doctest.REPORT_ONLY_FIRST_FAILURE`): + +.. code-block:: bash + + pytest --doctest-modules --doctest-report none + pytest --doctest-modules --doctest-report udiff + pytest --doctest-modules --doctest-report cdiff + pytest --doctest-modules --doctest-report ndiff + pytest --doctest-modules --doctest-report only_first_failure + + +pytest-specific features +------------------------ + +Some features are provided to make writing doctests easier or with better integration with +your existing test suite. Keep in mind however that by using those features you will make +your doctests incompatible with the standard ``doctests`` module. + +Using fixtures +^^^^^^^^^^^^^^ + +It is possible to use fixtures using the ``getfixture`` helper: + +.. code-block:: text + + # content of example.rst + >>> tmp = getfixture('tmp_path') + >>> ... + >>> + +Note that the fixture needs to be defined in a place visible by pytest, for example, a `conftest.py` +file or plugin; normal python files containing docstrings are not normally scanned for fixtures +unless explicitly configured by :confval:`python_files`. + +Also, the :ref:`usefixtures ` mark and fixtures marked as :ref:`autouse ` are supported +when executing text doctest files. + + +.. _`doctest_namespace`: + +'doctest_namespace' fixture +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``doctest_namespace`` fixture can be used to inject items into the +namespace in which your doctests run. It is intended to be used within +your own fixtures to provide the tests that use them with context. + +``doctest_namespace`` is a standard ``dict`` object into which you +place the objects you want to appear in the doctest namespace: + +.. code-block:: python + + # content of conftest.py + import numpy + + + @pytest.fixture(autouse=True) + def add_np(doctest_namespace): + doctest_namespace["np"] = numpy + +which can then be used in your doctests directly: + +.. code-block:: python + + # content of numpy.py + def arange(): + """ + >>> a = np.arange(10) + >>> len(a) + 10 + """ + pass + +Note that like the normal ``conftest.py``, the fixtures are discovered in the directory tree conftest is in. +Meaning that if you put your doctest with your source code, the relevant conftest.py needs to be in the same directory tree. +Fixtures will not be discovered in a sibling directory tree! + +Skipping tests +^^^^^^^^^^^^^^ + +For the same reasons one might want to skip normal tests, it is also possible to skip +tests inside doctests. + +To skip a single check inside a doctest you can use the standard +:data:`doctest.SKIP` directive: + +.. code-block:: python + + def test_random(y): + """ + >>> random.random() # doctest: +SKIP + 0.156231223 + + >>> 1 + 1 + 2 + """ + +This will skip the first check, but not the second. + +pytest also allows using the standard pytest functions :func:`pytest.skip` and +:func:`pytest.xfail` inside doctests, which might be useful because you can +then skip/xfail tests based on external conditions: + + +.. code-block:: text + + >>> import sys, pytest + >>> if sys.platform.startswith('win'): + ... pytest.skip('this doctest does not work on Windows') + ... + >>> import fcntl + >>> ... + +However using those functions is discouraged because it reduces the readability of the +docstring. + +.. note:: + + :func:`pytest.skip` and :func:`pytest.xfail` behave differently depending + if the doctests are in a Python file (in docstrings) or a text file containing + doctests intermingled with text: + + * Python modules (docstrings): the functions only act in that specific docstring, + letting the other docstrings in the same module execute as normal. + + * Text files: the functions will skip/xfail the checks for the rest of the entire + file. + + +Alternatives +------------ + +While the built-in pytest support provides a good set of functionalities for using +doctests, if you use them extensively you might be interested in those external packages +which add many more features, and include pytest integration: + +* `pytest-doctestplus `__: provides + advanced doctest support and enables the testing of reStructuredText (".rst") files. + +* `Sybil `__: provides a way to test examples in + your documentation by parsing them from the documentation source and evaluating + the parsed examples as part of your normal test run. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/existingtestsuite.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/existingtestsuite.rst new file mode 100644 index 0000000000..9909e7d113 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/existingtestsuite.rst @@ -0,0 +1,34 @@ +.. _existingtestsuite: + +How to use pytest with an existing test suite +============================================== + +Pytest can be used with most existing test suites, but its +behavior differs from other test runners such as :ref:`nose ` or +Python's default unittest framework. + +Before using this section you will want to :ref:`install pytest `. + +Running an existing test suite with pytest +--------------------------------------------- + +Say you want to contribute to an existing repository somewhere. +After pulling the code into your development space using some +flavor of version control and (optionally) setting up a virtualenv +you will want to run: + +.. code-block:: bash + + cd + pip install -e . # Environment dependent alternatives include + # 'python setup.py develop' and 'conda develop' + +in your project root. This will set up a symlink to your code in +site-packages, allowing you to edit your code while your tests +run against it as if it were installed. + +Setting up your project in development mode lets you avoid having to +reinstall every time you want to run your tests, and is less brittle than +mucking about with sys.path to point your tests at local code. + +Also consider using :ref:`tox `. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/failures.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/failures.rst new file mode 100644 index 0000000000..ef87550915 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/failures.rst @@ -0,0 +1,160 @@ +.. _how-to-handle-failures: + +How to handle test failures +============================= + +.. _maxfail: + +Stopping after the first (or N) failures +--------------------------------------------------- + +To stop the testing process after the first (N) failures: + +.. code-block:: bash + + pytest -x # stop after first failure + pytest --maxfail=2 # stop after two failures + + +.. _pdb-option: + +Using :doc:`python:library/pdb` with pytest +------------------------------------------- + +Dropping to :doc:`pdb ` on failures +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Python comes with a builtin Python debugger called :doc:`pdb `. ``pytest`` +allows one to drop into the :doc:`pdb ` prompt via a command line option: + +.. code-block:: bash + + pytest --pdb + +This will invoke the Python debugger on every failure (or KeyboardInterrupt). +Often you might only want to do this for the first failing test to understand +a certain failure situation: + +.. code-block:: bash + + pytest -x --pdb # drop to PDB on first failure, then end test session + pytest --pdb --maxfail=3 # drop to PDB for first three failures + +Note that on any failure the exception information is stored on +``sys.last_value``, ``sys.last_type`` and ``sys.last_traceback``. In +interactive use, this allows one to drop into postmortem debugging with +any debug tool. One can also manually access the exception information, +for example:: + + >>> import sys + >>> sys.last_traceback.tb_lineno + 42 + >>> sys.last_value + AssertionError('assert result == "ok"',) + + +.. _trace-option: + +Dropping to :doc:`pdb ` at the start of a test +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``pytest`` allows one to drop into the :doc:`pdb ` prompt immediately at the start of each test via a command line option: + +.. code-block:: bash + + pytest --trace + +This will invoke the Python debugger at the start of every test. + +.. _breakpoints: + +Setting breakpoints +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded: 2.4.0 + +To set a breakpoint in your code use the native Python ``import pdb;pdb.set_trace()`` call +in your code and pytest automatically disables its output capture for that test: + +* Output capture in other tests is not affected. +* Any prior test output that has already been captured and will be processed as + such. +* Output capture gets resumed when ending the debugger session (via the + ``continue`` command). + + +.. _`breakpoint-builtin`: + +Using the builtin breakpoint function +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Python 3.7 introduces a builtin ``breakpoint()`` function. +Pytest supports the use of ``breakpoint()`` with the following behaviours: + + - When ``breakpoint()`` is called and ``PYTHONBREAKPOINT`` is set to the default value, pytest will use the custom internal PDB trace UI instead of the system default ``Pdb``. + - When tests are complete, the system will default back to the system ``Pdb`` trace UI. + - With ``--pdb`` passed to pytest, the custom internal Pdb trace UI is used with both ``breakpoint()`` and failed tests/unhandled exceptions. + - ``--pdbcls`` can be used to specify a custom debugger class. + + +.. _faulthandler: + +Fault Handler +------------- + +.. versionadded:: 5.0 + +The :mod:`faulthandler` standard module +can be used to dump Python tracebacks on a segfault or after a timeout. + +The module is automatically enabled for pytest runs, unless the ``-p no:faulthandler`` is given +on the command-line. + +Also the :confval:`faulthandler_timeout=X` configuration option can be used +to dump the traceback of all threads if a test takes longer than ``X`` +seconds to finish (not available on Windows). + +.. note:: + + This functionality has been integrated from the external + `pytest-faulthandler `__ plugin, with two + small differences: + + * To disable it, use ``-p no:faulthandler`` instead of ``--no-faulthandler``: the former + can be used with any plugin, so it saves one option. + + * The ``--faulthandler-timeout`` command-line option has become the + :confval:`faulthandler_timeout` configuration option. It can still be configured from + the command-line using ``-o faulthandler_timeout=X``. + + +.. _unraisable: + +Warning about unraisable exceptions and unhandled thread exceptions +------------------------------------------------------------------- + +.. versionadded:: 6.2 + +.. note:: + + These features only work on Python>=3.8. + +Unhandled exceptions are exceptions that are raised in a situation in which +they cannot propagate to a caller. The most common case is an exception raised +in a :meth:`__del__ ` implementation. + +Unhandled thread exceptions are exceptions raised in a :class:`~threading.Thread` +but not handled, causing the thread to terminate uncleanly. + +Both types of exceptions are normally considered bugs, but may go unnoticed +because they don't cause the program itself to crash. Pytest detects these +conditions and issues a warning that is visible in the test run summary. + +The plugins are automatically enabled for pytest runs, unless the +``-p no:unraisableexception`` (for unraisable exceptions) and +``-p no:threadexception`` (for thread exceptions) options are given on the +command-line. + +The warnings may be silenced selectively using the :ref:`pytest.mark.filterwarnings ref` +mark. The warning categories are :class:`pytest.PytestUnraisableExceptionWarning` and +:class:`pytest.PytestUnhandledThreadExceptionWarning`. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/fixtures.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/fixtures.rst new file mode 100644 index 0000000000..0801387745 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/fixtures.rst @@ -0,0 +1,1887 @@ +.. _how-to-fixtures: + +How to use fixtures +==================== + +.. seealso:: :ref:`about-fixtures` +.. seealso:: :ref:`Fixtures reference ` + + +"Requesting" fixtures +--------------------- + +At a basic level, test functions request fixtures they require by declaring +them as arguments. + +When pytest goes to run a test, it looks at the parameters in that test +function's signature, and then searches for fixtures that have the same names as +those parameters. Once pytest finds them, it runs those fixtures, captures what +they returned (if anything), and passes those objects into the test function as +arguments. + + +Quick example +^^^^^^^^^^^^^ + +.. code-block:: python + + import pytest + + + class Fruit: + def __init__(self, name): + self.name = name + self.cubed = False + + def cube(self): + self.cubed = True + + + class FruitSalad: + def __init__(self, *fruit_bowl): + self.fruit = fruit_bowl + self._cube_fruit() + + def _cube_fruit(self): + for fruit in self.fruit: + fruit.cube() + + + # Arrange + @pytest.fixture + def fruit_bowl(): + return [Fruit("apple"), Fruit("banana")] + + + def test_fruit_salad(fruit_bowl): + # Act + fruit_salad = FruitSalad(*fruit_bowl) + + # Assert + assert all(fruit.cubed for fruit in fruit_salad.fruit) + +In this example, ``test_fruit_salad`` "**requests**" ``fruit_bowl`` (i.e. +``def test_fruit_salad(fruit_bowl):``), and when pytest sees this, it will +execute the ``fruit_bowl`` fixture function and pass the object it returns into +``test_fruit_salad`` as the ``fruit_bowl`` argument. + +Here's roughly +what's happening if we were to do it by hand: + +.. code-block:: python + + def fruit_bowl(): + return [Fruit("apple"), Fruit("banana")] + + + def test_fruit_salad(fruit_bowl): + # Act + fruit_salad = FruitSalad(*fruit_bowl) + + # Assert + assert all(fruit.cubed for fruit in fruit_salad.fruit) + + + # Arrange + bowl = fruit_bowl() + test_fruit_salad(fruit_bowl=bowl) + + +Fixtures can **request** other fixtures +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +One of pytest's greatest strengths is its extremely flexible fixture system. It +allows us to boil down complex requirements for tests into more simple and +organized functions, where we only need to have each one describe the things +they are dependent on. We'll get more into this further down, but for now, +here's a quick example to demonstrate how fixtures can use other fixtures: + +.. code-block:: python + + # contents of test_append.py + import pytest + + + # Arrange + @pytest.fixture + def first_entry(): + return "a" + + + # Arrange + @pytest.fixture + def order(first_entry): + return [first_entry] + + + def test_string(order): + # Act + order.append("b") + + # Assert + assert order == ["a", "b"] + + +Notice that this is the same example from above, but very little changed. The +fixtures in pytest **request** fixtures just like tests. All the same +**requesting** rules apply to fixtures that do for tests. Here's how this +example would work if we did it by hand: + +.. code-block:: python + + def first_entry(): + return "a" + + + def order(first_entry): + return [first_entry] + + + def test_string(order): + # Act + order.append("b") + + # Assert + assert order == ["a", "b"] + + + entry = first_entry() + the_list = order(first_entry=entry) + test_string(order=the_list) + +Fixtures are reusable +^^^^^^^^^^^^^^^^^^^^^ + +One of the things that makes pytest's fixture system so powerful, is that it +gives us the ability to define a generic setup step that can be reused over and +over, just like a normal function would be used. Two different tests can request +the same fixture and have pytest give each test their own result from that +fixture. + +This is extremely useful for making sure tests aren't affected by each other. We +can use this system to make sure each test gets its own fresh batch of data and +is starting from a clean state so it can provide consistent, repeatable results. + +Here's an example of how this can come in handy: + +.. code-block:: python + + # contents of test_append.py + import pytest + + + # Arrange + @pytest.fixture + def first_entry(): + return "a" + + + # Arrange + @pytest.fixture + def order(first_entry): + return [first_entry] + + + def test_string(order): + # Act + order.append("b") + + # Assert + assert order == ["a", "b"] + + + def test_int(order): + # Act + order.append(2) + + # Assert + assert order == ["a", 2] + + +Each test here is being given its own copy of that ``list`` object, +which means the ``order`` fixture is getting executed twice (the same +is true for the ``first_entry`` fixture). If we were to do this by hand as +well, it would look something like this: + +.. code-block:: python + + def first_entry(): + return "a" + + + def order(first_entry): + return [first_entry] + + + def test_string(order): + # Act + order.append("b") + + # Assert + assert order == ["a", "b"] + + + def test_int(order): + # Act + order.append(2) + + # Assert + assert order == ["a", 2] + + + entry = first_entry() + the_list = order(first_entry=entry) + test_string(order=the_list) + + entry = first_entry() + the_list = order(first_entry=entry) + test_int(order=the_list) + +A test/fixture can **request** more than one fixture at a time +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Tests and fixtures aren't limited to **requesting** a single fixture at a time. +They can request as many as they like. Here's another quick example to +demonstrate: + +.. code-block:: python + + # contents of test_append.py + import pytest + + + # Arrange + @pytest.fixture + def first_entry(): + return "a" + + + # Arrange + @pytest.fixture + def second_entry(): + return 2 + + + # Arrange + @pytest.fixture + def order(first_entry, second_entry): + return [first_entry, second_entry] + + + # Arrange + @pytest.fixture + def expected_list(): + return ["a", 2, 3.0] + + + def test_string(order, expected_list): + # Act + order.append(3.0) + + # Assert + assert order == expected_list + +Fixtures can be **requested** more than once per test (return values are cached) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Fixtures can also be **requested** more than once during the same test, and +pytest won't execute them again for that test. This means we can **request** +fixtures in multiple fixtures that are dependent on them (and even again in the +test itself) without those fixtures being executed more than once. + +.. code-block:: python + + # contents of test_append.py + import pytest + + + # Arrange + @pytest.fixture + def first_entry(): + return "a" + + + # Arrange + @pytest.fixture + def order(): + return [] + + + # Act + @pytest.fixture + def append_first(order, first_entry): + return order.append(first_entry) + + + def test_string_only(append_first, order, first_entry): + # Assert + assert order == [first_entry] + +If a **requested** fixture was executed once for every time it was **requested** +during a test, then this test would fail because both ``append_first`` and +``test_string_only`` would see ``order`` as an empty list (i.e. ``[]``), but +since the return value of ``order`` was cached (along with any side effects +executing it may have had) after the first time it was called, both the test and +``append_first`` were referencing the same object, and the test saw the effect +``append_first`` had on that object. + +.. _`autouse`: +.. _`autouse fixtures`: + +Autouse fixtures (fixtures you don't have to request) +----------------------------------------------------- + +Sometimes you may want to have a fixture (or even several) that you know all +your tests will depend on. "Autouse" fixtures are a convenient way to make all +tests automatically **request** them. This can cut out a +lot of redundant **requests**, and can even provide more advanced fixture usage +(more on that further down). + +We can make a fixture an autouse fixture by passing in ``autouse=True`` to the +fixture's decorator. Here's a simple example for how they can be used: + +.. code-block:: python + + # contents of test_append.py + import pytest + + + @pytest.fixture + def first_entry(): + return "a" + + + @pytest.fixture + def order(first_entry): + return [] + + + @pytest.fixture(autouse=True) + def append_first(order, first_entry): + return order.append(first_entry) + + + def test_string_only(order, first_entry): + assert order == [first_entry] + + + def test_string_and_int(order, first_entry): + order.append(2) + assert order == [first_entry, 2] + +In this example, the ``append_first`` fixture is an autouse fixture. Because it +happens automatically, both tests are affected by it, even though neither test +**requested** it. That doesn't mean they *can't* be **requested** though; just +that it isn't *necessary*. + +.. _smtpshared: + +Scope: sharing fixtures across classes, modules, packages or session +-------------------------------------------------------------------- + +.. regendoc:wipe + +Fixtures requiring network access depend on connectivity and are +usually time-expensive to create. Extending the previous example, we +can add a ``scope="module"`` parameter to the +:py:func:`@pytest.fixture ` invocation +to cause a ``smtp_connection`` fixture function, responsible to create a connection to a preexisting SMTP server, to only be invoked +once per test *module* (the default is to invoke once per test *function*). +Multiple test functions in a test module will thus +each receive the same ``smtp_connection`` fixture instance, thus saving time. +Possible values for ``scope`` are: ``function``, ``class``, ``module``, ``package`` or ``session``. + +The next example puts the fixture function into a separate ``conftest.py`` file +so that tests from multiple test modules in the directory can +access the fixture function: + +.. code-block:: python + + # content of conftest.py + import pytest + import smtplib + + + @pytest.fixture(scope="module") + def smtp_connection(): + return smtplib.SMTP("smtp.gmail.com", 587, timeout=5) + + +.. code-block:: python + + # content of test_module.py + + + def test_ehlo(smtp_connection): + response, msg = smtp_connection.ehlo() + assert response == 250 + assert b"smtp.gmail.com" in msg + assert 0 # for demo purposes + + + def test_noop(smtp_connection): + response, msg = smtp_connection.noop() + assert response == 250 + assert 0 # for demo purposes + +Here, the ``test_ehlo`` needs the ``smtp_connection`` fixture value. pytest +will discover and call the :py:func:`@pytest.fixture ` +marked ``smtp_connection`` fixture function. Running the test looks like this: + +.. code-block:: pytest + + $ pytest test_module.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 2 items + + test_module.py FF [100%] + + ================================= FAILURES ================================= + ________________________________ test_ehlo _________________________________ + + smtp_connection = + + def test_ehlo(smtp_connection): + response, msg = smtp_connection.ehlo() + assert response == 250 + assert b"smtp.gmail.com" in msg + > assert 0 # for demo purposes + E assert 0 + + test_module.py:7: AssertionError + ________________________________ test_noop _________________________________ + + smtp_connection = + + def test_noop(smtp_connection): + response, msg = smtp_connection.noop() + assert response == 250 + > assert 0 # for demo purposes + E assert 0 + + test_module.py:13: AssertionError + ========================= short test summary info ========================== + FAILED test_module.py::test_ehlo - assert 0 + FAILED test_module.py::test_noop - assert 0 + ============================ 2 failed in 0.12s ============================= + +You see the two ``assert 0`` failing and more importantly you can also see +that the **exactly same** ``smtp_connection`` object was passed into the +two test functions because pytest shows the incoming argument values in the +traceback. As a result, the two test functions using ``smtp_connection`` run +as quick as a single one because they reuse the same instance. + +If you decide that you rather want to have a session-scoped ``smtp_connection`` +instance, you can simply declare it: + +.. code-block:: python + + @pytest.fixture(scope="session") + def smtp_connection(): + # the returned fixture value will be shared for + # all tests requesting it + ... + + +Fixture scopes +^^^^^^^^^^^^^^ + +Fixtures are created when first requested by a test, and are destroyed based on their ``scope``: + +* ``function``: the default scope, the fixture is destroyed at the end of the test. +* ``class``: the fixture is destroyed during teardown of the last test in the class. +* ``module``: the fixture is destroyed during teardown of the last test in the module. +* ``package``: the fixture is destroyed during teardown of the last test in the package. +* ``session``: the fixture is destroyed at the end of the test session. + +.. note:: + + Pytest only caches one instance of a fixture at a time, which + means that when using a parametrized fixture, pytest may invoke a fixture more than once in + the given scope. + +.. _dynamic scope: + +Dynamic scope +^^^^^^^^^^^^^ + +.. versionadded:: 5.2 + +In some cases, you might want to change the scope of the fixture without changing the code. +To do that, pass a callable to ``scope``. The callable must return a string with a valid scope +and will be executed only once - during the fixture definition. It will be called with two +keyword arguments - ``fixture_name`` as a string and ``config`` with a configuration object. + +This can be especially useful when dealing with fixtures that need time for setup, like spawning +a docker container. You can use the command-line argument to control the scope of the spawned +containers for different environments. See the example below. + +.. code-block:: python + + def determine_scope(fixture_name, config): + if config.getoption("--keep-containers", None): + return "session" + return "function" + + + @pytest.fixture(scope=determine_scope) + def docker_container(): + yield spawn_container() + + + +.. _`finalization`: + +Teardown/Cleanup (AKA Fixture finalization) +------------------------------------------- + +When we run our tests, we'll want to make sure they clean up after themselves so +they don't mess with any other tests (and also so that we don't leave behind a +mountain of test data to bloat the system). Fixtures in pytest offer a very +useful teardown system, which allows us to define the specific steps necessary +for each fixture to clean up after itself. + +This system can be leveraged in two ways. + +.. _`yield fixtures`: + +1. ``yield`` fixtures (recommended) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. regendoc: wipe + +"Yield" fixtures ``yield`` instead of ``return``. With these +fixtures, we can run some code and pass an object back to the requesting +fixture/test, just like with the other fixtures. The only differences are: + +1. ``return`` is swapped out for ``yield``. +2. Any teardown code for that fixture is placed *after* the ``yield``. + +Once pytest figures out a linear order for the fixtures, it will run each one up +until it returns or yields, and then move on to the next fixture in the list to +do the same thing. + +Once the test is finished, pytest will go back down the list of fixtures, but in +the *reverse order*, taking each one that yielded, and running the code inside +it that was *after* the ``yield`` statement. + +As a simple example, consider this basic email module: + +.. code-block:: python + + # content of emaillib.py + class MailAdminClient: + def create_user(self): + return MailUser() + + def delete_user(self, user): + # do some cleanup + pass + + + class MailUser: + def __init__(self): + self.inbox = [] + + def send_email(self, email, other): + other.inbox.append(email) + + def clear_mailbox(self): + self.inbox.clear() + + + class Email: + def __init__(self, subject, body): + self.subject = subject + self.body = body + +Let's say we want to test sending email from one user to another. We'll have to +first make each user, then send the email from one user to the other, and +finally assert that the other user received that message in their inbox. If we +want to clean up after the test runs, we'll likely have to make sure the other +user's mailbox is emptied before deleting that user, otherwise the system may +complain. + +Here's what that might look like: + +.. code-block:: python + + # content of test_emaillib.py + import pytest + + from emaillib import Email, MailAdminClient + + + @pytest.fixture + def mail_admin(): + return MailAdminClient() + + + @pytest.fixture + def sending_user(mail_admin): + user = mail_admin.create_user() + yield user + mail_admin.delete_user(user) + + + @pytest.fixture + def receiving_user(mail_admin): + user = mail_admin.create_user() + yield user + mail_admin.delete_user(user) + + + def test_email_received(sending_user, receiving_user): + email = Email(subject="Hey!", body="How's it going?") + sending_user.send_email(email, receiving_user) + assert email in receiving_user.inbox + +Because ``receiving_user`` is the last fixture to run during setup, it's the first to run +during teardown. + +There is a risk that even having the order right on the teardown side of things +doesn't guarantee a safe cleanup. That's covered in a bit more detail in +:ref:`safe teardowns`. + +.. code-block:: pytest + + $ pytest -q test_emaillib.py + . [100%] + 1 passed in 0.12s + +Handling errors for yield fixture +""""""""""""""""""""""""""""""""" + +If a yield fixture raises an exception before yielding, pytest won't try to run +the teardown code after that yield fixture's ``yield`` statement. But, for every +fixture that has already run successfully for that test, pytest will still +attempt to tear them down as it normally would. + +2. Adding finalizers directly +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +While yield fixtures are considered to be the cleaner and more straightforward +option, there is another choice, and that is to add "finalizer" functions +directly to the test's `request-context`_ object. It brings a similar result as +yield fixtures, but requires a bit more verbosity. + +In order to use this approach, we have to request the `request-context`_ object +(just like we would request another fixture) in the fixture we need to add +teardown code for, and then pass a callable, containing that teardown code, to +its ``addfinalizer`` method. + +We have to be careful though, because pytest will run that finalizer once it's +been added, even if that fixture raises an exception after adding the finalizer. +So to make sure we don't run the finalizer code when we wouldn't need to, we +would only add the finalizer once the fixture would have done something that +we'd need to teardown. + +Here's how the previous example would look using the ``addfinalizer`` method: + +.. code-block:: python + + # content of test_emaillib.py + import pytest + + from emaillib import Email, MailAdminClient + + + @pytest.fixture + def mail_admin(): + return MailAdminClient() + + + @pytest.fixture + def sending_user(mail_admin): + user = mail_admin.create_user() + yield user + mail_admin.delete_user(user) + + + @pytest.fixture + def receiving_user(mail_admin, request): + user = mail_admin.create_user() + + def delete_user(): + mail_admin.delete_user(user) + + request.addfinalizer(delete_user) + return user + + + @pytest.fixture + def email(sending_user, receiving_user, request): + _email = Email(subject="Hey!", body="How's it going?") + sending_user.send_email(_email, receiving_user) + + def empty_mailbox(): + receiving_user.clear_mailbox() + + request.addfinalizer(empty_mailbox) + return _email + + + def test_email_received(receiving_user, email): + assert email in receiving_user.inbox + + +It's a bit longer than yield fixtures and a bit more complex, but it +does offer some nuances for when you're in a pinch. + +.. code-block:: pytest + + $ pytest -q test_emaillib.py + . [100%] + 1 passed in 0.12s + +.. _`safe teardowns`: + +Safe teardowns +-------------- + +The fixture system of pytest is *very* powerful, but it's still being run by a +computer, so it isn't able to figure out how to safely teardown everything we +throw at it. If we aren't careful, an error in the wrong spot might leave stuff +from our tests behind, and that can cause further issues pretty quickly. + +For example, consider the following tests (based off of the mail example from +above): + +.. code-block:: python + + # content of test_emaillib.py + import pytest + + from emaillib import Email, MailAdminClient + + + @pytest.fixture + def setup(): + mail_admin = MailAdminClient() + sending_user = mail_admin.create_user() + receiving_user = mail_admin.create_user() + email = Email(subject="Hey!", body="How's it going?") + sending_user.send_email(email, receiving_user) + yield receiving_user, email + receiving_user.clear_mailbox() + mail_admin.delete_user(sending_user) + mail_admin.delete_user(receiving_user) + + + def test_email_received(setup): + receiving_user, email = setup + assert email in receiving_user.inbox + +This version is a lot more compact, but it's also harder to read, doesn't have a +very descriptive fixture name, and none of the fixtures can be reused easily. + +There's also a more serious issue, which is that if any of those steps in the +setup raise an exception, none of the teardown code will run. + +One option might be to go with the ``addfinalizer`` method instead of yield +fixtures, but that might get pretty complex and difficult to maintain (and it +wouldn't be compact anymore). + +.. code-block:: pytest + + $ pytest -q test_emaillib.py + . [100%] + 1 passed in 0.12s + +.. _`safe fixture structure`: + +Safe fixture structure +^^^^^^^^^^^^^^^^^^^^^^ + +The safest and simplest fixture structure requires limiting fixtures to only +making one state-changing action each, and then bundling them together with +their teardown code, as :ref:`the email examples above ` showed. + +The chance that a state-changing operation can fail but still modify state is +negligible, as most of these operations tend to be `transaction +`_-based (at least at the +level of testing where state could be left behind). So if we make sure that any +successful state-changing action gets torn down by moving it to a separate +fixture function and separating it from other, potentially failing +state-changing actions, then our tests will stand the best chance at leaving +the test environment the way they found it. + +For an example, let's say we have a website with a login page, and we have +access to an admin API where we can generate users. For our test, we want to: + +1. Create a user through that admin API +2. Launch a browser using Selenium +3. Go to the login page of our site +4. Log in as the user we created +5. Assert that their name is in the header of the landing page + +We wouldn't want to leave that user in the system, nor would we want to leave +that browser session running, so we'll want to make sure the fixtures that +create those things clean up after themselves. + +Here's what that might look like: + +.. note:: + + For this example, certain fixtures (i.e. ``base_url`` and + ``admin_credentials``) are implied to exist elsewhere. So for now, let's + assume they exist, and we're just not looking at them. + +.. code-block:: python + + from uuid import uuid4 + from urllib.parse import urljoin + + from selenium.webdriver import Chrome + import pytest + + from src.utils.pages import LoginPage, LandingPage + from src.utils import AdminApiClient + from src.utils.data_types import User + + + @pytest.fixture + def admin_client(base_url, admin_credentials): + return AdminApiClient(base_url, **admin_credentials) + + + @pytest.fixture + def user(admin_client): + _user = User(name="Susan", username=f"testuser-{uuid4()}", password="P4$$word") + admin_client.create_user(_user) + yield _user + admin_client.delete_user(_user) + + + @pytest.fixture + def driver(): + _driver = Chrome() + yield _driver + _driver.quit() + + + @pytest.fixture + def login(driver, base_url, user): + driver.get(urljoin(base_url, "/login")) + page = LoginPage(driver) + page.login(user) + + + @pytest.fixture + def landing_page(driver, login): + return LandingPage(driver) + + + def test_name_on_landing_page_after_login(landing_page, user): + assert landing_page.header == f"Welcome, {user.name}!" + +The way the dependencies are laid out means it's unclear if the ``user`` +fixture would execute before the ``driver`` fixture. But that's ok, because +those are atomic operations, and so it doesn't matter which one runs first +because the sequence of events for the test is still `linearizable +`_. But what *does* matter is +that, no matter which one runs first, if the one raises an exception while the +other would not have, neither will have left anything behind. If ``driver`` +executes before ``user``, and ``user`` raises an exception, the driver will +still quit, and the user was never made. And if ``driver`` was the one to raise +the exception, then the driver would never have been started and the user would +never have been made. + +.. note: + + While the ``user`` fixture doesn't *actually* need to happen before the + ``driver`` fixture, if we made ``driver`` request ``user``, it might save + some time in the event that making the user raises an exception, since it + won't bother trying to start the driver, which is a fairly expensive + operation. + + +Running multiple ``assert`` statements safely +--------------------------------------------- + +Sometimes you may want to run multiple asserts after doing all that setup, which +makes sense as, in more complex systems, a single action can kick off multiple +behaviors. pytest has a convenient way of handling this and it combines a bunch +of what we've gone over so far. + +All that's needed is stepping up to a larger scope, then having the **act** +step defined as an autouse fixture, and finally, making sure all the fixtures +are targeting that higher level scope. + +Let's pull :ref:`an example from above `, and tweak it a +bit. Let's say that in addition to checking for a welcome message in the header, +we also want to check for a sign out button, and a link to the user's profile. + +Let's take a look at how we can structure that so we can run multiple asserts +without having to repeat all those steps again. + +.. note:: + + For this example, certain fixtures (i.e. ``base_url`` and + ``admin_credentials``) are implied to exist elsewhere. So for now, let's + assume they exist, and we're just not looking at them. + +.. code-block:: python + + # contents of tests/end_to_end/test_login.py + from uuid import uuid4 + from urllib.parse import urljoin + + from selenium.webdriver import Chrome + import pytest + + from src.utils.pages import LoginPage, LandingPage + from src.utils import AdminApiClient + from src.utils.data_types import User + + + @pytest.fixture(scope="class") + def admin_client(base_url, admin_credentials): + return AdminApiClient(base_url, **admin_credentials) + + + @pytest.fixture(scope="class") + def user(admin_client): + _user = User(name="Susan", username=f"testuser-{uuid4()}", password="P4$$word") + admin_client.create_user(_user) + yield _user + admin_client.delete_user(_user) + + + @pytest.fixture(scope="class") + def driver(): + _driver = Chrome() + yield _driver + _driver.quit() + + + @pytest.fixture(scope="class") + def landing_page(driver, login): + return LandingPage(driver) + + + class TestLandingPageSuccess: + @pytest.fixture(scope="class", autouse=True) + def login(self, driver, base_url, user): + driver.get(urljoin(base_url, "/login")) + page = LoginPage(driver) + page.login(user) + + def test_name_in_header(self, landing_page, user): + assert landing_page.header == f"Welcome, {user.name}!" + + def test_sign_out_button(self, landing_page): + assert landing_page.sign_out_button.is_displayed() + + def test_profile_link(self, landing_page, user): + profile_href = urljoin(base_url, f"/profile?id={user.profile_id}") + assert landing_page.profile_link.get_attribute("href") == profile_href + +Notice that the methods are only referencing ``self`` in the signature as a +formality. No state is tied to the actual test class as it might be in the +``unittest.TestCase`` framework. Everything is managed by the pytest fixture +system. + +Each method only has to request the fixtures that it actually needs without +worrying about order. This is because the **act** fixture is an autouse fixture, +and it made sure all the other fixtures executed before it. There's no more +changes of state that need to take place, so the tests are free to make as many +non-state-changing queries as they want without risking stepping on the toes of +the other tests. + +The ``login`` fixture is defined inside the class as well, because not every one +of the other tests in the module will be expecting a successful login, and the **act** may need to +be handled a little differently for another test class. For example, if we +wanted to write another test scenario around submitting bad credentials, we +could handle it by adding something like this to the test file: + +.. note: + + It's assumed that the page object for this (i.e. ``LoginPage``) raises a + custom exception, ``BadCredentialsException``, when it recognizes text + signifying that on the login form after attempting to log in. + +.. code-block:: python + + class TestLandingPageBadCredentials: + @pytest.fixture(scope="class") + def faux_user(self, user): + _user = deepcopy(user) + _user.password = "badpass" + return _user + + def test_raises_bad_credentials_exception(self, login_page, faux_user): + with pytest.raises(BadCredentialsException): + login_page.login(faux_user) + + +.. _`request-context`: + +Fixtures can introspect the requesting test context +------------------------------------------------------------- + +Fixture functions can accept the :py:class:`request <_pytest.fixtures.FixtureRequest>` object +to introspect the "requesting" test function, class or module context. +Further extending the previous ``smtp_connection`` fixture example, let's +read an optional server URL from the test module which uses our fixture: + +.. code-block:: python + + # content of conftest.py + import pytest + import smtplib + + + @pytest.fixture(scope="module") + def smtp_connection(request): + server = getattr(request.module, "smtpserver", "smtp.gmail.com") + smtp_connection = smtplib.SMTP(server, 587, timeout=5) + yield smtp_connection + print("finalizing {} ({})".format(smtp_connection, server)) + smtp_connection.close() + +We use the ``request.module`` attribute to optionally obtain an +``smtpserver`` attribute from the test module. If we just execute +again, nothing much has changed: + +.. code-block:: pytest + + $ pytest -s -q --tb=no test_module.py + FFfinalizing (smtp.gmail.com) + + ========================= short test summary info ========================== + FAILED test_module.py::test_ehlo - assert 0 + FAILED test_module.py::test_noop - assert 0 + 2 failed in 0.12s + +Let's quickly create another test module that actually sets the +server URL in its module namespace: + +.. code-block:: python + + # content of test_anothersmtp.py + + smtpserver = "mail.python.org" # will be read by smtp fixture + + + def test_showhelo(smtp_connection): + assert 0, smtp_connection.helo() + +Running it: + +.. code-block:: pytest + + $ pytest -qq --tb=short test_anothersmtp.py + F [100%] + ================================= FAILURES ================================= + ______________________________ test_showhelo _______________________________ + test_anothersmtp.py:6: in test_showhelo + assert 0, smtp_connection.helo() + E AssertionError: (250, b'mail.python.org') + E assert 0 + ------------------------- Captured stdout teardown ------------------------- + finalizing (mail.python.org) + ========================= short test summary info ========================== + FAILED test_anothersmtp.py::test_showhelo - AssertionError: (250, b'mail.... + +voila! The ``smtp_connection`` fixture function picked up our mail server name +from the module namespace. + +.. _`using-markers`: + +Using markers to pass data to fixtures +------------------------------------------------------------- + +Using the :py:class:`request <_pytest.fixtures.FixtureRequest>` object, a fixture can also access +markers which are applied to a test function. This can be useful to pass data +into a fixture from a test: + +.. code-block:: python + + import pytest + + + @pytest.fixture + def fixt(request): + marker = request.node.get_closest_marker("fixt_data") + if marker is None: + # Handle missing marker in some way... + data = None + else: + data = marker.args[0] + + # Do something with the data + return data + + + @pytest.mark.fixt_data(42) + def test_fixt(fixt): + assert fixt == 42 + +.. _`fixture-factory`: + +Factories as fixtures +------------------------------------------------------------- + +The "factory as fixture" pattern can help in situations where the result +of a fixture is needed multiple times in a single test. Instead of returning +data directly, the fixture instead returns a function which generates the data. +This function can then be called multiple times in the test. + +Factories can have parameters as needed: + +.. code-block:: python + + @pytest.fixture + def make_customer_record(): + def _make_customer_record(name): + return {"name": name, "orders": []} + + return _make_customer_record + + + def test_customer_records(make_customer_record): + customer_1 = make_customer_record("Lisa") + customer_2 = make_customer_record("Mike") + customer_3 = make_customer_record("Meredith") + +If the data created by the factory requires managing, the fixture can take care of that: + +.. code-block:: python + + @pytest.fixture + def make_customer_record(): + + created_records = [] + + def _make_customer_record(name): + record = models.Customer(name=name, orders=[]) + created_records.append(record) + return record + + yield _make_customer_record + + for record in created_records: + record.destroy() + + + def test_customer_records(make_customer_record): + customer_1 = make_customer_record("Lisa") + customer_2 = make_customer_record("Mike") + customer_3 = make_customer_record("Meredith") + + +.. _`fixture-parametrize`: + +Parametrizing fixtures +----------------------------------------------------------------- + +Fixture functions can be parametrized in which case they will be called +multiple times, each time executing the set of dependent tests, i.e. the +tests that depend on this fixture. Test functions usually do not need +to be aware of their re-running. Fixture parametrization helps to +write exhaustive functional tests for components which themselves can be +configured in multiple ways. + +Extending the previous example, we can flag the fixture to create two +``smtp_connection`` fixture instances which will cause all tests using the fixture +to run twice. The fixture function gets access to each parameter +through the special :py:class:`request ` object: + +.. code-block:: python + + # content of conftest.py + import pytest + import smtplib + + + @pytest.fixture(scope="module", params=["smtp.gmail.com", "mail.python.org"]) + def smtp_connection(request): + smtp_connection = smtplib.SMTP(request.param, 587, timeout=5) + yield smtp_connection + print("finalizing {}".format(smtp_connection)) + smtp_connection.close() + +The main change is the declaration of ``params`` with +:py:func:`@pytest.fixture `, a list of values +for each of which the fixture function will execute and can access +a value via ``request.param``. No test function code needs to change. +So let's just do another run: + +.. code-block:: pytest + + $ pytest -q test_module.py + FFFF [100%] + ================================= FAILURES ================================= + ________________________ test_ehlo[smtp.gmail.com] _________________________ + + smtp_connection = + + def test_ehlo(smtp_connection): + response, msg = smtp_connection.ehlo() + assert response == 250 + assert b"smtp.gmail.com" in msg + > assert 0 # for demo purposes + E assert 0 + + test_module.py:7: AssertionError + ________________________ test_noop[smtp.gmail.com] _________________________ + + smtp_connection = + + def test_noop(smtp_connection): + response, msg = smtp_connection.noop() + assert response == 250 + > assert 0 # for demo purposes + E assert 0 + + test_module.py:13: AssertionError + ________________________ test_ehlo[mail.python.org] ________________________ + + smtp_connection = + + def test_ehlo(smtp_connection): + response, msg = smtp_connection.ehlo() + assert response == 250 + > assert b"smtp.gmail.com" in msg + E AssertionError: assert b'smtp.gmail.com' in b'mail.python.org\nPIPELINING\nSIZE 51200000\nETRN\nSTARTTLS\nAUTH DIGEST-MD5 NTLM CRAM-MD5\nENHANCEDSTATUSCODES\n8BITMIME\nDSN\nSMTPUTF8\nCHUNKING' + + test_module.py:6: AssertionError + -------------------------- Captured stdout setup --------------------------- + finalizing + ________________________ test_noop[mail.python.org] ________________________ + + smtp_connection = + + def test_noop(smtp_connection): + response, msg = smtp_connection.noop() + assert response == 250 + > assert 0 # for demo purposes + E assert 0 + + test_module.py:13: AssertionError + ------------------------- Captured stdout teardown ------------------------- + finalizing + ========================= short test summary info ========================== + FAILED test_module.py::test_ehlo[smtp.gmail.com] - assert 0 + FAILED test_module.py::test_noop[smtp.gmail.com] - assert 0 + FAILED test_module.py::test_ehlo[mail.python.org] - AssertionError: asser... + FAILED test_module.py::test_noop[mail.python.org] - assert 0 + 4 failed in 0.12s + +We see that our two test functions each ran twice, against the different +``smtp_connection`` instances. Note also, that with the ``mail.python.org`` +connection the second test fails in ``test_ehlo`` because a +different server string is expected than what arrived. + +pytest will build a string that is the test ID for each fixture value +in a parametrized fixture, e.g. ``test_ehlo[smtp.gmail.com]`` and +``test_ehlo[mail.python.org]`` in the above examples. These IDs can +be used with ``-k`` to select specific cases to run, and they will +also identify the specific case when one is failing. Running pytest +with ``--collect-only`` will show the generated IDs. + +Numbers, strings, booleans and ``None`` will have their usual string +representation used in the test ID. For other objects, pytest will +make a string based on the argument name. It is possible to customise +the string used in a test ID for a certain fixture value by using the +``ids`` keyword argument: + +.. code-block:: python + + # content of test_ids.py + import pytest + + + @pytest.fixture(params=[0, 1], ids=["spam", "ham"]) + def a(request): + return request.param + + + def test_a(a): + pass + + + def idfn(fixture_value): + if fixture_value == 0: + return "eggs" + else: + return None + + + @pytest.fixture(params=[0, 1], ids=idfn) + def b(request): + return request.param + + + def test_b(b): + pass + +The above shows how ``ids`` can be either a list of strings to use or +a function which will be called with the fixture value and then +has to return a string to use. In the latter case if the function +returns ``None`` then pytest's auto-generated ID will be used. + +Running the above tests results in the following test IDs being used: + +.. code-block:: pytest + + $ pytest --collect-only + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 11 items + + + + + + + + + + + + + + + + + + ======================= 11 tests collected in 0.12s ======================== + +.. _`fixture-parametrize-marks`: + +Using marks with parametrized fixtures +-------------------------------------- + +:func:`pytest.param` can be used to apply marks in values sets of parametrized fixtures in the same way +that they can be used with :ref:`@pytest.mark.parametrize <@pytest.mark.parametrize>`. + +Example: + +.. code-block:: python + + # content of test_fixture_marks.py + import pytest + + + @pytest.fixture(params=[0, 1, pytest.param(2, marks=pytest.mark.skip)]) + def data_set(request): + return request.param + + + def test_data(data_set): + pass + +Running this test will *skip* the invocation of ``data_set`` with value ``2``: + +.. code-block:: pytest + + $ pytest test_fixture_marks.py -v + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 3 items + + test_fixture_marks.py::test_data[0] PASSED [ 33%] + test_fixture_marks.py::test_data[1] PASSED [ 66%] + test_fixture_marks.py::test_data[2] SKIPPED (unconditional skip) [100%] + + ======================= 2 passed, 1 skipped in 0.12s ======================= + +.. _`interdependent fixtures`: + +Modularity: using fixtures from a fixture function +---------------------------------------------------------- + +In addition to using fixtures in test functions, fixture functions +can use other fixtures themselves. This contributes to a modular design +of your fixtures and allows re-use of framework-specific fixtures across +many projects. As a simple example, we can extend the previous example +and instantiate an object ``app`` where we stick the already defined +``smtp_connection`` resource into it: + +.. code-block:: python + + # content of test_appsetup.py + + import pytest + + + class App: + def __init__(self, smtp_connection): + self.smtp_connection = smtp_connection + + + @pytest.fixture(scope="module") + def app(smtp_connection): + return App(smtp_connection) + + + def test_smtp_connection_exists(app): + assert app.smtp_connection + +Here we declare an ``app`` fixture which receives the previously defined +``smtp_connection`` fixture and instantiates an ``App`` object with it. Let's run it: + +.. code-block:: pytest + + $ pytest -v test_appsetup.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 2 items + + test_appsetup.py::test_smtp_connection_exists[smtp.gmail.com] PASSED [ 50%] + test_appsetup.py::test_smtp_connection_exists[mail.python.org] PASSED [100%] + + ============================ 2 passed in 0.12s ============================= + +Due to the parametrization of ``smtp_connection``, the test will run twice with two +different ``App`` instances and respective smtp servers. There is no +need for the ``app`` fixture to be aware of the ``smtp_connection`` +parametrization because pytest will fully analyse the fixture dependency graph. + +Note that the ``app`` fixture has a scope of ``module`` and uses a +module-scoped ``smtp_connection`` fixture. The example would still work if +``smtp_connection`` was cached on a ``session`` scope: it is fine for fixtures to use +"broader" scoped fixtures but not the other way round: +A session-scoped fixture could not use a module-scoped one in a +meaningful way. + + +.. _`automatic per-resource grouping`: + +Automatic grouping of tests by fixture instances +---------------------------------------------------------- + +.. regendoc: wipe + +pytest minimizes the number of active fixtures during test runs. +If you have a parametrized fixture, then all the tests using it will +first execute with one instance and then finalizers are called +before the next fixture instance is created. Among other things, +this eases testing of applications which create and use global state. + +The following example uses two parametrized fixtures, one of which is +scoped on a per-module basis, and all the functions perform ``print`` calls +to show the setup/teardown flow: + +.. code-block:: python + + # content of test_module.py + import pytest + + + @pytest.fixture(scope="module", params=["mod1", "mod2"]) + def modarg(request): + param = request.param + print(" SETUP modarg", param) + yield param + print(" TEARDOWN modarg", param) + + + @pytest.fixture(scope="function", params=[1, 2]) + def otherarg(request): + param = request.param + print(" SETUP otherarg", param) + yield param + print(" TEARDOWN otherarg", param) + + + def test_0(otherarg): + print(" RUN test0 with otherarg", otherarg) + + + def test_1(modarg): + print(" RUN test1 with modarg", modarg) + + + def test_2(otherarg, modarg): + print(" RUN test2 with otherarg {} and modarg {}".format(otherarg, modarg)) + + +Let's run the tests in verbose mode and with looking at the print-output: + +.. code-block:: pytest + + $ pytest -v -s test_module.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + cachedir: .pytest_cache + rootdir: /home/sweet/project + collecting ... collected 8 items + + test_module.py::test_0[1] SETUP otherarg 1 + RUN test0 with otherarg 1 + PASSED TEARDOWN otherarg 1 + + test_module.py::test_0[2] SETUP otherarg 2 + RUN test0 with otherarg 2 + PASSED TEARDOWN otherarg 2 + + test_module.py::test_1[mod1] SETUP modarg mod1 + RUN test1 with modarg mod1 + PASSED + test_module.py::test_2[mod1-1] SETUP otherarg 1 + RUN test2 with otherarg 1 and modarg mod1 + PASSED TEARDOWN otherarg 1 + + test_module.py::test_2[mod1-2] SETUP otherarg 2 + RUN test2 with otherarg 2 and modarg mod1 + PASSED TEARDOWN otherarg 2 + + test_module.py::test_1[mod2] TEARDOWN modarg mod1 + SETUP modarg mod2 + RUN test1 with modarg mod2 + PASSED + test_module.py::test_2[mod2-1] SETUP otherarg 1 + RUN test2 with otherarg 1 and modarg mod2 + PASSED TEARDOWN otherarg 1 + + test_module.py::test_2[mod2-2] SETUP otherarg 2 + RUN test2 with otherarg 2 and modarg mod2 + PASSED TEARDOWN otherarg 2 + TEARDOWN modarg mod2 + + + ============================ 8 passed in 0.12s ============================= + +You can see that the parametrized module-scoped ``modarg`` resource caused an +ordering of test execution that lead to the fewest possible "active" resources. +The finalizer for the ``mod1`` parametrized resource was executed before the +``mod2`` resource was setup. + +In particular notice that test_0 is completely independent and finishes first. +Then test_1 is executed with ``mod1``, then test_2 with ``mod1``, then test_1 +with ``mod2`` and finally test_2 with ``mod2``. + +The ``otherarg`` parametrized resource (having function scope) was set up before +and teared down after every test that used it. + + +.. _`usefixtures`: + +Use fixtures in classes and modules with ``usefixtures`` +-------------------------------------------------------- + +.. regendoc:wipe + +Sometimes test functions do not directly need access to a fixture object. +For example, tests may require to operate with an empty directory as the +current working directory but otherwise do not care for the concrete +directory. Here is how you can use the standard :mod:`tempfile` +and pytest fixtures to +achieve it. We separate the creation of the fixture into a :file:`conftest.py` +file: + +.. code-block:: python + + # content of conftest.py + + import os + import tempfile + + import pytest + + + @pytest.fixture + def cleandir(): + with tempfile.TemporaryDirectory() as newpath: + old_cwd = os.getcwd() + os.chdir(newpath) + yield + os.chdir(old_cwd) + +and declare its use in a test module via a ``usefixtures`` marker: + +.. code-block:: python + + # content of test_setenv.py + import os + import pytest + + + @pytest.mark.usefixtures("cleandir") + class TestDirectoryInit: + def test_cwd_starts_empty(self): + assert os.listdir(os.getcwd()) == [] + with open("myfile", "w") as f: + f.write("hello") + + def test_cwd_again_starts_empty(self): + assert os.listdir(os.getcwd()) == [] + +Due to the ``usefixtures`` marker, the ``cleandir`` fixture +will be required for the execution of each test method, just as if +you specified a "cleandir" function argument to each of them. Let's run it +to verify our fixture is activated and the tests pass: + +.. code-block:: pytest + + $ pytest -q + .. [100%] + 2 passed in 0.12s + +You can specify multiple fixtures like this: + +.. code-block:: python + + @pytest.mark.usefixtures("cleandir", "anotherfixture") + def test(): + ... + +and you may specify fixture usage at the test module level using :globalvar:`pytestmark`: + +.. code-block:: python + + pytestmark = pytest.mark.usefixtures("cleandir") + + +It is also possible to put fixtures required by all tests in your project +into an ini-file: + +.. code-block:: ini + + # content of pytest.ini + [pytest] + usefixtures = cleandir + + +.. warning:: + + Note this mark has no effect in **fixture functions**. For example, + this **will not work as expected**: + + .. code-block:: python + + @pytest.mark.usefixtures("my_other_fixture") + @pytest.fixture + def my_fixture_that_sadly_wont_use_my_other_fixture(): + ... + + Currently this will not generate any error or warning, but this is intended + to be handled by :issue:`3664`. + +.. _`override fixtures`: + +Overriding fixtures on various levels +------------------------------------- + +In relatively large test suite, you most likely need to ``override`` a ``global`` or ``root`` fixture with a ``locally`` +defined one, keeping the test code readable and maintainable. + +Override a fixture on a folder (conftest) level +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Given the tests file structure is: + +:: + + tests/ + __init__.py + + conftest.py + # content of tests/conftest.py + import pytest + + @pytest.fixture + def username(): + return 'username' + + test_something.py + # content of tests/test_something.py + def test_username(username): + assert username == 'username' + + subfolder/ + __init__.py + + conftest.py + # content of tests/subfolder/conftest.py + import pytest + + @pytest.fixture + def username(username): + return 'overridden-' + username + + test_something.py + # content of tests/subfolder/test_something.py + def test_username(username): + assert username == 'overridden-username' + +As you can see, a fixture with the same name can be overridden for certain test folder level. +Note that the ``base`` or ``super`` fixture can be accessed from the ``overriding`` +fixture easily - used in the example above. + +Override a fixture on a test module level +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Given the tests file structure is: + +:: + + tests/ + __init__.py + + conftest.py + # content of tests/conftest.py + import pytest + + @pytest.fixture + def username(): + return 'username' + + test_something.py + # content of tests/test_something.py + import pytest + + @pytest.fixture + def username(username): + return 'overridden-' + username + + def test_username(username): + assert username == 'overridden-username' + + test_something_else.py + # content of tests/test_something_else.py + import pytest + + @pytest.fixture + def username(username): + return 'overridden-else-' + username + + def test_username(username): + assert username == 'overridden-else-username' + +In the example above, a fixture with the same name can be overridden for certain test module. + + +Override a fixture with direct test parametrization +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Given the tests file structure is: + +:: + + tests/ + __init__.py + + conftest.py + # content of tests/conftest.py + import pytest + + @pytest.fixture + def username(): + return 'username' + + @pytest.fixture + def other_username(username): + return 'other-' + username + + test_something.py + # content of tests/test_something.py + import pytest + + @pytest.mark.parametrize('username', ['directly-overridden-username']) + def test_username(username): + assert username == 'directly-overridden-username' + + @pytest.mark.parametrize('username', ['directly-overridden-username-other']) + def test_username_other(other_username): + assert other_username == 'other-directly-overridden-username-other' + +In the example above, a fixture value is overridden by the test parameter value. Note that the value of the fixture +can be overridden this way even if the test doesn't use it directly (doesn't mention it in the function prototype). + + +Override a parametrized fixture with non-parametrized one and vice versa +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Given the tests file structure is: + +:: + + tests/ + __init__.py + + conftest.py + # content of tests/conftest.py + import pytest + + @pytest.fixture(params=['one', 'two', 'three']) + def parametrized_username(request): + return request.param + + @pytest.fixture + def non_parametrized_username(request): + return 'username' + + test_something.py + # content of tests/test_something.py + import pytest + + @pytest.fixture + def parametrized_username(): + return 'overridden-username' + + @pytest.fixture(params=['one', 'two', 'three']) + def non_parametrized_username(request): + return request.param + + def test_username(parametrized_username): + assert parametrized_username == 'overridden-username' + + def test_parametrized_username(non_parametrized_username): + assert non_parametrized_username in ['one', 'two', 'three'] + + test_something_else.py + # content of tests/test_something_else.py + def test_username(parametrized_username): + assert parametrized_username in ['one', 'two', 'three'] + + def test_username(non_parametrized_username): + assert non_parametrized_username == 'username' + +In the example above, a parametrized fixture is overridden with a non-parametrized version, and +a non-parametrized fixture is overridden with a parametrized version for certain test module. +The same applies for the test folder level obviously. + + +Using fixtures from other projects +---------------------------------- + +Usually projects that provide pytest support will use :ref:`entry points `, +so just installing those projects into an environment will make those fixtures available for use. + +In case you want to use fixtures from a project that does not use entry points, you can +define :globalvar:`pytest_plugins` in your top ``conftest.py`` file to register that module +as a plugin. + +Suppose you have some fixtures in ``mylibrary.fixtures`` and you want to reuse them into your +``app/tests`` directory. + +All you need to do is to define :globalvar:`pytest_plugins` in ``app/tests/conftest.py`` +pointing to that module. + +.. code-block:: python + + pytest_plugins = "mylibrary.fixtures" + +This effectively registers ``mylibrary.fixtures`` as a plugin, making all its fixtures and +hooks available to tests in ``app/tests``. + +.. note:: + + Sometimes users will *import* fixtures from other projects for use, however this is not + recommended: importing fixtures into a module will register them in pytest + as *defined* in that module. + + This has minor consequences, such as appearing multiple times in ``pytest --help``, + but it is not **recommended** because this behavior might change/stop working + in future versions. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/index.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/index.rst new file mode 100644 index 0000000000..6f52aaecdc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/index.rst @@ -0,0 +1,64 @@ +:orphan: + +.. _how-to: + +How-to guides +================ + +Core pytest functionality +------------------------- + +.. toctree:: + :maxdepth: 1 + + usage + assert + fixtures + mark + parametrize + tmp_path + monkeypatch + doctest + cache + +Test output and outcomes +---------------------------- + +.. toctree:: + :maxdepth: 1 + + failures + output + logging + capture-stdout-stderr + capture-warnings + skipping + +Plugins +---------------------------- + +.. toctree:: + :maxdepth: 1 + + plugins + writing_plugins + writing_hook_functions + +pytest and other test systems +----------------------------- + +.. toctree:: + :maxdepth: 1 + + existingtestsuite + unittest + nose + xunit_setup + +pytest development environment +------------------------------ + +.. toctree:: + :maxdepth: 1 + + bash-completion diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/logging.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/logging.rst new file mode 100644 index 0000000000..2e8734fa6a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/logging.rst @@ -0,0 +1,292 @@ +.. _logging: + +How to manage logging +--------------------- + +pytest captures log messages of level ``WARNING`` or above automatically and displays them in their own section +for each failed test in the same manner as captured stdout and stderr. + +Running without options: + +.. code-block:: bash + + pytest + +Shows failed tests like so: + +.. code-block:: pytest + + ----------------------- Captured stdlog call ---------------------- + test_reporting.py 26 WARNING text going to logger + ----------------------- Captured stdout call ---------------------- + text going to stdout + ----------------------- Captured stderr call ---------------------- + text going to stderr + ==================== 2 failed in 0.02 seconds ===================== + +By default each captured log message shows the module, line number, log level +and message. + +If desired the log and date format can be specified to +anything that the logging module supports by passing specific formatting options: + +.. code-block:: bash + + pytest --log-format="%(asctime)s %(levelname)s %(message)s" \ + --log-date-format="%Y-%m-%d %H:%M:%S" + +Shows failed tests like so: + +.. code-block:: pytest + + ----------------------- Captured stdlog call ---------------------- + 2010-04-10 14:48:44 WARNING text going to logger + ----------------------- Captured stdout call ---------------------- + text going to stdout + ----------------------- Captured stderr call ---------------------- + text going to stderr + ==================== 2 failed in 0.02 seconds ===================== + +These options can also be customized through ``pytest.ini`` file: + +.. code-block:: ini + + [pytest] + log_format = %(asctime)s %(levelname)s %(message)s + log_date_format = %Y-%m-%d %H:%M:%S + +Further it is possible to disable reporting of captured content (stdout, +stderr and logs) on failed tests completely with: + +.. code-block:: bash + + pytest --show-capture=no + + +caplog fixture +^^^^^^^^^^^^^^ + +Inside tests it is possible to change the log level for the captured log +messages. This is supported by the ``caplog`` fixture: + +.. code-block:: python + + def test_foo(caplog): + caplog.set_level(logging.INFO) + pass + +By default the level is set on the root logger, +however as a convenience it is also possible to set the log level of any +logger: + +.. code-block:: python + + def test_foo(caplog): + caplog.set_level(logging.CRITICAL, logger="root.baz") + pass + +The log levels set are restored automatically at the end of the test. + +It is also possible to use a context manager to temporarily change the log +level inside a ``with`` block: + +.. code-block:: python + + def test_bar(caplog): + with caplog.at_level(logging.INFO): + pass + +Again, by default the level of the root logger is affected but the level of any +logger can be changed instead with: + +.. code-block:: python + + def test_bar(caplog): + with caplog.at_level(logging.CRITICAL, logger="root.baz"): + pass + +Lastly all the logs sent to the logger during the test run are made available on +the fixture in the form of both the ``logging.LogRecord`` instances and the final log text. +This is useful for when you want to assert on the contents of a message: + +.. code-block:: python + + def test_baz(caplog): + func_under_test() + for record in caplog.records: + assert record.levelname != "CRITICAL" + assert "wally" not in caplog.text + +For all the available attributes of the log records see the +``logging.LogRecord`` class. + +You can also resort to ``record_tuples`` if all you want to do is to ensure, +that certain messages have been logged under a given logger name with a given +severity and message: + +.. code-block:: python + + def test_foo(caplog): + logging.getLogger().info("boo %s", "arg") + + assert caplog.record_tuples == [("root", logging.INFO, "boo arg")] + +You can call ``caplog.clear()`` to reset the captured log records in a test: + +.. code-block:: python + + def test_something_with_clearing_records(caplog): + some_method_that_creates_log_records() + caplog.clear() + your_test_method() + assert ["Foo"] == [rec.message for rec in caplog.records] + + +The ``caplog.records`` attribute contains records from the current stage only, so +inside the ``setup`` phase it contains only setup logs, same with the ``call`` and +``teardown`` phases. + +To access logs from other stages, use the ``caplog.get_records(when)`` method. As an example, +if you want to make sure that tests which use a certain fixture never log any warnings, you can inspect +the records for the ``setup`` and ``call`` stages during teardown like so: + +.. code-block:: python + + @pytest.fixture + def window(caplog): + window = create_window() + yield window + for when in ("setup", "call"): + messages = [ + x.message for x in caplog.get_records(when) if x.levelno == logging.WARNING + ] + if messages: + pytest.fail( + "warning messages encountered during testing: {}".format(messages) + ) + + + +The full API is available at :class:`pytest.LogCaptureFixture`. + + +.. _live_logs: + +Live Logs +^^^^^^^^^ + +By setting the :confval:`log_cli` configuration option to ``true``, pytest will output +logging records as they are emitted directly into the console. + +You can specify the logging level for which log records with equal or higher +level are printed to the console by passing ``--log-cli-level``. This setting +accepts the logging level names as seen in python's documentation or an integer +as the logging level num. + +Additionally, you can also specify ``--log-cli-format`` and +``--log-cli-date-format`` which mirror and default to ``--log-format`` and +``--log-date-format`` if not provided, but are applied only to the console +logging handler. + +All of the CLI log options can also be set in the configuration INI file. The +option names are: + +* ``log_cli_level`` +* ``log_cli_format`` +* ``log_cli_date_format`` + +If you need to record the whole test suite logging calls to a file, you can pass +``--log-file=/path/to/log/file``. This log file is opened in write mode which +means that it will be overwritten at each run tests session. + +You can also specify the logging level for the log file by passing +``--log-file-level``. This setting accepts the logging level names as seen in +python's documentation(ie, uppercased level names) or an integer as the logging +level num. + +Additionally, you can also specify ``--log-file-format`` and +``--log-file-date-format`` which are equal to ``--log-format`` and +``--log-date-format`` but are applied to the log file logging handler. + +All of the log file options can also be set in the configuration INI file. The +option names are: + +* ``log_file`` +* ``log_file_level`` +* ``log_file_format`` +* ``log_file_date_format`` + +You can call ``set_log_path()`` to customize the log_file path dynamically. This functionality +is considered **experimental**. + +.. _log_colors: + +Customizing Colors +^^^^^^^^^^^^^^^^^^ + +Log levels are colored if colored terminal output is enabled. Changing +from default colors or putting color on custom log levels is supported +through ``add_color_level()``. Example: + +.. code-block:: python + + @pytest.hookimpl + def pytest_configure(config): + logging_plugin = config.pluginmanager.get_plugin("logging-plugin") + + # Change color on existing log level + logging_plugin.log_cli_handler.formatter.add_color_level(logging.INFO, "cyan") + + # Add color to a custom log level (a custom log level `SPAM` is already set up) + logging_plugin.log_cli_handler.formatter.add_color_level(logging.SPAM, "blue") +.. warning:: + + This feature and its API are considered **experimental** and might change + between releases without a deprecation notice. +.. _log_release_notes: + +Release notes +^^^^^^^^^^^^^ + +This feature was introduced as a drop-in replacement for the +:pypi:`pytest-catchlog` plugin and they conflict +with each other. The backward compatibility API with ``pytest-capturelog`` +has been dropped when this feature was introduced, so if for that reason you +still need ``pytest-catchlog`` you can disable the internal feature by +adding to your ``pytest.ini``: + +.. code-block:: ini + + [pytest] + addopts=-p no:logging + + +.. _log_changes_3_4: + +Incompatible changes in pytest 3.4 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This feature was introduced in ``3.3`` and some **incompatible changes** have been +made in ``3.4`` after community feedback: + +* Log levels are no longer changed unless explicitly requested by the :confval:`log_level` configuration + or ``--log-level`` command-line options. This allows users to configure logger objects themselves. + Setting :confval:`log_level` will set the level that is captured globally so if a specific test requires + a lower level than this, use the ``caplog.set_level()`` functionality otherwise that test will be prone to + failure. +* :ref:`Live Logs ` is now disabled by default and can be enabled setting the + :confval:`log_cli` configuration option to ``true``. When enabled, the verbosity is increased so logging for each + test is visible. +* :ref:`Live Logs ` are now sent to ``sys.stdout`` and no longer require the ``-s`` command-line option + to work. + +If you want to partially restore the logging behavior of version ``3.3``, you can add this options to your ``ini`` +file: + +.. code-block:: ini + + [pytest] + log_cli=true + log_level=NOTSET + +More details about the discussion that lead to this changes can be read in :issue:`3013`. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/mark.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/mark.rst new file mode 100644 index 0000000000..33f9d18bfe --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/mark.rst @@ -0,0 +1,93 @@ +.. _mark: + +How to mark test functions with attributes +=========================================== + +By using the ``pytest.mark`` helper you can easily set +metadata on your test functions. You can find the full list of builtin markers +in the :ref:`API Reference`. Or you can list all the markers, including +builtin and custom, using the CLI - :code:`pytest --markers`. + +Here are some of the builtin markers: + +* :ref:`usefixtures ` - use fixtures on a test function or class +* :ref:`filterwarnings ` - filter certain warnings of a test function +* :ref:`skip ` - always skip a test function +* :ref:`skipif ` - skip a test function if a certain condition is met +* :ref:`xfail ` - produce an "expected failure" outcome if a certain + condition is met +* :ref:`parametrize ` - perform multiple calls + to the same test function. + +It's easy to create custom markers or to apply markers +to whole test classes or modules. Those markers can be used by plugins, and also +are commonly used to :ref:`select tests ` on the command-line with the ``-m`` option. + +See :ref:`mark examples` for examples which also serve as documentation. + +.. note:: + + Marks can only be applied to tests, having no effect on + :ref:`fixtures `. + + +Registering marks +----------------- + +You can register custom marks in your ``pytest.ini`` file like this: + +.. code-block:: ini + + [pytest] + markers = + slow: marks tests as slow (deselect with '-m "not slow"') + serial + +or in your ``pyproject.toml`` file like this: + +.. code-block:: toml + + [tool.pytest.ini_options] + markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "serial", + ] + +Note that everything past the ``:`` after the mark name is an optional description. + +Alternatively, you can register new markers programmatically in a +:ref:`pytest_configure ` hook: + +.. code-block:: python + + def pytest_configure(config): + config.addinivalue_line( + "markers", "env(name): mark test to run only on named environment" + ) + + +Registered marks appear in pytest's help text and do not emit warnings (see the next section). It +is recommended that third-party plugins always :ref:`register their markers `. + +.. _unknown-marks: + +Raising errors on unknown marks +------------------------------- + +Unregistered marks applied with the ``@pytest.mark.name_of_the_mark`` decorator +will always emit a warning in order to avoid silently doing something +surprising due to mistyped names. As described in the previous section, you can disable +the warning for custom marks by registering them in your ``pytest.ini`` file or +using a custom ``pytest_configure`` hook. + +When the ``--strict-markers`` command-line flag is passed, any unknown marks applied +with the ``@pytest.mark.name_of_the_mark`` decorator will trigger an error. You can +enforce this validation in your project by adding ``--strict-markers`` to ``addopts``: + +.. code-block:: ini + + [pytest] + addopts = --strict-markers + markers = + slow: marks tests as slow (deselect with '-m "not slow"') + serial diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/monkeypatch.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/monkeypatch.rst new file mode 100644 index 0000000000..9c61233f7e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/monkeypatch.rst @@ -0,0 +1,444 @@ +.. _monkeypatching: + +How to monkeypatch/mock modules and environments +================================================================ + +.. currentmodule:: _pytest.monkeypatch + +Sometimes tests need to invoke functionality which depends +on global settings or which invokes code which cannot be easily +tested such as network access. The ``monkeypatch`` fixture +helps you to safely set/delete an attribute, dictionary item or +environment variable, or to modify ``sys.path`` for importing. + +The ``monkeypatch`` fixture provides these helper methods for safely patching and mocking +functionality in tests: + +.. code-block:: python + + monkeypatch.setattr(obj, name, value, raising=True) + monkeypatch.setattr("somemodule.obj.name", value, raising=True) + monkeypatch.delattr(obj, name, raising=True) + monkeypatch.setitem(mapping, name, value) + monkeypatch.delitem(obj, name, raising=True) + monkeypatch.setenv(name, value, prepend=None) + monkeypatch.delenv(name, raising=True) + monkeypatch.syspath_prepend(path) + monkeypatch.chdir(path) + +All modifications will be undone after the requesting +test function or fixture has finished. The ``raising`` +parameter determines if a ``KeyError`` or ``AttributeError`` +will be raised if the target of the set/deletion operation does not exist. + +Consider the following scenarios: + +1. Modifying the behavior of a function or the property of a class for a test e.g. +there is an API call or database connection you will not make for a test but you know +what the expected output should be. Use :py:meth:`monkeypatch.setattr ` to patch the +function or property with your desired testing behavior. This can include your own functions. +Use :py:meth:`monkeypatch.delattr ` to remove the function or property for the test. + +2. Modifying the values of dictionaries e.g. you have a global configuration that +you want to modify for certain test cases. Use :py:meth:`monkeypatch.setitem ` to patch the +dictionary for the test. :py:meth:`monkeypatch.delitem ` can be used to remove items. + +3. Modifying environment variables for a test e.g. to test program behavior if an +environment variable is missing, or to set multiple values to a known variable. +:py:meth:`monkeypatch.setenv ` and :py:meth:`monkeypatch.delenv ` can be used for +these patches. + +4. Use ``monkeypatch.setenv("PATH", value, prepend=os.pathsep)`` to modify ``$PATH``, and +:py:meth:`monkeypatch.chdir ` to change the context of the current working directory +during a test. + +5. Use :py:meth:`monkeypatch.syspath_prepend ` to modify ``sys.path`` which will also +call ``pkg_resources.fixup_namespace_packages`` and :py:func:`importlib.invalidate_caches`. + +See the `monkeypatch blog post`_ for some introduction material +and a discussion of its motivation. + +.. _`monkeypatch blog post`: https://tetamap.wordpress.com//2009/03/03/monkeypatching-in-unit-tests-done-right/ + +Simple example: monkeypatching functions +---------------------------------------- + +Consider a scenario where you are working with user directories. In the context of +testing, you do not want your test to depend on the running user. ``monkeypatch`` +can be used to patch functions dependent on the user to always return a +specific value. + +In this example, :py:meth:`monkeypatch.setattr ` is used to patch ``Path.home`` +so that the known testing path ``Path("/abc")`` is always used when the test is run. +This removes any dependency on the running user for testing purposes. +:py:meth:`monkeypatch.setattr ` must be called before the function which will use +the patched function is called. +After the test function finishes the ``Path.home`` modification will be undone. + +.. code-block:: python + + # contents of test_module.py with source code and the test + from pathlib import Path + + + def getssh(): + """Simple function to return expanded homedir ssh path.""" + return Path.home() / ".ssh" + + + def test_getssh(monkeypatch): + # mocked return function to replace Path.home + # always return '/abc' + def mockreturn(): + return Path("/abc") + + # Application of the monkeypatch to replace Path.home + # with the behavior of mockreturn defined above. + monkeypatch.setattr(Path, "home", mockreturn) + + # Calling getssh() will use mockreturn in place of Path.home + # for this test with the monkeypatch. + x = getssh() + assert x == Path("/abc/.ssh") + +Monkeypatching returned objects: building mock classes +------------------------------------------------------ + +:py:meth:`monkeypatch.setattr ` can be used in conjunction with classes to mock returned +objects from functions instead of values. +Imagine a simple function to take an API url and return the json response. + +.. code-block:: python + + # contents of app.py, a simple API retrieval example + import requests + + + def get_json(url): + """Takes a URL, and returns the JSON.""" + r = requests.get(url) + return r.json() + +We need to mock ``r``, the returned response object for testing purposes. +The mock of ``r`` needs a ``.json()`` method which returns a dictionary. +This can be done in our test file by defining a class to represent ``r``. + +.. code-block:: python + + # contents of test_app.py, a simple test for our API retrieval + # import requests for the purposes of monkeypatching + import requests + + # our app.py that includes the get_json() function + # this is the previous code block example + import app + + # custom class to be the mock return value + # will override the requests.Response returned from requests.get + class MockResponse: + + # mock json() method always returns a specific testing dictionary + @staticmethod + def json(): + return {"mock_key": "mock_response"} + + + def test_get_json(monkeypatch): + + # Any arguments may be passed and mock_get() will always return our + # mocked object, which only has the .json() method. + def mock_get(*args, **kwargs): + return MockResponse() + + # apply the monkeypatch for requests.get to mock_get + monkeypatch.setattr(requests, "get", mock_get) + + # app.get_json, which contains requests.get, uses the monkeypatch + result = app.get_json("https://fakeurl") + assert result["mock_key"] == "mock_response" + + +``monkeypatch`` applies the mock for ``requests.get`` with our ``mock_get`` function. +The ``mock_get`` function returns an instance of the ``MockResponse`` class, which +has a ``json()`` method defined to return a known testing dictionary and does not +require any outside API connection. + +You can build the ``MockResponse`` class with the appropriate degree of complexity for +the scenario you are testing. For instance, it could include an ``ok`` property that +always returns ``True``, or return different values from the ``json()`` mocked method +based on input strings. + +This mock can be shared across tests using a ``fixture``: + +.. code-block:: python + + # contents of test_app.py, a simple test for our API retrieval + import pytest + import requests + + # app.py that includes the get_json() function + import app + + # custom class to be the mock return value of requests.get() + class MockResponse: + @staticmethod + def json(): + return {"mock_key": "mock_response"} + + + # monkeypatched requests.get moved to a fixture + @pytest.fixture + def mock_response(monkeypatch): + """Requests.get() mocked to return {'mock_key':'mock_response'}.""" + + def mock_get(*args, **kwargs): + return MockResponse() + + monkeypatch.setattr(requests, "get", mock_get) + + + # notice our test uses the custom fixture instead of monkeypatch directly + def test_get_json(mock_response): + result = app.get_json("https://fakeurl") + assert result["mock_key"] == "mock_response" + + +Furthermore, if the mock was designed to be applied to all tests, the ``fixture`` could +be moved to a ``conftest.py`` file and use the with ``autouse=True`` option. + + +Global patch example: preventing "requests" from remote operations +------------------------------------------------------------------ + +If you want to prevent the "requests" library from performing http +requests in all your tests, you can do: + +.. code-block:: python + + # contents of conftest.py + import pytest + + + @pytest.fixture(autouse=True) + def no_requests(monkeypatch): + """Remove requests.sessions.Session.request for all tests.""" + monkeypatch.delattr("requests.sessions.Session.request") + +This autouse fixture will be executed for each test function and it +will delete the method ``request.session.Session.request`` +so that any attempts within tests to create http requests will fail. + + +.. note:: + + Be advised that it is not recommended to patch builtin functions such as ``open``, + ``compile``, etc., because it might break pytest's internals. If that's + unavoidable, passing ``--tb=native``, ``--assert=plain`` and ``--capture=no`` might + help although there's no guarantee. + +.. note:: + + Mind that patching ``stdlib`` functions and some third-party libraries used by pytest + might break pytest itself, therefore in those cases it is recommended to use + :meth:`MonkeyPatch.context` to limit the patching to the block you want tested: + + .. code-block:: python + + import functools + + + def test_partial(monkeypatch): + with monkeypatch.context() as m: + m.setattr(functools, "partial", 3) + assert functools.partial == 3 + + See :issue:`3290` for details. + + +Monkeypatching environment variables +------------------------------------ + +If you are working with environment variables you often need to safely change the values +or delete them from the system for testing purposes. ``monkeypatch`` provides a mechanism +to do this using the ``setenv`` and ``delenv`` method. Our example code to test: + +.. code-block:: python + + # contents of our original code file e.g. code.py + import os + + + def get_os_user_lower(): + """Simple retrieval function. + Returns lowercase USER or raises OSError.""" + username = os.getenv("USER") + + if username is None: + raise OSError("USER environment is not set.") + + return username.lower() + +There are two potential paths. First, the ``USER`` environment variable is set to a +value. Second, the ``USER`` environment variable does not exist. Using ``monkeypatch`` +both paths can be safely tested without impacting the running environment: + +.. code-block:: python + + # contents of our test file e.g. test_code.py + import pytest + + + def test_upper_to_lower(monkeypatch): + """Set the USER env var to assert the behavior.""" + monkeypatch.setenv("USER", "TestingUser") + assert get_os_user_lower() == "testinguser" + + + def test_raise_exception(monkeypatch): + """Remove the USER env var and assert OSError is raised.""" + monkeypatch.delenv("USER", raising=False) + + with pytest.raises(OSError): + _ = get_os_user_lower() + +This behavior can be moved into ``fixture`` structures and shared across tests: + +.. code-block:: python + + # contents of our test file e.g. test_code.py + import pytest + + + @pytest.fixture + def mock_env_user(monkeypatch): + monkeypatch.setenv("USER", "TestingUser") + + + @pytest.fixture + def mock_env_missing(monkeypatch): + monkeypatch.delenv("USER", raising=False) + + + # notice the tests reference the fixtures for mocks + def test_upper_to_lower(mock_env_user): + assert get_os_user_lower() == "testinguser" + + + def test_raise_exception(mock_env_missing): + with pytest.raises(OSError): + _ = get_os_user_lower() + + +Monkeypatching dictionaries +--------------------------- + +:py:meth:`monkeypatch.setitem ` can be used to safely set the values of dictionaries +to specific values during tests. Take this simplified connection string example: + +.. code-block:: python + + # contents of app.py to generate a simple connection string + DEFAULT_CONFIG = {"user": "user1", "database": "db1"} + + + def create_connection_string(config=None): + """Creates a connection string from input or defaults.""" + config = config or DEFAULT_CONFIG + return f"User Id={config['user']}; Location={config['database']};" + +For testing purposes we can patch the ``DEFAULT_CONFIG`` dictionary to specific values. + +.. code-block:: python + + # contents of test_app.py + # app.py with the connection string function (prior code block) + import app + + + def test_connection(monkeypatch): + + # Patch the values of DEFAULT_CONFIG to specific + # testing values only for this test. + monkeypatch.setitem(app.DEFAULT_CONFIG, "user", "test_user") + monkeypatch.setitem(app.DEFAULT_CONFIG, "database", "test_db") + + # expected result based on the mocks + expected = "User Id=test_user; Location=test_db;" + + # the test uses the monkeypatched dictionary settings + result = app.create_connection_string() + assert result == expected + +You can use the :py:meth:`monkeypatch.delitem ` to remove values. + +.. code-block:: python + + # contents of test_app.py + import pytest + + # app.py with the connection string function + import app + + + def test_missing_user(monkeypatch): + + # patch the DEFAULT_CONFIG t be missing the 'user' key + monkeypatch.delitem(app.DEFAULT_CONFIG, "user", raising=False) + + # Key error expected because a config is not passed, and the + # default is now missing the 'user' entry. + with pytest.raises(KeyError): + _ = app.create_connection_string() + + +The modularity of fixtures gives you the flexibility to define +separate fixtures for each potential mock and reference them in the needed tests. + +.. code-block:: python + + # contents of test_app.py + import pytest + + # app.py with the connection string function + import app + + # all of the mocks are moved into separated fixtures + @pytest.fixture + def mock_test_user(monkeypatch): + """Set the DEFAULT_CONFIG user to test_user.""" + monkeypatch.setitem(app.DEFAULT_CONFIG, "user", "test_user") + + + @pytest.fixture + def mock_test_database(monkeypatch): + """Set the DEFAULT_CONFIG database to test_db.""" + monkeypatch.setitem(app.DEFAULT_CONFIG, "database", "test_db") + + + @pytest.fixture + def mock_missing_default_user(monkeypatch): + """Remove the user key from DEFAULT_CONFIG""" + monkeypatch.delitem(app.DEFAULT_CONFIG, "user", raising=False) + + + # tests reference only the fixture mocks that are needed + def test_connection(mock_test_user, mock_test_database): + + expected = "User Id=test_user; Location=test_db;" + + result = app.create_connection_string() + assert result == expected + + + def test_missing_user(mock_missing_default_user): + + with pytest.raises(KeyError): + _ = app.create_connection_string() + + +.. currentmodule:: _pytest.monkeypatch + +API Reference +------------- + +Consult the docs for the :class:`MonkeyPatch` class. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/nose.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/nose.rst new file mode 100644 index 0000000000..4bf8b06c32 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/nose.rst @@ -0,0 +1,79 @@ +.. _`noseintegration`: + +How to run tests written for nose +======================================= + +``pytest`` has basic support for running tests written for nose_. + +.. _nosestyle: + +Usage +------------- + +After :ref:`installation` type: + +.. code-block:: bash + + python setup.py develop # make sure tests can import our package + pytest # instead of 'nosetests' + +and you should be able to run your nose style tests and +make use of pytest's capabilities. + +Supported nose Idioms +---------------------- + +* setup and teardown at module/class/method level +* SkipTest exceptions and markers +* setup/teardown decorators +* ``__test__`` attribute on modules/classes/functions +* general usage of nose utilities + +Unsupported idioms / known issues +---------------------------------- + +- unittest-style ``setUp, tearDown, setUpClass, tearDownClass`` + are recognized only on ``unittest.TestCase`` classes but not + on plain classes. ``nose`` supports these methods also on plain + classes but pytest deliberately does not. As nose and pytest already + both support ``setup_class, teardown_class, setup_method, teardown_method`` + it doesn't seem useful to duplicate the unittest-API like nose does. + If you however rather think pytest should support the unittest-spelling on + plain classes please post to :issue:`377`. + +- nose imports test modules with the same import path (e.g. + ``tests.test_mode``) but different file system paths + (e.g. ``tests/test_mode.py`` and ``other/tests/test_mode.py``) + by extending sys.path/import semantics. pytest does not do that + but there is discussion in :issue:`268` for adding some support. Note that + `nose2 choose to avoid this sys.path/import hackery `_. + + If you place a conftest.py file in the root directory of your project + (as determined by pytest) pytest will run tests "nose style" against + the code below that directory by adding it to your ``sys.path`` instead of + running against your installed code. + + You may find yourself wanting to do this if you ran ``python setup.py install`` + to set up your project, as opposed to ``python setup.py develop`` or any of + the package manager equivalents. Installing with develop in a + virtual environment like tox is recommended over this pattern. + +- nose-style doctests are not collected and executed correctly, + also doctest fixtures don't work. + +- no nose-configuration is recognized. + +- ``yield``-based methods are unsupported as of pytest 4.1.0. They are + fundamentally incompatible with pytest because they don't support fixtures + properly since collection and test execution are separated. + +Migrating from nose to pytest +------------------------------ + +`nose2pytest `_ is a Python script +and pytest plugin to help convert Nose-based tests into pytest-based tests. +Specifically, the script transforms nose.tools.assert_* function calls into +raw assert statements, while preserving format of original arguments +as much as possible. + +.. _nose: https://nose.readthedocs.io/en/latest/ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/output.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/output.rst new file mode 100644 index 0000000000..4b90988f49 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/output.rst @@ -0,0 +1,710 @@ +.. _how-to-manage-output: + +Managing pytest's output +========================= + +.. _how-to-modifying-python-tb-printing: + +Modifying Python traceback printing +-------------------------------------------------- + +Examples for modifying traceback printing: + +.. code-block:: bash + + pytest --showlocals # show local variables in tracebacks + pytest -l # show local variables (shortcut) + + pytest --tb=auto # (default) 'long' tracebacks for the first and last + # entry, but 'short' style for the other entries + pytest --tb=long # exhaustive, informative traceback formatting + pytest --tb=short # shorter traceback format + pytest --tb=line # only one line per failure + pytest --tb=native # Python standard library formatting + pytest --tb=no # no traceback at all + +The ``--full-trace`` causes very long traces to be printed on error (longer +than ``--tb=long``). It also ensures that a stack trace is printed on +**KeyboardInterrupt** (Ctrl+C). +This is very useful if the tests are taking too long and you interrupt them +with Ctrl+C to find out where the tests are *hanging*. By default no output +will be shown (because KeyboardInterrupt is caught by pytest). By using this +option you make sure a trace is shown. + + +Verbosity +-------------------------------------------------- + +The ``-v`` flag controls the verbosity of pytest output in various aspects: test session progress, assertion +details when tests fail, fixtures details with ``--fixtures``, etc. + +.. regendoc:wipe + +Consider this simple file: + +.. code-block:: python + + # content of test_verbosity_example.py + def test_ok(): + pass + + + def test_words_fail(): + fruits1 = ["banana", "apple", "grapes", "melon", "kiwi"] + fruits2 = ["banana", "apple", "orange", "melon", "kiwi"] + assert fruits1 == fruits2 + + + def test_numbers_fail(): + number_to_text1 = {str(x): x for x in range(5)} + number_to_text2 = {str(x * 10): x * 10 for x in range(5)} + assert number_to_text1 == number_to_text2 + + + def test_long_text_fail(): + long_text = "Lorem ipsum dolor sit amet " * 10 + assert "hello world" in long_text + +Executing pytest normally gives us this output (we are skipping the header to focus on the rest): + +.. code-block:: pytest + + $ pytest --no-header + =========================== test session starts ============================ + collected 4 items + + test_verbosity_example.py .FFF [100%] + + ================================= FAILURES ================================= + _____________________________ test_words_fail ______________________________ + + def test_words_fail(): + fruits1 = ["banana", "apple", "grapes", "melon", "kiwi"] + fruits2 = ["banana", "apple", "orange", "melon", "kiwi"] + > assert fruits1 == fruits2 + E AssertionError: assert ['banana', 'a...elon', 'kiwi'] == ['banana', 'a...elon', 'kiwi'] + E At index 2 diff: 'grapes' != 'orange' + E Use -v to get the full diff + + test_verbosity_example.py:8: AssertionError + ____________________________ test_numbers_fail _____________________________ + + def test_numbers_fail(): + number_to_text1 = {str(x): x for x in range(5)} + number_to_text2 = {str(x * 10): x * 10 for x in range(5)} + > assert number_to_text1 == number_to_text2 + E AssertionError: assert {'0': 0, '1':..., '3': 3, ...} == {'0': 0, '10'...'30': 30, ...} + E Omitting 1 identical items, use -vv to show + E Left contains 4 more items: + E {'1': 1, '2': 2, '3': 3, '4': 4} + E Right contains 4 more items: + E {'10': 10, '20': 20, '30': 30, '40': 40} + E Use -v to get the full diff + + test_verbosity_example.py:14: AssertionError + ___________________________ test_long_text_fail ____________________________ + + def test_long_text_fail(): + long_text = "Lorem ipsum dolor sit amet " * 10 + > assert "hello world" in long_text + E AssertionError: assert 'hello world' in 'Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ips... sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet ' + + test_verbosity_example.py:19: AssertionError + ========================= short test summary info ========================== + FAILED test_verbosity_example.py::test_words_fail - AssertionError: asser... + FAILED test_verbosity_example.py::test_numbers_fail - AssertionError: ass... + FAILED test_verbosity_example.py::test_long_text_fail - AssertionError: a... + ======================= 3 failed, 1 passed in 0.12s ======================== + +Notice that: + +* Each test inside the file is shown by a single character in the output: ``.`` for passing, ``F`` for failure. +* ``test_words_fail`` failed, and we are shown a short summary indicating the index 2 of the two lists differ. +* ``test_numbers_fail`` failed, and we are shown a summary of left/right differences on dictionary items. Identical items are omitted. +* ``test_long_text_fail`` failed, and the right hand side of the ``in`` statement is truncated using ``...``` + because it is longer than an internal threshold (240 characters currently). + +Now we can increase pytest's verbosity: + +.. code-block:: pytest + + $ pytest --no-header -v + =========================== test session starts ============================ + collecting ... collected 4 items + + test_verbosity_example.py::test_ok PASSED [ 25%] + test_verbosity_example.py::test_words_fail FAILED [ 50%] + test_verbosity_example.py::test_numbers_fail FAILED [ 75%] + test_verbosity_example.py::test_long_text_fail FAILED [100%] + + ================================= FAILURES ================================= + _____________________________ test_words_fail ______________________________ + + def test_words_fail(): + fruits1 = ["banana", "apple", "grapes", "melon", "kiwi"] + fruits2 = ["banana", "apple", "orange", "melon", "kiwi"] + > assert fruits1 == fruits2 + E AssertionError: assert ['banana', 'a...elon', 'kiwi'] == ['banana', 'a...elon', 'kiwi'] + E At index 2 diff: 'grapes' != 'orange' + E Full diff: + E - ['banana', 'apple', 'orange', 'melon', 'kiwi'] + E ? ^ ^^ + E + ['banana', 'apple', 'grapes', 'melon', 'kiwi'] + E ? ^ ^ + + + test_verbosity_example.py:8: AssertionError + ____________________________ test_numbers_fail _____________________________ + + def test_numbers_fail(): + number_to_text1 = {str(x): x for x in range(5)} + number_to_text2 = {str(x * 10): x * 10 for x in range(5)} + > assert number_to_text1 == number_to_text2 + E AssertionError: assert {'0': 0, '1':..., '3': 3, ...} == {'0': 0, '10'...'30': 30, ...} + E Omitting 1 identical items, use -vv to show + E Left contains 4 more items: + E {'1': 1, '2': 2, '3': 3, '4': 4} + E Right contains 4 more items: + E {'10': 10, '20': 20, '30': 30, '40': 40} + E Full diff: + E - {'0': 0, '10': 10, '20': 20, '30': 30, '40': 40}... + E + E ...Full output truncated (3 lines hidden), use '-vv' to show + + test_verbosity_example.py:14: AssertionError + ___________________________ test_long_text_fail ____________________________ + + def test_long_text_fail(): + long_text = "Lorem ipsum dolor sit amet " * 10 + > assert "hello world" in long_text + E AssertionError: assert 'hello world' in 'Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet ' + + test_verbosity_example.py:19: AssertionError + ========================= short test summary info ========================== + FAILED test_verbosity_example.py::test_words_fail - AssertionError: asser... + FAILED test_verbosity_example.py::test_numbers_fail - AssertionError: ass... + FAILED test_verbosity_example.py::test_long_text_fail - AssertionError: a... + ======================= 3 failed, 1 passed in 0.12s ======================== + +Notice now that: + +* Each test inside the file gets its own line in the output. +* ``test_words_fail`` now shows the two failing lists in full, in addition to which index differs. +* ``test_numbers_fail`` now shows a text diff of the two dictionaries, truncated. +* ``test_long_text_fail`` no longer truncates the right hand side of the ``in`` statement, because the internal + threshold for truncation is larger now (2400 characters currently). + +Now if we increase verbosity even more: + +.. code-block:: pytest + + $ pytest --no-header -vv + =========================== test session starts ============================ + collecting ... collected 4 items + + test_verbosity_example.py::test_ok PASSED [ 25%] + test_verbosity_example.py::test_words_fail FAILED [ 50%] + test_verbosity_example.py::test_numbers_fail FAILED [ 75%] + test_verbosity_example.py::test_long_text_fail FAILED [100%] + + ================================= FAILURES ================================= + _____________________________ test_words_fail ______________________________ + + def test_words_fail(): + fruits1 = ["banana", "apple", "grapes", "melon", "kiwi"] + fruits2 = ["banana", "apple", "orange", "melon", "kiwi"] + > assert fruits1 == fruits2 + E AssertionError: assert ['banana', 'apple', 'grapes', 'melon', 'kiwi'] == ['banana', 'apple', 'orange', 'melon', 'kiwi'] + E At index 2 diff: 'grapes' != 'orange' + E Full diff: + E - ['banana', 'apple', 'orange', 'melon', 'kiwi'] + E ? ^ ^^ + E + ['banana', 'apple', 'grapes', 'melon', 'kiwi'] + E ? ^ ^ + + + test_verbosity_example.py:8: AssertionError + ____________________________ test_numbers_fail _____________________________ + + def test_numbers_fail(): + number_to_text1 = {str(x): x for x in range(5)} + number_to_text2 = {str(x * 10): x * 10 for x in range(5)} + > assert number_to_text1 == number_to_text2 + E AssertionError: assert {'0': 0, '1': 1, '2': 2, '3': 3, '4': 4} == {'0': 0, '10': 10, '20': 20, '30': 30, '40': 40} + E Common items: + E {'0': 0} + E Left contains 4 more items: + E {'1': 1, '2': 2, '3': 3, '4': 4} + E Right contains 4 more items: + E {'10': 10, '20': 20, '30': 30, '40': 40} + E Full diff: + E - {'0': 0, '10': 10, '20': 20, '30': 30, '40': 40} + E ? - - - - - - - - + E + {'0': 0, '1': 1, '2': 2, '3': 3, '4': 4} + + test_verbosity_example.py:14: AssertionError + ___________________________ test_long_text_fail ____________________________ + + def test_long_text_fail(): + long_text = "Lorem ipsum dolor sit amet " * 10 + > assert "hello world" in long_text + E AssertionError: assert 'hello world' in 'Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet Lorem ipsum dolor sit amet ' + + test_verbosity_example.py:19: AssertionError + ========================= short test summary info ========================== + FAILED test_verbosity_example.py::test_words_fail - AssertionError: asser... + FAILED test_verbosity_example.py::test_numbers_fail - AssertionError: ass... + FAILED test_verbosity_example.py::test_long_text_fail - AssertionError: a... + ======================= 3 failed, 1 passed in 0.12s ======================== + +Notice now that: + +* Each test inside the file gets its own line in the output. +* ``test_words_fail`` gives the same output as before in this case. +* ``test_numbers_fail`` now shows a full text diff of the two dictionaries. +* ``test_long_text_fail`` also doesn't truncate on the right hand side as before, but now pytest won't truncate any + text at all, regardless of its size. + +Those were examples of how verbosity affects normal test session output, but verbosity also is used in other +situations, for example you are shown even fixtures that start with ``_`` if you use ``pytest --fixtures -v``. + +Using higher verbosity levels (``-vvv``, ``-vvvv``, ...) is supported, but has no effect in pytest itself at the moment, +however some plugins might make use of higher verbosity. + +.. _`pytest.detailed_failed_tests_usage`: + +Producing a detailed summary report +-------------------------------------------------- + +The ``-r`` flag can be used to display a "short test summary info" at the end of the test session, +making it easy in large test suites to get a clear picture of all failures, skips, xfails, etc. + +It defaults to ``fE`` to list failures and errors. + +.. regendoc:wipe + +Example: + +.. code-block:: python + + # content of test_example.py + import pytest + + + @pytest.fixture + def error_fixture(): + assert 0 + + + def test_ok(): + print("ok") + + + def test_fail(): + assert 0 + + + def test_error(error_fixture): + pass + + + def test_skip(): + pytest.skip("skipping this test") + + + def test_xfail(): + pytest.xfail("xfailing this test") + + + @pytest.mark.xfail(reason="always xfail") + def test_xpass(): + pass + + +.. code-block:: pytest + + $ pytest -ra + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 6 items + + test_example.py .FEsxX [100%] + + ================================== ERRORS ================================== + _______________________ ERROR at setup of test_error _______________________ + + @pytest.fixture + def error_fixture(): + > assert 0 + E assert 0 + + test_example.py:6: AssertionError + ================================= FAILURES ================================= + ________________________________ test_fail _________________________________ + + def test_fail(): + > assert 0 + E assert 0 + + test_example.py:14: AssertionError + ========================= short test summary info ========================== + SKIPPED [1] test_example.py:22: skipping this test + XFAIL test_example.py::test_xfail + reason: xfailing this test + XPASS test_example.py::test_xpass always xfail + ERROR test_example.py::test_error - assert 0 + FAILED test_example.py::test_fail - assert 0 + == 1 failed, 1 passed, 1 skipped, 1 xfailed, 1 xpassed, 1 error in 0.12s === + +The ``-r`` options accepts a number of characters after it, with ``a`` used +above meaning "all except passes". + +Here is the full list of available characters that can be used: + + - ``f`` - failed + - ``E`` - error + - ``s`` - skipped + - ``x`` - xfailed + - ``X`` - xpassed + - ``p`` - passed + - ``P`` - passed with output + +Special characters for (de)selection of groups: + + - ``a`` - all except ``pP`` + - ``A`` - all + - ``N`` - none, this can be used to display nothing (since ``fE`` is the default) + +More than one character can be used, so for example to only see failed and skipped tests, you can execute: + +.. code-block:: pytest + + $ pytest -rfs + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 6 items + + test_example.py .FEsxX [100%] + + ================================== ERRORS ================================== + _______________________ ERROR at setup of test_error _______________________ + + @pytest.fixture + def error_fixture(): + > assert 0 + E assert 0 + + test_example.py:6: AssertionError + ================================= FAILURES ================================= + ________________________________ test_fail _________________________________ + + def test_fail(): + > assert 0 + E assert 0 + + test_example.py:14: AssertionError + ========================= short test summary info ========================== + FAILED test_example.py::test_fail - assert 0 + SKIPPED [1] test_example.py:22: skipping this test + == 1 failed, 1 passed, 1 skipped, 1 xfailed, 1 xpassed, 1 error in 0.12s === + +Using ``p`` lists the passing tests, whilst ``P`` adds an extra section "PASSES" with those tests that passed but had +captured output: + +.. code-block:: pytest + + $ pytest -rpP + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 6 items + + test_example.py .FEsxX [100%] + + ================================== ERRORS ================================== + _______________________ ERROR at setup of test_error _______________________ + + @pytest.fixture + def error_fixture(): + > assert 0 + E assert 0 + + test_example.py:6: AssertionError + ================================= FAILURES ================================= + ________________________________ test_fail _________________________________ + + def test_fail(): + > assert 0 + E assert 0 + + test_example.py:14: AssertionError + ================================== PASSES ================================== + _________________________________ test_ok __________________________________ + --------------------------- Captured stdout call --------------------------- + ok + ========================= short test summary info ========================== + PASSED test_example.py::test_ok + == 1 failed, 1 passed, 1 skipped, 1 xfailed, 1 xpassed, 1 error in 0.12s === + +Creating resultlog format files +-------------------------------------------------- + +To create plain-text machine-readable result files you can issue: + +.. code-block:: bash + + pytest --resultlog=path + +and look at the content at the ``path`` location. Such files are used e.g. +by the `PyPy-test`_ web page to show test results over several revisions. + +.. warning:: + + This option is rarely used and is scheduled for removal in pytest 6.0. + + If you use this option, consider using the new `pytest-reportlog `__ plugin instead. + + See :ref:`the deprecation docs ` for more information. + + +.. _`PyPy-test`: http://buildbot.pypy.org/summary + + +Creating JUnitXML format files +---------------------------------------------------- + +To create result files which can be read by Jenkins_ or other Continuous +integration servers, use this invocation: + +.. code-block:: bash + + pytest --junitxml=path + +to create an XML file at ``path``. + + + +To set the name of the root test suite xml item, you can configure the ``junit_suite_name`` option in your config file: + +.. code-block:: ini + + [pytest] + junit_suite_name = my_suite + +.. versionadded:: 4.0 + +JUnit XML specification seems to indicate that ``"time"`` attribute +should report total test execution times, including setup and teardown +(`1 `_, `2 +`_). +It is the default pytest behavior. To report just call durations +instead, configure the ``junit_duration_report`` option like this: + +.. code-block:: ini + + [pytest] + junit_duration_report = call + +.. _record_property example: + +record_property +~~~~~~~~~~~~~~~~~ + +If you want to log additional information for a test, you can use the +``record_property`` fixture: + +.. code-block:: python + + def test_function(record_property): + record_property("example_key", 1) + assert True + +This will add an extra property ``example_key="1"`` to the generated +``testcase`` tag: + +.. code-block:: xml + + + + + + + +Alternatively, you can integrate this functionality with custom markers: + +.. code-block:: python + + # content of conftest.py + + + def pytest_collection_modifyitems(session, config, items): + for item in items: + for marker in item.iter_markers(name="test_id"): + test_id = marker.args[0] + item.user_properties.append(("test_id", test_id)) + +And in your tests: + +.. code-block:: python + + # content of test_function.py + import pytest + + + @pytest.mark.test_id(1501) + def test_function(): + assert True + +Will result in: + +.. code-block:: xml + + + + + + + +.. warning:: + + Please note that using this feature will break schema verifications for the latest JUnitXML schema. + This might be a problem when used with some CI servers. + + +record_xml_attribute +~~~~~~~~~~~~~~~~~~~~~~~ + +To add an additional xml attribute to a testcase element, you can use +``record_xml_attribute`` fixture. This can also be used to override existing values: + +.. code-block:: python + + def test_function(record_xml_attribute): + record_xml_attribute("assertions", "REQ-1234") + record_xml_attribute("classname", "custom_classname") + print("hello world") + assert True + +Unlike ``record_property``, this will not add a new child element. +Instead, this will add an attribute ``assertions="REQ-1234"`` inside the generated +``testcase`` tag and override the default ``classname`` with ``"classname=custom_classname"``: + +.. code-block:: xml + + + + hello world + + + +.. warning:: + + ``record_xml_attribute`` is an experimental feature, and its interface might be replaced + by something more powerful and general in future versions. The + functionality per-se will be kept, however. + + Using this over ``record_xml_property`` can help when using ci tools to parse the xml report. + However, some parsers are quite strict about the elements and attributes that are allowed. + Many tools use an xsd schema (like the example below) to validate incoming xml. + Make sure you are using attribute names that are allowed by your parser. + + Below is the Scheme used by Jenkins to validate the XML report: + + .. code-block:: xml + + + + + + + + + + + + + + + + + + +.. warning:: + + Please note that using this feature will break schema verifications for the latest JUnitXML schema. + This might be a problem when used with some CI servers. + +.. _record_testsuite_property example: + +record_testsuite_property +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 4.5 + +If you want to add a properties node at the test-suite level, which may contains properties +that are relevant to all tests, you can use the ``record_testsuite_property`` session-scoped fixture: + +The ``record_testsuite_property`` session-scoped fixture can be used to add properties relevant +to all tests. + +.. code-block:: python + + import pytest + + + @pytest.fixture(scope="session", autouse=True) + def log_global_env_facts(record_testsuite_property): + record_testsuite_property("ARCH", "PPC") + record_testsuite_property("STORAGE_TYPE", "CEPH") + + + class TestMe: + def test_foo(self): + assert True + +The fixture is a callable which receives ``name`` and ``value`` of a ```` tag +added at the test-suite level of the generated xml: + +.. code-block:: xml + + + + + + + + + +``name`` must be a string, ``value`` will be converted to a string and properly xml-escaped. + +The generated XML is compatible with the latest ``xunit`` standard, contrary to `record_property`_ +and `record_xml_attribute`_. + + +Sending test report to an online pastebin service +-------------------------------------------------- + +**Creating a URL for each test failure**: + +.. code-block:: bash + + pytest --pastebin=failed + +This will submit test run information to a remote Paste service and +provide a URL for each failure. You may select tests as usual or add +for example ``-x`` if you only want to send one particular failure. + +**Creating a URL for a whole test session log**: + +.. code-block:: bash + + pytest --pastebin=all + +Currently only pasting to the https://bpaste.net/ service is implemented. + +.. versionchanged:: 5.2 + +If creating the URL fails for any reason, a warning is generated instead of failing the +entire test suite. + +.. _jenkins: https://jenkins-ci.org diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/parametrize.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/parametrize.rst new file mode 100644 index 0000000000..a0c9968428 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/parametrize.rst @@ -0,0 +1,298 @@ + +.. _`test generators`: +.. _`parametrizing-tests`: +.. _`parametrized test functions`: +.. _`parametrize`: + +.. _`parametrize-basics`: + +How to parametrize fixtures and test functions +========================================================================== + +pytest enables test parametrization at several levels: + +- :py:func:`pytest.fixture` allows one to :ref:`parametrize fixture + functions `. + +* `@pytest.mark.parametrize`_ allows one to define multiple sets of + arguments and fixtures at the test function or class. + +* `pytest_generate_tests`_ allows one to define custom parametrization + schemes or extensions. + +.. _parametrizemark: +.. _`@pytest.mark.parametrize`: + + +``@pytest.mark.parametrize``: parametrizing test functions +--------------------------------------------------------------------- + +.. regendoc: wipe + + + + Several improvements. + +The builtin :ref:`pytest.mark.parametrize ref` decorator enables +parametrization of arguments for a test function. Here is a typical example +of a test function that implements checking that a certain input leads +to an expected output: + +.. code-block:: python + + # content of test_expectation.py + import pytest + + + @pytest.mark.parametrize("test_input,expected", [("3+5", 8), ("2+4", 6), ("6*9", 42)]) + def test_eval(test_input, expected): + assert eval(test_input) == expected + +Here, the ``@parametrize`` decorator defines three different ``(test_input,expected)`` +tuples so that the ``test_eval`` function will run three times using +them in turn: + +.. code-block:: pytest + + $ pytest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 3 items + + test_expectation.py ..F [100%] + + ================================= FAILURES ================================= + ____________________________ test_eval[6*9-42] _____________________________ + + test_input = '6*9', expected = 42 + + @pytest.mark.parametrize("test_input,expected", [("3+5", 8), ("2+4", 6), ("6*9", 42)]) + def test_eval(test_input, expected): + > assert eval(test_input) == expected + E AssertionError: assert 54 == 42 + E + where 54 = eval('6*9') + + test_expectation.py:6: AssertionError + ========================= short test summary info ========================== + FAILED test_expectation.py::test_eval[6*9-42] - AssertionError: assert 54... + ======================= 1 failed, 2 passed in 0.12s ======================== + +.. note:: + + Parameter values are passed as-is to tests (no copy whatsoever). + + For example, if you pass a list or a dict as a parameter value, and + the test case code mutates it, the mutations will be reflected in subsequent + test case calls. + +.. note:: + + pytest by default escapes any non-ascii characters used in unicode strings + for the parametrization because it has several downsides. + If however you would like to use unicode strings in parametrization + and see them in the terminal as is (non-escaped), use this option + in your ``pytest.ini``: + + .. code-block:: ini + + [pytest] + disable_test_id_escaping_and_forfeit_all_rights_to_community_support = True + + Keep in mind however that this might cause unwanted side effects and + even bugs depending on the OS used and plugins currently installed, + so use it at your own risk. + + +As designed in this example, only one pair of input/output values fails +the simple test function. And as usual with test function arguments, +you can see the ``input`` and ``output`` values in the traceback. + +Note that you could also use the parametrize marker on a class or a module +(see :ref:`mark`) which would invoke several functions with the argument sets, +for instance: + + +.. code-block:: python + + import pytest + + + @pytest.mark.parametrize("n,expected", [(1, 2), (3, 4)]) + class TestClass: + def test_simple_case(self, n, expected): + assert n + 1 == expected + + def test_weird_simple_case(self, n, expected): + assert (n * 1) + 1 == expected + + +To parametrize all tests in a module, you can assign to the :globalvar:`pytestmark` global variable: + + +.. code-block:: python + + import pytest + + pytestmark = pytest.mark.parametrize("n,expected", [(1, 2), (3, 4)]) + + + class TestClass: + def test_simple_case(self, n, expected): + assert n + 1 == expected + + def test_weird_simple_case(self, n, expected): + assert (n * 1) + 1 == expected + + +It is also possible to mark individual test instances within parametrize, +for example with the builtin ``mark.xfail``: + +.. code-block:: python + + # content of test_expectation.py + import pytest + + + @pytest.mark.parametrize( + "test_input,expected", + [("3+5", 8), ("2+4", 6), pytest.param("6*9", 42, marks=pytest.mark.xfail)], + ) + def test_eval(test_input, expected): + assert eval(test_input) == expected + +Let's run this: + +.. code-block:: pytest + + $ pytest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 3 items + + test_expectation.py ..x [100%] + + ======================= 2 passed, 1 xfailed in 0.12s ======================= + +The one parameter set which caused a failure previously now +shows up as an "xfailed" (expected to fail) test. + +In case the values provided to ``parametrize`` result in an empty list - for +example, if they're dynamically generated by some function - the behaviour of +pytest is defined by the :confval:`empty_parameter_set_mark` option. + +To get all combinations of multiple parametrized arguments you can stack +``parametrize`` decorators: + +.. code-block:: python + + import pytest + + + @pytest.mark.parametrize("x", [0, 1]) + @pytest.mark.parametrize("y", [2, 3]) + def test_foo(x, y): + pass + +This will run the test with the arguments set to ``x=0/y=2``, ``x=1/y=2``, +``x=0/y=3``, and ``x=1/y=3`` exhausting parameters in the order of the decorators. + +.. _`pytest_generate_tests`: + +Basic ``pytest_generate_tests`` example +--------------------------------------------- + +Sometimes you may want to implement your own parametrization scheme +or implement some dynamism for determining the parameters or scope +of a fixture. For this, you can use the ``pytest_generate_tests`` hook +which is called when collecting a test function. Through the passed in +``metafunc`` object you can inspect the requesting test context and, most +importantly, you can call ``metafunc.parametrize()`` to cause +parametrization. + +For example, let's say we want to run a test taking string inputs which +we want to set via a new ``pytest`` command line option. Let's first write +a simple test accepting a ``stringinput`` fixture function argument: + +.. code-block:: python + + # content of test_strings.py + + + def test_valid_string(stringinput): + assert stringinput.isalpha() + +Now we add a ``conftest.py`` file containing the addition of a +command line option and the parametrization of our test function: + +.. code-block:: python + + # content of conftest.py + + + def pytest_addoption(parser): + parser.addoption( + "--stringinput", + action="append", + default=[], + help="list of stringinputs to pass to test functions", + ) + + + def pytest_generate_tests(metafunc): + if "stringinput" in metafunc.fixturenames: + metafunc.parametrize("stringinput", metafunc.config.getoption("stringinput")) + +If we now pass two stringinput values, our test will run twice: + +.. code-block:: pytest + + $ pytest -q --stringinput="hello" --stringinput="world" test_strings.py + .. [100%] + 2 passed in 0.12s + +Let's also run with a stringinput that will lead to a failing test: + +.. code-block:: pytest + + $ pytest -q --stringinput="!" test_strings.py + F [100%] + ================================= FAILURES ================================= + ___________________________ test_valid_string[!] ___________________________ + + stringinput = '!' + + def test_valid_string(stringinput): + > assert stringinput.isalpha() + E AssertionError: assert False + E + where False = () + E + where = '!'.isalpha + + test_strings.py:4: AssertionError + ========================= short test summary info ========================== + FAILED test_strings.py::test_valid_string[!] - AssertionError: assert False + 1 failed in 0.12s + +As expected our test function fails. + +If you don't specify a stringinput it will be skipped because +``metafunc.parametrize()`` will be called with an empty parameter +list: + +.. code-block:: pytest + + $ pytest -q -rs test_strings.py + s [100%] + ========================= short test summary info ========================== + SKIPPED [1] test_strings.py: got empty parameter set ['stringinput'], function test_valid_string at /home/sweet/project/test_strings.py:2 + 1 skipped in 0.12s + +Note that when calling ``metafunc.parametrize`` multiple times with different parameter sets, all parameter names across +those sets cannot be duplicated, otherwise an error will be raised. + +More examples +------------- + +For further examples, you might want to look at :ref:`more +parametrization examples `. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/plugins.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/plugins.rst new file mode 100644 index 0000000000..cae737e96e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/plugins.rst @@ -0,0 +1,136 @@ +.. _`external plugins`: +.. _`extplugins`: +.. _`using plugins`: + +How to install and use plugins +=============================== + +This section talks about installing and using third party plugins. +For writing your own plugins, please refer to :ref:`writing-plugins`. + +Installing a third party plugin can be easily done with ``pip``: + +.. code-block:: bash + + pip install pytest-NAME + pip uninstall pytest-NAME + +If a plugin is installed, ``pytest`` automatically finds and integrates it, +there is no need to activate it. + +Here is a little annotated list for some popular plugins: + +* :pypi:`pytest-django`: write tests + for :std:doc:`django ` apps, using pytest integration. + +* :pypi:`pytest-twisted`: write tests + for `twisted `_ apps, starting a reactor and + processing deferreds from test functions. + +* :pypi:`pytest-cov`: + coverage reporting, compatible with distributed testing + +* :pypi:`pytest-xdist`: + to distribute tests to CPUs and remote hosts, to run in boxed + mode which allows to survive segmentation faults, to run in + looponfailing mode, automatically re-running failing tests + on file changes. + +* :pypi:`pytest-instafail`: + to report failures while the test run is happening. + +* :pypi:`pytest-bdd`: + to write tests using behaviour-driven testing. + +* :pypi:`pytest-timeout`: + to timeout tests based on function marks or global definitions. + +* :pypi:`pytest-pep8`: + a ``--pep8`` option to enable PEP8 compliance checking. + +* :pypi:`pytest-flakes`: + check source code with pyflakes. + +* :pypi:`oejskit`: + a plugin to run javascript unittests in live browsers. + +To see a complete list of all plugins with their latest testing +status against different pytest and Python versions, please visit +:ref:`plugin-list`. + +You may also discover more plugins through a `pytest- pypi.org search`_. + +.. _`pytest- pypi.org search`: https://pypi.org/search/?q=pytest- + + +.. _`available installable plugins`: + +Requiring/Loading plugins in a test module or conftest file +----------------------------------------------------------- + +You can require plugins in a test module or a conftest file using :globalvar:`pytest_plugins`: + +.. code-block:: python + + pytest_plugins = ("myapp.testsupport.myplugin",) + +When the test module or conftest plugin is loaded the specified plugins +will be loaded as well. + +.. note:: + + Requiring plugins using a ``pytest_plugins`` variable in non-root + ``conftest.py`` files is deprecated. See + :ref:`full explanation ` + in the Writing plugins section. + +.. note:: + The name ``pytest_plugins`` is reserved and should not be used as a + name for a custom plugin module. + + +.. _`findpluginname`: + +Finding out which plugins are active +------------------------------------ + +If you want to find out which plugins are active in your +environment you can type: + +.. code-block:: bash + + pytest --trace-config + +and will get an extended test header which shows activated plugins +and their names. It will also print local plugins aka +:ref:`conftest.py ` files when they are loaded. + +.. _`cmdunregister`: + +Deactivating / unregistering a plugin by name +--------------------------------------------- + +You can prevent plugins from loading or unregister them: + +.. code-block:: bash + + pytest -p no:NAME + +This means that any subsequent try to activate/load the named +plugin will not work. + +If you want to unconditionally disable a plugin for a project, you can add +this option to your ``pytest.ini`` file: + +.. code-block:: ini + + [pytest] + addopts = -p no:NAME + +Alternatively to disable it only in certain environments (for example in a +CI server), you can set ``PYTEST_ADDOPTS`` environment variable to +``-p no:name``. + +See :ref:`findpluginname` for how to obtain the name of a plugin. + +.. _`builtin plugins`: diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/skipping.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/skipping.rst new file mode 100644 index 0000000000..e2f59c77ae --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/skipping.rst @@ -0,0 +1,430 @@ +.. _`skip and xfail`: + +.. _skipping: + +How to use skip and xfail to deal with tests that cannot succeed +================================================================= + +You can mark test functions that cannot be run on certain platforms +or that you expect to fail so pytest can deal with them accordingly and +present a summary of the test session, while keeping the test suite *green*. + +A **skip** means that you expect your test to pass only if some conditions are met, +otherwise pytest should skip running the test altogether. Common examples are skipping +windows-only tests on non-windows platforms, or skipping tests that depend on an external +resource which is not available at the moment (for example a database). + +An **xfail** means that you expect a test to fail for some reason. +A common example is a test for a feature not yet implemented, or a bug not yet fixed. +When a test passes despite being expected to fail (marked with ``pytest.mark.xfail``), +it's an **xpass** and will be reported in the test summary. + +``pytest`` counts and lists *skip* and *xfail* tests separately. Detailed +information about skipped/xfailed tests is not shown by default to avoid +cluttering the output. You can use the ``-r`` option to see details +corresponding to the "short" letters shown in the test progress: + +.. code-block:: bash + + pytest -rxXs # show extra info on xfailed, xpassed, and skipped tests + +More details on the ``-r`` option can be found by running ``pytest -h``. + +(See :ref:`how to change command line options defaults`) + +.. _skipif: +.. _skip: +.. _`condition booleans`: + +Skipping test functions +----------------------- + + + +The simplest way to skip a test function is to mark it with the ``skip`` decorator +which may be passed an optional ``reason``: + +.. code-block:: python + + @pytest.mark.skip(reason="no way of currently testing this") + def test_the_unknown(): + ... + + +Alternatively, it is also possible to skip imperatively during test execution or setup +by calling the ``pytest.skip(reason)`` function: + +.. code-block:: python + + def test_function(): + if not valid_config(): + pytest.skip("unsupported configuration") + +The imperative method is useful when it is not possible to evaluate the skip condition +during import time. + +It is also possible to skip the whole module using +``pytest.skip(reason, allow_module_level=True)`` at the module level: + +.. code-block:: python + + import sys + import pytest + + if not sys.platform.startswith("win"): + pytest.skip("skipping windows-only tests", allow_module_level=True) + + +**Reference**: :ref:`pytest.mark.skip ref` + +``skipif`` +~~~~~~~~~~ + + + +If you wish to skip something conditionally then you can use ``skipif`` instead. +Here is an example of marking a test function to be skipped +when run on an interpreter earlier than Python3.6: + +.. code-block:: python + + import sys + + + @pytest.mark.skipif(sys.version_info < (3, 7), reason="requires python3.7 or higher") + def test_function(): + ... + +If the condition evaluates to ``True`` during collection, the test function will be skipped, +with the specified reason appearing in the summary when using ``-rs``. + +You can share ``skipif`` markers between modules. Consider this test module: + +.. code-block:: python + + # content of test_mymodule.py + import mymodule + + minversion = pytest.mark.skipif( + mymodule.__versioninfo__ < (1, 1), reason="at least mymodule-1.1 required" + ) + + + @minversion + def test_function(): + ... + +You can import the marker and reuse it in another test module: + +.. code-block:: python + + # test_myothermodule.py + from test_mymodule import minversion + + + @minversion + def test_anotherfunction(): + ... + +For larger test suites it's usually a good idea to have one file +where you define the markers which you then consistently apply +throughout your test suite. + +Alternatively, you can use :ref:`condition strings +` instead of booleans, but they can't be shared between modules easily +so they are supported mainly for backward compatibility reasons. + +**Reference**: :ref:`pytest.mark.skipif ref` + + +Skip all test functions of a class or module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can use the ``skipif`` marker (as any other marker) on classes: + +.. code-block:: python + + @pytest.mark.skipif(sys.platform == "win32", reason="does not run on windows") + class TestPosixCalls: + def test_function(self): + "will not be setup or run under 'win32' platform" + +If the condition is ``True``, this marker will produce a skip result for +each of the test methods of that class. + +If you want to skip all test functions of a module, you may use the +:globalvar:`pytestmark` global: + +.. code-block:: python + + # test_module.py + pytestmark = pytest.mark.skipif(...) + +If multiple ``skipif`` decorators are applied to a test function, it +will be skipped if any of the skip conditions is true. + +.. _`whole class- or module level`: mark.html#scoped-marking + + +Skipping files or directories +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sometimes you may need to skip an entire file or directory, for example if the +tests rely on Python version-specific features or contain code that you do not +wish pytest to run. In this case, you must exclude the files and directories +from collection. Refer to :ref:`customizing-test-collection` for more +information. + + +Skipping on a missing import dependency +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can skip tests on a missing import by using :ref:`pytest.importorskip ref` +at module level, within a test, or test setup function. + +.. code-block:: python + + docutils = pytest.importorskip("docutils") + +If ``docutils`` cannot be imported here, this will lead to a skip outcome of +the test. You can also skip based on the version number of a library: + +.. code-block:: python + + docutils = pytest.importorskip("docutils", minversion="0.3") + +The version will be read from the specified +module's ``__version__`` attribute. + +Summary +~~~~~~~ + +Here's a quick guide on how to skip tests in a module in different situations: + +1. Skip all tests in a module unconditionally: + + .. code-block:: python + + pytestmark = pytest.mark.skip("all tests still WIP") + +2. Skip all tests in a module based on some condition: + + .. code-block:: python + + pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="tests for linux only") + +3. Skip all tests in a module if some import is missing: + + .. code-block:: python + + pexpect = pytest.importorskip("pexpect") + + +.. _xfail: + +XFail: mark test functions as expected to fail +---------------------------------------------- + +You can use the ``xfail`` marker to indicate that you +expect a test to fail: + +.. code-block:: python + + @pytest.mark.xfail + def test_function(): + ... + +This test will run but no traceback will be reported when it fails. Instead, terminal +reporting will list it in the "expected to fail" (``XFAIL``) or "unexpectedly +passing" (``XPASS``) sections. + +Alternatively, you can also mark a test as ``XFAIL`` from within the test or its setup function +imperatively: + +.. code-block:: python + + def test_function(): + if not valid_config(): + pytest.xfail("failing configuration (but should work)") + +.. code-block:: python + + def test_function2(): + import slow_module + + if slow_module.slow_function(): + pytest.xfail("slow_module taking too long") + +These two examples illustrate situations where you don't want to check for a condition +at the module level, which is when a condition would otherwise be evaluated for marks. + +This will make ``test_function`` ``XFAIL``. Note that no other code is executed after +the :func:`pytest.xfail` call, differently from the marker. That's because it is implemented +internally by raising a known exception. + +**Reference**: :ref:`pytest.mark.xfail ref` + + +``condition`` parameter +~~~~~~~~~~~~~~~~~~~~~~~ + +If a test is only expected to fail under a certain condition, you can pass +that condition as the first parameter: + +.. code-block:: python + + @pytest.mark.xfail(sys.platform == "win32", reason="bug in a 3rd party library") + def test_function(): + ... + +Note that you have to pass a reason as well (see the parameter description at +:ref:`pytest.mark.xfail ref`). + +``reason`` parameter +~~~~~~~~~~~~~~~~~~~~ + +You can specify the motive of an expected failure with the ``reason`` parameter: + +.. code-block:: python + + @pytest.mark.xfail(reason="known parser issue") + def test_function(): + ... + + +``raises`` parameter +~~~~~~~~~~~~~~~~~~~~ + +If you want to be more specific as to why the test is failing, you can specify +a single exception, or a tuple of exceptions, in the ``raises`` argument. + +.. code-block:: python + + @pytest.mark.xfail(raises=RuntimeError) + def test_function(): + ... + +Then the test will be reported as a regular failure if it fails with an +exception not mentioned in ``raises``. + +``run`` parameter +~~~~~~~~~~~~~~~~~ + +If a test should be marked as xfail and reported as such but should not be +even executed, use the ``run`` parameter as ``False``: + +.. code-block:: python + + @pytest.mark.xfail(run=False) + def test_function(): + ... + +This is specially useful for xfailing tests that are crashing the interpreter and should be +investigated later. + +.. _`xfail strict tutorial`: + +``strict`` parameter +~~~~~~~~~~~~~~~~~~~~ + +Both ``XFAIL`` and ``XPASS`` don't fail the test suite by default. +You can change this by setting the ``strict`` keyword-only parameter to ``True``: + +.. code-block:: python + + @pytest.mark.xfail(strict=True) + def test_function(): + ... + + +This will make ``XPASS`` ("unexpectedly passing") results from this test to fail the test suite. + +You can change the default value of the ``strict`` parameter using the +``xfail_strict`` ini option: + +.. code-block:: ini + + [pytest] + xfail_strict=true + + +Ignoring xfail +~~~~~~~~~~~~~~ + +By specifying on the commandline: + +.. code-block:: bash + + pytest --runxfail + +you can force the running and reporting of an ``xfail`` marked test +as if it weren't marked at all. This also causes :func:`pytest.xfail` to produce no effect. + +Examples +~~~~~~~~ + +Here is a simple test file with the several usages: + +.. literalinclude:: /example/xfail_demo.py + +Running it with the report-on-xfail option gives this output: + +.. FIXME: Use $ instead of ! again to re-enable regendoc once it's fixed: + https://github.com/pytest-dev/pytest/issues/8807 + +.. code-block:: pytest + + ! pytest -rx xfail_demo.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-6.x.y, py-1.x.y, pluggy-1.x.y + cachedir: $PYTHON_PREFIX/.pytest_cache + rootdir: $REGENDOC_TMPDIR/example + collected 7 items + + xfail_demo.py xxxxxxx [100%] + + ========================= short test summary info ========================== + XFAIL xfail_demo.py::test_hello + XFAIL xfail_demo.py::test_hello2 + reason: [NOTRUN] + XFAIL xfail_demo.py::test_hello3 + condition: hasattr(os, 'sep') + XFAIL xfail_demo.py::test_hello4 + bug 110 + XFAIL xfail_demo.py::test_hello5 + condition: pytest.__version__[0] != "17" + XFAIL xfail_demo.py::test_hello6 + reason: reason + XFAIL xfail_demo.py::test_hello7 + ============================ 7 xfailed in 0.12s ============================ + +.. _`skip/xfail with parametrize`: + +Skip/xfail with parametrize +--------------------------- + +It is possible to apply markers like skip and xfail to individual +test instances when using parametrize: + +.. code-block:: python + + import sys + import pytest + + + @pytest.mark.parametrize( + ("n", "expected"), + [ + (1, 2), + pytest.param(1, 0, marks=pytest.mark.xfail), + pytest.param(1, 3, marks=pytest.mark.xfail(reason="some bug")), + (2, 3), + (3, 4), + (4, 5), + pytest.param( + 10, 11, marks=pytest.mark.skipif(sys.version_info >= (3, 0), reason="py2k") + ), + ], + ) + def test_increment(n, expected): + assert n + 1 == expected diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/tmp_path.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/tmp_path.rst new file mode 100644 index 0000000000..ebd74d42e9 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/tmp_path.rst @@ -0,0 +1,139 @@ + +.. _`tmp_path handling`: +.. _tmp_path: + +How to use temporary directories and files in tests +=================================================== + +The ``tmp_path`` fixture +------------------------ + +You can use the ``tmp_path`` fixture which will +provide a temporary directory unique to the test invocation, +created in the `base temporary directory`_. + +``tmp_path`` is a :class:`pathlib.Path` object. Here is an example test usage: + +.. code-block:: python + + # content of test_tmp_path.py + CONTENT = "content" + + + def test_create_file(tmp_path): + d = tmp_path / "sub" + d.mkdir() + p = d / "hello.txt" + p.write_text(CONTENT) + assert p.read_text() == CONTENT + assert len(list(tmp_path.iterdir())) == 1 + assert 0 + +Running this would result in a passed test except for the last +``assert 0`` line which we use to look at values: + +.. code-block:: pytest + + $ pytest test_tmp_path.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 1 item + + test_tmp_path.py F [100%] + + ================================= FAILURES ================================= + _____________________________ test_create_file _____________________________ + + tmp_path = PosixPath('PYTEST_TMPDIR/test_create_file0') + + def test_create_file(tmp_path): + d = tmp_path / "sub" + d.mkdir() + p = d / "hello.txt" + p.write_text(CONTENT) + assert p.read_text() == CONTENT + assert len(list(tmp_path.iterdir())) == 1 + > assert 0 + E assert 0 + + test_tmp_path.py:11: AssertionError + ========================= short test summary info ========================== + FAILED test_tmp_path.py::test_create_file - assert 0 + ============================ 1 failed in 0.12s ============================= + +.. _`tmp_path_factory example`: + +The ``tmp_path_factory`` fixture +-------------------------------- + +The ``tmp_path_factory`` is a session-scoped fixture which can be used +to create arbitrary temporary directories from any other fixture or test. + +For example, suppose your test suite needs a large image on disk, which is +generated procedurally. Instead of computing the same image for each test +that uses it into its own ``tmp_path``, you can generate it once per-session +to save time: + +.. code-block:: python + + # contents of conftest.py + import pytest + + + @pytest.fixture(scope="session") + def image_file(tmp_path_factory): + img = compute_expensive_image() + fn = tmp_path_factory.mktemp("data") / "img.png" + img.save(fn) + return fn + + + # contents of test_image.py + def test_histogram(image_file): + img = load_image(image_file) + # compute and test histogram + +See :ref:`tmp_path_factory API ` for details. + +.. _`tmpdir and tmpdir_factory`: +.. _tmpdir: + +The ``tmpdir`` and ``tmpdir_factory`` fixtures +--------------------------------------------------- + +The ``tmpdir`` and ``tmpdir_factory`` fixtures are similar to ``tmp_path`` +and ``tmp_path_factory``, but use/return legacy `py.path.local`_ objects +rather than standard :class:`pathlib.Path` objects. These days, prefer to +use ``tmp_path`` and ``tmp_path_factory``. + +See :fixture:`tmpdir ` :fixture:`tmpdir_factory ` +API for details. + + +.. _`base temporary directory`: + +The default base temporary directory +----------------------------------------------- + +Temporary directories are by default created as sub-directories of +the system temporary directory. The base name will be ``pytest-NUM`` where +``NUM`` will be incremented with each test run. Moreover, entries older +than 3 temporary directories will be removed. + +You can override the default temporary directory setting like this: + +.. code-block:: bash + + pytest --basetemp=mydir + +.. warning:: + + The contents of ``mydir`` will be completely removed, so make sure to use a directory + for that purpose only. + +When distributing tests on the local machine using ``pytest-xdist``, care is taken to +automatically configure a basetemp directory for the sub processes such that all temporary +data lands below a single per-test run basetemp directory. + +.. _`py.path.local`: https://py.readthedocs.io/en/latest/path.html diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/unittest.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/unittest.rst new file mode 100644 index 0000000000..bff7511077 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/unittest.rst @@ -0,0 +1,251 @@ + +.. _`unittest.TestCase`: +.. _`unittest`: + +How to use ``unittest``-based tests with pytest +=============================================== + +``pytest`` supports running Python ``unittest``-based tests out of the box. +It's meant for leveraging existing ``unittest``-based test suites +to use pytest as a test runner and also allow to incrementally adapt +the test suite to take full advantage of pytest's features. + +To run an existing ``unittest``-style test suite using ``pytest``, type: + +.. code-block:: bash + + pytest tests + + +pytest will automatically collect ``unittest.TestCase`` subclasses and +their ``test`` methods in ``test_*.py`` or ``*_test.py`` files. + +Almost all ``unittest`` features are supported: + +* ``@unittest.skip`` style decorators; +* ``setUp/tearDown``; +* ``setUpClass/tearDownClass``; +* ``setUpModule/tearDownModule``; + +.. _`load_tests protocol`: https://docs.python.org/3/library/unittest.html#load-tests-protocol + +Up to this point pytest does not have support for the following features: + +* `load_tests protocol`_; +* :ref:`subtests `; + +Benefits out of the box +----------------------- + +By running your test suite with pytest you can make use of several features, +in most cases without having to modify existing code: + +* Obtain :ref:`more informative tracebacks `; +* :ref:`stdout and stderr ` capturing; +* :ref:`Test selection options ` using ``-k`` and ``-m`` flags; +* :ref:`maxfail`; +* :ref:`--pdb ` command-line option for debugging on test failures + (see :ref:`note ` below); +* Distribute tests to multiple CPUs using the :pypi:`pytest-xdist` plugin; +* Use :ref:`plain assert-statements ` instead of ``self.assert*`` functions + (:pypi:`unittest2pytest` is immensely helpful in this); + + +pytest features in ``unittest.TestCase`` subclasses +--------------------------------------------------- + +The following pytest features work in ``unittest.TestCase`` subclasses: + +* :ref:`Marks `: :ref:`skip `, :ref:`skipif `, :ref:`xfail `; +* :ref:`Auto-use fixtures `; + +The following pytest features **do not** work, and probably +never will due to different design philosophies: + +* :ref:`Fixtures ` (except for ``autouse`` fixtures, see :ref:`below `); +* :ref:`Parametrization `; +* :ref:`Custom hooks `; + + +Third party plugins may or may not work well, depending on the plugin and the test suite. + +.. _mixing-fixtures: + +Mixing pytest fixtures into ``unittest.TestCase`` subclasses using marks +------------------------------------------------------------------------ + +Running your unittest with ``pytest`` allows you to use its +:ref:`fixture mechanism ` with ``unittest.TestCase`` style +tests. Assuming you have at least skimmed the pytest fixture features, +let's jump-start into an example that integrates a pytest ``db_class`` +fixture, setting up a class-cached database object, and then reference +it from a unittest-style test: + +.. code-block:: python + + # content of conftest.py + + # we define a fixture function below and it will be "used" by + # referencing its name from tests + + import pytest + + + @pytest.fixture(scope="class") + def db_class(request): + class DummyDB: + pass + + # set a class attribute on the invoking test context + request.cls.db = DummyDB() + +This defines a fixture function ``db_class`` which - if used - is +called once for each test class and which sets the class-level +``db`` attribute to a ``DummyDB`` instance. The fixture function +achieves this by receiving a special ``request`` object which gives +access to :ref:`the requesting test context ` such +as the ``cls`` attribute, denoting the class from which the fixture +is used. This architecture de-couples fixture writing from actual test +code and allows re-use of the fixture by a minimal reference, the fixture +name. So let's write an actual ``unittest.TestCase`` class using our +fixture definition: + +.. code-block:: python + + # content of test_unittest_db.py + + import unittest + import pytest + + + @pytest.mark.usefixtures("db_class") + class MyTest(unittest.TestCase): + def test_method1(self): + assert hasattr(self, "db") + assert 0, self.db # fail for demo purposes + + def test_method2(self): + assert 0, self.db # fail for demo purposes + +The ``@pytest.mark.usefixtures("db_class")`` class-decorator makes sure that +the pytest fixture function ``db_class`` is called once per class. +Due to the deliberately failing assert statements, we can take a look at +the ``self.db`` values in the traceback: + +.. code-block:: pytest + + $ pytest test_unittest_db.py + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 2 items + + test_unittest_db.py FF [100%] + + ================================= FAILURES ================================= + ___________________________ MyTest.test_method1 ____________________________ + + self = + + def test_method1(self): + assert hasattr(self, "db") + > assert 0, self.db # fail for demo purposes + E AssertionError: .DummyDB object at 0xdeadbeef0001> + E assert 0 + + test_unittest_db.py:10: AssertionError + ___________________________ MyTest.test_method2 ____________________________ + + self = + + def test_method2(self): + > assert 0, self.db # fail for demo purposes + E AssertionError: .DummyDB object at 0xdeadbeef0001> + E assert 0 + + test_unittest_db.py:13: AssertionError + ========================= short test summary info ========================== + FAILED test_unittest_db.py::MyTest::test_method1 - AssertionError: ` + +In general, pytest is invoked with the command ``pytest`` (see below for :ref:`other ways to invoke pytest +`). This will execute all tests in all files whose names follow the form ``test_*.py`` or ``\*_test.py`` +in the current directory and its subdirectories. More generally, pytest follows :ref:`standard test discovery rules +`. + + +.. _select-tests: + +Specifying which tests to run +------------------------------ + +Pytest supports several ways to run and select tests from the command-line. + +**Run tests in a module** + +.. code-block:: bash + + pytest test_mod.py + +**Run tests in a directory** + +.. code-block:: bash + + pytest testing/ + +**Run tests by keyword expressions** + +.. code-block:: bash + + pytest -k "MyClass and not method" + +This will run tests which contain names that match the given *string expression* (case-insensitive), +which can include Python operators that use filenames, class names and function names as variables. +The example above will run ``TestMyClass.test_something`` but not ``TestMyClass.test_method_simple``. + +.. _nodeids: + +**Run tests by node ids** + +Each collected test is assigned a unique ``nodeid`` which consist of the module filename followed +by specifiers like class names, function names and parameters from parametrization, separated by ``::`` characters. + +To run a specific test within a module: + +.. code-block:: bash + + pytest test_mod.py::test_func + + +Another example specifying a test method in the command line: + +.. code-block:: bash + + pytest test_mod.py::TestClass::test_method + +**Run tests by marker expressions** + +.. code-block:: bash + + pytest -m slow + +Will run all tests which are decorated with the ``@pytest.mark.slow`` decorator. + +For more information see :ref:`marks `. + +**Run tests from packages** + +.. code-block:: bash + + pytest --pyargs pkg.testing + +This will import ``pkg.testing`` and use its filesystem location to find and run tests from. + + +Getting help on version, option names, environment variables +-------------------------------------------------------------- + +.. code-block:: bash + + pytest --version # shows where pytest was imported from + pytest --fixtures # show available builtin function arguments + pytest -h | --help # show help on command line and config file options + + +.. _durations: + +Profiling test execution duration +------------------------------------- + +.. versionchanged:: 6.0 + +To get a list of the slowest 10 test durations over 1.0s long: + +.. code-block:: bash + + pytest --durations=10 --durations-min=1.0 + +By default, pytest will not show test durations that are too small (<0.005s) unless ``-vv`` is passed on the command-line. + + +Managing loading of plugins +------------------------------- + +Early loading plugins +~~~~~~~~~~~~~~~~~~~~~~~ + +You can early-load plugins (internal and external) explicitly in the command-line with the ``-p`` option:: + + pytest -p mypluginmodule + +The option receives a ``name`` parameter, which can be: + +* A full module dotted name, for example ``myproject.plugins``. This dotted name must be importable. +* The entry-point name of a plugin. This is the name passed to ``setuptools`` when the plugin is + registered. For example to early-load the :pypi:`pytest-cov` plugin you can use:: + + pytest -p pytest_cov + + +Disabling plugins +~~~~~~~~~~~~~~~~~~ + +To disable loading specific plugins at invocation time, use the ``-p`` option +together with the prefix ``no:``. + +Example: to disable loading the plugin ``doctest``, which is responsible for +executing doctest tests from text files, invoke pytest like this: + +.. code-block:: bash + + pytest -p no:doctest + + +.. _invoke-other: + +Other ways of calling pytest +----------------------------------------------------- + +.. _invoke-python: + +Calling pytest through ``python -m pytest`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can invoke testing through the Python interpreter from the command line: + +.. code-block:: text + + python -m pytest [...] + +This is almost equivalent to invoking the command line script ``pytest [...]`` +directly, except that calling via ``python`` will also add the current directory to ``sys.path``. + + +.. _`pytest.main-usage`: + +Calling pytest from Python code +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can invoke ``pytest`` from Python code directly: + +.. code-block:: python + + retcode = pytest.main() + +this acts as if you would call "pytest" from the command line. +It will not raise :class:`SystemExit` but return the :ref:`exit code ` instead. +You can pass in options and arguments: + +.. code-block:: python + + retcode = pytest.main(["-x", "mytestdir"]) + +You can specify additional plugins to ``pytest.main``: + +.. code-block:: python + + # content of myinvoke.py + import pytest + import sys + + + class MyPlugin: + def pytest_sessionfinish(self): + print("*** test run reporting finishing") + + + if __name__ == "__main__": + sys.exit(pytest.main(["-qq"], plugins=[MyPlugin()])) + +Running it will show that ``MyPlugin`` was added and its +hook was invoked: + +.. code-block:: pytest + + $ python myinvoke.py + *** test run reporting finishing + + +.. note:: + + Calling ``pytest.main()`` will result in importing your tests and any modules + that they import. Due to the caching mechanism of python's import system, + making subsequent calls to ``pytest.main()`` from the same process will not + reflect changes to those files between the calls. For this reason, making + multiple calls to ``pytest.main()`` from the same process (in order to re-run + tests, for example) is not recommended. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/writing_hook_functions.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/writing_hook_functions.rst new file mode 100644 index 0000000000..f615fced86 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/writing_hook_functions.rst @@ -0,0 +1,352 @@ +.. _`writinghooks`: + +Writing hook functions +====================== + + +.. _validation: + +hook function validation and execution +-------------------------------------- + +pytest calls hook functions from registered plugins for any +given hook specification. Let's look at a typical hook function +for the ``pytest_collection_modifyitems(session, config, +items)`` hook which pytest calls after collection of all test items is +completed. + +When we implement a ``pytest_collection_modifyitems`` function in our plugin +pytest will during registration verify that you use argument +names which match the specification and bail out if not. + +Let's look at a possible implementation: + +.. code-block:: python + + def pytest_collection_modifyitems(config, items): + # called after collection is completed + # you can modify the ``items`` list + ... + +Here, ``pytest`` will pass in ``config`` (the pytest config object) +and ``items`` (the list of collected test items) but will not pass +in the ``session`` argument because we didn't list it in the function +signature. This dynamic "pruning" of arguments allows ``pytest`` to +be "future-compatible": we can introduce new hook named parameters without +breaking the signatures of existing hook implementations. It is one of +the reasons for the general long-lived compatibility of pytest plugins. + +Note that hook functions other than ``pytest_runtest_*`` are not +allowed to raise exceptions. Doing so will break the pytest run. + + + +.. _firstresult: + +firstresult: stop at first non-None result +------------------------------------------- + +Most calls to ``pytest`` hooks result in a **list of results** which contains +all non-None results of the called hook functions. + +Some hook specifications use the ``firstresult=True`` option so that the hook +call only executes until the first of N registered functions returns a +non-None result which is then taken as result of the overall hook call. +The remaining hook functions will not be called in this case. + +.. _`hookwrapper`: + +hookwrapper: executing around other hooks +------------------------------------------------- + +.. currentmodule:: _pytest.core + + + +pytest plugins can implement hook wrappers which wrap the execution +of other hook implementations. A hook wrapper is a generator function +which yields exactly once. When pytest invokes hooks it first executes +hook wrappers and passes the same arguments as to the regular hooks. + +At the yield point of the hook wrapper pytest will execute the next hook +implementations and return their result to the yield point in the form of +a :py:class:`Result ` instance which encapsulates a result or +exception info. The yield point itself will thus typically not raise +exceptions (unless there are bugs). + +Here is an example definition of a hook wrapper: + +.. code-block:: python + + import pytest + + + @pytest.hookimpl(hookwrapper=True) + def pytest_pyfunc_call(pyfuncitem): + do_something_before_next_hook_executes() + + outcome = yield + # outcome.excinfo may be None or a (cls, val, tb) tuple + + res = outcome.get_result() # will raise if outcome was exception + + post_process_result(res) + + outcome.force_result(new_res) # to override the return value to the plugin system + +Note that hook wrappers don't return results themselves, they merely +perform tracing or other side effects around the actual hook implementations. +If the result of the underlying hook is a mutable object, they may modify +that result but it's probably better to avoid it. + +For more information, consult the +:ref:`pluggy documentation about hookwrappers `. + +.. _plugin-hookorder: + +Hook function ordering / call example +------------------------------------- + +For any given hook specification there may be more than one +implementation and we thus generally view ``hook`` execution as a +``1:N`` function call where ``N`` is the number of registered functions. +There are ways to influence if a hook implementation comes before or +after others, i.e. the position in the ``N``-sized list of functions: + +.. code-block:: python + + # Plugin 1 + @pytest.hookimpl(tryfirst=True) + def pytest_collection_modifyitems(items): + # will execute as early as possible + ... + + + # Plugin 2 + @pytest.hookimpl(trylast=True) + def pytest_collection_modifyitems(items): + # will execute as late as possible + ... + + + # Plugin 3 + @pytest.hookimpl(hookwrapper=True) + def pytest_collection_modifyitems(items): + # will execute even before the tryfirst one above! + outcome = yield + # will execute after all non-hookwrappers executed + +Here is the order of execution: + +1. Plugin3's pytest_collection_modifyitems called until the yield point + because it is a hook wrapper. + +2. Plugin1's pytest_collection_modifyitems is called because it is marked + with ``tryfirst=True``. + +3. Plugin2's pytest_collection_modifyitems is called because it is marked + with ``trylast=True`` (but even without this mark it would come after + Plugin1). + +4. Plugin3's pytest_collection_modifyitems then executing the code after the yield + point. The yield receives a :py:class:`Result ` instance which encapsulates + the result from calling the non-wrappers. Wrappers shall not modify the result. + +It's possible to use ``tryfirst`` and ``trylast`` also in conjunction with +``hookwrapper=True`` in which case it will influence the ordering of hookwrappers +among each other. + + +Declaring new hooks +------------------------ + +.. note:: + + This is a quick overview on how to add new hooks and how they work in general, but a more complete + overview can be found in `the pluggy documentation `__. + +.. currentmodule:: _pytest.hookspec + +Plugins and ``conftest.py`` files may declare new hooks that can then be +implemented by other plugins in order to alter behaviour or interact with +the new plugin: + +.. autofunction:: pytest_addhooks + :noindex: + +Hooks are usually declared as do-nothing functions that contain only +documentation describing when the hook will be called and what return values +are expected. The names of the functions must start with `pytest_` otherwise pytest won't recognize them. + +Here's an example. Let's assume this code is in the ``sample_hook.py`` module. + +.. code-block:: python + + def pytest_my_hook(config): + """ + Receives the pytest config and does things with it + """ + +To register the hooks with pytest they need to be structured in their own module or class. This +class or module can then be passed to the ``pluginmanager`` using the ``pytest_addhooks`` function +(which itself is a hook exposed by pytest). + +.. code-block:: python + + def pytest_addhooks(pluginmanager): + """ This example assumes the hooks are grouped in the 'sample_hook' module. """ + from my_app.tests import sample_hook + + pluginmanager.add_hookspecs(sample_hook) + +For a real world example, see `newhooks.py`_ from `xdist `_. + +.. _`newhooks.py`: https://github.com/pytest-dev/pytest-xdist/blob/974bd566c599dc6a9ea291838c6f226197208b46/xdist/newhooks.py + +Hooks may be called both from fixtures or from other hooks. In both cases, hooks are called +through the ``hook`` object, available in the ``config`` object. Most hooks receive a +``config`` object directly, while fixtures may use the ``pytestconfig`` fixture which provides the same object. + +.. code-block:: python + + @pytest.fixture() + def my_fixture(pytestconfig): + # call the hook called "pytest_my_hook" + # 'result' will be a list of return values from all registered functions. + result = pytestconfig.hook.pytest_my_hook(config=pytestconfig) + +.. note:: + Hooks receive parameters using only keyword arguments. + +Now your hook is ready to be used. To register a function at the hook, other plugins or users must +now simply define the function ``pytest_my_hook`` with the correct signature in their ``conftest.py``. + +Example: + +.. code-block:: python + + def pytest_my_hook(config): + """ + Print all active hooks to the screen. + """ + print(config.hook) + + +.. _`addoptionhooks`: + + +Using hooks in pytest_addoption +------------------------------- + +Occasionally, it is necessary to change the way in which command line options +are defined by one plugin based on hooks in another plugin. For example, +a plugin may expose a command line option for which another plugin needs +to define the default value. The pluginmanager can be used to install and +use hooks to accomplish this. The plugin would define and add the hooks +and use pytest_addoption as follows: + +.. code-block:: python + + # contents of hooks.py + + # Use firstresult=True because we only want one plugin to define this + # default value + @hookspec(firstresult=True) + def pytest_config_file_default_value(): + """ Return the default value for the config file command line option. """ + + + # contents of myplugin.py + + + def pytest_addhooks(pluginmanager): + """ This example assumes the hooks are grouped in the 'hooks' module. """ + from . import hooks + + pluginmanager.add_hookspecs(hooks) + + + def pytest_addoption(parser, pluginmanager): + default_value = pluginmanager.hook.pytest_config_file_default_value() + parser.addoption( + "--config-file", + help="Config file to use, defaults to %(default)s", + default=default_value, + ) + +The conftest.py that is using myplugin would simply define the hook as follows: + +.. code-block:: python + + def pytest_config_file_default_value(): + return "config.yaml" + + +Optionally using hooks from 3rd party plugins +--------------------------------------------- + +Using new hooks from plugins as explained above might be a little tricky +because of the standard :ref:`validation mechanism `: +if you depend on a plugin that is not installed, validation will fail and +the error message will not make much sense to your users. + +One approach is to defer the hook implementation to a new plugin instead of +declaring the hook functions directly in your plugin module, for example: + +.. code-block:: python + + # contents of myplugin.py + + + class DeferPlugin: + """Simple plugin to defer pytest-xdist hook functions.""" + + def pytest_testnodedown(self, node, error): + """standard xdist hook function.""" + + + def pytest_configure(config): + if config.pluginmanager.hasplugin("xdist"): + config.pluginmanager.register(DeferPlugin()) + +This has the added benefit of allowing you to conditionally install hooks +depending on which plugins are installed. + +.. _plugin-stash: + +Storing data on items across hook functions +------------------------------------------- + +Plugins often need to store data on :class:`~pytest.Item`\s in one hook +implementation, and access it in another. One common solution is to just +assign some private attribute directly on the item, but type-checkers like +mypy frown upon this, and it may also cause conflicts with other plugins. +So pytest offers a better way to do this, :attr:`item.stash <_pytest.nodes.Node.stash>`. + +To use the "stash" in your plugins, first create "stash keys" somewhere at the +top level of your plugin: + +.. code-block:: python + + been_there_key = pytest.StashKey[bool]() + done_that_key = pytest.StashKey[str]() + +then use the keys to stash your data at some point: + +.. code-block:: python + + def pytest_runtest_setup(item: pytest.Item) -> None: + item.stash[been_there_key] = True + item.stash[done_that_key] = "no" + +and retrieve them at another point: + +.. code-block:: python + + def pytest_runtest_teardown(item: pytest.Item) -> None: + if not item.stash[been_there_key]: + print("Oh?") + item.stash[done_that_key] = "yes!" + +Stashes are available on all node types (like :class:`~pytest.Class`, +:class:`~pytest.Session`) and also on :class:`~pytest.Config`, if needed. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/writing_plugins.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/writing_plugins.rst new file mode 100644 index 0000000000..b2d2b6563d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/writing_plugins.rst @@ -0,0 +1,458 @@ +.. _plugins: +.. _`writing-plugins`: + +Writing plugins +=============== + +It is easy to implement `local conftest plugins`_ for your own project +or `pip-installable plugins`_ that can be used throughout many projects, +including third party projects. Please refer to :ref:`using plugins` if you +only want to use but not write plugins. + +A plugin contains one or multiple hook functions. :ref:`Writing hooks ` +explains the basics and details of how you can write a hook function yourself. +``pytest`` implements all aspects of configuration, collection, running and +reporting by calling :ref:`well specified hooks ` of the following plugins: + +* builtin plugins: loaded from pytest's internal ``_pytest`` directory. + +* :ref:`external plugins `: modules discovered through + `setuptools entry points`_ + +* `conftest.py plugins`_: modules auto-discovered in test directories + +In principle, each hook call is a ``1:N`` Python function call where ``N`` is the +number of registered implementation functions for a given specification. +All specifications and implementations follow the ``pytest_`` prefix +naming convention, making them easy to distinguish and find. + +.. _`pluginorder`: + +Plugin discovery order at tool startup +-------------------------------------- + +``pytest`` loads plugin modules at tool startup in the following way: + +1. by scanning the command line for the ``-p no:name`` option + and *blocking* that plugin from being loaded (even builtin plugins can + be blocked this way). This happens before normal command-line parsing. + +2. by loading all builtin plugins. + +3. by scanning the command line for the ``-p name`` option + and loading the specified plugin. This happens before normal command-line parsing. + +4. by loading all plugins registered through `setuptools entry points`_. + +5. by loading all plugins specified through the :envvar:`PYTEST_PLUGINS` environment variable. + +6. by loading all :file:`conftest.py` files as inferred by the command line + invocation: + + - if no test paths are specified, use the current dir as a test path + - if exists, load ``conftest.py`` and ``test*/conftest.py`` relative + to the directory part of the first test path. After the ``conftest.py`` + file is loaded, load all plugins specified in its + :globalvar:`pytest_plugins` variable if present. + + Note that pytest does not find ``conftest.py`` files in deeper nested + sub directories at tool startup. It is usually a good idea to keep + your ``conftest.py`` file in the top level test or project root directory. + +7. by recursively loading all plugins specified by the + :globalvar:`pytest_plugins` variable in ``conftest.py`` files. + + +.. _`pytest/plugin`: http://bitbucket.org/pytest-dev/pytest/src/tip/pytest/plugin/ +.. _`conftest.py plugins`: +.. _`localplugin`: +.. _`local conftest plugins`: + +conftest.py: local per-directory plugins +---------------------------------------- + +Local ``conftest.py`` plugins contain directory-specific hook +implementations. Hook Session and test running activities will +invoke all hooks defined in ``conftest.py`` files closer to the +root of the filesystem. Example of implementing the +``pytest_runtest_setup`` hook so that is called for tests in the ``a`` +sub directory but not for other directories:: + + a/conftest.py: + def pytest_runtest_setup(item): + # called for running each test in 'a' directory + print("setting up", item) + + a/test_sub.py: + def test_sub(): + pass + + test_flat.py: + def test_flat(): + pass + +Here is how you might run it:: + +     pytest test_flat.py --capture=no # will not show "setting up" + pytest a/test_sub.py --capture=no # will show "setting up" + +.. note:: + If you have ``conftest.py`` files which do not reside in a + python package directory (i.e. one containing an ``__init__.py``) then + "import conftest" can be ambiguous because there might be other + ``conftest.py`` files as well on your ``PYTHONPATH`` or ``sys.path``. + It is thus good practice for projects to either put ``conftest.py`` + under a package scope or to never import anything from a + ``conftest.py`` file. + + See also: :ref:`pythonpath`. + +.. note:: + Some hooks should be implemented only in plugins or conftest.py files situated at the + tests root directory due to how pytest discovers plugins during startup, + see the documentation of each hook for details. + +Writing your own plugin +----------------------- + +If you want to write a plugin, there are many real-life examples +you can copy from: + +* a custom collection example plugin: :ref:`yaml plugin` +* builtin plugins which provide pytest's own functionality +* many :ref:`external plugins ` providing additional features + +All of these plugins implement :ref:`hooks ` and/or :ref:`fixtures ` +to extend and add functionality. + +.. note:: + Make sure to check out the excellent + `cookiecutter-pytest-plugin `_ + project, which is a `cookiecutter template `_ + for authoring plugins. + + The template provides an excellent starting point with a working plugin, + tests running with tox, a comprehensive README file as well as a + pre-configured entry-point. + +Also consider :ref:`contributing your plugin to pytest-dev` +once it has some happy users other than yourself. + + +.. _`setuptools entry points`: +.. _`pip-installable plugins`: + +Making your plugin installable by others +---------------------------------------- + +If you want to make your plugin externally available, you +may define a so-called entry point for your distribution so +that ``pytest`` finds your plugin module. Entry points are +a feature that is provided by :std:doc:`setuptools:index`. pytest looks up +the ``pytest11`` entrypoint to discover its +plugins and you can thus make your plugin available by defining +it in your setuptools-invocation: + +.. sourcecode:: python + + # sample ./setup.py file + from setuptools import setup + + setup( + name="myproject", + packages=["myproject"], + # the following makes a plugin available to pytest + entry_points={"pytest11": ["name_of_plugin = myproject.pluginmodule"]}, + # custom PyPI classifier for pytest plugins + classifiers=["Framework :: Pytest"], + ) + +If a package is installed this way, ``pytest`` will load +``myproject.pluginmodule`` as a plugin which can define +:ref:`hooks `. + +.. note:: + + Make sure to include ``Framework :: Pytest`` in your list of + `PyPI classifiers `_ + to make it easy for users to find your plugin. + + +.. _assertion-rewriting: + +Assertion Rewriting +------------------- + +One of the main features of ``pytest`` is the use of plain assert +statements and the detailed introspection of expressions upon +assertion failures. This is provided by "assertion rewriting" which +modifies the parsed AST before it gets compiled to bytecode. This is +done via a :pep:`302` import hook which gets installed early on when +``pytest`` starts up and will perform this rewriting when modules get +imported. However, since we do not want to test different bytecode +from what you will run in production, this hook only rewrites test modules +themselves (as defined by the :confval:`python_files` configuration option), +and any modules which are part of plugins. +Any other imported module will not be rewritten and normal assertion behaviour +will happen. + +If you have assertion helpers in other modules where you would need +assertion rewriting to be enabled you need to ask ``pytest`` +explicitly to rewrite this module before it gets imported. + +.. autofunction:: pytest.register_assert_rewrite + :noindex: + +This is especially important when you write a pytest plugin which is +created using a package. The import hook only treats ``conftest.py`` +files and any modules which are listed in the ``pytest11`` entrypoint +as plugins. As an example consider the following package:: + + pytest_foo/__init__.py + pytest_foo/plugin.py + pytest_foo/helper.py + +With the following typical ``setup.py`` extract: + +.. code-block:: python + + setup(..., entry_points={"pytest11": ["foo = pytest_foo.plugin"]}, ...) + +In this case only ``pytest_foo/plugin.py`` will be rewritten. If the +helper module also contains assert statements which need to be +rewritten it needs to be marked as such, before it gets imported. +This is easiest by marking it for rewriting inside the +``__init__.py`` module, which will always be imported first when a +module inside a package is imported. This way ``plugin.py`` can still +import ``helper.py`` normally. The contents of +``pytest_foo/__init__.py`` will then need to look like this: + +.. code-block:: python + + import pytest + + pytest.register_assert_rewrite("pytest_foo.helper") + + +Requiring/Loading plugins in a test module or conftest file +----------------------------------------------------------- + +You can require plugins in a test module or a ``conftest.py`` file using :globalvar:`pytest_plugins`: + +.. code-block:: python + + pytest_plugins = ["name1", "name2"] + +When the test module or conftest plugin is loaded the specified plugins +will be loaded as well. Any module can be blessed as a plugin, including internal +application modules: + +.. code-block:: python + + pytest_plugins = "myapp.testsupport.myplugin" + +:globalvar:`pytest_plugins` are processed recursively, so note that in the example above +if ``myapp.testsupport.myplugin`` also declares :globalvar:`pytest_plugins`, the contents +of the variable will also be loaded as plugins, and so on. + +.. _`requiring plugins in non-root conftests`: + +.. note:: + Requiring plugins using :globalvar:`pytest_plugins` variable in non-root + ``conftest.py`` files is deprecated. + + This is important because ``conftest.py`` files implement per-directory + hook implementations, but once a plugin is imported, it will affect the + entire directory tree. In order to avoid confusion, defining + :globalvar:`pytest_plugins` in any ``conftest.py`` file which is not located in the + tests root directory is deprecated, and will raise a warning. + +This mechanism makes it easy to share fixtures within applications or even +external applications without the need to create external plugins using +the ``setuptools``'s entry point technique. + +Plugins imported by :globalvar:`pytest_plugins` will also automatically be marked +for assertion rewriting (see :func:`pytest.register_assert_rewrite`). +However for this to have any effect the module must not be +imported already; if it was already imported at the time the +:globalvar:`pytest_plugins` statement is processed, a warning will result and +assertions inside the plugin will not be rewritten. To fix this you +can either call :func:`pytest.register_assert_rewrite` yourself before +the module is imported, or you can arrange the code to delay the +importing until after the plugin is registered. + + +Accessing another plugin by name +-------------------------------- + +If a plugin wants to collaborate with code from +another plugin it can obtain a reference through +the plugin manager like this: + +.. sourcecode:: python + + plugin = config.pluginmanager.get_plugin("name_of_plugin") + +If you want to look at the names of existing plugins, use +the ``--trace-config`` option. + + +.. _registering-markers: + +Registering custom markers +-------------------------- + +If your plugin uses any markers, you should register them so that they appear in +pytest's help text and do not :ref:`cause spurious warnings `. +For example, the following plugin would register ``cool_marker`` and +``mark_with`` for all users: + +.. code-block:: python + + def pytest_configure(config): + config.addinivalue_line("markers", "cool_marker: this one is for cool tests.") + config.addinivalue_line( + "markers", "mark_with(arg, arg2): this marker takes arguments." + ) + + +Testing plugins +--------------- + +pytest comes with a plugin named ``pytester`` that helps you write tests for +your plugin code. The plugin is disabled by default, so you will have to enable +it before you can use it. + +You can do so by adding the following line to a ``conftest.py`` file in your +testing directory: + +.. code-block:: python + + # content of conftest.py + + pytest_plugins = ["pytester"] + +Alternatively you can invoke pytest with the ``-p pytester`` command line +option. + +This will allow you to use the :py:class:`pytester ` +fixture for testing your plugin code. + +Let's demonstrate what you can do with the plugin with an example. Imagine we +developed a plugin that provides a fixture ``hello`` which yields a function +and we can invoke this function with one optional parameter. It will return a +string value of ``Hello World!`` if we do not supply a value or ``Hello +{value}!`` if we do supply a string value. + +.. code-block:: python + + import pytest + + + def pytest_addoption(parser): + group = parser.getgroup("helloworld") + group.addoption( + "--name", + action="store", + dest="name", + default="World", + help='Default "name" for hello().', + ) + + + @pytest.fixture + def hello(request): + name = request.config.getoption("name") + + def _hello(name=None): + if not name: + name = request.config.getoption("name") + return "Hello {name}!".format(name=name) + + return _hello + + +Now the ``pytester`` fixture provides a convenient API for creating temporary +``conftest.py`` files and test files. It also allows us to run the tests and +return a result object, with which we can assert the tests' outcomes. + +.. code-block:: python + + def test_hello(pytester): + """Make sure that our plugin works.""" + + # create a temporary conftest.py file + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(params=[ + "Brianna", + "Andreas", + "Floris", + ]) + def name(request): + return request.param + """ + ) + + # create a temporary pytest test file + pytester.makepyfile( + """ + def test_hello_default(hello): + assert hello() == "Hello World!" + + def test_hello_name(hello, name): + assert hello(name) == "Hello {0}!".format(name) + """ + ) + + # run all tests with pytest + result = pytester.runpytest() + + # check that all 4 tests passed + result.assert_outcomes(passed=4) + + +Additionally it is possible to copy examples to the ``pytester``'s isolated environment +before running pytest on it. This way we can abstract the tested logic to separate files, +which is especially useful for longer tests and/or longer ``conftest.py`` files. + +Note that for ``pytester.copy_example`` to work we need to set `pytester_example_dir` +in our ``pytest.ini`` to tell pytest where to look for example files. + +.. code-block:: ini + + # content of pytest.ini + [pytest] + pytester_example_dir = . + + +.. code-block:: python + + # content of test_example.py + + + def test_plugin(pytester): + pytester.copy_example("test_example.py") + pytester.runpytest("-k", "test_example") + + + def test_example(): + pass + +.. code-block:: pytest + + $ pytest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project, configfile: pytest.ini + collected 2 items + + test_example.py .. [100%] + + ============================ 2 passed in 0.12s ============================= + +For more information about the result object that ``runpytest()`` returns, and +the methods that it provides please check out the :py:class:`RunResult +<_pytest.pytester.RunResult>` documentation. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/xunit_setup.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/xunit_setup.rst new file mode 100644 index 0000000000..5a97b2c85f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/how-to/xunit_setup.rst @@ -0,0 +1,117 @@ + +.. _`classic xunit`: +.. _xunitsetup: + +How to implement xunit-style set-up +======================================== + +This section describes a classic and popular way how you can implement +fixtures (setup and teardown test state) on a per-module/class/function basis. + + +.. note:: + + While these setup/teardown methods are simple and familiar to those + coming from a ``unittest`` or ``nose`` background, you may also consider + using pytest's more powerful :ref:`fixture mechanism + ` which leverages the concept of dependency injection, allowing + for a more modular and more scalable approach for managing test state, + especially for larger projects and for functional testing. You can + mix both fixture mechanisms in the same file but + test methods of ``unittest.TestCase`` subclasses + cannot receive fixture arguments. + + +Module level setup/teardown +-------------------------------------- + +If you have multiple test functions and test classes in a single +module you can optionally implement the following fixture methods +which will usually be called once for all the functions: + +.. code-block:: python + + def setup_module(module): + """ setup any state specific to the execution of the given module.""" + + + def teardown_module(module): + """teardown any state that was previously setup with a setup_module + method. + """ + +As of pytest-3.0, the ``module`` parameter is optional. + +Class level setup/teardown +---------------------------------- + +Similarly, the following methods are called at class level before +and after all test methods of the class are called: + +.. code-block:: python + + @classmethod + def setup_class(cls): + """setup any state specific to the execution of the given class (which + usually contains tests). + """ + + + @classmethod + def teardown_class(cls): + """teardown any state that was previously setup with a call to + setup_class. + """ + +Method and function level setup/teardown +----------------------------------------------- + +Similarly, the following methods are called around each method invocation: + +.. code-block:: python + + def setup_method(self, method): + """setup any state tied to the execution of the given method in a + class. setup_method is invoked for every test method of a class. + """ + + + def teardown_method(self, method): + """teardown any state that was previously setup with a setup_method + call. + """ + +As of pytest-3.0, the ``method`` parameter is optional. + +If you would rather define test functions directly at module level +you can also use the following functions to implement fixtures: + +.. code-block:: python + + def setup_function(function): + """setup any state tied to the execution of the given function. + Invoked for every test function in the module. + """ + + + def teardown_function(function): + """teardown any state that was previously setup with a setup_function + call. + """ + +As of pytest-3.0, the ``function`` parameter is optional. + +Remarks: + +* It is possible for setup/teardown pairs to be invoked multiple times + per testing process. + +* teardown functions are not called if the corresponding setup function existed + and failed/was skipped. + +* Prior to pytest-4.2, xunit-style functions did not obey the scope rules of fixtures, so + it was possible, for example, for a ``setup_method`` to be called before a + session-scoped autouse fixture. + + Now the xunit-style functions are integrated with the fixture mechanism and obey the proper + scope rules of fixtures involved in the call. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/cramer2.png b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/cramer2.png new file mode 100644 index 0000000000..6bf0e92e20 Binary files /dev/null and b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/cramer2.png differ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/favicon.png b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/favicon.png new file mode 100644 index 0000000000..5c8824d67d Binary files /dev/null and b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/favicon.png differ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/freiburg2.jpg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/freiburg2.jpg new file mode 100644 index 0000000000..3383d3023d Binary files /dev/null and b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/freiburg2.jpg differ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/gaynor3.png b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/gaynor3.png new file mode 100644 index 0000000000..a577c168b3 Binary files /dev/null and b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/gaynor3.png differ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/keleshev.png b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/keleshev.png new file mode 100644 index 0000000000..0d5e571e26 Binary files /dev/null and b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/keleshev.png differ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pullrequest.png b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pullrequest.png new file mode 100644 index 0000000000..4af293b213 Binary files /dev/null and b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pullrequest.png differ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pylib.png b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pylib.png new file mode 100644 index 0000000000..2e10d43886 Binary files /dev/null and b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pylib.png differ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pytest1.png b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pytest1.png new file mode 100644 index 0000000000..e8064a694c Binary files /dev/null and b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pytest1.png differ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pytest_logo_curves.svg b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pytest_logo_curves.svg new file mode 100644 index 0000000000..e05ceb1123 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/pytest_logo_curves.svg @@ -0,0 +1,29 @@ + + + + + diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/theuni.png b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/theuni.png new file mode 100644 index 0000000000..abeb737e79 Binary files /dev/null and b/testing/web-platform/tests/tools/third_party/pytest/doc/en/img/theuni.png differ diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/index.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/index.rst new file mode 100644 index 0000000000..d1b3d2e8a0 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/index.rst @@ -0,0 +1,148 @@ +:orphan: + +.. + .. sidebar:: Next Open Trainings + + - `Professional Testing with Python `_, via `Python Academy `_, February 1st to 3rd, 2022, Leipzig (Germany) and remote. + + Also see `previous talks and blogposts `_. + +.. _features: + +pytest: helps you write better programs +======================================= + +.. module:: pytest + +The ``pytest`` framework makes it easy to write small, readable tests, and can +scale to support complex functional testing for applications and libraries. + + +**Pythons**: ``pytest`` requires: Python 3.6, 3.7, 3.8, 3.9, or PyPy3. + +**PyPI package name**: :pypi:`pytest` + +**Documentation as PDF**: `download latest `_ + + +A quick example +--------------- + +.. code-block:: python + + # content of test_sample.py + def inc(x): + return x + 1 + + + def test_answer(): + assert inc(3) == 5 + + +To execute it: + +.. code-block:: pytest + + $ pytest + =========================== test session starts ============================ + platform linux -- Python 3.x.y, pytest-7.x.y, pluggy-1.x.y + rootdir: /home/sweet/project + collected 1 item + + test_sample.py F [100%] + + ================================= FAILURES ================================= + _______________________________ test_answer ________________________________ + + def test_answer(): + > assert inc(3) == 5 + E assert 4 == 5 + E + where 4 = inc(3) + + test_sample.py:6: AssertionError + ========================= short test summary info ========================== + FAILED test_sample.py::test_answer - assert 4 == 5 + ============================ 1 failed in 0.12s ============================= + +Due to ``pytest``'s detailed assertion introspection, only plain ``assert`` statements are used. +See :ref:`Get started ` for a basic introduction to using pytest. + + +Features +-------- + +- Detailed info on failing :ref:`assert statements ` (no need to remember ``self.assert*`` names) + +- :ref:`Auto-discovery ` of test modules and functions + +- :ref:`Modular fixtures ` for managing small or parametrized long-lived test resources + +- Can run :ref:`unittest ` (including trial) and :ref:`nose ` test suites out of the box + +- Python 3.6+ and PyPy 3 + +- Rich plugin architecture, with over 800+ :ref:`external plugins ` and thriving community + + +Documentation +------------- + +* :ref:`Get started ` - install pytest and grasp its basics just twenty minutes +* :ref:`How-to guides ` - step-by-step guides, covering a vast range of use-cases and needs +* :ref:`Reference guides ` - includes the complete pytest API reference, lists of plugins and more +* :ref:`Explanation ` - background, discussion of key topics, answers to higher-level questions + + +Bugs/Requests +------------- + +Please use the `GitHub issue tracker `_ to submit bugs or request features. + + +Changelog +--------- + +Consult the :ref:`Changelog ` page for fixes and enhancements of each version. + +Support pytest +-------------- + +`Open Collective`_ is an online funding platform for open and transparent communities. +It provides tools to raise money and share your finances in full transparency. + +It is the platform of choice for individuals and companies that want to make one-time or +monthly donations directly to the project. + +See more details in the `pytest collective`_. + +.. _Open Collective: https://opencollective.com +.. _pytest collective: https://opencollective.com/pytest + + +pytest for enterprise +--------------------- + +Available as part of the Tidelift Subscription. + +The maintainers of pytest and thousands of other packages are working with Tidelift to deliver commercial support and +maintenance for the open source dependencies you use to build your applications. +Save time, reduce risk, and improve code health, while paying the maintainers of the exact dependencies you use. + +`Learn more. `_ + +Security +~~~~~~~~ + +pytest has never been associated with a security vulnerability, but in any case, to report a +security vulnerability please use the `Tidelift security contact `_. +Tidelift will coordinate the fix and disclosure. + + +License +------- + +Copyright Holger Krekel and others, 2004. + +Distributed under the terms of the `MIT`_ license, pytest is free and open source software. + +.. _`MIT`: https://github.com/pytest-dev/pytest/blob/main/LICENSE diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/license.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/license.rst new file mode 100644 index 0000000000..acbfb8bdb1 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/license.rst @@ -0,0 +1,32 @@ +.. _license: + +License +------- + +Distributed under the terms of the `MIT`_ license, pytest is free and open source software. + +.. code-block:: text + + The MIT License (MIT) + + Copyright (c) 2004 Holger Krekel and others + + Permission is hereby granted, free of charge, to any person obtaining a copy of + this software and associated documentation files (the "Software"), to deal in + the Software without restriction, including without limitation the rights to + use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + of the Software, and to permit persons to whom the Software is furnished to do + so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +.. _`MIT`: https://github.com/pytest-dev/pytest/blob/main/LICENSE diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/naming20.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/naming20.rst new file mode 100644 index 0000000000..5a81df2698 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/naming20.rst @@ -0,0 +1,20 @@ + +.. _naming20: + +New pytest names in 2.0 (flat is better than nested) +---------------------------------------------------- + +If you used older version of the ``py`` distribution (which +included the py.test command line tool and Python name space) +you accessed helpers and possibly collection classes through +the ``py.test`` Python namespaces. The new ``pytest`` +Python module flaty provides the same objects, following +these renaming rules:: + + py.test.XYZ -> pytest.XYZ + py.test.collect.XYZ -> pytest.XYZ + py.test.cmdline.main -> pytest.main + +The old ``py.test.*`` ways to access functionality remain +valid but you are encouraged to do global renaming according +to the above rules in your test code. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/proposals/parametrize_with_fixtures.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/proposals/parametrize_with_fixtures.rst new file mode 100644 index 0000000000..f6814ec78d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/proposals/parametrize_with_fixtures.rst @@ -0,0 +1,164 @@ +:orphan: + +=================================== +PROPOSAL: Parametrize with fixtures +=================================== + +.. warning:: + + This document outlines a proposal around using fixtures as input + of parametrized tests or fixtures. + +Problem +------- + +As a user I have functional tests that I would like to run against various +scenarios. + +In this particular example we want to generate a new project based on a +cookiecutter template. We want to test default values but also data that +emulates user input. + +- use default values + +- emulate user input + + - specify 'author' + + - specify 'project_slug' + + - specify 'author' and 'project_slug' + +This is how a functional test could look like: + +.. code-block:: python + + import pytest + + + @pytest.fixture + def default_context(): + return {"extra_context": {}} + + + @pytest.fixture( + params=[ + {"author": "alice"}, + {"project_slug": "helloworld"}, + {"author": "bob", "project_slug": "foobar"}, + ] + ) + def extra_context(request): + return {"extra_context": request.param} + + + @pytest.fixture(params=["default", "extra"]) + def context(request): + if request.param == "default": + return request.getfuncargvalue("default_context") + else: + return request.getfuncargvalue("extra_context") + + + def test_generate_project(cookies, context): + """Call the cookiecutter API to generate a new project from a + template. + """ + result = cookies.bake(extra_context=context) + + assert result.exit_code == 0 + assert result.exception is None + assert result.project.isdir() + + +Issues +------ + +* By using ``request.getfuncargvalue()`` we rely on actual fixture function + execution to know what fixtures are involved, due to its dynamic nature +* More importantly, ``request.getfuncargvalue()`` cannot be combined with + parametrized fixtures, such as ``extra_context`` +* This is very inconvenient if you wish to extend an existing test suite by + certain parameters for fixtures that are already used by tests + +pytest version 3.0 reports an error if you try to run above code:: + + Failed: The requested fixture has no parameter defined for the current + test. + + Requested fixture 'extra_context' + + +Proposed solution +----------------- + +A new function that can be used in modules can be used to dynamically define +fixtures from existing ones. + +.. code-block:: python + + pytest.define_combined_fixture( + name="context", fixtures=["default_context", "extra_context"] + ) + +The new fixture ``context`` inherits the scope from the used fixtures and yield +the following values. + +- ``{}`` + +- ``{'author': 'alice'}`` + +- ``{'project_slug': 'helloworld'}`` + +- ``{'author': 'bob', 'project_slug': 'foobar'}`` + +Alternative approach +-------------------- + +A new helper function named ``fixture_request`` would tell pytest to yield +all parameters marked as a fixture. + +.. note:: + + The :pypi:`pytest-lazy-fixture` plugin implements a very + similar solution to the proposal below, make sure to check it out. + +.. code-block:: python + + @pytest.fixture( + params=[ + pytest.fixture_request("default_context"), + pytest.fixture_request("extra_context"), + ] + ) + def context(request): + """Returns all values for ``default_context``, one-by-one before it + does the same for ``extra_context``. + + request.param: + - {} + - {'author': 'alice'} + - {'project_slug': 'helloworld'} + - {'author': 'bob', 'project_slug': 'foobar'} + """ + return request.param + +The same helper can be used in combination with ``pytest.mark.parametrize``. + +.. code-block:: python + + + @pytest.mark.parametrize( + "context, expected_response_code", + [ + (pytest.fixture_request("default_context"), 0), + (pytest.fixture_request("extra_context"), 0), + ], + ) + def test_generate_project(cookies, context, exit_code): + """Call the cookiecutter API to generate a new project from a + template. + """ + result = cookies.bake(extra_context=context) + + assert result.exit_code == exit_code diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/py27-py34-deprecation.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/py27-py34-deprecation.rst new file mode 100644 index 0000000000..660b078e30 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/py27-py34-deprecation.rst @@ -0,0 +1,99 @@ +Python 2.7 and 3.4 support +========================== + +It is demanding on the maintainers of an open source project to support many Python versions, as +there's extra cost of keeping code compatible between all versions, while holding back on +features only made possible on newer Python versions. + +In case of Python 2 and 3, the difference between the languages makes it even more prominent, +because many new Python 3 features cannot be used in a Python 2/3 compatible code base. + +Python 2.7 EOL has been reached :pep:`in 2020 <0373#maintenance-releases>`, with +the last release made in April, 2020. + +Python 3.4 EOL has been reached :pep:`in 2019 <0429#release-schedule>`, with the last release made in March, 2019. + +For those reasons, in Jun 2019 it was decided that **pytest 4.6** series will be the last to support Python 2.7 and 3.4. + +What this means for general users +--------------------------------- + +Thanks to the `python_requires`_ setuptools option, +Python 2.7 and Python 3.4 users using a modern pip version +will install the last pytest 4.6.X version automatically even if 5.0 or later versions +are available on PyPI. + +Users should ensure they are using the latest pip and setuptools versions for this to work. + +Maintenance of 4.6.X versions +----------------------------- + +Until January 2020, the pytest core team ported many bug-fixes from the main release into the +``4.6.x`` branch, with several 4.6.X releases being made along the year. + +From now on, the core team will **no longer actively backport patches**, but the ``4.6.x`` +branch will continue to exist so the community itself can contribute patches. + +The core team will be happy to accept those patches, and make new 4.6.X releases **until mid-2020** +(but consider that date as a ballpark, after that date the team might still decide to make new releases +for critical bugs). + +.. _`python_requires`: https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires + +Technical aspects +~~~~~~~~~~~~~~~~~ + +(This section is a transcript from :issue:`5275`). + +In this section we describe the technical aspects of the Python 2.7 and 3.4 support plan. + +.. _what goes into 4.6.x releases: + +What goes into 4.6.X releases ++++++++++++++++++++++++++++++ + +New 4.6.X releases will contain bug fixes only. + +When will 4.6.X releases happen ++++++++++++++++++++++++++++++++ + +New 4.6.X releases will happen after we have a few bugs in place to release, or if a few weeks have +passed (say a single bug has been fixed a month after the latest 4.6.X release). + +No hard rules here, just ballpark. + +Who will handle applying bug fixes +++++++++++++++++++++++++++++++++++ + +We core maintainers expect that people still using Python 2.7/3.4 and being affected by +bugs to step up and provide patches and/or port bug fixes from the active branches. + +We will be happy to guide users interested in doing so, so please don't hesitate to ask. + +**Backporting changes into 4.6** + +Please follow these instructions: + +#. ``git fetch --all --prune`` + +#. ``git checkout origin/4.6.x -b backport-XXXX`` # use the PR number here + +#. Locate the merge commit on the PR, in the *merged* message, for example: + + nicoddemus merged commit 0f8b462 into pytest-dev:features + +#. ``git cherry-pick -m1 REVISION`` # use the revision you found above (``0f8b462``). + +#. Open a PR targeting ``4.6.x``: + + * Prefix the message with ``[4.6]`` so it is an obvious backport + * Delete the PR body, it usually contains a duplicate commit message. + +**Providing new PRs to 4.6** + +Fresh pull requests to ``4.6.x`` will be accepted provided that +the equivalent code in the active branches does not contain that bug (for example, a bug is specific +to Python 2 only). + +Bug fixes that also happen in the mainstream version should be first fixed +there, and then backported as per instructions above. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/pytest.ini b/testing/web-platform/tests/tools/third_party/pytest/doc/en/pytest.ini new file mode 100644 index 0000000000..7604360561 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +# just defined to prevent the root level tox.ini from kicking in diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/recwarn.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/recwarn.rst new file mode 100644 index 0000000000..513af0d450 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/recwarn.rst @@ -0,0 +1,3 @@ +:orphan: + +This page has been moved, please see :ref:`assertwarnings`. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/customize.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/customize.rst new file mode 100644 index 0000000000..fe10ca066b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/customize.rst @@ -0,0 +1,248 @@ +Configuration +============= + +Command line options and configuration file settings +----------------------------------------------------------------- + +You can get help on command line options and values in INI-style +configurations files by using the general help option: + +.. code-block:: bash + + pytest -h # prints options _and_ config file settings + +This will display command line and configuration file settings +which were registered by installed plugins. + +.. _`config file formats`: + +Configuration file formats +-------------------------- + +Many :ref:`pytest settings ` can be set in a *configuration file*, which +by convention resides in the root directory of your repository. + +A quick example of the configuration files supported by pytest: + +pytest.ini +~~~~~~~~~~ + +``pytest.ini`` files take precedence over other files, even when empty. + +.. code-block:: ini + + # pytest.ini + [pytest] + minversion = 6.0 + addopts = -ra -q + testpaths = + tests + integration + + +pyproject.toml +~~~~~~~~~~~~~~ + +.. versionadded:: 6.0 + +``pyproject.toml`` are considered for configuration when they contain a ``tool.pytest.ini_options`` table. + +.. code-block:: toml + + # pyproject.toml + [tool.pytest.ini_options] + minversion = "6.0" + addopts = "-ra -q" + testpaths = [ + "tests", + "integration", + ] + +.. note:: + + One might wonder why ``[tool.pytest.ini_options]`` instead of ``[tool.pytest]`` as is the + case with other tools. + + The reason is that the pytest team intends to fully utilize the rich TOML data format + for configuration in the future, reserving the ``[tool.pytest]`` table for that. + The ``ini_options`` table is being used, for now, as a bridge between the existing + ``.ini`` configuration system and the future configuration format. + +tox.ini +~~~~~~~ + +``tox.ini`` files are the configuration files of the `tox `__ project, +and can also be used to hold pytest configuration if they have a ``[pytest]`` section. + +.. code-block:: ini + + # tox.ini + [pytest] + minversion = 6.0 + addopts = -ra -q + testpaths = + tests + integration + + +setup.cfg +~~~~~~~~~ + +``setup.cfg`` files are general purpose configuration files, used originally by :doc:`distutils `, and can also be used to hold pytest configuration +if they have a ``[tool:pytest]`` section. + +.. code-block:: ini + + # setup.cfg + [tool:pytest] + minversion = 6.0 + addopts = -ra -q + testpaths = + tests + integration + +.. warning:: + + Usage of ``setup.cfg`` is not recommended unless for very simple use cases. ``.cfg`` + files use a different parser than ``pytest.ini`` and ``tox.ini`` which might cause hard to track + down problems. + When possible, it is recommended to use the latter files, or ``pyproject.toml``, to hold your + pytest configuration. + + +.. _rootdir: +.. _configfiles: + +Initialization: determining rootdir and configfile +-------------------------------------------------- + +pytest determines a ``rootdir`` for each test run which depends on +the command line arguments (specified test files, paths) and on +the existence of configuration files. The determined ``rootdir`` and ``configfile`` are +printed as part of the pytest header during startup. + +Here's a summary what ``pytest`` uses ``rootdir`` for: + +* Construct *nodeids* during collection; each test is assigned + a unique *nodeid* which is rooted at the ``rootdir`` and takes into account + the full path, class name, function name and parametrization (if any). + +* Is used by plugins as a stable location to store project/test run specific information; + for example, the internal :ref:`cache ` plugin creates a ``.pytest_cache`` subdirectory + in ``rootdir`` to store its cross-test run state. + +``rootdir`` is **NOT** used to modify ``sys.path``/``PYTHONPATH`` or +influence how modules are imported. See :ref:`pythonpath` for more details. + +The ``--rootdir=path`` command-line option can be used to force a specific directory. +Note that contrary to other command-line options, ``--rootdir`` cannot be used with +:confval:`addopts` inside ``pytest.ini`` because the ``rootdir`` is used to *find* ``pytest.ini`` +already. + +Finding the ``rootdir`` +~~~~~~~~~~~~~~~~~~~~~~~ + +Here is the algorithm which finds the rootdir from ``args``: + +- If ``-c`` is passed in the command-line, use that as configuration file, and its directory as ``rootdir``. + +- Determine the common ancestor directory for the specified ``args`` that are + recognised as paths that exist in the file system. If no such paths are + found, the common ancestor directory is set to the current working directory. + +- Look for ``pytest.ini``, ``pyproject.toml``, ``tox.ini``, and ``setup.cfg`` files in the ancestor + directory and upwards. If one is matched, it becomes the ``configfile`` and its + directory becomes the ``rootdir``. + +- If no configuration file was found, look for ``setup.py`` upwards from the common + ancestor directory to determine the ``rootdir``. + +- If no ``setup.py`` was found, look for ``pytest.ini``, ``pyproject.toml``, ``tox.ini``, and + ``setup.cfg`` in each of the specified ``args`` and upwards. If one is + matched, it becomes the ``configfile`` and its directory becomes the ``rootdir``. + +- If no ``configfile`` was found and no configuration argument is passed, use the already determined common ancestor as root + directory. This allows the use of pytest in structures that are not part of + a package and don't have any particular configuration file. + +If no ``args`` are given, pytest collects test below the current working +directory and also starts determining the ``rootdir`` from there. + +Files will only be matched for configuration if: + +* ``pytest.ini``: will always match and take precedence, even if empty. +* ``pyproject.toml``: contains a ``[tool.pytest.ini_options]`` table. +* ``tox.ini``: contains a ``[pytest]`` section. +* ``setup.cfg``: contains a ``[tool:pytest]`` section. + +The files are considered in the order above. Options from multiple ``configfiles`` candidates +are never merged - the first match wins. + +The :class:`Config ` object (accessible via hooks or through the :fixture:`pytestconfig` fixture) +will subsequently carry these attributes: + +- :attr:`config.rootpath `: the determined root directory, guaranteed to exist. + +- :attr:`config.inipath `: the determined ``configfile``, may be ``None`` + (it is named ``inipath`` for historical reasons). + +.. versionadded:: 6.1 + The ``config.rootpath`` and ``config.inipath`` properties. They are :class:`pathlib.Path` + versions of the older ``config.rootdir`` and ``config.inifile``, which have type + ``py.path.local``, and still exist for backward compatibility. + +The ``rootdir`` is used as a reference directory for constructing test +addresses ("nodeids") and can be used also by plugins for storing +per-testrun information. + +Example: + +.. code-block:: bash + + pytest path/to/testdir path/other/ + +will determine the common ancestor as ``path`` and then +check for configuration files as follows: + +.. code-block:: text + + # first look for pytest.ini files + path/pytest.ini + path/pyproject.toml # must contain a [tool.pytest.ini_options] table to match + path/tox.ini # must contain [pytest] section to match + path/setup.cfg # must contain [tool:pytest] section to match + pytest.ini + ... # all the way up to the root + + # now look for setup.py + path/setup.py + setup.py + ... # all the way up to the root + + +.. warning:: + + Custom pytest plugin commandline arguments may include a path, as in + ``pytest --log-output ../../test.log args``. Then ``args`` is mandatory, + otherwise pytest uses the folder of test.log for rootdir determination + (see also :issue:`1435`). + A dot ``.`` for referencing to the current working directory is also + possible. + + +.. _`how to change command line options defaults`: +.. _`adding default options`: + + +Builtin configuration file options +---------------------------------------------- + +For the full list of options consult the :ref:`reference documentation `. + +Syntax highlighting theme customization +--------------------------------------- + +The syntax highlighting themes used by pytest can be customized using two environment variables: + +- :envvar:`PYTEST_THEME` sets a `pygment style `_ to use. +- :envvar:`PYTEST_THEME_MODE` sets this style to *light* or *dark*. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/exit-codes.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/exit-codes.rst new file mode 100644 index 0000000000..b695ca3702 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/exit-codes.rst @@ -0,0 +1,26 @@ +.. _exit-codes: + +Exit codes +======================================================== + +Running ``pytest`` can result in six different exit codes: + +:Exit code 0: All tests were collected and passed successfully +:Exit code 1: Tests were collected and run but some of the tests failed +:Exit code 2: Test execution was interrupted by the user +:Exit code 3: Internal error happened while executing tests +:Exit code 4: pytest command line usage error +:Exit code 5: No tests were collected + +They are represented by the :class:`pytest.ExitCode` enum. The exit codes being a part of the public API can be imported and accessed directly using: + +.. code-block:: python + + from pytest import ExitCode + +.. note:: + + If you would like to customize the exit code in some scenarios, specially when + no tests are collected, consider using the + `pytest-custom_exit_code `__ + plugin. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/fixtures.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/fixtures.rst new file mode 100644 index 0000000000..d25979ab95 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/fixtures.rst @@ -0,0 +1,455 @@ +.. _reference-fixtures: +.. _fixture: +.. _fixtures: +.. _`@pytest.fixture`: +.. _`pytest.fixture`: + + +Fixtures reference +======================================================== + +.. seealso:: :ref:`about-fixtures` +.. seealso:: :ref:`how-to-fixtures` + + +.. currentmodule:: _pytest.python + +.. _`Dependency injection`: https://en.wikipedia.org/wiki/Dependency_injection + + +Built-in fixtures +----------------- + +:ref:`Fixtures ` are defined using the :ref:`@pytest.fixture +` decorator. Pytest has several useful built-in fixtures: + + :fixture:`capfd` + Capture, as text, output to file descriptors ``1`` and ``2``. + + :fixture:`capfdbinary` + Capture, as bytes, output to file descriptors ``1`` and ``2``. + + :fixture:`caplog` + Control logging and access log entries. + + :fixture:`capsys` + Capture, as text, output to ``sys.stdout`` and ``sys.stderr``. + + :fixture:`capsysbinary` + Capture, as bytes, output to ``sys.stdout`` and ``sys.stderr``. + + :fixture:`cache` + Store and retrieve values across pytest runs. + + :fixture:`doctest_namespace` + Provide a dict injected into the docstests namespace. + + :fixture:`monkeypatch` + Temporarily modify classes, functions, dictionaries, + ``os.environ``, and other objects. + + :fixture:`pytestconfig` + Access to configuration values, pluginmanager and plugin hooks. + + :fixture:`record_property` + Add extra properties to the test. + + :fixture:`record_testsuite_property` + Add extra properties to the test suite. + + :fixture:`recwarn` + Record warnings emitted by test functions. + + :fixture:`request` + Provide information on the executing test function. + + :fixture:`testdir` + Provide a temporary test directory to aid in running, and + testing, pytest plugins. + + :fixture:`tmp_path` + Provide a :class:`pathlib.Path` object to a temporary directory + which is unique to each test function. + + :fixture:`tmp_path_factory` + Make session-scoped temporary directories and return + :class:`pathlib.Path` objects. + + :fixture:`tmpdir` + Provide a :class:`py.path.local` object to a temporary + directory which is unique to each test function; + replaced by :fixture:`tmp_path`. + + .. _`py.path.local`: https://py.readthedocs.io/en/latest/path.html + + :fixture:`tmpdir_factory` + Make session-scoped temporary directories and return + :class:`py.path.local` objects; + replaced by :fixture:`tmp_path_factory`. + + +.. _`conftest.py`: +.. _`conftest`: + +Fixture availability +--------------------- + +Fixture availability is determined from the perspective of the test. A fixture +is only available for tests to request if they are in the scope that fixture is +defined in. If a fixture is defined inside a class, it can only be requested by +tests inside that class. But if a fixture is defined inside the global scope of +the module, than every test in that module, even if it's defined inside a class, +can request it. + +Similarly, a test can also only be affected by an autouse fixture if that test +is in the same scope that autouse fixture is defined in (see +:ref:`autouse order`). + +A fixture can also request any other fixture, no matter where it's defined, so +long as the test requesting them can see all fixtures involved. + +For example, here's a test file with a fixture (``outer``) that requests a +fixture (``inner``) from a scope it wasn't defined in: + +.. literalinclude:: /example/fixtures/test_fixtures_request_different_scope.py + +From the tests' perspectives, they have no problem seeing each of the fixtures +they're dependent on: + +.. image:: /example/fixtures/test_fixtures_request_different_scope.* + :align: center + +So when they run, ``outer`` will have no problem finding ``inner``, because +pytest searched from the tests' perspectives. + +.. note:: + The scope a fixture is defined in has no bearing on the order it will be + instantiated in: the order is mandated by the logic described + :ref:`here `. + +``conftest.py``: sharing fixtures across multiple files +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``conftest.py`` file serves as a means of providing fixtures for an entire +directory. Fixtures defined in a ``conftest.py`` can be used by any test +in that package without needing to import them (pytest will automatically +discover them). + +You can have multiple nested directories/packages containing your tests, and +each directory can have its own ``conftest.py`` with its own fixtures, adding on +to the ones provided by the ``conftest.py`` files in parent directories. + +For example, given a test file structure like this: + +:: + + tests/ + __init__.py + + conftest.py + # content of tests/conftest.py + import pytest + + @pytest.fixture + def order(): + return [] + + @pytest.fixture + def top(order, innermost): + order.append("top") + + test_top.py + # content of tests/test_top.py + import pytest + + @pytest.fixture + def innermost(order): + order.append("innermost top") + + def test_order(order, top): + assert order == ["innermost top", "top"] + + subpackage/ + __init__.py + + conftest.py + # content of tests/subpackage/conftest.py + import pytest + + @pytest.fixture + def mid(order): + order.append("mid subpackage") + + test_subpackage.py + # content of tests/subpackage/test_subpackage.py + import pytest + + @pytest.fixture + def innermost(order, mid): + order.append("innermost subpackage") + + def test_order(order, top): + assert order == ["mid subpackage", "innermost subpackage", "top"] + +The boundaries of the scopes can be visualized like this: + +.. image:: /example/fixtures/fixture_availability.* + :align: center + +The directories become their own sort of scope where fixtures that are defined +in a ``conftest.py`` file in that directory become available for that whole +scope. + +Tests are allowed to search upward (stepping outside a circle) for fixtures, but +can never go down (stepping inside a circle) to continue their search. So +``tests/subpackage/test_subpackage.py::test_order`` would be able to find the +``innermost`` fixture defined in ``tests/subpackage/test_subpackage.py``, but +the one defined in ``tests/test_top.py`` would be unavailable to it because it +would have to step down a level (step inside a circle) to find it. + +The first fixture the test finds is the one that will be used, so +:ref:`fixtures can be overridden ` if you need to change or +extend what one does for a particular scope. + +You can also use the ``conftest.py`` file to implement +:ref:`local per-directory plugins `. + +Fixtures from third-party plugins +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Fixtures don't have to be defined in this structure to be available for tests, +though. They can also be provided by third-party plugins that are installed, and +this is how many pytest plugins operate. As long as those plugins are installed, +the fixtures they provide can be requested from anywhere in your test suite. + +Because they're provided from outside the structure of your test suite, +third-party plugins don't really provide a scope like `conftest.py` files and +the directories in your test suite do. As a result, pytest will search for +fixtures stepping out through scopes as explained previously, only reaching +fixtures defined in plugins *last*. + +For example, given the following file structure: + +:: + + tests/ + __init__.py + + conftest.py + # content of tests/conftest.py + import pytest + + @pytest.fixture + def order(): + return [] + + subpackage/ + __init__.py + + conftest.py + # content of tests/subpackage/conftest.py + import pytest + + @pytest.fixture(autouse=True) + def mid(order, b_fix): + order.append("mid subpackage") + + test_subpackage.py + # content of tests/subpackage/test_subpackage.py + import pytest + + @pytest.fixture + def inner(order, mid, a_fix): + order.append("inner subpackage") + + def test_order(order, inner): + assert order == ["b_fix", "mid subpackage", "a_fix", "inner subpackage"] + +If ``plugin_a`` is installed and provides the fixture ``a_fix``, and +``plugin_b`` is installed and provides the fixture ``b_fix``, then this is what +the test's search for fixtures would look like: + +.. image:: /example/fixtures/fixture_availability_plugins.svg + :align: center + +pytest will only search for ``a_fix`` and ``b_fix`` in the plugins after +searching for them first in the scopes inside ``tests/``. + +.. note: + + pytest can tell you what fixtures are available for a given test if you call + ``pytests`` along with the test's name (or the scope it's in), and provide + the ``--fixtures`` flag, e.g. ``pytest --fixtures test_something.py`` + (fixtures with names that start with ``_`` will only be shown if you also + provide the ``-v`` flag). + + +.. _`fixture order`: + +Fixture instantiation order +--------------------------- + +When pytest wants to execute a test, once it knows what fixtures will be +executed, it has to figure out the order they'll be executed in. To do this, it +considers 3 factors: + +1. scope +2. dependencies +3. autouse + +Names of fixtures or tests, where they're defined, the order they're defined in, +and the order fixtures are requested in have no bearing on execution order +beyond coincidence. While pytest will try to make sure coincidences like these +stay consistent from run to run, it's not something that should be depended on. +If you want to control the order, it's safest to rely on these 3 things and make +sure dependencies are clearly established. + +Higher-scoped fixtures are executed first +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Within a function request for fixtures, those of higher-scopes (such as +``session``) are executed before lower-scoped fixtures (such as ``function`` or +``class``). + +Here's an example: + +.. literalinclude:: /example/fixtures/test_fixtures_order_scope.py + +The test will pass because the larger scoped fixtures are executing first. + +The order breaks down to this: + +.. image:: /example/fixtures/test_fixtures_order_scope.* + :align: center + +Fixtures of the same order execute based on dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When a fixture requests another fixture, the other fixture is executed first. +So if fixture ``a`` requests fixture ``b``, fixture ``b`` will execute first, +because ``a`` depends on ``b`` and can't operate without it. Even if ``a`` +doesn't need the result of ``b``, it can still request ``b`` if it needs to make +sure it is executed after ``b``. + +For example: + +.. literalinclude:: /example/fixtures/test_fixtures_order_dependencies.py + +If we map out what depends on what, we get something that look like this: + +.. image:: /example/fixtures/test_fixtures_order_dependencies.* + :align: center + +The rules provided by each fixture (as to what fixture(s) each one has to come +after) are comprehensive enough that it can be flattened to this: + +.. image:: /example/fixtures/test_fixtures_order_dependencies_flat.* + :align: center + +Enough information has to be provided through these requests in order for pytest +to be able to figure out a clear, linear chain of dependencies, and as a result, +an order of operations for a given test. If there's any ambiguity, and the order +of operations can be interpreted more than one way, you should assume pytest +could go with any one of those interpretations at any point. + +For example, if ``d`` didn't request ``c``, i.e.the graph would look like this: + +.. image:: /example/fixtures/test_fixtures_order_dependencies_unclear.* + :align: center + +Because nothing requested ``c`` other than ``g``, and ``g`` also requests ``f``, +it's now unclear if ``c`` should go before/after ``f``, ``e``, or ``d``. The +only rules that were set for ``c`` is that it must execute after ``b`` and +before ``g``. + +pytest doesn't know where ``c`` should go in the case, so it should be assumed +that it could go anywhere between ``g`` and ``b``. + +This isn't necessarily bad, but it's something to keep in mind. If the order +they execute in could affect the behavior a test is targeting, or could +otherwise influence the result of a test, then the order should be defined +explicitly in a way that allows pytest to linearize/"flatten" that order. + +.. _`autouse order`: + +Autouse fixtures are executed first within their scope +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Autouse fixtures are assumed to apply to every test that could reference them, +so they are executed before other fixtures in that scope. Fixtures that are +requested by autouse fixtures effectively become autouse fixtures themselves for +the tests that the real autouse fixture applies to. + +So if fixture ``a`` is autouse and fixture ``b`` is not, but fixture ``a`` +requests fixture ``b``, then fixture ``b`` will effectively be an autouse +fixture as well, but only for the tests that ``a`` applies to. + +In the last example, the graph became unclear if ``d`` didn't request ``c``. But +if ``c`` was autouse, then ``b`` and ``a`` would effectively also be autouse +because ``c`` depends on them. As a result, they would all be shifted above +non-autouse fixtures within that scope. + +So if the test file looked like this: + +.. literalinclude:: /example/fixtures/test_fixtures_order_autouse.py + +the graph would look like this: + +.. image:: /example/fixtures/test_fixtures_order_autouse.* + :align: center + +Because ``c`` can now be put above ``d`` in the graph, pytest can once again +linearize the graph to this: + +.. image:: /example/fixtures/test_fixtures_order_autouse_flat.* + :align: center + +In this example, ``c`` makes ``b`` and ``a`` effectively autouse fixtures as +well. + +Be careful with autouse, though, as an autouse fixture will automatically +execute for every test that can reach it, even if they don't request it. For +example, consider this file: + +.. literalinclude:: /example/fixtures/test_fixtures_order_autouse_multiple_scopes.py + +Even though nothing in ``TestClassWithoutC1Request`` is requesting ``c1``, it still +is executed for the tests inside it anyway: + +.. image:: /example/fixtures/test_fixtures_order_autouse_multiple_scopes.* + :align: center + +But just because one autouse fixture requested a non-autouse fixture, that +doesn't mean the non-autouse fixture becomes an autouse fixture for all contexts +that it can apply to. It only effectively becomes an autouse fixture for the +contexts the real autouse fixture (the one that requested the non-autouse +fixture) can apply to. + +For example, take a look at this test file: + +.. literalinclude:: /example/fixtures/test_fixtures_order_autouse_temp_effects.py + +It would break down to something like this: + +.. image:: /example/fixtures/test_fixtures_order_autouse_temp_effects.* + :align: center + +For ``test_req`` and ``test_no_req`` inside ``TestClassWithAutouse``, ``c3`` +effectively makes ``c2`` an autouse fixture, which is why ``c2`` and ``c3`` are +executed for both tests, despite not being requested, and why ``c2`` and ``c3`` +are executed before ``c1`` for ``test_req``. + +If this made ``c2`` an *actual* autouse fixture, then ``c2`` would also execute +for the tests inside ``TestClassWithoutAutouse``, since they can reference +``c2`` if they wanted to. But it doesn't, because from the perspective of the +``TestClassWithoutAutouse`` tests, ``c2`` isn't an autouse fixture, since they +can't see ``c3``. + + +.. note: + + pytest can tell you what order the fixtures will execute in for a given test + if you call ``pytests`` along with the test's name (or the scope it's in), + and provide the ``--setup-plan`` flag, e.g. + ``pytest --setup-plan test_something.py`` (fixtures with names that start + with ``_`` will only be shown if you also provide the ``-v`` flag). diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/index.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/index.rst new file mode 100644 index 0000000000..d964840031 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/index.rst @@ -0,0 +1,15 @@ +:orphan: + +.. _reference: + +Reference guides +================ + +.. toctree:: + :maxdepth: 1 + + fixtures + plugin_list + customize + reference + exit-codes diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/plugin_list.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/plugin_list.rst new file mode 100644 index 0000000000..ebf4009136 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/plugin_list.rst @@ -0,0 +1,7728 @@ + +.. _plugin-list: + +Plugin List +=========== + +PyPI projects that match "pytest-\*" are considered plugins and are listed +automatically. Packages classified as inactive are excluded. + +.. The following conditional uses a different format for this list when + creating a PDF, because otherwise the table gets far too wide for the + page. + +This list contains 963 plugins. + +.. only:: not latex + + =============================================== ======================================================================================================================================================================== ============== ===================== ================================================ + name summary last release status requires + =============================================== ======================================================================================================================================================================== ============== ===================== ================================================ + :pypi:`pytest-accept` A pytest-plugin for updating doctest outputs Nov 22, 2021 N/A pytest (>=6,<7) + :pypi:`pytest-adaptavist` pytest plugin for generating test execution results within Jira Test Management (tm4j) Nov 30, 2021 N/A pytest (>=5.4.0) + :pypi:`pytest-addons-test` 用于测试pytest的插件 Aug 02, 2021 N/A pytest (>=6.2.4,<7.0.0) + :pypi:`pytest-adf` Pytest plugin for writing Azure Data Factory integration tests May 10, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-adf-azure-identity` Pytest plugin for writing Azure Data Factory integration tests Mar 06, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-agent` Service that exposes a REST API that can be used to interract remotely with Pytest. It is shipped with a dashboard that enables running tests in a more convenient way. Nov 25, 2021 N/A N/A + :pypi:`pytest-aggreport` pytest plugin for pytest-repeat that generate aggregate report of the same test cases with additional statistics details. Mar 07, 2021 4 - Beta pytest (>=6.2.2) + :pypi:`pytest-aio` Pytest plugin for testing async python code Oct 20, 2021 4 - Beta pytest + :pypi:`pytest-aiofiles` pytest fixtures for writing aiofiles tests with pyfakefs May 14, 2017 5 - Production/Stable N/A + :pypi:`pytest-aiohttp` pytest plugin for aiohttp support Dec 05, 2017 N/A pytest + :pypi:`pytest-aiohttp-client` Pytest \`client\` fixture for the Aiohttp Nov 01, 2020 N/A pytest (>=6) + :pypi:`pytest-aioresponses` py.test integration for aioresponses Jul 29, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-aioworkers` A plugin to test aioworkers project with pytest Dec 04, 2019 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-airflow` pytest support for airflow. Apr 03, 2019 3 - Alpha pytest (>=4.4.0) + :pypi:`pytest-airflow-utils` Nov 15, 2021 N/A N/A + :pypi:`pytest-alembic` A pytest plugin for verifying alembic migrations. Dec 02, 2021 N/A pytest (>=1.0) + :pypi:`pytest-allclose` Pytest fixture extending Numpy's allclose function Jul 30, 2019 5 - Production/Stable pytest + :pypi:`pytest-allure-adaptor` Plugin for py.test to generate allure xml reports Jan 10, 2018 N/A pytest (>=2.7.3) + :pypi:`pytest-allure-adaptor2` Plugin for py.test to generate allure xml reports Oct 14, 2020 N/A pytest (>=2.7.3) + :pypi:`pytest-allure-dsl` pytest plugin to test case doc string dls instructions Oct 25, 2020 4 - Beta pytest + :pypi:`pytest-allure-spec-coverage` The pytest plugin aimed to display test coverage of the specs(requirements) in Allure Oct 26, 2021 N/A pytest + :pypi:`pytest-alphamoon` Static code checks used at Alphamoon Oct 21, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-android` This fixture provides a configured "driver" for Android Automated Testing, using uiautomator2. Feb 21, 2019 3 - Alpha pytest + :pypi:`pytest-anki` A pytest plugin for testing Anki add-ons Oct 14, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-annotate` pytest-annotate: Generate PyAnnotate annotations from your pytest tests. Nov 29, 2021 3 - Alpha pytest (<7.0.0,>=3.2.0) + :pypi:`pytest-ansible` Plugin for py.test to simplify calling ansible modules from tests or fixtures May 25, 2021 5 - Production/Stable N/A + :pypi:`pytest-ansible-playbook` Pytest fixture which runs given ansible playbook file. Mar 08, 2019 4 - Beta N/A + :pypi:`pytest-ansible-playbook-runner` Pytest fixture which runs given ansible playbook file. Dec 02, 2020 4 - Beta pytest (>=3.1.0) + :pypi:`pytest-antilru` Bust functools.lru_cache when running pytest to avoid test pollution Apr 11, 2019 5 - Production/Stable pytest + :pypi:`pytest-anyio` The pytest anyio plugin is built into anyio. You don't need this package. Jun 29, 2021 N/A pytest + :pypi:`pytest-anything` Pytest fixtures to assert anything and something Feb 18, 2021 N/A N/A + :pypi:`pytest-aoc` Downloads puzzle inputs for Advent of Code and synthesizes PyTest fixtures Nov 23, 2021 N/A pytest ; extra == 'test' + :pypi:`pytest-api` PyTest-API Python Web Framework built for testing purposes. May 04, 2021 N/A N/A + :pypi:`pytest-apistellar` apistellar plugin for pytest. Jun 18, 2019 N/A N/A + :pypi:`pytest-appengine` AppEngine integration that works well with pytest-django Feb 27, 2017 N/A N/A + :pypi:`pytest-appium` Pytest plugin for appium Dec 05, 2019 N/A N/A + :pypi:`pytest-approvaltests` A plugin to use approvaltests with pytest Feb 07, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-argus` pyest results colection plugin Jun 24, 2021 5 - Production/Stable pytest (>=6.2.4) + :pypi:`pytest-arraydiff` pytest plugin to help with comparing array output from tests Dec 06, 2018 4 - Beta pytest + :pypi:`pytest-asgi-server` Convenient ASGI client/server fixtures for Pytest Dec 12, 2020 N/A pytest (>=5.4.1) + :pypi:`pytest-asptest` test Answer Set Programming programs Apr 28, 2018 4 - Beta N/A + :pypi:`pytest-assertutil` pytest-assertutil May 10, 2019 N/A N/A + :pypi:`pytest-assert-utils` Useful assertion utilities for use with pytest Sep 21, 2021 3 - Alpha N/A + :pypi:`pytest-assume` A pytest plugin that allows multiple failures per test Jun 24, 2021 N/A pytest (>=2.7) + :pypi:`pytest-ast-back-to-python` A plugin for pytest devs to view how assertion rewriting recodes the AST Sep 29, 2019 4 - Beta N/A + :pypi:`pytest-astropy` Meta-package containing dependencies for testing Sep 21, 2021 5 - Production/Stable pytest (>=4.6) + :pypi:`pytest-astropy-header` pytest plugin to add diagnostic information to the header of the test output Dec 18, 2019 3 - Alpha pytest (>=2.8) + :pypi:`pytest-ast-transformer` May 04, 2019 3 - Alpha pytest + :pypi:`pytest-asyncio` Pytest support for asyncio. Oct 15, 2021 4 - Beta pytest (>=5.4.0) + :pypi:`pytest-asyncio-cooperative` Run all your asynchronous tests cooperatively. Oct 12, 2021 4 - Beta N/A + :pypi:`pytest-asyncio-network-simulator` pytest-asyncio-network-simulator: Plugin for pytest for simulator the network in tests Jul 31, 2018 3 - Alpha pytest (<3.7.0,>=3.3.2) + :pypi:`pytest-async-mongodb` pytest plugin for async MongoDB Oct 18, 2017 5 - Production/Stable pytest (>=2.5.2) + :pypi:`pytest-async-sqlalchemy` Database testing fixtures using the SQLAlchemy asyncio API Oct 07, 2021 4 - Beta pytest (>=6.0.0) + :pypi:`pytest-atomic` Skip rest of tests if previous test failed. Nov 24, 2018 4 - Beta N/A + :pypi:`pytest-attrib` pytest plugin to select tests based on attributes similar to the nose-attrib plugin May 24, 2016 4 - Beta N/A + :pypi:`pytest-austin` Austin plugin for pytest Oct 11, 2020 4 - Beta N/A + :pypi:`pytest-autochecklog` automatically check condition and log all the checks Apr 25, 2015 4 - Beta N/A + :pypi:`pytest-automation` pytest plugin for building a test suite, using YAML files to extend pytest parameterize functionality. Oct 01, 2021 N/A pytest + :pypi:`pytest-automock` Pytest plugin for automatical mocks creation Apr 22, 2020 N/A pytest ; extra == 'dev' + :pypi:`pytest-auto-parametrize` pytest plugin: avoid repeating arguments in parametrize Oct 02, 2016 3 - Alpha N/A + :pypi:`pytest-autotest` This fixture provides a configured "driver" for Android Automated Testing, using uiautomator2. Aug 25, 2021 N/A pytest + :pypi:`pytest-avoidance` Makes pytest skip tests that don not need rerunning May 23, 2019 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-aws` pytest plugin for testing AWS resource configurations Oct 04, 2017 4 - Beta N/A + :pypi:`pytest-aws-config` Protect your AWS credentials in unit tests May 28, 2021 N/A N/A + :pypi:`pytest-axe` pytest plugin for axe-selenium-python Nov 12, 2018 N/A pytest (>=3.0.0) + :pypi:`pytest-azurepipelines` Formatting PyTest output for Azure Pipelines UI Jul 23, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-bandit` A bandit plugin for pytest Feb 23, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-base-url` pytest plugin for URL based testing Jun 19, 2020 5 - Production/Stable pytest (>=2.7.3) + :pypi:`pytest-bdd` BDD for pytest Oct 25, 2021 6 - Mature pytest (>=4.3) + :pypi:`pytest-bdd-splinter` Common steps for pytest bdd and splinter integration Aug 12, 2019 5 - Production/Stable pytest (>=4.0.0) + :pypi:`pytest-bdd-web` A simple plugin to use with pytest Jan 02, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-bdd-wrappers` Feb 11, 2020 2 - Pre-Alpha N/A + :pypi:`pytest-beakerlib` A pytest plugin that reports test results to the BeakerLib framework Mar 17, 2017 5 - Production/Stable pytest + :pypi:`pytest-beds` Fixtures for testing Google Appengine (GAE) apps Jun 07, 2016 4 - Beta N/A + :pypi:`pytest-bench` Benchmark utility that plugs into pytest. Jul 21, 2014 3 - Alpha N/A + :pypi:`pytest-benchmark` A \`\`pytest\`\` fixture for benchmarking code. It will group the tests into rounds that are calibrated to the chosen timer. Apr 17, 2021 5 - Production/Stable pytest (>=3.8) + :pypi:`pytest-bg-process` Pytest plugin to initialize background process Aug 17, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-bigchaindb` A BigchainDB plugin for pytest. Aug 17, 2021 4 - Beta N/A + :pypi:`pytest-bigquery-mock` Provides a mock fixture for python bigquery client Aug 05, 2021 N/A pytest (>=5.0) + :pypi:`pytest-black` A pytest plugin to enable format checking with black Oct 05, 2020 4 - Beta N/A + :pypi:`pytest-black-multipy` Allow '--black' on older Pythons Jan 14, 2021 5 - Production/Stable pytest (!=3.7.3,>=3.5) ; extra == 'testing' + :pypi:`pytest-blame` A pytest plugin helps developers to debug by providing useful commits history. May 04, 2019 N/A pytest (>=4.4.0) + :pypi:`pytest-blender` Blender Pytest plugin. Oct 29, 2021 N/A pytest (==6.2.5) ; extra == 'dev' + :pypi:`pytest-blink1` Pytest plugin to emit notifications via the Blink(1) RGB LED Jan 07, 2018 4 - Beta N/A + :pypi:`pytest-blockage` Disable network requests during a test run. Feb 13, 2019 N/A pytest + :pypi:`pytest-blocker` pytest plugin to mark a test as blocker and skip all other tests Sep 07, 2015 4 - Beta N/A + :pypi:`pytest-board` Local continuous test runner with pytest and watchdog. Jan 20, 2019 N/A N/A + :pypi:`pytest-bpdb` A py.test plug-in to enable drop to bpdb debugger on test failure. Jan 19, 2015 2 - Pre-Alpha N/A + :pypi:`pytest-bravado` Pytest-bravado automatically generates from OpenAPI specification client fixtures. Jul 19, 2021 N/A N/A + :pypi:`pytest-breakword` Use breakword with pytest Aug 04, 2021 N/A pytest (>=6.2.4,<7.0.0) + :pypi:`pytest-breed-adapter` A simple plugin to connect with breed-server Nov 07, 2018 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-briefcase` A pytest plugin for running tests on a Briefcase project. Jun 14, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-browser` A pytest plugin for console based browser test selection just after the collection phase Dec 10, 2016 3 - Alpha N/A + :pypi:`pytest-browsermob-proxy` BrowserMob proxy plugin for py.test. Jun 11, 2013 4 - Beta N/A + :pypi:`pytest-browserstack-local` \`\`py.test\`\` plugin to run \`\`BrowserStackLocal\`\` in background. Feb 09, 2018 N/A N/A + :pypi:`pytest-bug` Pytest plugin for marking tests as a bug Jun 02, 2020 5 - Production/Stable pytest (>=3.6.0) + :pypi:`pytest-bugtong-tag` pytest-bugtong-tag is a plugin for pytest Apr 23, 2021 N/A N/A + :pypi:`pytest-bugzilla` py.test bugzilla integration plugin May 05, 2010 4 - Beta N/A + :pypi:`pytest-bugzilla-notifier` A plugin that allows you to execute create, update, and read information from BugZilla bugs Jun 15, 2018 4 - Beta pytest (>=2.9.2) + :pypi:`pytest-buildkite` Plugin for pytest that automatically publishes coverage and pytest report annotations to Buildkite. Jul 13, 2019 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-builtin-types` Nov 17, 2021 N/A pytest + :pypi:`pytest-bwrap` Run your tests in Bubblewrap sandboxes Oct 26, 2018 3 - Alpha N/A + :pypi:`pytest-cache` pytest plugin with mechanisms for caching across test runs Jun 04, 2013 3 - Alpha N/A + :pypi:`pytest-cache-assert` Cache assertion data to simplify regression testing of complex serializable data Nov 03, 2021 4 - Beta pytest (>=5) + :pypi:`pytest-cagoule` Pytest plugin to only run tests affected by changes Jan 01, 2020 3 - Alpha N/A + :pypi:`pytest-camel-collect` Enable CamelCase-aware pytest class collection Aug 02, 2020 N/A pytest (>=2.9) + :pypi:`pytest-canonical-data` A plugin which allows to compare results with canonical results, based on previous runs May 08, 2020 2 - Pre-Alpha pytest (>=3.5.0) + :pypi:`pytest-caprng` A plugin that replays pRNG state on failure. May 02, 2018 4 - Beta N/A + :pypi:`pytest-capture-deprecatedwarnings` pytest plugin to capture all deprecatedwarnings and put them in one file Apr 30, 2019 N/A N/A + :pypi:`pytest-capturelogs` A sample Python project Sep 11, 2021 3 - Alpha N/A + :pypi:`pytest-cases` Separate test code from test cases in pytest. Nov 08, 2021 5 - Production/Stable N/A + :pypi:`pytest-cassandra` Cassandra CCM Test Fixtures for pytest Nov 04, 2017 1 - Planning N/A + :pypi:`pytest-catchlog` py.test plugin to catch log messages. This is a fork of pytest-capturelog. Jan 24, 2016 4 - Beta pytest (>=2.6) + :pypi:`pytest-catch-server` Pytest plugin with server for catching HTTP requests. Dec 12, 2019 5 - Production/Stable N/A + :pypi:`pytest-celery` pytest-celery a shim pytest plugin to enable celery.contrib.pytest May 06, 2021 N/A N/A + :pypi:`pytest-chainmaker` pytest plugin for chainmaker Oct 15, 2021 N/A N/A + :pypi:`pytest-chalice` A set of py.test fixtures for AWS Chalice Jul 01, 2020 4 - Beta N/A + :pypi:`pytest-change-report` turn . into √,turn F into x Sep 14, 2020 N/A pytest + :pypi:`pytest-chdir` A pytest fixture for changing current working directory Jan 28, 2020 N/A pytest (>=5.0.0,<6.0.0) + :pypi:`pytest-checkdocs` check the README when running tests Jul 31, 2021 5 - Production/Stable pytest (>=4.6) ; extra == 'testing' + :pypi:`pytest-checkipdb` plugin to check if there are ipdb debugs left Jul 22, 2020 5 - Production/Stable pytest (>=2.9.2) + :pypi:`pytest-check-links` Check links in files Jul 29, 2020 N/A pytest (>=4.6) + :pypi:`pytest-check-mk` pytest plugin to test Check_MK checks Nov 19, 2015 4 - Beta pytest + :pypi:`pytest-circleci` py.test plugin for CircleCI May 03, 2019 N/A N/A + :pypi:`pytest-circleci-parallelized` Parallelize pytest across CircleCI workers. Mar 26, 2019 N/A N/A + :pypi:`pytest-ckan` Backport of CKAN 2.9 pytest plugin and fixtures to CAKN 2.8 Apr 28, 2020 4 - Beta pytest + :pypi:`pytest-clarity` A plugin providing an alternative, colourful diff output for failing assertions. Jun 11, 2021 N/A N/A + :pypi:`pytest-cldf` Easy quality control for CLDF datasets using pytest May 06, 2019 N/A N/A + :pypi:`pytest-click` Py.test plugin for Click Aug 29, 2020 5 - Production/Stable pytest (>=5.0) + :pypi:`pytest-clld` Nov 29, 2021 N/A pytest (>=3.6) + :pypi:`pytest-cloud` Distributed tests planner plugin for pytest testing framework. Oct 05, 2020 6 - Mature N/A + :pypi:`pytest-cloudflare-worker` pytest plugin for testing cloudflare workers Mar 30, 2021 4 - Beta pytest (>=6.0.0) + :pypi:`pytest-cobra` PyTest plugin for testing Smart Contracts for Ethereum blockchain. Jun 29, 2019 3 - Alpha pytest (<4.0.0,>=3.7.1) + :pypi:`pytest-codeblocks` Test code blocks in your READMEs Oct 13, 2021 4 - Beta pytest (>=6) + :pypi:`pytest-codecheckers` pytest plugin to add source code sanity checks (pep8 and friends) Feb 13, 2010 N/A N/A + :pypi:`pytest-codecov` Pytest plugin for uploading pytest-cov results to codecov.io Oct 27, 2021 4 - Beta pytest (>=4.6.0) + :pypi:`pytest-codegen` Automatically create pytest test signatures Aug 23, 2020 2 - Pre-Alpha N/A + :pypi:`pytest-codestyle` pytest plugin to run pycodestyle Mar 23, 2020 3 - Alpha N/A + :pypi:`pytest-collect-formatter` Formatter for pytest collect output Mar 29, 2021 5 - Production/Stable N/A + :pypi:`pytest-collect-formatter2` Formatter for pytest collect output May 31, 2021 5 - Production/Stable N/A + :pypi:`pytest-colordots` Colorizes the progress indicators Oct 06, 2017 5 - Production/Stable N/A + :pypi:`pytest-commander` An interactive GUI test runner for PyTest Aug 17, 2021 N/A pytest (<7.0.0,>=6.2.4) + :pypi:`pytest-common-subject` pytest framework for testing different aspects of a common method Nov 12, 2020 N/A pytest (>=3.6,<7) + :pypi:`pytest-concurrent` Concurrently execute test cases with multithread, multiprocess and gevent Jan 12, 2019 4 - Beta pytest (>=3.1.1) + :pypi:`pytest-config` Base configurations and utilities for developing your Python project test suite with pytest. Nov 07, 2014 5 - Production/Stable N/A + :pypi:`pytest-confluence-report` Package stands for pytest plugin to upload results into Confluence page. Nov 06, 2020 N/A N/A + :pypi:`pytest-console-scripts` Pytest plugin for testing console scripts Sep 28, 2021 4 - Beta N/A + :pypi:`pytest-consul` pytest plugin with fixtures for testing consul aware apps Nov 24, 2018 3 - Alpha pytest + :pypi:`pytest-container` Pytest fixtures for writing container based tests Nov 19, 2021 3 - Alpha pytest (>=3.10) + :pypi:`pytest-contextfixture` Define pytest fixtures as context managers. Mar 12, 2013 4 - Beta N/A + :pypi:`pytest-contexts` A plugin to run tests written with the Contexts framework using pytest May 19, 2021 4 - Beta N/A + :pypi:`pytest-cookies` The pytest plugin for your Cookiecutter templates. 🍪 May 24, 2021 5 - Production/Stable pytest (>=3.3.0) + :pypi:`pytest-couchdbkit` py.test extension for per-test couchdb databases using couchdbkit Apr 17, 2012 N/A N/A + :pypi:`pytest-count` count erros and send email Jan 12, 2018 4 - Beta N/A + :pypi:`pytest-cov` Pytest plugin for measuring coverage. Oct 04, 2021 5 - Production/Stable pytest (>=4.6) + :pypi:`pytest-cover` Pytest plugin for measuring coverage. Forked from \`pytest-cov\`. Aug 01, 2015 5 - Production/Stable N/A + :pypi:`pytest-coverage` Jun 17, 2015 N/A N/A + :pypi:`pytest-coverage-context` Coverage dynamic context support for PyTest, including sub-processes Jan 04, 2021 4 - Beta pytest (>=6.1.0) + :pypi:`pytest-cov-exclude` Pytest plugin for excluding tests based on coverage data Apr 29, 2016 4 - Beta pytest (>=2.8.0,<2.9.0); extra == 'dev' + :pypi:`pytest-cpp` Use pytest's runner to discover and execute C++ tests Dec 03, 2021 5 - Production/Stable pytest (!=5.4.0,!=5.4.1) + :pypi:`pytest-cram` Run cram tests with pytest. Aug 08, 2020 N/A N/A + :pypi:`pytest-crate` Manages CrateDB instances during your integration tests May 28, 2019 3 - Alpha pytest (>=4.0) + :pypi:`pytest-cricri` A Cricri plugin for pytest. Jan 27, 2018 N/A pytest + :pypi:`pytest-crontab` add crontab task in crontab Dec 09, 2019 N/A N/A + :pypi:`pytest-csv` CSV output for pytest. Apr 22, 2021 N/A pytest (>=6.0) + :pypi:`pytest-curio` Pytest support for curio. Oct 07, 2020 N/A N/A + :pypi:`pytest-curl-report` pytest plugin to generate curl command line report Dec 11, 2016 4 - Beta N/A + :pypi:`pytest-custom-concurrency` Custom grouping concurrence for pytest Feb 08, 2021 N/A N/A + :pypi:`pytest-custom-exit-code` Exit pytest test session with custom exit code in different scenarios Aug 07, 2019 4 - Beta pytest (>=4.0.2) + :pypi:`pytest-custom-nodeid` Custom grouping for pytest-xdist, rename test cases name and test cases nodeid, support allure report Mar 07, 2021 N/A N/A + :pypi:`pytest-custom-report` Configure the symbols displayed for test outcomes Jan 30, 2019 N/A pytest + :pypi:`pytest-custom-scheduling` Custom grouping for pytest-xdist, rename test cases name and test cases nodeid, support allure report Mar 01, 2021 N/A N/A + :pypi:`pytest-cython` A plugin for testing Cython extension modules Jan 26, 2021 4 - Beta pytest (>=2.7.3) + :pypi:`pytest-darker` A pytest plugin for checking of modified code using Darker Aug 16, 2020 N/A pytest (>=6.0.1) ; extra == 'test' + :pypi:`pytest-dash` pytest fixtures to run dash applications. Mar 18, 2019 N/A N/A + :pypi:`pytest-data` Useful functions for managing data for pytest fixtures Nov 01, 2016 5 - Production/Stable N/A + :pypi:`pytest-databricks` Pytest plugin for remote Databricks notebooks testing Jul 29, 2020 N/A pytest + :pypi:`pytest-datadir` pytest plugin for test data directories and files Oct 22, 2019 5 - Production/Stable pytest (>=2.7.0) + :pypi:`pytest-datadir-mgr` Manager for test data providing downloads, caching of generated files, and a context for temp directories. Aug 16, 2021 5 - Production/Stable pytest + :pypi:`pytest-datadir-ng` Fixtures for pytest allowing test functions/methods to easily retrieve test resources from the local filesystem. Dec 25, 2019 5 - Production/Stable pytest + :pypi:`pytest-data-file` Fixture "data" and "case_data" for test from yaml file Dec 04, 2019 N/A N/A + :pypi:`pytest-datafiles` py.test plugin to create a 'tmpdir' containing predefined files/directories. Oct 07, 2018 5 - Production/Stable pytest (>=3.6) + :pypi:`pytest-datafixtures` Data fixtures for pytest made simple Dec 05, 2020 5 - Production/Stable N/A + :pypi:`pytest-data-from-files` pytest plugin to provide data from files loaded automatically Oct 13, 2021 4 - Beta pytest + :pypi:`pytest-dataplugin` A pytest plugin for managing an archive of test data. Sep 16, 2017 1 - Planning N/A + :pypi:`pytest-datarecorder` A py.test plugin recording and comparing test output. Apr 20, 2020 5 - Production/Stable pytest + :pypi:`pytest-datatest` A pytest plugin for test driven data-wrangling (this is the development version of datatest's pytest integration). Oct 15, 2020 4 - Beta pytest (>=3.3) + :pypi:`pytest-db` Session scope fixture "db" for mysql query or change Dec 04, 2019 N/A N/A + :pypi:`pytest-dbfixtures` Databases fixtures plugin for py.test. Dec 07, 2016 4 - Beta N/A + :pypi:`pytest-db-plugin` Nov 27, 2021 N/A pytest (>=5.0) + :pypi:`pytest-dbt-adapter` A pytest plugin for testing dbt adapter plugins Nov 24, 2021 N/A pytest (<7,>=6) + :pypi:`pytest-dbus-notification` D-BUS notifications for pytest results. Mar 05, 2014 5 - Production/Stable N/A + :pypi:`pytest-deadfixtures` A simple plugin to list unused fixtures in pytest Jul 23, 2020 5 - Production/Stable N/A + :pypi:`pytest-deepcov` deepcov Mar 30, 2021 N/A N/A + :pypi:`pytest-defer` Aug 24, 2021 N/A N/A + :pypi:`pytest-demo-plugin` pytest示例插件 May 15, 2021 N/A N/A + :pypi:`pytest-dependency` Manage dependencies of tests Feb 14, 2020 4 - Beta N/A + :pypi:`pytest-depends` Tests that depend on other tests Apr 05, 2020 5 - Production/Stable pytest (>=3) + :pypi:`pytest-deprecate` Mark tests as testing a deprecated feature with a warning note. Jul 01, 2019 N/A N/A + :pypi:`pytest-describe` Describe-style plugin for pytest Nov 13, 2021 4 - Beta pytest (>=4.0.0) + :pypi:`pytest-describe-it` plugin for rich text descriptions Jul 19, 2019 4 - Beta pytest + :pypi:`pytest-devpi-server` DevPI server fixture for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-diamond` pytest plugin for diamond Aug 31, 2015 4 - Beta N/A + :pypi:`pytest-dicom` pytest plugin to provide DICOM fixtures Dec 19, 2018 3 - Alpha pytest + :pypi:`pytest-dictsdiff` Jul 26, 2019 N/A N/A + :pypi:`pytest-diff` A simple plugin to use with pytest Mar 30, 2019 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-disable` pytest plugin to disable a test and skip it from testrun Sep 10, 2015 4 - Beta N/A + :pypi:`pytest-disable-plugin` Disable plugins per test Feb 28, 2019 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-discord` A pytest plugin to notify test results to a Discord channel. Mar 20, 2021 3 - Alpha pytest (!=6.0.0,<7,>=3.3.2) + :pypi:`pytest-django` A Django plugin for pytest. Dec 02, 2021 5 - Production/Stable pytest (>=5.4.0) + :pypi:`pytest-django-ahead` A Django plugin for pytest. Oct 27, 2016 5 - Production/Stable pytest (>=2.9) + :pypi:`pytest-djangoapp` Nice pytest plugin to help you with Django pluggable application testing. Aug 04, 2021 4 - Beta N/A + :pypi:`pytest-django-cache-xdist` A djangocachexdist plugin for pytest May 12, 2020 4 - Beta N/A + :pypi:`pytest-django-casperjs` Integrate CasperJS with your django tests as a pytest fixture. Mar 15, 2015 2 - Pre-Alpha N/A + :pypi:`pytest-django-dotenv` Pytest plugin used to setup environment variables with django-dotenv Nov 26, 2019 4 - Beta pytest (>=2.6.0) + :pypi:`pytest-django-factories` Factories for your Django models that can be used as Pytest fixtures. Nov 12, 2020 4 - Beta N/A + :pypi:`pytest-django-gcir` A Django plugin for pytest. Mar 06, 2018 5 - Production/Stable N/A + :pypi:`pytest-django-haystack` Cleanup your Haystack indexes between tests Sep 03, 2017 5 - Production/Stable pytest (>=2.3.4) + :pypi:`pytest-django-ifactory` A model instance factory for pytest-django Jan 13, 2021 3 - Alpha N/A + :pypi:`pytest-django-lite` The bare minimum to integrate py.test with Django. Jan 30, 2014 N/A N/A + :pypi:`pytest-django-liveserver-ssl` Jul 30, 2021 3 - Alpha N/A + :pypi:`pytest-django-model` A Simple Way to Test your Django Models Feb 14, 2019 4 - Beta N/A + :pypi:`pytest-django-ordering` A pytest plugin for preserving the order in which Django runs tests. Jul 25, 2019 5 - Production/Stable pytest (>=2.3.0) + :pypi:`pytest-django-queries` Generate performance reports from your django database performance tests. Mar 01, 2021 N/A N/A + :pypi:`pytest-djangorestframework` A djangorestframework plugin for pytest Aug 11, 2019 4 - Beta N/A + :pypi:`pytest-django-rq` A pytest plugin to help writing unit test for django-rq Apr 13, 2020 4 - Beta N/A + :pypi:`pytest-django-sqlcounts` py.test plugin for reporting the number of SQLs executed per django testcase. Jun 16, 2015 4 - Beta N/A + :pypi:`pytest-django-testing-postgresql` Use a temporary PostgreSQL database with pytest-django Dec 05, 2019 3 - Alpha N/A + :pypi:`pytest-doc` A documentation plugin for py.test. Jun 28, 2015 5 - Production/Stable N/A + :pypi:`pytest-docgen` An RST Documentation Generator for pytest-based test suites Apr 17, 2020 N/A N/A + :pypi:`pytest-docker` Simple pytest fixtures for Docker and docker-compose based tests Jun 14, 2021 N/A pytest (<7.0,>=4.0) + :pypi:`pytest-docker-butla` Jun 16, 2019 3 - Alpha N/A + :pypi:`pytest-dockerc` Run, manage and stop Docker Compose project from Docker API Oct 09, 2020 5 - Production/Stable pytest (>=3.0) + :pypi:`pytest-docker-compose` Manages Docker containers during your integration tests Jan 26, 2021 5 - Production/Stable pytest (>=3.3) + :pypi:`pytest-docker-db` A plugin to use docker databases for pytests Mar 20, 2021 5 - Production/Stable pytest (>=3.1.1) + :pypi:`pytest-docker-fixtures` pytest docker fixtures Nov 23, 2021 3 - Alpha N/A + :pypi:`pytest-docker-git-fixtures` Pytest fixtures for testing with git scm. Mar 11, 2021 4 - Beta pytest + :pypi:`pytest-docker-pexpect` pytest plugin for writing functional tests with pexpect and docker Jan 14, 2019 N/A pytest + :pypi:`pytest-docker-postgresql` A simple plugin to use with pytest Sep 24, 2019 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-docker-py` Easy to use, simple to extend, pytest plugin that minimally leverages docker-py. Nov 27, 2018 N/A pytest (==4.0.0) + :pypi:`pytest-docker-registry-fixtures` Pytest fixtures for testing with docker registries. Mar 04, 2021 4 - Beta pytest + :pypi:`pytest-docker-tools` Docker integration tests for pytest Jul 23, 2021 4 - Beta pytest (>=6.0.1,<7.0.0) + :pypi:`pytest-docs` Documentation tool for pytest Nov 11, 2018 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-docstyle` pytest plugin to run pydocstyle Mar 23, 2020 3 - Alpha N/A + :pypi:`pytest-doctest-custom` A py.test plugin for customizing string representations of doctest results. Jul 25, 2016 4 - Beta N/A + :pypi:`pytest-doctest-ellipsis-markers` Setup additional values for ELLIPSIS_MARKER for doctests Jan 12, 2018 4 - Beta N/A + :pypi:`pytest-doctest-import` A simple pytest plugin to import names and add them to the doctest namespace. Nov 13, 2018 4 - Beta pytest (>=3.3.0) + :pypi:`pytest-doctestplus` Pytest plugin with advanced doctest features. Nov 16, 2021 3 - Alpha pytest (>=4.6) + :pypi:`pytest-doctest-ufunc` A plugin to run doctests in docstrings of Numpy ufuncs Aug 02, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-dolphin` Some extra stuff that we use ininternally Nov 30, 2016 4 - Beta pytest (==3.0.4) + :pypi:`pytest-doorstop` A pytest plugin for adding test results into doorstop items. Jun 09, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-dotenv` A py.test plugin that parses environment files before running tests Jun 16, 2020 4 - Beta pytest (>=5.0.0) + :pypi:`pytest-drf` A Django REST framework plugin for pytest. Nov 12, 2020 5 - Production/Stable pytest (>=3.6) + :pypi:`pytest-drivings` Tool to allow webdriver automation to be ran locally or remotely Jan 13, 2021 N/A N/A + :pypi:`pytest-drop-dup-tests` A Pytest plugin to drop duplicated tests during collection May 23, 2020 4 - Beta pytest (>=2.7) + :pypi:`pytest-dummynet` A py.test plugin providing access to a dummynet. Oct 13, 2021 5 - Production/Stable pytest + :pypi:`pytest-dump2json` A pytest plugin for dumping test results to json. Jun 29, 2015 N/A N/A + :pypi:`pytest-duration-insights` Jun 25, 2021 N/A N/A + :pypi:`pytest-dynamicrerun` A pytest plugin to rerun tests dynamically based off of test outcome and output. Aug 15, 2020 4 - Beta N/A + :pypi:`pytest-dynamodb` DynamoDB fixtures for pytest Jun 03, 2021 5 - Production/Stable pytest + :pypi:`pytest-easy-addoption` pytest-easy-addoption: Easy way to work with pytest addoption Jan 22, 2020 N/A N/A + :pypi:`pytest-easy-api` Simple API testing with pytest Mar 26, 2018 N/A N/A + :pypi:`pytest-easyMPI` Package that supports mpi tests in pytest Oct 21, 2020 N/A N/A + :pypi:`pytest-easyread` pytest plugin that makes terminal printouts of the reports easier to read Nov 17, 2017 N/A N/A + :pypi:`pytest-easy-server` Pytest plugin for easy testing against servers May 01, 2021 4 - Beta pytest (<5.0.0,>=4.3.1) ; python_version < "3.5" + :pypi:`pytest-ec2` Pytest execution on EC2 instance Oct 22, 2019 3 - Alpha N/A + :pypi:`pytest-echo` pytest plugin with mechanisms for echoing environment variables, package version and generic attributes Jan 08, 2020 5 - Production/Stable N/A + :pypi:`pytest-elasticsearch` Elasticsearch fixtures and fixture factories for Pytest. May 12, 2021 5 - Production/Stable pytest (>=3.0.0) + :pypi:`pytest-elements` Tool to help automate user interfaces Jan 13, 2021 N/A pytest (>=5.4,<6.0) + :pypi:`pytest-elk-reporter` A simple plugin to use with pytest Jan 24, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-email` Send execution result email Jul 08, 2020 N/A pytest + :pypi:`pytest-embedded` pytest embedded plugin Nov 29, 2021 N/A pytest (>=6.2.0) + :pypi:`pytest-embedded-idf` pytest embedded plugin for esp-idf project Nov 29, 2021 N/A N/A + :pypi:`pytest-embedded-jtag` pytest embedded plugin for testing with jtag Nov 29, 2021 N/A N/A + :pypi:`pytest-embedded-qemu` pytest embedded plugin for qemu, not target chip Nov 29, 2021 N/A N/A + :pypi:`pytest-embedded-qemu-idf` pytest embedded plugin for esp-idf project by qemu, not target chip Jun 29, 2021 N/A N/A + :pypi:`pytest-embedded-serial` pytest embedded plugin for testing serial ports Nov 29, 2021 N/A N/A + :pypi:`pytest-embedded-serial-esp` pytest embedded plugin for testing espressif boards via serial ports Nov 29, 2021 N/A N/A + :pypi:`pytest-emoji` A pytest plugin that adds emojis to your test result report Feb 19, 2019 4 - Beta pytest (>=4.2.1) + :pypi:`pytest-emoji-output` Pytest plugin to represent test output with emoji support Oct 10, 2021 4 - Beta pytest (==6.0.1) + :pypi:`pytest-enabler` Enable installed pytest plugins Nov 08, 2021 5 - Production/Stable pytest (>=6) ; extra == 'testing' + :pypi:`pytest-encode` set your encoding and logger Nov 06, 2021 N/A N/A + :pypi:`pytest-encode-kane` set your encoding and logger Nov 16, 2021 N/A pytest + :pypi:`pytest-enhancements` Improvements for pytest (rejected upstream) Oct 30, 2019 4 - Beta N/A + :pypi:`pytest-env` py.test plugin that allows you to add environment variables. Jun 16, 2017 4 - Beta N/A + :pypi:`pytest-envfiles` A py.test plugin that parses environment files before running tests Oct 08, 2015 3 - Alpha N/A + :pypi:`pytest-env-info` Push information about the running pytest into envvars Nov 25, 2017 4 - Beta pytest (>=3.1.1) + :pypi:`pytest-envraw` py.test plugin that allows you to add environment variables. Aug 27, 2020 4 - Beta pytest (>=2.6.0) + :pypi:`pytest-envvars` Pytest plugin to validate use of envvars on your tests Jun 13, 2020 5 - Production/Stable pytest (>=3.0.0) + :pypi:`pytest-env-yaml` Apr 02, 2019 N/A N/A + :pypi:`pytest-eradicate` pytest plugin to check for commented out code Sep 08, 2020 N/A pytest (>=2.4.2) + :pypi:`pytest-error-for-skips` Pytest plugin to treat skipped tests a test failure Dec 19, 2019 4 - Beta pytest (>=4.6) + :pypi:`pytest-eth` PyTest plugin for testing Smart Contracts for Ethereum Virtual Machine (EVM). Aug 14, 2020 1 - Planning N/A + :pypi:`pytest-ethereum` pytest-ethereum: Pytest library for ethereum projects. Jun 24, 2019 3 - Alpha pytest (==3.3.2); extra == 'dev' + :pypi:`pytest-eucalyptus` Pytest Plugin for BDD Aug 13, 2019 N/A pytest (>=4.2.0) + :pypi:`pytest-eventlet` Applies eventlet monkey-patch as a pytest plugin. Oct 04, 2021 N/A pytest ; extra == 'dev' + :pypi:`pytest-excel` pytest plugin for generating excel reports Oct 06, 2020 5 - Production/Stable N/A + :pypi:`pytest-exceptional` Better exceptions Mar 16, 2017 4 - Beta N/A + :pypi:`pytest-exception-script` Walk your code through exception script to check it's resiliency to failures. Aug 04, 2020 3 - Alpha pytest + :pypi:`pytest-executable` pytest plugin for testing executables Nov 10, 2021 4 - Beta pytest (<6.3,>=4.3) + :pypi:`pytest-expect` py.test plugin to store test expectations and mark tests based on them Apr 21, 2016 4 - Beta N/A + :pypi:`pytest-expecter` Better testing with expecter and pytest. Jul 08, 2020 5 - Production/Stable N/A + :pypi:`pytest-expectr` This plugin is used to expect multiple assert using pytest framework. Oct 05, 2018 N/A pytest (>=2.4.2) + :pypi:`pytest-explicit` A Pytest plugin to ignore certain marked tests by default Jun 15, 2021 5 - Production/Stable pytest + :pypi:`pytest-exploratory` Interactive console for pytest. Aug 03, 2021 N/A pytest (>=5.3) + :pypi:`pytest-external-blockers` a special outcome for tests that are blocked for external reasons Oct 05, 2021 N/A pytest + :pypi:`pytest-extra-durations` A pytest plugin to get durations on a per-function basis and per module basis. Apr 21, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-fabric` Provides test utilities to run fabric task tests by using docker containers Sep 12, 2018 5 - Production/Stable N/A + :pypi:`pytest-factory` Use factories for test setup with py.test Sep 06, 2020 3 - Alpha pytest (>4.3) + :pypi:`pytest-factoryboy` Factory Boy support for pytest. Dec 30, 2020 6 - Mature pytest (>=4.6) + :pypi:`pytest-factoryboy-fixtures` Generates pytest fixtures that allow the use of type hinting Jun 25, 2020 N/A N/A + :pypi:`pytest-factoryboy-state` Simple factoryboy random state management Dec 11, 2020 4 - Beta pytest (>=5.0) + :pypi:`pytest-failed-screenshot` Test case fails,take a screenshot,save it,attach it to the allure Apr 21, 2021 N/A N/A + :pypi:`pytest-failed-to-verify` A pytest plugin that helps better distinguishing real test failures from setup flakiness. Aug 08, 2019 5 - Production/Stable pytest (>=4.1.0) + :pypi:`pytest-faker` Faker integration with the pytest framework. Dec 19, 2016 6 - Mature N/A + :pypi:`pytest-falcon` Pytest helpers for Falcon. Sep 07, 2016 4 - Beta N/A + :pypi:`pytest-falcon-client` Pytest \`client\` fixture for the Falcon Framework Mar 19, 2019 N/A N/A + :pypi:`pytest-fantasy` Pytest plugin for Flask Fantasy Framework Mar 14, 2019 N/A N/A + :pypi:`pytest-fastapi` Dec 27, 2020 N/A N/A + :pypi:`pytest-fastest` Use SCM and coverage to run only needed tests Mar 05, 2020 N/A N/A + :pypi:`pytest-fast-first` Pytest plugin that runs fast tests first Apr 02, 2021 3 - Alpha pytest + :pypi:`pytest-faulthandler` py.test plugin that activates the fault handler module for tests (dummy package) Jul 04, 2019 6 - Mature pytest (>=5.0) + :pypi:`pytest-fauxfactory` Integration of fauxfactory into pytest. Dec 06, 2017 5 - Production/Stable pytest (>=3.2) + :pypi:`pytest-figleaf` py.test figleaf coverage plugin Jan 18, 2010 5 - Production/Stable N/A + :pypi:`pytest-filecov` A pytest plugin to detect unused files Jun 27, 2021 4 - Beta pytest + :pypi:`pytest-filedata` easily load data from files Jan 17, 2019 4 - Beta N/A + :pypi:`pytest-filemarker` A pytest plugin that runs marked tests when files change. Dec 01, 2020 N/A pytest + :pypi:`pytest-filter-case` run test cases filter by mark Nov 05, 2020 N/A N/A + :pypi:`pytest-filter-subpackage` Pytest plugin for filtering based on sub-packages Jan 09, 2020 3 - Alpha pytest (>=3.0) + :pypi:`pytest-find-dependencies` A pytest plugin to find dependencies between tests Apr 21, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-finer-verdicts` A pytest plugin to treat non-assertion failures as test errors. Jun 18, 2020 N/A pytest (>=5.4.3) + :pypi:`pytest-firefox` pytest plugin to manipulate firefox Aug 08, 2017 3 - Alpha pytest (>=3.0.2) + :pypi:`pytest-fixture-config` Fixture configuration utils for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-fixture-maker` Pytest plugin to load fixtures from YAML files Sep 21, 2021 N/A N/A + :pypi:`pytest-fixture-marker` A pytest plugin to add markers based on fixtures used. Oct 11, 2020 5 - Production/Stable N/A + :pypi:`pytest-fixture-order` pytest plugin to control fixture evaluation order Aug 25, 2020 N/A pytest (>=3.0) + :pypi:`pytest-fixtures` Common fixtures for pytest May 01, 2019 5 - Production/Stable N/A + :pypi:`pytest-fixture-tools` Plugin for pytest which provides tools for fixtures Aug 18, 2020 6 - Mature pytest + :pypi:`pytest-fixture-typecheck` A pytest plugin to assert type annotations at runtime. Aug 24, 2021 N/A pytest + :pypi:`pytest-flake8` pytest plugin to check FLAKE8 requirements Dec 16, 2020 4 - Beta pytest (>=3.5) + :pypi:`pytest-flake8-path` A pytest fixture for testing flake8 plugins. Aug 11, 2021 5 - Production/Stable pytest + :pypi:`pytest-flakefinder` Runs tests multiple times to expose flakiness. Jul 28, 2020 4 - Beta pytest (>=2.7.1) + :pypi:`pytest-flakes` pytest plugin to check source code with pyflakes Dec 02, 2021 5 - Production/Stable pytest (>=5) + :pypi:`pytest-flaptastic` Flaptastic py.test plugin Mar 17, 2019 N/A N/A + :pypi:`pytest-flask` A set of py.test fixtures to test Flask applications. Feb 27, 2021 5 - Production/Stable pytest (>=5.2) + :pypi:`pytest-flask-sqlalchemy` A pytest plugin for preserving test isolation in Flask-SQlAlchemy using database transactions. Apr 04, 2019 4 - Beta pytest (>=3.2.1) + :pypi:`pytest-flask-sqlalchemy-transactions` Run tests in transactions using pytest, Flask, and SQLalchemy. Aug 02, 2018 4 - Beta pytest (>=3.2.1) + :pypi:`pytest-flyte` Pytest fixtures for simplifying Flyte integration testing May 03, 2021 N/A pytest + :pypi:`pytest-focus` A pytest plugin that alerts user of failed test cases with screen notifications May 04, 2019 4 - Beta pytest + :pypi:`pytest-forcefail` py.test plugin to make the test failing regardless of pytest.mark.xfail May 15, 2018 4 - Beta N/A + :pypi:`pytest-forward-compatability` A name to avoid typosquating pytest-foward-compatibility Sep 06, 2020 N/A N/A + :pypi:`pytest-forward-compatibility` A pytest plugin to shim pytest commandline options for fowards compatibility Sep 29, 2020 N/A N/A + :pypi:`pytest-freezegun` Wrap tests with fixtures in freeze_time Jul 19, 2020 4 - Beta pytest (>=3.0.0) + :pypi:`pytest-freeze-reqs` Check if requirement files are frozen Apr 29, 2021 N/A N/A + :pypi:`pytest-frozen-uuids` Deterministically frozen UUID's for your tests Oct 19, 2021 N/A pytest (>=3.0) + :pypi:`pytest-func-cov` Pytest plugin for measuring function coverage Apr 15, 2021 3 - Alpha pytest (>=5) + :pypi:`pytest-funparam` An alternative way to parametrize test cases. Dec 02, 2021 4 - Beta pytest >=4.6.0 + :pypi:`pytest-fxa` pytest plugin for Firefox Accounts Aug 28, 2018 5 - Production/Stable N/A + :pypi:`pytest-fxtest` Oct 27, 2020 N/A N/A + :pypi:`pytest-gc` The garbage collector plugin for py.test Feb 01, 2018 N/A N/A + :pypi:`pytest-gcov` Uses gcov to measure test coverage of a C library Feb 01, 2018 3 - Alpha N/A + :pypi:`pytest-gevent` Ensure that gevent is properly patched when invoking pytest Feb 25, 2020 N/A pytest + :pypi:`pytest-gherkin` A flexible framework for executing BDD gherkin tests Jul 27, 2019 3 - Alpha pytest (>=5.0.0) + :pypi:`pytest-ghostinspector` For finding/executing Ghost Inspector tests May 17, 2016 3 - Alpha N/A + :pypi:`pytest-girder` A set of pytest fixtures for testing Girder applications. Nov 30, 2021 N/A N/A + :pypi:`pytest-git` Git repository fixture for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-gitcov` Pytest plugin for reporting on coverage of the last git commit. Jan 11, 2020 2 - Pre-Alpha N/A + :pypi:`pytest-git-fixtures` Pytest fixtures for testing with git. Mar 11, 2021 4 - Beta pytest + :pypi:`pytest-github` Plugin for py.test that associates tests with github issues using a marker. Mar 07, 2019 5 - Production/Stable N/A + :pypi:`pytest-github-actions-annotate-failures` pytest plugin to annotate failed tests with a workflow command for GitHub Actions Oct 24, 2021 N/A pytest (>=4.0.0) + :pypi:`pytest-gitignore` py.test plugin to ignore the same files as git Jul 17, 2015 4 - Beta N/A + :pypi:`pytest-glamor-allure` Extends allure-pytest functionality Nov 26, 2021 4 - Beta pytest + :pypi:`pytest-gnupg-fixtures` Pytest fixtures for testing with gnupg. Mar 04, 2021 4 - Beta pytest + :pypi:`pytest-golden` Plugin for pytest that offloads expected outputs to data files Nov 23, 2020 N/A pytest (>=6.1.2,<7.0.0) + :pypi:`pytest-graphql-schema` Get graphql schema as fixture for pytest Oct 18, 2019 N/A N/A + :pypi:`pytest-greendots` Green progress dots Feb 08, 2014 3 - Alpha N/A + :pypi:`pytest-growl` Growl notifications for pytest results. Jan 13, 2014 5 - Production/Stable N/A + :pypi:`pytest-grpc` pytest plugin for grpc May 01, 2020 N/A pytest (>=3.6.0) + :pypi:`pytest-hammertime` Display "🔨 " instead of "." for passed pytest tests. Jul 28, 2018 N/A pytest + :pypi:`pytest-harvest` Store data created during your pytest tests execution, and retrieve it at the end of the session, e.g. for applicative benchmarking purposes. Apr 01, 2021 5 - Production/Stable N/A + :pypi:`pytest-helm-chart` A plugin to provide different types and configs of Kubernetes clusters that can be used for testing. Jun 15, 2020 4 - Beta pytest (>=5.4.2,<6.0.0) + :pypi:`pytest-helm-charts` A plugin to provide different types and configs of Kubernetes clusters that can be used for testing. Oct 26, 2021 4 - Beta pytest (>=6.1.2,<7.0.0) + :pypi:`pytest-helper` Functions to help in using the pytest testing framework May 31, 2019 5 - Production/Stable N/A + :pypi:`pytest-helpers` pytest helpers May 17, 2020 N/A pytest + :pypi:`pytest-helpers-namespace` Pytest Helpers Namespace Plugin Apr 29, 2021 5 - Production/Stable pytest (>=6.0.0) + :pypi:`pytest-hidecaptured` Hide captured output May 04, 2018 4 - Beta pytest (>=2.8.5) + :pypi:`pytest-historic` Custom report to display pytest historical execution records Apr 08, 2020 N/A pytest + :pypi:`pytest-historic-hook` Custom listener to store execution results into MYSQL DB, which is used for pytest-historic report Apr 08, 2020 N/A pytest + :pypi:`pytest-homeassistant` A pytest plugin for use with homeassistant custom components. Aug 12, 2020 4 - Beta N/A + :pypi:`pytest-homeassistant-custom-component` Experimental package to automatically extract test plugins for Home Assistant custom components Nov 20, 2021 3 - Alpha pytest (==6.2.5) + :pypi:`pytest-honors` Report on tests that honor constraints, and guard against regressions Mar 06, 2020 4 - Beta N/A + :pypi:`pytest-hoverfly` Simplify working with Hoverfly from pytest Jul 12, 2021 N/A pytest (>=5.0) + :pypi:`pytest-hoverfly-wrapper` Integrates the Hoverfly HTTP proxy into Pytest Aug 29, 2021 4 - Beta N/A + :pypi:`pytest-hpfeeds` Helpers for testing hpfeeds in your python project Aug 27, 2021 4 - Beta pytest (>=6.2.4,<7.0.0) + :pypi:`pytest-html` pytest plugin for generating HTML reports Dec 13, 2020 5 - Production/Stable pytest (!=6.0.0,>=5.0) + :pypi:`pytest-html-lee` optimized pytest plugin for generating HTML reports Jun 30, 2020 5 - Production/Stable pytest (>=5.0) + :pypi:`pytest-html-profiling` Pytest plugin for generating HTML reports with per-test profiling and optionally call graph visualizations. Based on pytest-html by Dave Hunt. Feb 11, 2020 5 - Production/Stable pytest (>=3.0) + :pypi:`pytest-html-reporter` Generates a static html report based on pytest framework Apr 25, 2021 N/A N/A + :pypi:`pytest-html-thread` pytest plugin for generating HTML reports Dec 29, 2020 5 - Production/Stable N/A + :pypi:`pytest-http` Fixture "http" for http requests Dec 05, 2019 N/A N/A + :pypi:`pytest-httpbin` Easily test your HTTP library against a local copy of httpbin Feb 11, 2019 5 - Production/Stable N/A + :pypi:`pytest-http-mocker` Pytest plugin for http mocking (via https://github.com/vilus/mocker) Oct 20, 2019 N/A N/A + :pypi:`pytest-httpretty` A thin wrapper of HTTPretty for pytest Feb 16, 2014 3 - Alpha N/A + :pypi:`pytest-httpserver` pytest-httpserver is a httpserver for pytest Oct 18, 2021 3 - Alpha pytest ; extra == 'dev' + :pypi:`pytest-httpx` Send responses to httpx. Nov 16, 2021 5 - Production/Stable pytest (==6.*) + :pypi:`pytest-httpx-blockage` Disable httpx requests during a test run Nov 16, 2021 N/A pytest (>=6.2.5) + :pypi:`pytest-hue` Visualise PyTest status via your Phillips Hue lights May 09, 2019 N/A N/A + :pypi:`pytest-hylang` Pytest plugin to allow running tests written in hylang Mar 28, 2021 N/A pytest + :pypi:`pytest-hypo-25` help hypo module for pytest Jan 12, 2020 3 - Alpha N/A + :pypi:`pytest-ibutsu` A plugin to sent pytest results to an Ibutsu server Jun 16, 2021 4 - Beta pytest + :pypi:`pytest-icdiff` use icdiff for better error messages in pytest assertions Apr 08, 2020 4 - Beta N/A + :pypi:`pytest-idapro` A pytest plugin for idapython. Allows a pytest setup to run tests outside and inside IDA in an automated manner by runnig pytest inside IDA and by mocking idapython api Nov 03, 2018 N/A N/A + :pypi:`pytest-idempotent` Pytest plugin for testing function idempotence. Nov 26, 2021 N/A N/A + :pypi:`pytest-ignore-flaky` ignore failures from flaky tests (pytest plugin) Apr 23, 2021 5 - Production/Stable N/A + :pypi:`pytest-image-diff` Jul 28, 2021 3 - Alpha pytest + :pypi:`pytest-incremental` an incremental test runner (pytest plugin) Apr 24, 2021 5 - Production/Stable N/A + :pypi:`pytest-influxdb` Plugin for influxdb and pytest integration. Apr 20, 2021 N/A N/A + :pypi:`pytest-info-collector` pytest plugin to collect information from tests May 26, 2019 3 - Alpha N/A + :pypi:`pytest-informative-node` display more node ininformation. Apr 25, 2019 4 - Beta N/A + :pypi:`pytest-infrastructure` pytest stack validation prior to testing executing Apr 12, 2020 4 - Beta N/A + :pypi:`pytest-ini` Reuse pytest.ini to store env variables Sep 30, 2021 N/A N/A + :pypi:`pytest-inmanta` A py.test plugin providing fixtures to simplify inmanta modules testing. Aug 17, 2021 5 - Production/Stable N/A + :pypi:`pytest-inmanta-extensions` Inmanta tests package May 27, 2021 5 - Production/Stable N/A + :pypi:`pytest-Inomaly` A simple image diff plugin for pytest Feb 13, 2018 4 - Beta N/A + :pypi:`pytest-insta` A practical snapshot testing plugin for pytest Apr 07, 2021 N/A pytest (>=6.0.2,<7.0.0) + :pypi:`pytest-instafail` pytest plugin to show failures instantly Jun 14, 2020 4 - Beta pytest (>=2.9) + :pypi:`pytest-instrument` pytest plugin to instrument tests Apr 05, 2020 5 - Production/Stable pytest (>=5.1.0) + :pypi:`pytest-integration` Organizing pytests by integration or not Apr 16, 2020 N/A N/A + :pypi:`pytest-integration-mark` Automatic integration test marking and excluding plugin for pytest Jul 19, 2021 N/A pytest (>=5.2,<7.0) + :pypi:`pytest-interactive` A pytest plugin for console based interactive test selection just after the collection phase Nov 30, 2017 3 - Alpha N/A + :pypi:`pytest-intercept-remote` Pytest plugin for intercepting outgoing connection requests during pytest run. May 24, 2021 4 - Beta pytest (>=4.6) + :pypi:`pytest-invenio` Pytest fixtures for Invenio. May 11, 2021 5 - Production/Stable pytest (<7,>=6) + :pypi:`pytest-involve` Run tests covering a specific file or changeset Feb 02, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-ipdb` A py.test plug-in to enable drop to ipdb debugger on test failure. Sep 02, 2014 2 - Pre-Alpha N/A + :pypi:`pytest-ipynb` THIS PROJECT IS ABANDONED Jan 29, 2019 3 - Alpha N/A + :pypi:`pytest-isort` py.test plugin to check import ordering using isort Apr 27, 2021 5 - Production/Stable N/A + :pypi:`pytest-it` Pytest plugin to display test reports as a plaintext spec, inspired by Rspec: https://github.com/mattduck/pytest-it. Jan 22, 2020 4 - Beta N/A + :pypi:`pytest-iterassert` Nicer list and iterable assertion messages for pytest May 11, 2020 3 - Alpha N/A + :pypi:`pytest-jasmine` Run jasmine tests from your pytest test suite Nov 04, 2017 1 - Planning N/A + :pypi:`pytest-jest` A custom jest-pytest oriented Pytest reporter May 22, 2018 4 - Beta pytest (>=3.3.2) + :pypi:`pytest-jira` py.test JIRA integration plugin, using markers Dec 02, 2021 3 - Alpha N/A + :pypi:`pytest-jira-xray` pytest plugin to integrate tests with JIRA XRAY Nov 28, 2021 3 - Alpha pytest + :pypi:`pytest-jobserver` Limit parallel tests with posix jobserver. May 15, 2019 5 - Production/Stable pytest + :pypi:`pytest-joke` Test failures are better served with humor. Oct 08, 2019 4 - Beta pytest (>=4.2.1) + :pypi:`pytest-json` Generate JSON test reports Jan 18, 2016 4 - Beta N/A + :pypi:`pytest-jsonlint` UNKNOWN Aug 04, 2016 N/A N/A + :pypi:`pytest-json-report` A pytest plugin to report test results as JSON files Sep 24, 2021 4 - Beta pytest (>=3.8.0) + :pypi:`pytest-kafka` Zookeeper, Kafka server, and Kafka consumer fixtures for Pytest Aug 24, 2021 N/A pytest + :pypi:`pytest-kafkavents` A plugin to send pytest events to Kafka Sep 08, 2021 4 - Beta pytest + :pypi:`pytest-kind` Kubernetes test support with KIND for pytest Jan 24, 2021 5 - Production/Stable N/A + :pypi:`pytest-kivy` Kivy GUI tests fixtures using pytest Jul 06, 2021 4 - Beta pytest (>=3.6) + :pypi:`pytest-knows` A pytest plugin that can automaticly skip test case based on dependence info calculated by trace Aug 22, 2014 N/A N/A + :pypi:`pytest-konira` Run Konira DSL tests with py.test Oct 09, 2011 N/A N/A + :pypi:`pytest-krtech-common` pytest krtech common library Nov 28, 2016 4 - Beta N/A + :pypi:`pytest-kwparametrize` Alternate syntax for @pytest.mark.parametrize with test cases as dictionaries and default value fallbacks Jan 22, 2021 N/A pytest (>=6) + :pypi:`pytest-lambda` Define pytest fixtures with lambda functions. Aug 23, 2021 3 - Alpha pytest (>=3.6,<7) + :pypi:`pytest-lamp` Jan 06, 2017 3 - Alpha N/A + :pypi:`pytest-layab` Pytest fixtures for layab. Oct 05, 2020 5 - Production/Stable N/A + :pypi:`pytest-lazy-fixture` It helps to use fixtures in pytest.mark.parametrize Feb 01, 2020 4 - Beta pytest (>=3.2.5) + :pypi:`pytest-ldap` python-ldap fixtures for pytest Aug 18, 2020 N/A pytest + :pypi:`pytest-leaks` A pytest plugin to trace resource leaks. Nov 27, 2019 1 - Planning N/A + :pypi:`pytest-level` Select tests of a given level or lower Oct 21, 2019 N/A pytest + :pypi:`pytest-libfaketime` A python-libfaketime plugin for pytest. Dec 22, 2018 4 - Beta pytest (>=3.0.0) + :pypi:`pytest-libiio` A pytest plugin to manage interfacing with libiio contexts Oct 29, 2021 4 - Beta N/A + :pypi:`pytest-libnotify` Pytest plugin that shows notifications about the test run Apr 02, 2021 3 - Alpha pytest + :pypi:`pytest-ligo` Jan 16, 2020 4 - Beta N/A + :pypi:`pytest-lineno` A pytest plugin to show the line numbers of test functions Dec 04, 2020 N/A pytest + :pypi:`pytest-line-profiler` Profile code executed by pytest May 03, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-lisa` Pytest plugin for organizing tests. Jan 21, 2021 3 - Alpha pytest (>=6.1.2,<7.0.0) + :pypi:`pytest-listener` A simple network listener May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-litf` A pytest plugin that stream output in LITF format Jan 18, 2021 4 - Beta pytest (>=3.1.1) + :pypi:`pytest-live` Live results for pytest Mar 08, 2020 N/A pytest + :pypi:`pytest-localftpserver` A PyTest plugin which provides an FTP fixture for your tests Aug 25, 2021 5 - Production/Stable pytest + :pypi:`pytest-localserver` py.test plugin to test server connections locally. Nov 19, 2021 4 - Beta N/A + :pypi:`pytest-localstack` Pytest plugin for AWS integration tests Aug 22, 2019 4 - Beta pytest (>=3.3.0) + :pypi:`pytest-lockable` lockable resource plugin for pytest Nov 09, 2021 5 - Production/Stable pytest + :pypi:`pytest-locker` Used to lock object during testing. Essentially changing assertions from being hard coded to asserting that nothing changed Oct 29, 2021 N/A pytest (>=5.4) + :pypi:`pytest-log` print log Aug 15, 2021 N/A pytest (>=3.8) + :pypi:`pytest-logbook` py.test plugin to capture logbook log messages Nov 23, 2015 5 - Production/Stable pytest (>=2.8) + :pypi:`pytest-logdog` Pytest plugin to test logging Jun 15, 2021 1 - Planning pytest (>=6.2.0) + :pypi:`pytest-logfest` Pytest plugin providing three logger fixtures with basic or full writing to log files Jul 21, 2019 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-logger` Plugin configuring handlers for loggers from Python logging module. Jul 25, 2019 4 - Beta pytest (>=3.2) + :pypi:`pytest-logging` Configures logging and allows tweaking the log level with a py.test flag Nov 04, 2015 4 - Beta N/A + :pypi:`pytest-log-report` Package for creating a pytest test run reprot Dec 26, 2019 N/A N/A + :pypi:`pytest-manual-marker` pytest marker for marking manual tests Oct 11, 2021 3 - Alpha pytest (>=6) + :pypi:`pytest-markdown` Test your markdown docs with pytest Jan 15, 2021 4 - Beta pytest (>=6.0.1,<7.0.0) + :pypi:`pytest-marker-bugzilla` py.test bugzilla integration plugin, using markers Jan 09, 2020 N/A N/A + :pypi:`pytest-markers-presence` A simple plugin to detect missed pytest tags and markers" Feb 04, 2021 4 - Beta pytest (>=6.0) + :pypi:`pytest-markfiltration` UNKNOWN Nov 08, 2011 3 - Alpha N/A + :pypi:`pytest-mark-no-py3` pytest plugin and bowler codemod to help migrate tests to Python 3 May 17, 2019 N/A pytest + :pypi:`pytest-marks` UNKNOWN Nov 23, 2012 3 - Alpha N/A + :pypi:`pytest-matcher` Match test output against patterns stored in files Apr 23, 2020 5 - Production/Stable pytest (>=3.4) + :pypi:`pytest-match-skip` Skip matching marks. Matches partial marks using wildcards. May 15, 2019 4 - Beta pytest (>=4.4.1) + :pypi:`pytest-mat-report` this is report Jan 20, 2021 N/A N/A + :pypi:`pytest-matrix` Provide tools for generating tests from combinations of fixtures. Jun 24, 2020 5 - Production/Stable pytest (>=5.4.3,<6.0.0) + :pypi:`pytest-mccabe` pytest plugin to run the mccabe code complexity checker. Jul 22, 2020 3 - Alpha pytest (>=5.4.0) + :pypi:`pytest-md` Plugin for generating Markdown reports for pytest results Jul 11, 2019 3 - Alpha pytest (>=4.2.1) + :pypi:`pytest-md-report` A pytest plugin to make a test results report with Markdown table format. May 04, 2021 4 - Beta pytest (!=6.0.0,<7,>=3.3.2) + :pypi:`pytest-memprof` Estimates memory consumption of test functions Mar 29, 2019 4 - Beta N/A + :pypi:`pytest-menu` A pytest plugin for console based interactive test selection just after the collection phase Oct 04, 2017 3 - Alpha pytest (>=2.4.2) + :pypi:`pytest-mercurial` pytest plugin to write integration tests for projects using Mercurial Python internals Nov 21, 2020 1 - Planning N/A + :pypi:`pytest-message` Pytest plugin for sending report message of marked tests execution Nov 04, 2021 N/A pytest (>=6.2.5) + :pypi:`pytest-messenger` Pytest to Slack reporting plugin Dec 16, 2020 5 - Production/Stable N/A + :pypi:`pytest-metadata` pytest plugin for test session metadata Nov 27, 2020 5 - Production/Stable pytest (>=2.9.0) + :pypi:`pytest-metrics` Custom metrics report for pytest Apr 04, 2020 N/A pytest + :pypi:`pytest-mimesis` Mimesis integration with the pytest test runner Mar 21, 2020 5 - Production/Stable pytest (>=4.2) + :pypi:`pytest-minecraft` A pytest plugin for running tests against Minecraft releases Sep 26, 2020 N/A pytest (>=6.0.1,<7.0.0) + :pypi:`pytest-missing-fixtures` Pytest plugin that creates missing fixtures Oct 14, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-ml` Test your machine learning! May 04, 2019 4 - Beta N/A + :pypi:`pytest-mocha` pytest plugin to display test execution output like a mochajs Apr 02, 2020 4 - Beta pytest (>=5.4.0) + :pypi:`pytest-mock` Thin-wrapper around the mock package for easier use with pytest May 06, 2021 5 - Production/Stable pytest (>=5.0) + :pypi:`pytest-mock-api` A mock API server with configurable routes and responses available as a fixture. Feb 13, 2019 1 - Planning pytest (>=4.0.0) + :pypi:`pytest-mock-generator` A pytest fixture wrapper for https://pypi.org/project/mock-generator Aug 10, 2021 5 - Production/Stable N/A + :pypi:`pytest-mock-helper` Help you mock HTTP call and generate mock code Jan 24, 2018 N/A pytest + :pypi:`pytest-mockito` Base fixtures for mockito Jul 11, 2018 4 - Beta N/A + :pypi:`pytest-mockredis` An in-memory mock of a Redis server that runs in a separate thread. This is to be used for unit-tests that require a Redis database. Jan 02, 2018 2 - Pre-Alpha N/A + :pypi:`pytest-mock-resources` A pytest plugin for easily instantiating reproducible mock resources. Dec 03, 2021 N/A pytest (>=1.0) + :pypi:`pytest-mock-server` Mock server plugin for pytest Apr 06, 2020 4 - Beta N/A + :pypi:`pytest-mockservers` A set of fixtures to test your requests to HTTP/UDP servers Mar 31, 2020 N/A pytest (>=4.3.0) + :pypi:`pytest-modifyjunit` Utility for adding additional properties to junit xml for IDM QE Jan 10, 2019 N/A N/A + :pypi:`pytest-modifyscope` pytest plugin to modify fixture scope Apr 12, 2020 N/A pytest + :pypi:`pytest-molecule` PyTest Molecule Plugin :: discover and run molecule tests Oct 06, 2021 5 - Production/Stable N/A + :pypi:`pytest-mongo` MongoDB process and client fixtures plugin for Pytest. Jun 07, 2021 5 - Production/Stable pytest + :pypi:`pytest-mongodb` pytest plugin for MongoDB fixtures Dec 07, 2019 5 - Production/Stable pytest (>=2.5.2) + :pypi:`pytest-monitor` Pytest plugin for analyzing resource usage. Aug 24, 2021 5 - Production/Stable pytest + :pypi:`pytest-monkeyplus` pytest's monkeypatch subclass with extra functionalities Sep 18, 2012 5 - Production/Stable N/A + :pypi:`pytest-monkeytype` pytest-monkeytype: Generate Monkeytype annotations from your pytest tests. Jul 29, 2020 4 - Beta N/A + :pypi:`pytest-moto` Fixtures for integration tests of AWS services,uses moto mocking library. Aug 28, 2015 1 - Planning N/A + :pypi:`pytest-motor` A pytest plugin for motor, the non-blocking MongoDB driver. Jul 21, 2021 3 - Alpha pytest + :pypi:`pytest-mp` A test batcher for multiprocessed Pytest runs May 23, 2018 4 - Beta pytest + :pypi:`pytest-mpi` pytest plugin to collect information from tests Mar 14, 2021 3 - Alpha pytest + :pypi:`pytest-mpl` pytest plugin to help with testing figures output from Matplotlib Jul 02, 2021 4 - Beta pytest + :pypi:`pytest-mproc` low-startup-overhead, scalable, distributed-testing pytest plugin Mar 07, 2021 4 - Beta pytest + :pypi:`pytest-multi-check` Pytest-плагин, реализует возможность мульти проверок и мягких проверок Jun 03, 2021 N/A pytest + :pypi:`pytest-multihost` Utility for writing multi-host tests for pytest Apr 07, 2020 4 - Beta N/A + :pypi:`pytest-multilog` Multi-process logs handling and other helpers for pytest Jun 10, 2021 N/A N/A + :pypi:`pytest-multithreading` a pytest plugin for th and concurrent testing Aug 12, 2021 N/A pytest (>=3.6) + :pypi:`pytest-mutagen` Add the mutation testing feature to pytest Jul 24, 2020 N/A pytest (>=5.4) + :pypi:`pytest-mypy` Mypy static type checker plugin for Pytest Mar 21, 2021 4 - Beta pytest (>=3.5) + :pypi:`pytest-mypyd` Mypy static type checker plugin for Pytest Aug 20, 2019 4 - Beta pytest (<4.7,>=2.8) ; python_version < "3.5" + :pypi:`pytest-mypy-plugins` pytest plugin for writing tests for mypy plugins Oct 19, 2021 3 - Alpha pytest (>=6.0.0) + :pypi:`pytest-mypy-plugins-shim` Substitute for "pytest-mypy-plugins" for Python implementations which aren't supported by mypy. Apr 12, 2021 N/A N/A + :pypi:`pytest-mypy-testing` Pytest plugin to check mypy output. Jun 13, 2021 N/A pytest + :pypi:`pytest-mysql` MySQL process and client fixtures for pytest Nov 22, 2021 5 - Production/Stable pytest + :pypi:`pytest-needle` pytest plugin for visual testing websites using selenium Dec 10, 2018 4 - Beta pytest (<5.0.0,>=3.0.0) + :pypi:`pytest-neo` pytest-neo is a plugin for pytest that shows tests like screen of Matrix. Apr 23, 2019 3 - Alpha pytest (>=3.7.2) + :pypi:`pytest-network` A simple plugin to disable network on socket level. May 07, 2020 N/A N/A + :pypi:`pytest-never-sleep` pytest plugin helps to avoid adding tests without mock \`time.sleep\` May 05, 2021 3 - Alpha pytest (>=3.5.1) + :pypi:`pytest-nginx` nginx fixture for pytest Aug 12, 2017 5 - Production/Stable N/A + :pypi:`pytest-nginx-iplweb` nginx fixture for pytest - iplweb temporary fork Mar 01, 2019 5 - Production/Stable N/A + :pypi:`pytest-ngrok` Jan 22, 2020 3 - Alpha N/A + :pypi:`pytest-ngsfixtures` pytest ngs fixtures Sep 06, 2019 2 - Pre-Alpha pytest (>=5.0.0) + :pypi:`pytest-nice` A pytest plugin that alerts user of failed test cases with screen notifications May 04, 2019 4 - Beta pytest + :pypi:`pytest-nice-parametrize` A small snippet for nicer PyTest's Parametrize Apr 17, 2021 5 - Production/Stable N/A + :pypi:`pytest-nlcov` Pytest plugin to get the coverage of the new lines (based on git diff) only Jul 07, 2021 N/A N/A + :pypi:`pytest-nocustom` Run all tests without custom markers Jul 07, 2021 5 - Production/Stable N/A + :pypi:`pytest-nodev` Test-driven source code search for Python. Jul 21, 2016 4 - Beta pytest (>=2.8.1) + :pypi:`pytest-nogarbage` Ensure a test produces no garbage Aug 29, 2021 5 - Production/Stable pytest (>=4.6.0) + :pypi:`pytest-notebook` A pytest plugin for testing Jupyter Notebooks Sep 16, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-notice` Send pytest execution result email Nov 05, 2020 N/A N/A + :pypi:`pytest-notification` A pytest plugin for sending a desktop notification and playing a sound upon completion of tests Jun 19, 2020 N/A pytest (>=4) + :pypi:`pytest-notifier` A pytest plugin to notify test result Jun 12, 2020 3 - Alpha pytest + :pypi:`pytest-notimplemented` Pytest markers for not implemented features and tests. Aug 27, 2019 N/A pytest (>=5.1,<6.0) + :pypi:`pytest-notion` A PyTest Reporter to send test runs to Notion.so Aug 07, 2019 N/A N/A + :pypi:`pytest-nunit` A pytest plugin for generating NUnit3 test result XML output Aug 04, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-ochrus` pytest results data-base and HTML reporter Feb 21, 2018 4 - Beta N/A + :pypi:`pytest-odoo` py.test plugin to run Odoo tests Nov 04, 2021 4 - Beta pytest (>=2.9) + :pypi:`pytest-odoo-fixtures` Project description Jun 25, 2019 N/A N/A + :pypi:`pytest-oerp` pytest plugin to test OpenERP modules Feb 28, 2012 3 - Alpha N/A + :pypi:`pytest-ok` The ultimate pytest output plugin Apr 01, 2019 4 - Beta N/A + :pypi:`pytest-only` Use @pytest.mark.only to run a single test Jan 19, 2020 N/A N/A + :pypi:`pytest-oot` Run object-oriented tests in a simple format Sep 18, 2016 4 - Beta N/A + :pypi:`pytest-openfiles` Pytest plugin for detecting inadvertent open file handles Apr 16, 2020 3 - Alpha pytest (>=4.6) + :pypi:`pytest-opentmi` pytest plugin for publish results to opentmi Nov 04, 2021 5 - Production/Stable pytest (>=5.0) + :pypi:`pytest-operator` Fixtures for Operators Oct 26, 2021 N/A N/A + :pypi:`pytest-optional` include/exclude values of fixtures in pytest Oct 07, 2015 N/A N/A + :pypi:`pytest-optional-tests` Easy declaration of optional tests (i.e., that are not run by default) Jul 09, 2019 4 - Beta pytest (>=4.5.0) + :pypi:`pytest-orchestration` A pytest plugin for orchestrating tests Jul 18, 2019 N/A N/A + :pypi:`pytest-order` pytest plugin to run your tests in a specific order May 30, 2021 4 - Beta pytest (>=5.0) + :pypi:`pytest-ordering` pytest plugin to run your tests in a specific order Nov 14, 2018 4 - Beta pytest + :pypi:`pytest-osxnotify` OS X notifications for py.test results. May 15, 2015 N/A N/A + :pypi:`pytest-otel` pytest-otel report OpenTelemetry traces about test executed Dec 03, 2021 N/A N/A + :pypi:`pytest-pact` A simple plugin to use with pytest Jan 07, 2019 4 - Beta N/A + :pypi:`pytest-pahrametahrize` Parametrize your tests with a Boston accent. Nov 24, 2021 4 - Beta pytest (>=6.0,<7.0) + :pypi:`pytest-parallel` a pytest plugin for parallel and concurrent testing Oct 10, 2021 3 - Alpha pytest (>=3.0.0) + :pypi:`pytest-parallel-39` a pytest plugin for parallel and concurrent testing Jul 12, 2021 3 - Alpha pytest (>=3.0.0) + :pypi:`pytest-param` pytest plugin to test all, first, last or random params Sep 11, 2016 4 - Beta pytest (>=2.6.0) + :pypi:`pytest-paramark` Configure pytest fixtures using a combination of"parametrize" and markers Jan 10, 2020 4 - Beta pytest (>=4.5.0) + :pypi:`pytest-parametrization` Simpler PyTest parametrization Nov 30, 2021 5 - Production/Stable pytest + :pypi:`pytest-parametrize-cases` A more user-friendly way to write parametrized tests. Dec 12, 2020 N/A pytest (>=6.1.2,<7.0.0) + :pypi:`pytest-parametrized` Pytest plugin for parametrizing tests with default iterables. Oct 19, 2020 5 - Production/Stable pytest + :pypi:`pytest-parawtf` Finally spell paramete?ri[sz]e correctly Dec 03, 2018 4 - Beta pytest (>=3.6.0) + :pypi:`pytest-pass` Check out https://github.com/elilutsky/pytest-pass Dec 04, 2019 N/A N/A + :pypi:`pytest-passrunner` Pytest plugin providing the 'run_on_pass' marker Feb 10, 2021 5 - Production/Stable pytest (>=4.6.0) + :pypi:`pytest-paste-config` Allow setting the path to a paste config file Sep 18, 2013 3 - Alpha N/A + :pypi:`pytest-patches` A contextmanager pytest fixture for handling multiple mock patches Aug 30, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-pdb` pytest plugin which adds pdb helper commands related to pytest. Jul 31, 2018 N/A N/A + :pypi:`pytest-peach` pytest plugin for fuzzing with Peach API Security Apr 12, 2019 4 - Beta pytest (>=2.8.7) + :pypi:`pytest-pep257` py.test plugin for pep257 Jul 09, 2016 N/A N/A + :pypi:`pytest-pep8` pytest plugin to check PEP8 requirements Apr 27, 2014 N/A N/A + :pypi:`pytest-percent` Change the exit code of pytest test sessions when a required percent of tests pass. May 21, 2020 N/A pytest (>=5.2.0) + :pypi:`pytest-perf` pytest-perf Jun 27, 2021 5 - Production/Stable pytest (>=4.6) ; extra == 'testing' + :pypi:`pytest-performance` A simple plugin to ensure the execution of critical sections of code has not been impacted Sep 11, 2020 5 - Production/Stable pytest (>=3.7.0) + :pypi:`pytest-persistence` Pytest tool for persistent objects Nov 06, 2021 N/A N/A + :pypi:`pytest-pgsql` Pytest plugins and helpers for tests using a Postgres database. May 13, 2020 5 - Production/Stable pytest (>=3.0.0) + :pypi:`pytest-phmdoctest` pytest plugin to test Python examples in Markdown using phmdoctest. Nov 10, 2021 4 - Beta pytest (>=6.2) ; extra == 'test' + :pypi:`pytest-picked` Run the tests related to the changed files Dec 23, 2020 N/A pytest (>=3.5.0) + :pypi:`pytest-pigeonhole` Jun 25, 2018 5 - Production/Stable pytest (>=3.4) + :pypi:`pytest-pikachu` Show surprise when tests are passing Aug 05, 2021 5 - Production/Stable pytest + :pypi:`pytest-pilot` Slice in your test base thanks to powerful markers. Oct 09, 2020 5 - Production/Stable N/A + :pypi:`pytest-pings` 🦊 The pytest plugin for Firefox Telemetry 📊 Jun 29, 2019 3 - Alpha pytest (>=5.0.0) + :pypi:`pytest-pinned` A simple pytest plugin for pinning tests Sep 17, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-pinpoint` A pytest plugin which runs SBFL algorithms to detect faults. Sep 25, 2020 N/A pytest (>=4.4.0) + :pypi:`pytest-pipeline` Pytest plugin for functional testing of data analysispipelines Jan 24, 2017 3 - Alpha N/A + :pypi:`pytest-platform-markers` Markers for pytest to skip tests on specific platforms Sep 09, 2019 4 - Beta pytest (>=3.6.0) + :pypi:`pytest-play` pytest plugin that let you automate actions and assertions with test metrics reporting executing plain YAML files Jun 12, 2019 5 - Production/Stable N/A + :pypi:`pytest-playbook` Pytest plugin for reading playbooks. Jan 21, 2021 3 - Alpha pytest (>=6.1.2,<7.0.0) + :pypi:`pytest-playwright` A pytest wrapper with fixtures for Playwright to automate web browsers Oct 28, 2021 N/A pytest + :pypi:`pytest-playwrights` A pytest wrapper with fixtures for Playwright to automate web browsers Dec 02, 2021 N/A N/A + :pypi:`pytest-playwright-snapshot` A pytest wrapper for snapshot testing with playwright Aug 19, 2021 N/A N/A + :pypi:`pytest-plt` Fixtures for quickly making Matplotlib plots in tests Aug 17, 2020 5 - Production/Stable pytest + :pypi:`pytest-plugin-helpers` A plugin to help developing and testing other plugins Nov 23, 2019 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-plus` PyTest Plus Plugin :: extends pytest functionality Mar 19, 2020 5 - Production/Stable pytest (>=3.50) + :pypi:`pytest-pmisc` Mar 21, 2019 5 - Production/Stable N/A + :pypi:`pytest-pointers` Pytest plugin to define functions you test with special marks for better navigation and reports Oct 14, 2021 N/A N/A + :pypi:`pytest-polarion-cfme` pytest plugin for collecting test cases and recording test results Nov 13, 2017 3 - Alpha N/A + :pypi:`pytest-polarion-collect` pytest plugin for collecting polarion test cases data Jun 18, 2020 3 - Alpha pytest + :pypi:`pytest-polecat` Provides Polecat pytest fixtures Aug 12, 2019 4 - Beta N/A + :pypi:`pytest-ponyorm` PonyORM in Pytest Oct 31, 2018 N/A pytest (>=3.1.1) + :pypi:`pytest-poo` Visualize your crappy tests Mar 25, 2021 5 - Production/Stable pytest (>=2.3.4) + :pypi:`pytest-poo-fail` Visualize your failed tests with poo Feb 12, 2015 5 - Production/Stable N/A + :pypi:`pytest-pop` A pytest plugin to help with testing pop projects Aug 19, 2021 5 - Production/Stable pytest + :pypi:`pytest-portion` Select a portion of the collected tests Jan 28, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-postgres` Run PostgreSQL in Docker container in Pytest. Mar 22, 2020 N/A pytest + :pypi:`pytest-postgresql` Postgresql fixtures and fixture factories for Pytest. Nov 05, 2021 5 - Production/Stable pytest (>=3.0.0) + :pypi:`pytest-power` pytest plugin with powerful fixtures Dec 31, 2020 N/A pytest (>=5.4) + :pypi:`pytest-pretty-terminal` pytest plugin for generating prettier terminal output Nov 24, 2021 N/A pytest (>=3.4.1) + :pypi:`pytest-pride` Minitest-style test colors Apr 02, 2016 3 - Alpha N/A + :pypi:`pytest-print` pytest-print adds the printer fixture you can use to print messages to the user (directly to the pytest runner, not stdout) Jun 17, 2021 5 - Production/Stable pytest (>=6) + :pypi:`pytest-profiling` Profiling plugin for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-progress` pytest plugin for instant test progress status Nov 09, 2021 5 - Production/Stable pytest (>=2.7) + :pypi:`pytest-prometheus` Report test pass / failures to a Prometheus PushGateway Oct 03, 2017 N/A N/A + :pypi:`pytest-prosper` Test helpers for Prosper projects Sep 24, 2018 N/A N/A + :pypi:`pytest-pspec` A rspec format reporter for Python ptest Jun 02, 2020 4 - Beta pytest (>=3.0.0) + :pypi:`pytest-psqlgraph` pytest plugin for testing applications that use psqlgraph Oct 19, 2021 4 - Beta pytest (>=6.0) + :pypi:`pytest-ptera` Use ptera probes in tests Oct 20, 2021 N/A pytest (>=6.2.4,<7.0.0) + :pypi:`pytest-pudb` Pytest PuDB debugger integration Oct 25, 2018 3 - Alpha pytest (>=2.0) + :pypi:`pytest-purkinje` py.test plugin for purkinje test runner Oct 28, 2017 2 - Pre-Alpha N/A + :pypi:`pytest-pycharm` Plugin for py.test to enter PyCharm debugger on uncaught exceptions Aug 13, 2020 5 - Production/Stable pytest (>=2.3) + :pypi:`pytest-pycodestyle` pytest plugin to run pycodestyle Aug 10, 2020 3 - Alpha N/A + :pypi:`pytest-pydev` py.test plugin to connect to a remote debug server with PyDev or PyCharm. Nov 15, 2017 3 - Alpha N/A + :pypi:`pytest-pydocstyle` pytest plugin to run pydocstyle Aug 10, 2020 3 - Alpha N/A + :pypi:`pytest-pylint` pytest plugin to check source code with pylint Nov 09, 2020 5 - Production/Stable pytest (>=5.4) + :pypi:`pytest-pypi` Easily test your HTTP library against a local copy of pypi Mar 04, 2018 3 - Alpha N/A + :pypi:`pytest-pypom-navigation` Core engine for cookiecutter-qa and pytest-play packages Feb 18, 2019 4 - Beta pytest (>=3.0.7) + :pypi:`pytest-pyppeteer` A plugin to run pyppeteer in pytest. Feb 16, 2021 4 - Beta pytest (>=6.0.2) + :pypi:`pytest-pyq` Pytest fixture "q" for pyq Mar 10, 2020 5 - Production/Stable N/A + :pypi:`pytest-pyramid` pytest_pyramid - provides fixtures for testing pyramid applications with pytest test suite Oct 15, 2021 5 - Production/Stable pytest + :pypi:`pytest-pyramid-server` Pyramid server fixture for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-pyright` Pytest plugin for type checking code with Pyright Aug 16, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-pytestrail` Pytest plugin for interaction with TestRail Aug 27, 2020 4 - Beta pytest (>=3.8.0) + :pypi:`pytest-pythonpath` pytest plugin for adding to the PYTHONPATH from command line or configs. Aug 22, 2018 5 - Production/Stable N/A + :pypi:`pytest-pytorch` pytest plugin for a better developer experience when working with the PyTorch test suite May 25, 2021 4 - Beta pytest + :pypi:`pytest-qasync` Pytest support for qasync. Jul 12, 2021 4 - Beta pytest (>=5.4.0) + :pypi:`pytest-qatouch` Pytest plugin for uploading test results to your QA Touch Testrun. Jun 26, 2021 4 - Beta pytest (>=6.2.0) + :pypi:`pytest-qgis` A pytest plugin for testing QGIS python plugins Nov 25, 2021 5 - Production/Stable pytest (>=6.2.3) + :pypi:`pytest-qml` Run QML Tests with pytest Dec 02, 2020 4 - Beta pytest (>=6.0.0) + :pypi:`pytest-qr` pytest plugin to generate test result QR codes Nov 25, 2021 4 - Beta N/A + :pypi:`pytest-qt` pytest support for PyQt and PySide applications Jun 13, 2021 5 - Production/Stable pytest (>=3.0.0) + :pypi:`pytest-qt-app` QT app fixture for py.test Dec 23, 2015 5 - Production/Stable N/A + :pypi:`pytest-quarantine` A plugin for pytest to manage expected test failures Nov 24, 2019 5 - Production/Stable pytest (>=4.6) + :pypi:`pytest-quickcheck` pytest plugin to generate random data inspired by QuickCheck Nov 15, 2020 4 - Beta pytest (<6.0.0,>=4.0) + :pypi:`pytest-rabbitmq` RabbitMQ process and client fixtures for pytest Jun 02, 2021 5 - Production/Stable pytest (>=3.0.0) + :pypi:`pytest-race` Race conditions tester for pytest Nov 21, 2016 4 - Beta N/A + :pypi:`pytest-rage` pytest plugin to implement PEP712 Oct 21, 2011 3 - Alpha N/A + :pypi:`pytest-railflow-testrail-reporter` Generate json reports along with specified metadata defined in test markers. Dec 02, 2021 5 - Production/Stable pytest + :pypi:`pytest-raises` An implementation of pytest.raises as a pytest.mark fixture Apr 23, 2020 N/A pytest (>=3.2.2) + :pypi:`pytest-raisesregexp` Simple pytest plugin to look for regex in Exceptions Dec 18, 2015 N/A N/A + :pypi:`pytest-raisin` Plugin enabling the use of exception instances with pytest.raises Jun 25, 2020 N/A pytest + :pypi:`pytest-random` py.test plugin to randomize tests Apr 28, 2013 3 - Alpha N/A + :pypi:`pytest-randomly` Pytest plugin to randomly order tests and control random.seed. Nov 30, 2021 5 - Production/Stable pytest + :pypi:`pytest-randomness` Pytest plugin about random seed management May 30, 2019 3 - Alpha N/A + :pypi:`pytest-random-num` Randomise the order in which pytest tests are run with some control over the randomness Oct 19, 2020 5 - Production/Stable N/A + :pypi:`pytest-random-order` Randomise the order in which pytest tests are run with some control over the randomness Nov 30, 2018 5 - Production/Stable pytest (>=3.0.0) + :pypi:`pytest-readme` Test your README.md file Dec 28, 2014 5 - Production/Stable N/A + :pypi:`pytest-reana` Pytest fixtures for REANA. Nov 22, 2021 3 - Alpha N/A + :pypi:`pytest-recording` A pytest plugin that allows you recording of network interactions via VCR.py Jul 08, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-recordings` Provides pytest plugins for reporting request/response traffic, screenshots, and more to ReportPortal Aug 13, 2020 N/A N/A + :pypi:`pytest-redis` Redis fixtures and fixture factories for Pytest. Nov 03, 2021 5 - Production/Stable pytest + :pypi:`pytest-redislite` Pytest plugin for testing code using Redis Sep 19, 2021 4 - Beta pytest + :pypi:`pytest-redmine` Pytest plugin for redmine Mar 19, 2018 1 - Planning N/A + :pypi:`pytest-ref` A plugin to store reference files to ease regression testing Nov 23, 2019 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-reference-formatter` Conveniently run pytest with a dot-formatted test reference. Oct 01, 2019 4 - Beta N/A + :pypi:`pytest-regressions` Easy to use fixtures to write regression tests. Jan 27, 2021 5 - Production/Stable pytest (>=3.5.0) + :pypi:`pytest-regtest` pytest plugin for regression tests Jun 03, 2021 N/A N/A + :pypi:`pytest-relative-order` a pytest plugin that sorts tests using "before" and "after" markers May 17, 2021 4 - Beta N/A + :pypi:`pytest-relaxed` Relaxed test discovery/organization for pytest Jun 14, 2019 5 - Production/Stable pytest (<5,>=3) + :pypi:`pytest-remfiles` Pytest plugin to create a temporary directory with remote files Jul 01, 2019 5 - Production/Stable N/A + :pypi:`pytest-remotedata` Pytest plugin for controlling remote data access. Jul 20, 2019 3 - Alpha pytest (>=3.1) + :pypi:`pytest-remote-response` Pytest plugin for capturing and mocking connection requests. Jun 30, 2021 4 - Beta pytest (>=4.6) + :pypi:`pytest-remove-stale-bytecode` py.test plugin to remove stale byte code files. Mar 04, 2020 4 - Beta pytest + :pypi:`pytest-reorder` Reorder tests depending on their paths and names. May 31, 2018 4 - Beta pytest + :pypi:`pytest-repeat` pytest plugin for repeating tests Oct 31, 2020 5 - Production/Stable pytest (>=3.6) + :pypi:`pytest-replay` Saves previous test runs and allow re-execute previous pytest runs to reproduce crashes or flaky tests Jun 09, 2021 4 - Beta pytest (>=3.0.0) + :pypi:`pytest-repo-health` A pytest plugin to report on repository standards conformance Nov 23, 2021 3 - Alpha pytest + :pypi:`pytest-report` Creates json report that is compatible with atom.io's linter message format May 11, 2016 4 - Beta N/A + :pypi:`pytest-reporter` Generate Pytest reports with templates Jul 22, 2021 4 - Beta pytest + :pypi:`pytest-reporter-html1` A basic HTML report template for Pytest Jun 08, 2021 4 - Beta N/A + :pypi:`pytest-reportinfra` Pytest plugin for reportinfra Aug 11, 2019 3 - Alpha N/A + :pypi:`pytest-reporting` A plugin to report summarized results in a table format Oct 25, 2019 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-reportlog` Replacement for the --resultlog option, focused in simplicity and extensibility Dec 11, 2020 3 - Alpha pytest (>=5.2) + :pypi:`pytest-report-me` A pytest plugin to generate report. Dec 31, 2020 N/A pytest + :pypi:`pytest-report-parameters` pytest plugin for adding tests' parameters to junit report Jun 18, 2020 3 - Alpha pytest (>=2.4.2) + :pypi:`pytest-reportportal` Agent for Reporting results of tests to the Report Portal Jun 18, 2021 N/A pytest (>=3.8.0) + :pypi:`pytest-reqs` pytest plugin to check pinned requirements May 12, 2019 N/A pytest (>=2.4.2) + :pypi:`pytest-requests` A simple plugin to use with pytest Jun 24, 2019 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-reraise` Make multi-threaded pytest test cases fail when they should Jun 17, 2021 5 - Production/Stable pytest (>=4.6) + :pypi:`pytest-rerun` Re-run only changed files in specified branch Jul 08, 2019 N/A pytest (>=3.6) + :pypi:`pytest-rerunfailures` pytest plugin to re-run tests to eliminate flaky failures Sep 17, 2021 5 - Production/Stable pytest (>=5.3) + :pypi:`pytest-resilient-circuits` Resilient Circuits fixtures for PyTest. Nov 15, 2021 N/A N/A + :pypi:`pytest-resource` Load resource fixture plugin to use with pytest Nov 14, 2018 4 - Beta N/A + :pypi:`pytest-resource-path` Provides path for uniform access to test resources in isolated directory May 01, 2021 5 - Production/Stable pytest (>=3.5.0) + :pypi:`pytest-responsemock` Simplified requests calls mocking for pytest Oct 10, 2020 5 - Production/Stable N/A + :pypi:`pytest-responses` py.test integration for responses Apr 26, 2021 N/A pytest (>=2.5) + :pypi:`pytest-restrict` Pytest plugin to restrict the test types allowed Aug 12, 2021 5 - Production/Stable pytest + :pypi:`pytest-rethinkdb` A RethinkDB plugin for pytest. Jul 24, 2016 4 - Beta N/A + :pypi:`pytest-reverse` Pytest plugin to reverse test order. Aug 12, 2021 5 - Production/Stable pytest + :pypi:`pytest-ringo` pytest plugin to test webapplications using the Ringo webframework Sep 27, 2017 3 - Alpha N/A + :pypi:`pytest-rng` Fixtures for seeding tests and making randomness reproducible Aug 08, 2019 5 - Production/Stable pytest + :pypi:`pytest-roast` pytest plugin for ROAST configuration override and fixtures Jul 29, 2021 5 - Production/Stable pytest + :pypi:`pytest-rocketchat` Pytest to Rocket.Chat reporting plugin Apr 18, 2021 5 - Production/Stable N/A + :pypi:`pytest-rotest` Pytest integration with rotest Sep 08, 2019 N/A pytest (>=3.5.0) + :pypi:`pytest-rpc` Extend py.test for RPC OpenStack testing. Feb 22, 2019 4 - Beta pytest (~=3.6) + :pypi:`pytest-rst` Test code from RST documents with pytest Sep 21, 2021 N/A pytest + :pypi:`pytest-rt` pytest data collector plugin for Testgr Sep 04, 2021 N/A N/A + :pypi:`pytest-rts` Coverage-based regression test selection (RTS) plugin for pytest May 17, 2021 N/A pytest + :pypi:`pytest-run-changed` Pytest plugin that runs changed tests only Apr 02, 2021 3 - Alpha pytest + :pypi:`pytest-runfailed` implement a --failed option for pytest Mar 24, 2016 N/A N/A + :pypi:`pytest-runner` Invoke py.test as distutils command with dependency resolution May 19, 2021 5 - Production/Stable pytest (>=4.6) ; extra == 'testing' + :pypi:`pytest-runtime-xfail` Call runtime_xfail() to mark running test as xfail. Aug 26, 2021 N/A N/A + :pypi:`pytest-salt` Pytest Salt Plugin Jan 27, 2020 4 - Beta N/A + :pypi:`pytest-salt-containers` A Pytest plugin that builds and creates docker containers Nov 09, 2016 4 - Beta N/A + :pypi:`pytest-salt-factories` Pytest Salt Plugin Sep 16, 2021 4 - Beta pytest (>=6.0.0) + :pypi:`pytest-salt-from-filenames` Simple PyTest Plugin For Salt's Test Suite Specifically Jan 29, 2019 4 - Beta pytest (>=4.1) + :pypi:`pytest-salt-runtests-bridge` Simple PyTest Plugin For Salt's Test Suite Specifically Dec 05, 2019 4 - Beta pytest (>=4.1) + :pypi:`pytest-sanic` a pytest plugin for Sanic Oct 25, 2021 N/A pytest (>=5.2) + :pypi:`pytest-sanity` Dec 07, 2020 N/A N/A + :pypi:`pytest-sa-pg` May 14, 2019 N/A N/A + :pypi:`pytest-sbase` A complete web automation framework for end-to-end testing. Dec 03, 2021 5 - Production/Stable N/A + :pypi:`pytest-scenario` pytest plugin for test scenarios Feb 06, 2017 3 - Alpha N/A + :pypi:`pytest-schema` 👍 Validate return values against a schema-like object in testing Aug 31, 2020 5 - Production/Stable pytest (>=3.5.0) + :pypi:`pytest-securestore` An encrypted password store for use within pytest cases Nov 08, 2021 4 - Beta N/A + :pypi:`pytest-select` A pytest plugin which allows to (de-)select tests from a file. Jan 18, 2019 3 - Alpha pytest (>=3.0) + :pypi:`pytest-selenium` pytest plugin for Selenium Sep 19, 2020 5 - Production/Stable pytest (>=5.0.0) + :pypi:`pytest-seleniumbase` A complete web automation framework for end-to-end testing. Dec 03, 2021 5 - Production/Stable N/A + :pypi:`pytest-selenium-enhancer` pytest plugin for Selenium Nov 26, 2020 5 - Production/Stable N/A + :pypi:`pytest-selenium-pdiff` A pytest package implementing perceptualdiff for Selenium tests. Apr 06, 2017 2 - Pre-Alpha N/A + :pypi:`pytest-send-email` Send pytest execution result email Dec 04, 2019 N/A N/A + :pypi:`pytest-sentry` A pytest plugin to send testrun information to Sentry.io Apr 21, 2021 N/A pytest + :pypi:`pytest-server-fixtures` Extensible server fixures for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-serverless` Automatically mocks resources from serverless.yml in pytest using moto. Nov 27, 2021 4 - Beta N/A + :pypi:`pytest-services` Services plugin for pytest testing framework Oct 30, 2020 6 - Mature N/A + :pypi:`pytest-session2file` pytest-session2file (aka: pytest-session_to_file for v0.1.0 - v0.1.2) is a py.test plugin for capturing and saving to file the stdout of py.test. Jan 26, 2021 3 - Alpha pytest + :pypi:`pytest-session-fixture-globalize` py.test plugin to make session fixtures behave as if written in conftest, even if it is written in some modules May 15, 2018 4 - Beta N/A + :pypi:`pytest-session_to_file` pytest-session_to_file is a py.test plugin for capturing and saving to file the stdout of py.test. Oct 01, 2015 3 - Alpha N/A + :pypi:`pytest-sftpserver` py.test plugin to locally test sftp server connections. Sep 16, 2019 4 - Beta N/A + :pypi:`pytest-shard` Dec 11, 2020 4 - Beta pytest + :pypi:`pytest-shell` A pytest plugin to help with testing shell scripts / black box commands Nov 07, 2021 N/A N/A + :pypi:`pytest-sheraf` Versatile ZODB abstraction layer - pytest fixtures Feb 11, 2020 N/A pytest + :pypi:`pytest-sherlock` pytest plugin help to find coupled tests Nov 18, 2021 5 - Production/Stable pytest (>=3.5.1) + :pypi:`pytest-shortcuts` Expand command-line shortcuts listed in pytest configuration Oct 29, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-shutil` A goodie-bag of unix shell and environment tools for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-simplehttpserver` Simple pytest fixture to spin up an HTTP server Jun 24, 2021 4 - Beta N/A + :pypi:`pytest-simple-plugin` Simple pytest plugin Nov 27, 2019 N/A N/A + :pypi:`pytest-simple-settings` simple-settings plugin for pytest Nov 17, 2020 4 - Beta pytest + :pypi:`pytest-single-file-logging` Allow for multiple processes to log to a single file May 05, 2016 4 - Beta pytest (>=2.8.1) + :pypi:`pytest-skip-markers` Pytest Salt Plugin Oct 04, 2021 4 - Beta pytest (>=6.0.0) + :pypi:`pytest-skipper` A plugin that selects only tests with changes in execution path Mar 26, 2017 3 - Alpha pytest (>=3.0.6) + :pypi:`pytest-skippy` Automatically skip tests that don't need to run! Jan 27, 2018 3 - Alpha pytest (>=2.3.4) + :pypi:`pytest-skip-slow` A pytest plugin to skip \`@pytest.mark.slow\` tests by default. Sep 28, 2021 N/A N/A + :pypi:`pytest-slack` Pytest to Slack reporting plugin Dec 15, 2020 5 - Production/Stable N/A + :pypi:`pytest-slow` A pytest plugin to skip \`@pytest.mark.slow\` tests by default. Sep 28, 2021 N/A N/A + :pypi:`pytest-smartcollect` A plugin for collecting tests that touch changed code Oct 04, 2018 N/A pytest (>=3.5.0) + :pypi:`pytest-smartcov` Smart coverage plugin for pytest. Sep 30, 2017 3 - Alpha N/A + :pypi:`pytest-smtp` Send email with pytest execution result Feb 20, 2021 N/A pytest + :pypi:`pytest-snail` Plugin for adding a marker to slow running tests. 🐌 Nov 04, 2019 3 - Alpha pytest (>=5.0.1) + :pypi:`pytest-snapci` py.test plugin for Snap-CI Nov 12, 2015 N/A N/A + :pypi:`pytest-snapshot` A plugin for snapshot testing with pytest. Dec 02, 2021 4 - Beta pytest (>=3.0.0) + :pypi:`pytest-snmpserver` May 12, 2021 N/A N/A + :pypi:`pytest-socket` Pytest Plugin to disable socket calls during tests Aug 28, 2021 4 - Beta pytest (>=3.6.3) + :pypi:`pytest-soft-assertions` May 05, 2020 3 - Alpha pytest + :pypi:`pytest-solr` Solr process and client fixtures for py.test. May 11, 2020 3 - Alpha pytest (>=3.0.0) + :pypi:`pytest-sorter` A simple plugin to first execute tests that historically failed more Apr 20, 2021 4 - Beta pytest (>=3.1.1) + :pypi:`pytest-sourceorder` Test-ordering plugin for pytest Sep 01, 2021 4 - Beta pytest + :pypi:`pytest-spark` pytest plugin to run the tests with support of pyspark. Feb 23, 2020 4 - Beta pytest + :pypi:`pytest-spawner` py.test plugin to spawn process and communicate with them. Jul 31, 2015 4 - Beta N/A + :pypi:`pytest-spec` Library pytest-spec is a pytest plugin to display test execution output like a SPECIFICATION. May 04, 2021 N/A N/A + :pypi:`pytest-sphinx` Doctest plugin for pytest with support for Sphinx-specific doctest-directives Aug 05, 2020 4 - Beta N/A + :pypi:`pytest-spiratest` Exports unit tests as test runs in SpiraTest/Team/Plan Oct 13, 2021 N/A N/A + :pypi:`pytest-splinter` Splinter plugin for pytest testing framework Dec 25, 2020 6 - Mature N/A + :pypi:`pytest-split` Pytest plugin which splits the test suite to equally sized sub suites based on test execution time. Nov 09, 2021 4 - Beta pytest (>=5,<7) + :pypi:`pytest-splitio` Split.io SDK integration for e2e tests Sep 22, 2020 N/A pytest (<7,>=5.0) + :pypi:`pytest-split-tests` A Pytest plugin for running a subset of your tests by splitting them in to equally sized groups. Forked from Mark Adams' original project pytest-test-groups. Jul 30, 2021 5 - Production/Stable pytest (>=2.5) + :pypi:`pytest-split-tests-tresorit` Feb 22, 2021 1 - Planning N/A + :pypi:`pytest-splunk-addon` A Dynamic test tool for Splunk Apps and Add-ons Nov 29, 2021 N/A pytest (>5.4.0,<6.3) + :pypi:`pytest-splunk-addon-ui-smartx` Library to support testing Splunk Add-on UX Oct 07, 2021 N/A N/A + :pypi:`pytest-splunk-env` pytest fixtures for interaction with Splunk Enterprise and Splunk Cloud Oct 22, 2020 N/A pytest (>=6.1.1,<7.0.0) + :pypi:`pytest-sqitch` sqitch for pytest Apr 06, 2020 4 - Beta N/A + :pypi:`pytest-sqlalchemy` pytest plugin with sqlalchemy related fixtures Mar 13, 2018 3 - Alpha N/A + :pypi:`pytest-sql-bigquery` Yet another SQL-testing framework for BigQuery provided by pytest plugin Dec 19, 2019 N/A pytest + :pypi:`pytest-srcpaths` Add paths to sys.path Oct 15, 2021 N/A N/A + :pypi:`pytest-ssh` pytest plugin for ssh command run May 27, 2019 N/A pytest + :pypi:`pytest-start-from` Start pytest run from a given point Apr 11, 2016 N/A N/A + :pypi:`pytest-statsd` pytest plugin for reporting to graphite Nov 30, 2018 5 - Production/Stable pytest (>=3.0.0) + :pypi:`pytest-stepfunctions` A small description May 08, 2021 4 - Beta pytest + :pypi:`pytest-steps` Create step-wise / incremental tests in pytest. Sep 23, 2021 5 - Production/Stable N/A + :pypi:`pytest-stepwise` Run a test suite one failing test at a time. Dec 01, 2015 4 - Beta N/A + :pypi:`pytest-stoq` A plugin to pytest stoq Feb 09, 2021 4 - Beta N/A + :pypi:`pytest-stress` A Pytest plugin that allows you to loop tests for a user defined amount of time. Dec 07, 2019 4 - Beta pytest (>=3.6.0) + :pypi:`pytest-structlog` Structured logging assertions Sep 21, 2021 N/A pytest + :pypi:`pytest-structmpd` provide structured temporary directory Oct 17, 2018 N/A N/A + :pypi:`pytest-stub` Stub packages, modules and attributes. Apr 28, 2020 5 - Production/Stable N/A + :pypi:`pytest-stubprocess` Provide stub implementations for subprocesses in Python tests Sep 17, 2018 3 - Alpha pytest (>=3.5.0) + :pypi:`pytest-study` A pytest plugin to organize long run tests (named studies) without interfering the regular tests Sep 26, 2017 3 - Alpha pytest (>=2.0) + :pypi:`pytest-subprocess` A plugin to fake subprocess for pytest Nov 07, 2021 5 - Production/Stable pytest (>=4.0.0) + :pypi:`pytest-subtesthack` A hack to explicitly set up and tear down fixtures. Mar 02, 2021 N/A N/A + :pypi:`pytest-subtests` unittest subTest() support and subtests fixture May 29, 2021 4 - Beta pytest (>=5.3.0) + :pypi:`pytest-subunit` pytest-subunit is a plugin for py.test which outputs testsresult in subunit format. Aug 29, 2017 N/A N/A + :pypi:`pytest-sugar` pytest-sugar is a plugin for pytest that changes the default look and feel of pytest (e.g. progressbar, show tests that fail instantly). Jul 06, 2020 3 - Alpha N/A + :pypi:`pytest-sugar-bugfix159` Workaround for https://github.com/Frozenball/pytest-sugar/issues/159 Nov 07, 2018 5 - Production/Stable pytest (!=3.7.3,>=3.5); extra == 'testing' + :pypi:`pytest-super-check` Pytest plugin to check your TestCase classes call super in setUp, tearDown, etc. Aug 12, 2021 5 - Production/Stable pytest + :pypi:`pytest-svn` SVN repository fixture for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-symbols` pytest-symbols is a pytest plugin that adds support for passing test environment symbols into pytest tests. Nov 20, 2017 3 - Alpha N/A + :pypi:`pytest-takeltest` Fixtures for ansible, testinfra and molecule Oct 13, 2021 N/A N/A + :pypi:`pytest-talisker` Nov 28, 2021 N/A N/A + :pypi:`pytest-tap` Test Anything Protocol (TAP) reporting plugin for pytest Oct 27, 2021 5 - Production/Stable pytest (>=3.0) + :pypi:`pytest-tape` easy assertion with expected results saved to yaml files Mar 17, 2021 4 - Beta N/A + :pypi:`pytest-target` Pytest plugin for remote target orchestration. Jan 21, 2021 3 - Alpha pytest (>=6.1.2,<7.0.0) + :pypi:`pytest-tblineinfo` tblineinfo is a py.test plugin that insert the node id in the final py.test report when --tb=line option is used Dec 01, 2015 3 - Alpha pytest (>=2.0) + :pypi:`pytest-teamcity-logblock` py.test plugin to introduce block structure in teamcity build log, if output is not captured May 15, 2018 4 - Beta N/A + :pypi:`pytest-telegram` Pytest to Telegram reporting plugin Dec 10, 2020 5 - Production/Stable N/A + :pypi:`pytest-tempdir` Predictable and repeatable tempdir support. Oct 11, 2019 4 - Beta pytest (>=2.8.1) + :pypi:`pytest-terraform` A pytest plugin for using terraform fixtures Nov 10, 2021 N/A pytest (>=6.0) + :pypi:`pytest-terraform-fixture` generate terraform resources to use with pytest Nov 14, 2018 4 - Beta N/A + :pypi:`pytest-testbook` A plugin to run tests written in Jupyter notebook Dec 11, 2016 3 - Alpha N/A + :pypi:`pytest-testconfig` Test configuration plugin for pytest. Jan 11, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-testdirectory` A py.test plugin providing temporary directories in unit tests. Nov 06, 2018 5 - Production/Stable pytest + :pypi:`pytest-testdox` A testdox format reporter for pytest Oct 13, 2020 5 - Production/Stable pytest (>=3.7.0) + :pypi:`pytest-test-groups` A Pytest plugin for running a subset of your tests by splitting them in to equally sized groups. Oct 25, 2016 5 - Production/Stable N/A + :pypi:`pytest-testinfra` Test infrastructures Jun 20, 2021 5 - Production/Stable pytest (!=3.0.2) + :pypi:`pytest-testlink-adaptor` pytest reporting plugin for testlink Dec 20, 2018 4 - Beta pytest (>=2.6) + :pypi:`pytest-testmon` selects tests affected by changed files and methods Oct 22, 2021 4 - Beta N/A + :pypi:`pytest-testobject` Plugin to use TestObject Suites with Pytest Sep 24, 2019 4 - Beta pytest (>=3.1.1) + :pypi:`pytest-testrail` pytest plugin for creating TestRail runs and adding results Aug 27, 2020 N/A pytest (>=3.6) + :pypi:`pytest-testrail2` A small example package Nov 17, 2020 N/A pytest (>=5) + :pypi:`pytest-testrail-api` Плагин Pytest, для интеграции с TestRail Nov 30, 2021 N/A pytest (>=5.5) + :pypi:`pytest-testrail-api-client` TestRail Api Python Client Dec 03, 2021 N/A pytest + :pypi:`pytest-testrail-appetize` pytest plugin for creating TestRail runs and adding results Sep 29, 2021 N/A N/A + :pypi:`pytest-testrail-client` pytest plugin for Testrail Sep 29, 2020 5 - Production/Stable N/A + :pypi:`pytest-testrail-e2e` pytest plugin for creating TestRail runs and adding results Oct 11, 2021 N/A pytest (>=3.6) + :pypi:`pytest-testrail-ns` pytest plugin for creating TestRail runs and adding results Oct 08, 2021 N/A pytest (>=3.6) + :pypi:`pytest-testrail-plugin` PyTest plugin for TestRail Apr 21, 2020 3 - Alpha pytest + :pypi:`pytest-testrail-reporter` Sep 10, 2018 N/A N/A + :pypi:`pytest-testreport` Nov 12, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-testslide` TestSlide fixture for pytest Jan 07, 2021 5 - Production/Stable pytest (~=6.2) + :pypi:`pytest-test-this` Plugin for py.test to run relevant tests, based on naively checking if a test contains a reference to the symbol you supply Sep 15, 2019 2 - Pre-Alpha pytest (>=2.3) + :pypi:`pytest-test-utils` Nov 30, 2021 N/A pytest (>=5) + :pypi:`pytest-tesults` Tesults plugin for pytest Jul 31, 2021 5 - Production/Stable pytest (>=3.5.0) + :pypi:`pytest-tezos` pytest-ligo Jan 16, 2020 4 - Beta N/A + :pypi:`pytest-thawgun` Pytest plugin for time travel May 26, 2020 3 - Alpha N/A + :pypi:`pytest-threadleak` Detects thread leaks Sep 08, 2017 4 - Beta N/A + :pypi:`pytest-tick` Ticking on tests Aug 31, 2021 5 - Production/Stable pytest (>=6.2.5,<7.0.0) + :pypi:`pytest-timeit` A pytest plugin to time test function runs Oct 13, 2016 4 - Beta N/A + :pypi:`pytest-timeout` pytest plugin to abort hanging tests Oct 11, 2021 5 - Production/Stable pytest (>=5.0.0) + :pypi:`pytest-timeouts` Linux-only Pytest plugin to control durations of various test case execution phases Sep 21, 2019 5 - Production/Stable N/A + :pypi:`pytest-timer` A timer plugin for pytest Jun 02, 2021 N/A N/A + :pypi:`pytest-timestamper` Pytest plugin to add a timestamp prefix to the pytest output Jun 06, 2021 N/A N/A + :pypi:`pytest-tipsi-django` Nov 17, 2021 4 - Beta pytest (>=6.0.0) + :pypi:`pytest-tipsi-testing` Better fixtures management. Various helpers Nov 04, 2020 4 - Beta pytest (>=3.3.0) + :pypi:`pytest-tldr` A pytest plugin that limits the output to just the things you need. Mar 12, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-tm4j-reporter` Cloud Jira Test Management (TM4J) PyTest reporter plugin Sep 01, 2020 N/A pytest + :pypi:`pytest-tmreport` this is a vue-element ui report for pytest Nov 17, 2021 N/A N/A + :pypi:`pytest-todo` A small plugin for the pytest testing framework, marking TODO comments as failure May 23, 2019 4 - Beta pytest + :pypi:`pytest-tomato` Mar 01, 2019 5 - Production/Stable N/A + :pypi:`pytest-toolbelt` This is just a collection of utilities for pytest, but don't really belong in pytest proper. Aug 12, 2019 3 - Alpha N/A + :pypi:`pytest-toolbox` Numerous useful plugins for pytest. Apr 07, 2018 N/A pytest (>=3.5.0) + :pypi:`pytest-tornado` A py.test plugin providing fixtures and markers to simplify testing of asynchronous tornado applications. Jun 17, 2020 5 - Production/Stable pytest (>=3.6) + :pypi:`pytest-tornado5` A py.test plugin providing fixtures and markers to simplify testing of asynchronous tornado applications. Nov 16, 2018 5 - Production/Stable pytest (>=3.6) + :pypi:`pytest-tornado-yen3` A py.test plugin providing fixtures and markers to simplify testing of asynchronous tornado applications. Oct 15, 2018 5 - Production/Stable N/A + :pypi:`pytest-tornasync` py.test plugin for testing Python 3.5+ Tornado code Jul 15, 2019 3 - Alpha pytest (>=3.0) + :pypi:`pytest-track` Feb 26, 2021 3 - Alpha pytest (>=3.0) + :pypi:`pytest-translations` Test your translation files. Nov 05, 2021 5 - Production/Stable N/A + :pypi:`pytest-travis-fold` Folds captured output sections in Travis CI build log Nov 29, 2017 4 - Beta pytest (>=2.6.0) + :pypi:`pytest-trello` Plugin for py.test that integrates trello using markers Nov 20, 2015 5 - Production/Stable N/A + :pypi:`pytest-trepan` Pytest plugin for trepan debugger. Jul 28, 2018 5 - Production/Stable N/A + :pypi:`pytest-trialtemp` py.test plugin for using the same _trial_temp working directory as trial Jun 08, 2015 N/A N/A + :pypi:`pytest-trio` Pytest plugin for trio Oct 16, 2020 N/A N/A + :pypi:`pytest-tspwplib` A simple plugin to use with tspwplib Jan 08, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-tstcls` Test Class Base Mar 23, 2020 5 - Production/Stable N/A + :pypi:`pytest-twisted` A twisted plugin for pytest. Aug 30, 2021 5 - Production/Stable pytest (>=2.3) + :pypi:`pytest-typhoon-xray` Typhoon HIL plugin for pytest Nov 03, 2021 4 - Beta N/A + :pypi:`pytest-tytest` Typhoon HIL plugin for pytest May 25, 2020 4 - Beta pytest (>=5.4.2) + :pypi:`pytest-ubersmith` Easily mock calls to ubersmith at the \`requests\` level. Apr 13, 2015 N/A N/A + :pypi:`pytest-ui` Text User Interface for running python tests Jul 05, 2021 4 - Beta pytest + :pypi:`pytest-unhandled-exception-exit-code` Plugin for py.test set a different exit code on uncaught exceptions Jun 22, 2020 5 - Production/Stable pytest (>=2.3) + :pypi:`pytest-unittest-filter` A pytest plugin for filtering unittest-based test classes Jan 12, 2019 4 - Beta pytest (>=3.1.0) + :pypi:`pytest-unmarked` Run only unmarked tests Aug 27, 2019 5 - Production/Stable N/A + :pypi:`pytest-unordered` Test equality of unordered collections in pytest Mar 28, 2021 4 - Beta N/A + :pypi:`pytest-upload-report` pytest-upload-report is a plugin for pytest that upload your test report for test results. Jun 18, 2021 5 - Production/Stable N/A + :pypi:`pytest-utils` Some helpers for pytest. Dec 04, 2021 4 - Beta pytest (>=6.2.5,<7.0.0) + :pypi:`pytest-vagrant` A py.test plugin providing access to vagrant. Sep 07, 2021 5 - Production/Stable pytest + :pypi:`pytest-valgrind` May 19, 2021 N/A N/A + :pypi:`pytest-variables` pytest plugin for providing variables to tests/fixtures Oct 23, 2019 5 - Production/Stable pytest (>=2.4.2) + :pypi:`pytest-variant` Variant support for Pytest Jun 20, 2021 N/A N/A + :pypi:`pytest-vcr` Plugin for managing VCR.py cassettes Apr 26, 2019 5 - Production/Stable pytest (>=3.6.0) + :pypi:`pytest-vcr-delete-on-fail` A pytest plugin that automates vcrpy cassettes deletion on test failure. Aug 13, 2021 4 - Beta pytest (>=6.2.2,<7.0.0) + :pypi:`pytest-vcrpandas` Test from HTTP interactions to dataframe processed. Jan 12, 2019 4 - Beta pytest + :pypi:`pytest-venv` py.test fixture for creating a virtual environment Aug 04, 2020 4 - Beta pytest + :pypi:`pytest-ver` Pytest module with Verification Report Aug 30, 2021 2 - Pre-Alpha N/A + :pypi:`pytest-verbose-parametrize` More descriptive output for parametrized py.test tests May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-vimqf` A simple pytest plugin that will shrink pytest output when specified, to fit vim quickfix window. Feb 08, 2021 4 - Beta pytest (>=6.2.2,<7.0.0) + :pypi:`pytest-virtualenv` Virtualenv fixture for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-voluptuous` Pytest plugin for asserting data against voluptuous schema. Jun 09, 2020 N/A pytest + :pypi:`pytest-vscodedebug` A pytest plugin to easily enable debugging tests within Visual Studio Code Dec 04, 2020 4 - Beta N/A + :pypi:`pytest-vts` pytest plugin for automatic recording of http stubbed tests Jun 05, 2019 N/A pytest (>=2.3) + :pypi:`pytest-vw` pytest-vw makes your failing test cases succeed under CI tools scrutiny Oct 07, 2015 4 - Beta N/A + :pypi:`pytest-vyper` Plugin for the vyper smart contract language. May 28, 2020 2 - Pre-Alpha N/A + :pypi:`pytest-wa-e2e-plugin` Pytest plugin for testing whatsapp bots with end to end tests Feb 18, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-watch` Local continuous test runner with pytest and watchdog. May 20, 2018 N/A N/A + :pypi:`pytest-watcher` Continiously runs pytest on changes in \*.py files Sep 18, 2021 3 - Alpha N/A + :pypi:`pytest-wdl` Pytest plugin for testing WDL workflows. Nov 17, 2020 5 - Production/Stable N/A + :pypi:`pytest-webdriver` Selenium webdriver fixture for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-wetest` Welian API Automation test framework pytest plugin Nov 10, 2018 4 - Beta N/A + :pypi:`pytest-whirlwind` Testing Tornado. Jun 12, 2020 N/A N/A + :pypi:`pytest-wholenodeid` pytest addon for displaying the whole node id for failures Aug 26, 2015 4 - Beta pytest (>=2.0) + :pypi:`pytest-win32consoletitle` Pytest progress in console title (Win32 only) Aug 08, 2021 N/A N/A + :pypi:`pytest-winnotify` Windows tray notifications for py.test results. Apr 22, 2016 N/A N/A + :pypi:`pytest-with-docker` pytest with docker helpers. Nov 09, 2021 N/A pytest + :pypi:`pytest-workflow` A pytest plugin for configuring workflow/pipeline tests using YAML files Dec 03, 2021 5 - Production/Stable pytest (>=5.4.0) + :pypi:`pytest-xdist` pytest xdist plugin for distributed testing and loop-on-failing modes Sep 21, 2021 5 - Production/Stable pytest (>=6.0.0) + :pypi:`pytest-xdist-debug-for-graingert` pytest xdist plugin for distributed testing and loop-on-failing modes Jul 24, 2019 5 - Production/Stable pytest (>=4.4.0) + :pypi:`pytest-xdist-forked` forked from pytest-xdist Feb 10, 2020 5 - Production/Stable pytest (>=4.4.0) + :pypi:`pytest-xdist-tracker` pytest plugin helps to reproduce failures for particular xdist node Nov 18, 2021 3 - Alpha pytest (>=3.5.1) + :pypi:`pytest-xfaillist` Maintain a xfaillist in an additional file to avoid merge-conflicts. Sep 17, 2021 N/A pytest (>=6.2.2,<7.0.0) + :pypi:`pytest-xfiles` Pytest fixtures providing data read from function, module or package related (x)files. Feb 27, 2018 N/A N/A + :pypi:`pytest-xlog` Extended logging for test and decorators May 31, 2020 4 - Beta N/A + :pypi:`pytest-xpara` An extended parametrizing plugin of pytest. Oct 30, 2017 3 - Alpha pytest + :pypi:`pytest-xprocess` A pytest plugin for managing processes across test runs. Jul 28, 2021 4 - Beta pytest (>=2.8) + :pypi:`pytest-xray` May 30, 2019 3 - Alpha N/A + :pypi:`pytest-xrayjira` Mar 17, 2020 3 - Alpha pytest (==4.3.1) + :pypi:`pytest-xray-server` Oct 27, 2021 3 - Alpha pytest (>=5.3.1) + :pypi:`pytest-xvfb` A pytest plugin to run Xvfb for tests. Jun 09, 2020 4 - Beta pytest (>=2.8.1) + :pypi:`pytest-yaml` This plugin is used to load yaml output to your test using pytest framework. Oct 05, 2018 N/A pytest + :pypi:`pytest-yamltree` Create or check file/directory trees described by YAML Mar 02, 2020 4 - Beta pytest (>=3.1.1) + :pypi:`pytest-yamlwsgi` Run tests against wsgi apps defined in yaml May 11, 2010 N/A N/A + :pypi:`pytest-yapf` Run yapf Jul 06, 2017 4 - Beta pytest (>=3.1.1) + :pypi:`pytest-yapf3` Validate your Python file format with yapf Aug 03, 2020 5 - Production/Stable pytest (>=5.4) + :pypi:`pytest-yield` PyTest plugin to run tests concurrently, each \`yield\` switch context to other one Jan 23, 2019 N/A N/A + :pypi:`pytest-yuk` Display tests you are uneasy with, using 🤢/🤮 for pass/fail of tests marked with yuk. Mar 26, 2021 N/A N/A + :pypi:`pytest-zafira` A Zafira plugin for pytest Sep 18, 2019 5 - Production/Stable pytest (==4.1.1) + :pypi:`pytest-zap` OWASP ZAP plugin for py.test. May 12, 2014 4 - Beta N/A + :pypi:`pytest-zebrunner` Pytest connector for Zebrunner reporting Dec 02, 2021 5 - Production/Stable pytest (>=4.5.0) + :pypi:`pytest-zigzag` Extend py.test for RPC OpenStack testing. Feb 27, 2019 4 - Beta pytest (~=3.6) + =============================================== ======================================================================================================================================================================== ============== ===================== ================================================ + +.. only:: latex + + + :pypi:`pytest-accept` + *last release*: Nov 22, 2021, + *status*: N/A, + *requires*: pytest (>=6,<7) + + A pytest-plugin for updating doctest outputs + + :pypi:`pytest-adaptavist` + *last release*: Nov 30, 2021, + *status*: N/A, + *requires*: pytest (>=5.4.0) + + pytest plugin for generating test execution results within Jira Test Management (tm4j) + + :pypi:`pytest-addons-test` + *last release*: Aug 02, 2021, + *status*: N/A, + *requires*: pytest (>=6.2.4,<7.0.0) + + 用于测试pytest的插件 + + :pypi:`pytest-adf` + *last release*: May 10, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Pytest plugin for writing Azure Data Factory integration tests + + :pypi:`pytest-adf-azure-identity` + *last release*: Mar 06, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Pytest plugin for writing Azure Data Factory integration tests + + :pypi:`pytest-agent` + *last release*: Nov 25, 2021, + *status*: N/A, + *requires*: N/A + + Service that exposes a REST API that can be used to interract remotely with Pytest. It is shipped with a dashboard that enables running tests in a more convenient way. + + :pypi:`pytest-aggreport` + *last release*: Mar 07, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.2.2) + + pytest plugin for pytest-repeat that generate aggregate report of the same test cases with additional statistics details. + + :pypi:`pytest-aio` + *last release*: Oct 20, 2021, + *status*: 4 - Beta, + *requires*: pytest + + Pytest plugin for testing async python code + + :pypi:`pytest-aiofiles` + *last release*: May 14, 2017, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest fixtures for writing aiofiles tests with pyfakefs + + :pypi:`pytest-aiohttp` + *last release*: Dec 05, 2017, + *status*: N/A, + *requires*: pytest + + pytest plugin for aiohttp support + + :pypi:`pytest-aiohttp-client` + *last release*: Nov 01, 2020, + *status*: N/A, + *requires*: pytest (>=6) + + Pytest \`client\` fixture for the Aiohttp + + :pypi:`pytest-aioresponses` + *last release*: Jul 29, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + py.test integration for aioresponses + + :pypi:`pytest-aioworkers` + *last release*: Dec 04, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A plugin to test aioworkers project with pytest + + :pypi:`pytest-airflow` + *last release*: Apr 03, 2019, + *status*: 3 - Alpha, + *requires*: pytest (>=4.4.0) + + pytest support for airflow. + + :pypi:`pytest-airflow-utils` + *last release*: Nov 15, 2021, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-alembic` + *last release*: Dec 02, 2021, + *status*: N/A, + *requires*: pytest (>=1.0) + + A pytest plugin for verifying alembic migrations. + + :pypi:`pytest-allclose` + *last release*: Jul 30, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + Pytest fixture extending Numpy's allclose function + + :pypi:`pytest-allure-adaptor` + *last release*: Jan 10, 2018, + *status*: N/A, + *requires*: pytest (>=2.7.3) + + Plugin for py.test to generate allure xml reports + + :pypi:`pytest-allure-adaptor2` + *last release*: Oct 14, 2020, + *status*: N/A, + *requires*: pytest (>=2.7.3) + + Plugin for py.test to generate allure xml reports + + :pypi:`pytest-allure-dsl` + *last release*: Oct 25, 2020, + *status*: 4 - Beta, + *requires*: pytest + + pytest plugin to test case doc string dls instructions + + :pypi:`pytest-allure-spec-coverage` + *last release*: Oct 26, 2021, + *status*: N/A, + *requires*: pytest + + The pytest plugin aimed to display test coverage of the specs(requirements) in Allure + + :pypi:`pytest-alphamoon` + *last release*: Oct 21, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Static code checks used at Alphamoon + + :pypi:`pytest-android` + *last release*: Feb 21, 2019, + *status*: 3 - Alpha, + *requires*: pytest + + This fixture provides a configured "driver" for Android Automated Testing, using uiautomator2. + + :pypi:`pytest-anki` + *last release*: Oct 14, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A pytest plugin for testing Anki add-ons + + :pypi:`pytest-annotate` + *last release*: Nov 29, 2021, + *status*: 3 - Alpha, + *requires*: pytest (<7.0.0,>=3.2.0) + + pytest-annotate: Generate PyAnnotate annotations from your pytest tests. + + :pypi:`pytest-ansible` + *last release*: May 25, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + Plugin for py.test to simplify calling ansible modules from tests or fixtures + + :pypi:`pytest-ansible-playbook` + *last release*: Mar 08, 2019, + *status*: 4 - Beta, + *requires*: N/A + + Pytest fixture which runs given ansible playbook file. + + :pypi:`pytest-ansible-playbook-runner` + *last release*: Dec 02, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.1.0) + + Pytest fixture which runs given ansible playbook file. + + :pypi:`pytest-antilru` + *last release*: Apr 11, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + Bust functools.lru_cache when running pytest to avoid test pollution + + :pypi:`pytest-anyio` + *last release*: Jun 29, 2021, + *status*: N/A, + *requires*: pytest + + The pytest anyio plugin is built into anyio. You don't need this package. + + :pypi:`pytest-anything` + *last release*: Feb 18, 2021, + *status*: N/A, + *requires*: N/A + + Pytest fixtures to assert anything and something + + :pypi:`pytest-aoc` + *last release*: Nov 23, 2021, + *status*: N/A, + *requires*: pytest ; extra == 'test' + + Downloads puzzle inputs for Advent of Code and synthesizes PyTest fixtures + + :pypi:`pytest-api` + *last release*: May 04, 2021, + *status*: N/A, + *requires*: N/A + + PyTest-API Python Web Framework built for testing purposes. + + :pypi:`pytest-apistellar` + *last release*: Jun 18, 2019, + *status*: N/A, + *requires*: N/A + + apistellar plugin for pytest. + + :pypi:`pytest-appengine` + *last release*: Feb 27, 2017, + *status*: N/A, + *requires*: N/A + + AppEngine integration that works well with pytest-django + + :pypi:`pytest-appium` + *last release*: Dec 05, 2019, + *status*: N/A, + *requires*: N/A + + Pytest plugin for appium + + :pypi:`pytest-approvaltests` + *last release*: Feb 07, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A plugin to use approvaltests with pytest + + :pypi:`pytest-argus` + *last release*: Jun 24, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=6.2.4) + + pyest results colection plugin + + :pypi:`pytest-arraydiff` + *last release*: Dec 06, 2018, + *status*: 4 - Beta, + *requires*: pytest + + pytest plugin to help with comparing array output from tests + + :pypi:`pytest-asgi-server` + *last release*: Dec 12, 2020, + *status*: N/A, + *requires*: pytest (>=5.4.1) + + Convenient ASGI client/server fixtures for Pytest + + :pypi:`pytest-asptest` + *last release*: Apr 28, 2018, + *status*: 4 - Beta, + *requires*: N/A + + test Answer Set Programming programs + + :pypi:`pytest-assertutil` + *last release*: May 10, 2019, + *status*: N/A, + *requires*: N/A + + pytest-assertutil + + :pypi:`pytest-assert-utils` + *last release*: Sep 21, 2021, + *status*: 3 - Alpha, + *requires*: N/A + + Useful assertion utilities for use with pytest + + :pypi:`pytest-assume` + *last release*: Jun 24, 2021, + *status*: N/A, + *requires*: pytest (>=2.7) + + A pytest plugin that allows multiple failures per test + + :pypi:`pytest-ast-back-to-python` + *last release*: Sep 29, 2019, + *status*: 4 - Beta, + *requires*: N/A + + A plugin for pytest devs to view how assertion rewriting recodes the AST + + :pypi:`pytest-astropy` + *last release*: Sep 21, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.6) + + Meta-package containing dependencies for testing + + :pypi:`pytest-astropy-header` + *last release*: Dec 18, 2019, + *status*: 3 - Alpha, + *requires*: pytest (>=2.8) + + pytest plugin to add diagnostic information to the header of the test output + + :pypi:`pytest-ast-transformer` + *last release*: May 04, 2019, + *status*: 3 - Alpha, + *requires*: pytest + + + + :pypi:`pytest-asyncio` + *last release*: Oct 15, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=5.4.0) + + Pytest support for asyncio. + + :pypi:`pytest-asyncio-cooperative` + *last release*: Oct 12, 2021, + *status*: 4 - Beta, + *requires*: N/A + + Run all your asynchronous tests cooperatively. + + :pypi:`pytest-asyncio-network-simulator` + *last release*: Jul 31, 2018, + *status*: 3 - Alpha, + *requires*: pytest (<3.7.0,>=3.3.2) + + pytest-asyncio-network-simulator: Plugin for pytest for simulator the network in tests + + :pypi:`pytest-async-mongodb` + *last release*: Oct 18, 2017, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.5.2) + + pytest plugin for async MongoDB + + :pypi:`pytest-async-sqlalchemy` + *last release*: Oct 07, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.0.0) + + Database testing fixtures using the SQLAlchemy asyncio API + + :pypi:`pytest-atomic` + *last release*: Nov 24, 2018, + *status*: 4 - Beta, + *requires*: N/A + + Skip rest of tests if previous test failed. + + :pypi:`pytest-attrib` + *last release*: May 24, 2016, + *status*: 4 - Beta, + *requires*: N/A + + pytest plugin to select tests based on attributes similar to the nose-attrib plugin + + :pypi:`pytest-austin` + *last release*: Oct 11, 2020, + *status*: 4 - Beta, + *requires*: N/A + + Austin plugin for pytest + + :pypi:`pytest-autochecklog` + *last release*: Apr 25, 2015, + *status*: 4 - Beta, + *requires*: N/A + + automatically check condition and log all the checks + + :pypi:`pytest-automation` + *last release*: Oct 01, 2021, + *status*: N/A, + *requires*: pytest + + pytest plugin for building a test suite, using YAML files to extend pytest parameterize functionality. + + :pypi:`pytest-automock` + *last release*: Apr 22, 2020, + *status*: N/A, + *requires*: pytest ; extra == 'dev' + + Pytest plugin for automatical mocks creation + + :pypi:`pytest-auto-parametrize` + *last release*: Oct 02, 2016, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin: avoid repeating arguments in parametrize + + :pypi:`pytest-autotest` + *last release*: Aug 25, 2021, + *status*: N/A, + *requires*: pytest + + This fixture provides a configured "driver" for Android Automated Testing, using uiautomator2. + + :pypi:`pytest-avoidance` + *last release*: May 23, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Makes pytest skip tests that don not need rerunning + + :pypi:`pytest-aws` + *last release*: Oct 04, 2017, + *status*: 4 - Beta, + *requires*: N/A + + pytest plugin for testing AWS resource configurations + + :pypi:`pytest-aws-config` + *last release*: May 28, 2021, + *status*: N/A, + *requires*: N/A + + Protect your AWS credentials in unit tests + + :pypi:`pytest-axe` + *last release*: Nov 12, 2018, + *status*: N/A, + *requires*: pytest (>=3.0.0) + + pytest plugin for axe-selenium-python + + :pypi:`pytest-azurepipelines` + *last release*: Jul 23, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Formatting PyTest output for Azure Pipelines UI + + :pypi:`pytest-bandit` + *last release*: Feb 23, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A bandit plugin for pytest + + :pypi:`pytest-base-url` + *last release*: Jun 19, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.7.3) + + pytest plugin for URL based testing + + :pypi:`pytest-bdd` + *last release*: Oct 25, 2021, + *status*: 6 - Mature, + *requires*: pytest (>=4.3) + + BDD for pytest + + :pypi:`pytest-bdd-splinter` + *last release*: Aug 12, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.0.0) + + Common steps for pytest bdd and splinter integration + + :pypi:`pytest-bdd-web` + *last release*: Jan 02, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A simple plugin to use with pytest + + :pypi:`pytest-bdd-wrappers` + *last release*: Feb 11, 2020, + *status*: 2 - Pre-Alpha, + *requires*: N/A + + + + :pypi:`pytest-beakerlib` + *last release*: Mar 17, 2017, + *status*: 5 - Production/Stable, + *requires*: pytest + + A pytest plugin that reports test results to the BeakerLib framework + + :pypi:`pytest-beds` + *last release*: Jun 07, 2016, + *status*: 4 - Beta, + *requires*: N/A + + Fixtures for testing Google Appengine (GAE) apps + + :pypi:`pytest-bench` + *last release*: Jul 21, 2014, + *status*: 3 - Alpha, + *requires*: N/A + + Benchmark utility that plugs into pytest. + + :pypi:`pytest-benchmark` + *last release*: Apr 17, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.8) + + A \`\`pytest\`\` fixture for benchmarking code. It will group the tests into rounds that are calibrated to the chosen timer. + + :pypi:`pytest-bg-process` + *last release*: Aug 17, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Pytest plugin to initialize background process + + :pypi:`pytest-bigchaindb` + *last release*: Aug 17, 2021, + *status*: 4 - Beta, + *requires*: N/A + + A BigchainDB plugin for pytest. + + :pypi:`pytest-bigquery-mock` + *last release*: Aug 05, 2021, + *status*: N/A, + *requires*: pytest (>=5.0) + + Provides a mock fixture for python bigquery client + + :pypi:`pytest-black` + *last release*: Oct 05, 2020, + *status*: 4 - Beta, + *requires*: N/A + + A pytest plugin to enable format checking with black + + :pypi:`pytest-black-multipy` + *last release*: Jan 14, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (!=3.7.3,>=3.5) ; extra == 'testing' + + Allow '--black' on older Pythons + + :pypi:`pytest-blame` + *last release*: May 04, 2019, + *status*: N/A, + *requires*: pytest (>=4.4.0) + + A pytest plugin helps developers to debug by providing useful commits history. + + :pypi:`pytest-blender` + *last release*: Oct 29, 2021, + *status*: N/A, + *requires*: pytest (==6.2.5) ; extra == 'dev' + + Blender Pytest plugin. + + :pypi:`pytest-blink1` + *last release*: Jan 07, 2018, + *status*: 4 - Beta, + *requires*: N/A + + Pytest plugin to emit notifications via the Blink(1) RGB LED + + :pypi:`pytest-blockage` + *last release*: Feb 13, 2019, + *status*: N/A, + *requires*: pytest + + Disable network requests during a test run. + + :pypi:`pytest-blocker` + *last release*: Sep 07, 2015, + *status*: 4 - Beta, + *requires*: N/A + + pytest plugin to mark a test as blocker and skip all other tests + + :pypi:`pytest-board` + *last release*: Jan 20, 2019, + *status*: N/A, + *requires*: N/A + + Local continuous test runner with pytest and watchdog. + + :pypi:`pytest-bpdb` + *last release*: Jan 19, 2015, + *status*: 2 - Pre-Alpha, + *requires*: N/A + + A py.test plug-in to enable drop to bpdb debugger on test failure. + + :pypi:`pytest-bravado` + *last release*: Jul 19, 2021, + *status*: N/A, + *requires*: N/A + + Pytest-bravado automatically generates from OpenAPI specification client fixtures. + + :pypi:`pytest-breakword` + *last release*: Aug 04, 2021, + *status*: N/A, + *requires*: pytest (>=6.2.4,<7.0.0) + + Use breakword with pytest + + :pypi:`pytest-breed-adapter` + *last release*: Nov 07, 2018, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A simple plugin to connect with breed-server + + :pypi:`pytest-briefcase` + *last release*: Jun 14, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A pytest plugin for running tests on a Briefcase project. + + :pypi:`pytest-browser` + *last release*: Dec 10, 2016, + *status*: 3 - Alpha, + *requires*: N/A + + A pytest plugin for console based browser test selection just after the collection phase + + :pypi:`pytest-browsermob-proxy` + *last release*: Jun 11, 2013, + *status*: 4 - Beta, + *requires*: N/A + + BrowserMob proxy plugin for py.test. + + :pypi:`pytest-browserstack-local` + *last release*: Feb 09, 2018, + *status*: N/A, + *requires*: N/A + + \`\`py.test\`\` plugin to run \`\`BrowserStackLocal\`\` in background. + + :pypi:`pytest-bug` + *last release*: Jun 02, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.6.0) + + Pytest plugin for marking tests as a bug + + :pypi:`pytest-bugtong-tag` + *last release*: Apr 23, 2021, + *status*: N/A, + *requires*: N/A + + pytest-bugtong-tag is a plugin for pytest + + :pypi:`pytest-bugzilla` + *last release*: May 05, 2010, + *status*: 4 - Beta, + *requires*: N/A + + py.test bugzilla integration plugin + + :pypi:`pytest-bugzilla-notifier` + *last release*: Jun 15, 2018, + *status*: 4 - Beta, + *requires*: pytest (>=2.9.2) + + A plugin that allows you to execute create, update, and read information from BugZilla bugs + + :pypi:`pytest-buildkite` + *last release*: Jul 13, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Plugin for pytest that automatically publishes coverage and pytest report annotations to Buildkite. + + :pypi:`pytest-builtin-types` + *last release*: Nov 17, 2021, + *status*: N/A, + *requires*: pytest + + + + :pypi:`pytest-bwrap` + *last release*: Oct 26, 2018, + *status*: 3 - Alpha, + *requires*: N/A + + Run your tests in Bubblewrap sandboxes + + :pypi:`pytest-cache` + *last release*: Jun 04, 2013, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin with mechanisms for caching across test runs + + :pypi:`pytest-cache-assert` + *last release*: Nov 03, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=5) + + Cache assertion data to simplify regression testing of complex serializable data + + :pypi:`pytest-cagoule` + *last release*: Jan 01, 2020, + *status*: 3 - Alpha, + *requires*: N/A + + Pytest plugin to only run tests affected by changes + + :pypi:`pytest-camel-collect` + *last release*: Aug 02, 2020, + *status*: N/A, + *requires*: pytest (>=2.9) + + Enable CamelCase-aware pytest class collection + + :pypi:`pytest-canonical-data` + *last release*: May 08, 2020, + *status*: 2 - Pre-Alpha, + *requires*: pytest (>=3.5.0) + + A plugin which allows to compare results with canonical results, based on previous runs + + :pypi:`pytest-caprng` + *last release*: May 02, 2018, + *status*: 4 - Beta, + *requires*: N/A + + A plugin that replays pRNG state on failure. + + :pypi:`pytest-capture-deprecatedwarnings` + *last release*: Apr 30, 2019, + *status*: N/A, + *requires*: N/A + + pytest plugin to capture all deprecatedwarnings and put them in one file + + :pypi:`pytest-capturelogs` + *last release*: Sep 11, 2021, + *status*: 3 - Alpha, + *requires*: N/A + + A sample Python project + + :pypi:`pytest-cases` + *last release*: Nov 08, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + Separate test code from test cases in pytest. + + :pypi:`pytest-cassandra` + *last release*: Nov 04, 2017, + *status*: 1 - Planning, + *requires*: N/A + + Cassandra CCM Test Fixtures for pytest + + :pypi:`pytest-catchlog` + *last release*: Jan 24, 2016, + *status*: 4 - Beta, + *requires*: pytest (>=2.6) + + py.test plugin to catch log messages. This is a fork of pytest-capturelog. + + :pypi:`pytest-catch-server` + *last release*: Dec 12, 2019, + *status*: 5 - Production/Stable, + *requires*: N/A + + Pytest plugin with server for catching HTTP requests. + + :pypi:`pytest-celery` + *last release*: May 06, 2021, + *status*: N/A, + *requires*: N/A + + pytest-celery a shim pytest plugin to enable celery.contrib.pytest + + :pypi:`pytest-chainmaker` + *last release*: Oct 15, 2021, + *status*: N/A, + *requires*: N/A + + pytest plugin for chainmaker + + :pypi:`pytest-chalice` + *last release*: Jul 01, 2020, + *status*: 4 - Beta, + *requires*: N/A + + A set of py.test fixtures for AWS Chalice + + :pypi:`pytest-change-report` + *last release*: Sep 14, 2020, + *status*: N/A, + *requires*: pytest + + turn . into √,turn F into x + + :pypi:`pytest-chdir` + *last release*: Jan 28, 2020, + *status*: N/A, + *requires*: pytest (>=5.0.0,<6.0.0) + + A pytest fixture for changing current working directory + + :pypi:`pytest-checkdocs` + *last release*: Jul 31, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.6) ; extra == 'testing' + + check the README when running tests + + :pypi:`pytest-checkipdb` + *last release*: Jul 22, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.9.2) + + plugin to check if there are ipdb debugs left + + :pypi:`pytest-check-links` + *last release*: Jul 29, 2020, + *status*: N/A, + *requires*: pytest (>=4.6) + + Check links in files + + :pypi:`pytest-check-mk` + *last release*: Nov 19, 2015, + *status*: 4 - Beta, + *requires*: pytest + + pytest plugin to test Check_MK checks + + :pypi:`pytest-circleci` + *last release*: May 03, 2019, + *status*: N/A, + *requires*: N/A + + py.test plugin for CircleCI + + :pypi:`pytest-circleci-parallelized` + *last release*: Mar 26, 2019, + *status*: N/A, + *requires*: N/A + + Parallelize pytest across CircleCI workers. + + :pypi:`pytest-ckan` + *last release*: Apr 28, 2020, + *status*: 4 - Beta, + *requires*: pytest + + Backport of CKAN 2.9 pytest plugin and fixtures to CAKN 2.8 + + :pypi:`pytest-clarity` + *last release*: Jun 11, 2021, + *status*: N/A, + *requires*: N/A + + A plugin providing an alternative, colourful diff output for failing assertions. + + :pypi:`pytest-cldf` + *last release*: May 06, 2019, + *status*: N/A, + *requires*: N/A + + Easy quality control for CLDF datasets using pytest + + :pypi:`pytest-click` + *last release*: Aug 29, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.0) + + Py.test plugin for Click + + :pypi:`pytest-clld` + *last release*: Nov 29, 2021, + *status*: N/A, + *requires*: pytest (>=3.6) + + + + :pypi:`pytest-cloud` + *last release*: Oct 05, 2020, + *status*: 6 - Mature, + *requires*: N/A + + Distributed tests planner plugin for pytest testing framework. + + :pypi:`pytest-cloudflare-worker` + *last release*: Mar 30, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.0.0) + + pytest plugin for testing cloudflare workers + + :pypi:`pytest-cobra` + *last release*: Jun 29, 2019, + *status*: 3 - Alpha, + *requires*: pytest (<4.0.0,>=3.7.1) + + PyTest plugin for testing Smart Contracts for Ethereum blockchain. + + :pypi:`pytest-codeblocks` + *last release*: Oct 13, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6) + + Test code blocks in your READMEs + + :pypi:`pytest-codecheckers` + *last release*: Feb 13, 2010, + *status*: N/A, + *requires*: N/A + + pytest plugin to add source code sanity checks (pep8 and friends) + + :pypi:`pytest-codecov` + *last release*: Oct 27, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=4.6.0) + + Pytest plugin for uploading pytest-cov results to codecov.io + + :pypi:`pytest-codegen` + *last release*: Aug 23, 2020, + *status*: 2 - Pre-Alpha, + *requires*: N/A + + Automatically create pytest test signatures + + :pypi:`pytest-codestyle` + *last release*: Mar 23, 2020, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin to run pycodestyle + + :pypi:`pytest-collect-formatter` + *last release*: Mar 29, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + Formatter for pytest collect output + + :pypi:`pytest-collect-formatter2` + *last release*: May 31, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + Formatter for pytest collect output + + :pypi:`pytest-colordots` + *last release*: Oct 06, 2017, + *status*: 5 - Production/Stable, + *requires*: N/A + + Colorizes the progress indicators + + :pypi:`pytest-commander` + *last release*: Aug 17, 2021, + *status*: N/A, + *requires*: pytest (<7.0.0,>=6.2.4) + + An interactive GUI test runner for PyTest + + :pypi:`pytest-common-subject` + *last release*: Nov 12, 2020, + *status*: N/A, + *requires*: pytest (>=3.6,<7) + + pytest framework for testing different aspects of a common method + + :pypi:`pytest-concurrent` + *last release*: Jan 12, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.1.1) + + Concurrently execute test cases with multithread, multiprocess and gevent + + :pypi:`pytest-config` + *last release*: Nov 07, 2014, + *status*: 5 - Production/Stable, + *requires*: N/A + + Base configurations and utilities for developing your Python project test suite with pytest. + + :pypi:`pytest-confluence-report` + *last release*: Nov 06, 2020, + *status*: N/A, + *requires*: N/A + + Package stands for pytest plugin to upload results into Confluence page. + + :pypi:`pytest-console-scripts` + *last release*: Sep 28, 2021, + *status*: 4 - Beta, + *requires*: N/A + + Pytest plugin for testing console scripts + + :pypi:`pytest-consul` + *last release*: Nov 24, 2018, + *status*: 3 - Alpha, + *requires*: pytest + + pytest plugin with fixtures for testing consul aware apps + + :pypi:`pytest-container` + *last release*: Nov 19, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=3.10) + + Pytest fixtures for writing container based tests + + :pypi:`pytest-contextfixture` + *last release*: Mar 12, 2013, + *status*: 4 - Beta, + *requires*: N/A + + Define pytest fixtures as context managers. + + :pypi:`pytest-contexts` + *last release*: May 19, 2021, + *status*: 4 - Beta, + *requires*: N/A + + A plugin to run tests written with the Contexts framework using pytest + + :pypi:`pytest-cookies` + *last release*: May 24, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.3.0) + + The pytest plugin for your Cookiecutter templates. 🍪 + + :pypi:`pytest-couchdbkit` + *last release*: Apr 17, 2012, + *status*: N/A, + *requires*: N/A + + py.test extension for per-test couchdb databases using couchdbkit + + :pypi:`pytest-count` + *last release*: Jan 12, 2018, + *status*: 4 - Beta, + *requires*: N/A + + count erros and send email + + :pypi:`pytest-cov` + *last release*: Oct 04, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.6) + + Pytest plugin for measuring coverage. + + :pypi:`pytest-cover` + *last release*: Aug 01, 2015, + *status*: 5 - Production/Stable, + *requires*: N/A + + Pytest plugin for measuring coverage. Forked from \`pytest-cov\`. + + :pypi:`pytest-coverage` + *last release*: Jun 17, 2015, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-coverage-context` + *last release*: Jan 04, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.1.0) + + Coverage dynamic context support for PyTest, including sub-processes + + :pypi:`pytest-cov-exclude` + *last release*: Apr 29, 2016, + *status*: 4 - Beta, + *requires*: pytest (>=2.8.0,<2.9.0); extra == 'dev' + + Pytest plugin for excluding tests based on coverage data + + :pypi:`pytest-cpp` + *last release*: Dec 03, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (!=5.4.0,!=5.4.1) + + Use pytest's runner to discover and execute C++ tests + + :pypi:`pytest-cram` + *last release*: Aug 08, 2020, + *status*: N/A, + *requires*: N/A + + Run cram tests with pytest. + + :pypi:`pytest-crate` + *last release*: May 28, 2019, + *status*: 3 - Alpha, + *requires*: pytest (>=4.0) + + Manages CrateDB instances during your integration tests + + :pypi:`pytest-cricri` + *last release*: Jan 27, 2018, + *status*: N/A, + *requires*: pytest + + A Cricri plugin for pytest. + + :pypi:`pytest-crontab` + *last release*: Dec 09, 2019, + *status*: N/A, + *requires*: N/A + + add crontab task in crontab + + :pypi:`pytest-csv` + *last release*: Apr 22, 2021, + *status*: N/A, + *requires*: pytest (>=6.0) + + CSV output for pytest. + + :pypi:`pytest-curio` + *last release*: Oct 07, 2020, + *status*: N/A, + *requires*: N/A + + Pytest support for curio. + + :pypi:`pytest-curl-report` + *last release*: Dec 11, 2016, + *status*: 4 - Beta, + *requires*: N/A + + pytest plugin to generate curl command line report + + :pypi:`pytest-custom-concurrency` + *last release*: Feb 08, 2021, + *status*: N/A, + *requires*: N/A + + Custom grouping concurrence for pytest + + :pypi:`pytest-custom-exit-code` + *last release*: Aug 07, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=4.0.2) + + Exit pytest test session with custom exit code in different scenarios + + :pypi:`pytest-custom-nodeid` + *last release*: Mar 07, 2021, + *status*: N/A, + *requires*: N/A + + Custom grouping for pytest-xdist, rename test cases name and test cases nodeid, support allure report + + :pypi:`pytest-custom-report` + *last release*: Jan 30, 2019, + *status*: N/A, + *requires*: pytest + + Configure the symbols displayed for test outcomes + + :pypi:`pytest-custom-scheduling` + *last release*: Mar 01, 2021, + *status*: N/A, + *requires*: N/A + + Custom grouping for pytest-xdist, rename test cases name and test cases nodeid, support allure report + + :pypi:`pytest-cython` + *last release*: Jan 26, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=2.7.3) + + A plugin for testing Cython extension modules + + :pypi:`pytest-darker` + *last release*: Aug 16, 2020, + *status*: N/A, + *requires*: pytest (>=6.0.1) ; extra == 'test' + + A pytest plugin for checking of modified code using Darker + + :pypi:`pytest-dash` + *last release*: Mar 18, 2019, + *status*: N/A, + *requires*: N/A + + pytest fixtures to run dash applications. + + :pypi:`pytest-data` + *last release*: Nov 01, 2016, + *status*: 5 - Production/Stable, + *requires*: N/A + + Useful functions for managing data for pytest fixtures + + :pypi:`pytest-databricks` + *last release*: Jul 29, 2020, + *status*: N/A, + *requires*: pytest + + Pytest plugin for remote Databricks notebooks testing + + :pypi:`pytest-datadir` + *last release*: Oct 22, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.7.0) + + pytest plugin for test data directories and files + + :pypi:`pytest-datadir-mgr` + *last release*: Aug 16, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + Manager for test data providing downloads, caching of generated files, and a context for temp directories. + + :pypi:`pytest-datadir-ng` + *last release*: Dec 25, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + Fixtures for pytest allowing test functions/methods to easily retrieve test resources from the local filesystem. + + :pypi:`pytest-data-file` + *last release*: Dec 04, 2019, + *status*: N/A, + *requires*: N/A + + Fixture "data" and "case_data" for test from yaml file + + :pypi:`pytest-datafiles` + *last release*: Oct 07, 2018, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.6) + + py.test plugin to create a 'tmpdir' containing predefined files/directories. + + :pypi:`pytest-datafixtures` + *last release*: Dec 05, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Data fixtures for pytest made simple + + :pypi:`pytest-data-from-files` + *last release*: Oct 13, 2021, + *status*: 4 - Beta, + *requires*: pytest + + pytest plugin to provide data from files loaded automatically + + :pypi:`pytest-dataplugin` + *last release*: Sep 16, 2017, + *status*: 1 - Planning, + *requires*: N/A + + A pytest plugin for managing an archive of test data. + + :pypi:`pytest-datarecorder` + *last release*: Apr 20, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest + + A py.test plugin recording and comparing test output. + + :pypi:`pytest-datatest` + *last release*: Oct 15, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.3) + + A pytest plugin for test driven data-wrangling (this is the development version of datatest's pytest integration). + + :pypi:`pytest-db` + *last release*: Dec 04, 2019, + *status*: N/A, + *requires*: N/A + + Session scope fixture "db" for mysql query or change + + :pypi:`pytest-dbfixtures` + *last release*: Dec 07, 2016, + *status*: 4 - Beta, + *requires*: N/A + + Databases fixtures plugin for py.test. + + :pypi:`pytest-db-plugin` + *last release*: Nov 27, 2021, + *status*: N/A, + *requires*: pytest (>=5.0) + + + + :pypi:`pytest-dbt-adapter` + *last release*: Nov 24, 2021, + *status*: N/A, + *requires*: pytest (<7,>=6) + + A pytest plugin for testing dbt adapter plugins + + :pypi:`pytest-dbus-notification` + *last release*: Mar 05, 2014, + *status*: 5 - Production/Stable, + *requires*: N/A + + D-BUS notifications for pytest results. + + :pypi:`pytest-deadfixtures` + *last release*: Jul 23, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + A simple plugin to list unused fixtures in pytest + + :pypi:`pytest-deepcov` + *last release*: Mar 30, 2021, + *status*: N/A, + *requires*: N/A + + deepcov + + :pypi:`pytest-defer` + *last release*: Aug 24, 2021, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-demo-plugin` + *last release*: May 15, 2021, + *status*: N/A, + *requires*: N/A + + pytest示例插件 + + :pypi:`pytest-dependency` + *last release*: Feb 14, 2020, + *status*: 4 - Beta, + *requires*: N/A + + Manage dependencies of tests + + :pypi:`pytest-depends` + *last release*: Apr 05, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3) + + Tests that depend on other tests + + :pypi:`pytest-deprecate` + *last release*: Jul 01, 2019, + *status*: N/A, + *requires*: N/A + + Mark tests as testing a deprecated feature with a warning note. + + :pypi:`pytest-describe` + *last release*: Nov 13, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=4.0.0) + + Describe-style plugin for pytest + + :pypi:`pytest-describe-it` + *last release*: Jul 19, 2019, + *status*: 4 - Beta, + *requires*: pytest + + plugin for rich text descriptions + + :pypi:`pytest-devpi-server` + *last release*: May 28, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + DevPI server fixture for py.test + + :pypi:`pytest-diamond` + *last release*: Aug 31, 2015, + *status*: 4 - Beta, + *requires*: N/A + + pytest plugin for diamond + + :pypi:`pytest-dicom` + *last release*: Dec 19, 2018, + *status*: 3 - Alpha, + *requires*: pytest + + pytest plugin to provide DICOM fixtures + + :pypi:`pytest-dictsdiff` + *last release*: Jul 26, 2019, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-diff` + *last release*: Mar 30, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A simple plugin to use with pytest + + :pypi:`pytest-disable` + *last release*: Sep 10, 2015, + *status*: 4 - Beta, + *requires*: N/A + + pytest plugin to disable a test and skip it from testrun + + :pypi:`pytest-disable-plugin` + *last release*: Feb 28, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Disable plugins per test + + :pypi:`pytest-discord` + *last release*: Mar 20, 2021, + *status*: 3 - Alpha, + *requires*: pytest (!=6.0.0,<7,>=3.3.2) + + A pytest plugin to notify test results to a Discord channel. + + :pypi:`pytest-django` + *last release*: Dec 02, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.4.0) + + A Django plugin for pytest. + + :pypi:`pytest-django-ahead` + *last release*: Oct 27, 2016, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.9) + + A Django plugin for pytest. + + :pypi:`pytest-djangoapp` + *last release*: Aug 04, 2021, + *status*: 4 - Beta, + *requires*: N/A + + Nice pytest plugin to help you with Django pluggable application testing. + + :pypi:`pytest-django-cache-xdist` + *last release*: May 12, 2020, + *status*: 4 - Beta, + *requires*: N/A + + A djangocachexdist plugin for pytest + + :pypi:`pytest-django-casperjs` + *last release*: Mar 15, 2015, + *status*: 2 - Pre-Alpha, + *requires*: N/A + + Integrate CasperJS with your django tests as a pytest fixture. + + :pypi:`pytest-django-dotenv` + *last release*: Nov 26, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=2.6.0) + + Pytest plugin used to setup environment variables with django-dotenv + + :pypi:`pytest-django-factories` + *last release*: Nov 12, 2020, + *status*: 4 - Beta, + *requires*: N/A + + Factories for your Django models that can be used as Pytest fixtures. + + :pypi:`pytest-django-gcir` + *last release*: Mar 06, 2018, + *status*: 5 - Production/Stable, + *requires*: N/A + + A Django plugin for pytest. + + :pypi:`pytest-django-haystack` + *last release*: Sep 03, 2017, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.3.4) + + Cleanup your Haystack indexes between tests + + :pypi:`pytest-django-ifactory` + *last release*: Jan 13, 2021, + *status*: 3 - Alpha, + *requires*: N/A + + A model instance factory for pytest-django + + :pypi:`pytest-django-lite` + *last release*: Jan 30, 2014, + *status*: N/A, + *requires*: N/A + + The bare minimum to integrate py.test with Django. + + :pypi:`pytest-django-liveserver-ssl` + *last release*: Jul 30, 2021, + *status*: 3 - Alpha, + *requires*: N/A + + + + :pypi:`pytest-django-model` + *last release*: Feb 14, 2019, + *status*: 4 - Beta, + *requires*: N/A + + A Simple Way to Test your Django Models + + :pypi:`pytest-django-ordering` + *last release*: Jul 25, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.3.0) + + A pytest plugin for preserving the order in which Django runs tests. + + :pypi:`pytest-django-queries` + *last release*: Mar 01, 2021, + *status*: N/A, + *requires*: N/A + + Generate performance reports from your django database performance tests. + + :pypi:`pytest-djangorestframework` + *last release*: Aug 11, 2019, + *status*: 4 - Beta, + *requires*: N/A + + A djangorestframework plugin for pytest + + :pypi:`pytest-django-rq` + *last release*: Apr 13, 2020, + *status*: 4 - Beta, + *requires*: N/A + + A pytest plugin to help writing unit test for django-rq + + :pypi:`pytest-django-sqlcounts` + *last release*: Jun 16, 2015, + *status*: 4 - Beta, + *requires*: N/A + + py.test plugin for reporting the number of SQLs executed per django testcase. + + :pypi:`pytest-django-testing-postgresql` + *last release*: Dec 05, 2019, + *status*: 3 - Alpha, + *requires*: N/A + + Use a temporary PostgreSQL database with pytest-django + + :pypi:`pytest-doc` + *last release*: Jun 28, 2015, + *status*: 5 - Production/Stable, + *requires*: N/A + + A documentation plugin for py.test. + + :pypi:`pytest-docgen` + *last release*: Apr 17, 2020, + *status*: N/A, + *requires*: N/A + + An RST Documentation Generator for pytest-based test suites + + :pypi:`pytest-docker` + *last release*: Jun 14, 2021, + *status*: N/A, + *requires*: pytest (<7.0,>=4.0) + + Simple pytest fixtures for Docker and docker-compose based tests + + :pypi:`pytest-docker-butla` + *last release*: Jun 16, 2019, + *status*: 3 - Alpha, + *requires*: N/A + + + + :pypi:`pytest-dockerc` + *last release*: Oct 09, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.0) + + Run, manage and stop Docker Compose project from Docker API + + :pypi:`pytest-docker-compose` + *last release*: Jan 26, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.3) + + Manages Docker containers during your integration tests + + :pypi:`pytest-docker-db` + *last release*: Mar 20, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.1.1) + + A plugin to use docker databases for pytests + + :pypi:`pytest-docker-fixtures` + *last release*: Nov 23, 2021, + *status*: 3 - Alpha, + *requires*: N/A + + pytest docker fixtures + + :pypi:`pytest-docker-git-fixtures` + *last release*: Mar 11, 2021, + *status*: 4 - Beta, + *requires*: pytest + + Pytest fixtures for testing with git scm. + + :pypi:`pytest-docker-pexpect` + *last release*: Jan 14, 2019, + *status*: N/A, + *requires*: pytest + + pytest plugin for writing functional tests with pexpect and docker + + :pypi:`pytest-docker-postgresql` + *last release*: Sep 24, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A simple plugin to use with pytest + + :pypi:`pytest-docker-py` + *last release*: Nov 27, 2018, + *status*: N/A, + *requires*: pytest (==4.0.0) + + Easy to use, simple to extend, pytest plugin that minimally leverages docker-py. + + :pypi:`pytest-docker-registry-fixtures` + *last release*: Mar 04, 2021, + *status*: 4 - Beta, + *requires*: pytest + + Pytest fixtures for testing with docker registries. + + :pypi:`pytest-docker-tools` + *last release*: Jul 23, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.0.1,<7.0.0) + + Docker integration tests for pytest + + :pypi:`pytest-docs` + *last release*: Nov 11, 2018, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Documentation tool for pytest + + :pypi:`pytest-docstyle` + *last release*: Mar 23, 2020, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin to run pydocstyle + + :pypi:`pytest-doctest-custom` + *last release*: Jul 25, 2016, + *status*: 4 - Beta, + *requires*: N/A + + A py.test plugin for customizing string representations of doctest results. + + :pypi:`pytest-doctest-ellipsis-markers` + *last release*: Jan 12, 2018, + *status*: 4 - Beta, + *requires*: N/A + + Setup additional values for ELLIPSIS_MARKER for doctests + + :pypi:`pytest-doctest-import` + *last release*: Nov 13, 2018, + *status*: 4 - Beta, + *requires*: pytest (>=3.3.0) + + A simple pytest plugin to import names and add them to the doctest namespace. + + :pypi:`pytest-doctestplus` + *last release*: Nov 16, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=4.6) + + Pytest plugin with advanced doctest features. + + :pypi:`pytest-doctest-ufunc` + *last release*: Aug 02, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A plugin to run doctests in docstrings of Numpy ufuncs + + :pypi:`pytest-dolphin` + *last release*: Nov 30, 2016, + *status*: 4 - Beta, + *requires*: pytest (==3.0.4) + + Some extra stuff that we use ininternally + + :pypi:`pytest-doorstop` + *last release*: Jun 09, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A pytest plugin for adding test results into doorstop items. + + :pypi:`pytest-dotenv` + *last release*: Jun 16, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=5.0.0) + + A py.test plugin that parses environment files before running tests + + :pypi:`pytest-drf` + *last release*: Nov 12, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.6) + + A Django REST framework plugin for pytest. + + :pypi:`pytest-drivings` + *last release*: Jan 13, 2021, + *status*: N/A, + *requires*: N/A + + Tool to allow webdriver automation to be ran locally or remotely + + :pypi:`pytest-drop-dup-tests` + *last release*: May 23, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=2.7) + + A Pytest plugin to drop duplicated tests during collection + + :pypi:`pytest-dummynet` + *last release*: Oct 13, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + A py.test plugin providing access to a dummynet. + + :pypi:`pytest-dump2json` + *last release*: Jun 29, 2015, + *status*: N/A, + *requires*: N/A + + A pytest plugin for dumping test results to json. + + :pypi:`pytest-duration-insights` + *last release*: Jun 25, 2021, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-dynamicrerun` + *last release*: Aug 15, 2020, + *status*: 4 - Beta, + *requires*: N/A + + A pytest plugin to rerun tests dynamically based off of test outcome and output. + + :pypi:`pytest-dynamodb` + *last release*: Jun 03, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + DynamoDB fixtures for pytest + + :pypi:`pytest-easy-addoption` + *last release*: Jan 22, 2020, + *status*: N/A, + *requires*: N/A + + pytest-easy-addoption: Easy way to work with pytest addoption + + :pypi:`pytest-easy-api` + *last release*: Mar 26, 2018, + *status*: N/A, + *requires*: N/A + + Simple API testing with pytest + + :pypi:`pytest-easyMPI` + *last release*: Oct 21, 2020, + *status*: N/A, + *requires*: N/A + + Package that supports mpi tests in pytest + + :pypi:`pytest-easyread` + *last release*: Nov 17, 2017, + *status*: N/A, + *requires*: N/A + + pytest plugin that makes terminal printouts of the reports easier to read + + :pypi:`pytest-easy-server` + *last release*: May 01, 2021, + *status*: 4 - Beta, + *requires*: pytest (<5.0.0,>=4.3.1) ; python_version < "3.5" + + Pytest plugin for easy testing against servers + + :pypi:`pytest-ec2` + *last release*: Oct 22, 2019, + *status*: 3 - Alpha, + *requires*: N/A + + Pytest execution on EC2 instance + + :pypi:`pytest-echo` + *last release*: Jan 08, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest plugin with mechanisms for echoing environment variables, package version and generic attributes + + :pypi:`pytest-elasticsearch` + *last release*: May 12, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.0.0) + + Elasticsearch fixtures and fixture factories for Pytest. + + :pypi:`pytest-elements` + *last release*: Jan 13, 2021, + *status*: N/A, + *requires*: pytest (>=5.4,<6.0) + + Tool to help automate user interfaces + + :pypi:`pytest-elk-reporter` + *last release*: Jan 24, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A simple plugin to use with pytest + + :pypi:`pytest-email` + *last release*: Jul 08, 2020, + *status*: N/A, + *requires*: pytest + + Send execution result email + + :pypi:`pytest-embedded` + *last release*: Nov 29, 2021, + *status*: N/A, + *requires*: pytest (>=6.2.0) + + pytest embedded plugin + + :pypi:`pytest-embedded-idf` + *last release*: Nov 29, 2021, + *status*: N/A, + *requires*: N/A + + pytest embedded plugin for esp-idf project + + :pypi:`pytest-embedded-jtag` + *last release*: Nov 29, 2021, + *status*: N/A, + *requires*: N/A + + pytest embedded plugin for testing with jtag + + :pypi:`pytest-embedded-qemu` + *last release*: Nov 29, 2021, + *status*: N/A, + *requires*: N/A + + pytest embedded plugin for qemu, not target chip + + :pypi:`pytest-embedded-qemu-idf` + *last release*: Jun 29, 2021, + *status*: N/A, + *requires*: N/A + + pytest embedded plugin for esp-idf project by qemu, not target chip + + :pypi:`pytest-embedded-serial` + *last release*: Nov 29, 2021, + *status*: N/A, + *requires*: N/A + + pytest embedded plugin for testing serial ports + + :pypi:`pytest-embedded-serial-esp` + *last release*: Nov 29, 2021, + *status*: N/A, + *requires*: N/A + + pytest embedded plugin for testing espressif boards via serial ports + + :pypi:`pytest-emoji` + *last release*: Feb 19, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=4.2.1) + + A pytest plugin that adds emojis to your test result report + + :pypi:`pytest-emoji-output` + *last release*: Oct 10, 2021, + *status*: 4 - Beta, + *requires*: pytest (==6.0.1) + + Pytest plugin to represent test output with emoji support + + :pypi:`pytest-enabler` + *last release*: Nov 08, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=6) ; extra == 'testing' + + Enable installed pytest plugins + + :pypi:`pytest-encode` + *last release*: Nov 06, 2021, + *status*: N/A, + *requires*: N/A + + set your encoding and logger + + :pypi:`pytest-encode-kane` + *last release*: Nov 16, 2021, + *status*: N/A, + *requires*: pytest + + set your encoding and logger + + :pypi:`pytest-enhancements` + *last release*: Oct 30, 2019, + *status*: 4 - Beta, + *requires*: N/A + + Improvements for pytest (rejected upstream) + + :pypi:`pytest-env` + *last release*: Jun 16, 2017, + *status*: 4 - Beta, + *requires*: N/A + + py.test plugin that allows you to add environment variables. + + :pypi:`pytest-envfiles` + *last release*: Oct 08, 2015, + *status*: 3 - Alpha, + *requires*: N/A + + A py.test plugin that parses environment files before running tests + + :pypi:`pytest-env-info` + *last release*: Nov 25, 2017, + *status*: 4 - Beta, + *requires*: pytest (>=3.1.1) + + Push information about the running pytest into envvars + + :pypi:`pytest-envraw` + *last release*: Aug 27, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=2.6.0) + + py.test plugin that allows you to add environment variables. + + :pypi:`pytest-envvars` + *last release*: Jun 13, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.0.0) + + Pytest plugin to validate use of envvars on your tests + + :pypi:`pytest-env-yaml` + *last release*: Apr 02, 2019, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-eradicate` + *last release*: Sep 08, 2020, + *status*: N/A, + *requires*: pytest (>=2.4.2) + + pytest plugin to check for commented out code + + :pypi:`pytest-error-for-skips` + *last release*: Dec 19, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=4.6) + + Pytest plugin to treat skipped tests a test failure + + :pypi:`pytest-eth` + *last release*: Aug 14, 2020, + *status*: 1 - Planning, + *requires*: N/A + + PyTest plugin for testing Smart Contracts for Ethereum Virtual Machine (EVM). + + :pypi:`pytest-ethereum` + *last release*: Jun 24, 2019, + *status*: 3 - Alpha, + *requires*: pytest (==3.3.2); extra == 'dev' + + pytest-ethereum: Pytest library for ethereum projects. + + :pypi:`pytest-eucalyptus` + *last release*: Aug 13, 2019, + *status*: N/A, + *requires*: pytest (>=4.2.0) + + Pytest Plugin for BDD + + :pypi:`pytest-eventlet` + *last release*: Oct 04, 2021, + *status*: N/A, + *requires*: pytest ; extra == 'dev' + + Applies eventlet monkey-patch as a pytest plugin. + + :pypi:`pytest-excel` + *last release*: Oct 06, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest plugin for generating excel reports + + :pypi:`pytest-exceptional` + *last release*: Mar 16, 2017, + *status*: 4 - Beta, + *requires*: N/A + + Better exceptions + + :pypi:`pytest-exception-script` + *last release*: Aug 04, 2020, + *status*: 3 - Alpha, + *requires*: pytest + + Walk your code through exception script to check it's resiliency to failures. + + :pypi:`pytest-executable` + *last release*: Nov 10, 2021, + *status*: 4 - Beta, + *requires*: pytest (<6.3,>=4.3) + + pytest plugin for testing executables + + :pypi:`pytest-expect` + *last release*: Apr 21, 2016, + *status*: 4 - Beta, + *requires*: N/A + + py.test plugin to store test expectations and mark tests based on them + + :pypi:`pytest-expecter` + *last release*: Jul 08, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Better testing with expecter and pytest. + + :pypi:`pytest-expectr` + *last release*: Oct 05, 2018, + *status*: N/A, + *requires*: pytest (>=2.4.2) + + This plugin is used to expect multiple assert using pytest framework. + + :pypi:`pytest-explicit` + *last release*: Jun 15, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + A Pytest plugin to ignore certain marked tests by default + + :pypi:`pytest-exploratory` + *last release*: Aug 03, 2021, + *status*: N/A, + *requires*: pytest (>=5.3) + + Interactive console for pytest. + + :pypi:`pytest-external-blockers` + *last release*: Oct 05, 2021, + *status*: N/A, + *requires*: pytest + + a special outcome for tests that are blocked for external reasons + + :pypi:`pytest-extra-durations` + *last release*: Apr 21, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A pytest plugin to get durations on a per-function basis and per module basis. + + :pypi:`pytest-fabric` + *last release*: Sep 12, 2018, + *status*: 5 - Production/Stable, + *requires*: N/A + + Provides test utilities to run fabric task tests by using docker containers + + :pypi:`pytest-factory` + *last release*: Sep 06, 2020, + *status*: 3 - Alpha, + *requires*: pytest (>4.3) + + Use factories for test setup with py.test + + :pypi:`pytest-factoryboy` + *last release*: Dec 30, 2020, + *status*: 6 - Mature, + *requires*: pytest (>=4.6) + + Factory Boy support for pytest. + + :pypi:`pytest-factoryboy-fixtures` + *last release*: Jun 25, 2020, + *status*: N/A, + *requires*: N/A + + Generates pytest fixtures that allow the use of type hinting + + :pypi:`pytest-factoryboy-state` + *last release*: Dec 11, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=5.0) + + Simple factoryboy random state management + + :pypi:`pytest-failed-screenshot` + *last release*: Apr 21, 2021, + *status*: N/A, + *requires*: N/A + + Test case fails,take a screenshot,save it,attach it to the allure + + :pypi:`pytest-failed-to-verify` + *last release*: Aug 08, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.1.0) + + A pytest plugin that helps better distinguishing real test failures from setup flakiness. + + :pypi:`pytest-faker` + *last release*: Dec 19, 2016, + *status*: 6 - Mature, + *requires*: N/A + + Faker integration with the pytest framework. + + :pypi:`pytest-falcon` + *last release*: Sep 07, 2016, + *status*: 4 - Beta, + *requires*: N/A + + Pytest helpers for Falcon. + + :pypi:`pytest-falcon-client` + *last release*: Mar 19, 2019, + *status*: N/A, + *requires*: N/A + + Pytest \`client\` fixture for the Falcon Framework + + :pypi:`pytest-fantasy` + *last release*: Mar 14, 2019, + *status*: N/A, + *requires*: N/A + + Pytest plugin for Flask Fantasy Framework + + :pypi:`pytest-fastapi` + *last release*: Dec 27, 2020, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-fastest` + *last release*: Mar 05, 2020, + *status*: N/A, + *requires*: N/A + + Use SCM and coverage to run only needed tests + + :pypi:`pytest-fast-first` + *last release*: Apr 02, 2021, + *status*: 3 - Alpha, + *requires*: pytest + + Pytest plugin that runs fast tests first + + :pypi:`pytest-faulthandler` + *last release*: Jul 04, 2019, + *status*: 6 - Mature, + *requires*: pytest (>=5.0) + + py.test plugin that activates the fault handler module for tests (dummy package) + + :pypi:`pytest-fauxfactory` + *last release*: Dec 06, 2017, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.2) + + Integration of fauxfactory into pytest. + + :pypi:`pytest-figleaf` + *last release*: Jan 18, 2010, + *status*: 5 - Production/Stable, + *requires*: N/A + + py.test figleaf coverage plugin + + :pypi:`pytest-filecov` + *last release*: Jun 27, 2021, + *status*: 4 - Beta, + *requires*: pytest + + A pytest plugin to detect unused files + + :pypi:`pytest-filedata` + *last release*: Jan 17, 2019, + *status*: 4 - Beta, + *requires*: N/A + + easily load data from files + + :pypi:`pytest-filemarker` + *last release*: Dec 01, 2020, + *status*: N/A, + *requires*: pytest + + A pytest plugin that runs marked tests when files change. + + :pypi:`pytest-filter-case` + *last release*: Nov 05, 2020, + *status*: N/A, + *requires*: N/A + + run test cases filter by mark + + :pypi:`pytest-filter-subpackage` + *last release*: Jan 09, 2020, + *status*: 3 - Alpha, + *requires*: pytest (>=3.0) + + Pytest plugin for filtering based on sub-packages + + :pypi:`pytest-find-dependencies` + *last release*: Apr 21, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A pytest plugin to find dependencies between tests + + :pypi:`pytest-finer-verdicts` + *last release*: Jun 18, 2020, + *status*: N/A, + *requires*: pytest (>=5.4.3) + + A pytest plugin to treat non-assertion failures as test errors. + + :pypi:`pytest-firefox` + *last release*: Aug 08, 2017, + *status*: 3 - Alpha, + *requires*: pytest (>=3.0.2) + + pytest plugin to manipulate firefox + + :pypi:`pytest-fixture-config` + *last release*: May 28, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + Fixture configuration utils for py.test + + :pypi:`pytest-fixture-maker` + *last release*: Sep 21, 2021, + *status*: N/A, + *requires*: N/A + + Pytest plugin to load fixtures from YAML files + + :pypi:`pytest-fixture-marker` + *last release*: Oct 11, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + A pytest plugin to add markers based on fixtures used. + + :pypi:`pytest-fixture-order` + *last release*: Aug 25, 2020, + *status*: N/A, + *requires*: pytest (>=3.0) + + pytest plugin to control fixture evaluation order + + :pypi:`pytest-fixtures` + *last release*: May 01, 2019, + *status*: 5 - Production/Stable, + *requires*: N/A + + Common fixtures for pytest + + :pypi:`pytest-fixture-tools` + *last release*: Aug 18, 2020, + *status*: 6 - Mature, + *requires*: pytest + + Plugin for pytest which provides tools for fixtures + + :pypi:`pytest-fixture-typecheck` + *last release*: Aug 24, 2021, + *status*: N/A, + *requires*: pytest + + A pytest plugin to assert type annotations at runtime. + + :pypi:`pytest-flake8` + *last release*: Dec 16, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5) + + pytest plugin to check FLAKE8 requirements + + :pypi:`pytest-flake8-path` + *last release*: Aug 11, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + A pytest fixture for testing flake8 plugins. + + :pypi:`pytest-flakefinder` + *last release*: Jul 28, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=2.7.1) + + Runs tests multiple times to expose flakiness. + + :pypi:`pytest-flakes` + *last release*: Dec 02, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5) + + pytest plugin to check source code with pyflakes + + :pypi:`pytest-flaptastic` + *last release*: Mar 17, 2019, + *status*: N/A, + *requires*: N/A + + Flaptastic py.test plugin + + :pypi:`pytest-flask` + *last release*: Feb 27, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.2) + + A set of py.test fixtures to test Flask applications. + + :pypi:`pytest-flask-sqlalchemy` + *last release*: Apr 04, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.2.1) + + A pytest plugin for preserving test isolation in Flask-SQlAlchemy using database transactions. + + :pypi:`pytest-flask-sqlalchemy-transactions` + *last release*: Aug 02, 2018, + *status*: 4 - Beta, + *requires*: pytest (>=3.2.1) + + Run tests in transactions using pytest, Flask, and SQLalchemy. + + :pypi:`pytest-flyte` + *last release*: May 03, 2021, + *status*: N/A, + *requires*: pytest + + Pytest fixtures for simplifying Flyte integration testing + + :pypi:`pytest-focus` + *last release*: May 04, 2019, + *status*: 4 - Beta, + *requires*: pytest + + A pytest plugin that alerts user of failed test cases with screen notifications + + :pypi:`pytest-forcefail` + *last release*: May 15, 2018, + *status*: 4 - Beta, + *requires*: N/A + + py.test plugin to make the test failing regardless of pytest.mark.xfail + + :pypi:`pytest-forward-compatability` + *last release*: Sep 06, 2020, + *status*: N/A, + *requires*: N/A + + A name to avoid typosquating pytest-foward-compatibility + + :pypi:`pytest-forward-compatibility` + *last release*: Sep 29, 2020, + *status*: N/A, + *requires*: N/A + + A pytest plugin to shim pytest commandline options for fowards compatibility + + :pypi:`pytest-freezegun` + *last release*: Jul 19, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.0.0) + + Wrap tests with fixtures in freeze_time + + :pypi:`pytest-freeze-reqs` + *last release*: Apr 29, 2021, + *status*: N/A, + *requires*: N/A + + Check if requirement files are frozen + + :pypi:`pytest-frozen-uuids` + *last release*: Oct 19, 2021, + *status*: N/A, + *requires*: pytest (>=3.0) + + Deterministically frozen UUID's for your tests + + :pypi:`pytest-func-cov` + *last release*: Apr 15, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=5) + + Pytest plugin for measuring function coverage + + :pypi:`pytest-funparam` + *last release*: Dec 02, 2021, + *status*: 4 - Beta, + *requires*: pytest >=4.6.0 + + An alternative way to parametrize test cases. + + :pypi:`pytest-fxa` + *last release*: Aug 28, 2018, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest plugin for Firefox Accounts + + :pypi:`pytest-fxtest` + *last release*: Oct 27, 2020, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-gc` + *last release*: Feb 01, 2018, + *status*: N/A, + *requires*: N/A + + The garbage collector plugin for py.test + + :pypi:`pytest-gcov` + *last release*: Feb 01, 2018, + *status*: 3 - Alpha, + *requires*: N/A + + Uses gcov to measure test coverage of a C library + + :pypi:`pytest-gevent` + *last release*: Feb 25, 2020, + *status*: N/A, + *requires*: pytest + + Ensure that gevent is properly patched when invoking pytest + + :pypi:`pytest-gherkin` + *last release*: Jul 27, 2019, + *status*: 3 - Alpha, + *requires*: pytest (>=5.0.0) + + A flexible framework for executing BDD gherkin tests + + :pypi:`pytest-ghostinspector` + *last release*: May 17, 2016, + *status*: 3 - Alpha, + *requires*: N/A + + For finding/executing Ghost Inspector tests + + :pypi:`pytest-girder` + *last release*: Nov 30, 2021, + *status*: N/A, + *requires*: N/A + + A set of pytest fixtures for testing Girder applications. + + :pypi:`pytest-git` + *last release*: May 28, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + Git repository fixture for py.test + + :pypi:`pytest-gitcov` + *last release*: Jan 11, 2020, + *status*: 2 - Pre-Alpha, + *requires*: N/A + + Pytest plugin for reporting on coverage of the last git commit. + + :pypi:`pytest-git-fixtures` + *last release*: Mar 11, 2021, + *status*: 4 - Beta, + *requires*: pytest + + Pytest fixtures for testing with git. + + :pypi:`pytest-github` + *last release*: Mar 07, 2019, + *status*: 5 - Production/Stable, + *requires*: N/A + + Plugin for py.test that associates tests with github issues using a marker. + + :pypi:`pytest-github-actions-annotate-failures` + *last release*: Oct 24, 2021, + *status*: N/A, + *requires*: pytest (>=4.0.0) + + pytest plugin to annotate failed tests with a workflow command for GitHub Actions + + :pypi:`pytest-gitignore` + *last release*: Jul 17, 2015, + *status*: 4 - Beta, + *requires*: N/A + + py.test plugin to ignore the same files as git + + :pypi:`pytest-glamor-allure` + *last release*: Nov 26, 2021, + *status*: 4 - Beta, + *requires*: pytest + + Extends allure-pytest functionality + + :pypi:`pytest-gnupg-fixtures` + *last release*: Mar 04, 2021, + *status*: 4 - Beta, + *requires*: pytest + + Pytest fixtures for testing with gnupg. + + :pypi:`pytest-golden` + *last release*: Nov 23, 2020, + *status*: N/A, + *requires*: pytest (>=6.1.2,<7.0.0) + + Plugin for pytest that offloads expected outputs to data files + + :pypi:`pytest-graphql-schema` + *last release*: Oct 18, 2019, + *status*: N/A, + *requires*: N/A + + Get graphql schema as fixture for pytest + + :pypi:`pytest-greendots` + *last release*: Feb 08, 2014, + *status*: 3 - Alpha, + *requires*: N/A + + Green progress dots + + :pypi:`pytest-growl` + *last release*: Jan 13, 2014, + *status*: 5 - Production/Stable, + *requires*: N/A + + Growl notifications for pytest results. + + :pypi:`pytest-grpc` + *last release*: May 01, 2020, + *status*: N/A, + *requires*: pytest (>=3.6.0) + + pytest plugin for grpc + + :pypi:`pytest-hammertime` + *last release*: Jul 28, 2018, + *status*: N/A, + *requires*: pytest + + Display "🔨 " instead of "." for passed pytest tests. + + :pypi:`pytest-harvest` + *last release*: Apr 01, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + Store data created during your pytest tests execution, and retrieve it at the end of the session, e.g. for applicative benchmarking purposes. + + :pypi:`pytest-helm-chart` + *last release*: Jun 15, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=5.4.2,<6.0.0) + + A plugin to provide different types and configs of Kubernetes clusters that can be used for testing. + + :pypi:`pytest-helm-charts` + *last release*: Oct 26, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.1.2,<7.0.0) + + A plugin to provide different types and configs of Kubernetes clusters that can be used for testing. + + :pypi:`pytest-helper` + *last release*: May 31, 2019, + *status*: 5 - Production/Stable, + *requires*: N/A + + Functions to help in using the pytest testing framework + + :pypi:`pytest-helpers` + *last release*: May 17, 2020, + *status*: N/A, + *requires*: pytest + + pytest helpers + + :pypi:`pytest-helpers-namespace` + *last release*: Apr 29, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=6.0.0) + + Pytest Helpers Namespace Plugin + + :pypi:`pytest-hidecaptured` + *last release*: May 04, 2018, + *status*: 4 - Beta, + *requires*: pytest (>=2.8.5) + + Hide captured output + + :pypi:`pytest-historic` + *last release*: Apr 08, 2020, + *status*: N/A, + *requires*: pytest + + Custom report to display pytest historical execution records + + :pypi:`pytest-historic-hook` + *last release*: Apr 08, 2020, + *status*: N/A, + *requires*: pytest + + Custom listener to store execution results into MYSQL DB, which is used for pytest-historic report + + :pypi:`pytest-homeassistant` + *last release*: Aug 12, 2020, + *status*: 4 - Beta, + *requires*: N/A + + A pytest plugin for use with homeassistant custom components. + + :pypi:`pytest-homeassistant-custom-component` + *last release*: Nov 20, 2021, + *status*: 3 - Alpha, + *requires*: pytest (==6.2.5) + + Experimental package to automatically extract test plugins for Home Assistant custom components + + :pypi:`pytest-honors` + *last release*: Mar 06, 2020, + *status*: 4 - Beta, + *requires*: N/A + + Report on tests that honor constraints, and guard against regressions + + :pypi:`pytest-hoverfly` + *last release*: Jul 12, 2021, + *status*: N/A, + *requires*: pytest (>=5.0) + + Simplify working with Hoverfly from pytest + + :pypi:`pytest-hoverfly-wrapper` + *last release*: Aug 29, 2021, + *status*: 4 - Beta, + *requires*: N/A + + Integrates the Hoverfly HTTP proxy into Pytest + + :pypi:`pytest-hpfeeds` + *last release*: Aug 27, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.2.4,<7.0.0) + + Helpers for testing hpfeeds in your python project + + :pypi:`pytest-html` + *last release*: Dec 13, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (!=6.0.0,>=5.0) + + pytest plugin for generating HTML reports + + :pypi:`pytest-html-lee` + *last release*: Jun 30, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.0) + + optimized pytest plugin for generating HTML reports + + :pypi:`pytest-html-profiling` + *last release*: Feb 11, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.0) + + Pytest plugin for generating HTML reports with per-test profiling and optionally call graph visualizations. Based on pytest-html by Dave Hunt. + + :pypi:`pytest-html-reporter` + *last release*: Apr 25, 2021, + *status*: N/A, + *requires*: N/A + + Generates a static html report based on pytest framework + + :pypi:`pytest-html-thread` + *last release*: Dec 29, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest plugin for generating HTML reports + + :pypi:`pytest-http` + *last release*: Dec 05, 2019, + *status*: N/A, + *requires*: N/A + + Fixture "http" for http requests + + :pypi:`pytest-httpbin` + *last release*: Feb 11, 2019, + *status*: 5 - Production/Stable, + *requires*: N/A + + Easily test your HTTP library against a local copy of httpbin + + :pypi:`pytest-http-mocker` + *last release*: Oct 20, 2019, + *status*: N/A, + *requires*: N/A + + Pytest plugin for http mocking (via https://github.com/vilus/mocker) + + :pypi:`pytest-httpretty` + *last release*: Feb 16, 2014, + *status*: 3 - Alpha, + *requires*: N/A + + A thin wrapper of HTTPretty for pytest + + :pypi:`pytest-httpserver` + *last release*: Oct 18, 2021, + *status*: 3 - Alpha, + *requires*: pytest ; extra == 'dev' + + pytest-httpserver is a httpserver for pytest + + :pypi:`pytest-httpx` + *last release*: Nov 16, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (==6.*) + + Send responses to httpx. + + :pypi:`pytest-httpx-blockage` + *last release*: Nov 16, 2021, + *status*: N/A, + *requires*: pytest (>=6.2.5) + + Disable httpx requests during a test run + + :pypi:`pytest-hue` + *last release*: May 09, 2019, + *status*: N/A, + *requires*: N/A + + Visualise PyTest status via your Phillips Hue lights + + :pypi:`pytest-hylang` + *last release*: Mar 28, 2021, + *status*: N/A, + *requires*: pytest + + Pytest plugin to allow running tests written in hylang + + :pypi:`pytest-hypo-25` + *last release*: Jan 12, 2020, + *status*: 3 - Alpha, + *requires*: N/A + + help hypo module for pytest + + :pypi:`pytest-ibutsu` + *last release*: Jun 16, 2021, + *status*: 4 - Beta, + *requires*: pytest + + A plugin to sent pytest results to an Ibutsu server + + :pypi:`pytest-icdiff` + *last release*: Apr 08, 2020, + *status*: 4 - Beta, + *requires*: N/A + + use icdiff for better error messages in pytest assertions + + :pypi:`pytest-idapro` + *last release*: Nov 03, 2018, + *status*: N/A, + *requires*: N/A + + A pytest plugin for idapython. Allows a pytest setup to run tests outside and inside IDA in an automated manner by runnig pytest inside IDA and by mocking idapython api + + :pypi:`pytest-idempotent` + *last release*: Nov 26, 2021, + *status*: N/A, + *requires*: N/A + + Pytest plugin for testing function idempotence. + + :pypi:`pytest-ignore-flaky` + *last release*: Apr 23, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + ignore failures from flaky tests (pytest plugin) + + :pypi:`pytest-image-diff` + *last release*: Jul 28, 2021, + *status*: 3 - Alpha, + *requires*: pytest + + + + :pypi:`pytest-incremental` + *last release*: Apr 24, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + an incremental test runner (pytest plugin) + + :pypi:`pytest-influxdb` + *last release*: Apr 20, 2021, + *status*: N/A, + *requires*: N/A + + Plugin for influxdb and pytest integration. + + :pypi:`pytest-info-collector` + *last release*: May 26, 2019, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin to collect information from tests + + :pypi:`pytest-informative-node` + *last release*: Apr 25, 2019, + *status*: 4 - Beta, + *requires*: N/A + + display more node ininformation. + + :pypi:`pytest-infrastructure` + *last release*: Apr 12, 2020, + *status*: 4 - Beta, + *requires*: N/A + + pytest stack validation prior to testing executing + + :pypi:`pytest-ini` + *last release*: Sep 30, 2021, + *status*: N/A, + *requires*: N/A + + Reuse pytest.ini to store env variables + + :pypi:`pytest-inmanta` + *last release*: Aug 17, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + A py.test plugin providing fixtures to simplify inmanta modules testing. + + :pypi:`pytest-inmanta-extensions` + *last release*: May 27, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + Inmanta tests package + + :pypi:`pytest-Inomaly` + *last release*: Feb 13, 2018, + *status*: 4 - Beta, + *requires*: N/A + + A simple image diff plugin for pytest + + :pypi:`pytest-insta` + *last release*: Apr 07, 2021, + *status*: N/A, + *requires*: pytest (>=6.0.2,<7.0.0) + + A practical snapshot testing plugin for pytest + + :pypi:`pytest-instafail` + *last release*: Jun 14, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=2.9) + + pytest plugin to show failures instantly + + :pypi:`pytest-instrument` + *last release*: Apr 05, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.1.0) + + pytest plugin to instrument tests + + :pypi:`pytest-integration` + *last release*: Apr 16, 2020, + *status*: N/A, + *requires*: N/A + + Organizing pytests by integration or not + + :pypi:`pytest-integration-mark` + *last release*: Jul 19, 2021, + *status*: N/A, + *requires*: pytest (>=5.2,<7.0) + + Automatic integration test marking and excluding plugin for pytest + + :pypi:`pytest-interactive` + *last release*: Nov 30, 2017, + *status*: 3 - Alpha, + *requires*: N/A + + A pytest plugin for console based interactive test selection just after the collection phase + + :pypi:`pytest-intercept-remote` + *last release*: May 24, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=4.6) + + Pytest plugin for intercepting outgoing connection requests during pytest run. + + :pypi:`pytest-invenio` + *last release*: May 11, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (<7,>=6) + + Pytest fixtures for Invenio. + + :pypi:`pytest-involve` + *last release*: Feb 02, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Run tests covering a specific file or changeset + + :pypi:`pytest-ipdb` + *last release*: Sep 02, 2014, + *status*: 2 - Pre-Alpha, + *requires*: N/A + + A py.test plug-in to enable drop to ipdb debugger on test failure. + + :pypi:`pytest-ipynb` + *last release*: Jan 29, 2019, + *status*: 3 - Alpha, + *requires*: N/A + + THIS PROJECT IS ABANDONED + + :pypi:`pytest-isort` + *last release*: Apr 27, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + py.test plugin to check import ordering using isort + + :pypi:`pytest-it` + *last release*: Jan 22, 2020, + *status*: 4 - Beta, + *requires*: N/A + + Pytest plugin to display test reports as a plaintext spec, inspired by Rspec: https://github.com/mattduck/pytest-it. + + :pypi:`pytest-iterassert` + *last release*: May 11, 2020, + *status*: 3 - Alpha, + *requires*: N/A + + Nicer list and iterable assertion messages for pytest + + :pypi:`pytest-jasmine` + *last release*: Nov 04, 2017, + *status*: 1 - Planning, + *requires*: N/A + + Run jasmine tests from your pytest test suite + + :pypi:`pytest-jest` + *last release*: May 22, 2018, + *status*: 4 - Beta, + *requires*: pytest (>=3.3.2) + + A custom jest-pytest oriented Pytest reporter + + :pypi:`pytest-jira` + *last release*: Dec 02, 2021, + *status*: 3 - Alpha, + *requires*: N/A + + py.test JIRA integration plugin, using markers + + :pypi:`pytest-jira-xray` + *last release*: Nov 28, 2021, + *status*: 3 - Alpha, + *requires*: pytest + + pytest plugin to integrate tests with JIRA XRAY + + :pypi:`pytest-jobserver` + *last release*: May 15, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + Limit parallel tests with posix jobserver. + + :pypi:`pytest-joke` + *last release*: Oct 08, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=4.2.1) + + Test failures are better served with humor. + + :pypi:`pytest-json` + *last release*: Jan 18, 2016, + *status*: 4 - Beta, + *requires*: N/A + + Generate JSON test reports + + :pypi:`pytest-jsonlint` + *last release*: Aug 04, 2016, + *status*: N/A, + *requires*: N/A + + UNKNOWN + + :pypi:`pytest-json-report` + *last release*: Sep 24, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.8.0) + + A pytest plugin to report test results as JSON files + + :pypi:`pytest-kafka` + *last release*: Aug 24, 2021, + *status*: N/A, + *requires*: pytest + + Zookeeper, Kafka server, and Kafka consumer fixtures for Pytest + + :pypi:`pytest-kafkavents` + *last release*: Sep 08, 2021, + *status*: 4 - Beta, + *requires*: pytest + + A plugin to send pytest events to Kafka + + :pypi:`pytest-kind` + *last release*: Jan 24, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + Kubernetes test support with KIND for pytest + + :pypi:`pytest-kivy` + *last release*: Jul 06, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.6) + + Kivy GUI tests fixtures using pytest + + :pypi:`pytest-knows` + *last release*: Aug 22, 2014, + *status*: N/A, + *requires*: N/A + + A pytest plugin that can automaticly skip test case based on dependence info calculated by trace + + :pypi:`pytest-konira` + *last release*: Oct 09, 2011, + *status*: N/A, + *requires*: N/A + + Run Konira DSL tests with py.test + + :pypi:`pytest-krtech-common` + *last release*: Nov 28, 2016, + *status*: 4 - Beta, + *requires*: N/A + + pytest krtech common library + + :pypi:`pytest-kwparametrize` + *last release*: Jan 22, 2021, + *status*: N/A, + *requires*: pytest (>=6) + + Alternate syntax for @pytest.mark.parametrize with test cases as dictionaries and default value fallbacks + + :pypi:`pytest-lambda` + *last release*: Aug 23, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=3.6,<7) + + Define pytest fixtures with lambda functions. + + :pypi:`pytest-lamp` + *last release*: Jan 06, 2017, + *status*: 3 - Alpha, + *requires*: N/A + + + + :pypi:`pytest-layab` + *last release*: Oct 05, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Pytest fixtures for layab. + + :pypi:`pytest-lazy-fixture` + *last release*: Feb 01, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.2.5) + + It helps to use fixtures in pytest.mark.parametrize + + :pypi:`pytest-ldap` + *last release*: Aug 18, 2020, + *status*: N/A, + *requires*: pytest + + python-ldap fixtures for pytest + + :pypi:`pytest-leaks` + *last release*: Nov 27, 2019, + *status*: 1 - Planning, + *requires*: N/A + + A pytest plugin to trace resource leaks. + + :pypi:`pytest-level` + *last release*: Oct 21, 2019, + *status*: N/A, + *requires*: pytest + + Select tests of a given level or lower + + :pypi:`pytest-libfaketime` + *last release*: Dec 22, 2018, + *status*: 4 - Beta, + *requires*: pytest (>=3.0.0) + + A python-libfaketime plugin for pytest. + + :pypi:`pytest-libiio` + *last release*: Oct 29, 2021, + *status*: 4 - Beta, + *requires*: N/A + + A pytest plugin to manage interfacing with libiio contexts + + :pypi:`pytest-libnotify` + *last release*: Apr 02, 2021, + *status*: 3 - Alpha, + *requires*: pytest + + Pytest plugin that shows notifications about the test run + + :pypi:`pytest-ligo` + *last release*: Jan 16, 2020, + *status*: 4 - Beta, + *requires*: N/A + + + + :pypi:`pytest-lineno` + *last release*: Dec 04, 2020, + *status*: N/A, + *requires*: pytest + + A pytest plugin to show the line numbers of test functions + + :pypi:`pytest-line-profiler` + *last release*: May 03, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Profile code executed by pytest + + :pypi:`pytest-lisa` + *last release*: Jan 21, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=6.1.2,<7.0.0) + + Pytest plugin for organizing tests. + + :pypi:`pytest-listener` + *last release*: May 28, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + A simple network listener + + :pypi:`pytest-litf` + *last release*: Jan 18, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.1.1) + + A pytest plugin that stream output in LITF format + + :pypi:`pytest-live` + *last release*: Mar 08, 2020, + *status*: N/A, + *requires*: pytest + + Live results for pytest + + :pypi:`pytest-localftpserver` + *last release*: Aug 25, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + A PyTest plugin which provides an FTP fixture for your tests + + :pypi:`pytest-localserver` + *last release*: Nov 19, 2021, + *status*: 4 - Beta, + *requires*: N/A + + py.test plugin to test server connections locally. + + :pypi:`pytest-localstack` + *last release*: Aug 22, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.3.0) + + Pytest plugin for AWS integration tests + + :pypi:`pytest-lockable` + *last release*: Nov 09, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + lockable resource plugin for pytest + + :pypi:`pytest-locker` + *last release*: Oct 29, 2021, + *status*: N/A, + *requires*: pytest (>=5.4) + + Used to lock object during testing. Essentially changing assertions from being hard coded to asserting that nothing changed + + :pypi:`pytest-log` + *last release*: Aug 15, 2021, + *status*: N/A, + *requires*: pytest (>=3.8) + + print log + + :pypi:`pytest-logbook` + *last release*: Nov 23, 2015, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.8) + + py.test plugin to capture logbook log messages + + :pypi:`pytest-logdog` + *last release*: Jun 15, 2021, + *status*: 1 - Planning, + *requires*: pytest (>=6.2.0) + + Pytest plugin to test logging + + :pypi:`pytest-logfest` + *last release*: Jul 21, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Pytest plugin providing three logger fixtures with basic or full writing to log files + + :pypi:`pytest-logger` + *last release*: Jul 25, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.2) + + Plugin configuring handlers for loggers from Python logging module. + + :pypi:`pytest-logging` + *last release*: Nov 04, 2015, + *status*: 4 - Beta, + *requires*: N/A + + Configures logging and allows tweaking the log level with a py.test flag + + :pypi:`pytest-log-report` + *last release*: Dec 26, 2019, + *status*: N/A, + *requires*: N/A + + Package for creating a pytest test run reprot + + :pypi:`pytest-manual-marker` + *last release*: Oct 11, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=6) + + pytest marker for marking manual tests + + :pypi:`pytest-markdown` + *last release*: Jan 15, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.0.1,<7.0.0) + + Test your markdown docs with pytest + + :pypi:`pytest-marker-bugzilla` + *last release*: Jan 09, 2020, + *status*: N/A, + *requires*: N/A + + py.test bugzilla integration plugin, using markers + + :pypi:`pytest-markers-presence` + *last release*: Feb 04, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.0) + + A simple plugin to detect missed pytest tags and markers" + + :pypi:`pytest-markfiltration` + *last release*: Nov 08, 2011, + *status*: 3 - Alpha, + *requires*: N/A + + UNKNOWN + + :pypi:`pytest-mark-no-py3` + *last release*: May 17, 2019, + *status*: N/A, + *requires*: pytest + + pytest plugin and bowler codemod to help migrate tests to Python 3 + + :pypi:`pytest-marks` + *last release*: Nov 23, 2012, + *status*: 3 - Alpha, + *requires*: N/A + + UNKNOWN + + :pypi:`pytest-matcher` + *last release*: Apr 23, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.4) + + Match test output against patterns stored in files + + :pypi:`pytest-match-skip` + *last release*: May 15, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=4.4.1) + + Skip matching marks. Matches partial marks using wildcards. + + :pypi:`pytest-mat-report` + *last release*: Jan 20, 2021, + *status*: N/A, + *requires*: N/A + + this is report + + :pypi:`pytest-matrix` + *last release*: Jun 24, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.4.3,<6.0.0) + + Provide tools for generating tests from combinations of fixtures. + + :pypi:`pytest-mccabe` + *last release*: Jul 22, 2020, + *status*: 3 - Alpha, + *requires*: pytest (>=5.4.0) + + pytest plugin to run the mccabe code complexity checker. + + :pypi:`pytest-md` + *last release*: Jul 11, 2019, + *status*: 3 - Alpha, + *requires*: pytest (>=4.2.1) + + Plugin for generating Markdown reports for pytest results + + :pypi:`pytest-md-report` + *last release*: May 04, 2021, + *status*: 4 - Beta, + *requires*: pytest (!=6.0.0,<7,>=3.3.2) + + A pytest plugin to make a test results report with Markdown table format. + + :pypi:`pytest-memprof` + *last release*: Mar 29, 2019, + *status*: 4 - Beta, + *requires*: N/A + + Estimates memory consumption of test functions + + :pypi:`pytest-menu` + *last release*: Oct 04, 2017, + *status*: 3 - Alpha, + *requires*: pytest (>=2.4.2) + + A pytest plugin for console based interactive test selection just after the collection phase + + :pypi:`pytest-mercurial` + *last release*: Nov 21, 2020, + *status*: 1 - Planning, + *requires*: N/A + + pytest plugin to write integration tests for projects using Mercurial Python internals + + :pypi:`pytest-message` + *last release*: Nov 04, 2021, + *status*: N/A, + *requires*: pytest (>=6.2.5) + + Pytest plugin for sending report message of marked tests execution + + :pypi:`pytest-messenger` + *last release*: Dec 16, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Pytest to Slack reporting plugin + + :pypi:`pytest-metadata` + *last release*: Nov 27, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.9.0) + + pytest plugin for test session metadata + + :pypi:`pytest-metrics` + *last release*: Apr 04, 2020, + *status*: N/A, + *requires*: pytest + + Custom metrics report for pytest + + :pypi:`pytest-mimesis` + *last release*: Mar 21, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.2) + + Mimesis integration with the pytest test runner + + :pypi:`pytest-minecraft` + *last release*: Sep 26, 2020, + *status*: N/A, + *requires*: pytest (>=6.0.1,<7.0.0) + + A pytest plugin for running tests against Minecraft releases + + :pypi:`pytest-missing-fixtures` + *last release*: Oct 14, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Pytest plugin that creates missing fixtures + + :pypi:`pytest-ml` + *last release*: May 04, 2019, + *status*: 4 - Beta, + *requires*: N/A + + Test your machine learning! + + :pypi:`pytest-mocha` + *last release*: Apr 02, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=5.4.0) + + pytest plugin to display test execution output like a mochajs + + :pypi:`pytest-mock` + *last release*: May 06, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.0) + + Thin-wrapper around the mock package for easier use with pytest + + :pypi:`pytest-mock-api` + *last release*: Feb 13, 2019, + *status*: 1 - Planning, + *requires*: pytest (>=4.0.0) + + A mock API server with configurable routes and responses available as a fixture. + + :pypi:`pytest-mock-generator` + *last release*: Aug 10, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + A pytest fixture wrapper for https://pypi.org/project/mock-generator + + :pypi:`pytest-mock-helper` + *last release*: Jan 24, 2018, + *status*: N/A, + *requires*: pytest + + Help you mock HTTP call and generate mock code + + :pypi:`pytest-mockito` + *last release*: Jul 11, 2018, + *status*: 4 - Beta, + *requires*: N/A + + Base fixtures for mockito + + :pypi:`pytest-mockredis` + *last release*: Jan 02, 2018, + *status*: 2 - Pre-Alpha, + *requires*: N/A + + An in-memory mock of a Redis server that runs in a separate thread. This is to be used for unit-tests that require a Redis database. + + :pypi:`pytest-mock-resources` + *last release*: Dec 03, 2021, + *status*: N/A, + *requires*: pytest (>=1.0) + + A pytest plugin for easily instantiating reproducible mock resources. + + :pypi:`pytest-mock-server` + *last release*: Apr 06, 2020, + *status*: 4 - Beta, + *requires*: N/A + + Mock server plugin for pytest + + :pypi:`pytest-mockservers` + *last release*: Mar 31, 2020, + *status*: N/A, + *requires*: pytest (>=4.3.0) + + A set of fixtures to test your requests to HTTP/UDP servers + + :pypi:`pytest-modifyjunit` + *last release*: Jan 10, 2019, + *status*: N/A, + *requires*: N/A + + Utility for adding additional properties to junit xml for IDM QE + + :pypi:`pytest-modifyscope` + *last release*: Apr 12, 2020, + *status*: N/A, + *requires*: pytest + + pytest plugin to modify fixture scope + + :pypi:`pytest-molecule` + *last release*: Oct 06, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + PyTest Molecule Plugin :: discover and run molecule tests + + :pypi:`pytest-mongo` + *last release*: Jun 07, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + MongoDB process and client fixtures plugin for Pytest. + + :pypi:`pytest-mongodb` + *last release*: Dec 07, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.5.2) + + pytest plugin for MongoDB fixtures + + :pypi:`pytest-monitor` + *last release*: Aug 24, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + Pytest plugin for analyzing resource usage. + + :pypi:`pytest-monkeyplus` + *last release*: Sep 18, 2012, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest's monkeypatch subclass with extra functionalities + + :pypi:`pytest-monkeytype` + *last release*: Jul 29, 2020, + *status*: 4 - Beta, + *requires*: N/A + + pytest-monkeytype: Generate Monkeytype annotations from your pytest tests. + + :pypi:`pytest-moto` + *last release*: Aug 28, 2015, + *status*: 1 - Planning, + *requires*: N/A + + Fixtures for integration tests of AWS services,uses moto mocking library. + + :pypi:`pytest-motor` + *last release*: Jul 21, 2021, + *status*: 3 - Alpha, + *requires*: pytest + + A pytest plugin for motor, the non-blocking MongoDB driver. + + :pypi:`pytest-mp` + *last release*: May 23, 2018, + *status*: 4 - Beta, + *requires*: pytest + + A test batcher for multiprocessed Pytest runs + + :pypi:`pytest-mpi` + *last release*: Mar 14, 2021, + *status*: 3 - Alpha, + *requires*: pytest + + pytest plugin to collect information from tests + + :pypi:`pytest-mpl` + *last release*: Jul 02, 2021, + *status*: 4 - Beta, + *requires*: pytest + + pytest plugin to help with testing figures output from Matplotlib + + :pypi:`pytest-mproc` + *last release*: Mar 07, 2021, + *status*: 4 - Beta, + *requires*: pytest + + low-startup-overhead, scalable, distributed-testing pytest plugin + + :pypi:`pytest-multi-check` + *last release*: Jun 03, 2021, + *status*: N/A, + *requires*: pytest + + Pytest-плагин, реализует возможность мульти проверок и мягких проверок + + :pypi:`pytest-multihost` + *last release*: Apr 07, 2020, + *status*: 4 - Beta, + *requires*: N/A + + Utility for writing multi-host tests for pytest + + :pypi:`pytest-multilog` + *last release*: Jun 10, 2021, + *status*: N/A, + *requires*: N/A + + Multi-process logs handling and other helpers for pytest + + :pypi:`pytest-multithreading` + *last release*: Aug 12, 2021, + *status*: N/A, + *requires*: pytest (>=3.6) + + a pytest plugin for th and concurrent testing + + :pypi:`pytest-mutagen` + *last release*: Jul 24, 2020, + *status*: N/A, + *requires*: pytest (>=5.4) + + Add the mutation testing feature to pytest + + :pypi:`pytest-mypy` + *last release*: Mar 21, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5) + + Mypy static type checker plugin for Pytest + + :pypi:`pytest-mypyd` + *last release*: Aug 20, 2019, + *status*: 4 - Beta, + *requires*: pytest (<4.7,>=2.8) ; python_version < "3.5" + + Mypy static type checker plugin for Pytest + + :pypi:`pytest-mypy-plugins` + *last release*: Oct 19, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=6.0.0) + + pytest plugin for writing tests for mypy plugins + + :pypi:`pytest-mypy-plugins-shim` + *last release*: Apr 12, 2021, + *status*: N/A, + *requires*: N/A + + Substitute for "pytest-mypy-plugins" for Python implementations which aren't supported by mypy. + + :pypi:`pytest-mypy-testing` + *last release*: Jun 13, 2021, + *status*: N/A, + *requires*: pytest + + Pytest plugin to check mypy output. + + :pypi:`pytest-mysql` + *last release*: Nov 22, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + MySQL process and client fixtures for pytest + + :pypi:`pytest-needle` + *last release*: Dec 10, 2018, + *status*: 4 - Beta, + *requires*: pytest (<5.0.0,>=3.0.0) + + pytest plugin for visual testing websites using selenium + + :pypi:`pytest-neo` + *last release*: Apr 23, 2019, + *status*: 3 - Alpha, + *requires*: pytest (>=3.7.2) + + pytest-neo is a plugin for pytest that shows tests like screen of Matrix. + + :pypi:`pytest-network` + *last release*: May 07, 2020, + *status*: N/A, + *requires*: N/A + + A simple plugin to disable network on socket level. + + :pypi:`pytest-never-sleep` + *last release*: May 05, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=3.5.1) + + pytest plugin helps to avoid adding tests without mock \`time.sleep\` + + :pypi:`pytest-nginx` + *last release*: Aug 12, 2017, + *status*: 5 - Production/Stable, + *requires*: N/A + + nginx fixture for pytest + + :pypi:`pytest-nginx-iplweb` + *last release*: Mar 01, 2019, + *status*: 5 - Production/Stable, + *requires*: N/A + + nginx fixture for pytest - iplweb temporary fork + + :pypi:`pytest-ngrok` + *last release*: Jan 22, 2020, + *status*: 3 - Alpha, + *requires*: N/A + + + + :pypi:`pytest-ngsfixtures` + *last release*: Sep 06, 2019, + *status*: 2 - Pre-Alpha, + *requires*: pytest (>=5.0.0) + + pytest ngs fixtures + + :pypi:`pytest-nice` + *last release*: May 04, 2019, + *status*: 4 - Beta, + *requires*: pytest + + A pytest plugin that alerts user of failed test cases with screen notifications + + :pypi:`pytest-nice-parametrize` + *last release*: Apr 17, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + A small snippet for nicer PyTest's Parametrize + + :pypi:`pytest-nlcov` + *last release*: Jul 07, 2021, + *status*: N/A, + *requires*: N/A + + Pytest plugin to get the coverage of the new lines (based on git diff) only + + :pypi:`pytest-nocustom` + *last release*: Jul 07, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + Run all tests without custom markers + + :pypi:`pytest-nodev` + *last release*: Jul 21, 2016, + *status*: 4 - Beta, + *requires*: pytest (>=2.8.1) + + Test-driven source code search for Python. + + :pypi:`pytest-nogarbage` + *last release*: Aug 29, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.6.0) + + Ensure a test produces no garbage + + :pypi:`pytest-notebook` + *last release*: Sep 16, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A pytest plugin for testing Jupyter Notebooks + + :pypi:`pytest-notice` + *last release*: Nov 05, 2020, + *status*: N/A, + *requires*: N/A + + Send pytest execution result email + + :pypi:`pytest-notification` + *last release*: Jun 19, 2020, + *status*: N/A, + *requires*: pytest (>=4) + + A pytest plugin for sending a desktop notification and playing a sound upon completion of tests + + :pypi:`pytest-notifier` + *last release*: Jun 12, 2020, + *status*: 3 - Alpha, + *requires*: pytest + + A pytest plugin to notify test result + + :pypi:`pytest-notimplemented` + *last release*: Aug 27, 2019, + *status*: N/A, + *requires*: pytest (>=5.1,<6.0) + + Pytest markers for not implemented features and tests. + + :pypi:`pytest-notion` + *last release*: Aug 07, 2019, + *status*: N/A, + *requires*: N/A + + A PyTest Reporter to send test runs to Notion.so + + :pypi:`pytest-nunit` + *last release*: Aug 04, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A pytest plugin for generating NUnit3 test result XML output + + :pypi:`pytest-ochrus` + *last release*: Feb 21, 2018, + *status*: 4 - Beta, + *requires*: N/A + + pytest results data-base and HTML reporter + + :pypi:`pytest-odoo` + *last release*: Nov 04, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=2.9) + + py.test plugin to run Odoo tests + + :pypi:`pytest-odoo-fixtures` + *last release*: Jun 25, 2019, + *status*: N/A, + *requires*: N/A + + Project description + + :pypi:`pytest-oerp` + *last release*: Feb 28, 2012, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin to test OpenERP modules + + :pypi:`pytest-ok` + *last release*: Apr 01, 2019, + *status*: 4 - Beta, + *requires*: N/A + + The ultimate pytest output plugin + + :pypi:`pytest-only` + *last release*: Jan 19, 2020, + *status*: N/A, + *requires*: N/A + + Use @pytest.mark.only to run a single test + + :pypi:`pytest-oot` + *last release*: Sep 18, 2016, + *status*: 4 - Beta, + *requires*: N/A + + Run object-oriented tests in a simple format + + :pypi:`pytest-openfiles` + *last release*: Apr 16, 2020, + *status*: 3 - Alpha, + *requires*: pytest (>=4.6) + + Pytest plugin for detecting inadvertent open file handles + + :pypi:`pytest-opentmi` + *last release*: Nov 04, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.0) + + pytest plugin for publish results to opentmi + + :pypi:`pytest-operator` + *last release*: Oct 26, 2021, + *status*: N/A, + *requires*: N/A + + Fixtures for Operators + + :pypi:`pytest-optional` + *last release*: Oct 07, 2015, + *status*: N/A, + *requires*: N/A + + include/exclude values of fixtures in pytest + + :pypi:`pytest-optional-tests` + *last release*: Jul 09, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=4.5.0) + + Easy declaration of optional tests (i.e., that are not run by default) + + :pypi:`pytest-orchestration` + *last release*: Jul 18, 2019, + *status*: N/A, + *requires*: N/A + + A pytest plugin for orchestrating tests + + :pypi:`pytest-order` + *last release*: May 30, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=5.0) + + pytest plugin to run your tests in a specific order + + :pypi:`pytest-ordering` + *last release*: Nov 14, 2018, + *status*: 4 - Beta, + *requires*: pytest + + pytest plugin to run your tests in a specific order + + :pypi:`pytest-osxnotify` + *last release*: May 15, 2015, + *status*: N/A, + *requires*: N/A + + OS X notifications for py.test results. + + :pypi:`pytest-otel` + *last release*: Dec 03, 2021, + *status*: N/A, + *requires*: N/A + + pytest-otel report OpenTelemetry traces about test executed + + :pypi:`pytest-pact` + *last release*: Jan 07, 2019, + *status*: 4 - Beta, + *requires*: N/A + + A simple plugin to use with pytest + + :pypi:`pytest-pahrametahrize` + *last release*: Nov 24, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.0,<7.0) + + Parametrize your tests with a Boston accent. + + :pypi:`pytest-parallel` + *last release*: Oct 10, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=3.0.0) + + a pytest plugin for parallel and concurrent testing + + :pypi:`pytest-parallel-39` + *last release*: Jul 12, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=3.0.0) + + a pytest plugin for parallel and concurrent testing + + :pypi:`pytest-param` + *last release*: Sep 11, 2016, + *status*: 4 - Beta, + *requires*: pytest (>=2.6.0) + + pytest plugin to test all, first, last or random params + + :pypi:`pytest-paramark` + *last release*: Jan 10, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=4.5.0) + + Configure pytest fixtures using a combination of"parametrize" and markers + + :pypi:`pytest-parametrization` + *last release*: Nov 30, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + Simpler PyTest parametrization + + :pypi:`pytest-parametrize-cases` + *last release*: Dec 12, 2020, + *status*: N/A, + *requires*: pytest (>=6.1.2,<7.0.0) + + A more user-friendly way to write parametrized tests. + + :pypi:`pytest-parametrized` + *last release*: Oct 19, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest + + Pytest plugin for parametrizing tests with default iterables. + + :pypi:`pytest-parawtf` + *last release*: Dec 03, 2018, + *status*: 4 - Beta, + *requires*: pytest (>=3.6.0) + + Finally spell paramete?ri[sz]e correctly + + :pypi:`pytest-pass` + *last release*: Dec 04, 2019, + *status*: N/A, + *requires*: N/A + + Check out https://github.com/elilutsky/pytest-pass + + :pypi:`pytest-passrunner` + *last release*: Feb 10, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.6.0) + + Pytest plugin providing the 'run_on_pass' marker + + :pypi:`pytest-paste-config` + *last release*: Sep 18, 2013, + *status*: 3 - Alpha, + *requires*: N/A + + Allow setting the path to a paste config file + + :pypi:`pytest-patches` + *last release*: Aug 30, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A contextmanager pytest fixture for handling multiple mock patches + + :pypi:`pytest-pdb` + *last release*: Jul 31, 2018, + *status*: N/A, + *requires*: N/A + + pytest plugin which adds pdb helper commands related to pytest. + + :pypi:`pytest-peach` + *last release*: Apr 12, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=2.8.7) + + pytest plugin for fuzzing with Peach API Security + + :pypi:`pytest-pep257` + *last release*: Jul 09, 2016, + *status*: N/A, + *requires*: N/A + + py.test plugin for pep257 + + :pypi:`pytest-pep8` + *last release*: Apr 27, 2014, + *status*: N/A, + *requires*: N/A + + pytest plugin to check PEP8 requirements + + :pypi:`pytest-percent` + *last release*: May 21, 2020, + *status*: N/A, + *requires*: pytest (>=5.2.0) + + Change the exit code of pytest test sessions when a required percent of tests pass. + + :pypi:`pytest-perf` + *last release*: Jun 27, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.6) ; extra == 'testing' + + pytest-perf + + :pypi:`pytest-performance` + *last release*: Sep 11, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.7.0) + + A simple plugin to ensure the execution of critical sections of code has not been impacted + + :pypi:`pytest-persistence` + *last release*: Nov 06, 2021, + *status*: N/A, + *requires*: N/A + + Pytest tool for persistent objects + + :pypi:`pytest-pgsql` + *last release*: May 13, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.0.0) + + Pytest plugins and helpers for tests using a Postgres database. + + :pypi:`pytest-phmdoctest` + *last release*: Nov 10, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.2) ; extra == 'test' + + pytest plugin to test Python examples in Markdown using phmdoctest. + + :pypi:`pytest-picked` + *last release*: Dec 23, 2020, + *status*: N/A, + *requires*: pytest (>=3.5.0) + + Run the tests related to the changed files + + :pypi:`pytest-pigeonhole` + *last release*: Jun 25, 2018, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.4) + + + + :pypi:`pytest-pikachu` + *last release*: Aug 05, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + Show surprise when tests are passing + + :pypi:`pytest-pilot` + *last release*: Oct 09, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Slice in your test base thanks to powerful markers. + + :pypi:`pytest-pings` + *last release*: Jun 29, 2019, + *status*: 3 - Alpha, + *requires*: pytest (>=5.0.0) + + 🦊 The pytest plugin for Firefox Telemetry 📊 + + :pypi:`pytest-pinned` + *last release*: Sep 17, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A simple pytest plugin for pinning tests + + :pypi:`pytest-pinpoint` + *last release*: Sep 25, 2020, + *status*: N/A, + *requires*: pytest (>=4.4.0) + + A pytest plugin which runs SBFL algorithms to detect faults. + + :pypi:`pytest-pipeline` + *last release*: Jan 24, 2017, + *status*: 3 - Alpha, + *requires*: N/A + + Pytest plugin for functional testing of data analysispipelines + + :pypi:`pytest-platform-markers` + *last release*: Sep 09, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.6.0) + + Markers for pytest to skip tests on specific platforms + + :pypi:`pytest-play` + *last release*: Jun 12, 2019, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest plugin that let you automate actions and assertions with test metrics reporting executing plain YAML files + + :pypi:`pytest-playbook` + *last release*: Jan 21, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=6.1.2,<7.0.0) + + Pytest plugin for reading playbooks. + + :pypi:`pytest-playwright` + *last release*: Oct 28, 2021, + *status*: N/A, + *requires*: pytest + + A pytest wrapper with fixtures for Playwright to automate web browsers + + :pypi:`pytest-playwrights` + *last release*: Dec 02, 2021, + *status*: N/A, + *requires*: N/A + + A pytest wrapper with fixtures for Playwright to automate web browsers + + :pypi:`pytest-playwright-snapshot` + *last release*: Aug 19, 2021, + *status*: N/A, + *requires*: N/A + + A pytest wrapper for snapshot testing with playwright + + :pypi:`pytest-plt` + *last release*: Aug 17, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest + + Fixtures for quickly making Matplotlib plots in tests + + :pypi:`pytest-plugin-helpers` + *last release*: Nov 23, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A plugin to help developing and testing other plugins + + :pypi:`pytest-plus` + *last release*: Mar 19, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.50) + + PyTest Plus Plugin :: extends pytest functionality + + :pypi:`pytest-pmisc` + *last release*: Mar 21, 2019, + *status*: 5 - Production/Stable, + *requires*: N/A + + + + :pypi:`pytest-pointers` + *last release*: Oct 14, 2021, + *status*: N/A, + *requires*: N/A + + Pytest plugin to define functions you test with special marks for better navigation and reports + + :pypi:`pytest-polarion-cfme` + *last release*: Nov 13, 2017, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin for collecting test cases and recording test results + + :pypi:`pytest-polarion-collect` + *last release*: Jun 18, 2020, + *status*: 3 - Alpha, + *requires*: pytest + + pytest plugin for collecting polarion test cases data + + :pypi:`pytest-polecat` + *last release*: Aug 12, 2019, + *status*: 4 - Beta, + *requires*: N/A + + Provides Polecat pytest fixtures + + :pypi:`pytest-ponyorm` + *last release*: Oct 31, 2018, + *status*: N/A, + *requires*: pytest (>=3.1.1) + + PonyORM in Pytest + + :pypi:`pytest-poo` + *last release*: Mar 25, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.3.4) + + Visualize your crappy tests + + :pypi:`pytest-poo-fail` + *last release*: Feb 12, 2015, + *status*: 5 - Production/Stable, + *requires*: N/A + + Visualize your failed tests with poo + + :pypi:`pytest-pop` + *last release*: Aug 19, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + A pytest plugin to help with testing pop projects + + :pypi:`pytest-portion` + *last release*: Jan 28, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Select a portion of the collected tests + + :pypi:`pytest-postgres` + *last release*: Mar 22, 2020, + *status*: N/A, + *requires*: pytest + + Run PostgreSQL in Docker container in Pytest. + + :pypi:`pytest-postgresql` + *last release*: Nov 05, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.0.0) + + Postgresql fixtures and fixture factories for Pytest. + + :pypi:`pytest-power` + *last release*: Dec 31, 2020, + *status*: N/A, + *requires*: pytest (>=5.4) + + pytest plugin with powerful fixtures + + :pypi:`pytest-pretty-terminal` + *last release*: Nov 24, 2021, + *status*: N/A, + *requires*: pytest (>=3.4.1) + + pytest plugin for generating prettier terminal output + + :pypi:`pytest-pride` + *last release*: Apr 02, 2016, + *status*: 3 - Alpha, + *requires*: N/A + + Minitest-style test colors + + :pypi:`pytest-print` + *last release*: Jun 17, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=6) + + pytest-print adds the printer fixture you can use to print messages to the user (directly to the pytest runner, not stdout) + + :pypi:`pytest-profiling` + *last release*: May 28, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + Profiling plugin for py.test + + :pypi:`pytest-progress` + *last release*: Nov 09, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.7) + + pytest plugin for instant test progress status + + :pypi:`pytest-prometheus` + *last release*: Oct 03, 2017, + *status*: N/A, + *requires*: N/A + + Report test pass / failures to a Prometheus PushGateway + + :pypi:`pytest-prosper` + *last release*: Sep 24, 2018, + *status*: N/A, + *requires*: N/A + + Test helpers for Prosper projects + + :pypi:`pytest-pspec` + *last release*: Jun 02, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.0.0) + + A rspec format reporter for Python ptest + + :pypi:`pytest-psqlgraph` + *last release*: Oct 19, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.0) + + pytest plugin for testing applications that use psqlgraph + + :pypi:`pytest-ptera` + *last release*: Oct 20, 2021, + *status*: N/A, + *requires*: pytest (>=6.2.4,<7.0.0) + + Use ptera probes in tests + + :pypi:`pytest-pudb` + *last release*: Oct 25, 2018, + *status*: 3 - Alpha, + *requires*: pytest (>=2.0) + + Pytest PuDB debugger integration + + :pypi:`pytest-purkinje` + *last release*: Oct 28, 2017, + *status*: 2 - Pre-Alpha, + *requires*: N/A + + py.test plugin for purkinje test runner + + :pypi:`pytest-pycharm` + *last release*: Aug 13, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.3) + + Plugin for py.test to enter PyCharm debugger on uncaught exceptions + + :pypi:`pytest-pycodestyle` + *last release*: Aug 10, 2020, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin to run pycodestyle + + :pypi:`pytest-pydev` + *last release*: Nov 15, 2017, + *status*: 3 - Alpha, + *requires*: N/A + + py.test plugin to connect to a remote debug server with PyDev or PyCharm. + + :pypi:`pytest-pydocstyle` + *last release*: Aug 10, 2020, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin to run pydocstyle + + :pypi:`pytest-pylint` + *last release*: Nov 09, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.4) + + pytest plugin to check source code with pylint + + :pypi:`pytest-pypi` + *last release*: Mar 04, 2018, + *status*: 3 - Alpha, + *requires*: N/A + + Easily test your HTTP library against a local copy of pypi + + :pypi:`pytest-pypom-navigation` + *last release*: Feb 18, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.0.7) + + Core engine for cookiecutter-qa and pytest-play packages + + :pypi:`pytest-pyppeteer` + *last release*: Feb 16, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.0.2) + + A plugin to run pyppeteer in pytest. + + :pypi:`pytest-pyq` + *last release*: Mar 10, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Pytest fixture "q" for pyq + + :pypi:`pytest-pyramid` + *last release*: Oct 15, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + pytest_pyramid - provides fixtures for testing pyramid applications with pytest test suite + + :pypi:`pytest-pyramid-server` + *last release*: May 28, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + Pyramid server fixture for py.test + + :pypi:`pytest-pyright` + *last release*: Aug 16, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Pytest plugin for type checking code with Pyright + + :pypi:`pytest-pytestrail` + *last release*: Aug 27, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.8.0) + + Pytest plugin for interaction with TestRail + + :pypi:`pytest-pythonpath` + *last release*: Aug 22, 2018, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest plugin for adding to the PYTHONPATH from command line or configs. + + :pypi:`pytest-pytorch` + *last release*: May 25, 2021, + *status*: 4 - Beta, + *requires*: pytest + + pytest plugin for a better developer experience when working with the PyTorch test suite + + :pypi:`pytest-qasync` + *last release*: Jul 12, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=5.4.0) + + Pytest support for qasync. + + :pypi:`pytest-qatouch` + *last release*: Jun 26, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.2.0) + + Pytest plugin for uploading test results to your QA Touch Testrun. + + :pypi:`pytest-qgis` + *last release*: Nov 25, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=6.2.3) + + A pytest plugin for testing QGIS python plugins + + :pypi:`pytest-qml` + *last release*: Dec 02, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=6.0.0) + + Run QML Tests with pytest + + :pypi:`pytest-qr` + *last release*: Nov 25, 2021, + *status*: 4 - Beta, + *requires*: N/A + + pytest plugin to generate test result QR codes + + :pypi:`pytest-qt` + *last release*: Jun 13, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.0.0) + + pytest support for PyQt and PySide applications + + :pypi:`pytest-qt-app` + *last release*: Dec 23, 2015, + *status*: 5 - Production/Stable, + *requires*: N/A + + QT app fixture for py.test + + :pypi:`pytest-quarantine` + *last release*: Nov 24, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.6) + + A plugin for pytest to manage expected test failures + + :pypi:`pytest-quickcheck` + *last release*: Nov 15, 2020, + *status*: 4 - Beta, + *requires*: pytest (<6.0.0,>=4.0) + + pytest plugin to generate random data inspired by QuickCheck + + :pypi:`pytest-rabbitmq` + *last release*: Jun 02, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.0.0) + + RabbitMQ process and client fixtures for pytest + + :pypi:`pytest-race` + *last release*: Nov 21, 2016, + *status*: 4 - Beta, + *requires*: N/A + + Race conditions tester for pytest + + :pypi:`pytest-rage` + *last release*: Oct 21, 2011, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin to implement PEP712 + + :pypi:`pytest-railflow-testrail-reporter` + *last release*: Dec 02, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + Generate json reports along with specified metadata defined in test markers. + + :pypi:`pytest-raises` + *last release*: Apr 23, 2020, + *status*: N/A, + *requires*: pytest (>=3.2.2) + + An implementation of pytest.raises as a pytest.mark fixture + + :pypi:`pytest-raisesregexp` + *last release*: Dec 18, 2015, + *status*: N/A, + *requires*: N/A + + Simple pytest plugin to look for regex in Exceptions + + :pypi:`pytest-raisin` + *last release*: Jun 25, 2020, + *status*: N/A, + *requires*: pytest + + Plugin enabling the use of exception instances with pytest.raises + + :pypi:`pytest-random` + *last release*: Apr 28, 2013, + *status*: 3 - Alpha, + *requires*: N/A + + py.test plugin to randomize tests + + :pypi:`pytest-randomly` + *last release*: Nov 30, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + Pytest plugin to randomly order tests and control random.seed. + + :pypi:`pytest-randomness` + *last release*: May 30, 2019, + *status*: 3 - Alpha, + *requires*: N/A + + Pytest plugin about random seed management + + :pypi:`pytest-random-num` + *last release*: Oct 19, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Randomise the order in which pytest tests are run with some control over the randomness + + :pypi:`pytest-random-order` + *last release*: Nov 30, 2018, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.0.0) + + Randomise the order in which pytest tests are run with some control over the randomness + + :pypi:`pytest-readme` + *last release*: Dec 28, 2014, + *status*: 5 - Production/Stable, + *requires*: N/A + + Test your README.md file + + :pypi:`pytest-reana` + *last release*: Nov 22, 2021, + *status*: 3 - Alpha, + *requires*: N/A + + Pytest fixtures for REANA. + + :pypi:`pytest-recording` + *last release*: Jul 08, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A pytest plugin that allows you recording of network interactions via VCR.py + + :pypi:`pytest-recordings` + *last release*: Aug 13, 2020, + *status*: N/A, + *requires*: N/A + + Provides pytest plugins for reporting request/response traffic, screenshots, and more to ReportPortal + + :pypi:`pytest-redis` + *last release*: Nov 03, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + Redis fixtures and fixture factories for Pytest. + + :pypi:`pytest-redislite` + *last release*: Sep 19, 2021, + *status*: 4 - Beta, + *requires*: pytest + + Pytest plugin for testing code using Redis + + :pypi:`pytest-redmine` + *last release*: Mar 19, 2018, + *status*: 1 - Planning, + *requires*: N/A + + Pytest plugin for redmine + + :pypi:`pytest-ref` + *last release*: Nov 23, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A plugin to store reference files to ease regression testing + + :pypi:`pytest-reference-formatter` + *last release*: Oct 01, 2019, + *status*: 4 - Beta, + *requires*: N/A + + Conveniently run pytest with a dot-formatted test reference. + + :pypi:`pytest-regressions` + *last release*: Jan 27, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.5.0) + + Easy to use fixtures to write regression tests. + + :pypi:`pytest-regtest` + *last release*: Jun 03, 2021, + *status*: N/A, + *requires*: N/A + + pytest plugin for regression tests + + :pypi:`pytest-relative-order` + *last release*: May 17, 2021, + *status*: 4 - Beta, + *requires*: N/A + + a pytest plugin that sorts tests using "before" and "after" markers + + :pypi:`pytest-relaxed` + *last release*: Jun 14, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest (<5,>=3) + + Relaxed test discovery/organization for pytest + + :pypi:`pytest-remfiles` + *last release*: Jul 01, 2019, + *status*: 5 - Production/Stable, + *requires*: N/A + + Pytest plugin to create a temporary directory with remote files + + :pypi:`pytest-remotedata` + *last release*: Jul 20, 2019, + *status*: 3 - Alpha, + *requires*: pytest (>=3.1) + + Pytest plugin for controlling remote data access. + + :pypi:`pytest-remote-response` + *last release*: Jun 30, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=4.6) + + Pytest plugin for capturing and mocking connection requests. + + :pypi:`pytest-remove-stale-bytecode` + *last release*: Mar 04, 2020, + *status*: 4 - Beta, + *requires*: pytest + + py.test plugin to remove stale byte code files. + + :pypi:`pytest-reorder` + *last release*: May 31, 2018, + *status*: 4 - Beta, + *requires*: pytest + + Reorder tests depending on their paths and names. + + :pypi:`pytest-repeat` + *last release*: Oct 31, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.6) + + pytest plugin for repeating tests + + :pypi:`pytest-replay` + *last release*: Jun 09, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.0.0) + + Saves previous test runs and allow re-execute previous pytest runs to reproduce crashes or flaky tests + + :pypi:`pytest-repo-health` + *last release*: Nov 23, 2021, + *status*: 3 - Alpha, + *requires*: pytest + + A pytest plugin to report on repository standards conformance + + :pypi:`pytest-report` + *last release*: May 11, 2016, + *status*: 4 - Beta, + *requires*: N/A + + Creates json report that is compatible with atom.io's linter message format + + :pypi:`pytest-reporter` + *last release*: Jul 22, 2021, + *status*: 4 - Beta, + *requires*: pytest + + Generate Pytest reports with templates + + :pypi:`pytest-reporter-html1` + *last release*: Jun 08, 2021, + *status*: 4 - Beta, + *requires*: N/A + + A basic HTML report template for Pytest + + :pypi:`pytest-reportinfra` + *last release*: Aug 11, 2019, + *status*: 3 - Alpha, + *requires*: N/A + + Pytest plugin for reportinfra + + :pypi:`pytest-reporting` + *last release*: Oct 25, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A plugin to report summarized results in a table format + + :pypi:`pytest-reportlog` + *last release*: Dec 11, 2020, + *status*: 3 - Alpha, + *requires*: pytest (>=5.2) + + Replacement for the --resultlog option, focused in simplicity and extensibility + + :pypi:`pytest-report-me` + *last release*: Dec 31, 2020, + *status*: N/A, + *requires*: pytest + + A pytest plugin to generate report. + + :pypi:`pytest-report-parameters` + *last release*: Jun 18, 2020, + *status*: 3 - Alpha, + *requires*: pytest (>=2.4.2) + + pytest plugin for adding tests' parameters to junit report + + :pypi:`pytest-reportportal` + *last release*: Jun 18, 2021, + *status*: N/A, + *requires*: pytest (>=3.8.0) + + Agent for Reporting results of tests to the Report Portal + + :pypi:`pytest-reqs` + *last release*: May 12, 2019, + *status*: N/A, + *requires*: pytest (>=2.4.2) + + pytest plugin to check pinned requirements + + :pypi:`pytest-requests` + *last release*: Jun 24, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A simple plugin to use with pytest + + :pypi:`pytest-reraise` + *last release*: Jun 17, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.6) + + Make multi-threaded pytest test cases fail when they should + + :pypi:`pytest-rerun` + *last release*: Jul 08, 2019, + *status*: N/A, + *requires*: pytest (>=3.6) + + Re-run only changed files in specified branch + + :pypi:`pytest-rerunfailures` + *last release*: Sep 17, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.3) + + pytest plugin to re-run tests to eliminate flaky failures + + :pypi:`pytest-resilient-circuits` + *last release*: Nov 15, 2021, + *status*: N/A, + *requires*: N/A + + Resilient Circuits fixtures for PyTest. + + :pypi:`pytest-resource` + *last release*: Nov 14, 2018, + *status*: 4 - Beta, + *requires*: N/A + + Load resource fixture plugin to use with pytest + + :pypi:`pytest-resource-path` + *last release*: May 01, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.5.0) + + Provides path for uniform access to test resources in isolated directory + + :pypi:`pytest-responsemock` + *last release*: Oct 10, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Simplified requests calls mocking for pytest + + :pypi:`pytest-responses` + *last release*: Apr 26, 2021, + *status*: N/A, + *requires*: pytest (>=2.5) + + py.test integration for responses + + :pypi:`pytest-restrict` + *last release*: Aug 12, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + Pytest plugin to restrict the test types allowed + + :pypi:`pytest-rethinkdb` + *last release*: Jul 24, 2016, + *status*: 4 - Beta, + *requires*: N/A + + A RethinkDB plugin for pytest. + + :pypi:`pytest-reverse` + *last release*: Aug 12, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + Pytest plugin to reverse test order. + + :pypi:`pytest-ringo` + *last release*: Sep 27, 2017, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin to test webapplications using the Ringo webframework + + :pypi:`pytest-rng` + *last release*: Aug 08, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + Fixtures for seeding tests and making randomness reproducible + + :pypi:`pytest-roast` + *last release*: Jul 29, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + pytest plugin for ROAST configuration override and fixtures + + :pypi:`pytest-rocketchat` + *last release*: Apr 18, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + Pytest to Rocket.Chat reporting plugin + + :pypi:`pytest-rotest` + *last release*: Sep 08, 2019, + *status*: N/A, + *requires*: pytest (>=3.5.0) + + Pytest integration with rotest + + :pypi:`pytest-rpc` + *last release*: Feb 22, 2019, + *status*: 4 - Beta, + *requires*: pytest (~=3.6) + + Extend py.test for RPC OpenStack testing. + + :pypi:`pytest-rst` + *last release*: Sep 21, 2021, + *status*: N/A, + *requires*: pytest + + Test code from RST documents with pytest + + :pypi:`pytest-rt` + *last release*: Sep 04, 2021, + *status*: N/A, + *requires*: N/A + + pytest data collector plugin for Testgr + + :pypi:`pytest-rts` + *last release*: May 17, 2021, + *status*: N/A, + *requires*: pytest + + Coverage-based regression test selection (RTS) plugin for pytest + + :pypi:`pytest-run-changed` + *last release*: Apr 02, 2021, + *status*: 3 - Alpha, + *requires*: pytest + + Pytest plugin that runs changed tests only + + :pypi:`pytest-runfailed` + *last release*: Mar 24, 2016, + *status*: N/A, + *requires*: N/A + + implement a --failed option for pytest + + :pypi:`pytest-runner` + *last release*: May 19, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.6) ; extra == 'testing' + + Invoke py.test as distutils command with dependency resolution + + :pypi:`pytest-runtime-xfail` + *last release*: Aug 26, 2021, + *status*: N/A, + *requires*: N/A + + Call runtime_xfail() to mark running test as xfail. + + :pypi:`pytest-salt` + *last release*: Jan 27, 2020, + *status*: 4 - Beta, + *requires*: N/A + + Pytest Salt Plugin + + :pypi:`pytest-salt-containers` + *last release*: Nov 09, 2016, + *status*: 4 - Beta, + *requires*: N/A + + A Pytest plugin that builds and creates docker containers + + :pypi:`pytest-salt-factories` + *last release*: Sep 16, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.0.0) + + Pytest Salt Plugin + + :pypi:`pytest-salt-from-filenames` + *last release*: Jan 29, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=4.1) + + Simple PyTest Plugin For Salt's Test Suite Specifically + + :pypi:`pytest-salt-runtests-bridge` + *last release*: Dec 05, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=4.1) + + Simple PyTest Plugin For Salt's Test Suite Specifically + + :pypi:`pytest-sanic` + *last release*: Oct 25, 2021, + *status*: N/A, + *requires*: pytest (>=5.2) + + a pytest plugin for Sanic + + :pypi:`pytest-sanity` + *last release*: Dec 07, 2020, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-sa-pg` + *last release*: May 14, 2019, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-sbase` + *last release*: Dec 03, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + A complete web automation framework for end-to-end testing. + + :pypi:`pytest-scenario` + *last release*: Feb 06, 2017, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin for test scenarios + + :pypi:`pytest-schema` + *last release*: Aug 31, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.5.0) + + 👍 Validate return values against a schema-like object in testing + + :pypi:`pytest-securestore` + *last release*: Nov 08, 2021, + *status*: 4 - Beta, + *requires*: N/A + + An encrypted password store for use within pytest cases + + :pypi:`pytest-select` + *last release*: Jan 18, 2019, + *status*: 3 - Alpha, + *requires*: pytest (>=3.0) + + A pytest plugin which allows to (de-)select tests from a file. + + :pypi:`pytest-selenium` + *last release*: Sep 19, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.0.0) + + pytest plugin for Selenium + + :pypi:`pytest-seleniumbase` + *last release*: Dec 03, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + A complete web automation framework for end-to-end testing. + + :pypi:`pytest-selenium-enhancer` + *last release*: Nov 26, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest plugin for Selenium + + :pypi:`pytest-selenium-pdiff` + *last release*: Apr 06, 2017, + *status*: 2 - Pre-Alpha, + *requires*: N/A + + A pytest package implementing perceptualdiff for Selenium tests. + + :pypi:`pytest-send-email` + *last release*: Dec 04, 2019, + *status*: N/A, + *requires*: N/A + + Send pytest execution result email + + :pypi:`pytest-sentry` + *last release*: Apr 21, 2021, + *status*: N/A, + *requires*: pytest + + A pytest plugin to send testrun information to Sentry.io + + :pypi:`pytest-server-fixtures` + *last release*: May 28, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + Extensible server fixures for py.test + + :pypi:`pytest-serverless` + *last release*: Nov 27, 2021, + *status*: 4 - Beta, + *requires*: N/A + + Automatically mocks resources from serverless.yml in pytest using moto. + + :pypi:`pytest-services` + *last release*: Oct 30, 2020, + *status*: 6 - Mature, + *requires*: N/A + + Services plugin for pytest testing framework + + :pypi:`pytest-session2file` + *last release*: Jan 26, 2021, + *status*: 3 - Alpha, + *requires*: pytest + + pytest-session2file (aka: pytest-session_to_file for v0.1.0 - v0.1.2) is a py.test plugin for capturing and saving to file the stdout of py.test. + + :pypi:`pytest-session-fixture-globalize` + *last release*: May 15, 2018, + *status*: 4 - Beta, + *requires*: N/A + + py.test plugin to make session fixtures behave as if written in conftest, even if it is written in some modules + + :pypi:`pytest-session_to_file` + *last release*: Oct 01, 2015, + *status*: 3 - Alpha, + *requires*: N/A + + pytest-session_to_file is a py.test plugin for capturing and saving to file the stdout of py.test. + + :pypi:`pytest-sftpserver` + *last release*: Sep 16, 2019, + *status*: 4 - Beta, + *requires*: N/A + + py.test plugin to locally test sftp server connections. + + :pypi:`pytest-shard` + *last release*: Dec 11, 2020, + *status*: 4 - Beta, + *requires*: pytest + + + + :pypi:`pytest-shell` + *last release*: Nov 07, 2021, + *status*: N/A, + *requires*: N/A + + A pytest plugin to help with testing shell scripts / black box commands + + :pypi:`pytest-sheraf` + *last release*: Feb 11, 2020, + *status*: N/A, + *requires*: pytest + + Versatile ZODB abstraction layer - pytest fixtures + + :pypi:`pytest-sherlock` + *last release*: Nov 18, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.5.1) + + pytest plugin help to find coupled tests + + :pypi:`pytest-shortcuts` + *last release*: Oct 29, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Expand command-line shortcuts listed in pytest configuration + + :pypi:`pytest-shutil` + *last release*: May 28, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + A goodie-bag of unix shell and environment tools for py.test + + :pypi:`pytest-simplehttpserver` + *last release*: Jun 24, 2021, + *status*: 4 - Beta, + *requires*: N/A + + Simple pytest fixture to spin up an HTTP server + + :pypi:`pytest-simple-plugin` + *last release*: Nov 27, 2019, + *status*: N/A, + *requires*: N/A + + Simple pytest plugin + + :pypi:`pytest-simple-settings` + *last release*: Nov 17, 2020, + *status*: 4 - Beta, + *requires*: pytest + + simple-settings plugin for pytest + + :pypi:`pytest-single-file-logging` + *last release*: May 05, 2016, + *status*: 4 - Beta, + *requires*: pytest (>=2.8.1) + + Allow for multiple processes to log to a single file + + :pypi:`pytest-skip-markers` + *last release*: Oct 04, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.0.0) + + Pytest Salt Plugin + + :pypi:`pytest-skipper` + *last release*: Mar 26, 2017, + *status*: 3 - Alpha, + *requires*: pytest (>=3.0.6) + + A plugin that selects only tests with changes in execution path + + :pypi:`pytest-skippy` + *last release*: Jan 27, 2018, + *status*: 3 - Alpha, + *requires*: pytest (>=2.3.4) + + Automatically skip tests that don't need to run! + + :pypi:`pytest-skip-slow` + *last release*: Sep 28, 2021, + *status*: N/A, + *requires*: N/A + + A pytest plugin to skip \`@pytest.mark.slow\` tests by default. + + :pypi:`pytest-slack` + *last release*: Dec 15, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Pytest to Slack reporting plugin + + :pypi:`pytest-slow` + *last release*: Sep 28, 2021, + *status*: N/A, + *requires*: N/A + + A pytest plugin to skip \`@pytest.mark.slow\` tests by default. + + :pypi:`pytest-smartcollect` + *last release*: Oct 04, 2018, + *status*: N/A, + *requires*: pytest (>=3.5.0) + + A plugin for collecting tests that touch changed code + + :pypi:`pytest-smartcov` + *last release*: Sep 30, 2017, + *status*: 3 - Alpha, + *requires*: N/A + + Smart coverage plugin for pytest. + + :pypi:`pytest-smtp` + *last release*: Feb 20, 2021, + *status*: N/A, + *requires*: pytest + + Send email with pytest execution result + + :pypi:`pytest-snail` + *last release*: Nov 04, 2019, + *status*: 3 - Alpha, + *requires*: pytest (>=5.0.1) + + Plugin for adding a marker to slow running tests. 🐌 + + :pypi:`pytest-snapci` + *last release*: Nov 12, 2015, + *status*: N/A, + *requires*: N/A + + py.test plugin for Snap-CI + + :pypi:`pytest-snapshot` + *last release*: Dec 02, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.0.0) + + A plugin for snapshot testing with pytest. + + :pypi:`pytest-snmpserver` + *last release*: May 12, 2021, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-socket` + *last release*: Aug 28, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.6.3) + + Pytest Plugin to disable socket calls during tests + + :pypi:`pytest-soft-assertions` + *last release*: May 05, 2020, + *status*: 3 - Alpha, + *requires*: pytest + + + + :pypi:`pytest-solr` + *last release*: May 11, 2020, + *status*: 3 - Alpha, + *requires*: pytest (>=3.0.0) + + Solr process and client fixtures for py.test. + + :pypi:`pytest-sorter` + *last release*: Apr 20, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.1.1) + + A simple plugin to first execute tests that historically failed more + + :pypi:`pytest-sourceorder` + *last release*: Sep 01, 2021, + *status*: 4 - Beta, + *requires*: pytest + + Test-ordering plugin for pytest + + :pypi:`pytest-spark` + *last release*: Feb 23, 2020, + *status*: 4 - Beta, + *requires*: pytest + + pytest plugin to run the tests with support of pyspark. + + :pypi:`pytest-spawner` + *last release*: Jul 31, 2015, + *status*: 4 - Beta, + *requires*: N/A + + py.test plugin to spawn process and communicate with them. + + :pypi:`pytest-spec` + *last release*: May 04, 2021, + *status*: N/A, + *requires*: N/A + + Library pytest-spec is a pytest plugin to display test execution output like a SPECIFICATION. + + :pypi:`pytest-sphinx` + *last release*: Aug 05, 2020, + *status*: 4 - Beta, + *requires*: N/A + + Doctest plugin for pytest with support for Sphinx-specific doctest-directives + + :pypi:`pytest-spiratest` + *last release*: Oct 13, 2021, + *status*: N/A, + *requires*: N/A + + Exports unit tests as test runs in SpiraTest/Team/Plan + + :pypi:`pytest-splinter` + *last release*: Dec 25, 2020, + *status*: 6 - Mature, + *requires*: N/A + + Splinter plugin for pytest testing framework + + :pypi:`pytest-split` + *last release*: Nov 09, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=5,<7) + + Pytest plugin which splits the test suite to equally sized sub suites based on test execution time. + + :pypi:`pytest-splitio` + *last release*: Sep 22, 2020, + *status*: N/A, + *requires*: pytest (<7,>=5.0) + + Split.io SDK integration for e2e tests + + :pypi:`pytest-split-tests` + *last release*: Jul 30, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.5) + + A Pytest plugin for running a subset of your tests by splitting them in to equally sized groups. Forked from Mark Adams' original project pytest-test-groups. + + :pypi:`pytest-split-tests-tresorit` + *last release*: Feb 22, 2021, + *status*: 1 - Planning, + *requires*: N/A + + + + :pypi:`pytest-splunk-addon` + *last release*: Nov 29, 2021, + *status*: N/A, + *requires*: pytest (>5.4.0,<6.3) + + A Dynamic test tool for Splunk Apps and Add-ons + + :pypi:`pytest-splunk-addon-ui-smartx` + *last release*: Oct 07, 2021, + *status*: N/A, + *requires*: N/A + + Library to support testing Splunk Add-on UX + + :pypi:`pytest-splunk-env` + *last release*: Oct 22, 2020, + *status*: N/A, + *requires*: pytest (>=6.1.1,<7.0.0) + + pytest fixtures for interaction with Splunk Enterprise and Splunk Cloud + + :pypi:`pytest-sqitch` + *last release*: Apr 06, 2020, + *status*: 4 - Beta, + *requires*: N/A + + sqitch for pytest + + :pypi:`pytest-sqlalchemy` + *last release*: Mar 13, 2018, + *status*: 3 - Alpha, + *requires*: N/A + + pytest plugin with sqlalchemy related fixtures + + :pypi:`pytest-sql-bigquery` + *last release*: Dec 19, 2019, + *status*: N/A, + *requires*: pytest + + Yet another SQL-testing framework for BigQuery provided by pytest plugin + + :pypi:`pytest-srcpaths` + *last release*: Oct 15, 2021, + *status*: N/A, + *requires*: N/A + + Add paths to sys.path + + :pypi:`pytest-ssh` + *last release*: May 27, 2019, + *status*: N/A, + *requires*: pytest + + pytest plugin for ssh command run + + :pypi:`pytest-start-from` + *last release*: Apr 11, 2016, + *status*: N/A, + *requires*: N/A + + Start pytest run from a given point + + :pypi:`pytest-statsd` + *last release*: Nov 30, 2018, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.0.0) + + pytest plugin for reporting to graphite + + :pypi:`pytest-stepfunctions` + *last release*: May 08, 2021, + *status*: 4 - Beta, + *requires*: pytest + + A small description + + :pypi:`pytest-steps` + *last release*: Sep 23, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + Create step-wise / incremental tests in pytest. + + :pypi:`pytest-stepwise` + *last release*: Dec 01, 2015, + *status*: 4 - Beta, + *requires*: N/A + + Run a test suite one failing test at a time. + + :pypi:`pytest-stoq` + *last release*: Feb 09, 2021, + *status*: 4 - Beta, + *requires*: N/A + + A plugin to pytest stoq + + :pypi:`pytest-stress` + *last release*: Dec 07, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.6.0) + + A Pytest plugin that allows you to loop tests for a user defined amount of time. + + :pypi:`pytest-structlog` + *last release*: Sep 21, 2021, + *status*: N/A, + *requires*: pytest + + Structured logging assertions + + :pypi:`pytest-structmpd` + *last release*: Oct 17, 2018, + *status*: N/A, + *requires*: N/A + + provide structured temporary directory + + :pypi:`pytest-stub` + *last release*: Apr 28, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Stub packages, modules and attributes. + + :pypi:`pytest-stubprocess` + *last release*: Sep 17, 2018, + *status*: 3 - Alpha, + *requires*: pytest (>=3.5.0) + + Provide stub implementations for subprocesses in Python tests + + :pypi:`pytest-study` + *last release*: Sep 26, 2017, + *status*: 3 - Alpha, + *requires*: pytest (>=2.0) + + A pytest plugin to organize long run tests (named studies) without interfering the regular tests + + :pypi:`pytest-subprocess` + *last release*: Nov 07, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.0.0) + + A plugin to fake subprocess for pytest + + :pypi:`pytest-subtesthack` + *last release*: Mar 02, 2021, + *status*: N/A, + *requires*: N/A + + A hack to explicitly set up and tear down fixtures. + + :pypi:`pytest-subtests` + *last release*: May 29, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=5.3.0) + + unittest subTest() support and subtests fixture + + :pypi:`pytest-subunit` + *last release*: Aug 29, 2017, + *status*: N/A, + *requires*: N/A + + pytest-subunit is a plugin for py.test which outputs testsresult in subunit format. + + :pypi:`pytest-sugar` + *last release*: Jul 06, 2020, + *status*: 3 - Alpha, + *requires*: N/A + + pytest-sugar is a plugin for pytest that changes the default look and feel of pytest (e.g. progressbar, show tests that fail instantly). + + :pypi:`pytest-sugar-bugfix159` + *last release*: Nov 07, 2018, + *status*: 5 - Production/Stable, + *requires*: pytest (!=3.7.3,>=3.5); extra == 'testing' + + Workaround for https://github.com/Frozenball/pytest-sugar/issues/159 + + :pypi:`pytest-super-check` + *last release*: Aug 12, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + Pytest plugin to check your TestCase classes call super in setUp, tearDown, etc. + + :pypi:`pytest-svn` + *last release*: May 28, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + SVN repository fixture for py.test + + :pypi:`pytest-symbols` + *last release*: Nov 20, 2017, + *status*: 3 - Alpha, + *requires*: N/A + + pytest-symbols is a pytest plugin that adds support for passing test environment symbols into pytest tests. + + :pypi:`pytest-takeltest` + *last release*: Oct 13, 2021, + *status*: N/A, + *requires*: N/A + + Fixtures for ansible, testinfra and molecule + + :pypi:`pytest-talisker` + *last release*: Nov 28, 2021, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-tap` + *last release*: Oct 27, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.0) + + Test Anything Protocol (TAP) reporting plugin for pytest + + :pypi:`pytest-tape` + *last release*: Mar 17, 2021, + *status*: 4 - Beta, + *requires*: N/A + + easy assertion with expected results saved to yaml files + + :pypi:`pytest-target` + *last release*: Jan 21, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=6.1.2,<7.0.0) + + Pytest plugin for remote target orchestration. + + :pypi:`pytest-tblineinfo` + *last release*: Dec 01, 2015, + *status*: 3 - Alpha, + *requires*: pytest (>=2.0) + + tblineinfo is a py.test plugin that insert the node id in the final py.test report when --tb=line option is used + + :pypi:`pytest-teamcity-logblock` + *last release*: May 15, 2018, + *status*: 4 - Beta, + *requires*: N/A + + py.test plugin to introduce block structure in teamcity build log, if output is not captured + + :pypi:`pytest-telegram` + *last release*: Dec 10, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Pytest to Telegram reporting plugin + + :pypi:`pytest-tempdir` + *last release*: Oct 11, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=2.8.1) + + Predictable and repeatable tempdir support. + + :pypi:`pytest-terraform` + *last release*: Nov 10, 2021, + *status*: N/A, + *requires*: pytest (>=6.0) + + A pytest plugin for using terraform fixtures + + :pypi:`pytest-terraform-fixture` + *last release*: Nov 14, 2018, + *status*: 4 - Beta, + *requires*: N/A + + generate terraform resources to use with pytest + + :pypi:`pytest-testbook` + *last release*: Dec 11, 2016, + *status*: 3 - Alpha, + *requires*: N/A + + A plugin to run tests written in Jupyter notebook + + :pypi:`pytest-testconfig` + *last release*: Jan 11, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Test configuration plugin for pytest. + + :pypi:`pytest-testdirectory` + *last release*: Nov 06, 2018, + *status*: 5 - Production/Stable, + *requires*: pytest + + A py.test plugin providing temporary directories in unit tests. + + :pypi:`pytest-testdox` + *last release*: Oct 13, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.7.0) + + A testdox format reporter for pytest + + :pypi:`pytest-test-groups` + *last release*: Oct 25, 2016, + *status*: 5 - Production/Stable, + *requires*: N/A + + A Pytest plugin for running a subset of your tests by splitting them in to equally sized groups. + + :pypi:`pytest-testinfra` + *last release*: Jun 20, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (!=3.0.2) + + Test infrastructures + + :pypi:`pytest-testlink-adaptor` + *last release*: Dec 20, 2018, + *status*: 4 - Beta, + *requires*: pytest (>=2.6) + + pytest reporting plugin for testlink + + :pypi:`pytest-testmon` + *last release*: Oct 22, 2021, + *status*: 4 - Beta, + *requires*: N/A + + selects tests affected by changed files and methods + + :pypi:`pytest-testobject` + *last release*: Sep 24, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.1.1) + + Plugin to use TestObject Suites with Pytest + + :pypi:`pytest-testrail` + *last release*: Aug 27, 2020, + *status*: N/A, + *requires*: pytest (>=3.6) + + pytest plugin for creating TestRail runs and adding results + + :pypi:`pytest-testrail2` + *last release*: Nov 17, 2020, + *status*: N/A, + *requires*: pytest (>=5) + + A small example package + + :pypi:`pytest-testrail-api` + *last release*: Nov 30, 2021, + *status*: N/A, + *requires*: pytest (>=5.5) + + Плагин Pytest, для интеграции с TestRail + + :pypi:`pytest-testrail-api-client` + *last release*: Dec 03, 2021, + *status*: N/A, + *requires*: pytest + + TestRail Api Python Client + + :pypi:`pytest-testrail-appetize` + *last release*: Sep 29, 2021, + *status*: N/A, + *requires*: N/A + + pytest plugin for creating TestRail runs and adding results + + :pypi:`pytest-testrail-client` + *last release*: Sep 29, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest plugin for Testrail + + :pypi:`pytest-testrail-e2e` + *last release*: Oct 11, 2021, + *status*: N/A, + *requires*: pytest (>=3.6) + + pytest plugin for creating TestRail runs and adding results + + :pypi:`pytest-testrail-ns` + *last release*: Oct 08, 2021, + *status*: N/A, + *requires*: pytest (>=3.6) + + pytest plugin for creating TestRail runs and adding results + + :pypi:`pytest-testrail-plugin` + *last release*: Apr 21, 2020, + *status*: 3 - Alpha, + *requires*: pytest + + PyTest plugin for TestRail + + :pypi:`pytest-testrail-reporter` + *last release*: Sep 10, 2018, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-testreport` + *last release*: Nov 12, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + + + :pypi:`pytest-testslide` + *last release*: Jan 07, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (~=6.2) + + TestSlide fixture for pytest + + :pypi:`pytest-test-this` + *last release*: Sep 15, 2019, + *status*: 2 - Pre-Alpha, + *requires*: pytest (>=2.3) + + Plugin for py.test to run relevant tests, based on naively checking if a test contains a reference to the symbol you supply + + :pypi:`pytest-test-utils` + *last release*: Nov 30, 2021, + *status*: N/A, + *requires*: pytest (>=5) + + + + :pypi:`pytest-tesults` + *last release*: Jul 31, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.5.0) + + Tesults plugin for pytest + + :pypi:`pytest-tezos` + *last release*: Jan 16, 2020, + *status*: 4 - Beta, + *requires*: N/A + + pytest-ligo + + :pypi:`pytest-thawgun` + *last release*: May 26, 2020, + *status*: 3 - Alpha, + *requires*: N/A + + Pytest plugin for time travel + + :pypi:`pytest-threadleak` + *last release*: Sep 08, 2017, + *status*: 4 - Beta, + *requires*: N/A + + Detects thread leaks + + :pypi:`pytest-tick` + *last release*: Aug 31, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=6.2.5,<7.0.0) + + Ticking on tests + + :pypi:`pytest-timeit` + *last release*: Oct 13, 2016, + *status*: 4 - Beta, + *requires*: N/A + + A pytest plugin to time test function runs + + :pypi:`pytest-timeout` + *last release*: Oct 11, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.0.0) + + pytest plugin to abort hanging tests + + :pypi:`pytest-timeouts` + *last release*: Sep 21, 2019, + *status*: 5 - Production/Stable, + *requires*: N/A + + Linux-only Pytest plugin to control durations of various test case execution phases + + :pypi:`pytest-timer` + *last release*: Jun 02, 2021, + *status*: N/A, + *requires*: N/A + + A timer plugin for pytest + + :pypi:`pytest-timestamper` + *last release*: Jun 06, 2021, + *status*: N/A, + *requires*: N/A + + Pytest plugin to add a timestamp prefix to the pytest output + + :pypi:`pytest-tipsi-django` + *last release*: Nov 17, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.0.0) + + + + :pypi:`pytest-tipsi-testing` + *last release*: Nov 04, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.3.0) + + Better fixtures management. Various helpers + + :pypi:`pytest-tldr` + *last release*: Mar 12, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A pytest plugin that limits the output to just the things you need. + + :pypi:`pytest-tm4j-reporter` + *last release*: Sep 01, 2020, + *status*: N/A, + *requires*: pytest + + Cloud Jira Test Management (TM4J) PyTest reporter plugin + + :pypi:`pytest-tmreport` + *last release*: Nov 17, 2021, + *status*: N/A, + *requires*: N/A + + this is a vue-element ui report for pytest + + :pypi:`pytest-todo` + *last release*: May 23, 2019, + *status*: 4 - Beta, + *requires*: pytest + + A small plugin for the pytest testing framework, marking TODO comments as failure + + :pypi:`pytest-tomato` + *last release*: Mar 01, 2019, + *status*: 5 - Production/Stable, + *requires*: N/A + + + + :pypi:`pytest-toolbelt` + *last release*: Aug 12, 2019, + *status*: 3 - Alpha, + *requires*: N/A + + This is just a collection of utilities for pytest, but don't really belong in pytest proper. + + :pypi:`pytest-toolbox` + *last release*: Apr 07, 2018, + *status*: N/A, + *requires*: pytest (>=3.5.0) + + Numerous useful plugins for pytest. + + :pypi:`pytest-tornado` + *last release*: Jun 17, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.6) + + A py.test plugin providing fixtures and markers to simplify testing of asynchronous tornado applications. + + :pypi:`pytest-tornado5` + *last release*: Nov 16, 2018, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.6) + + A py.test plugin providing fixtures and markers to simplify testing of asynchronous tornado applications. + + :pypi:`pytest-tornado-yen3` + *last release*: Oct 15, 2018, + *status*: 5 - Production/Stable, + *requires*: N/A + + A py.test plugin providing fixtures and markers to simplify testing of asynchronous tornado applications. + + :pypi:`pytest-tornasync` + *last release*: Jul 15, 2019, + *status*: 3 - Alpha, + *requires*: pytest (>=3.0) + + py.test plugin for testing Python 3.5+ Tornado code + + :pypi:`pytest-track` + *last release*: Feb 26, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=3.0) + + + + :pypi:`pytest-translations` + *last release*: Nov 05, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + Test your translation files. + + :pypi:`pytest-travis-fold` + *last release*: Nov 29, 2017, + *status*: 4 - Beta, + *requires*: pytest (>=2.6.0) + + Folds captured output sections in Travis CI build log + + :pypi:`pytest-trello` + *last release*: Nov 20, 2015, + *status*: 5 - Production/Stable, + *requires*: N/A + + Plugin for py.test that integrates trello using markers + + :pypi:`pytest-trepan` + *last release*: Jul 28, 2018, + *status*: 5 - Production/Stable, + *requires*: N/A + + Pytest plugin for trepan debugger. + + :pypi:`pytest-trialtemp` + *last release*: Jun 08, 2015, + *status*: N/A, + *requires*: N/A + + py.test plugin for using the same _trial_temp working directory as trial + + :pypi:`pytest-trio` + *last release*: Oct 16, 2020, + *status*: N/A, + *requires*: N/A + + Pytest plugin for trio + + :pypi:`pytest-tspwplib` + *last release*: Jan 08, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + A simple plugin to use with tspwplib + + :pypi:`pytest-tstcls` + *last release*: Mar 23, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Test Class Base + + :pypi:`pytest-twisted` + *last release*: Aug 30, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.3) + + A twisted plugin for pytest. + + :pypi:`pytest-typhoon-xray` + *last release*: Nov 03, 2021, + *status*: 4 - Beta, + *requires*: N/A + + Typhoon HIL plugin for pytest + + :pypi:`pytest-tytest` + *last release*: May 25, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=5.4.2) + + Typhoon HIL plugin for pytest + + :pypi:`pytest-ubersmith` + *last release*: Apr 13, 2015, + *status*: N/A, + *requires*: N/A + + Easily mock calls to ubersmith at the \`requests\` level. + + :pypi:`pytest-ui` + *last release*: Jul 05, 2021, + *status*: 4 - Beta, + *requires*: pytest + + Text User Interface for running python tests + + :pypi:`pytest-unhandled-exception-exit-code` + *last release*: Jun 22, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.3) + + Plugin for py.test set a different exit code on uncaught exceptions + + :pypi:`pytest-unittest-filter` + *last release*: Jan 12, 2019, + *status*: 4 - Beta, + *requires*: pytest (>=3.1.0) + + A pytest plugin for filtering unittest-based test classes + + :pypi:`pytest-unmarked` + *last release*: Aug 27, 2019, + *status*: 5 - Production/Stable, + *requires*: N/A + + Run only unmarked tests + + :pypi:`pytest-unordered` + *last release*: Mar 28, 2021, + *status*: 4 - Beta, + *requires*: N/A + + Test equality of unordered collections in pytest + + :pypi:`pytest-upload-report` + *last release*: Jun 18, 2021, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest-upload-report is a plugin for pytest that upload your test report for test results. + + :pypi:`pytest-utils` + *last release*: Dec 04, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.2.5,<7.0.0) + + Some helpers for pytest. + + :pypi:`pytest-vagrant` + *last release*: Sep 07, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest + + A py.test plugin providing access to vagrant. + + :pypi:`pytest-valgrind` + *last release*: May 19, 2021, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-variables` + *last release*: Oct 23, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest (>=2.4.2) + + pytest plugin for providing variables to tests/fixtures + + :pypi:`pytest-variant` + *last release*: Jun 20, 2021, + *status*: N/A, + *requires*: N/A + + Variant support for Pytest + + :pypi:`pytest-vcr` + *last release*: Apr 26, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest (>=3.6.0) + + Plugin for managing VCR.py cassettes + + :pypi:`pytest-vcr-delete-on-fail` + *last release*: Aug 13, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.2.2,<7.0.0) + + A pytest plugin that automates vcrpy cassettes deletion on test failure. + + :pypi:`pytest-vcrpandas` + *last release*: Jan 12, 2019, + *status*: 4 - Beta, + *requires*: pytest + + Test from HTTP interactions to dataframe processed. + + :pypi:`pytest-venv` + *last release*: Aug 04, 2020, + *status*: 4 - Beta, + *requires*: pytest + + py.test fixture for creating a virtual environment + + :pypi:`pytest-ver` + *last release*: Aug 30, 2021, + *status*: 2 - Pre-Alpha, + *requires*: N/A + + Pytest module with Verification Report + + :pypi:`pytest-verbose-parametrize` + *last release*: May 28, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + More descriptive output for parametrized py.test tests + + :pypi:`pytest-vimqf` + *last release*: Feb 08, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=6.2.2,<7.0.0) + + A simple pytest plugin that will shrink pytest output when specified, to fit vim quickfix window. + + :pypi:`pytest-virtualenv` + *last release*: May 28, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + Virtualenv fixture for py.test + + :pypi:`pytest-voluptuous` + *last release*: Jun 09, 2020, + *status*: N/A, + *requires*: pytest + + Pytest plugin for asserting data against voluptuous schema. + + :pypi:`pytest-vscodedebug` + *last release*: Dec 04, 2020, + *status*: 4 - Beta, + *requires*: N/A + + A pytest plugin to easily enable debugging tests within Visual Studio Code + + :pypi:`pytest-vts` + *last release*: Jun 05, 2019, + *status*: N/A, + *requires*: pytest (>=2.3) + + pytest plugin for automatic recording of http stubbed tests + + :pypi:`pytest-vw` + *last release*: Oct 07, 2015, + *status*: 4 - Beta, + *requires*: N/A + + pytest-vw makes your failing test cases succeed under CI tools scrutiny + + :pypi:`pytest-vyper` + *last release*: May 28, 2020, + *status*: 2 - Pre-Alpha, + *requires*: N/A + + Plugin for the vyper smart contract language. + + :pypi:`pytest-wa-e2e-plugin` + *last release*: Feb 18, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.5.0) + + Pytest plugin for testing whatsapp bots with end to end tests + + :pypi:`pytest-watch` + *last release*: May 20, 2018, + *status*: N/A, + *requires*: N/A + + Local continuous test runner with pytest and watchdog. + + :pypi:`pytest-watcher` + *last release*: Sep 18, 2021, + *status*: 3 - Alpha, + *requires*: N/A + + Continiously runs pytest on changes in \*.py files + + :pypi:`pytest-wdl` + *last release*: Nov 17, 2020, + *status*: 5 - Production/Stable, + *requires*: N/A + + Pytest plugin for testing WDL workflows. + + :pypi:`pytest-webdriver` + *last release*: May 28, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest + + Selenium webdriver fixture for py.test + + :pypi:`pytest-wetest` + *last release*: Nov 10, 2018, + *status*: 4 - Beta, + *requires*: N/A + + Welian API Automation test framework pytest plugin + + :pypi:`pytest-whirlwind` + *last release*: Jun 12, 2020, + *status*: N/A, + *requires*: N/A + + Testing Tornado. + + :pypi:`pytest-wholenodeid` + *last release*: Aug 26, 2015, + *status*: 4 - Beta, + *requires*: pytest (>=2.0) + + pytest addon for displaying the whole node id for failures + + :pypi:`pytest-win32consoletitle` + *last release*: Aug 08, 2021, + *status*: N/A, + *requires*: N/A + + Pytest progress in console title (Win32 only) + + :pypi:`pytest-winnotify` + *last release*: Apr 22, 2016, + *status*: N/A, + *requires*: N/A + + Windows tray notifications for py.test results. + + :pypi:`pytest-with-docker` + *last release*: Nov 09, 2021, + *status*: N/A, + *requires*: pytest + + pytest with docker helpers. + + :pypi:`pytest-workflow` + *last release*: Dec 03, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.4.0) + + A pytest plugin for configuring workflow/pipeline tests using YAML files + + :pypi:`pytest-xdist` + *last release*: Sep 21, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=6.0.0) + + pytest xdist plugin for distributed testing and loop-on-failing modes + + :pypi:`pytest-xdist-debug-for-graingert` + *last release*: Jul 24, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.4.0) + + pytest xdist plugin for distributed testing and loop-on-failing modes + + :pypi:`pytest-xdist-forked` + *last release*: Feb 10, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.4.0) + + forked from pytest-xdist + + :pypi:`pytest-xdist-tracker` + *last release*: Nov 18, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=3.5.1) + + pytest plugin helps to reproduce failures for particular xdist node + + :pypi:`pytest-xfaillist` + *last release*: Sep 17, 2021, + *status*: N/A, + *requires*: pytest (>=6.2.2,<7.0.0) + + Maintain a xfaillist in an additional file to avoid merge-conflicts. + + :pypi:`pytest-xfiles` + *last release*: Feb 27, 2018, + *status*: N/A, + *requires*: N/A + + Pytest fixtures providing data read from function, module or package related (x)files. + + :pypi:`pytest-xlog` + *last release*: May 31, 2020, + *status*: 4 - Beta, + *requires*: N/A + + Extended logging for test and decorators + + :pypi:`pytest-xpara` + *last release*: Oct 30, 2017, + *status*: 3 - Alpha, + *requires*: pytest + + An extended parametrizing plugin of pytest. + + :pypi:`pytest-xprocess` + *last release*: Jul 28, 2021, + *status*: 4 - Beta, + *requires*: pytest (>=2.8) + + A pytest plugin for managing processes across test runs. + + :pypi:`pytest-xray` + *last release*: May 30, 2019, + *status*: 3 - Alpha, + *requires*: N/A + + + + :pypi:`pytest-xrayjira` + *last release*: Mar 17, 2020, + *status*: 3 - Alpha, + *requires*: pytest (==4.3.1) + + + + :pypi:`pytest-xray-server` + *last release*: Oct 27, 2021, + *status*: 3 - Alpha, + *requires*: pytest (>=5.3.1) + + + + :pypi:`pytest-xvfb` + *last release*: Jun 09, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=2.8.1) + + A pytest plugin to run Xvfb for tests. + + :pypi:`pytest-yaml` + *last release*: Oct 05, 2018, + *status*: N/A, + *requires*: pytest + + This plugin is used to load yaml output to your test using pytest framework. + + :pypi:`pytest-yamltree` + *last release*: Mar 02, 2020, + *status*: 4 - Beta, + *requires*: pytest (>=3.1.1) + + Create or check file/directory trees described by YAML + + :pypi:`pytest-yamlwsgi` + *last release*: May 11, 2010, + *status*: N/A, + *requires*: N/A + + Run tests against wsgi apps defined in yaml + + :pypi:`pytest-yapf` + *last release*: Jul 06, 2017, + *status*: 4 - Beta, + *requires*: pytest (>=3.1.1) + + Run yapf + + :pypi:`pytest-yapf3` + *last release*: Aug 03, 2020, + *status*: 5 - Production/Stable, + *requires*: pytest (>=5.4) + + Validate your Python file format with yapf + + :pypi:`pytest-yield` + *last release*: Jan 23, 2019, + *status*: N/A, + *requires*: N/A + + PyTest plugin to run tests concurrently, each \`yield\` switch context to other one + + :pypi:`pytest-yuk` + *last release*: Mar 26, 2021, + *status*: N/A, + *requires*: N/A + + Display tests you are uneasy with, using 🤢/🤮 for pass/fail of tests marked with yuk. + + :pypi:`pytest-zafira` + *last release*: Sep 18, 2019, + *status*: 5 - Production/Stable, + *requires*: pytest (==4.1.1) + + A Zafira plugin for pytest + + :pypi:`pytest-zap` + *last release*: May 12, 2014, + *status*: 4 - Beta, + *requires*: N/A + + OWASP ZAP plugin for py.test. + + :pypi:`pytest-zebrunner` + *last release*: Dec 02, 2021, + *status*: 5 - Production/Stable, + *requires*: pytest (>=4.5.0) + + Pytest connector for Zebrunner reporting + + :pypi:`pytest-zigzag` + *last release*: Feb 27, 2019, + *status*: 4 - Beta, + *requires*: pytest (~=3.6) + + Extend py.test for RPC OpenStack testing. diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/reference.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/reference.rst new file mode 100644 index 0000000000..0d80c80680 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/reference/reference.rst @@ -0,0 +1,2101 @@ +.. _`api-reference`: + +API Reference +============= + +This page contains the full reference to pytest's API. + +.. contents:: + :depth: 3 + :local: + +Constants +--------- + +pytest.__version__ +~~~~~~~~~~~~~~~~~~ + +The current pytest version, as a string:: + + >>> import pytest + >>> pytest.__version__ + '7.0.0' + + +.. _`version-tuple`: + +pytest.version_tuple +~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 7.0 + +The current pytest version, as a tuple:: + + >>> import pytest + >>> pytest.version_tuple + (7, 0, 0) + +For pre-releases, the last component will be a string with the prerelease version:: + + >>> import pytest + >>> pytest.version_tuple + (7, 0, '0rc1') + + +Functions +--------- + +pytest.approx +~~~~~~~~~~~~~ + +.. autofunction:: pytest.approx + +pytest.fail +~~~~~~~~~~~ + +**Tutorial**: :ref:`skipping` + +.. autofunction:: pytest.fail(reason, [pytrace=True, msg=None]) + +pytest.skip +~~~~~~~~~~~ + +.. autofunction:: pytest.skip(reason, [allow_module_level=False, msg=None]) + +.. _`pytest.importorskip ref`: + +pytest.importorskip +~~~~~~~~~~~~~~~~~~~ + +.. autofunction:: pytest.importorskip + +pytest.xfail +~~~~~~~~~~~~ + +.. autofunction:: pytest.xfail + +pytest.exit +~~~~~~~~~~~ + +.. autofunction:: pytest.exit(reason, [returncode=False, msg=None]) + +pytest.main +~~~~~~~~~~~ + +.. autofunction:: pytest.main + +pytest.param +~~~~~~~~~~~~ + +.. autofunction:: pytest.param(*values, [id], [marks]) + +pytest.raises +~~~~~~~~~~~~~ + +**Tutorial**: :ref:`assertraises`. + +.. autofunction:: pytest.raises(expected_exception: Exception [, *, match]) + :with: excinfo + +pytest.deprecated_call +~~~~~~~~~~~~~~~~~~~~~~ + +**Tutorial**: :ref:`ensuring_function_triggers`. + +.. autofunction:: pytest.deprecated_call() + :with: + +pytest.register_assert_rewrite +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Tutorial**: :ref:`assertion-rewriting`. + +.. autofunction:: pytest.register_assert_rewrite + +pytest.warns +~~~~~~~~~~~~ + +**Tutorial**: :ref:`assertwarnings` + +.. autofunction:: pytest.warns(expected_warning: Exception, [match]) + :with: + +pytest.freeze_includes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Tutorial**: :ref:`freezing-pytest`. + +.. autofunction:: pytest.freeze_includes + +.. _`marks ref`: + +Marks +----- + +Marks can be used apply meta data to *test functions* (but not fixtures), which can then be accessed by +fixtures or plugins. + + + + +.. _`pytest.mark.filterwarnings ref`: + +pytest.mark.filterwarnings +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Tutorial**: :ref:`filterwarnings`. + +Add warning filters to marked test items. + +.. py:function:: pytest.mark.filterwarnings(filter) + + :keyword str filter: + A *warning specification string*, which is composed of contents of the tuple ``(action, message, category, module, lineno)`` + as specified in :ref:`python:warning-filter` section of + the Python documentation, separated by ``":"``. Optional fields can be omitted. + Module names passed for filtering are not regex-escaped. + + For example: + + .. code-block:: python + + @pytest.mark.filterwarnings("ignore:.*usage will be deprecated.*:DeprecationWarning") + def test_foo(): + ... + + +.. _`pytest.mark.parametrize ref`: + +pytest.mark.parametrize +~~~~~~~~~~~~~~~~~~~~~~~ + +:ref:`parametrize`. + +This mark has the same signature as :py:meth:`pytest.Metafunc.parametrize`; see there. + + +.. _`pytest.mark.skip ref`: + +pytest.mark.skip +~~~~~~~~~~~~~~~~ + +:ref:`skip`. + +Unconditionally skip a test function. + +.. py:function:: pytest.mark.skip(reason=None) + + :keyword str reason: Reason why the test function is being skipped. + + +.. _`pytest.mark.skipif ref`: + +pytest.mark.skipif +~~~~~~~~~~~~~~~~~~ + +:ref:`skipif`. + +Skip a test function if a condition is ``True``. + +.. py:function:: pytest.mark.skipif(condition, *, reason=None) + + :type condition: bool or str + :param condition: ``True/False`` if the condition should be skipped or a :ref:`condition string `. + :keyword str reason: Reason why the test function is being skipped. + + +.. _`pytest.mark.usefixtures ref`: + +pytest.mark.usefixtures +~~~~~~~~~~~~~~~~~~~~~~~ + +**Tutorial**: :ref:`usefixtures`. + +Mark a test function as using the given fixture names. + +.. py:function:: pytest.mark.usefixtures(*names) + + :param args: The names of the fixture to use, as strings. + +.. note:: + + When using `usefixtures` in hooks, it can only load fixtures when applied to a test function before test setup + (for example in the `pytest_collection_modifyitems` hook). + + Also note that this mark has no effect when applied to **fixtures**. + + + +.. _`pytest.mark.xfail ref`: + +pytest.mark.xfail +~~~~~~~~~~~~~~~~~~ + +**Tutorial**: :ref:`xfail`. + +Marks a test function as *expected to fail*. + +.. py:function:: pytest.mark.xfail(condition=None, *, reason=None, raises=None, run=True, strict=False) + + :type condition: bool or str + :param condition: + Condition for marking the test function as xfail (``True/False`` or a + :ref:`condition string `). If a bool, you also have + to specify ``reason`` (see :ref:`condition string `). + :keyword str reason: + Reason why the test function is marked as xfail. + :keyword Type[Exception] raises: + Exception subclass expected to be raised by the test function; other exceptions will fail the test. + :keyword bool run: + If the test function should actually be executed. If ``False``, the function will always xfail and will + not be executed (useful if a function is segfaulting). + :keyword bool strict: + * If ``False`` (the default) the function will be shown in the terminal output as ``xfailed`` if it fails + and as ``xpass`` if it passes. In both cases this will not cause the test suite to fail as a whole. This + is particularly useful to mark *flaky* tests (tests that fail at random) to be tackled later. + * If ``True``, the function will be shown in the terminal output as ``xfailed`` if it fails, but if it + unexpectedly passes then it will **fail** the test suite. This is particularly useful to mark functions + that are always failing and there should be a clear indication if they unexpectedly start to pass (for example + a new release of a library fixes a known bug). + + +Custom marks +~~~~~~~~~~~~ + +Marks are created dynamically using the factory object ``pytest.mark`` and applied as a decorator. + +For example: + +.. code-block:: python + + @pytest.mark.timeout(10, "slow", method="thread") + def test_function(): + ... + +Will create and attach a :class:`Mark ` object to the collected +:class:`Item `, which can then be accessed by fixtures or hooks with +:meth:`Node.iter_markers <_pytest.nodes.Node.iter_markers>`. The ``mark`` object will have the following attributes: + +.. code-block:: python + + mark.args == (10, "slow") + mark.kwargs == {"method": "thread"} + +Example for using multiple custom markers: + +.. code-block:: python + + @pytest.mark.timeout(10, "slow", method="thread") + @pytest.mark.slow + def test_function(): + ... + +When :meth:`Node.iter_markers <_pytest.nodes.Node.iter_markers>` or :meth:`Node.iter_markers <_pytest.nodes.Node.iter_markers_with_node>` is used with multiple markers, the marker closest to the function will be iterated over first. The above example will result in ``@pytest.mark.slow`` followed by ``@pytest.mark.timeout(...)``. + +.. _`fixtures-api`: + +Fixtures +-------- + +**Tutorial**: :ref:`fixture`. + +Fixtures are requested by test functions or other fixtures by declaring them as argument names. + + +Example of a test requiring a fixture: + +.. code-block:: python + + def test_output(capsys): + print("hello") + out, err = capsys.readouterr() + assert out == "hello\n" + + +Example of a fixture requiring another fixture: + +.. code-block:: python + + @pytest.fixture + def db_session(tmp_path): + fn = tmp_path / "db.file" + return connect(fn) + +For more details, consult the full :ref:`fixtures docs `. + + +.. _`pytest.fixture-api`: + +@pytest.fixture +~~~~~~~~~~~~~~~ + +.. autofunction:: pytest.fixture + :decorator: + + +.. fixture:: cache + +config.cache +~~~~~~~~~~~~ + +**Tutorial**: :ref:`cache`. + +The ``config.cache`` object allows other plugins and fixtures +to store and retrieve values across test runs. To access it from fixtures +request ``pytestconfig`` into your fixture and get it with ``pytestconfig.cache``. + +Under the hood, the cache plugin uses the simple +``dumps``/``loads`` API of the :py:mod:`json` stdlib module. + +``config.cache`` is an instance of :class:`pytest.Cache`: + +.. autoclass:: pytest.Cache() + :members: + + +.. fixture:: capsys + +capsys +~~~~~~ + +:ref:`captures`. + +.. autofunction:: _pytest.capture.capsys() + :no-auto-options: + + Returns an instance of :class:`CaptureFixture[str] `. + + Example: + + .. code-block:: python + + def test_output(capsys): + print("hello") + captured = capsys.readouterr() + assert captured.out == "hello\n" + +.. autoclass:: pytest.CaptureFixture() + :members: + + +.. fixture:: capsysbinary + +capsysbinary +~~~~~~~~~~~~ + +:ref:`captures`. + +.. autofunction:: _pytest.capture.capsysbinary() + :no-auto-options: + + Returns an instance of :class:`CaptureFixture[bytes] `. + + Example: + + .. code-block:: python + + def test_output(capsysbinary): + print("hello") + captured = capsysbinary.readouterr() + assert captured.out == b"hello\n" + + +.. fixture:: capfd + +capfd +~~~~~~ + +:ref:`captures`. + +.. autofunction:: _pytest.capture.capfd() + :no-auto-options: + + Returns an instance of :class:`CaptureFixture[str] `. + + Example: + + .. code-block:: python + + def test_system_echo(capfd): + os.system('echo "hello"') + captured = capfd.readouterr() + assert captured.out == "hello\n" + + +.. fixture:: capfdbinary + +capfdbinary +~~~~~~~~~~~~ + +:ref:`captures`. + +.. autofunction:: _pytest.capture.capfdbinary() + :no-auto-options: + + Returns an instance of :class:`CaptureFixture[bytes] `. + + Example: + + .. code-block:: python + + def test_system_echo(capfdbinary): + os.system('echo "hello"') + captured = capfdbinary.readouterr() + assert captured.out == b"hello\n" + + +.. fixture:: doctest_namespace + +doctest_namespace +~~~~~~~~~~~~~~~~~ + +:ref:`doctest`. + +.. autofunction:: _pytest.doctest.doctest_namespace() + + Usually this fixture is used in conjunction with another ``autouse`` fixture: + + .. code-block:: python + + @pytest.fixture(autouse=True) + def add_np(doctest_namespace): + doctest_namespace["np"] = numpy + + For more details: :ref:`doctest_namespace`. + + +.. fixture:: request + +request +~~~~~~~ + +:ref:`request example`. + +The ``request`` fixture is a special fixture providing information of the requesting test function. + +.. autoclass:: pytest.FixtureRequest() + :members: + + +.. fixture:: pytestconfig + +pytestconfig +~~~~~~~~~~~~ + +.. autofunction:: _pytest.fixtures.pytestconfig() + + +.. fixture:: record_property + +record_property +~~~~~~~~~~~~~~~~~~~ + +**Tutorial**: :ref:`record_property example`. + +.. autofunction:: _pytest.junitxml.record_property() + + +.. fixture:: record_testsuite_property + +record_testsuite_property +~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Tutorial**: :ref:`record_testsuite_property example`. + +.. autofunction:: _pytest.junitxml.record_testsuite_property() + + +.. fixture:: caplog + +caplog +~~~~~~ + +:ref:`logging`. + +.. autofunction:: _pytest.logging.caplog() + :no-auto-options: + + Returns a :class:`pytest.LogCaptureFixture` instance. + +.. autoclass:: pytest.LogCaptureFixture() + :members: + + +.. fixture:: monkeypatch + +monkeypatch +~~~~~~~~~~~ + +:ref:`monkeypatching`. + +.. autofunction:: _pytest.monkeypatch.monkeypatch() + :no-auto-options: + + Returns a :class:`~pytest.MonkeyPatch` instance. + +.. autoclass:: pytest.MonkeyPatch + :members: + + +.. fixture:: pytester + +pytester +~~~~~~~~ + +.. versionadded:: 6.2 + +Provides a :class:`~pytest.Pytester` instance that can be used to run and test pytest itself. + +It provides an empty directory where pytest can be executed in isolation, and contains facilities +to write tests, configuration files, and match against expected output. + +To use it, include in your topmost ``conftest.py`` file: + +.. code-block:: python + + pytest_plugins = "pytester" + + + +.. autoclass:: pytest.Pytester() + :members: + +.. autoclass:: pytest.RunResult() + :members: + +.. autoclass:: pytest.LineMatcher() + :members: + :special-members: __str__ + +.. autoclass:: pytest.HookRecorder() + :members: + +.. autoclass:: pytest.RecordedHookCall() + :members: + +.. fixture:: testdir + +testdir +~~~~~~~ + +Identical to :fixture:`pytester`, but provides an instance whose methods return +legacy ``py.path.local`` objects instead when applicable. + +New code should avoid using :fixture:`testdir` in favor of :fixture:`pytester`. + +.. autoclass:: pytest.Testdir() + :members: + + +.. fixture:: recwarn + +recwarn +~~~~~~~ + +**Tutorial**: :ref:`assertwarnings` + +.. autofunction:: _pytest.recwarn.recwarn() + :no-auto-options: + +.. autoclass:: pytest.WarningsRecorder() + :members: + +Each recorded warning is an instance of :class:`warnings.WarningMessage`. + +.. note:: + ``DeprecationWarning`` and ``PendingDeprecationWarning`` are treated + differently; see :ref:`ensuring_function_triggers`. + + +.. fixture:: tmp_path + +tmp_path +~~~~~~~~ + +:ref:`tmp_path` + +.. autofunction:: _pytest.tmpdir.tmp_path() + :no-auto-options: + + +.. fixture:: tmp_path_factory + +tmp_path_factory +~~~~~~~~~~~~~~~~ + +:ref:`tmp_path_factory example` + +.. _`tmp_path_factory factory api`: + +``tmp_path_factory`` is an instance of :class:`~pytest.TempPathFactory`: + +.. autoclass:: pytest.TempPathFactory() + :members: + + +.. fixture:: tmpdir + +tmpdir +~~~~~~ + +:ref:`tmpdir and tmpdir_factory` + +.. autofunction:: _pytest.legacypath.LegacyTmpdirPlugin.tmpdir() + :no-auto-options: + + +.. fixture:: tmpdir_factory + +tmpdir_factory +~~~~~~~~~~~~~~ + +:ref:`tmpdir and tmpdir_factory` + +``tmpdir_factory`` is an instance of :class:`~pytest.TempdirFactory`: + +.. autoclass:: pytest.TempdirFactory() + :members: + + +.. _`hook-reference`: + +Hooks +----- + +:ref:`writing-plugins`. + +.. currentmodule:: _pytest.hookspec + +Reference to all hooks which can be implemented by :ref:`conftest.py files ` and :ref:`plugins `. + +Bootstrapping hooks +~~~~~~~~~~~~~~~~~~~ + +Bootstrapping hooks called for plugins registered early enough (internal and setuptools plugins). + +.. hook:: pytest_load_initial_conftests +.. autofunction:: pytest_load_initial_conftests +.. hook:: pytest_cmdline_preparse +.. autofunction:: pytest_cmdline_preparse +.. hook:: pytest_cmdline_parse +.. autofunction:: pytest_cmdline_parse +.. hook:: pytest_cmdline_main +.. autofunction:: pytest_cmdline_main + +.. _`initialization-hooks`: + +Initialization hooks +~~~~~~~~~~~~~~~~~~~~ + +Initialization hooks called for plugins and ``conftest.py`` files. + +.. hook:: pytest_addoption +.. autofunction:: pytest_addoption +.. hook:: pytest_addhooks +.. autofunction:: pytest_addhooks +.. hook:: pytest_configure +.. autofunction:: pytest_configure +.. hook:: pytest_unconfigure +.. autofunction:: pytest_unconfigure +.. hook:: pytest_sessionstart +.. autofunction:: pytest_sessionstart +.. hook:: pytest_sessionfinish +.. autofunction:: pytest_sessionfinish + +.. hook:: pytest_plugin_registered +.. autofunction:: pytest_plugin_registered + +Collection hooks +~~~~~~~~~~~~~~~~ + +``pytest`` calls the following hooks for collecting files and directories: + +.. hook:: pytest_collection +.. autofunction:: pytest_collection +.. hook:: pytest_ignore_collect +.. autofunction:: pytest_ignore_collect +.. hook:: pytest_collect_file +.. autofunction:: pytest_collect_file +.. hook:: pytest_pycollect_makemodule +.. autofunction:: pytest_pycollect_makemodule + +For influencing the collection of objects in Python modules +you can use the following hook: + +.. hook:: pytest_pycollect_makeitem +.. autofunction:: pytest_pycollect_makeitem +.. hook:: pytest_generate_tests +.. autofunction:: pytest_generate_tests +.. hook:: pytest_make_parametrize_id +.. autofunction:: pytest_make_parametrize_id + +Hooks for influencing test skipping: + +.. hook:: pytest_markeval_namespace +.. autofunction:: pytest_markeval_namespace + +After collection is complete, you can modify the order of +items, delete or otherwise amend the test items: + +.. hook:: pytest_collection_modifyitems +.. autofunction:: pytest_collection_modifyitems + +.. note:: + If this hook is implemented in ``conftest.py`` files, it always receives all collected items, not only those + under the ``conftest.py`` where it is implemented. + +.. autofunction:: pytest_collection_finish + +Test running (runtest) hooks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All runtest related hooks receive a :py:class:`pytest.Item ` object. + +.. hook:: pytest_runtestloop +.. autofunction:: pytest_runtestloop +.. hook:: pytest_runtest_protocol +.. autofunction:: pytest_runtest_protocol +.. hook:: pytest_runtest_logstart +.. autofunction:: pytest_runtest_logstart +.. hook:: pytest_runtest_logfinish +.. autofunction:: pytest_runtest_logfinish +.. hook:: pytest_runtest_setup +.. autofunction:: pytest_runtest_setup +.. hook:: pytest_runtest_call +.. autofunction:: pytest_runtest_call +.. hook:: pytest_runtest_teardown +.. autofunction:: pytest_runtest_teardown +.. hook:: pytest_runtest_makereport +.. autofunction:: pytest_runtest_makereport + +For deeper understanding you may look at the default implementation of +these hooks in ``_pytest.runner`` and maybe also +in ``_pytest.pdb`` which interacts with ``_pytest.capture`` +and its input/output capturing in order to immediately drop +into interactive debugging when a test failure occurs. + +.. hook:: pytest_pyfunc_call +.. autofunction:: pytest_pyfunc_call + +Reporting hooks +~~~~~~~~~~~~~~~ + +Session related reporting hooks: + +.. hook:: pytest_collectstart +.. autofunction:: pytest_collectstart +.. hook:: pytest_make_collect_report +.. autofunction:: pytest_make_collect_report +.. hook:: pytest_itemcollected +.. autofunction:: pytest_itemcollected +.. hook:: pytest_collectreport +.. autofunction:: pytest_collectreport +.. hook:: pytest_deselected +.. autofunction:: pytest_deselected +.. hook:: pytest_report_header +.. autofunction:: pytest_report_header +.. hook:: pytest_report_collectionfinish +.. autofunction:: pytest_report_collectionfinish +.. hook:: pytest_report_teststatus +.. autofunction:: pytest_report_teststatus +.. hook:: pytest_report_to_serializable +.. autofunction:: pytest_report_to_serializable +.. hook:: pytest_report_from_serializable +.. autofunction:: pytest_report_from_serializable +.. hook:: pytest_terminal_summary +.. autofunction:: pytest_terminal_summary +.. hook:: pytest_fixture_setup +.. autofunction:: pytest_fixture_setup +.. hook:: pytest_fixture_post_finalizer +.. autofunction:: pytest_fixture_post_finalizer +.. hook:: pytest_warning_captured +.. autofunction:: pytest_warning_captured +.. hook:: pytest_warning_recorded +.. autofunction:: pytest_warning_recorded + +Central hook for reporting about test execution: + +.. hook:: pytest_runtest_logreport +.. autofunction:: pytest_runtest_logreport + +Assertion related hooks: + +.. hook:: pytest_assertrepr_compare +.. autofunction:: pytest_assertrepr_compare +.. hook:: pytest_assertion_pass +.. autofunction:: pytest_assertion_pass + + +Debugging/Interaction hooks +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are few hooks which can be used for special +reporting or interaction with exceptions: + +.. hook:: pytest_internalerror +.. autofunction:: pytest_internalerror +.. hook:: pytest_keyboard_interrupt +.. autofunction:: pytest_keyboard_interrupt +.. hook:: pytest_exception_interact +.. autofunction:: pytest_exception_interact +.. hook:: pytest_enter_pdb +.. autofunction:: pytest_enter_pdb +.. hook:: pytest_leave_pdb +.. autofunction:: pytest_leave_pdb + + +Objects +------- + +Full reference to objects accessible from :ref:`fixtures ` or :ref:`hooks `. + + +CallInfo +~~~~~~~~ + +.. autoclass:: pytest.CallInfo() + :members: + + +Class +~~~~~ + +.. autoclass:: pytest.Class() + :members: + :show-inheritance: + +Collector +~~~~~~~~~ + +.. autoclass:: pytest.Collector() + :members: + :show-inheritance: + +CollectReport +~~~~~~~~~~~~~ + +.. autoclass:: pytest.CollectReport() + :members: + :show-inheritance: + :inherited-members: + +Config +~~~~~~ + +.. autoclass:: pytest.Config() + :members: + +ExceptionInfo +~~~~~~~~~~~~~ + +.. autoclass:: pytest.ExceptionInfo() + :members: + + +ExitCode +~~~~~~~~ + +.. autoclass:: pytest.ExitCode + :members: + +File +~~~~ + +.. autoclass:: pytest.File() + :members: + :show-inheritance: + + +FixtureDef +~~~~~~~~~~ + +.. autoclass:: _pytest.fixtures.FixtureDef() + :members: + :show-inheritance: + +FSCollector +~~~~~~~~~~~ + +.. autoclass:: _pytest.nodes.FSCollector() + :members: + :show-inheritance: + +Function +~~~~~~~~ + +.. autoclass:: pytest.Function() + :members: + :show-inheritance: + +FunctionDefinition +~~~~~~~~~~~~~~~~~~ + +.. autoclass:: _pytest.python.FunctionDefinition() + :members: + :show-inheritance: + +Item +~~~~ + +.. autoclass:: pytest.Item() + :members: + :show-inheritance: + +MarkDecorator +~~~~~~~~~~~~~ + +.. autoclass:: pytest.MarkDecorator() + :members: + + +MarkGenerator +~~~~~~~~~~~~~ + +.. autoclass:: pytest.MarkGenerator() + :members: + + +Mark +~~~~ + +.. autoclass:: pytest.Mark() + :members: + + +Metafunc +~~~~~~~~ + +.. autoclass:: pytest.Metafunc() + :members: + +Module +~~~~~~ + +.. autoclass:: pytest.Module() + :members: + :show-inheritance: + +Node +~~~~ + +.. autoclass:: _pytest.nodes.Node() + :members: + +Parser +~~~~~~ + +.. autoclass:: pytest.Parser() + :members: + +OptionGroup +~~~~~~~~~~~ + +.. autoclass:: pytest.OptionGroup() + :members: + +PytestPluginManager +~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: pytest.PytestPluginManager() + :members: + :undoc-members: + :inherited-members: + :show-inheritance: + +Session +~~~~~~~ + +.. autoclass:: pytest.Session() + :members: + :show-inheritance: + +TestReport +~~~~~~~~~~ + +.. autoclass:: pytest.TestReport() + :members: + :show-inheritance: + :inherited-members: + +_Result +~~~~~~~ + +Result object used within :ref:`hook wrappers `, see :py:class:`_Result in the pluggy documentation ` for more information. + +Stash +~~~~~ + +.. autoclass:: pytest.Stash + :special-members: __setitem__, __getitem__, __delitem__, __contains__, __len__ + :members: + +.. autoclass:: pytest.StashKey + :show-inheritance: + :members: + + +Global Variables +---------------- + +pytest treats some global variables in a special manner when defined in a test module or +``conftest.py`` files. + + +.. globalvar:: collect_ignore + +**Tutorial**: :ref:`customizing-test-collection` + +Can be declared in *conftest.py files* to exclude test directories or modules. +Needs to be a list of paths (``str``, :class:`pathlib.Path` or any :class:`os.PathLike`). + +.. code-block:: python + + collect_ignore = ["setup.py"] + + +.. globalvar:: collect_ignore_glob + +**Tutorial**: :ref:`customizing-test-collection` + +Can be declared in *conftest.py files* to exclude test directories or modules +with Unix shell-style wildcards. Needs to be ``list[str]`` where ``str`` can +contain glob patterns. + +.. code-block:: python + + collect_ignore_glob = ["*_ignore.py"] + + +.. globalvar:: pytest_plugins + +**Tutorial**: :ref:`available installable plugins` + +Can be declared at the **global** level in *test modules* and *conftest.py files* to register additional plugins. +Can be either a ``str`` or ``Sequence[str]``. + +.. code-block:: python + + pytest_plugins = "myapp.testsupport.myplugin" + +.. code-block:: python + + pytest_plugins = ("myapp.testsupport.tools", "myapp.testsupport.regression") + + +.. globalvar:: pytestmark + +**Tutorial**: :ref:`scoped-marking` + +Can be declared at the **global** level in *test modules* to apply one or more :ref:`marks ` to all +test functions and methods. Can be either a single mark or a list of marks (applied in left-to-right order). + +.. code-block:: python + + import pytest + + pytestmark = pytest.mark.webtest + + +.. code-block:: python + + import pytest + + pytestmark = [pytest.mark.integration, pytest.mark.slow] + + +Environment Variables +--------------------- + +Environment variables that can be used to change pytest's behavior. + +.. envvar:: PYTEST_ADDOPTS + +This contains a command-line (parsed by the py:mod:`shlex` module) that will be **prepended** to the command line given +by the user, see :ref:`adding default options` for more information. + +.. envvar:: PYTEST_CURRENT_TEST + +This is not meant to be set by users, but is set by pytest internally with the name of the current test so other +processes can inspect it, see :ref:`pytest current test env` for more information. + +.. envvar:: PYTEST_DEBUG + +When set, pytest will print tracing and debug information. + +.. envvar:: PYTEST_DISABLE_PLUGIN_AUTOLOAD + +When set, disables plugin auto-loading through setuptools entrypoints. Only explicitly specified plugins will be +loaded. + +.. envvar:: PYTEST_PLUGINS + +Contains comma-separated list of modules that should be loaded as plugins: + +.. code-block:: bash + + export PYTEST_PLUGINS=mymodule.plugin,xdist + +.. envvar:: PYTEST_THEME + +Sets a `pygment style `_ to use for the code output. + +.. envvar:: PYTEST_THEME_MODE + +Sets the :envvar:`PYTEST_THEME` to be either *dark* or *light*. + +.. envvar:: PY_COLORS + +When set to ``1``, pytest will use color in terminal output. +When set to ``0``, pytest will not use color. +``PY_COLORS`` takes precedence over ``NO_COLOR`` and ``FORCE_COLOR``. + +.. envvar:: NO_COLOR + +When set (regardless of value), pytest will not use color in terminal output. +``PY_COLORS`` takes precedence over ``NO_COLOR``, which takes precedence over ``FORCE_COLOR``. +See `no-color.org `__ for other libraries supporting this community standard. + +.. envvar:: FORCE_COLOR + +When set (regardless of value), pytest will use color in terminal output. +``PY_COLORS`` and ``NO_COLOR`` take precedence over ``FORCE_COLOR``. + +Exceptions +---------- + +.. autoclass:: pytest.UsageError() + :show-inheritance: + +.. _`warnings ref`: + +Warnings +-------- + +Custom warnings generated in some situations such as improper usage or deprecated features. + +.. autoclass:: pytest.PytestWarning + :show-inheritance: + +.. autoclass:: pytest.PytestAssertRewriteWarning + :show-inheritance: + +.. autoclass:: pytest.PytestCacheWarning + :show-inheritance: + +.. autoclass:: pytest.PytestCollectionWarning + :show-inheritance: + +.. autoclass:: pytest.PytestConfigWarning + :show-inheritance: + +.. autoclass:: pytest.PytestDeprecationWarning + :show-inheritance: + +.. autoclass:: pytest.PytestExperimentalApiWarning + :show-inheritance: + +.. autoclass:: pytest.PytestUnhandledCoroutineWarning + :show-inheritance: + +.. autoclass:: pytest.PytestUnknownMarkWarning + :show-inheritance: + +.. autoclass:: pytest.PytestUnraisableExceptionWarning + :show-inheritance: + +.. autoclass:: pytest.PytestUnhandledThreadExceptionWarning + :show-inheritance: + + +Consult the :ref:`internal-warnings` section in the documentation for more information. + + +.. _`ini options ref`: + +Configuration Options +--------------------- + +Here is a list of builtin configuration options that may be written in a ``pytest.ini``, ``pyproject.toml``, ``tox.ini`` or ``setup.cfg`` +file, usually located at the root of your repository. To see each file format in details, see +:ref:`config file formats`. + +.. warning:: + Usage of ``setup.cfg`` is not recommended except for very simple use cases. ``.cfg`` + files use a different parser than ``pytest.ini`` and ``tox.ini`` which might cause hard to track + down problems. + When possible, it is recommended to use the latter files, or ``pyproject.toml``, to hold your pytest configuration. + +Configuration options may be overwritten in the command-line by using ``-o/--override-ini``, which can also be +passed multiple times. The expected format is ``name=value``. For example:: + + pytest -o console_output_style=classic -o cache_dir=/tmp/mycache + + +.. confval:: addopts + + Add the specified ``OPTS`` to the set of command line arguments as if they + had been specified by the user. Example: if you have this ini file content: + + .. code-block:: ini + + # content of pytest.ini + [pytest] + addopts = --maxfail=2 -rf # exit after 2 failures, report fail info + + issuing ``pytest test_hello.py`` actually means: + + .. code-block:: bash + + pytest --maxfail=2 -rf test_hello.py + + Default is to add no options. + + +.. confval:: cache_dir + + Sets a directory where stores content of cache plugin. Default directory is + ``.pytest_cache`` which is created in :ref:`rootdir `. Directory may be + relative or absolute path. If setting relative path, then directory is created + relative to :ref:`rootdir `. Additionally path may contain environment + variables, that will be expanded. For more information about cache plugin + please refer to :ref:`cache_provider`. + +.. confval:: console_output_style + + Sets the console output style while running tests: + + * ``classic``: classic pytest output. + * ``progress``: like classic pytest output, but with a progress indicator. + * ``count``: like progress, but shows progress as the number of tests completed instead of a percent. + + The default is ``progress``, but you can fallback to ``classic`` if you prefer or + the new mode is causing unexpected problems: + + .. code-block:: ini + + # content of pytest.ini + [pytest] + console_output_style = classic + + +.. confval:: doctest_encoding + + + + Default encoding to use to decode text files with docstrings. + :ref:`See how pytest handles doctests `. + + +.. confval:: doctest_optionflags + + One or more doctest flag names from the standard ``doctest`` module. + :ref:`See how pytest handles doctests `. + + +.. confval:: empty_parameter_set_mark + + + + Allows to pick the action for empty parametersets in parameterization + + * ``skip`` skips tests with an empty parameterset (default) + * ``xfail`` marks tests with an empty parameterset as xfail(run=False) + * ``fail_at_collect`` raises an exception if parametrize collects an empty parameter set + + .. code-block:: ini + + # content of pytest.ini + [pytest] + empty_parameter_set_mark = xfail + + .. note:: + + The default value of this option is planned to change to ``xfail`` in future releases + as this is considered less error prone, see :issue:`3155` for more details. + + +.. confval:: faulthandler_timeout + + Dumps the tracebacks of all threads if a test takes longer than ``X`` seconds to run (including + fixture setup and teardown). Implemented using the :func:`faulthandler.dump_traceback_later` function, + so all caveats there apply. + + .. code-block:: ini + + # content of pytest.ini + [pytest] + faulthandler_timeout=5 + + For more information please refer to :ref:`faulthandler`. + +.. confval:: filterwarnings + + + + Sets a list of filters and actions that should be taken for matched + warnings. By default all warnings emitted during the test session + will be displayed in a summary at the end of the test session. + + .. code-block:: ini + + # content of pytest.ini + [pytest] + filterwarnings = + error + ignore::DeprecationWarning + + This tells pytest to ignore deprecation warnings and turn all other warnings + into errors. For more information please refer to :ref:`warnings`. + + +.. confval:: junit_duration_report + + .. versionadded:: 4.1 + + Configures how durations are recorded into the JUnit XML report: + + * ``total`` (the default): duration times reported include setup, call, and teardown times. + * ``call``: duration times reported include only call times, excluding setup and teardown. + + .. code-block:: ini + + [pytest] + junit_duration_report = call + + +.. confval:: junit_family + + .. versionadded:: 4.2 + .. versionchanged:: 6.1 + Default changed to ``xunit2``. + + Configures the format of the generated JUnit XML file. The possible options are: + + * ``xunit1`` (or ``legacy``): produces old style output, compatible with the xunit 1.0 format. + * ``xunit2``: produces `xunit 2.0 style output `__, which should be more compatible with latest Jenkins versions. **This is the default**. + + .. code-block:: ini + + [pytest] + junit_family = xunit2 + + +.. confval:: junit_logging + + .. versionadded:: 3.5 + .. versionchanged:: 5.4 + ``log``, ``all``, ``out-err`` options added. + + Configures if captured output should be written to the JUnit XML file. Valid values are: + + * ``log``: write only ``logging`` captured output. + * ``system-out``: write captured ``stdout`` contents. + * ``system-err``: write captured ``stderr`` contents. + * ``out-err``: write both captured ``stdout`` and ``stderr`` contents. + * ``all``: write captured ``logging``, ``stdout`` and ``stderr`` contents. + * ``no`` (the default): no captured output is written. + + .. code-block:: ini + + [pytest] + junit_logging = system-out + + +.. confval:: junit_log_passing_tests + + .. versionadded:: 4.6 + + If ``junit_logging != "no"``, configures if the captured output should be written + to the JUnit XML file for **passing** tests. Default is ``True``. + + .. code-block:: ini + + [pytest] + junit_log_passing_tests = False + + +.. confval:: junit_suite_name + + To set the name of the root test suite xml item, you can configure the ``junit_suite_name`` option in your config file: + + .. code-block:: ini + + [pytest] + junit_suite_name = my_suite + +.. confval:: log_auto_indent + + Allow selective auto-indentation of multiline log messages. + + Supports command line option ``--log-auto-indent [value]`` + and config option ``log_auto_indent = [value]`` to set the + auto-indentation behavior for all logging. + + ``[value]`` can be: + * True or "On" - Dynamically auto-indent multiline log messages + * False or "Off" or 0 - Do not auto-indent multiline log messages (the default behavior) + * [positive integer] - auto-indent multiline log messages by [value] spaces + + .. code-block:: ini + + [pytest] + log_auto_indent = False + + Supports passing kwarg ``extra={"auto_indent": [value]}`` to + calls to ``logging.log()`` to specify auto-indentation behavior for + a specific entry in the log. ``extra`` kwarg overrides the value specified + on the command line or in the config. + +.. confval:: log_cli + + Enable log display during test run (also known as :ref:`"live logging" `). + The default is ``False``. + + .. code-block:: ini + + [pytest] + log_cli = True + +.. confval:: log_cli_date_format + + + + Sets a :py:func:`time.strftime`-compatible string that will be used when formatting dates for live logging. + + .. code-block:: ini + + [pytest] + log_cli_date_format = %Y-%m-%d %H:%M:%S + + For more information, see :ref:`live_logs`. + +.. confval:: log_cli_format + + + + Sets a :py:mod:`logging`-compatible string used to format live logging messages. + + .. code-block:: ini + + [pytest] + log_cli_format = %(asctime)s %(levelname)s %(message)s + + For more information, see :ref:`live_logs`. + + +.. confval:: log_cli_level + + + + Sets the minimum log message level that should be captured for live logging. The integer value or + the names of the levels can be used. + + .. code-block:: ini + + [pytest] + log_cli_level = INFO + + For more information, see :ref:`live_logs`. + + +.. confval:: log_date_format + + + + Sets a :py:func:`time.strftime`-compatible string that will be used when formatting dates for logging capture. + + .. code-block:: ini + + [pytest] + log_date_format = %Y-%m-%d %H:%M:%S + + For more information, see :ref:`logging`. + + +.. confval:: log_file + + + + Sets a file name relative to the ``pytest.ini`` file where log messages should be written to, in addition + to the other logging facilities that are active. + + .. code-block:: ini + + [pytest] + log_file = logs/pytest-logs.txt + + For more information, see :ref:`logging`. + + +.. confval:: log_file_date_format + + + + Sets a :py:func:`time.strftime`-compatible string that will be used when formatting dates for the logging file. + + .. code-block:: ini + + [pytest] + log_file_date_format = %Y-%m-%d %H:%M:%S + + For more information, see :ref:`logging`. + +.. confval:: log_file_format + + + + Sets a :py:mod:`logging`-compatible string used to format logging messages redirected to the logging file. + + .. code-block:: ini + + [pytest] + log_file_format = %(asctime)s %(levelname)s %(message)s + + For more information, see :ref:`logging`. + +.. confval:: log_file_level + + + + Sets the minimum log message level that should be captured for the logging file. The integer value or + the names of the levels can be used. + + .. code-block:: ini + + [pytest] + log_file_level = INFO + + For more information, see :ref:`logging`. + + +.. confval:: log_format + + + + Sets a :py:mod:`logging`-compatible string used to format captured logging messages. + + .. code-block:: ini + + [pytest] + log_format = %(asctime)s %(levelname)s %(message)s + + For more information, see :ref:`logging`. + + +.. confval:: log_level + + + + Sets the minimum log message level that should be captured for logging capture. The integer value or + the names of the levels can be used. + + .. code-block:: ini + + [pytest] + log_level = INFO + + For more information, see :ref:`logging`. + + +.. confval:: markers + + When the ``--strict-markers`` or ``--strict`` command-line arguments are used, + only known markers - defined in code by core pytest or some plugin - are allowed. + + You can list additional markers in this setting to add them to the whitelist, + in which case you probably want to add ``--strict-markers`` to ``addopts`` + to avoid future regressions: + + .. code-block:: ini + + [pytest] + addopts = --strict-markers + markers = + slow + serial + + .. note:: + The use of ``--strict-markers`` is highly preferred. ``--strict`` was kept for + backward compatibility only and may be confusing for others as it only applies to + markers and not to other options. + +.. confval:: minversion + + Specifies a minimal pytest version required for running tests. + + .. code-block:: ini + + # content of pytest.ini + [pytest] + minversion = 3.0 # will fail if we run with pytest-2.8 + + +.. confval:: norecursedirs + + Set the directory basename patterns to avoid when recursing + for test discovery. The individual (fnmatch-style) patterns are + applied to the basename of a directory to decide if to recurse into it. + Pattern matching characters:: + + * matches everything + ? matches any single character + [seq] matches any character in seq + [!seq] matches any char not in seq + + Default patterns are ``'*.egg'``, ``'.*'``, ``'_darcs'``, ``'build'``, + ``'CVS'``, ``'dist'``, ``'node_modules'``, ``'venv'``, ``'{arch}'``. + Setting a ``norecursedirs`` replaces the default. Here is an example of + how to avoid certain directories: + + .. code-block:: ini + + [pytest] + norecursedirs = .svn _build tmp* + + This would tell ``pytest`` to not look into typical subversion or + sphinx-build directories or into any ``tmp`` prefixed directory. + + Additionally, ``pytest`` will attempt to intelligently identify and ignore a + virtualenv by the presence of an activation script. Any directory deemed to + be the root of a virtual environment will not be considered during test + collection unless ``‑‑collect‑in‑virtualenv`` is given. Note also that + ``norecursedirs`` takes precedence over ``‑‑collect‑in‑virtualenv``; e.g. if + you intend to run tests in a virtualenv with a base directory that matches + ``'.*'`` you *must* override ``norecursedirs`` in addition to using the + ``‑‑collect‑in‑virtualenv`` flag. + + +.. confval:: python_classes + + One or more name prefixes or glob-style patterns determining which classes + are considered for test collection. Search for multiple glob patterns by + adding a space between patterns. By default, pytest will consider any + class prefixed with ``Test`` as a test collection. Here is an example of how + to collect tests from classes that end in ``Suite``: + + .. code-block:: ini + + [pytest] + python_classes = *Suite + + Note that ``unittest.TestCase`` derived classes are always collected + regardless of this option, as ``unittest``'s own collection framework is used + to collect those tests. + + +.. confval:: python_files + + One or more Glob-style file patterns determining which python files + are considered as test modules. Search for multiple glob patterns by + adding a space between patterns: + + .. code-block:: ini + + [pytest] + python_files = test_*.py check_*.py example_*.py + + Or one per line: + + .. code-block:: ini + + [pytest] + python_files = + test_*.py + check_*.py + example_*.py + + By default, files matching ``test_*.py`` and ``*_test.py`` will be considered + test modules. + + +.. confval:: python_functions + + One or more name prefixes or glob-patterns determining which test functions + and methods are considered tests. Search for multiple glob patterns by + adding a space between patterns. By default, pytest will consider any + function prefixed with ``test`` as a test. Here is an example of how + to collect test functions and methods that end in ``_test``: + + .. code-block:: ini + + [pytest] + python_functions = *_test + + Note that this has no effect on methods that live on a ``unittest.TestCase`` + derived class, as ``unittest``'s own collection framework is used + to collect those tests. + + See :ref:`change naming conventions` for more detailed examples. + + +.. confval:: pythonpath + + Sets list of directories that should be added to the python search path. + Directories will be added to the head of :data:`sys.path`. + Similar to the :envvar:`PYTHONPATH` environment variable, the directories will be + included in where Python will look for imported modules. + Paths are relative to the :ref:`rootdir ` directory. + Directories remain in path for the duration of the test session. + + .. code-block:: ini + + [pytest] + pythonpath = src1 src2 + + +.. confval:: required_plugins + + A space separated list of plugins that must be present for pytest to run. + Plugins can be listed with or without version specifiers directly following + their name. Whitespace between different version specifiers is not allowed. + If any one of the plugins is not found, emit an error. + + .. code-block:: ini + + [pytest] + required_plugins = pytest-django>=3.0.0,<4.0.0 pytest-html pytest-xdist>=1.0.0 + + +.. confval:: testpaths + + + + Sets list of directories that should be searched for tests when + no specific directories, files or test ids are given in the command line when + executing pytest from the :ref:`rootdir ` directory. + Useful when all project tests are in a known location to speed up + test collection and to avoid picking up undesired tests by accident. + + .. code-block:: ini + + [pytest] + testpaths = testing doc + + This tells pytest to only look for tests in ``testing`` and ``doc`` + directories when executing from the root directory. + + +.. confval:: usefixtures + + List of fixtures that will be applied to all test functions; this is semantically the same to apply + the ``@pytest.mark.usefixtures`` marker to all test functions. + + + .. code-block:: ini + + [pytest] + usefixtures = + clean_db + + +.. confval:: xfail_strict + + If set to ``True``, tests marked with ``@pytest.mark.xfail`` that actually succeed will by default fail the + test suite. + For more information, see :ref:`xfail strict tutorial`. + + + .. code-block:: ini + + [pytest] + xfail_strict = True + + +.. _`command-line-flags`: + +Command-line Flags +------------------ + +All the command-line flags can be obtained by running ``pytest --help``:: + + $ pytest --help + usage: pytest [options] [file_or_dir] [file_or_dir] [...] + + positional arguments: + file_or_dir + + general: + -k EXPRESSION only run tests which match the given substring + expression. An expression is a python evaluatable + expression where all names are substring-matched + against test names and their parent classes. + Example: -k 'test_method or test_other' matches all + test functions and classes whose name contains + 'test_method' or 'test_other', while -k 'not + test_method' matches those that don't contain + 'test_method' in their names. -k 'not test_method + and not test_other' will eliminate the matches. + Additionally keywords are matched to classes and + functions containing extra names in their + 'extra_keyword_matches' set, as well as functions + which have names assigned directly to them. The + matching is case-insensitive. + -m MARKEXPR only run tests matching given mark expression. + For example: -m 'mark1 and not mark2'. + --markers show markers (builtin, plugin and per-project ones). + -x, --exitfirst exit instantly on first error or failed test. + --fixtures, --funcargs + show available fixtures, sorted by plugin appearance + (fixtures with leading '_' are only shown with '-v') + --fixtures-per-test show fixtures per test + --pdb start the interactive Python debugger on errors or + KeyboardInterrupt. + --pdbcls=modulename:classname + specify a custom interactive Python debugger for use + with --pdb.For example: + --pdbcls=IPython.terminal.debugger:TerminalPdb + --trace Immediately break when running each test. + --capture=method per-test capturing method: one of fd|sys|no|tee-sys. + -s shortcut for --capture=no. + --runxfail report the results of xfail tests as if they were + not marked + --lf, --last-failed rerun only the tests that failed at the last run (or + all if none failed) + --ff, --failed-first run all tests, but run the last failures first. + This may re-order tests and thus lead to repeated + fixture setup/teardown. + --nf, --new-first run tests from new files first, then the rest of the + tests sorted by file mtime + --cache-show=[CACHESHOW] + show cache contents, don't perform collection or + tests. Optional argument: glob (default: '*'). + --cache-clear remove all cache contents at start of test run. + --lfnf={all,none}, --last-failed-no-failures={all,none} + which tests to run with no previously (known) + failures. + --sw, --stepwise exit on test failure and continue from last failing + test next time + --sw-skip, --stepwise-skip + ignore the first failing test but stop on the next + failing test. + implicitly enables --stepwise. + + reporting: + --durations=N show N slowest setup/test durations (N=0 for all). + --durations-min=N Minimal duration in seconds for inclusion in slowest + list. Default 0.005 + -v, --verbose increase verbosity. + --no-header disable header + --no-summary disable summary + -q, --quiet decrease verbosity. + --verbosity=VERBOSE set verbosity. Default is 0. + -r chars show extra test summary info as specified by chars: + (f)ailed, (E)rror, (s)kipped, (x)failed, (X)passed, + (p)assed, (P)assed with output, (a)ll except passed + (p/P), or (A)ll. (w)arnings are enabled by default + (see --disable-warnings), 'N' can be used to reset + the list. (default: 'fE'). + --disable-warnings, --disable-pytest-warnings + disable warnings summary + -l, --showlocals show locals in tracebacks (disabled by default). + --tb=style traceback print mode + (auto/long/short/line/native/no). + --show-capture={no,stdout,stderr,log,all} + Controls how captured stdout/stderr/log is shown on + failed tests. Default is 'all'. + --full-trace don't cut any tracebacks (default is to cut). + --color=color color terminal output (yes/no/auto). + --code-highlight={yes,no} + Whether code should be highlighted (only if --color + is also enabled) + --pastebin=mode send failed|all info to bpaste.net pastebin service. + --junit-xml=path create junit-xml style report file at given path. + --junit-prefix=str prepend prefix to classnames in junit-xml output + + pytest-warnings: + -W PYTHONWARNINGS, --pythonwarnings=PYTHONWARNINGS + set which warnings to report, see -W option of + python itself. + --maxfail=num exit after first num failures or errors. + --strict-config any warnings encountered while parsing the `pytest` + section of the configuration file raise errors. + --strict-markers markers not registered in the `markers` section of + the configuration file raise errors. + --strict (deprecated) alias to --strict-markers. + -c file load configuration from `file` instead of trying to + locate one of the implicit configuration files. + --continue-on-collection-errors + Force test execution even if collection errors + occur. + --rootdir=ROOTDIR Define root directory for tests. Can be relative + path: 'root_dir', './root_dir', + 'root_dir/another_dir/'; absolute path: + '/home/user/root_dir'; path with variables: + '$HOME/root_dir'. + + collection: + --collect-only, --co only collect tests, don't execute them. + --pyargs try to interpret all arguments as python packages. + --ignore=path ignore path during collection (multi-allowed). + --ignore-glob=path ignore path pattern during collection (multi- + allowed). + --deselect=nodeid_prefix + deselect item (via node id prefix) during collection + (multi-allowed). + --confcutdir=dir only load conftest.py's relative to specified dir. + --noconftest Don't load any conftest.py files. + --keep-duplicates Keep duplicate tests. + --collect-in-virtualenv + Don't ignore tests in a local virtualenv directory + --import-mode={prepend,append,importlib} + prepend/append to sys.path when importing test + modules and conftest files, default is to prepend. + --doctest-modules run doctests in all .py modules + --doctest-report={none,cdiff,ndiff,udiff,only_first_failure} + choose another output format for diffs on doctest + failure + --doctest-glob=pat doctests file matching pattern, default: test*.txt + --doctest-ignore-import-errors + ignore doctest ImportErrors + --doctest-continue-on-failure + for a given doctest, continue to run after the first + failure + + test session debugging and configuration: + --basetemp=dir base temporary directory for this test run.(warning: + this directory is removed if it exists) + -V, --version display pytest version and information about + plugins. When given twice, also display information + about plugins. + -h, --help show help message and configuration info + -p name early-load given plugin module name or entry point + (multi-allowed). + To avoid loading of plugins, use the `no:` prefix, + e.g. `no:doctest`. + --trace-config trace considerations of conftest.py files. + --debug=[DEBUG_FILE_NAME] + store internal tracing debug information in this log + file. + This file is opened with 'w' and truncated as a + result, care advised. + Defaults to 'pytestdebug.log'. + -o OVERRIDE_INI, --override-ini=OVERRIDE_INI + override ini option with "option=value" style, e.g. + `-o xfail_strict=True -o cache_dir=cache`. + --assert=MODE Control assertion debugging tools. + 'plain' performs no assertion debugging. + 'rewrite' (the default) rewrites assert statements + in test modules on import to provide assert + expression information. + --setup-only only setup fixtures, do not execute tests. + --setup-show show setup of fixtures while executing tests. + --setup-plan show what fixtures and tests would be executed but + don't execute anything. + + logging: + --log-level=LEVEL level of messages to catch/display. + Not set by default, so it depends on the root/parent + log handler's effective level, where it is "WARNING" + by default. + --log-format=LOG_FORMAT + log format as used by the logging module. + --log-date-format=LOG_DATE_FORMAT + log date format as used by the logging module. + --log-cli-level=LOG_CLI_LEVEL + cli logging level. + --log-cli-format=LOG_CLI_FORMAT + log format as used by the logging module. + --log-cli-date-format=LOG_CLI_DATE_FORMAT + log date format as used by the logging module. + --log-file=LOG_FILE path to a file when logging will be written to. + --log-file-level=LOG_FILE_LEVEL + log file logging level. + --log-file-format=LOG_FILE_FORMAT + log format as used by the logging module. + --log-file-date-format=LOG_FILE_DATE_FORMAT + log date format as used by the logging module. + --log-auto-indent=LOG_AUTO_INDENT + Auto-indent multiline messages passed to the logging + module. Accepts true|on, false|off or an integer. + + [pytest] ini-options in the first pytest.ini|tox.ini|setup.cfg file found: + + markers (linelist): markers for test functions + empty_parameter_set_mark (string): + default marker for empty parametersets + norecursedirs (args): directory patterns to avoid for recursion + testpaths (args): directories to search for tests when no files or + directories are given in the command line. + filterwarnings (linelist): + Each line specifies a pattern for + warnings.filterwarnings. Processed after + -W/--pythonwarnings. + usefixtures (args): list of default fixtures to be used with this + project + python_files (args): glob-style file patterns for Python test module + discovery + python_classes (args): + prefixes or glob names for Python test class + discovery + python_functions (args): + prefixes or glob names for Python test function and + method discovery + disable_test_id_escaping_and_forfeit_all_rights_to_community_support (bool): + disable string escape non-ascii characters, might + cause unwanted side effects(use at your own risk) + console_output_style (string): + console output: "classic", or with additional + progress information ("progress" (percentage) | + "count"). + xfail_strict (bool): default for the strict parameter of xfail markers + when not given explicitly (default: False) + enable_assertion_pass_hook (bool): + Enables the pytest_assertion_pass hook.Make sure to + delete any previously generated pyc cache files. + junit_suite_name (string): + Test suite name for JUnit report + junit_logging (string): + Write captured log messages to JUnit report: one of + no|log|system-out|system-err|out-err|all + junit_log_passing_tests (bool): + Capture log information for passing tests to JUnit + report: + junit_duration_report (string): + Duration time to report: one of total|call + junit_family (string): + Emit XML for schema: one of legacy|xunit1|xunit2 + doctest_optionflags (args): + option flags for doctests + doctest_encoding (string): + encoding used for doctest files + cache_dir (string): cache directory path. + log_level (string): default value for --log-level + log_format (string): default value for --log-format + log_date_format (string): + default value for --log-date-format + log_cli (bool): enable log display during test run (also known as + "live logging"). + log_cli_level (string): + default value for --log-cli-level + log_cli_format (string): + default value for --log-cli-format + log_cli_date_format (string): + default value for --log-cli-date-format + log_file (string): default value for --log-file + log_file_level (string): + default value for --log-file-level + log_file_format (string): + default value for --log-file-format + log_file_date_format (string): + default value for --log-file-date-format + log_auto_indent (string): + default value for --log-auto-indent + pythonpath (paths): Add paths to sys.path + faulthandler_timeout (string): + Dump the traceback of all threads if a test takes + more than TIMEOUT seconds to finish. + addopts (args): extra command line options + minversion (string): minimally required pytest version + required_plugins (args): + plugins that must be present for pytest to run + + environment variables: + PYTEST_ADDOPTS extra command line options + PYTEST_PLUGINS comma-separated plugins to load during startup + PYTEST_DISABLE_PLUGIN_AUTOLOAD set to disable plugin auto-loading + PYTEST_DEBUG set to enable debug tracing of pytest's internals + + + to see available markers type: pytest --markers + to see available fixtures type: pytest --fixtures + (shown according to specified file_or_dir or current dir if not specified; fixtures with leading '_' are only shown with the '-v' option diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/requirements.txt b/testing/web-platform/tests/tools/third_party/pytest/doc/en/requirements.txt new file mode 100644 index 0000000000..5b49cb7fcc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/requirements.txt @@ -0,0 +1,7 @@ +pallets-sphinx-themes +pluggy>=1.0 +pygments-pytest>=2.2.0 +sphinx-removed-in>=0.2.0 +sphinx>=3.1,<4 +sphinxcontrib-trio +sphinxcontrib-svg2pdfconverter diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/sponsor.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/sponsor.rst new file mode 100644 index 0000000000..8362a7f0a3 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/sponsor.rst @@ -0,0 +1,26 @@ +Sponsor +======= + +pytest is maintained by a team of volunteers from all around the world in their free time. While +we work on pytest because we love the project and use it daily at our daily jobs, monetary +compensation when possible is welcome to justify time away from friends, family and personal time. + +Money is also used to fund local sprints, merchandising (stickers to distribute in conferences for example) +and every few years a large sprint involving all members. + +OpenCollective +-------------- + +`Open Collective`_ is an online funding platform for open and transparent communities. +It provide tools to raise money and share your finances in full transparency. + +It is the platform of choice for individuals and companies that want to make one-time or +monthly donations directly to the project. + +See more details in the `pytest collective`_. + + +.. _Tidelift: https://tidelift.com +.. _Tidelift subscription: https://tidelift.com/subscription/pkg/pypi-pytest +.. _Open Collective: https://opencollective.com +.. _pytest collective: https://opencollective.com/pytest diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/talks.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/talks.rst new file mode 100644 index 0000000000..6843c82bab --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/talks.rst @@ -0,0 +1,109 @@ + +Talks and Tutorials +========================== + +Books +--------------------------------------------- + +- `pytest Quick Start Guide, by Bruno Oliveira (2018) + `_. + +- `Python Testing with pytest, by Brian Okken (2017) + `_. + +Talks and blog postings +--------------------------------------------- + +- Webinar: `pytest: Test Driven Development für Python (German) `_, Florian Bruhin, via mylearning.ch, 2020 + +- Webinar: `Simplify Your Tests with Fixtures `_, Oliver Bestwalter, via JetBrains, 2020 + +- Training: `Introduction to pytest - simple, rapid and fun testing with Python `_, Florian Bruhin, PyConDE 2019 + +- Abridged metaprogramming classics - this episode: pytest, Oliver Bestwalter, PyConDE 2019 (`repository `__, `recording `__) + +- Testing PySide/PyQt code easily using the pytest framework, Florian Bruhin, Qt World Summit 2019 (`slides `__, `recording `__) + +- `pytest: recommendations, basic packages for testing in Python and Django, Andreu Vallbona, PyBCN June 2019 `_. + +- pytest: recommendations, basic packages for testing in Python and Django, Andreu Vallbona, PyconES 2017 (`slides in english `_, `video in spanish `_) + +- `pytest advanced, Andrew Svetlov (Russian, PyCon Russia, 2016) + `_. + +- `Pythonic testing, Igor Starikov (Russian, PyNsk, November 2016) + `_. + +- `pytest - Rapid Simple Testing, Florian Bruhin, Swiss Python Summit 2016 + `_. + +- `Improve your testing with Pytest and Mock, Gabe Hollombe, PyCon SG 2015 + `_. + +- `Introduction to pytest, Andreas Pelme, EuroPython 2014 + `_. + +- `Advanced Uses of py.test Fixtures, Floris Bruynooghe, EuroPython + 2014 `_. + +- `Why i use py.test and maybe you should too, Andy Todd, Pycon AU 2013 + `_ + +- `3-part blog series about pytest from @pydanny alias Daniel Greenfeld (January + 2014) `_ + +- `pytest: helps you write better Django apps, Andreas Pelme, DjangoCon + Europe 2014 `_. + +- `Testing Django Applications with pytest, Andreas Pelme, EuroPython + 2013 `_. + +- `Testes pythonics com py.test, Vinicius Belchior Assef Neto, Plone + Conf 2013, Brazil `_. + +- `Introduction to py.test fixtures, FOSDEM 2013, Floris Bruynooghe + `_. + +- `pytest feature and release highlights, Holger Krekel (GERMAN, October 2013) + `_ + +- `pytest introduction from Brian Okken (January 2013) + `_ + +- pycon australia 2012 pytest talk from Brianna Laugher (`video `_, `slides `_, `code `_) +- `pycon 2012 US talk video from Holger Krekel `_ + +- `monkey patching done right`_ (blog post, consult `monkeypatch plugin`_ for up-to-date API) + +Test parametrization: + +- `generating parametrized tests with fixtures`_. +- `test generators and cached setup`_ +- `parametrizing tests, generalized`_ (blog post) +- `putting test-hooks into local or global plugins`_ (blog post) + +Assertion introspection: + +- `(07/2011) Behind the scenes of pytest's new assertion rewriting + `_ + +Distributed testing: + +- `simultaneously test your code on all platforms`_ (blog entry) + +Plugin specific examples: + +- `skipping slow tests by default in pytest`_ (blog entry) + +- `many examples in the docs for plugins`_ + +.. _`skipping slow tests by default in pytest`: http://bruynooghe.blogspot.com/2009/12/skipping-slow-test-by-default-in-pytest.html +.. _`many examples in the docs for plugins`: plugins.html +.. _`monkeypatch plugin`: monkeypatch.html +.. _`application setup in test functions with fixtures`: fixture.html#interdependent-fixtures +.. _`simultaneously test your code on all platforms`: https://tetamap.wordpress.com//2009/03/23/new-simultanously-test-your-code-on-all-platforms/ +.. _`monkey patching done right`: https://tetamap.wordpress.com//2009/03/03/monkeypatching-in-unit-tests-done-right/ +.. _`putting test-hooks into local or global plugins`: https://tetamap.wordpress.com/2009/05/14/putting-test-hooks-into-local-and-global-plugins/ +.. _`parametrizing tests, generalized`: https://tetamap.wordpress.com/2009/05/13/parametrizing-python-tests-generalized/ +.. _`generating parametrized tests with fixtures`: parametrize.html#test-generators +.. _`test generators and cached setup`: http://bruynooghe.blogspot.com/2010/06/pytest-test-generators-and-cached-setup.html diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/tidelift.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/tidelift.rst new file mode 100644 index 0000000000..8ce55e97b3 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/tidelift.rst @@ -0,0 +1,45 @@ +pytest for enterprise +===================== + +`Tidelift`_ is working with the maintainers of pytest and thousands of other +open source projects to deliver commercial support and maintenance for the open source dependencies you use +to build your applications. Save time, reduce risk, and improve code health, while paying the maintainers of the +exact dependencies you use. + +`Get more details `_ + +The Tidelift Subscription is a managed open source subscription for application dependencies covering millions of open source projects across JavaScript, Python, Java, PHP, Ruby, .NET, and more. + +Your subscription includes: + +* **Security updates** + + - Tidelift's security response team coordinates patches for new breaking security vulnerabilities and alerts immediately through a private channel, so your software supply chain is always secure. + +* **Licensing verification and indemnification** + + - Tidelift verifies license information to enable easy policy enforcement and adds intellectual property indemnification to cover creators and users in case something goes wrong. You always have a 100% up-to-date bill of materials for your dependencies to share with your legal team, customers, or partners. + +* **Maintenance and code improvement** + + - Tidelift ensures the software you rely on keeps working as long as you need it to work. Your managed dependencies are actively maintained and we recruit additional maintainers where required. + +* **Package selection and version guidance** + + - Tidelift helps you choose the best open source packages from the start—and then guide you through updates to stay on the best releases as new issues arise. + +* **Roadmap input** + + - Take a seat at the table with the creators behind the software you use. Tidelift's participating maintainers earn more income as their software is used by more subscribers, so they're interested in knowing what you need. + +* **Tooling and cloud integration** + + - Tidelift works with GitHub, GitLab, BitBucket, and every cloud platform (and other deployment targets, too). + +The end result? All of the capabilities you expect from commercial-grade software, for the full breadth of open +source you use. That means less time grappling with esoteric open source trivia, and more time building your own +applications—and your business. + +`Request a demo `_ + +.. _Tidelift: https://tidelift.com diff --git a/testing/web-platform/tests/tools/third_party/pytest/doc/en/yieldfixture.rst b/testing/web-platform/tests/tools/third_party/pytest/doc/en/yieldfixture.rst new file mode 100644 index 0000000000..47590f9db9 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/doc/en/yieldfixture.rst @@ -0,0 +1,18 @@ +:orphan: + +.. _yieldfixture: + +"yield_fixture" functions +--------------------------------------------------------------- + + + + + +.. important:: + Since pytest-3.0, fixtures using the normal ``fixture`` decorator can use a ``yield`` + statement to provide fixture values and execute teardown code, exactly like ``yield_fixture`` + in previous versions. + + Marking functions as ``yield_fixture`` is still supported, but deprecated and should not + be used in new code. diff --git a/testing/web-platform/tests/tools/third_party/pytest/extra/get_issues.py b/testing/web-platform/tests/tools/third_party/pytest/extra/get_issues.py new file mode 100644 index 0000000000..4aaa3c3ec3 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/extra/get_issues.py @@ -0,0 +1,85 @@ +import json +from pathlib import Path + +import requests + +issues_url = "https://api.github.com/repos/pytest-dev/pytest/issues" + + +def get_issues(): + issues = [] + url = issues_url + while 1: + get_data = {"state": "all"} + r = requests.get(url, params=get_data) + data = r.json() + if r.status_code == 403: + # API request limit exceeded + print(data["message"]) + exit(1) + issues.extend(data) + + # Look for next page + links = requests.utils.parse_header_links(r.headers["Link"]) + another_page = False + for link in links: + if link["rel"] == "next": + url = link["url"] + another_page = True + if not another_page: + return issues + + +def main(args): + cachefile = Path(args.cache) + if not cachefile.exists() or args.refresh: + issues = get_issues() + cachefile.write_text(json.dumps(issues), "utf-8") + else: + issues = json.loads(cachefile.read_text("utf-8")) + + open_issues = [x for x in issues if x["state"] == "open"] + + open_issues.sort(key=lambda x: x["number"]) + report(open_issues) + + +def _get_kind(issue): + labels = [label["name"] for label in issue["labels"]] + for key in ("bug", "enhancement", "proposal"): + if key in labels: + return key + return "issue" + + +def report(issues): + for issue in issues: + title = issue["title"] + # body = issue["body"] + kind = _get_kind(issue) + status = issue["state"] + number = issue["number"] + link = "https://github.com/pytest-dev/pytest/issues/%s/" % number + print("----") + print(status, kind, link) + print(title) + # print() + # lines = body.split("\n") + # print("\n".join(lines[:3])) + # if len(lines) > 3 or len(body) > 240: + # print("...") + print("\n\nFound %s open issues" % len(issues)) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser("process bitbucket issues") + parser.add_argument( + "--refresh", action="store_true", help="invalidate cache, refresh issues" + ) + parser.add_argument( + "--cache", action="store", default="issues.json", help="cache file" + ) + args = parser.parse_args() + main(args) diff --git a/testing/web-platform/tests/tools/third_party/pytest/extra/setup-py.test/setup.py b/testing/web-platform/tests/tools/third_party/pytest/extra/setup-py.test/setup.py new file mode 100644 index 0000000000..d0560ce1f5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/extra/setup-py.test/setup.py @@ -0,0 +1,11 @@ +import sys +from distutils.core import setup + +if __name__ == "__main__": + if "sdist" not in sys.argv[1:]: + raise ValueError("please use 'pytest' pypi package instead of 'py.test'") + setup( + name="py.test", + version="0.0", + description="please use 'pytest' for installation", + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/pyproject.toml b/testing/web-platform/tests/tools/third_party/pytest/pyproject.toml new file mode 100644 index 0000000000..5d32b755c7 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/pyproject.toml @@ -0,0 +1,116 @@ +[build-system] +requires = [ + # sync with setup.py until we discard non-pep-517/518 + "setuptools>=45.0", + "setuptools-scm[toml]>=6.2.3", + "wheel", +] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "src/_pytest/_version.py" + +[tool.pytest.ini_options] +minversion = "2.0" +addopts = "-rfEX -p pytester --strict-markers" +python_files = ["test_*.py", "*_test.py", "testing/python/*.py"] +python_classes = ["Test", "Acceptance"] +python_functions = ["test"] +# NOTE: "doc" is not included here, but gets tested explicitly via "doctesting". +testpaths = ["testing"] +norecursedirs = ["testing/example_scripts"] +xfail_strict = true +filterwarnings = [ + "error", + "default:Using or importing the ABCs:DeprecationWarning:unittest2.*", + # produced by older pyparsing<=2.2.0. + "default:Using or importing the ABCs:DeprecationWarning:pyparsing.*", + "default:the imp module is deprecated in favour of importlib:DeprecationWarning:nose.*", + # distutils is deprecated in 3.10, scheduled for removal in 3.12 + "ignore:The distutils package is deprecated:DeprecationWarning", + # produced by python3.6/site.py itself (3.6.7 on Travis, could not trigger it with 3.6.8)." + "ignore:.*U.*mode is deprecated:DeprecationWarning:(?!(pytest|_pytest))", + # produced by pytest-xdist + "ignore:.*type argument to addoption.*:DeprecationWarning", + # produced on execnet (pytest-xdist) + "ignore:.*inspect.getargspec.*deprecated, use inspect.signature.*:DeprecationWarning", + # pytest's own futurewarnings + "ignore::pytest.PytestExperimentalApiWarning", + # Do not cause SyntaxError for invalid escape sequences in py37. + # Those are caught/handled by pyupgrade, and not easy to filter with the + # module being the filename (with .py removed). + "default:invalid escape sequence:DeprecationWarning", + # ignore use of unregistered marks, because we use many to test the implementation + "ignore::_pytest.warning_types.PytestUnknownMarkWarning", + # https://github.com/benjaminp/six/issues/341 + "ignore:_SixMetaPathImporter\\.exec_module\\(\\) not found; falling back to load_module\\(\\):ImportWarning", + # https://github.com/benjaminp/six/pull/352 + "ignore:_SixMetaPathImporter\\.find_spec\\(\\) not found; falling back to find_module\\(\\):ImportWarning", + # https://github.com/pypa/setuptools/pull/2517 + "ignore:VendorImporter\\.find_spec\\(\\) not found; falling back to find_module\\(\\):ImportWarning", + # https://github.com/pytest-dev/execnet/pull/127 + "ignore:isSet\\(\\) is deprecated, use is_set\\(\\) instead:DeprecationWarning", +] +pytester_example_dir = "testing/example_scripts" +markers = [ + # dummy markers for testing + "foo", + "bar", + "baz", + # conftest.py reorders tests moving slow ones to the end of the list + "slow", + # experimental mark for all tests using pexpect + "uses_pexpect", +] + + +[tool.towncrier] +package = "pytest" +package_dir = "src" +filename = "doc/en/changelog.rst" +directory = "changelog/" +title_format = "pytest {version} ({project_date})" +template = "changelog/_template.rst" + + [[tool.towncrier.type]] + directory = "breaking" + name = "Breaking Changes" + showcontent = true + + [[tool.towncrier.type]] + directory = "deprecation" + name = "Deprecations" + showcontent = true + + [[tool.towncrier.type]] + directory = "feature" + name = "Features" + showcontent = true + + [[tool.towncrier.type]] + directory = "improvement" + name = "Improvements" + showcontent = true + + [[tool.towncrier.type]] + directory = "bugfix" + name = "Bug Fixes" + showcontent = true + + [[tool.towncrier.type]] + directory = "vendor" + name = "Vendored Libraries" + showcontent = true + + [[tool.towncrier.type]] + directory = "doc" + name = "Improved Documentation" + showcontent = true + + [[tool.towncrier.type]] + directory = "trivial" + name = "Trivial/Internal Changes" + showcontent = true + +[tool.black] +target-version = ['py36'] diff --git a/testing/web-platform/tests/tools/third_party/pytest/scripts/prepare-release-pr.py b/testing/web-platform/tests/tools/third_party/pytest/scripts/prepare-release-pr.py new file mode 100644 index 0000000000..7a80de7eda --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/scripts/prepare-release-pr.py @@ -0,0 +1,174 @@ +""" +This script is part of the pytest release process which is triggered manually in the Actions +tab of the repository. + +The user will need to enter the base branch to start the release from (for example +``6.1.x`` or ``main``) and if it should be a major release. + +The appropriate version will be obtained based on the given branch automatically. + +After that, it will create a release using the `release` tox environment, and push a new PR. + +**Token**: currently the token from the GitHub Actions is used, pushed with +`pytest bot ` commit author. +""" +import argparse +import re +from pathlib import Path +from subprocess import check_call +from subprocess import check_output +from subprocess import run + +from colorama import Fore +from colorama import init +from github3.repos import Repository + + +class InvalidFeatureRelease(Exception): + pass + + +SLUG = "pytest-dev/pytest" + +PR_BODY = """\ +Created automatically from manual trigger. + +Once all builds pass and it has been **approved** by one or more maintainers, the build +can be released by pushing a tag `{version}` to this repository. +""" + + +def login(token: str) -> Repository: + import github3 + + github = github3.login(token=token) + owner, repo = SLUG.split("/") + return github.repository(owner, repo) + + +def prepare_release_pr( + base_branch: str, is_major: bool, token: str, prerelease: str +) -> None: + print() + print(f"Processing release for branch {Fore.CYAN}{base_branch}") + + check_call(["git", "checkout", f"origin/{base_branch}"]) + + changelog = Path("changelog") + + features = list(changelog.glob("*.feature.rst")) + breaking = list(changelog.glob("*.breaking.rst")) + is_feature_release = bool(features or breaking) + + try: + version = find_next_version( + base_branch, is_major, is_feature_release, prerelease + ) + except InvalidFeatureRelease as e: + print(f"{Fore.RED}{e}") + raise SystemExit(1) + + print(f"Version: {Fore.CYAN}{version}") + + release_branch = f"release-{version}" + + run( + ["git", "config", "user.name", "pytest bot"], + check=True, + ) + run( + ["git", "config", "user.email", "pytestbot@gmail.com"], + check=True, + ) + + run( + ["git", "checkout", "-b", release_branch, f"origin/{base_branch}"], + check=True, + ) + + print(f"Branch {Fore.CYAN}{release_branch}{Fore.RESET} created.") + + if is_major: + template_name = "release.major.rst" + elif prerelease: + template_name = "release.pre.rst" + elif is_feature_release: + template_name = "release.minor.rst" + else: + template_name = "release.patch.rst" + + # important to use tox here because we have changed branches, so dependencies + # might have changed as well + cmdline = [ + "tox", + "-e", + "release", + "--", + version, + template_name, + release_branch, # doc_version + "--skip-check-links", + ] + print("Running", " ".join(cmdline)) + run( + cmdline, + check=True, + ) + + oauth_url = f"https://{token}:x-oauth-basic@github.com/{SLUG}.git" + run( + ["git", "push", oauth_url, f"HEAD:{release_branch}", "--force"], + check=True, + ) + print(f"Branch {Fore.CYAN}{release_branch}{Fore.RESET} pushed.") + + body = PR_BODY.format(version=version) + repo = login(token) + pr = repo.create_pull( + f"Prepare release {version}", + base=base_branch, + head=release_branch, + body=body, + ) + print(f"Pull request {Fore.CYAN}{pr.url}{Fore.RESET} created.") + + +def find_next_version( + base_branch: str, is_major: bool, is_feature_release: bool, prerelease: str +) -> str: + output = check_output(["git", "tag"], encoding="UTF-8") + valid_versions = [] + for v in output.splitlines(): + m = re.match(r"\d.\d.\d+$", v.strip()) + if m: + valid_versions.append(tuple(int(x) for x in v.split("."))) + + valid_versions.sort() + last_version = valid_versions[-1] + + if is_major: + return f"{last_version[0]+1}.0.0{prerelease}" + elif is_feature_release: + return f"{last_version[0]}.{last_version[1] + 1}.0{prerelease}" + else: + return f"{last_version[0]}.{last_version[1]}.{last_version[2] + 1}{prerelease}" + + +def main() -> None: + init(autoreset=True) + parser = argparse.ArgumentParser() + parser.add_argument("base_branch") + parser.add_argument("token") + parser.add_argument("--major", action="store_true", default=False) + parser.add_argument("--prerelease", default="") + options = parser.parse_args() + prepare_release_pr( + base_branch=options.base_branch, + is_major=options.major, + token=options.token, + prerelease=options.prerelease, + ) + + +if __name__ == "__main__": + main() diff --git a/testing/web-platform/tests/tools/third_party/pytest/scripts/publish-gh-release-notes.py b/testing/web-platform/tests/tools/third_party/pytest/scripts/publish-gh-release-notes.py new file mode 100644 index 0000000000..68cbd7adff --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/scripts/publish-gh-release-notes.py @@ -0,0 +1,102 @@ +""" +Script used to publish GitHub release notes extracted from CHANGELOG.rst. + +This script is meant to be executed after a successful deployment in GitHub actions. + +Uses the following environment variables: + +* GIT_TAG: the name of the tag of the current commit. +* GH_RELEASE_NOTES_TOKEN: a personal access token with 'repo' permissions. + + Create one at: + + https://github.com/settings/tokens + + This token should be set in a secret in the repository, which is exposed as an + environment variable in the main.yml workflow file. + +The script also requires ``pandoc`` to be previously installed in the system. + +Requires Python3.6+. +""" +import os +import re +import sys +from pathlib import Path + +import github3 +import pypandoc + + +def publish_github_release(slug, token, tag_name, body): + github = github3.login(token=token) + owner, repo = slug.split("/") + repo = github.repository(owner, repo) + return repo.create_release(tag_name=tag_name, body=body) + + +def parse_changelog(tag_name): + p = Path(__file__).parent.parent / "doc/en/changelog.rst" + changelog_lines = p.read_text(encoding="UTF-8").splitlines() + + title_regex = re.compile(r"pytest (\d\.\d+\.\d+) \(\d{4}-\d{2}-\d{2}\)") + consuming_version = False + version_lines = [] + for line in changelog_lines: + m = title_regex.match(line) + if m: + # found the version we want: start to consume lines until we find the next version title + if m.group(1) == tag_name: + consuming_version = True + # found a new version title while parsing the version we want: break out + elif consuming_version: + break + if consuming_version: + version_lines.append(line) + + return "\n".join(version_lines) + + +def convert_rst_to_md(text): + return pypandoc.convert_text( + text, "md", format="rst", extra_args=["--wrap=preserve"] + ) + + +def main(argv): + if len(argv) > 1: + tag_name = argv[1] + else: + tag_name = os.environ.get("GITHUB_REF") + if not tag_name: + print("tag_name not given and $GITHUB_REF not set", file=sys.stderr) + return 1 + if tag_name.startswith("refs/tags/"): + tag_name = tag_name[len("refs/tags/") :] + + token = os.environ.get("GH_RELEASE_NOTES_TOKEN") + if not token: + print("GH_RELEASE_NOTES_TOKEN not set", file=sys.stderr) + return 1 + + slug = os.environ.get("GITHUB_REPOSITORY") + if not slug: + print("GITHUB_REPOSITORY not set", file=sys.stderr) + return 1 + + rst_body = parse_changelog(tag_name) + md_body = convert_rst_to_md(rst_body) + if not publish_github_release(slug, token, tag_name, md_body): + print("Could not publish release notes:", file=sys.stderr) + print(md_body, file=sys.stderr) + return 5 + + print() + print(f"Release notes for {tag_name} published successfully:") + print(f"https://github.com/{slug}/releases/tag/{tag_name}") + print() + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/testing/web-platform/tests/tools/third_party/pytest/scripts/release.major.rst b/testing/web-platform/tests/tools/third_party/pytest/scripts/release.major.rst new file mode 100644 index 0000000000..76e447f0c6 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/scripts/release.major.rst @@ -0,0 +1,24 @@ +pytest-{version} +======================================= + +The pytest team is proud to announce the {version} release! + +This release contains new features, improvements, bug fixes, and breaking changes, so users +are encouraged to take a look at the CHANGELOG carefully: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from PyPI via: + + pip install -U pytest + +Thanks to all of the contributors to this release: + +{contributors} + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/scripts/release.minor.rst b/testing/web-platform/tests/tools/third_party/pytest/scripts/release.minor.rst new file mode 100644 index 0000000000..9a06d3d414 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/scripts/release.minor.rst @@ -0,0 +1,24 @@ +pytest-{version} +======================================= + +The pytest team is proud to announce the {version} release! + +This release contains new features, improvements, and bug fixes, +the full list of changes is available in the changelog: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from PyPI via: + + pip install -U pytest + +Thanks to all of the contributors to this release: + +{contributors} + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/scripts/release.patch.rst b/testing/web-platform/tests/tools/third_party/pytest/scripts/release.patch.rst new file mode 100644 index 0000000000..59fbe50ce0 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/scripts/release.patch.rst @@ -0,0 +1,17 @@ +pytest-{version} +======================================= + +pytest {version} has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +{contributors} + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/scripts/release.pre.rst b/testing/web-platform/tests/tools/third_party/pytest/scripts/release.pre.rst new file mode 100644 index 0000000000..960fae7e4f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/scripts/release.pre.rst @@ -0,0 +1,29 @@ +pytest-{version} +======================================= + +The pytest team is proud to announce the {version} prerelease! + +This is a prerelease, not intended for production use, but to test the upcoming features and improvements +in order to catch any major problems before the final version is released to the major public. + +We appreciate your help testing this out before the final release, making sure to report any +regressions to our issue tracker: + +https://github.com/pytest-dev/pytest/issues + +When doing so, please include the string ``[prerelease]`` in the title. + +You can upgrade from PyPI via: + + pip install pytest=={version} + +Users are encouraged to take a look at the CHANGELOG carefully: + + https://docs.pytest.org/en/{doc_version}/changelog.html + +Thanks to all the contributors to this release: + +{contributors} + +Happy testing, +The pytest Development Team diff --git a/testing/web-platform/tests/tools/third_party/pytest/scripts/release.py b/testing/web-platform/tests/tools/third_party/pytest/scripts/release.py new file mode 100644 index 0000000000..19fef42842 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/scripts/release.py @@ -0,0 +1,131 @@ +"""Invoke development tasks.""" +import argparse +import os +from pathlib import Path +from subprocess import call +from subprocess import check_call +from subprocess import check_output + +from colorama import Fore +from colorama import init + + +def announce(version, template_name, doc_version): + """Generates a new release announcement entry in the docs.""" + # Get our list of authors + stdout = check_output(["git", "describe", "--abbrev=0", "--tags"]) + stdout = stdout.decode("utf-8") + last_version = stdout.strip() + + stdout = check_output(["git", "log", f"{last_version}..HEAD", "--format=%aN"]) + stdout = stdout.decode("utf-8") + + contributors = { + name + for name in stdout.splitlines() + if not name.endswith("[bot]") and name != "pytest bot" + } + + template_text = ( + Path(__file__).parent.joinpath(template_name).read_text(encoding="UTF-8") + ) + + contributors_text = "\n".join(f"* {name}" for name in sorted(contributors)) + "\n" + text = template_text.format( + version=version, contributors=contributors_text, doc_version=doc_version + ) + + target = Path(__file__).parent.joinpath(f"../doc/en/announce/release-{version}.rst") + target.write_text(text, encoding="UTF-8") + print(f"{Fore.CYAN}[generate.announce] {Fore.RESET}Generated {target.name}") + + # Update index with the new release entry + index_path = Path(__file__).parent.joinpath("../doc/en/announce/index.rst") + lines = index_path.read_text(encoding="UTF-8").splitlines() + indent = " " + for index, line in enumerate(lines): + if line.startswith(f"{indent}release-"): + new_line = indent + target.stem + if line != new_line: + lines.insert(index, new_line) + index_path.write_text("\n".join(lines) + "\n", encoding="UTF-8") + print( + f"{Fore.CYAN}[generate.announce] {Fore.RESET}Updated {index_path.name}" + ) + else: + print( + f"{Fore.CYAN}[generate.announce] {Fore.RESET}Skip {index_path.name} (already contains release)" + ) + break + + check_call(["git", "add", str(target)]) + + +def regen(version): + """Call regendoc tool to update examples and pytest output in the docs.""" + print(f"{Fore.CYAN}[generate.regen] {Fore.RESET}Updating docs") + check_call( + ["tox", "-e", "regen"], + env={**os.environ, "SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST": version}, + ) + + +def fix_formatting(): + """Runs pre-commit in all files to ensure they are formatted correctly""" + print( + f"{Fore.CYAN}[generate.fix linting] {Fore.RESET}Fixing formatting using pre-commit" + ) + call(["pre-commit", "run", "--all-files"]) + + +def check_links(): + """Runs sphinx-build to check links""" + print(f"{Fore.CYAN}[generate.check_links] {Fore.RESET}Checking links") + check_call(["tox", "-e", "docs-checklinks"]) + + +def pre_release(version, template_name, doc_version, *, skip_check_links): + """Generates new docs, release announcements and creates a local tag.""" + announce(version, template_name, doc_version) + regen(version) + changelog(version, write_out=True) + fix_formatting() + if not skip_check_links: + check_links() + + msg = f"Prepare release version {version}" + check_call(["git", "commit", "-a", "-m", msg]) + + print() + print(f"{Fore.CYAN}[generate.pre_release] {Fore.GREEN}All done!") + print() + print("Please push your branch and open a PR.") + + +def changelog(version, write_out=False): + addopts = [] if write_out else ["--draft"] + check_call(["towncrier", "--yes", "--version", version] + addopts) + + +def main(): + init(autoreset=True) + parser = argparse.ArgumentParser() + parser.add_argument("version", help="Release version") + parser.add_argument( + "template_name", help="Name of template file to use for release announcement" + ) + parser.add_argument( + "doc_version", help="For prereleases, the version to link to in the docs" + ) + parser.add_argument("--skip-check-links", action="store_true", default=False) + options = parser.parse_args() + pre_release( + options.version, + options.template_name, + options.doc_version, + skip_check_links=options.skip_check_links, + ) + + +if __name__ == "__main__": + main() diff --git a/testing/web-platform/tests/tools/third_party/pytest/scripts/towncrier-draft-to-file.py b/testing/web-platform/tests/tools/third_party/pytest/scripts/towncrier-draft-to-file.py new file mode 100644 index 0000000000..81507b40b7 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/scripts/towncrier-draft-to-file.py @@ -0,0 +1,15 @@ +import sys +from subprocess import call + + +def main(): + """ + Platform agnostic wrapper script for towncrier. + Fixes the issue (#7251) where windows users are unable to natively run tox -e docs to build pytest docs. + """ + with open("doc/en/_changelog_towncrier_draft.rst", "w") as draft_file: + return call(("towncrier", "--draft"), stdout=draft_file) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/testing/web-platform/tests/tools/third_party/pytest/scripts/update-plugin-list.py b/testing/web-platform/tests/tools/third_party/pytest/scripts/update-plugin-list.py new file mode 100644 index 0000000000..c034c72420 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/scripts/update-plugin-list.py @@ -0,0 +1,140 @@ +import datetime +import pathlib +import re +from textwrap import dedent +from textwrap import indent + +import packaging.version +import requests +import tabulate +import wcwidth +from tqdm import tqdm + +FILE_HEAD = r""" +.. _plugin-list: + +Plugin List +=========== + +PyPI projects that match "pytest-\*" are considered plugins and are listed +automatically. Packages classified as inactive are excluded. + +.. The following conditional uses a different format for this list when + creating a PDF, because otherwise the table gets far too wide for the + page. + +""" +DEVELOPMENT_STATUS_CLASSIFIERS = ( + "Development Status :: 1 - Planning", + "Development Status :: 2 - Pre-Alpha", + "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", + "Development Status :: 6 - Mature", + "Development Status :: 7 - Inactive", +) + + +def escape_rst(text: str) -> str: + """Rudimentary attempt to escape special RST characters to appear as + plain text.""" + text = ( + text.replace("*", "\\*") + .replace("<", "\\<") + .replace(">", "\\>") + .replace("`", "\\`") + ) + text = re.sub(r"_\b", "", text) + return text + + +def iter_plugins(): + regex = r">([\d\w-]*)" + response = requests.get("https://pypi.org/simple") + + matches = list( + match + for match in re.finditer(regex, response.text) + if match.groups()[0].startswith("pytest-") + ) + + for match in tqdm(matches, smoothing=0): + name = match.groups()[0] + response = requests.get(f"https://pypi.org/pypi/{name}/json") + if response.status_code == 404: + # Some packages, like pytest-azurepipelines42, are included in https://pypi.org/simple but + # return 404 on the JSON API. Skip. + continue + response.raise_for_status() + info = response.json()["info"] + if "Development Status :: 7 - Inactive" in info["classifiers"]: + continue + for classifier in DEVELOPMENT_STATUS_CLASSIFIERS: + if classifier in info["classifiers"]: + status = classifier[22:] + break + else: + status = "N/A" + requires = "N/A" + if info["requires_dist"]: + for requirement in info["requires_dist"]: + if requirement == "pytest" or "pytest " in requirement: + requires = requirement + break + releases = response.json()["releases"] + for release in sorted(releases, key=packaging.version.parse, reverse=True): + if releases[release]: + release_date = datetime.date.fromisoformat( + releases[release][-1]["upload_time_iso_8601"].split("T")[0] + ) + last_release = release_date.strftime("%b %d, %Y") + break + name = f':pypi:`{info["name"]}`' + summary = escape_rst(info["summary"].replace("\n", "")) + yield { + "name": name, + "summary": summary.strip(), + "last release": last_release, + "status": status, + "requires": requires, + } + + +def plugin_definitions(plugins): + """Return RST for the plugin list that fits better on a vertical page.""" + + for plugin in plugins: + yield dedent( + f""" + {plugin['name']} + *last release*: {plugin["last release"]}, + *status*: {plugin["status"]}, + *requires*: {plugin["requires"]} + + {plugin["summary"]} + """ + ) + + +def main(): + plugins = list(iter_plugins()) + + reference_dir = pathlib.Path("doc", "en", "reference") + + plugin_list = reference_dir / "plugin_list.rst" + with plugin_list.open("w") as f: + f.write(FILE_HEAD) + f.write(f"This list contains {len(plugins)} plugins.\n\n") + f.write(".. only:: not latex\n\n") + + wcwidth # reference library that must exist for tabulate to work + plugin_table = tabulate.tabulate(plugins, headers="keys", tablefmt="rst") + f.write(indent(plugin_table, " ")) + f.write("\n\n") + + f.write(".. only:: latex\n\n") + f.write(indent("".join(plugin_definitions(plugins)), " ")) + + +if __name__ == "__main__": + main() diff --git a/testing/web-platform/tests/tools/third_party/pytest/setup.cfg b/testing/web-platform/tests/tools/third_party/pytest/setup.cfg new file mode 100644 index 0000000000..26a5d2e63e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/setup.cfg @@ -0,0 +1,105 @@ +[metadata] +name = pytest +description = pytest: simple powerful testing with Python +long_description = file: README.rst +long_description_content_type = text/x-rst +url = https://docs.pytest.org/en/latest/ +author = Holger Krekel, Bruno Oliveira, Ronny Pfannschmidt, Floris Bruynooghe, Brianna Laugher, Florian Bruhin and others +license = MIT +license_file = LICENSE +platforms = unix, linux, osx, cygwin, win32 +classifiers = + Development Status :: 6 - Mature + Intended Audience :: Developers + License :: OSI Approved :: MIT License + Operating System :: MacOS :: MacOS X + Operating System :: Microsoft :: Windows + Operating System :: POSIX + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Topic :: Software Development :: Libraries + Topic :: Software Development :: Testing + Topic :: Utilities +keywords = test, unittest +project_urls = + Changelog=https://docs.pytest.org/en/stable/changelog.html + Twitter=https://twitter.com/pytestdotorg + Source=https://github.com/pytest-dev/pytest + Tracker=https://github.com/pytest-dev/pytest/issues + +[options] +packages = + _pytest + _pytest._code + _pytest._io + _pytest.assertion + _pytest.config + _pytest.mark + pytest +install_requires = + attrs>=19.2.0 + iniconfig + packaging + pluggy>=0.12,<2.0 + py>=1.8.2 + tomli>=1.0.0 + atomicwrites>=1.0;sys_platform=="win32" + colorama;sys_platform=="win32" + importlib-metadata>=0.12;python_version<"3.8" +python_requires = >=3.6 +package_dir = + =src +setup_requires = + setuptools + setuptools-scm>=6.0 +zip_safe = no + +[options.entry_points] +console_scripts = + pytest=pytest:console_main + py.test=pytest:console_main + +[options.extras_require] +testing = + argcomplete + hypothesis>=3.56 + mock + nose + pygments>=2.7.2 + requests + xmlschema + +[options.package_data] +_pytest = py.typed +pytest = py.typed + +[build_sphinx] +source_dir = doc/en/ +build_dir = doc/build +all_files = 1 + +[check-manifest] +ignore = + src/_pytest/_version.py + +[devpi:upload] +formats = sdist.tgz,bdist_wheel + +[mypy] +mypy_path = src +check_untyped_defs = True +disallow_any_generics = True +ignore_missing_imports = True +no_implicit_optional = True +show_error_codes = True +strict_equality = True +warn_redundant_casts = True +warn_return_any = True +warn_unreachable = True +warn_unused_configs = True +no_implicit_reexport = True diff --git a/testing/web-platform/tests/tools/third_party/pytest/setup.py b/testing/web-platform/tests/tools/third_party/pytest/setup.py new file mode 100644 index 0000000000..7f1a1763ca --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/setup.py @@ -0,0 +1,4 @@ +from setuptools import setup + +if __name__ == "__main__": + setup() diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/__init__.py new file mode 100644 index 0000000000..8a406c5c75 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/__init__.py @@ -0,0 +1,9 @@ +__all__ = ["__version__", "version_tuple"] + +try: + from ._version import version as __version__, version_tuple +except ImportError: # pragma: no cover + # broken installation, we don't even try + # unknown only works because we do poor mans version compare + __version__ = "unknown" + version_tuple = (0, 0, "unknown") # type:ignore[assignment] diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_argcomplete.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_argcomplete.py new file mode 100644 index 0000000000..41d9d9407c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_argcomplete.py @@ -0,0 +1,117 @@ +"""Allow bash-completion for argparse with argcomplete if installed. + +Needs argcomplete>=0.5.6 for python 3.2/3.3 (older versions fail +to find the magic string, so _ARGCOMPLETE env. var is never set, and +this does not need special code). + +Function try_argcomplete(parser) should be called directly before +the call to ArgumentParser.parse_args(). + +The filescompleter is what you normally would use on the positional +arguments specification, in order to get "dirname/" after "dirn" +instead of the default "dirname ": + + optparser.add_argument(Config._file_or_dir, nargs='*').completer=filescompleter + +Other, application specific, completers should go in the file +doing the add_argument calls as they need to be specified as .completer +attributes as well. (If argcomplete is not installed, the function the +attribute points to will not be used). + +SPEEDUP +======= + +The generic argcomplete script for bash-completion +(/etc/bash_completion.d/python-argcomplete.sh) +uses a python program to determine startup script generated by pip. +You can speed up completion somewhat by changing this script to include + # PYTHON_ARGCOMPLETE_OK +so the python-argcomplete-check-easy-install-script does not +need to be called to find the entry point of the code and see if that is +marked with PYTHON_ARGCOMPLETE_OK. + +INSTALL/DEBUGGING +================= + +To include this support in another application that has setup.py generated +scripts: + +- Add the line: + # PYTHON_ARGCOMPLETE_OK + near the top of the main python entry point. + +- Include in the file calling parse_args(): + from _argcomplete import try_argcomplete, filescompleter + Call try_argcomplete just before parse_args(), and optionally add + filescompleter to the positional arguments' add_argument(). + +If things do not work right away: + +- Switch on argcomplete debugging with (also helpful when doing custom + completers): + export _ARC_DEBUG=1 + +- Run: + python-argcomplete-check-easy-install-script $(which appname) + echo $? + will echo 0 if the magic line has been found, 1 if not. + +- Sometimes it helps to find early on errors using: + _ARGCOMPLETE=1 _ARC_DEBUG=1 appname + which should throw a KeyError: 'COMPLINE' (which is properly set by the + global argcomplete script). +""" +import argparse +import os +import sys +from glob import glob +from typing import Any +from typing import List +from typing import Optional + + +class FastFilesCompleter: + """Fast file completer class.""" + + def __init__(self, directories: bool = True) -> None: + self.directories = directories + + def __call__(self, prefix: str, **kwargs: Any) -> List[str]: + # Only called on non option completions. + if os.path.sep in prefix[1:]: + prefix_dir = len(os.path.dirname(prefix) + os.path.sep) + else: + prefix_dir = 0 + completion = [] + globbed = [] + if "*" not in prefix and "?" not in prefix: + # We are on unix, otherwise no bash. + if not prefix or prefix[-1] == os.path.sep: + globbed.extend(glob(prefix + ".*")) + prefix += "*" + globbed.extend(glob(prefix)) + for x in sorted(globbed): + if os.path.isdir(x): + x += "/" + # Append stripping the prefix (like bash, not like compgen). + completion.append(x[prefix_dir:]) + return completion + + +if os.environ.get("_ARGCOMPLETE"): + try: + import argcomplete.completers + except ImportError: + sys.exit(-1) + filescompleter: Optional[FastFilesCompleter] = FastFilesCompleter() + + def try_argcomplete(parser: argparse.ArgumentParser) -> None: + argcomplete.autocomplete(parser, always_complete_options=False) + + +else: + + def try_argcomplete(parser: argparse.ArgumentParser) -> None: + pass + + filescompleter = None diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_code/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_code/__init__.py new file mode 100644 index 0000000000..511d0dde66 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_code/__init__.py @@ -0,0 +1,22 @@ +"""Python inspection/code generation API.""" +from .code import Code +from .code import ExceptionInfo +from .code import filter_traceback +from .code import Frame +from .code import getfslineno +from .code import Traceback +from .code import TracebackEntry +from .source import getrawcode +from .source import Source + +__all__ = [ + "Code", + "ExceptionInfo", + "filter_traceback", + "Frame", + "getfslineno", + "getrawcode", + "Traceback", + "TracebackEntry", + "Source", +] diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_code/code.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_code/code.py new file mode 100644 index 0000000000..5b758a8848 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_code/code.py @@ -0,0 +1,1274 @@ +import ast +import inspect +import os +import re +import sys +import traceback +from inspect import CO_VARARGS +from inspect import CO_VARKEYWORDS +from io import StringIO +from pathlib import Path +from traceback import format_exception_only +from types import CodeType +from types import FrameType +from types import TracebackType +from typing import Any +from typing import Callable +from typing import ClassVar +from typing import Dict +from typing import Generic +from typing import Iterable +from typing import List +from typing import Mapping +from typing import Optional +from typing import overload +from typing import Pattern +from typing import Sequence +from typing import Set +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import TypeVar +from typing import Union +from weakref import ref + +import attr +import pluggy + +import _pytest +from _pytest._code.source import findsource +from _pytest._code.source import getrawcode +from _pytest._code.source import getstatementrange_ast +from _pytest._code.source import Source +from _pytest._io import TerminalWriter +from _pytest._io.saferepr import safeformat +from _pytest._io.saferepr import saferepr +from _pytest.compat import final +from _pytest.compat import get_real_func +from _pytest.deprecated import check_ispytest +from _pytest.pathlib import absolutepath +from _pytest.pathlib import bestrelpath + +if TYPE_CHECKING: + from typing_extensions import Literal + from typing_extensions import SupportsIndex + from weakref import ReferenceType + + _TracebackStyle = Literal["long", "short", "line", "no", "native", "value", "auto"] + + +class Code: + """Wrapper around Python code objects.""" + + __slots__ = ("raw",) + + def __init__(self, obj: CodeType) -> None: + self.raw = obj + + @classmethod + def from_function(cls, obj: object) -> "Code": + return cls(getrawcode(obj)) + + def __eq__(self, other): + return self.raw == other.raw + + # Ignore type because of https://github.com/python/mypy/issues/4266. + __hash__ = None # type: ignore + + @property + def firstlineno(self) -> int: + return self.raw.co_firstlineno - 1 + + @property + def name(self) -> str: + return self.raw.co_name + + @property + def path(self) -> Union[Path, str]: + """Return a path object pointing to source code, or an ``str`` in + case of ``OSError`` / non-existing file.""" + if not self.raw.co_filename: + return "" + try: + p = absolutepath(self.raw.co_filename) + # maybe don't try this checking + if not p.exists(): + raise OSError("path check failed.") + return p + except OSError: + # XXX maybe try harder like the weird logic + # in the standard lib [linecache.updatecache] does? + return self.raw.co_filename + + @property + def fullsource(self) -> Optional["Source"]: + """Return a _pytest._code.Source object for the full source file of the code.""" + full, _ = findsource(self.raw) + return full + + def source(self) -> "Source": + """Return a _pytest._code.Source object for the code object's source only.""" + # return source only for that part of code + return Source(self.raw) + + def getargs(self, var: bool = False) -> Tuple[str, ...]: + """Return a tuple with the argument names for the code object. + + If 'var' is set True also return the names of the variable and + keyword arguments when present. + """ + # Handy shortcut for getting args. + raw = self.raw + argcount = raw.co_argcount + if var: + argcount += raw.co_flags & CO_VARARGS + argcount += raw.co_flags & CO_VARKEYWORDS + return raw.co_varnames[:argcount] + + +class Frame: + """Wrapper around a Python frame holding f_locals and f_globals + in which expressions can be evaluated.""" + + __slots__ = ("raw",) + + def __init__(self, frame: FrameType) -> None: + self.raw = frame + + @property + def lineno(self) -> int: + return self.raw.f_lineno - 1 + + @property + def f_globals(self) -> Dict[str, Any]: + return self.raw.f_globals + + @property + def f_locals(self) -> Dict[str, Any]: + return self.raw.f_locals + + @property + def code(self) -> Code: + return Code(self.raw.f_code) + + @property + def statement(self) -> "Source": + """Statement this frame is at.""" + if self.code.fullsource is None: + return Source("") + return self.code.fullsource.getstatement(self.lineno) + + def eval(self, code, **vars): + """Evaluate 'code' in the frame. + + 'vars' are optional additional local variables. + + Returns the result of the evaluation. + """ + f_locals = self.f_locals.copy() + f_locals.update(vars) + return eval(code, self.f_globals, f_locals) + + def repr(self, object: object) -> str: + """Return a 'safe' (non-recursive, one-line) string repr for 'object'.""" + return saferepr(object) + + def getargs(self, var: bool = False): + """Return a list of tuples (name, value) for all arguments. + + If 'var' is set True, also include the variable and keyword arguments + when present. + """ + retval = [] + for arg in self.code.getargs(var): + try: + retval.append((arg, self.f_locals[arg])) + except KeyError: + pass # this can occur when using Psyco + return retval + + +class TracebackEntry: + """A single entry in a Traceback.""" + + __slots__ = ("_rawentry", "_excinfo", "_repr_style") + + def __init__( + self, + rawentry: TracebackType, + excinfo: Optional["ReferenceType[ExceptionInfo[BaseException]]"] = None, + ) -> None: + self._rawentry = rawentry + self._excinfo = excinfo + self._repr_style: Optional['Literal["short", "long"]'] = None + + @property + def lineno(self) -> int: + return self._rawentry.tb_lineno - 1 + + def set_repr_style(self, mode: "Literal['short', 'long']") -> None: + assert mode in ("short", "long") + self._repr_style = mode + + @property + def frame(self) -> Frame: + return Frame(self._rawentry.tb_frame) + + @property + def relline(self) -> int: + return self.lineno - self.frame.code.firstlineno + + def __repr__(self) -> str: + return "" % (self.frame.code.path, self.lineno + 1) + + @property + def statement(self) -> "Source": + """_pytest._code.Source object for the current statement.""" + source = self.frame.code.fullsource + assert source is not None + return source.getstatement(self.lineno) + + @property + def path(self) -> Union[Path, str]: + """Path to the source code.""" + return self.frame.code.path + + @property + def locals(self) -> Dict[str, Any]: + """Locals of underlying frame.""" + return self.frame.f_locals + + def getfirstlinesource(self) -> int: + return self.frame.code.firstlineno + + def getsource( + self, astcache: Optional[Dict[Union[str, Path], ast.AST]] = None + ) -> Optional["Source"]: + """Return failing source code.""" + # we use the passed in astcache to not reparse asttrees + # within exception info printing + source = self.frame.code.fullsource + if source is None: + return None + key = astnode = None + if astcache is not None: + key = self.frame.code.path + if key is not None: + astnode = astcache.get(key, None) + start = self.getfirstlinesource() + try: + astnode, _, end = getstatementrange_ast( + self.lineno, source, astnode=astnode + ) + except SyntaxError: + end = self.lineno + 1 + else: + if key is not None and astcache is not None: + astcache[key] = astnode + return source[start:end] + + source = property(getsource) + + def ishidden(self) -> bool: + """Return True if the current frame has a var __tracebackhide__ + resolving to True. + + If __tracebackhide__ is a callable, it gets called with the + ExceptionInfo instance and can decide whether to hide the traceback. + + Mostly for internal use. + """ + tbh: Union[ + bool, Callable[[Optional[ExceptionInfo[BaseException]]], bool] + ] = False + for maybe_ns_dct in (self.frame.f_locals, self.frame.f_globals): + # in normal cases, f_locals and f_globals are dictionaries + # however via `exec(...)` / `eval(...)` they can be other types + # (even incorrect types!). + # as such, we suppress all exceptions while accessing __tracebackhide__ + try: + tbh = maybe_ns_dct["__tracebackhide__"] + except Exception: + pass + else: + break + if tbh and callable(tbh): + return tbh(None if self._excinfo is None else self._excinfo()) + return tbh + + def __str__(self) -> str: + name = self.frame.code.name + try: + line = str(self.statement).lstrip() + except KeyboardInterrupt: + raise + except BaseException: + line = "???" + # This output does not quite match Python's repr for traceback entries, + # but changing it to do so would break certain plugins. See + # https://github.com/pytest-dev/pytest/pull/7535/ for details. + return " File %r:%d in %s\n %s\n" % ( + str(self.path), + self.lineno + 1, + name, + line, + ) + + @property + def name(self) -> str: + """co_name of underlying code.""" + return self.frame.code.raw.co_name + + +class Traceback(List[TracebackEntry]): + """Traceback objects encapsulate and offer higher level access to Traceback entries.""" + + def __init__( + self, + tb: Union[TracebackType, Iterable[TracebackEntry]], + excinfo: Optional["ReferenceType[ExceptionInfo[BaseException]]"] = None, + ) -> None: + """Initialize from given python traceback object and ExceptionInfo.""" + self._excinfo = excinfo + if isinstance(tb, TracebackType): + + def f(cur: TracebackType) -> Iterable[TracebackEntry]: + cur_: Optional[TracebackType] = cur + while cur_ is not None: + yield TracebackEntry(cur_, excinfo=excinfo) + cur_ = cur_.tb_next + + super().__init__(f(tb)) + else: + super().__init__(tb) + + def cut( + self, + path: Optional[Union["os.PathLike[str]", str]] = None, + lineno: Optional[int] = None, + firstlineno: Optional[int] = None, + excludepath: Optional["os.PathLike[str]"] = None, + ) -> "Traceback": + """Return a Traceback instance wrapping part of this Traceback. + + By providing any combination of path, lineno and firstlineno, the + first frame to start the to-be-returned traceback is determined. + + This allows cutting the first part of a Traceback instance e.g. + for formatting reasons (removing some uninteresting bits that deal + with handling of the exception/traceback). + """ + path_ = None if path is None else os.fspath(path) + excludepath_ = None if excludepath is None else os.fspath(excludepath) + for x in self: + code = x.frame.code + codepath = code.path + if path is not None and str(codepath) != path_: + continue + if ( + excludepath is not None + and isinstance(codepath, Path) + and excludepath_ in (str(p) for p in codepath.parents) # type: ignore[operator] + ): + continue + if lineno is not None and x.lineno != lineno: + continue + if firstlineno is not None and x.frame.code.firstlineno != firstlineno: + continue + return Traceback(x._rawentry, self._excinfo) + return self + + @overload + def __getitem__(self, key: "SupportsIndex") -> TracebackEntry: + ... + + @overload + def __getitem__(self, key: slice) -> "Traceback": + ... + + def __getitem__( + self, key: Union["SupportsIndex", slice] + ) -> Union[TracebackEntry, "Traceback"]: + if isinstance(key, slice): + return self.__class__(super().__getitem__(key)) + else: + return super().__getitem__(key) + + def filter( + self, fn: Callable[[TracebackEntry], bool] = lambda x: not x.ishidden() + ) -> "Traceback": + """Return a Traceback instance with certain items removed + + fn is a function that gets a single argument, a TracebackEntry + instance, and should return True when the item should be added + to the Traceback, False when not. + + By default this removes all the TracebackEntries which are hidden + (see ishidden() above). + """ + return Traceback(filter(fn, self), self._excinfo) + + def getcrashentry(self) -> TracebackEntry: + """Return last non-hidden traceback entry that lead to the exception of a traceback.""" + for i in range(-1, -len(self) - 1, -1): + entry = self[i] + if not entry.ishidden(): + return entry + return self[-1] + + def recursionindex(self) -> Optional[int]: + """Return the index of the frame/TracebackEntry where recursion originates if + appropriate, None if no recursion occurred.""" + cache: Dict[Tuple[Any, int, int], List[Dict[str, Any]]] = {} + for i, entry in enumerate(self): + # id for the code.raw is needed to work around + # the strange metaprogramming in the decorator lib from pypi + # which generates code objects that have hash/value equality + # XXX needs a test + key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno + # print "checking for recursion at", key + values = cache.setdefault(key, []) + if values: + f = entry.frame + loc = f.f_locals + for otherloc in values: + if otherloc == loc: + return i + values.append(entry.frame.f_locals) + return None + + +E = TypeVar("E", bound=BaseException, covariant=True) + + +@final +@attr.s(repr=False, init=False, auto_attribs=True) +class ExceptionInfo(Generic[E]): + """Wraps sys.exc_info() objects and offers help for navigating the traceback.""" + + _assert_start_repr: ClassVar = "AssertionError('assert " + + _excinfo: Optional[Tuple[Type["E"], "E", TracebackType]] + _striptext: str + _traceback: Optional[Traceback] + + def __init__( + self, + excinfo: Optional[Tuple[Type["E"], "E", TracebackType]], + striptext: str = "", + traceback: Optional[Traceback] = None, + *, + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + self._excinfo = excinfo + self._striptext = striptext + self._traceback = traceback + + @classmethod + def from_exc_info( + cls, + exc_info: Tuple[Type[E], E, TracebackType], + exprinfo: Optional[str] = None, + ) -> "ExceptionInfo[E]": + """Return an ExceptionInfo for an existing exc_info tuple. + + .. warning:: + + Experimental API + + :param exprinfo: + A text string helping to determine if we should strip + ``AssertionError`` from the output. Defaults to the exception + message/``__str__()``. + """ + _striptext = "" + if exprinfo is None and isinstance(exc_info[1], AssertionError): + exprinfo = getattr(exc_info[1], "msg", None) + if exprinfo is None: + exprinfo = saferepr(exc_info[1]) + if exprinfo and exprinfo.startswith(cls._assert_start_repr): + _striptext = "AssertionError: " + + return cls(exc_info, _striptext, _ispytest=True) + + @classmethod + def from_current( + cls, exprinfo: Optional[str] = None + ) -> "ExceptionInfo[BaseException]": + """Return an ExceptionInfo matching the current traceback. + + .. warning:: + + Experimental API + + :param exprinfo: + A text string helping to determine if we should strip + ``AssertionError`` from the output. Defaults to the exception + message/``__str__()``. + """ + tup = sys.exc_info() + assert tup[0] is not None, "no current exception" + assert tup[1] is not None, "no current exception" + assert tup[2] is not None, "no current exception" + exc_info = (tup[0], tup[1], tup[2]) + return ExceptionInfo.from_exc_info(exc_info, exprinfo) + + @classmethod + def for_later(cls) -> "ExceptionInfo[E]": + """Return an unfilled ExceptionInfo.""" + return cls(None, _ispytest=True) + + def fill_unfilled(self, exc_info: Tuple[Type[E], E, TracebackType]) -> None: + """Fill an unfilled ExceptionInfo created with ``for_later()``.""" + assert self._excinfo is None, "ExceptionInfo was already filled" + self._excinfo = exc_info + + @property + def type(self) -> Type[E]: + """The exception class.""" + assert ( + self._excinfo is not None + ), ".type can only be used after the context manager exits" + return self._excinfo[0] + + @property + def value(self) -> E: + """The exception value.""" + assert ( + self._excinfo is not None + ), ".value can only be used after the context manager exits" + return self._excinfo[1] + + @property + def tb(self) -> TracebackType: + """The exception raw traceback.""" + assert ( + self._excinfo is not None + ), ".tb can only be used after the context manager exits" + return self._excinfo[2] + + @property + def typename(self) -> str: + """The type name of the exception.""" + assert ( + self._excinfo is not None + ), ".typename can only be used after the context manager exits" + return self.type.__name__ + + @property + def traceback(self) -> Traceback: + """The traceback.""" + if self._traceback is None: + self._traceback = Traceback(self.tb, excinfo=ref(self)) + return self._traceback + + @traceback.setter + def traceback(self, value: Traceback) -> None: + self._traceback = value + + def __repr__(self) -> str: + if self._excinfo is None: + return "" + return "<{} {} tblen={}>".format( + self.__class__.__name__, saferepr(self._excinfo[1]), len(self.traceback) + ) + + def exconly(self, tryshort: bool = False) -> str: + """Return the exception as a string. + + When 'tryshort' resolves to True, and the exception is an + AssertionError, only the actual exception part of the exception + representation is returned (so 'AssertionError: ' is removed from + the beginning). + """ + lines = format_exception_only(self.type, self.value) + text = "".join(lines) + text = text.rstrip() + if tryshort: + if text.startswith(self._striptext): + text = text[len(self._striptext) :] + return text + + def errisinstance( + self, exc: Union[Type[BaseException], Tuple[Type[BaseException], ...]] + ) -> bool: + """Return True if the exception is an instance of exc. + + Consider using ``isinstance(excinfo.value, exc)`` instead. + """ + return isinstance(self.value, exc) + + def _getreprcrash(self) -> "ReprFileLocation": + exconly = self.exconly(tryshort=True) + entry = self.traceback.getcrashentry() + path, lineno = entry.frame.code.raw.co_filename, entry.lineno + return ReprFileLocation(path, lineno + 1, exconly) + + def getrepr( + self, + showlocals: bool = False, + style: "_TracebackStyle" = "long", + abspath: bool = False, + tbfilter: bool = True, + funcargs: bool = False, + truncate_locals: bool = True, + chain: bool = True, + ) -> Union["ReprExceptionInfo", "ExceptionChainRepr"]: + """Return str()able representation of this exception info. + + :param bool showlocals: + Show locals per traceback entry. + Ignored if ``style=="native"``. + + :param str style: + long|short|no|native|value traceback style. + + :param bool abspath: + If paths should be changed to absolute or left unchanged. + + :param bool tbfilter: + Hide entries that contain a local variable ``__tracebackhide__==True``. + Ignored if ``style=="native"``. + + :param bool funcargs: + Show fixtures ("funcargs" for legacy purposes) per traceback entry. + + :param bool truncate_locals: + With ``showlocals==True``, make sure locals can be safely represented as strings. + + :param bool chain: + If chained exceptions in Python 3 should be shown. + + .. versionchanged:: 3.9 + + Added the ``chain`` parameter. + """ + if style == "native": + return ReprExceptionInfo( + ReprTracebackNative( + traceback.format_exception( + self.type, self.value, self.traceback[0]._rawentry + ) + ), + self._getreprcrash(), + ) + + fmt = FormattedExcinfo( + showlocals=showlocals, + style=style, + abspath=abspath, + tbfilter=tbfilter, + funcargs=funcargs, + truncate_locals=truncate_locals, + chain=chain, + ) + return fmt.repr_excinfo(self) + + def match(self, regexp: Union[str, Pattern[str]]) -> "Literal[True]": + """Check whether the regular expression `regexp` matches the string + representation of the exception using :func:`python:re.search`. + + If it matches `True` is returned, otherwise an `AssertionError` is raised. + """ + __tracebackhide__ = True + msg = "Regex pattern {!r} does not match {!r}." + if regexp == str(self.value): + msg += " Did you mean to `re.escape()` the regex?" + assert re.search(regexp, str(self.value)), msg.format(regexp, str(self.value)) + # Return True to allow for "assert excinfo.match()". + return True + + +@attr.s(auto_attribs=True) +class FormattedExcinfo: + """Presenting information about failing Functions and Generators.""" + + # for traceback entries + flow_marker: ClassVar = ">" + fail_marker: ClassVar = "E" + + showlocals: bool = False + style: "_TracebackStyle" = "long" + abspath: bool = True + tbfilter: bool = True + funcargs: bool = False + truncate_locals: bool = True + chain: bool = True + astcache: Dict[Union[str, Path], ast.AST] = attr.ib( + factory=dict, init=False, repr=False + ) + + def _getindent(self, source: "Source") -> int: + # Figure out indent for the given source. + try: + s = str(source.getstatement(len(source) - 1)) + except KeyboardInterrupt: + raise + except BaseException: + try: + s = str(source[-1]) + except KeyboardInterrupt: + raise + except BaseException: + return 0 + return 4 + (len(s) - len(s.lstrip())) + + def _getentrysource(self, entry: TracebackEntry) -> Optional["Source"]: + source = entry.getsource(self.astcache) + if source is not None: + source = source.deindent() + return source + + def repr_args(self, entry: TracebackEntry) -> Optional["ReprFuncArgs"]: + if self.funcargs: + args = [] + for argname, argvalue in entry.frame.getargs(var=True): + args.append((argname, saferepr(argvalue))) + return ReprFuncArgs(args) + return None + + def get_source( + self, + source: Optional["Source"], + line_index: int = -1, + excinfo: Optional[ExceptionInfo[BaseException]] = None, + short: bool = False, + ) -> List[str]: + """Return formatted and marked up source lines.""" + lines = [] + if source is None or line_index >= len(source.lines): + source = Source("???") + line_index = 0 + if line_index < 0: + line_index += len(source) + space_prefix = " " + if short: + lines.append(space_prefix + source.lines[line_index].strip()) + else: + for line in source.lines[:line_index]: + lines.append(space_prefix + line) + lines.append(self.flow_marker + " " + source.lines[line_index]) + for line in source.lines[line_index + 1 :]: + lines.append(space_prefix + line) + if excinfo is not None: + indent = 4 if short else self._getindent(source) + lines.extend(self.get_exconly(excinfo, indent=indent, markall=True)) + return lines + + def get_exconly( + self, + excinfo: ExceptionInfo[BaseException], + indent: int = 4, + markall: bool = False, + ) -> List[str]: + lines = [] + indentstr = " " * indent + # Get the real exception information out. + exlines = excinfo.exconly(tryshort=True).split("\n") + failindent = self.fail_marker + indentstr[1:] + for line in exlines: + lines.append(failindent + line) + if not markall: + failindent = indentstr + return lines + + def repr_locals(self, locals: Mapping[str, object]) -> Optional["ReprLocals"]: + if self.showlocals: + lines = [] + keys = [loc for loc in locals if loc[0] != "@"] + keys.sort() + for name in keys: + value = locals[name] + if name == "__builtins__": + lines.append("__builtins__ = ") + else: + # This formatting could all be handled by the + # _repr() function, which is only reprlib.Repr in + # disguise, so is very configurable. + if self.truncate_locals: + str_repr = saferepr(value) + else: + str_repr = safeformat(value) + # if len(str_repr) < 70 or not isinstance(value, (list, tuple, dict)): + lines.append(f"{name:<10} = {str_repr}") + # else: + # self._line("%-10s =\\" % (name,)) + # # XXX + # pprint.pprint(value, stream=self.excinfowriter) + return ReprLocals(lines) + return None + + def repr_traceback_entry( + self, + entry: TracebackEntry, + excinfo: Optional[ExceptionInfo[BaseException]] = None, + ) -> "ReprEntry": + lines: List[str] = [] + style = entry._repr_style if entry._repr_style is not None else self.style + if style in ("short", "long"): + source = self._getentrysource(entry) + if source is None: + source = Source("???") + line_index = 0 + else: + line_index = entry.lineno - entry.getfirstlinesource() + short = style == "short" + reprargs = self.repr_args(entry) if not short else None + s = self.get_source(source, line_index, excinfo, short=short) + lines.extend(s) + if short: + message = "in %s" % (entry.name) + else: + message = excinfo and excinfo.typename or "" + entry_path = entry.path + path = self._makepath(entry_path) + reprfileloc = ReprFileLocation(path, entry.lineno + 1, message) + localsrepr = self.repr_locals(entry.locals) + return ReprEntry(lines, reprargs, localsrepr, reprfileloc, style) + elif style == "value": + if excinfo: + lines.extend(str(excinfo.value).split("\n")) + return ReprEntry(lines, None, None, None, style) + else: + if excinfo: + lines.extend(self.get_exconly(excinfo, indent=4)) + return ReprEntry(lines, None, None, None, style) + + def _makepath(self, path: Union[Path, str]) -> str: + if not self.abspath and isinstance(path, Path): + try: + np = bestrelpath(Path.cwd(), path) + except OSError: + return str(path) + if len(np) < len(str(path)): + return np + return str(path) + + def repr_traceback(self, excinfo: ExceptionInfo[BaseException]) -> "ReprTraceback": + traceback = excinfo.traceback + if self.tbfilter: + traceback = traceback.filter() + + if isinstance(excinfo.value, RecursionError): + traceback, extraline = self._truncate_recursive_traceback(traceback) + else: + extraline = None + + last = traceback[-1] + entries = [] + if self.style == "value": + reprentry = self.repr_traceback_entry(last, excinfo) + entries.append(reprentry) + return ReprTraceback(entries, None, style=self.style) + + for index, entry in enumerate(traceback): + einfo = (last == entry) and excinfo or None + reprentry = self.repr_traceback_entry(entry, einfo) + entries.append(reprentry) + return ReprTraceback(entries, extraline, style=self.style) + + def _truncate_recursive_traceback( + self, traceback: Traceback + ) -> Tuple[Traceback, Optional[str]]: + """Truncate the given recursive traceback trying to find the starting + point of the recursion. + + The detection is done by going through each traceback entry and + finding the point in which the locals of the frame are equal to the + locals of a previous frame (see ``recursionindex()``). + + Handle the situation where the recursion process might raise an + exception (for example comparing numpy arrays using equality raises a + TypeError), in which case we do our best to warn the user of the + error and show a limited traceback. + """ + try: + recursionindex = traceback.recursionindex() + except Exception as e: + max_frames = 10 + extraline: Optional[str] = ( + "!!! Recursion error detected, but an error occurred locating the origin of recursion.\n" + " The following exception happened when comparing locals in the stack frame:\n" + " {exc_type}: {exc_msg}\n" + " Displaying first and last {max_frames} stack frames out of {total}." + ).format( + exc_type=type(e).__name__, + exc_msg=str(e), + max_frames=max_frames, + total=len(traceback), + ) + # Type ignored because adding two instances of a List subtype + # currently incorrectly has type List instead of the subtype. + traceback = traceback[:max_frames] + traceback[-max_frames:] # type: ignore + else: + if recursionindex is not None: + extraline = "!!! Recursion detected (same locals & position)" + traceback = traceback[: recursionindex + 1] + else: + extraline = None + + return traceback, extraline + + def repr_excinfo( + self, excinfo: ExceptionInfo[BaseException] + ) -> "ExceptionChainRepr": + repr_chain: List[ + Tuple[ReprTraceback, Optional[ReprFileLocation], Optional[str]] + ] = [] + e: Optional[BaseException] = excinfo.value + excinfo_: Optional[ExceptionInfo[BaseException]] = excinfo + descr = None + seen: Set[int] = set() + while e is not None and id(e) not in seen: + seen.add(id(e)) + if excinfo_: + reprtraceback = self.repr_traceback(excinfo_) + reprcrash: Optional[ReprFileLocation] = ( + excinfo_._getreprcrash() if self.style != "value" else None + ) + else: + # Fallback to native repr if the exception doesn't have a traceback: + # ExceptionInfo objects require a full traceback to work. + reprtraceback = ReprTracebackNative( + traceback.format_exception(type(e), e, None) + ) + reprcrash = None + + repr_chain += [(reprtraceback, reprcrash, descr)] + if e.__cause__ is not None and self.chain: + e = e.__cause__ + excinfo_ = ( + ExceptionInfo.from_exc_info((type(e), e, e.__traceback__)) + if e.__traceback__ + else None + ) + descr = "The above exception was the direct cause of the following exception:" + elif ( + e.__context__ is not None and not e.__suppress_context__ and self.chain + ): + e = e.__context__ + excinfo_ = ( + ExceptionInfo.from_exc_info((type(e), e, e.__traceback__)) + if e.__traceback__ + else None + ) + descr = "During handling of the above exception, another exception occurred:" + else: + e = None + repr_chain.reverse() + return ExceptionChainRepr(repr_chain) + + +@attr.s(eq=False, auto_attribs=True) +class TerminalRepr: + def __str__(self) -> str: + # FYI this is called from pytest-xdist's serialization of exception + # information. + io = StringIO() + tw = TerminalWriter(file=io) + self.toterminal(tw) + return io.getvalue().strip() + + def __repr__(self) -> str: + return f"<{self.__class__} instance at {id(self):0x}>" + + def toterminal(self, tw: TerminalWriter) -> None: + raise NotImplementedError() + + +# This class is abstract -- only subclasses are instantiated. +@attr.s(eq=False) +class ExceptionRepr(TerminalRepr): + # Provided by subclasses. + reprcrash: Optional["ReprFileLocation"] + reprtraceback: "ReprTraceback" + + def __attrs_post_init__(self) -> None: + self.sections: List[Tuple[str, str, str]] = [] + + def addsection(self, name: str, content: str, sep: str = "-") -> None: + self.sections.append((name, content, sep)) + + def toterminal(self, tw: TerminalWriter) -> None: + for name, content, sep in self.sections: + tw.sep(sep, name) + tw.line(content) + + +@attr.s(eq=False, auto_attribs=True) +class ExceptionChainRepr(ExceptionRepr): + chain: Sequence[Tuple["ReprTraceback", Optional["ReprFileLocation"], Optional[str]]] + + def __attrs_post_init__(self) -> None: + super().__attrs_post_init__() + # reprcrash and reprtraceback of the outermost (the newest) exception + # in the chain. + self.reprtraceback = self.chain[-1][0] + self.reprcrash = self.chain[-1][1] + + def toterminal(self, tw: TerminalWriter) -> None: + for element in self.chain: + element[0].toterminal(tw) + if element[2] is not None: + tw.line("") + tw.line(element[2], yellow=True) + super().toterminal(tw) + + +@attr.s(eq=False, auto_attribs=True) +class ReprExceptionInfo(ExceptionRepr): + reprtraceback: "ReprTraceback" + reprcrash: "ReprFileLocation" + + def toterminal(self, tw: TerminalWriter) -> None: + self.reprtraceback.toterminal(tw) + super().toterminal(tw) + + +@attr.s(eq=False, auto_attribs=True) +class ReprTraceback(TerminalRepr): + reprentries: Sequence[Union["ReprEntry", "ReprEntryNative"]] + extraline: Optional[str] + style: "_TracebackStyle" + + entrysep: ClassVar = "_ " + + def toterminal(self, tw: TerminalWriter) -> None: + # The entries might have different styles. + for i, entry in enumerate(self.reprentries): + if entry.style == "long": + tw.line("") + entry.toterminal(tw) + if i < len(self.reprentries) - 1: + next_entry = self.reprentries[i + 1] + if ( + entry.style == "long" + or entry.style == "short" + and next_entry.style == "long" + ): + tw.sep(self.entrysep) + + if self.extraline: + tw.line(self.extraline) + + +class ReprTracebackNative(ReprTraceback): + def __init__(self, tblines: Sequence[str]) -> None: + self.style = "native" + self.reprentries = [ReprEntryNative(tblines)] + self.extraline = None + + +@attr.s(eq=False, auto_attribs=True) +class ReprEntryNative(TerminalRepr): + lines: Sequence[str] + + style: ClassVar["_TracebackStyle"] = "native" + + def toterminal(self, tw: TerminalWriter) -> None: + tw.write("".join(self.lines)) + + +@attr.s(eq=False, auto_attribs=True) +class ReprEntry(TerminalRepr): + lines: Sequence[str] + reprfuncargs: Optional["ReprFuncArgs"] + reprlocals: Optional["ReprLocals"] + reprfileloc: Optional["ReprFileLocation"] + style: "_TracebackStyle" + + def _write_entry_lines(self, tw: TerminalWriter) -> None: + """Write the source code portions of a list of traceback entries with syntax highlighting. + + Usually entries are lines like these: + + " x = 1" + "> assert x == 2" + "E assert 1 == 2" + + This function takes care of rendering the "source" portions of it (the lines without + the "E" prefix) using syntax highlighting, taking care to not highlighting the ">" + character, as doing so might break line continuations. + """ + + if not self.lines: + return + + # separate indents and source lines that are not failures: we want to + # highlight the code but not the indentation, which may contain markers + # such as "> assert 0" + fail_marker = f"{FormattedExcinfo.fail_marker} " + indent_size = len(fail_marker) + indents: List[str] = [] + source_lines: List[str] = [] + failure_lines: List[str] = [] + for index, line in enumerate(self.lines): + is_failure_line = line.startswith(fail_marker) + if is_failure_line: + # from this point on all lines are considered part of the failure + failure_lines.extend(self.lines[index:]) + break + else: + if self.style == "value": + source_lines.append(line) + else: + indents.append(line[:indent_size]) + source_lines.append(line[indent_size:]) + + tw._write_source(source_lines, indents) + + # failure lines are always completely red and bold + for line in failure_lines: + tw.line(line, bold=True, red=True) + + def toterminal(self, tw: TerminalWriter) -> None: + if self.style == "short": + assert self.reprfileloc is not None + self.reprfileloc.toterminal(tw) + self._write_entry_lines(tw) + if self.reprlocals: + self.reprlocals.toterminal(tw, indent=" " * 8) + return + + if self.reprfuncargs: + self.reprfuncargs.toterminal(tw) + + self._write_entry_lines(tw) + + if self.reprlocals: + tw.line("") + self.reprlocals.toterminal(tw) + if self.reprfileloc: + if self.lines: + tw.line("") + self.reprfileloc.toterminal(tw) + + def __str__(self) -> str: + return "{}\n{}\n{}".format( + "\n".join(self.lines), self.reprlocals, self.reprfileloc + ) + + +@attr.s(eq=False, auto_attribs=True) +class ReprFileLocation(TerminalRepr): + path: str = attr.ib(converter=str) + lineno: int + message: str + + def toterminal(self, tw: TerminalWriter) -> None: + # Filename and lineno output for each entry, using an output format + # that most editors understand. + msg = self.message + i = msg.find("\n") + if i != -1: + msg = msg[:i] + tw.write(self.path, bold=True, red=True) + tw.line(f":{self.lineno}: {msg}") + + +@attr.s(eq=False, auto_attribs=True) +class ReprLocals(TerminalRepr): + lines: Sequence[str] + + def toterminal(self, tw: TerminalWriter, indent="") -> None: + for line in self.lines: + tw.line(indent + line) + + +@attr.s(eq=False, auto_attribs=True) +class ReprFuncArgs(TerminalRepr): + args: Sequence[Tuple[str, object]] + + def toterminal(self, tw: TerminalWriter) -> None: + if self.args: + linesofar = "" + for name, value in self.args: + ns = f"{name} = {value}" + if len(ns) + len(linesofar) + 2 > tw.fullwidth: + if linesofar: + tw.line(linesofar) + linesofar = ns + else: + if linesofar: + linesofar += ", " + ns + else: + linesofar = ns + if linesofar: + tw.line(linesofar) + tw.line("") + + +def getfslineno(obj: object) -> Tuple[Union[str, Path], int]: + """Return source location (path, lineno) for the given object. + + If the source cannot be determined return ("", -1). + + The line number is 0-based. + """ + # xxx let decorators etc specify a sane ordering + # NOTE: this used to be done in _pytest.compat.getfslineno, initially added + # in 6ec13a2b9. It ("place_as") appears to be something very custom. + obj = get_real_func(obj) + if hasattr(obj, "place_as"): + obj = obj.place_as # type: ignore[attr-defined] + + try: + code = Code.from_function(obj) + except TypeError: + try: + fn = inspect.getsourcefile(obj) or inspect.getfile(obj) # type: ignore[arg-type] + except TypeError: + return "", -1 + + fspath = fn and absolutepath(fn) or "" + lineno = -1 + if fspath: + try: + _, lineno = findsource(obj) + except OSError: + pass + return fspath, lineno + + return code.path, code.firstlineno + + +# Relative paths that we use to filter traceback entries from appearing to the user; +# see filter_traceback. +# note: if we need to add more paths than what we have now we should probably use a list +# for better maintenance. + +_PLUGGY_DIR = Path(pluggy.__file__.rstrip("oc")) +# pluggy is either a package or a single module depending on the version +if _PLUGGY_DIR.name == "__init__.py": + _PLUGGY_DIR = _PLUGGY_DIR.parent +_PYTEST_DIR = Path(_pytest.__file__).parent + + +def filter_traceback(entry: TracebackEntry) -> bool: + """Return True if a TracebackEntry instance should be included in tracebacks. + + We hide traceback entries of: + + * dynamically generated code (no code to show up for it); + * internal traceback from pytest or its internal libraries, py and pluggy. + """ + # entry.path might sometimes return a str object when the entry + # points to dynamically generated code. + # See https://bitbucket.org/pytest-dev/py/issues/71. + raw_filename = entry.frame.code.raw.co_filename + is_generated = "<" in raw_filename and ">" in raw_filename + if is_generated: + return False + + # entry.path might point to a non-existing file, in which case it will + # also return a str object. See #1133. + p = Path(entry.path) + + parents = p.parents + if _PLUGGY_DIR in parents: + return False + if _PYTEST_DIR in parents: + return False + + return True diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_code/source.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_code/source.py new file mode 100644 index 0000000000..208cfb8003 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_code/source.py @@ -0,0 +1,217 @@ +import ast +import inspect +import textwrap +import tokenize +import types +import warnings +from bisect import bisect_right +from typing import Iterable +from typing import Iterator +from typing import List +from typing import Optional +from typing import overload +from typing import Tuple +from typing import Union + + +class Source: + """An immutable object holding a source code fragment. + + When using Source(...), the source lines are deindented. + """ + + def __init__(self, obj: object = None) -> None: + if not obj: + self.lines: List[str] = [] + elif isinstance(obj, Source): + self.lines = obj.lines + elif isinstance(obj, (tuple, list)): + self.lines = deindent(x.rstrip("\n") for x in obj) + elif isinstance(obj, str): + self.lines = deindent(obj.split("\n")) + else: + try: + rawcode = getrawcode(obj) + src = inspect.getsource(rawcode) + except TypeError: + src = inspect.getsource(obj) # type: ignore[arg-type] + self.lines = deindent(src.split("\n")) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Source): + return NotImplemented + return self.lines == other.lines + + # Ignore type because of https://github.com/python/mypy/issues/4266. + __hash__ = None # type: ignore + + @overload + def __getitem__(self, key: int) -> str: + ... + + @overload + def __getitem__(self, key: slice) -> "Source": + ... + + def __getitem__(self, key: Union[int, slice]) -> Union[str, "Source"]: + if isinstance(key, int): + return self.lines[key] + else: + if key.step not in (None, 1): + raise IndexError("cannot slice a Source with a step") + newsource = Source() + newsource.lines = self.lines[key.start : key.stop] + return newsource + + def __iter__(self) -> Iterator[str]: + return iter(self.lines) + + def __len__(self) -> int: + return len(self.lines) + + def strip(self) -> "Source": + """Return new Source object with trailing and leading blank lines removed.""" + start, end = 0, len(self) + while start < end and not self.lines[start].strip(): + start += 1 + while end > start and not self.lines[end - 1].strip(): + end -= 1 + source = Source() + source.lines[:] = self.lines[start:end] + return source + + def indent(self, indent: str = " " * 4) -> "Source": + """Return a copy of the source object with all lines indented by the + given indent-string.""" + newsource = Source() + newsource.lines = [(indent + line) for line in self.lines] + return newsource + + def getstatement(self, lineno: int) -> "Source": + """Return Source statement which contains the given linenumber + (counted from 0).""" + start, end = self.getstatementrange(lineno) + return self[start:end] + + def getstatementrange(self, lineno: int) -> Tuple[int, int]: + """Return (start, end) tuple which spans the minimal statement region + which containing the given lineno.""" + if not (0 <= lineno < len(self)): + raise IndexError("lineno out of range") + ast, start, end = getstatementrange_ast(lineno, self) + return start, end + + def deindent(self) -> "Source": + """Return a new Source object deindented.""" + newsource = Source() + newsource.lines[:] = deindent(self.lines) + return newsource + + def __str__(self) -> str: + return "\n".join(self.lines) + + +# +# helper functions +# + + +def findsource(obj) -> Tuple[Optional[Source], int]: + try: + sourcelines, lineno = inspect.findsource(obj) + except Exception: + return None, -1 + source = Source() + source.lines = [line.rstrip() for line in sourcelines] + return source, lineno + + +def getrawcode(obj: object, trycall: bool = True) -> types.CodeType: + """Return code object for given function.""" + try: + return obj.__code__ # type: ignore[attr-defined,no-any-return] + except AttributeError: + pass + if trycall: + call = getattr(obj, "__call__", None) + if call and not isinstance(obj, type): + return getrawcode(call, trycall=False) + raise TypeError(f"could not get code object for {obj!r}") + + +def deindent(lines: Iterable[str]) -> List[str]: + return textwrap.dedent("\n".join(lines)).splitlines() + + +def get_statement_startend2(lineno: int, node: ast.AST) -> Tuple[int, Optional[int]]: + # Flatten all statements and except handlers into one lineno-list. + # AST's line numbers start indexing at 1. + values: List[int] = [] + for x in ast.walk(node): + if isinstance(x, (ast.stmt, ast.ExceptHandler)): + # Before Python 3.8, the lineno of a decorated class or function pointed at the decorator. + # Since Python 3.8, the lineno points to the class/def, so need to include the decorators. + if isinstance(x, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + for d in x.decorator_list: + values.append(d.lineno - 1) + values.append(x.lineno - 1) + for name in ("finalbody", "orelse"): + val: Optional[List[ast.stmt]] = getattr(x, name, None) + if val: + # Treat the finally/orelse part as its own statement. + values.append(val[0].lineno - 1 - 1) + values.sort() + insert_index = bisect_right(values, lineno) + start = values[insert_index - 1] + if insert_index >= len(values): + end = None + else: + end = values[insert_index] + return start, end + + +def getstatementrange_ast( + lineno: int, + source: Source, + assertion: bool = False, + astnode: Optional[ast.AST] = None, +) -> Tuple[ast.AST, int, int]: + if astnode is None: + content = str(source) + # See #4260: + # Don't produce duplicate warnings when compiling source to find AST. + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + astnode = ast.parse(content, "source", "exec") + + start, end = get_statement_startend2(lineno, astnode) + # We need to correct the end: + # - ast-parsing strips comments + # - there might be empty lines + # - we might have lesser indented code blocks at the end + if end is None: + end = len(source.lines) + + if end > start + 1: + # Make sure we don't span differently indented code blocks + # by using the BlockFinder helper used which inspect.getsource() uses itself. + block_finder = inspect.BlockFinder() + # If we start with an indented line, put blockfinder to "started" mode. + block_finder.started = source.lines[start][0].isspace() + it = ((x + "\n") for x in source.lines[start:end]) + try: + for tok in tokenize.generate_tokens(lambda: next(it)): + block_finder.tokeneater(*tok) + except (inspect.EndOfBlock, IndentationError): + end = block_finder.last + start + except Exception: + pass + + # The end might still point to a comment or empty line, correct it. + while end: + line = source.lines[end - 1].lstrip() + if line.startswith("#") or not line: + end -= 1 + else: + break + return astnode, start, end diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/__init__.py new file mode 100644 index 0000000000..db001e918c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/__init__.py @@ -0,0 +1,8 @@ +from .terminalwriter import get_terminal_width +from .terminalwriter import TerminalWriter + + +__all__ = [ + "TerminalWriter", + "get_terminal_width", +] diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/saferepr.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/saferepr.py new file mode 100644 index 0000000000..e7ff5cab20 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/saferepr.py @@ -0,0 +1,153 @@ +import pprint +import reprlib +from typing import Any +from typing import Dict +from typing import IO +from typing import Optional + + +def _try_repr_or_str(obj: object) -> str: + try: + return repr(obj) + except (KeyboardInterrupt, SystemExit): + raise + except BaseException: + return f'{type(obj).__name__}("{obj}")' + + +def _format_repr_exception(exc: BaseException, obj: object) -> str: + try: + exc_info = _try_repr_or_str(exc) + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as exc: + exc_info = f"unpresentable exception ({_try_repr_or_str(exc)})" + return "<[{} raised in repr()] {} object at 0x{:x}>".format( + exc_info, type(obj).__name__, id(obj) + ) + + +def _ellipsize(s: str, maxsize: int) -> str: + if len(s) > maxsize: + i = max(0, (maxsize - 3) // 2) + j = max(0, maxsize - 3 - i) + return s[:i] + "..." + s[len(s) - j :] + return s + + +class SafeRepr(reprlib.Repr): + """ + repr.Repr that limits the resulting size of repr() and includes + information on exceptions raised during the call. + """ + + def __init__(self, maxsize: Optional[int]) -> None: + """ + :param maxsize: + If not None, will truncate the resulting repr to that specific size, using ellipsis + somewhere in the middle to hide the extra text. + If None, will not impose any size limits on the returning repr. + """ + super().__init__() + # ``maxstring`` is used by the superclass, and needs to be an int; using a + # very large number in case maxsize is None, meaning we want to disable + # truncation. + self.maxstring = maxsize if maxsize is not None else 1_000_000_000 + self.maxsize = maxsize + + def repr(self, x: object) -> str: + try: + s = super().repr(x) + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as exc: + s = _format_repr_exception(exc, x) + if self.maxsize is not None: + s = _ellipsize(s, self.maxsize) + return s + + def repr_instance(self, x: object, level: int) -> str: + try: + s = repr(x) + except (KeyboardInterrupt, SystemExit): + raise + except BaseException as exc: + s = _format_repr_exception(exc, x) + if self.maxsize is not None: + s = _ellipsize(s, self.maxsize) + return s + + +def safeformat(obj: object) -> str: + """Return a pretty printed string for the given object. + + Failing __repr__ functions of user instances will be represented + with a short exception info. + """ + try: + return pprint.pformat(obj) + except Exception as exc: + return _format_repr_exception(exc, obj) + + +# Maximum size of overall repr of objects to display during assertion errors. +DEFAULT_REPR_MAX_SIZE = 240 + + +def saferepr(obj: object, maxsize: Optional[int] = DEFAULT_REPR_MAX_SIZE) -> str: + """Return a size-limited safe repr-string for the given object. + + Failing __repr__ functions of user instances will be represented + with a short exception info and 'saferepr' generally takes + care to never raise exceptions itself. + + This function is a wrapper around the Repr/reprlib functionality of the + stdlib. + """ + return SafeRepr(maxsize).repr(obj) + + +class AlwaysDispatchingPrettyPrinter(pprint.PrettyPrinter): + """PrettyPrinter that always dispatches (regardless of width).""" + + def _format( + self, + object: object, + stream: IO[str], + indent: int, + allowance: int, + context: Dict[int, Any], + level: int, + ) -> None: + # Type ignored because _dispatch is private. + p = self._dispatch.get(type(object).__repr__, None) # type: ignore[attr-defined] + + objid = id(object) + if objid in context or p is None: + # Type ignored because _format is private. + super()._format( # type: ignore[misc] + object, + stream, + indent, + allowance, + context, + level, + ) + return + + context[objid] = 1 + p(self, object, stream, indent, allowance, context, level + 1) + del context[objid] + + +def _pformat_dispatch( + object: object, + indent: int = 1, + width: int = 80, + depth: Optional[int] = None, + *, + compact: bool = False, +) -> str: + return AlwaysDispatchingPrettyPrinter( + indent=indent, width=width, depth=depth, compact=compact + ).pformat(object) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/terminalwriter.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/terminalwriter.py new file mode 100644 index 0000000000..379035d858 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/terminalwriter.py @@ -0,0 +1,233 @@ +"""Helper functions for writing to terminals and files.""" +import os +import shutil +import sys +from typing import Optional +from typing import Sequence +from typing import TextIO + +from .wcwidth import wcswidth +from _pytest.compat import final + + +# This code was initially copied from py 1.8.1, file _io/terminalwriter.py. + + +def get_terminal_width() -> int: + width, _ = shutil.get_terminal_size(fallback=(80, 24)) + + # The Windows get_terminal_size may be bogus, let's sanify a bit. + if width < 40: + width = 80 + + return width + + +def should_do_markup(file: TextIO) -> bool: + if os.environ.get("PY_COLORS") == "1": + return True + if os.environ.get("PY_COLORS") == "0": + return False + if "NO_COLOR" in os.environ: + return False + if "FORCE_COLOR" in os.environ: + return True + return ( + hasattr(file, "isatty") and file.isatty() and os.environ.get("TERM") != "dumb" + ) + + +@final +class TerminalWriter: + _esctable = dict( + black=30, + red=31, + green=32, + yellow=33, + blue=34, + purple=35, + cyan=36, + white=37, + Black=40, + Red=41, + Green=42, + Yellow=43, + Blue=44, + Purple=45, + Cyan=46, + White=47, + bold=1, + light=2, + blink=5, + invert=7, + ) + + def __init__(self, file: Optional[TextIO] = None) -> None: + if file is None: + file = sys.stdout + if hasattr(file, "isatty") and file.isatty() and sys.platform == "win32": + try: + import colorama + except ImportError: + pass + else: + file = colorama.AnsiToWin32(file).stream + assert file is not None + self._file = file + self.hasmarkup = should_do_markup(file) + self._current_line = "" + self._terminal_width: Optional[int] = None + self.code_highlight = True + + @property + def fullwidth(self) -> int: + if self._terminal_width is not None: + return self._terminal_width + return get_terminal_width() + + @fullwidth.setter + def fullwidth(self, value: int) -> None: + self._terminal_width = value + + @property + def width_of_current_line(self) -> int: + """Return an estimate of the width so far in the current line.""" + return wcswidth(self._current_line) + + def markup(self, text: str, **markup: bool) -> str: + for name in markup: + if name not in self._esctable: + raise ValueError(f"unknown markup: {name!r}") + if self.hasmarkup: + esc = [self._esctable[name] for name, on in markup.items() if on] + if esc: + text = "".join("\x1b[%sm" % cod for cod in esc) + text + "\x1b[0m" + return text + + def sep( + self, + sepchar: str, + title: Optional[str] = None, + fullwidth: Optional[int] = None, + **markup: bool, + ) -> None: + if fullwidth is None: + fullwidth = self.fullwidth + # The goal is to have the line be as long as possible + # under the condition that len(line) <= fullwidth. + if sys.platform == "win32": + # If we print in the last column on windows we are on a + # new line but there is no way to verify/neutralize this + # (we may not know the exact line width). + # So let's be defensive to avoid empty lines in the output. + fullwidth -= 1 + if title is not None: + # we want 2 + 2*len(fill) + len(title) <= fullwidth + # i.e. 2 + 2*len(sepchar)*N + len(title) <= fullwidth + # 2*len(sepchar)*N <= fullwidth - len(title) - 2 + # N <= (fullwidth - len(title) - 2) // (2*len(sepchar)) + N = max((fullwidth - len(title) - 2) // (2 * len(sepchar)), 1) + fill = sepchar * N + line = f"{fill} {title} {fill}" + else: + # we want len(sepchar)*N <= fullwidth + # i.e. N <= fullwidth // len(sepchar) + line = sepchar * (fullwidth // len(sepchar)) + # In some situations there is room for an extra sepchar at the right, + # in particular if we consider that with a sepchar like "_ " the + # trailing space is not important at the end of the line. + if len(line) + len(sepchar.rstrip()) <= fullwidth: + line += sepchar.rstrip() + + self.line(line, **markup) + + def write(self, msg: str, *, flush: bool = False, **markup: bool) -> None: + if msg: + current_line = msg.rsplit("\n", 1)[-1] + if "\n" in msg: + self._current_line = current_line + else: + self._current_line += current_line + + msg = self.markup(msg, **markup) + + try: + self._file.write(msg) + except UnicodeEncodeError: + # Some environments don't support printing general Unicode + # strings, due to misconfiguration or otherwise; in that case, + # print the string escaped to ASCII. + # When the Unicode situation improves we should consider + # letting the error propagate instead of masking it (see #7475 + # for one brief attempt). + msg = msg.encode("unicode-escape").decode("ascii") + self._file.write(msg) + + if flush: + self.flush() + + def line(self, s: str = "", **markup: bool) -> None: + self.write(s, **markup) + self.write("\n") + + def flush(self) -> None: + self._file.flush() + + def _write_source(self, lines: Sequence[str], indents: Sequence[str] = ()) -> None: + """Write lines of source code possibly highlighted. + + Keeping this private for now because the API is clunky. We should discuss how + to evolve the terminal writer so we can have more precise color support, for example + being able to write part of a line in one color and the rest in another, and so on. + """ + if indents and len(indents) != len(lines): + raise ValueError( + "indents size ({}) should have same size as lines ({})".format( + len(indents), len(lines) + ) + ) + if not indents: + indents = [""] * len(lines) + source = "\n".join(lines) + new_lines = self._highlight(source).splitlines() + for indent, new_line in zip(indents, new_lines): + self.line(indent + new_line) + + def _highlight(self, source: str) -> str: + """Highlight the given source code if we have markup support.""" + from _pytest.config.exceptions import UsageError + + if not self.hasmarkup or not self.code_highlight: + return source + try: + from pygments.formatters.terminal import TerminalFormatter + from pygments.lexers.python import PythonLexer + from pygments import highlight + import pygments.util + except ImportError: + return source + else: + try: + highlighted: str = highlight( + source, + PythonLexer(), + TerminalFormatter( + bg=os.getenv("PYTEST_THEME_MODE", "dark"), + style=os.getenv("PYTEST_THEME"), + ), + ) + return highlighted + except pygments.util.ClassNotFound: + raise UsageError( + "PYTEST_THEME environment variable had an invalid value: '{}'. " + "Only valid pygment styles are allowed.".format( + os.getenv("PYTEST_THEME") + ) + ) + except pygments.util.OptionError: + raise UsageError( + "PYTEST_THEME_MODE environment variable had an invalid value: '{}'. " + "The only allowed values are 'dark' and 'light'.".format( + os.getenv("PYTEST_THEME_MODE") + ) + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/wcwidth.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/wcwidth.py new file mode 100644 index 0000000000..e5c7bf4d86 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_io/wcwidth.py @@ -0,0 +1,55 @@ +import unicodedata +from functools import lru_cache + + +@lru_cache(100) +def wcwidth(c: str) -> int: + """Determine how many columns are needed to display a character in a terminal. + + Returns -1 if the character is not printable. + Returns 0, 1 or 2 for other characters. + """ + o = ord(c) + + # ASCII fast path. + if 0x20 <= o < 0x07F: + return 1 + + # Some Cf/Zp/Zl characters which should be zero-width. + if ( + o == 0x0000 + or 0x200B <= o <= 0x200F + or 0x2028 <= o <= 0x202E + or 0x2060 <= o <= 0x2063 + ): + return 0 + + category = unicodedata.category(c) + + # Control characters. + if category == "Cc": + return -1 + + # Combining characters with zero width. + if category in ("Me", "Mn"): + return 0 + + # Full/Wide east asian characters. + if unicodedata.east_asian_width(c) in ("F", "W"): + return 2 + + return 1 + + +def wcswidth(s: str) -> int: + """Determine how many columns are needed to display a string in a terminal. + + Returns -1 if the string contains non-printable characters. + """ + width = 0 + for c in unicodedata.normalize("NFC", s): + wc = wcwidth(c) + if wc < 0: + return -1 + width += wc + return width diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_version.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_version.py new file mode 100644 index 0000000000..5515abadad --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/_version.py @@ -0,0 +1,5 @@ +# coding: utf-8 +# file generated by setuptools_scm +# don't change, don't track in version control +version = '7.0.1' +version_tuple = (7, 0, 1) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/__init__.py new file mode 100644 index 0000000000..480a26ad86 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/__init__.py @@ -0,0 +1,181 @@ +"""Support for presenting detailed information in failing assertions.""" +import sys +from typing import Any +from typing import Generator +from typing import List +from typing import Optional +from typing import TYPE_CHECKING + +from _pytest.assertion import rewrite +from _pytest.assertion import truncate +from _pytest.assertion import util +from _pytest.assertion.rewrite import assertstate_key +from _pytest.config import Config +from _pytest.config import hookimpl +from _pytest.config.argparsing import Parser +from _pytest.nodes import Item + +if TYPE_CHECKING: + from _pytest.main import Session + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("debugconfig") + group.addoption( + "--assert", + action="store", + dest="assertmode", + choices=("rewrite", "plain"), + default="rewrite", + metavar="MODE", + help=( + "Control assertion debugging tools.\n" + "'plain' performs no assertion debugging.\n" + "'rewrite' (the default) rewrites assert statements in test modules" + " on import to provide assert expression information." + ), + ) + parser.addini( + "enable_assertion_pass_hook", + type="bool", + default=False, + help="Enables the pytest_assertion_pass hook." + "Make sure to delete any previously generated pyc cache files.", + ) + + +def register_assert_rewrite(*names: str) -> None: + """Register one or more module names to be rewritten on import. + + This function will make sure that this module or all modules inside + the package will get their assert statements rewritten. + Thus you should make sure to call this before the module is + actually imported, usually in your __init__.py if you are a plugin + using a package. + + :raises TypeError: If the given module names are not strings. + """ + for name in names: + if not isinstance(name, str): + msg = "expected module names as *args, got {0} instead" # type: ignore[unreachable] + raise TypeError(msg.format(repr(names))) + for hook in sys.meta_path: + if isinstance(hook, rewrite.AssertionRewritingHook): + importhook = hook + break + else: + # TODO(typing): Add a protocol for mark_rewrite() and use it + # for importhook and for PytestPluginManager.rewrite_hook. + importhook = DummyRewriteHook() # type: ignore + importhook.mark_rewrite(*names) + + +class DummyRewriteHook: + """A no-op import hook for when rewriting is disabled.""" + + def mark_rewrite(self, *names: str) -> None: + pass + + +class AssertionState: + """State for the assertion plugin.""" + + def __init__(self, config: Config, mode) -> None: + self.mode = mode + self.trace = config.trace.root.get("assertion") + self.hook: Optional[rewrite.AssertionRewritingHook] = None + + +def install_importhook(config: Config) -> rewrite.AssertionRewritingHook: + """Try to install the rewrite hook, raise SystemError if it fails.""" + config.stash[assertstate_key] = AssertionState(config, "rewrite") + config.stash[assertstate_key].hook = hook = rewrite.AssertionRewritingHook(config) + sys.meta_path.insert(0, hook) + config.stash[assertstate_key].trace("installed rewrite import hook") + + def undo() -> None: + hook = config.stash[assertstate_key].hook + if hook is not None and hook in sys.meta_path: + sys.meta_path.remove(hook) + + config.add_cleanup(undo) + return hook + + +def pytest_collection(session: "Session") -> None: + # This hook is only called when test modules are collected + # so for example not in the managing process of pytest-xdist + # (which does not collect test modules). + assertstate = session.config.stash.get(assertstate_key, None) + if assertstate: + if assertstate.hook is not None: + assertstate.hook.set_session(session) + + +@hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_protocol(item: Item) -> Generator[None, None, None]: + """Setup the pytest_assertrepr_compare and pytest_assertion_pass hooks. + + The rewrite module will use util._reprcompare if it exists to use custom + reporting via the pytest_assertrepr_compare hook. This sets up this custom + comparison for the test. + """ + + ihook = item.ihook + + def callbinrepr(op, left: object, right: object) -> Optional[str]: + """Call the pytest_assertrepr_compare hook and prepare the result. + + This uses the first result from the hook and then ensures the + following: + * Overly verbose explanations are truncated unless configured otherwise + (eg. if running in verbose mode). + * Embedded newlines are escaped to help util.format_explanation() + later. + * If the rewrite mode is used embedded %-characters are replaced + to protect later % formatting. + + The result can be formatted by util.format_explanation() for + pretty printing. + """ + hook_result = ihook.pytest_assertrepr_compare( + config=item.config, op=op, left=left, right=right + ) + for new_expl in hook_result: + if new_expl: + new_expl = truncate.truncate_if_required(new_expl, item) + new_expl = [line.replace("\n", "\\n") for line in new_expl] + res = "\n~".join(new_expl) + if item.config.getvalue("assertmode") == "rewrite": + res = res.replace("%", "%%") + return res + return None + + saved_assert_hooks = util._reprcompare, util._assertion_pass + util._reprcompare = callbinrepr + util._config = item.config + + if ihook.pytest_assertion_pass.get_hookimpls(): + + def call_assertion_pass_hook(lineno: int, orig: str, expl: str) -> None: + ihook.pytest_assertion_pass(item=item, lineno=lineno, orig=orig, expl=expl) + + util._assertion_pass = call_assertion_pass_hook + + yield + + util._reprcompare, util._assertion_pass = saved_assert_hooks + util._config = None + + +def pytest_sessionfinish(session: "Session") -> None: + assertstate = session.config.stash.get(assertstate_key, None) + if assertstate: + if assertstate.hook is not None: + assertstate.hook.set_session(None) + + +def pytest_assertrepr_compare( + config: Config, op: str, left: Any, right: Any +) -> Optional[List[str]]: + return util.assertrepr_compare(config=config, op=op, left=left, right=right) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/rewrite.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/rewrite.py new file mode 100644 index 0000000000..88ac6cab36 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/rewrite.py @@ -0,0 +1,1136 @@ +"""Rewrite assertion AST to produce nice error messages.""" +import ast +import errno +import functools +import importlib.abc +import importlib.machinery +import importlib.util +import io +import itertools +import marshal +import os +import struct +import sys +import tokenize +import types +from pathlib import Path +from pathlib import PurePath +from typing import Callable +from typing import Dict +from typing import IO +from typing import Iterable +from typing import Iterator +from typing import List +from typing import Optional +from typing import Sequence +from typing import Set +from typing import Tuple +from typing import TYPE_CHECKING +from typing import Union + +from _pytest._io.saferepr import DEFAULT_REPR_MAX_SIZE +from _pytest._io.saferepr import saferepr +from _pytest._version import version +from _pytest.assertion import util +from _pytest.assertion.util import ( # noqa: F401 + format_explanation as _format_explanation, +) +from _pytest.config import Config +from _pytest.main import Session +from _pytest.pathlib import absolutepath +from _pytest.pathlib import fnmatch_ex +from _pytest.stash import StashKey + +if TYPE_CHECKING: + from _pytest.assertion import AssertionState + + +assertstate_key = StashKey["AssertionState"]() + + +# pytest caches rewritten pycs in pycache dirs +PYTEST_TAG = f"{sys.implementation.cache_tag}-pytest-{version}" +PYC_EXT = ".py" + (__debug__ and "c" or "o") +PYC_TAIL = "." + PYTEST_TAG + PYC_EXT + + +class AssertionRewritingHook(importlib.abc.MetaPathFinder, importlib.abc.Loader): + """PEP302/PEP451 import hook which rewrites asserts.""" + + def __init__(self, config: Config) -> None: + self.config = config + try: + self.fnpats = config.getini("python_files") + except ValueError: + self.fnpats = ["test_*.py", "*_test.py"] + self.session: Optional[Session] = None + self._rewritten_names: Dict[str, Path] = {} + self._must_rewrite: Set[str] = set() + # flag to guard against trying to rewrite a pyc file while we are already writing another pyc file, + # which might result in infinite recursion (#3506) + self._writing_pyc = False + self._basenames_to_check_rewrite = {"conftest"} + self._marked_for_rewrite_cache: Dict[str, bool] = {} + self._session_paths_checked = False + + def set_session(self, session: Optional[Session]) -> None: + self.session = session + self._session_paths_checked = False + + # Indirection so we can mock calls to find_spec originated from the hook during testing + _find_spec = importlib.machinery.PathFinder.find_spec + + def find_spec( + self, + name: str, + path: Optional[Sequence[Union[str, bytes]]] = None, + target: Optional[types.ModuleType] = None, + ) -> Optional[importlib.machinery.ModuleSpec]: + if self._writing_pyc: + return None + state = self.config.stash[assertstate_key] + if self._early_rewrite_bailout(name, state): + return None + state.trace("find_module called for: %s" % name) + + # Type ignored because mypy is confused about the `self` binding here. + spec = self._find_spec(name, path) # type: ignore + if ( + # the import machinery could not find a file to import + spec is None + # this is a namespace package (without `__init__.py`) + # there's nothing to rewrite there + # python3.6: `namespace` + # python3.7+: `None` + or spec.origin == "namespace" + or spec.origin is None + # we can only rewrite source files + or not isinstance(spec.loader, importlib.machinery.SourceFileLoader) + # if the file doesn't exist, we can't rewrite it + or not os.path.exists(spec.origin) + ): + return None + else: + fn = spec.origin + + if not self._should_rewrite(name, fn, state): + return None + + return importlib.util.spec_from_file_location( + name, + fn, + loader=self, + submodule_search_locations=spec.submodule_search_locations, + ) + + def create_module( + self, spec: importlib.machinery.ModuleSpec + ) -> Optional[types.ModuleType]: + return None # default behaviour is fine + + def exec_module(self, module: types.ModuleType) -> None: + assert module.__spec__ is not None + assert module.__spec__.origin is not None + fn = Path(module.__spec__.origin) + state = self.config.stash[assertstate_key] + + self._rewritten_names[module.__name__] = fn + + # The requested module looks like a test file, so rewrite it. This is + # the most magical part of the process: load the source, rewrite the + # asserts, and load the rewritten source. We also cache the rewritten + # module code in a special pyc. We must be aware of the possibility of + # concurrent pytest processes rewriting and loading pycs. To avoid + # tricky race conditions, we maintain the following invariant: The + # cached pyc is always a complete, valid pyc. Operations on it must be + # atomic. POSIX's atomic rename comes in handy. + write = not sys.dont_write_bytecode + cache_dir = get_cache_dir(fn) + if write: + ok = try_makedirs(cache_dir) + if not ok: + write = False + state.trace(f"read only directory: {cache_dir}") + + cache_name = fn.name[:-3] + PYC_TAIL + pyc = cache_dir / cache_name + # Notice that even if we're in a read-only directory, I'm going + # to check for a cached pyc. This may not be optimal... + co = _read_pyc(fn, pyc, state.trace) + if co is None: + state.trace(f"rewriting {fn!r}") + source_stat, co = _rewrite_test(fn, self.config) + if write: + self._writing_pyc = True + try: + _write_pyc(state, co, source_stat, pyc) + finally: + self._writing_pyc = False + else: + state.trace(f"found cached rewritten pyc for {fn}") + exec(co, module.__dict__) + + def _early_rewrite_bailout(self, name: str, state: "AssertionState") -> bool: + """A fast way to get out of rewriting modules. + + Profiling has shown that the call to PathFinder.find_spec (inside of + the find_spec from this class) is a major slowdown, so, this method + tries to filter what we're sure won't be rewritten before getting to + it. + """ + if self.session is not None and not self._session_paths_checked: + self._session_paths_checked = True + for initial_path in self.session._initialpaths: + # Make something as c:/projects/my_project/path.py -> + # ['c:', 'projects', 'my_project', 'path.py'] + parts = str(initial_path).split(os.path.sep) + # add 'path' to basenames to be checked. + self._basenames_to_check_rewrite.add(os.path.splitext(parts[-1])[0]) + + # Note: conftest already by default in _basenames_to_check_rewrite. + parts = name.split(".") + if parts[-1] in self._basenames_to_check_rewrite: + return False + + # For matching the name it must be as if it was a filename. + path = PurePath(os.path.sep.join(parts) + ".py") + + for pat in self.fnpats: + # if the pattern contains subdirectories ("tests/**.py" for example) we can't bail out based + # on the name alone because we need to match against the full path + if os.path.dirname(pat): + return False + if fnmatch_ex(pat, path): + return False + + if self._is_marked_for_rewrite(name, state): + return False + + state.trace(f"early skip of rewriting module: {name}") + return True + + def _should_rewrite(self, name: str, fn: str, state: "AssertionState") -> bool: + # always rewrite conftest files + if os.path.basename(fn) == "conftest.py": + state.trace(f"rewriting conftest file: {fn!r}") + return True + + if self.session is not None: + if self.session.isinitpath(absolutepath(fn)): + state.trace(f"matched test file (was specified on cmdline): {fn!r}") + return True + + # modules not passed explicitly on the command line are only + # rewritten if they match the naming convention for test files + fn_path = PurePath(fn) + for pat in self.fnpats: + if fnmatch_ex(pat, fn_path): + state.trace(f"matched test file {fn!r}") + return True + + return self._is_marked_for_rewrite(name, state) + + def _is_marked_for_rewrite(self, name: str, state: "AssertionState") -> bool: + try: + return self._marked_for_rewrite_cache[name] + except KeyError: + for marked in self._must_rewrite: + if name == marked or name.startswith(marked + "."): + state.trace(f"matched marked file {name!r} (from {marked!r})") + self._marked_for_rewrite_cache[name] = True + return True + + self._marked_for_rewrite_cache[name] = False + return False + + def mark_rewrite(self, *names: str) -> None: + """Mark import names as needing to be rewritten. + + The named module or package as well as any nested modules will + be rewritten on import. + """ + already_imported = ( + set(names).intersection(sys.modules).difference(self._rewritten_names) + ) + for name in already_imported: + mod = sys.modules[name] + if not AssertionRewriter.is_rewrite_disabled( + mod.__doc__ or "" + ) and not isinstance(mod.__loader__, type(self)): + self._warn_already_imported(name) + self._must_rewrite.update(names) + self._marked_for_rewrite_cache.clear() + + def _warn_already_imported(self, name: str) -> None: + from _pytest.warning_types import PytestAssertRewriteWarning + + self.config.issue_config_time_warning( + PytestAssertRewriteWarning( + "Module already imported so cannot be rewritten: %s" % name + ), + stacklevel=5, + ) + + def get_data(self, pathname: Union[str, bytes]) -> bytes: + """Optional PEP302 get_data API.""" + with open(pathname, "rb") as f: + return f.read() + + if sys.version_info >= (3, 10): + + def get_resource_reader(self, name: str) -> importlib.abc.TraversableResources: # type: ignore + if sys.version_info < (3, 11): + from importlib.readers import FileReader + else: + from importlib.resources.readers import FileReader + + return FileReader(types.SimpleNamespace(path=self._rewritten_names[name])) + + +def _write_pyc_fp( + fp: IO[bytes], source_stat: os.stat_result, co: types.CodeType +) -> None: + # Technically, we don't have to have the same pyc format as + # (C)Python, since these "pycs" should never be seen by builtin + # import. However, there's little reason to deviate. + fp.write(importlib.util.MAGIC_NUMBER) + # https://www.python.org/dev/peps/pep-0552/ + if sys.version_info >= (3, 7): + flags = b"\x00\x00\x00\x00" + fp.write(flags) + # as of now, bytecode header expects 32-bit numbers for size and mtime (#4903) + mtime = int(source_stat.st_mtime) & 0xFFFFFFFF + size = source_stat.st_size & 0xFFFFFFFF + # " bool: + try: + with atomic_write(os.fspath(pyc), mode="wb", overwrite=True) as fp: + _write_pyc_fp(fp, source_stat, co) + except OSError as e: + state.trace(f"error writing pyc file at {pyc}: {e}") + # we ignore any failure to write the cache file + # there are many reasons, permission-denied, pycache dir being a + # file etc. + return False + return True + + +else: + + def _write_pyc( + state: "AssertionState", + co: types.CodeType, + source_stat: os.stat_result, + pyc: Path, + ) -> bool: + proc_pyc = f"{pyc}.{os.getpid()}" + try: + fp = open(proc_pyc, "wb") + except OSError as e: + state.trace(f"error writing pyc file at {proc_pyc}: errno={e.errno}") + return False + + try: + _write_pyc_fp(fp, source_stat, co) + os.rename(proc_pyc, pyc) + except OSError as e: + state.trace(f"error writing pyc file at {pyc}: {e}") + # we ignore any failure to write the cache file + # there are many reasons, permission-denied, pycache dir being a + # file etc. + return False + finally: + fp.close() + return True + + +def _rewrite_test(fn: Path, config: Config) -> Tuple[os.stat_result, types.CodeType]: + """Read and rewrite *fn* and return the code object.""" + stat = os.stat(fn) + source = fn.read_bytes() + strfn = str(fn) + tree = ast.parse(source, filename=strfn) + rewrite_asserts(tree, source, strfn, config) + co = compile(tree, strfn, "exec", dont_inherit=True) + return stat, co + + +def _read_pyc( + source: Path, pyc: Path, trace: Callable[[str], None] = lambda x: None +) -> Optional[types.CodeType]: + """Possibly read a pytest pyc containing rewritten code. + + Return rewritten code if successful or None if not. + """ + try: + fp = open(pyc, "rb") + except OSError: + return None + with fp: + # https://www.python.org/dev/peps/pep-0552/ + has_flags = sys.version_info >= (3, 7) + try: + stat_result = os.stat(source) + mtime = int(stat_result.st_mtime) + size = stat_result.st_size + data = fp.read(16 if has_flags else 12) + except OSError as e: + trace(f"_read_pyc({source}): OSError {e}") + return None + # Check for invalid or out of date pyc file. + if len(data) != (16 if has_flags else 12): + trace("_read_pyc(%s): invalid pyc (too short)" % source) + return None + if data[:4] != importlib.util.MAGIC_NUMBER: + trace("_read_pyc(%s): invalid pyc (bad magic number)" % source) + return None + if has_flags and data[4:8] != b"\x00\x00\x00\x00": + trace("_read_pyc(%s): invalid pyc (unsupported flags)" % source) + return None + mtime_data = data[8 if has_flags else 4 : 12 if has_flags else 8] + if int.from_bytes(mtime_data, "little") != mtime & 0xFFFFFFFF: + trace("_read_pyc(%s): out of date" % source) + return None + size_data = data[12 if has_flags else 8 : 16 if has_flags else 12] + if int.from_bytes(size_data, "little") != size & 0xFFFFFFFF: + trace("_read_pyc(%s): invalid pyc (incorrect size)" % source) + return None + try: + co = marshal.load(fp) + except Exception as e: + trace(f"_read_pyc({source}): marshal.load error {e}") + return None + if not isinstance(co, types.CodeType): + trace("_read_pyc(%s): not a code object" % source) + return None + return co + + +def rewrite_asserts( + mod: ast.Module, + source: bytes, + module_path: Optional[str] = None, + config: Optional[Config] = None, +) -> None: + """Rewrite the assert statements in mod.""" + AssertionRewriter(module_path, config, source).run(mod) + + +def _saferepr(obj: object) -> str: + r"""Get a safe repr of an object for assertion error messages. + + The assertion formatting (util.format_explanation()) requires + newlines to be escaped since they are a special character for it. + Normally assertion.util.format_explanation() does this but for a + custom repr it is possible to contain one of the special escape + sequences, especially '\n{' and '\n}' are likely to be present in + JSON reprs. + """ + maxsize = _get_maxsize_for_saferepr(util._config) + return saferepr(obj, maxsize=maxsize).replace("\n", "\\n") + + +def _get_maxsize_for_saferepr(config: Optional[Config]) -> Optional[int]: + """Get `maxsize` configuration for saferepr based on the given config object.""" + verbosity = config.getoption("verbose") if config is not None else 0 + if verbosity >= 2: + return None + if verbosity >= 1: + return DEFAULT_REPR_MAX_SIZE * 10 + return DEFAULT_REPR_MAX_SIZE + + +def _format_assertmsg(obj: object) -> str: + r"""Format the custom assertion message given. + + For strings this simply replaces newlines with '\n~' so that + util.format_explanation() will preserve them instead of escaping + newlines. For other objects saferepr() is used first. + """ + # reprlib appears to have a bug which means that if a string + # contains a newline it gets escaped, however if an object has a + # .__repr__() which contains newlines it does not get escaped. + # However in either case we want to preserve the newline. + replaces = [("\n", "\n~"), ("%", "%%")] + if not isinstance(obj, str): + obj = saferepr(obj) + replaces.append(("\\n", "\n~")) + + for r1, r2 in replaces: + obj = obj.replace(r1, r2) + + return obj + + +def _should_repr_global_name(obj: object) -> bool: + if callable(obj): + return False + + try: + return not hasattr(obj, "__name__") + except Exception: + return True + + +def _format_boolop(explanations: Iterable[str], is_or: bool) -> str: + explanation = "(" + (is_or and " or " or " and ").join(explanations) + ")" + return explanation.replace("%", "%%") + + +def _call_reprcompare( + ops: Sequence[str], + results: Sequence[bool], + expls: Sequence[str], + each_obj: Sequence[object], +) -> str: + for i, res, expl in zip(range(len(ops)), results, expls): + try: + done = not res + except Exception: + done = True + if done: + break + if util._reprcompare is not None: + custom = util._reprcompare(ops[i], each_obj[i], each_obj[i + 1]) + if custom is not None: + return custom + return expl + + +def _call_assertion_pass(lineno: int, orig: str, expl: str) -> None: + if util._assertion_pass is not None: + util._assertion_pass(lineno, orig, expl) + + +def _check_if_assertion_pass_impl() -> bool: + """Check if any plugins implement the pytest_assertion_pass hook + in order not to generate explanation unnecessarily (might be expensive).""" + return True if util._assertion_pass else False + + +UNARY_MAP = {ast.Not: "not %s", ast.Invert: "~%s", ast.USub: "-%s", ast.UAdd: "+%s"} + +BINOP_MAP = { + ast.BitOr: "|", + ast.BitXor: "^", + ast.BitAnd: "&", + ast.LShift: "<<", + ast.RShift: ">>", + ast.Add: "+", + ast.Sub: "-", + ast.Mult: "*", + ast.Div: "/", + ast.FloorDiv: "//", + ast.Mod: "%%", # escaped for string formatting + ast.Eq: "==", + ast.NotEq: "!=", + ast.Lt: "<", + ast.LtE: "<=", + ast.Gt: ">", + ast.GtE: ">=", + ast.Pow: "**", + ast.Is: "is", + ast.IsNot: "is not", + ast.In: "in", + ast.NotIn: "not in", + ast.MatMult: "@", +} + + +def traverse_node(node: ast.AST) -> Iterator[ast.AST]: + """Recursively yield node and all its children in depth-first order.""" + yield node + for child in ast.iter_child_nodes(node): + yield from traverse_node(child) + + +@functools.lru_cache(maxsize=1) +def _get_assertion_exprs(src: bytes) -> Dict[int, str]: + """Return a mapping from {lineno: "assertion test expression"}.""" + ret: Dict[int, str] = {} + + depth = 0 + lines: List[str] = [] + assert_lineno: Optional[int] = None + seen_lines: Set[int] = set() + + def _write_and_reset() -> None: + nonlocal depth, lines, assert_lineno, seen_lines + assert assert_lineno is not None + ret[assert_lineno] = "".join(lines).rstrip().rstrip("\\") + depth = 0 + lines = [] + assert_lineno = None + seen_lines = set() + + tokens = tokenize.tokenize(io.BytesIO(src).readline) + for tp, source, (lineno, offset), _, line in tokens: + if tp == tokenize.NAME and source == "assert": + assert_lineno = lineno + elif assert_lineno is not None: + # keep track of depth for the assert-message `,` lookup + if tp == tokenize.OP and source in "([{": + depth += 1 + elif tp == tokenize.OP and source in ")]}": + depth -= 1 + + if not lines: + lines.append(line[offset:]) + seen_lines.add(lineno) + # a non-nested comma separates the expression from the message + elif depth == 0 and tp == tokenize.OP and source == ",": + # one line assert with message + if lineno in seen_lines and len(lines) == 1: + offset_in_trimmed = offset + len(lines[-1]) - len(line) + lines[-1] = lines[-1][:offset_in_trimmed] + # multi-line assert with message + elif lineno in seen_lines: + lines[-1] = lines[-1][:offset] + # multi line assert with escapd newline before message + else: + lines.append(line[:offset]) + _write_and_reset() + elif tp in {tokenize.NEWLINE, tokenize.ENDMARKER}: + _write_and_reset() + elif lines and lineno not in seen_lines: + lines.append(line) + seen_lines.add(lineno) + + return ret + + +class AssertionRewriter(ast.NodeVisitor): + """Assertion rewriting implementation. + + The main entrypoint is to call .run() with an ast.Module instance, + this will then find all the assert statements and rewrite them to + provide intermediate values and a detailed assertion error. See + http://pybites.blogspot.be/2011/07/behind-scenes-of-pytests-new-assertion.html + for an overview of how this works. + + The entry point here is .run() which will iterate over all the + statements in an ast.Module and for each ast.Assert statement it + finds call .visit() with it. Then .visit_Assert() takes over and + is responsible for creating new ast statements to replace the + original assert statement: it rewrites the test of an assertion + to provide intermediate values and replace it with an if statement + which raises an assertion error with a detailed explanation in + case the expression is false and calls pytest_assertion_pass hook + if expression is true. + + For this .visit_Assert() uses the visitor pattern to visit all the + AST nodes of the ast.Assert.test field, each visit call returning + an AST node and the corresponding explanation string. During this + state is kept in several instance attributes: + + :statements: All the AST statements which will replace the assert + statement. + + :variables: This is populated by .variable() with each variable + used by the statements so that they can all be set to None at + the end of the statements. + + :variable_counter: Counter to create new unique variables needed + by statements. Variables are created using .variable() and + have the form of "@py_assert0". + + :expl_stmts: The AST statements which will be executed to get + data from the assertion. This is the code which will construct + the detailed assertion message that is used in the AssertionError + or for the pytest_assertion_pass hook. + + :explanation_specifiers: A dict filled by .explanation_param() + with %-formatting placeholders and their corresponding + expressions to use in the building of an assertion message. + This is used by .pop_format_context() to build a message. + + :stack: A stack of the explanation_specifiers dicts maintained by + .push_format_context() and .pop_format_context() which allows + to build another %-formatted string while already building one. + + This state is reset on every new assert statement visited and used + by the other visitors. + """ + + def __init__( + self, module_path: Optional[str], config: Optional[Config], source: bytes + ) -> None: + super().__init__() + self.module_path = module_path + self.config = config + if config is not None: + self.enable_assertion_pass_hook = config.getini( + "enable_assertion_pass_hook" + ) + else: + self.enable_assertion_pass_hook = False + self.source = source + + def run(self, mod: ast.Module) -> None: + """Find all assert statements in *mod* and rewrite them.""" + if not mod.body: + # Nothing to do. + return + + # We'll insert some special imports at the top of the module, but after any + # docstrings and __future__ imports, so first figure out where that is. + doc = getattr(mod, "docstring", None) + expect_docstring = doc is None + if doc is not None and self.is_rewrite_disabled(doc): + return + pos = 0 + lineno = 1 + for item in mod.body: + if ( + expect_docstring + and isinstance(item, ast.Expr) + and isinstance(item.value, ast.Str) + ): + doc = item.value.s + if self.is_rewrite_disabled(doc): + return + expect_docstring = False + elif ( + isinstance(item, ast.ImportFrom) + and item.level == 0 + and item.module == "__future__" + ): + pass + else: + break + pos += 1 + # Special case: for a decorated function, set the lineno to that of the + # first decorator, not the `def`. Issue #4984. + if isinstance(item, ast.FunctionDef) and item.decorator_list: + lineno = item.decorator_list[0].lineno + else: + lineno = item.lineno + # Now actually insert the special imports. + if sys.version_info >= (3, 10): + aliases = [ + ast.alias("builtins", "@py_builtins", lineno=lineno, col_offset=0), + ast.alias( + "_pytest.assertion.rewrite", + "@pytest_ar", + lineno=lineno, + col_offset=0, + ), + ] + else: + aliases = [ + ast.alias("builtins", "@py_builtins"), + ast.alias("_pytest.assertion.rewrite", "@pytest_ar"), + ] + imports = [ + ast.Import([alias], lineno=lineno, col_offset=0) for alias in aliases + ] + mod.body[pos:pos] = imports + + # Collect asserts. + nodes: List[ast.AST] = [mod] + while nodes: + node = nodes.pop() + for name, field in ast.iter_fields(node): + if isinstance(field, list): + new: List[ast.AST] = [] + for i, child in enumerate(field): + if isinstance(child, ast.Assert): + # Transform assert. + new.extend(self.visit(child)) + else: + new.append(child) + if isinstance(child, ast.AST): + nodes.append(child) + setattr(node, name, new) + elif ( + isinstance(field, ast.AST) + # Don't recurse into expressions as they can't contain + # asserts. + and not isinstance(field, ast.expr) + ): + nodes.append(field) + + @staticmethod + def is_rewrite_disabled(docstring: str) -> bool: + return "PYTEST_DONT_REWRITE" in docstring + + def variable(self) -> str: + """Get a new variable.""" + # Use a character invalid in python identifiers to avoid clashing. + name = "@py_assert" + str(next(self.variable_counter)) + self.variables.append(name) + return name + + def assign(self, expr: ast.expr) -> ast.Name: + """Give *expr* a name.""" + name = self.variable() + self.statements.append(ast.Assign([ast.Name(name, ast.Store())], expr)) + return ast.Name(name, ast.Load()) + + def display(self, expr: ast.expr) -> ast.expr: + """Call saferepr on the expression.""" + return self.helper("_saferepr", expr) + + def helper(self, name: str, *args: ast.expr) -> ast.expr: + """Call a helper in this module.""" + py_name = ast.Name("@pytest_ar", ast.Load()) + attr = ast.Attribute(py_name, name, ast.Load()) + return ast.Call(attr, list(args), []) + + def builtin(self, name: str) -> ast.Attribute: + """Return the builtin called *name*.""" + builtin_name = ast.Name("@py_builtins", ast.Load()) + return ast.Attribute(builtin_name, name, ast.Load()) + + def explanation_param(self, expr: ast.expr) -> str: + """Return a new named %-formatting placeholder for expr. + + This creates a %-formatting placeholder for expr in the + current formatting context, e.g. ``%(py0)s``. The placeholder + and expr are placed in the current format context so that it + can be used on the next call to .pop_format_context(). + """ + specifier = "py" + str(next(self.variable_counter)) + self.explanation_specifiers[specifier] = expr + return "%(" + specifier + ")s" + + def push_format_context(self) -> None: + """Create a new formatting context. + + The format context is used for when an explanation wants to + have a variable value formatted in the assertion message. In + this case the value required can be added using + .explanation_param(). Finally .pop_format_context() is used + to format a string of %-formatted values as added by + .explanation_param(). + """ + self.explanation_specifiers: Dict[str, ast.expr] = {} + self.stack.append(self.explanation_specifiers) + + def pop_format_context(self, expl_expr: ast.expr) -> ast.Name: + """Format the %-formatted string with current format context. + + The expl_expr should be an str ast.expr instance constructed from + the %-placeholders created by .explanation_param(). This will + add the required code to format said string to .expl_stmts and + return the ast.Name instance of the formatted string. + """ + current = self.stack.pop() + if self.stack: + self.explanation_specifiers = self.stack[-1] + keys = [ast.Str(key) for key in current.keys()] + format_dict = ast.Dict(keys, list(current.values())) + form = ast.BinOp(expl_expr, ast.Mod(), format_dict) + name = "@py_format" + str(next(self.variable_counter)) + if self.enable_assertion_pass_hook: + self.format_variables.append(name) + self.expl_stmts.append(ast.Assign([ast.Name(name, ast.Store())], form)) + return ast.Name(name, ast.Load()) + + def generic_visit(self, node: ast.AST) -> Tuple[ast.Name, str]: + """Handle expressions we don't have custom code for.""" + assert isinstance(node, ast.expr) + res = self.assign(node) + return res, self.explanation_param(self.display(res)) + + def visit_Assert(self, assert_: ast.Assert) -> List[ast.stmt]: + """Return the AST statements to replace the ast.Assert instance. + + This rewrites the test of an assertion to provide + intermediate values and replace it with an if statement which + raises an assertion error with a detailed explanation in case + the expression is false. + """ + if isinstance(assert_.test, ast.Tuple) and len(assert_.test.elts) >= 1: + from _pytest.warning_types import PytestAssertRewriteWarning + import warnings + + # TODO: This assert should not be needed. + assert self.module_path is not None + warnings.warn_explicit( + PytestAssertRewriteWarning( + "assertion is always true, perhaps remove parentheses?" + ), + category=None, + filename=self.module_path, + lineno=assert_.lineno, + ) + + self.statements: List[ast.stmt] = [] + self.variables: List[str] = [] + self.variable_counter = itertools.count() + + if self.enable_assertion_pass_hook: + self.format_variables: List[str] = [] + + self.stack: List[Dict[str, ast.expr]] = [] + self.expl_stmts: List[ast.stmt] = [] + self.push_format_context() + # Rewrite assert into a bunch of statements. + top_condition, explanation = self.visit(assert_.test) + + negation = ast.UnaryOp(ast.Not(), top_condition) + + if self.enable_assertion_pass_hook: # Experimental pytest_assertion_pass hook + msg = self.pop_format_context(ast.Str(explanation)) + + # Failed + if assert_.msg: + assertmsg = self.helper("_format_assertmsg", assert_.msg) + gluestr = "\n>assert " + else: + assertmsg = ast.Str("") + gluestr = "assert " + err_explanation = ast.BinOp(ast.Str(gluestr), ast.Add(), msg) + err_msg = ast.BinOp(assertmsg, ast.Add(), err_explanation) + err_name = ast.Name("AssertionError", ast.Load()) + fmt = self.helper("_format_explanation", err_msg) + exc = ast.Call(err_name, [fmt], []) + raise_ = ast.Raise(exc, None) + statements_fail = [] + statements_fail.extend(self.expl_stmts) + statements_fail.append(raise_) + + # Passed + fmt_pass = self.helper("_format_explanation", msg) + orig = _get_assertion_exprs(self.source)[assert_.lineno] + hook_call_pass = ast.Expr( + self.helper( + "_call_assertion_pass", + ast.Num(assert_.lineno), + ast.Str(orig), + fmt_pass, + ) + ) + # If any hooks implement assert_pass hook + hook_impl_test = ast.If( + self.helper("_check_if_assertion_pass_impl"), + self.expl_stmts + [hook_call_pass], + [], + ) + statements_pass = [hook_impl_test] + + # Test for assertion condition + main_test = ast.If(negation, statements_fail, statements_pass) + self.statements.append(main_test) + if self.format_variables: + variables = [ + ast.Name(name, ast.Store()) for name in self.format_variables + ] + clear_format = ast.Assign(variables, ast.NameConstant(None)) + self.statements.append(clear_format) + + else: # Original assertion rewriting + # Create failure message. + body = self.expl_stmts + self.statements.append(ast.If(negation, body, [])) + if assert_.msg: + assertmsg = self.helper("_format_assertmsg", assert_.msg) + explanation = "\n>assert " + explanation + else: + assertmsg = ast.Str("") + explanation = "assert " + explanation + template = ast.BinOp(assertmsg, ast.Add(), ast.Str(explanation)) + msg = self.pop_format_context(template) + fmt = self.helper("_format_explanation", msg) + err_name = ast.Name("AssertionError", ast.Load()) + exc = ast.Call(err_name, [fmt], []) + raise_ = ast.Raise(exc, None) + + body.append(raise_) + + # Clear temporary variables by setting them to None. + if self.variables: + variables = [ast.Name(name, ast.Store()) for name in self.variables] + clear = ast.Assign(variables, ast.NameConstant(None)) + self.statements.append(clear) + # Fix locations (line numbers/column offsets). + for stmt in self.statements: + for node in traverse_node(stmt): + ast.copy_location(node, assert_) + return self.statements + + def visit_Name(self, name: ast.Name) -> Tuple[ast.Name, str]: + # Display the repr of the name if it's a local variable or + # _should_repr_global_name() thinks it's acceptable. + locs = ast.Call(self.builtin("locals"), [], []) + inlocs = ast.Compare(ast.Str(name.id), [ast.In()], [locs]) + dorepr = self.helper("_should_repr_global_name", name) + test = ast.BoolOp(ast.Or(), [inlocs, dorepr]) + expr = ast.IfExp(test, self.display(name), ast.Str(name.id)) + return name, self.explanation_param(expr) + + def visit_BoolOp(self, boolop: ast.BoolOp) -> Tuple[ast.Name, str]: + res_var = self.variable() + expl_list = self.assign(ast.List([], ast.Load())) + app = ast.Attribute(expl_list, "append", ast.Load()) + is_or = int(isinstance(boolop.op, ast.Or)) + body = save = self.statements + fail_save = self.expl_stmts + levels = len(boolop.values) - 1 + self.push_format_context() + # Process each operand, short-circuiting if needed. + for i, v in enumerate(boolop.values): + if i: + fail_inner: List[ast.stmt] = [] + # cond is set in a prior loop iteration below + self.expl_stmts.append(ast.If(cond, fail_inner, [])) # noqa + self.expl_stmts = fail_inner + self.push_format_context() + res, expl = self.visit(v) + body.append(ast.Assign([ast.Name(res_var, ast.Store())], res)) + expl_format = self.pop_format_context(ast.Str(expl)) + call = ast.Call(app, [expl_format], []) + self.expl_stmts.append(ast.Expr(call)) + if i < levels: + cond: ast.expr = res + if is_or: + cond = ast.UnaryOp(ast.Not(), cond) + inner: List[ast.stmt] = [] + self.statements.append(ast.If(cond, inner, [])) + self.statements = body = inner + self.statements = save + self.expl_stmts = fail_save + expl_template = self.helper("_format_boolop", expl_list, ast.Num(is_or)) + expl = self.pop_format_context(expl_template) + return ast.Name(res_var, ast.Load()), self.explanation_param(expl) + + def visit_UnaryOp(self, unary: ast.UnaryOp) -> Tuple[ast.Name, str]: + pattern = UNARY_MAP[unary.op.__class__] + operand_res, operand_expl = self.visit(unary.operand) + res = self.assign(ast.UnaryOp(unary.op, operand_res)) + return res, pattern % (operand_expl,) + + def visit_BinOp(self, binop: ast.BinOp) -> Tuple[ast.Name, str]: + symbol = BINOP_MAP[binop.op.__class__] + left_expr, left_expl = self.visit(binop.left) + right_expr, right_expl = self.visit(binop.right) + explanation = f"({left_expl} {symbol} {right_expl})" + res = self.assign(ast.BinOp(left_expr, binop.op, right_expr)) + return res, explanation + + def visit_Call(self, call: ast.Call) -> Tuple[ast.Name, str]: + new_func, func_expl = self.visit(call.func) + arg_expls = [] + new_args = [] + new_kwargs = [] + for arg in call.args: + res, expl = self.visit(arg) + arg_expls.append(expl) + new_args.append(res) + for keyword in call.keywords: + res, expl = self.visit(keyword.value) + new_kwargs.append(ast.keyword(keyword.arg, res)) + if keyword.arg: + arg_expls.append(keyword.arg + "=" + expl) + else: # **args have `arg` keywords with an .arg of None + arg_expls.append("**" + expl) + + expl = "{}({})".format(func_expl, ", ".join(arg_expls)) + new_call = ast.Call(new_func, new_args, new_kwargs) + res = self.assign(new_call) + res_expl = self.explanation_param(self.display(res)) + outer_expl = f"{res_expl}\n{{{res_expl} = {expl}\n}}" + return res, outer_expl + + def visit_Starred(self, starred: ast.Starred) -> Tuple[ast.Starred, str]: + # A Starred node can appear in a function call. + res, expl = self.visit(starred.value) + new_starred = ast.Starred(res, starred.ctx) + return new_starred, "*" + expl + + def visit_Attribute(self, attr: ast.Attribute) -> Tuple[ast.Name, str]: + if not isinstance(attr.ctx, ast.Load): + return self.generic_visit(attr) + value, value_expl = self.visit(attr.value) + res = self.assign(ast.Attribute(value, attr.attr, ast.Load())) + res_expl = self.explanation_param(self.display(res)) + pat = "%s\n{%s = %s.%s\n}" + expl = pat % (res_expl, res_expl, value_expl, attr.attr) + return res, expl + + def visit_Compare(self, comp: ast.Compare) -> Tuple[ast.expr, str]: + self.push_format_context() + left_res, left_expl = self.visit(comp.left) + if isinstance(comp.left, (ast.Compare, ast.BoolOp)): + left_expl = f"({left_expl})" + res_variables = [self.variable() for i in range(len(comp.ops))] + load_names = [ast.Name(v, ast.Load()) for v in res_variables] + store_names = [ast.Name(v, ast.Store()) for v in res_variables] + it = zip(range(len(comp.ops)), comp.ops, comp.comparators) + expls = [] + syms = [] + results = [left_res] + for i, op, next_operand in it: + next_res, next_expl = self.visit(next_operand) + if isinstance(next_operand, (ast.Compare, ast.BoolOp)): + next_expl = f"({next_expl})" + results.append(next_res) + sym = BINOP_MAP[op.__class__] + syms.append(ast.Str(sym)) + expl = f"{left_expl} {sym} {next_expl}" + expls.append(ast.Str(expl)) + res_expr = ast.Compare(left_res, [op], [next_res]) + self.statements.append(ast.Assign([store_names[i]], res_expr)) + left_res, left_expl = next_res, next_expl + # Use pytest.assertion.util._reprcompare if that's available. + expl_call = self.helper( + "_call_reprcompare", + ast.Tuple(syms, ast.Load()), + ast.Tuple(load_names, ast.Load()), + ast.Tuple(expls, ast.Load()), + ast.Tuple(results, ast.Load()), + ) + if len(comp.ops) > 1: + res: ast.expr = ast.BoolOp(ast.And(), load_names) + else: + res = load_names[0] + return res, self.explanation_param(self.pop_format_context(expl_call)) + + +def try_makedirs(cache_dir: Path) -> bool: + """Attempt to create the given directory and sub-directories exist. + + Returns True if successful or if it already exists. + """ + try: + os.makedirs(cache_dir, exist_ok=True) + except (FileNotFoundError, NotADirectoryError, FileExistsError): + # One of the path components was not a directory: + # - we're in a zip file + # - it is a file + return False + except PermissionError: + return False + except OSError as e: + # as of now, EROFS doesn't have an equivalent OSError-subclass + if e.errno == errno.EROFS: + return False + raise + return True + + +def get_cache_dir(file_path: Path) -> Path: + """Return the cache directory to write .pyc files for the given .py file path.""" + if sys.version_info >= (3, 8) and sys.pycache_prefix: + # given: + # prefix = '/tmp/pycs' + # path = '/home/user/proj/test_app.py' + # we want: + # '/tmp/pycs/home/user/proj' + return Path(sys.pycache_prefix) / Path(*file_path.parts[1:-1]) + else: + # classic pycache directory + return file_path.parent / "__pycache__" diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/truncate.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/truncate.py new file mode 100644 index 0000000000..ce148dca09 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/truncate.py @@ -0,0 +1,94 @@ +"""Utilities for truncating assertion output. + +Current default behaviour is to truncate assertion explanations at +~8 terminal lines, unless running in "-vv" mode or running on CI. +""" +from typing import List +from typing import Optional + +from _pytest.assertion import util +from _pytest.nodes import Item + + +DEFAULT_MAX_LINES = 8 +DEFAULT_MAX_CHARS = 8 * 80 +USAGE_MSG = "use '-vv' to show" + + +def truncate_if_required( + explanation: List[str], item: Item, max_length: Optional[int] = None +) -> List[str]: + """Truncate this assertion explanation if the given test item is eligible.""" + if _should_truncate_item(item): + return _truncate_explanation(explanation) + return explanation + + +def _should_truncate_item(item: Item) -> bool: + """Whether or not this test item is eligible for truncation.""" + verbose = item.config.option.verbose + return verbose < 2 and not util.running_on_ci() + + +def _truncate_explanation( + input_lines: List[str], + max_lines: Optional[int] = None, + max_chars: Optional[int] = None, +) -> List[str]: + """Truncate given list of strings that makes up the assertion explanation. + + Truncates to either 8 lines, or 640 characters - whichever the input reaches + first. The remaining lines will be replaced by a usage message. + """ + + if max_lines is None: + max_lines = DEFAULT_MAX_LINES + if max_chars is None: + max_chars = DEFAULT_MAX_CHARS + + # Check if truncation required + input_char_count = len("".join(input_lines)) + if len(input_lines) <= max_lines and input_char_count <= max_chars: + return input_lines + + # Truncate first to max_lines, and then truncate to max_chars if max_chars + # is exceeded. + truncated_explanation = input_lines[:max_lines] + truncated_explanation = _truncate_by_char_count(truncated_explanation, max_chars) + + # Add ellipsis to final line + truncated_explanation[-1] = truncated_explanation[-1] + "..." + + # Append useful message to explanation + truncated_line_count = len(input_lines) - len(truncated_explanation) + truncated_line_count += 1 # Account for the part-truncated final line + msg = "...Full output truncated" + if truncated_line_count == 1: + msg += f" ({truncated_line_count} line hidden)" + else: + msg += f" ({truncated_line_count} lines hidden)" + msg += f", {USAGE_MSG}" + truncated_explanation.extend(["", str(msg)]) + return truncated_explanation + + +def _truncate_by_char_count(input_lines: List[str], max_chars: int) -> List[str]: + # Check if truncation required + if len("".join(input_lines)) <= max_chars: + return input_lines + + # Find point at which input length exceeds total allowed length + iterated_char_count = 0 + for iterated_index, input_line in enumerate(input_lines): + if iterated_char_count + len(input_line) > max_chars: + break + iterated_char_count += len(input_line) + + # Create truncated explanation with modified final line + truncated_result = input_lines[:iterated_index] + final_line = input_lines[iterated_index] + if final_line: + final_line_truncate_point = max_chars - iterated_char_count + final_line = final_line[:final_line_truncate_point] + truncated_result.append(final_line) + return truncated_result diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/util.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/util.py new file mode 100644 index 0000000000..19f1089c20 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/assertion/util.py @@ -0,0 +1,498 @@ +"""Utilities for assertion debugging.""" +import collections.abc +import os +import pprint +from typing import AbstractSet +from typing import Any +from typing import Callable +from typing import Iterable +from typing import List +from typing import Mapping +from typing import Optional +from typing import Sequence + +import _pytest._code +from _pytest import outcomes +from _pytest._io.saferepr import _pformat_dispatch +from _pytest._io.saferepr import safeformat +from _pytest._io.saferepr import saferepr +from _pytest.config import Config + +# The _reprcompare attribute on the util module is used by the new assertion +# interpretation code and assertion rewriter to detect this plugin was +# loaded and in turn call the hooks defined here as part of the +# DebugInterpreter. +_reprcompare: Optional[Callable[[str, object, object], Optional[str]]] = None + +# Works similarly as _reprcompare attribute. Is populated with the hook call +# when pytest_runtest_setup is called. +_assertion_pass: Optional[Callable[[int, str, str], None]] = None + +# Config object which is assigned during pytest_runtest_protocol. +_config: Optional[Config] = None + + +def format_explanation(explanation: str) -> str: + r"""Format an explanation. + + Normally all embedded newlines are escaped, however there are + three exceptions: \n{, \n} and \n~. The first two are intended + cover nested explanations, see function and attribute explanations + for examples (.visit_Call(), visit_Attribute()). The last one is + for when one explanation needs to span multiple lines, e.g. when + displaying diffs. + """ + lines = _split_explanation(explanation) + result = _format_lines(lines) + return "\n".join(result) + + +def _split_explanation(explanation: str) -> List[str]: + r"""Return a list of individual lines in the explanation. + + This will return a list of lines split on '\n{', '\n}' and '\n~'. + Any other newlines will be escaped and appear in the line as the + literal '\n' characters. + """ + raw_lines = (explanation or "").split("\n") + lines = [raw_lines[0]] + for values in raw_lines[1:]: + if values and values[0] in ["{", "}", "~", ">"]: + lines.append(values) + else: + lines[-1] += "\\n" + values + return lines + + +def _format_lines(lines: Sequence[str]) -> List[str]: + """Format the individual lines. + + This will replace the '{', '}' and '~' characters of our mini formatting + language with the proper 'where ...', 'and ...' and ' + ...' text, taking + care of indentation along the way. + + Return a list of formatted lines. + """ + result = list(lines[:1]) + stack = [0] + stackcnt = [0] + for line in lines[1:]: + if line.startswith("{"): + if stackcnt[-1]: + s = "and " + else: + s = "where " + stack.append(len(result)) + stackcnt[-1] += 1 + stackcnt.append(0) + result.append(" +" + " " * (len(stack) - 1) + s + line[1:]) + elif line.startswith("}"): + stack.pop() + stackcnt.pop() + result[stack[-1]] += line[1:] + else: + assert line[0] in ["~", ">"] + stack[-1] += 1 + indent = len(stack) if line.startswith("~") else len(stack) - 1 + result.append(" " * indent + line[1:]) + assert len(stack) == 1 + return result + + +def issequence(x: Any) -> bool: + return isinstance(x, collections.abc.Sequence) and not isinstance(x, str) + + +def istext(x: Any) -> bool: + return isinstance(x, str) + + +def isdict(x: Any) -> bool: + return isinstance(x, dict) + + +def isset(x: Any) -> bool: + return isinstance(x, (set, frozenset)) + + +def isnamedtuple(obj: Any) -> bool: + return isinstance(obj, tuple) and getattr(obj, "_fields", None) is not None + + +def isdatacls(obj: Any) -> bool: + return getattr(obj, "__dataclass_fields__", None) is not None + + +def isattrs(obj: Any) -> bool: + return getattr(obj, "__attrs_attrs__", None) is not None + + +def isiterable(obj: Any) -> bool: + try: + iter(obj) + return not istext(obj) + except TypeError: + return False + + +def assertrepr_compare(config, op: str, left: Any, right: Any) -> Optional[List[str]]: + """Return specialised explanations for some operators/operands.""" + verbose = config.getoption("verbose") + if verbose > 1: + left_repr = safeformat(left) + right_repr = safeformat(right) + else: + # XXX: "15 chars indentation" is wrong + # ("E AssertionError: assert "); should use term width. + maxsize = ( + 80 - 15 - len(op) - 2 + ) // 2 # 15 chars indentation, 1 space around op + left_repr = saferepr(left, maxsize=maxsize) + right_repr = saferepr(right, maxsize=maxsize) + + summary = f"{left_repr} {op} {right_repr}" + + explanation = None + try: + if op == "==": + explanation = _compare_eq_any(left, right, verbose) + elif op == "not in": + if istext(left) and istext(right): + explanation = _notin_text(left, right, verbose) + except outcomes.Exit: + raise + except Exception: + explanation = [ + "(pytest_assertion plugin: representation of details failed: {}.".format( + _pytest._code.ExceptionInfo.from_current()._getreprcrash() + ), + " Probably an object has a faulty __repr__.)", + ] + + if not explanation: + return None + + return [summary] + explanation + + +def _compare_eq_any(left: Any, right: Any, verbose: int = 0) -> List[str]: + explanation = [] + if istext(left) and istext(right): + explanation = _diff_text(left, right, verbose) + else: + from _pytest.python_api import ApproxBase + + if isinstance(left, ApproxBase) or isinstance(right, ApproxBase): + # Although the common order should be obtained == expected, this ensures both ways + approx_side = left if isinstance(left, ApproxBase) else right + other_side = right if isinstance(left, ApproxBase) else left + + explanation = approx_side._repr_compare(other_side) + elif type(left) == type(right) and ( + isdatacls(left) or isattrs(left) or isnamedtuple(left) + ): + # Note: unlike dataclasses/attrs, namedtuples compare only the + # field values, not the type or field names. But this branch + # intentionally only handles the same-type case, which was often + # used in older code bases before dataclasses/attrs were available. + explanation = _compare_eq_cls(left, right, verbose) + elif issequence(left) and issequence(right): + explanation = _compare_eq_sequence(left, right, verbose) + elif isset(left) and isset(right): + explanation = _compare_eq_set(left, right, verbose) + elif isdict(left) and isdict(right): + explanation = _compare_eq_dict(left, right, verbose) + elif verbose > 0: + explanation = _compare_eq_verbose(left, right) + + if isiterable(left) and isiterable(right): + expl = _compare_eq_iterable(left, right, verbose) + explanation.extend(expl) + + return explanation + + +def _diff_text(left: str, right: str, verbose: int = 0) -> List[str]: + """Return the explanation for the diff between text. + + Unless --verbose is used this will skip leading and trailing + characters which are identical to keep the diff minimal. + """ + from difflib import ndiff + + explanation: List[str] = [] + + if verbose < 1: + i = 0 # just in case left or right has zero length + for i in range(min(len(left), len(right))): + if left[i] != right[i]: + break + if i > 42: + i -= 10 # Provide some context + explanation = [ + "Skipping %s identical leading characters in diff, use -v to show" % i + ] + left = left[i:] + right = right[i:] + if len(left) == len(right): + for i in range(len(left)): + if left[-i] != right[-i]: + break + if i > 42: + i -= 10 # Provide some context + explanation += [ + "Skipping {} identical trailing " + "characters in diff, use -v to show".format(i) + ] + left = left[:-i] + right = right[:-i] + keepends = True + if left.isspace() or right.isspace(): + left = repr(str(left)) + right = repr(str(right)) + explanation += ["Strings contain only whitespace, escaping them using repr()"] + # "right" is the expected base against which we compare "left", + # see https://github.com/pytest-dev/pytest/issues/3333 + explanation += [ + line.strip("\n") + for line in ndiff(right.splitlines(keepends), left.splitlines(keepends)) + ] + return explanation + + +def _compare_eq_verbose(left: Any, right: Any) -> List[str]: + keepends = True + left_lines = repr(left).splitlines(keepends) + right_lines = repr(right).splitlines(keepends) + + explanation: List[str] = [] + explanation += ["+" + line for line in left_lines] + explanation += ["-" + line for line in right_lines] + + return explanation + + +def _surrounding_parens_on_own_lines(lines: List[str]) -> None: + """Move opening/closing parenthesis/bracket to own lines.""" + opening = lines[0][:1] + if opening in ["(", "[", "{"]: + lines[0] = " " + lines[0][1:] + lines[:] = [opening] + lines + closing = lines[-1][-1:] + if closing in [")", "]", "}"]: + lines[-1] = lines[-1][:-1] + "," + lines[:] = lines + [closing] + + +def _compare_eq_iterable( + left: Iterable[Any], right: Iterable[Any], verbose: int = 0 +) -> List[str]: + if not verbose and not running_on_ci(): + return ["Use -v to get the full diff"] + # dynamic import to speedup pytest + import difflib + + left_formatting = pprint.pformat(left).splitlines() + right_formatting = pprint.pformat(right).splitlines() + + # Re-format for different output lengths. + lines_left = len(left_formatting) + lines_right = len(right_formatting) + if lines_left != lines_right: + left_formatting = _pformat_dispatch(left).splitlines() + right_formatting = _pformat_dispatch(right).splitlines() + + if lines_left > 1 or lines_right > 1: + _surrounding_parens_on_own_lines(left_formatting) + _surrounding_parens_on_own_lines(right_formatting) + + explanation = ["Full diff:"] + # "right" is the expected base against which we compare "left", + # see https://github.com/pytest-dev/pytest/issues/3333 + explanation.extend( + line.rstrip() for line in difflib.ndiff(right_formatting, left_formatting) + ) + return explanation + + +def _compare_eq_sequence( + left: Sequence[Any], right: Sequence[Any], verbose: int = 0 +) -> List[str]: + comparing_bytes = isinstance(left, bytes) and isinstance(right, bytes) + explanation: List[str] = [] + len_left = len(left) + len_right = len(right) + for i in range(min(len_left, len_right)): + if left[i] != right[i]: + if comparing_bytes: + # when comparing bytes, we want to see their ascii representation + # instead of their numeric values (#5260) + # using a slice gives us the ascii representation: + # >>> s = b'foo' + # >>> s[0] + # 102 + # >>> s[0:1] + # b'f' + left_value = left[i : i + 1] + right_value = right[i : i + 1] + else: + left_value = left[i] + right_value = right[i] + + explanation += [f"At index {i} diff: {left_value!r} != {right_value!r}"] + break + + if comparing_bytes: + # when comparing bytes, it doesn't help to show the "sides contain one or more + # items" longer explanation, so skip it + + return explanation + + len_diff = len_left - len_right + if len_diff: + if len_diff > 0: + dir_with_more = "Left" + extra = saferepr(left[len_right]) + else: + len_diff = 0 - len_diff + dir_with_more = "Right" + extra = saferepr(right[len_left]) + + if len_diff == 1: + explanation += [f"{dir_with_more} contains one more item: {extra}"] + else: + explanation += [ + "%s contains %d more items, first extra item: %s" + % (dir_with_more, len_diff, extra) + ] + return explanation + + +def _compare_eq_set( + left: AbstractSet[Any], right: AbstractSet[Any], verbose: int = 0 +) -> List[str]: + explanation = [] + diff_left = left - right + diff_right = right - left + if diff_left: + explanation.append("Extra items in the left set:") + for item in diff_left: + explanation.append(saferepr(item)) + if diff_right: + explanation.append("Extra items in the right set:") + for item in diff_right: + explanation.append(saferepr(item)) + return explanation + + +def _compare_eq_dict( + left: Mapping[Any, Any], right: Mapping[Any, Any], verbose: int = 0 +) -> List[str]: + explanation: List[str] = [] + set_left = set(left) + set_right = set(right) + common = set_left.intersection(set_right) + same = {k: left[k] for k in common if left[k] == right[k]} + if same and verbose < 2: + explanation += ["Omitting %s identical items, use -vv to show" % len(same)] + elif same: + explanation += ["Common items:"] + explanation += pprint.pformat(same).splitlines() + diff = {k for k in common if left[k] != right[k]} + if diff: + explanation += ["Differing items:"] + for k in diff: + explanation += [saferepr({k: left[k]}) + " != " + saferepr({k: right[k]})] + extra_left = set_left - set_right + len_extra_left = len(extra_left) + if len_extra_left: + explanation.append( + "Left contains %d more item%s:" + % (len_extra_left, "" if len_extra_left == 1 else "s") + ) + explanation.extend( + pprint.pformat({k: left[k] for k in extra_left}).splitlines() + ) + extra_right = set_right - set_left + len_extra_right = len(extra_right) + if len_extra_right: + explanation.append( + "Right contains %d more item%s:" + % (len_extra_right, "" if len_extra_right == 1 else "s") + ) + explanation.extend( + pprint.pformat({k: right[k] for k in extra_right}).splitlines() + ) + return explanation + + +def _compare_eq_cls(left: Any, right: Any, verbose: int) -> List[str]: + if isdatacls(left): + all_fields = left.__dataclass_fields__ + fields_to_check = [field for field, info in all_fields.items() if info.compare] + elif isattrs(left): + all_fields = left.__attrs_attrs__ + fields_to_check = [field.name for field in all_fields if getattr(field, "eq")] + elif isnamedtuple(left): + fields_to_check = left._fields + else: + assert False + + indent = " " + same = [] + diff = [] + for field in fields_to_check: + if getattr(left, field) == getattr(right, field): + same.append(field) + else: + diff.append(field) + + explanation = [] + if same or diff: + explanation += [""] + if same and verbose < 2: + explanation.append("Omitting %s identical items, use -vv to show" % len(same)) + elif same: + explanation += ["Matching attributes:"] + explanation += pprint.pformat(same).splitlines() + if diff: + explanation += ["Differing attributes:"] + explanation += pprint.pformat(diff).splitlines() + for field in diff: + field_left = getattr(left, field) + field_right = getattr(right, field) + explanation += [ + "", + "Drill down into differing attribute %s:" % field, + ("%s%s: %r != %r") % (indent, field, field_left, field_right), + ] + explanation += [ + indent + line + for line in _compare_eq_any(field_left, field_right, verbose) + ] + return explanation + + +def _notin_text(term: str, text: str, verbose: int = 0) -> List[str]: + index = text.find(term) + head = text[:index] + tail = text[index + len(term) :] + correct_text = head + tail + diff = _diff_text(text, correct_text, verbose) + newdiff = ["%s is contained here:" % saferepr(term, maxsize=42)] + for line in diff: + if line.startswith("Skipping"): + continue + if line.startswith("- "): + continue + if line.startswith("+ "): + newdiff.append(" " + line[2:]) + else: + newdiff.append(line) + return newdiff + + +def running_on_ci() -> bool: + """Check if we're currently running on a CI system.""" + env_vars = ["CI", "BUILD_NUMBER"] + return any(var in os.environ for var in env_vars) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/cacheprovider.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/cacheprovider.py new file mode 100644 index 0000000000..681d02b409 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/cacheprovider.py @@ -0,0 +1,580 @@ +"""Implementation of the cache provider.""" +# This plugin was not named "cache" to avoid conflicts with the external +# pytest-cache version. +import json +import os +from pathlib import Path +from typing import Dict +from typing import Generator +from typing import Iterable +from typing import List +from typing import Optional +from typing import Set +from typing import Union + +import attr + +from .pathlib import resolve_from_str +from .pathlib import rm_rf +from .reports import CollectReport +from _pytest import nodes +from _pytest._io import TerminalWriter +from _pytest.compat import final +from _pytest.config import Config +from _pytest.config import ExitCode +from _pytest.config import hookimpl +from _pytest.config.argparsing import Parser +from _pytest.deprecated import check_ispytest +from _pytest.fixtures import fixture +from _pytest.fixtures import FixtureRequest +from _pytest.main import Session +from _pytest.python import Module +from _pytest.python import Package +from _pytest.reports import TestReport + + +README_CONTENT = """\ +# pytest cache directory # + +This directory contains data from the pytest's cache plugin, +which provides the `--lf` and `--ff` options, as well as the `cache` fixture. + +**Do not** commit this to version control. + +See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information. +""" + +CACHEDIR_TAG_CONTENT = b"""\ +Signature: 8a477f597d28d172789f06886806bc55 +# This file is a cache directory tag created by pytest. +# For information about cache directory tags, see: +# https://bford.info/cachedir/spec.html +""" + + +@final +@attr.s(init=False, auto_attribs=True) +class Cache: + _cachedir: Path = attr.ib(repr=False) + _config: Config = attr.ib(repr=False) + + # Sub-directory under cache-dir for directories created by `mkdir()`. + _CACHE_PREFIX_DIRS = "d" + + # Sub-directory under cache-dir for values created by `set()`. + _CACHE_PREFIX_VALUES = "v" + + def __init__( + self, cachedir: Path, config: Config, *, _ispytest: bool = False + ) -> None: + check_ispytest(_ispytest) + self._cachedir = cachedir + self._config = config + + @classmethod + def for_config(cls, config: Config, *, _ispytest: bool = False) -> "Cache": + """Create the Cache instance for a Config. + + :meta private: + """ + check_ispytest(_ispytest) + cachedir = cls.cache_dir_from_config(config, _ispytest=True) + if config.getoption("cacheclear") and cachedir.is_dir(): + cls.clear_cache(cachedir, _ispytest=True) + return cls(cachedir, config, _ispytest=True) + + @classmethod + def clear_cache(cls, cachedir: Path, _ispytest: bool = False) -> None: + """Clear the sub-directories used to hold cached directories and values. + + :meta private: + """ + check_ispytest(_ispytest) + for prefix in (cls._CACHE_PREFIX_DIRS, cls._CACHE_PREFIX_VALUES): + d = cachedir / prefix + if d.is_dir(): + rm_rf(d) + + @staticmethod + def cache_dir_from_config(config: Config, *, _ispytest: bool = False) -> Path: + """Get the path to the cache directory for a Config. + + :meta private: + """ + check_ispytest(_ispytest) + return resolve_from_str(config.getini("cache_dir"), config.rootpath) + + def warn(self, fmt: str, *, _ispytest: bool = False, **args: object) -> None: + """Issue a cache warning. + + :meta private: + """ + check_ispytest(_ispytest) + import warnings + from _pytest.warning_types import PytestCacheWarning + + warnings.warn( + PytestCacheWarning(fmt.format(**args) if args else fmt), + self._config.hook, + stacklevel=3, + ) + + def mkdir(self, name: str) -> Path: + """Return a directory path object with the given name. + + If the directory does not yet exist, it will be created. You can use + it to manage files to e.g. store/retrieve database dumps across test + sessions. + + .. versionadded:: 7.0 + + :param name: + Must be a string not containing a ``/`` separator. + Make sure the name contains your plugin or application + identifiers to prevent clashes with other cache users. + """ + path = Path(name) + if len(path.parts) > 1: + raise ValueError("name is not allowed to contain path separators") + res = self._cachedir.joinpath(self._CACHE_PREFIX_DIRS, path) + res.mkdir(exist_ok=True, parents=True) + return res + + def _getvaluepath(self, key: str) -> Path: + return self._cachedir.joinpath(self._CACHE_PREFIX_VALUES, Path(key)) + + def get(self, key: str, default): + """Return the cached value for the given key. + + If no value was yet cached or the value cannot be read, the specified + default is returned. + + :param key: + Must be a ``/`` separated value. Usually the first + name is the name of your plugin or your application. + :param default: + The value to return in case of a cache-miss or invalid cache value. + """ + path = self._getvaluepath(key) + try: + with path.open("r") as f: + return json.load(f) + except (ValueError, OSError): + return default + + def set(self, key: str, value: object) -> None: + """Save value for the given key. + + :param key: + Must be a ``/`` separated value. Usually the first + name is the name of your plugin or your application. + :param value: + Must be of any combination of basic python types, + including nested types like lists of dictionaries. + """ + path = self._getvaluepath(key) + try: + if path.parent.is_dir(): + cache_dir_exists_already = True + else: + cache_dir_exists_already = self._cachedir.exists() + path.parent.mkdir(exist_ok=True, parents=True) + except OSError: + self.warn("could not create cache path {path}", path=path, _ispytest=True) + return + if not cache_dir_exists_already: + self._ensure_supporting_files() + data = json.dumps(value, indent=2) + try: + f = path.open("w") + except OSError: + self.warn("cache could not write path {path}", path=path, _ispytest=True) + else: + with f: + f.write(data) + + def _ensure_supporting_files(self) -> None: + """Create supporting files in the cache dir that are not really part of the cache.""" + readme_path = self._cachedir / "README.md" + readme_path.write_text(README_CONTENT) + + gitignore_path = self._cachedir.joinpath(".gitignore") + msg = "# Created by pytest automatically.\n*\n" + gitignore_path.write_text(msg, encoding="UTF-8") + + cachedir_tag_path = self._cachedir.joinpath("CACHEDIR.TAG") + cachedir_tag_path.write_bytes(CACHEDIR_TAG_CONTENT) + + +class LFPluginCollWrapper: + def __init__(self, lfplugin: "LFPlugin") -> None: + self.lfplugin = lfplugin + self._collected_at_least_one_failure = False + + @hookimpl(hookwrapper=True) + def pytest_make_collect_report(self, collector: nodes.Collector): + if isinstance(collector, Session): + out = yield + res: CollectReport = out.get_result() + + # Sort any lf-paths to the beginning. + lf_paths = self.lfplugin._last_failed_paths + + res.result = sorted( + res.result, + # use stable sort to priorize last failed + key=lambda x: x.path in lf_paths, + reverse=True, + ) + return + + elif isinstance(collector, Module): + if collector.path in self.lfplugin._last_failed_paths: + out = yield + res = out.get_result() + result = res.result + lastfailed = self.lfplugin.lastfailed + + # Only filter with known failures. + if not self._collected_at_least_one_failure: + if not any(x.nodeid in lastfailed for x in result): + return + self.lfplugin.config.pluginmanager.register( + LFPluginCollSkipfiles(self.lfplugin), "lfplugin-collskip" + ) + self._collected_at_least_one_failure = True + + session = collector.session + result[:] = [ + x + for x in result + if x.nodeid in lastfailed + # Include any passed arguments (not trivial to filter). + or session.isinitpath(x.path) + # Keep all sub-collectors. + or isinstance(x, nodes.Collector) + ] + return + yield + + +class LFPluginCollSkipfiles: + def __init__(self, lfplugin: "LFPlugin") -> None: + self.lfplugin = lfplugin + + @hookimpl + def pytest_make_collect_report( + self, collector: nodes.Collector + ) -> Optional[CollectReport]: + # Packages are Modules, but _last_failed_paths only contains + # test-bearing paths and doesn't try to include the paths of their + # packages, so don't filter them. + if isinstance(collector, Module) and not isinstance(collector, Package): + if collector.path not in self.lfplugin._last_failed_paths: + self.lfplugin._skipped_files += 1 + + return CollectReport( + collector.nodeid, "passed", longrepr=None, result=[] + ) + return None + + +class LFPlugin: + """Plugin which implements the --lf (run last-failing) option.""" + + def __init__(self, config: Config) -> None: + self.config = config + active_keys = "lf", "failedfirst" + self.active = any(config.getoption(key) for key in active_keys) + assert config.cache + self.lastfailed: Dict[str, bool] = config.cache.get("cache/lastfailed", {}) + self._previously_failed_count: Optional[int] = None + self._report_status: Optional[str] = None + self._skipped_files = 0 # count skipped files during collection due to --lf + + if config.getoption("lf"): + self._last_failed_paths = self.get_last_failed_paths() + config.pluginmanager.register( + LFPluginCollWrapper(self), "lfplugin-collwrapper" + ) + + def get_last_failed_paths(self) -> Set[Path]: + """Return a set with all Paths()s of the previously failed nodeids.""" + rootpath = self.config.rootpath + result = {rootpath / nodeid.split("::")[0] for nodeid in self.lastfailed} + return {x for x in result if x.exists()} + + def pytest_report_collectionfinish(self) -> Optional[str]: + if self.active and self.config.getoption("verbose") >= 0: + return "run-last-failure: %s" % self._report_status + return None + + def pytest_runtest_logreport(self, report: TestReport) -> None: + if (report.when == "call" and report.passed) or report.skipped: + self.lastfailed.pop(report.nodeid, None) + elif report.failed: + self.lastfailed[report.nodeid] = True + + def pytest_collectreport(self, report: CollectReport) -> None: + passed = report.outcome in ("passed", "skipped") + if passed: + if report.nodeid in self.lastfailed: + self.lastfailed.pop(report.nodeid) + self.lastfailed.update((item.nodeid, True) for item in report.result) + else: + self.lastfailed[report.nodeid] = True + + @hookimpl(hookwrapper=True, tryfirst=True) + def pytest_collection_modifyitems( + self, config: Config, items: List[nodes.Item] + ) -> Generator[None, None, None]: + yield + + if not self.active: + return + + if self.lastfailed: + previously_failed = [] + previously_passed = [] + for item in items: + if item.nodeid in self.lastfailed: + previously_failed.append(item) + else: + previously_passed.append(item) + self._previously_failed_count = len(previously_failed) + + if not previously_failed: + # Running a subset of all tests with recorded failures + # only outside of it. + self._report_status = "%d known failures not in selected tests" % ( + len(self.lastfailed), + ) + else: + if self.config.getoption("lf"): + items[:] = previously_failed + config.hook.pytest_deselected(items=previously_passed) + else: # --failedfirst + items[:] = previously_failed + previously_passed + + noun = "failure" if self._previously_failed_count == 1 else "failures" + suffix = " first" if self.config.getoption("failedfirst") else "" + self._report_status = "rerun previous {count} {noun}{suffix}".format( + count=self._previously_failed_count, suffix=suffix, noun=noun + ) + + if self._skipped_files > 0: + files_noun = "file" if self._skipped_files == 1 else "files" + self._report_status += " (skipped {files} {files_noun})".format( + files=self._skipped_files, files_noun=files_noun + ) + else: + self._report_status = "no previously failed tests, " + if self.config.getoption("last_failed_no_failures") == "none": + self._report_status += "deselecting all items." + config.hook.pytest_deselected(items=items[:]) + items[:] = [] + else: + self._report_status += "not deselecting items." + + def pytest_sessionfinish(self, session: Session) -> None: + config = self.config + if config.getoption("cacheshow") or hasattr(config, "workerinput"): + return + + assert config.cache is not None + saved_lastfailed = config.cache.get("cache/lastfailed", {}) + if saved_lastfailed != self.lastfailed: + config.cache.set("cache/lastfailed", self.lastfailed) + + +class NFPlugin: + """Plugin which implements the --nf (run new-first) option.""" + + def __init__(self, config: Config) -> None: + self.config = config + self.active = config.option.newfirst + assert config.cache is not None + self.cached_nodeids = set(config.cache.get("cache/nodeids", [])) + + @hookimpl(hookwrapper=True, tryfirst=True) + def pytest_collection_modifyitems( + self, items: List[nodes.Item] + ) -> Generator[None, None, None]: + yield + + if self.active: + new_items: Dict[str, nodes.Item] = {} + other_items: Dict[str, nodes.Item] = {} + for item in items: + if item.nodeid not in self.cached_nodeids: + new_items[item.nodeid] = item + else: + other_items[item.nodeid] = item + + items[:] = self._get_increasing_order( + new_items.values() + ) + self._get_increasing_order(other_items.values()) + self.cached_nodeids.update(new_items) + else: + self.cached_nodeids.update(item.nodeid for item in items) + + def _get_increasing_order(self, items: Iterable[nodes.Item]) -> List[nodes.Item]: + return sorted(items, key=lambda item: item.path.stat().st_mtime, reverse=True) # type: ignore[no-any-return] + + def pytest_sessionfinish(self) -> None: + config = self.config + if config.getoption("cacheshow") or hasattr(config, "workerinput"): + return + + if config.getoption("collectonly"): + return + + assert config.cache is not None + config.cache.set("cache/nodeids", sorted(self.cached_nodeids)) + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("general") + group.addoption( + "--lf", + "--last-failed", + action="store_true", + dest="lf", + help="rerun only the tests that failed " + "at the last run (or all if none failed)", + ) + group.addoption( + "--ff", + "--failed-first", + action="store_true", + dest="failedfirst", + help="run all tests, but run the last failures first.\n" + "This may re-order tests and thus lead to " + "repeated fixture setup/teardown.", + ) + group.addoption( + "--nf", + "--new-first", + action="store_true", + dest="newfirst", + help="run tests from new files first, then the rest of the tests " + "sorted by file mtime", + ) + group.addoption( + "--cache-show", + action="append", + nargs="?", + dest="cacheshow", + help=( + "show cache contents, don't perform collection or tests. " + "Optional argument: glob (default: '*')." + ), + ) + group.addoption( + "--cache-clear", + action="store_true", + dest="cacheclear", + help="remove all cache contents at start of test run.", + ) + cache_dir_default = ".pytest_cache" + if "TOX_ENV_DIR" in os.environ: + cache_dir_default = os.path.join(os.environ["TOX_ENV_DIR"], cache_dir_default) + parser.addini("cache_dir", default=cache_dir_default, help="cache directory path.") + group.addoption( + "--lfnf", + "--last-failed-no-failures", + action="store", + dest="last_failed_no_failures", + choices=("all", "none"), + default="all", + help="which tests to run with no previously (known) failures.", + ) + + +def pytest_cmdline_main(config: Config) -> Optional[Union[int, ExitCode]]: + if config.option.cacheshow: + from _pytest.main import wrap_session + + return wrap_session(config, cacheshow) + return None + + +@hookimpl(tryfirst=True) +def pytest_configure(config: Config) -> None: + config.cache = Cache.for_config(config, _ispytest=True) + config.pluginmanager.register(LFPlugin(config), "lfplugin") + config.pluginmanager.register(NFPlugin(config), "nfplugin") + + +@fixture +def cache(request: FixtureRequest) -> Cache: + """Return a cache object that can persist state between testing sessions. + + cache.get(key, default) + cache.set(key, value) + + Keys must be ``/`` separated strings, where the first part is usually the + name of your plugin or application to avoid clashes with other cache users. + + Values can be any object handled by the json stdlib module. + """ + assert request.config.cache is not None + return request.config.cache + + +def pytest_report_header(config: Config) -> Optional[str]: + """Display cachedir with --cache-show and if non-default.""" + if config.option.verbose > 0 or config.getini("cache_dir") != ".pytest_cache": + assert config.cache is not None + cachedir = config.cache._cachedir + # TODO: evaluate generating upward relative paths + # starting with .., ../.. if sensible + + try: + displaypath = cachedir.relative_to(config.rootpath) + except ValueError: + displaypath = cachedir + return f"cachedir: {displaypath}" + return None + + +def cacheshow(config: Config, session: Session) -> int: + from pprint import pformat + + assert config.cache is not None + + tw = TerminalWriter() + tw.line("cachedir: " + str(config.cache._cachedir)) + if not config.cache._cachedir.is_dir(): + tw.line("cache is empty") + return 0 + + glob = config.option.cacheshow[0] + if glob is None: + glob = "*" + + dummy = object() + basedir = config.cache._cachedir + vdir = basedir / Cache._CACHE_PREFIX_VALUES + tw.sep("-", "cache values for %r" % glob) + for valpath in sorted(x for x in vdir.rglob(glob) if x.is_file()): + key = str(valpath.relative_to(vdir)) + val = config.cache.get(key, dummy) + if val is dummy: + tw.line("%s contains unreadable content, will be ignored" % key) + else: + tw.line("%s contains:" % key) + for line in pformat(val).splitlines(): + tw.line(" " + line) + + ddir = basedir / Cache._CACHE_PREFIX_DIRS + if ddir.is_dir(): + contents = sorted(ddir.rglob(glob)) + tw.sep("-", "cache directories for %r" % glob) + for p in contents: + # if p.is_dir(): + # print("%s/" % p.relative_to(basedir)) + if p.is_file(): + key = str(p.relative_to(basedir)) + tw.line(f"{key} is a file of length {p.stat().st_size:d}") + return 0 diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/capture.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/capture.py new file mode 100644 index 0000000000..884f035e29 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/capture.py @@ -0,0 +1,942 @@ +"""Per-test stdout/stderr capturing mechanism.""" +import contextlib +import functools +import io +import os +import sys +from io import UnsupportedOperation +from tempfile import TemporaryFile +from typing import Any +from typing import AnyStr +from typing import Generator +from typing import Generic +from typing import Iterator +from typing import Optional +from typing import TextIO +from typing import Tuple +from typing import TYPE_CHECKING +from typing import Union + +from _pytest.compat import final +from _pytest.config import Config +from _pytest.config import hookimpl +from _pytest.config.argparsing import Parser +from _pytest.deprecated import check_ispytest +from _pytest.fixtures import fixture +from _pytest.fixtures import SubRequest +from _pytest.nodes import Collector +from _pytest.nodes import File +from _pytest.nodes import Item + +if TYPE_CHECKING: + from typing_extensions import Literal + + _CaptureMethod = Literal["fd", "sys", "no", "tee-sys"] + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("general") + group._addoption( + "--capture", + action="store", + default="fd", + metavar="method", + choices=["fd", "sys", "no", "tee-sys"], + help="per-test capturing method: one of fd|sys|no|tee-sys.", + ) + group._addoption( + "-s", + action="store_const", + const="no", + dest="capture", + help="shortcut for --capture=no.", + ) + + +def _colorama_workaround() -> None: + """Ensure colorama is imported so that it attaches to the correct stdio + handles on Windows. + + colorama uses the terminal on import time. So if something does the + first import of colorama while I/O capture is active, colorama will + fail in various ways. + """ + if sys.platform.startswith("win32"): + try: + import colorama # noqa: F401 + except ImportError: + pass + + +def _py36_windowsconsoleio_workaround(stream: TextIO) -> None: + """Workaround for Windows Unicode console handling on Python>=3.6. + + Python 3.6 implemented Unicode console handling for Windows. This works + by reading/writing to the raw console handle using + ``{Read,Write}ConsoleW``. + + The problem is that we are going to ``dup2`` over the stdio file + descriptors when doing ``FDCapture`` and this will ``CloseHandle`` the + handles used by Python to write to the console. Though there is still some + weirdness and the console handle seems to only be closed randomly and not + on the first call to ``CloseHandle``, or maybe it gets reopened with the + same handle value when we suspend capturing. + + The workaround in this case will reopen stdio with a different fd which + also means a different handle by replicating the logic in + "Py_lifecycle.c:initstdio/create_stdio". + + :param stream: + In practice ``sys.stdout`` or ``sys.stderr``, but given + here as parameter for unittesting purposes. + + See https://github.com/pytest-dev/py/issues/103. + """ + if not sys.platform.startswith("win32") or hasattr(sys, "pypy_version_info"): + return + + # Bail out if ``stream`` doesn't seem like a proper ``io`` stream (#2666). + if not hasattr(stream, "buffer"): # type: ignore[unreachable] + return + + buffered = hasattr(stream.buffer, "raw") + raw_stdout = stream.buffer.raw if buffered else stream.buffer # type: ignore[attr-defined] + + if not isinstance(raw_stdout, io._WindowsConsoleIO): # type: ignore[attr-defined] + return + + def _reopen_stdio(f, mode): + if not buffered and mode[0] == "w": + buffering = 0 + else: + buffering = -1 + + return io.TextIOWrapper( + open(os.dup(f.fileno()), mode, buffering), # type: ignore[arg-type] + f.encoding, + f.errors, + f.newlines, + f.line_buffering, + ) + + sys.stdin = _reopen_stdio(sys.stdin, "rb") + sys.stdout = _reopen_stdio(sys.stdout, "wb") + sys.stderr = _reopen_stdio(sys.stderr, "wb") + + +@hookimpl(hookwrapper=True) +def pytest_load_initial_conftests(early_config: Config): + ns = early_config.known_args_namespace + if ns.capture == "fd": + _py36_windowsconsoleio_workaround(sys.stdout) + _colorama_workaround() + pluginmanager = early_config.pluginmanager + capman = CaptureManager(ns.capture) + pluginmanager.register(capman, "capturemanager") + + # Make sure that capturemanager is properly reset at final shutdown. + early_config.add_cleanup(capman.stop_global_capturing) + + # Finally trigger conftest loading but while capturing (issue #93). + capman.start_global_capturing() + outcome = yield + capman.suspend_global_capture() + if outcome.excinfo is not None: + out, err = capman.read_global_capture() + sys.stdout.write(out) + sys.stderr.write(err) + + +# IO Helpers. + + +class EncodedFile(io.TextIOWrapper): + __slots__ = () + + @property + def name(self) -> str: + # Ensure that file.name is a string. Workaround for a Python bug + # fixed in >=3.7.4: https://bugs.python.org/issue36015 + return repr(self.buffer) + + @property + def mode(self) -> str: + # TextIOWrapper doesn't expose a mode, but at least some of our + # tests check it. + return self.buffer.mode.replace("b", "") + + +class CaptureIO(io.TextIOWrapper): + def __init__(self) -> None: + super().__init__(io.BytesIO(), encoding="UTF-8", newline="", write_through=True) + + def getvalue(self) -> str: + assert isinstance(self.buffer, io.BytesIO) + return self.buffer.getvalue().decode("UTF-8") + + +class TeeCaptureIO(CaptureIO): + def __init__(self, other: TextIO) -> None: + self._other = other + super().__init__() + + def write(self, s: str) -> int: + super().write(s) + return self._other.write(s) + + +class DontReadFromInput: + encoding = None + + def read(self, *args): + raise OSError( + "pytest: reading from stdin while output is captured! Consider using `-s`." + ) + + readline = read + readlines = read + __next__ = read + + def __iter__(self): + return self + + def fileno(self) -> int: + raise UnsupportedOperation("redirected stdin is pseudofile, has no fileno()") + + def isatty(self) -> bool: + return False + + def close(self) -> None: + pass + + @property + def buffer(self): + return self + + +# Capture classes. + + +patchsysdict = {0: "stdin", 1: "stdout", 2: "stderr"} + + +class NoCapture: + EMPTY_BUFFER = None + __init__ = start = done = suspend = resume = lambda *args: None + + +class SysCaptureBinary: + + EMPTY_BUFFER = b"" + + def __init__(self, fd: int, tmpfile=None, *, tee: bool = False) -> None: + name = patchsysdict[fd] + self._old = getattr(sys, name) + self.name = name + if tmpfile is None: + if name == "stdin": + tmpfile = DontReadFromInput() + else: + tmpfile = CaptureIO() if not tee else TeeCaptureIO(self._old) + self.tmpfile = tmpfile + self._state = "initialized" + + def repr(self, class_name: str) -> str: + return "<{} {} _old={} _state={!r} tmpfile={!r}>".format( + class_name, + self.name, + hasattr(self, "_old") and repr(self._old) or "", + self._state, + self.tmpfile, + ) + + def __repr__(self) -> str: + return "<{} {} _old={} _state={!r} tmpfile={!r}>".format( + self.__class__.__name__, + self.name, + hasattr(self, "_old") and repr(self._old) or "", + self._state, + self.tmpfile, + ) + + def _assert_state(self, op: str, states: Tuple[str, ...]) -> None: + assert ( + self._state in states + ), "cannot {} in state {!r}: expected one of {}".format( + op, self._state, ", ".join(states) + ) + + def start(self) -> None: + self._assert_state("start", ("initialized",)) + setattr(sys, self.name, self.tmpfile) + self._state = "started" + + def snap(self): + self._assert_state("snap", ("started", "suspended")) + self.tmpfile.seek(0) + res = self.tmpfile.buffer.read() + self.tmpfile.seek(0) + self.tmpfile.truncate() + return res + + def done(self) -> None: + self._assert_state("done", ("initialized", "started", "suspended", "done")) + if self._state == "done": + return + setattr(sys, self.name, self._old) + del self._old + self.tmpfile.close() + self._state = "done" + + def suspend(self) -> None: + self._assert_state("suspend", ("started", "suspended")) + setattr(sys, self.name, self._old) + self._state = "suspended" + + def resume(self) -> None: + self._assert_state("resume", ("started", "suspended")) + if self._state == "started": + return + setattr(sys, self.name, self.tmpfile) + self._state = "started" + + def writeorg(self, data) -> None: + self._assert_state("writeorg", ("started", "suspended")) + self._old.flush() + self._old.buffer.write(data) + self._old.buffer.flush() + + +class SysCapture(SysCaptureBinary): + EMPTY_BUFFER = "" # type: ignore[assignment] + + def snap(self): + res = self.tmpfile.getvalue() + self.tmpfile.seek(0) + self.tmpfile.truncate() + return res + + def writeorg(self, data): + self._assert_state("writeorg", ("started", "suspended")) + self._old.write(data) + self._old.flush() + + +class FDCaptureBinary: + """Capture IO to/from a given OS-level file descriptor. + + snap() produces `bytes`. + """ + + EMPTY_BUFFER = b"" + + def __init__(self, targetfd: int) -> None: + self.targetfd = targetfd + + try: + os.fstat(targetfd) + except OSError: + # FD capturing is conceptually simple -- create a temporary file, + # redirect the FD to it, redirect back when done. But when the + # target FD is invalid it throws a wrench into this lovely scheme. + # + # Tests themselves shouldn't care if the FD is valid, FD capturing + # should work regardless of external circumstances. So falling back + # to just sys capturing is not a good option. + # + # Further complications are the need to support suspend() and the + # possibility of FD reuse (e.g. the tmpfile getting the very same + # target FD). The following approach is robust, I believe. + self.targetfd_invalid: Optional[int] = os.open(os.devnull, os.O_RDWR) + os.dup2(self.targetfd_invalid, targetfd) + else: + self.targetfd_invalid = None + self.targetfd_save = os.dup(targetfd) + + if targetfd == 0: + self.tmpfile = open(os.devnull) + self.syscapture = SysCapture(targetfd) + else: + self.tmpfile = EncodedFile( + TemporaryFile(buffering=0), + encoding="utf-8", + errors="replace", + newline="", + write_through=True, + ) + if targetfd in patchsysdict: + self.syscapture = SysCapture(targetfd, self.tmpfile) + else: + self.syscapture = NoCapture() + + self._state = "initialized" + + def __repr__(self) -> str: + return "<{} {} oldfd={} _state={!r} tmpfile={!r}>".format( + self.__class__.__name__, + self.targetfd, + self.targetfd_save, + self._state, + self.tmpfile, + ) + + def _assert_state(self, op: str, states: Tuple[str, ...]) -> None: + assert ( + self._state in states + ), "cannot {} in state {!r}: expected one of {}".format( + op, self._state, ", ".join(states) + ) + + def start(self) -> None: + """Start capturing on targetfd using memorized tmpfile.""" + self._assert_state("start", ("initialized",)) + os.dup2(self.tmpfile.fileno(), self.targetfd) + self.syscapture.start() + self._state = "started" + + def snap(self): + self._assert_state("snap", ("started", "suspended")) + self.tmpfile.seek(0) + res = self.tmpfile.buffer.read() + self.tmpfile.seek(0) + self.tmpfile.truncate() + return res + + def done(self) -> None: + """Stop capturing, restore streams, return original capture file, + seeked to position zero.""" + self._assert_state("done", ("initialized", "started", "suspended", "done")) + if self._state == "done": + return + os.dup2(self.targetfd_save, self.targetfd) + os.close(self.targetfd_save) + if self.targetfd_invalid is not None: + if self.targetfd_invalid != self.targetfd: + os.close(self.targetfd) + os.close(self.targetfd_invalid) + self.syscapture.done() + self.tmpfile.close() + self._state = "done" + + def suspend(self) -> None: + self._assert_state("suspend", ("started", "suspended")) + if self._state == "suspended": + return + self.syscapture.suspend() + os.dup2(self.targetfd_save, self.targetfd) + self._state = "suspended" + + def resume(self) -> None: + self._assert_state("resume", ("started", "suspended")) + if self._state == "started": + return + self.syscapture.resume() + os.dup2(self.tmpfile.fileno(), self.targetfd) + self._state = "started" + + def writeorg(self, data): + """Write to original file descriptor.""" + self._assert_state("writeorg", ("started", "suspended")) + os.write(self.targetfd_save, data) + + +class FDCapture(FDCaptureBinary): + """Capture IO to/from a given OS-level file descriptor. + + snap() produces text. + """ + + # Ignore type because it doesn't match the type in the superclass (bytes). + EMPTY_BUFFER = "" # type: ignore + + def snap(self): + self._assert_state("snap", ("started", "suspended")) + self.tmpfile.seek(0) + res = self.tmpfile.read() + self.tmpfile.seek(0) + self.tmpfile.truncate() + return res + + def writeorg(self, data): + """Write to original file descriptor.""" + super().writeorg(data.encode("utf-8")) # XXX use encoding of original stream + + +# MultiCapture + + +# This class was a namedtuple, but due to mypy limitation[0] it could not be +# made generic, so was replaced by a regular class which tries to emulate the +# pertinent parts of a namedtuple. If the mypy limitation is ever lifted, can +# make it a namedtuple again. +# [0]: https://github.com/python/mypy/issues/685 +@final +@functools.total_ordering +class CaptureResult(Generic[AnyStr]): + """The result of :method:`CaptureFixture.readouterr`.""" + + __slots__ = ("out", "err") + + def __init__(self, out: AnyStr, err: AnyStr) -> None: + self.out: AnyStr = out + self.err: AnyStr = err + + def __len__(self) -> int: + return 2 + + def __iter__(self) -> Iterator[AnyStr]: + return iter((self.out, self.err)) + + def __getitem__(self, item: int) -> AnyStr: + return tuple(self)[item] + + def _replace( + self, *, out: Optional[AnyStr] = None, err: Optional[AnyStr] = None + ) -> "CaptureResult[AnyStr]": + return CaptureResult( + out=self.out if out is None else out, err=self.err if err is None else err + ) + + def count(self, value: AnyStr) -> int: + return tuple(self).count(value) + + def index(self, value) -> int: + return tuple(self).index(value) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, (CaptureResult, tuple)): + return NotImplemented + return tuple(self) == tuple(other) + + def __hash__(self) -> int: + return hash(tuple(self)) + + def __lt__(self, other: object) -> bool: + if not isinstance(other, (CaptureResult, tuple)): + return NotImplemented + return tuple(self) < tuple(other) + + def __repr__(self) -> str: + return f"CaptureResult(out={self.out!r}, err={self.err!r})" + + +class MultiCapture(Generic[AnyStr]): + _state = None + _in_suspended = False + + def __init__(self, in_, out, err) -> None: + self.in_ = in_ + self.out = out + self.err = err + + def __repr__(self) -> str: + return "".format( + self.out, + self.err, + self.in_, + self._state, + self._in_suspended, + ) + + def start_capturing(self) -> None: + self._state = "started" + if self.in_: + self.in_.start() + if self.out: + self.out.start() + if self.err: + self.err.start() + + def pop_outerr_to_orig(self) -> Tuple[AnyStr, AnyStr]: + """Pop current snapshot out/err capture and flush to orig streams.""" + out, err = self.readouterr() + if out: + self.out.writeorg(out) + if err: + self.err.writeorg(err) + return out, err + + def suspend_capturing(self, in_: bool = False) -> None: + self._state = "suspended" + if self.out: + self.out.suspend() + if self.err: + self.err.suspend() + if in_ and self.in_: + self.in_.suspend() + self._in_suspended = True + + def resume_capturing(self) -> None: + self._state = "started" + if self.out: + self.out.resume() + if self.err: + self.err.resume() + if self._in_suspended: + self.in_.resume() + self._in_suspended = False + + def stop_capturing(self) -> None: + """Stop capturing and reset capturing streams.""" + if self._state == "stopped": + raise ValueError("was already stopped") + self._state = "stopped" + if self.out: + self.out.done() + if self.err: + self.err.done() + if self.in_: + self.in_.done() + + def is_started(self) -> bool: + """Whether actively capturing -- not suspended or stopped.""" + return self._state == "started" + + def readouterr(self) -> CaptureResult[AnyStr]: + out = self.out.snap() if self.out else "" + err = self.err.snap() if self.err else "" + return CaptureResult(out, err) + + +def _get_multicapture(method: "_CaptureMethod") -> MultiCapture[str]: + if method == "fd": + return MultiCapture(in_=FDCapture(0), out=FDCapture(1), err=FDCapture(2)) + elif method == "sys": + return MultiCapture(in_=SysCapture(0), out=SysCapture(1), err=SysCapture(2)) + elif method == "no": + return MultiCapture(in_=None, out=None, err=None) + elif method == "tee-sys": + return MultiCapture( + in_=None, out=SysCapture(1, tee=True), err=SysCapture(2, tee=True) + ) + raise ValueError(f"unknown capturing method: {method!r}") + + +# CaptureManager and CaptureFixture + + +class CaptureManager: + """The capture plugin. + + Manages that the appropriate capture method is enabled/disabled during + collection and each test phase (setup, call, teardown). After each of + those points, the captured output is obtained and attached to the + collection/runtest report. + + There are two levels of capture: + + * global: enabled by default and can be suppressed by the ``-s`` + option. This is always enabled/disabled during collection and each test + phase. + + * fixture: when a test function or one of its fixture depend on the + ``capsys`` or ``capfd`` fixtures. In this case special handling is + needed to ensure the fixtures take precedence over the global capture. + """ + + def __init__(self, method: "_CaptureMethod") -> None: + self._method = method + self._global_capturing: Optional[MultiCapture[str]] = None + self._capture_fixture: Optional[CaptureFixture[Any]] = None + + def __repr__(self) -> str: + return "".format( + self._method, self._global_capturing, self._capture_fixture + ) + + def is_capturing(self) -> Union[str, bool]: + if self.is_globally_capturing(): + return "global" + if self._capture_fixture: + return "fixture %s" % self._capture_fixture.request.fixturename + return False + + # Global capturing control + + def is_globally_capturing(self) -> bool: + return self._method != "no" + + def start_global_capturing(self) -> None: + assert self._global_capturing is None + self._global_capturing = _get_multicapture(self._method) + self._global_capturing.start_capturing() + + def stop_global_capturing(self) -> None: + if self._global_capturing is not None: + self._global_capturing.pop_outerr_to_orig() + self._global_capturing.stop_capturing() + self._global_capturing = None + + def resume_global_capture(self) -> None: + # During teardown of the python process, and on rare occasions, capture + # attributes can be `None` while trying to resume global capture. + if self._global_capturing is not None: + self._global_capturing.resume_capturing() + + def suspend_global_capture(self, in_: bool = False) -> None: + if self._global_capturing is not None: + self._global_capturing.suspend_capturing(in_=in_) + + def suspend(self, in_: bool = False) -> None: + # Need to undo local capsys-et-al if it exists before disabling global capture. + self.suspend_fixture() + self.suspend_global_capture(in_) + + def resume(self) -> None: + self.resume_global_capture() + self.resume_fixture() + + def read_global_capture(self) -> CaptureResult[str]: + assert self._global_capturing is not None + return self._global_capturing.readouterr() + + # Fixture Control + + def set_fixture(self, capture_fixture: "CaptureFixture[Any]") -> None: + if self._capture_fixture: + current_fixture = self._capture_fixture.request.fixturename + requested_fixture = capture_fixture.request.fixturename + capture_fixture.request.raiseerror( + "cannot use {} and {} at the same time".format( + requested_fixture, current_fixture + ) + ) + self._capture_fixture = capture_fixture + + def unset_fixture(self) -> None: + self._capture_fixture = None + + def activate_fixture(self) -> None: + """If the current item is using ``capsys`` or ``capfd``, activate + them so they take precedence over the global capture.""" + if self._capture_fixture: + self._capture_fixture._start() + + def deactivate_fixture(self) -> None: + """Deactivate the ``capsys`` or ``capfd`` fixture of this item, if any.""" + if self._capture_fixture: + self._capture_fixture.close() + + def suspend_fixture(self) -> None: + if self._capture_fixture: + self._capture_fixture._suspend() + + def resume_fixture(self) -> None: + if self._capture_fixture: + self._capture_fixture._resume() + + # Helper context managers + + @contextlib.contextmanager + def global_and_fixture_disabled(self) -> Generator[None, None, None]: + """Context manager to temporarily disable global and current fixture capturing.""" + do_fixture = self._capture_fixture and self._capture_fixture._is_started() + if do_fixture: + self.suspend_fixture() + do_global = self._global_capturing and self._global_capturing.is_started() + if do_global: + self.suspend_global_capture() + try: + yield + finally: + if do_global: + self.resume_global_capture() + if do_fixture: + self.resume_fixture() + + @contextlib.contextmanager + def item_capture(self, when: str, item: Item) -> Generator[None, None, None]: + self.resume_global_capture() + self.activate_fixture() + try: + yield + finally: + self.deactivate_fixture() + self.suspend_global_capture(in_=False) + + out, err = self.read_global_capture() + item.add_report_section(when, "stdout", out) + item.add_report_section(when, "stderr", err) + + # Hooks + + @hookimpl(hookwrapper=True) + def pytest_make_collect_report(self, collector: Collector): + if isinstance(collector, File): + self.resume_global_capture() + outcome = yield + self.suspend_global_capture() + out, err = self.read_global_capture() + rep = outcome.get_result() + if out: + rep.sections.append(("Captured stdout", out)) + if err: + rep.sections.append(("Captured stderr", err)) + else: + yield + + @hookimpl(hookwrapper=True) + def pytest_runtest_setup(self, item: Item) -> Generator[None, None, None]: + with self.item_capture("setup", item): + yield + + @hookimpl(hookwrapper=True) + def pytest_runtest_call(self, item: Item) -> Generator[None, None, None]: + with self.item_capture("call", item): + yield + + @hookimpl(hookwrapper=True) + def pytest_runtest_teardown(self, item: Item) -> Generator[None, None, None]: + with self.item_capture("teardown", item): + yield + + @hookimpl(tryfirst=True) + def pytest_keyboard_interrupt(self) -> None: + self.stop_global_capturing() + + @hookimpl(tryfirst=True) + def pytest_internalerror(self) -> None: + self.stop_global_capturing() + + +class CaptureFixture(Generic[AnyStr]): + """Object returned by the :fixture:`capsys`, :fixture:`capsysbinary`, + :fixture:`capfd` and :fixture:`capfdbinary` fixtures.""" + + def __init__( + self, captureclass, request: SubRequest, *, _ispytest: bool = False + ) -> None: + check_ispytest(_ispytest) + self.captureclass = captureclass + self.request = request + self._capture: Optional[MultiCapture[AnyStr]] = None + self._captured_out = self.captureclass.EMPTY_BUFFER + self._captured_err = self.captureclass.EMPTY_BUFFER + + def _start(self) -> None: + if self._capture is None: + self._capture = MultiCapture( + in_=None, + out=self.captureclass(1), + err=self.captureclass(2), + ) + self._capture.start_capturing() + + def close(self) -> None: + if self._capture is not None: + out, err = self._capture.pop_outerr_to_orig() + self._captured_out += out + self._captured_err += err + self._capture.stop_capturing() + self._capture = None + + def readouterr(self) -> CaptureResult[AnyStr]: + """Read and return the captured output so far, resetting the internal + buffer. + + :returns: + The captured content as a namedtuple with ``out`` and ``err`` + string attributes. + """ + captured_out, captured_err = self._captured_out, self._captured_err + if self._capture is not None: + out, err = self._capture.readouterr() + captured_out += out + captured_err += err + self._captured_out = self.captureclass.EMPTY_BUFFER + self._captured_err = self.captureclass.EMPTY_BUFFER + return CaptureResult(captured_out, captured_err) + + def _suspend(self) -> None: + """Suspend this fixture's own capturing temporarily.""" + if self._capture is not None: + self._capture.suspend_capturing() + + def _resume(self) -> None: + """Resume this fixture's own capturing temporarily.""" + if self._capture is not None: + self._capture.resume_capturing() + + def _is_started(self) -> bool: + """Whether actively capturing -- not disabled or closed.""" + if self._capture is not None: + return self._capture.is_started() + return False + + @contextlib.contextmanager + def disabled(self) -> Generator[None, None, None]: + """Temporarily disable capturing while inside the ``with`` block.""" + capmanager = self.request.config.pluginmanager.getplugin("capturemanager") + with capmanager.global_and_fixture_disabled(): + yield + + +# The fixtures. + + +@fixture +def capsys(request: SubRequest) -> Generator[CaptureFixture[str], None, None]: + """Enable text capturing of writes to ``sys.stdout`` and ``sys.stderr``. + + The captured output is made available via ``capsys.readouterr()`` method + calls, which return a ``(out, err)`` namedtuple. + ``out`` and ``err`` will be ``text`` objects. + """ + capman = request.config.pluginmanager.getplugin("capturemanager") + capture_fixture = CaptureFixture[str](SysCapture, request, _ispytest=True) + capman.set_fixture(capture_fixture) + capture_fixture._start() + yield capture_fixture + capture_fixture.close() + capman.unset_fixture() + + +@fixture +def capsysbinary(request: SubRequest) -> Generator[CaptureFixture[bytes], None, None]: + """Enable bytes capturing of writes to ``sys.stdout`` and ``sys.stderr``. + + The captured output is made available via ``capsysbinary.readouterr()`` + method calls, which return a ``(out, err)`` namedtuple. + ``out`` and ``err`` will be ``bytes`` objects. + """ + capman = request.config.pluginmanager.getplugin("capturemanager") + capture_fixture = CaptureFixture[bytes](SysCaptureBinary, request, _ispytest=True) + capman.set_fixture(capture_fixture) + capture_fixture._start() + yield capture_fixture + capture_fixture.close() + capman.unset_fixture() + + +@fixture +def capfd(request: SubRequest) -> Generator[CaptureFixture[str], None, None]: + """Enable text capturing of writes to file descriptors ``1`` and ``2``. + + The captured output is made available via ``capfd.readouterr()`` method + calls, which return a ``(out, err)`` namedtuple. + ``out`` and ``err`` will be ``text`` objects. + """ + capman = request.config.pluginmanager.getplugin("capturemanager") + capture_fixture = CaptureFixture[str](FDCapture, request, _ispytest=True) + capman.set_fixture(capture_fixture) + capture_fixture._start() + yield capture_fixture + capture_fixture.close() + capman.unset_fixture() + + +@fixture +def capfdbinary(request: SubRequest) -> Generator[CaptureFixture[bytes], None, None]: + """Enable bytes capturing of writes to file descriptors ``1`` and ``2``. + + The captured output is made available via ``capfd.readouterr()`` method + calls, which return a ``(out, err)`` namedtuple. + ``out`` and ``err`` will be ``byte`` objects. + """ + capman = request.config.pluginmanager.getplugin("capturemanager") + capture_fixture = CaptureFixture[bytes](FDCaptureBinary, request, _ispytest=True) + capman.set_fixture(capture_fixture) + capture_fixture._start() + yield capture_fixture + capture_fixture.close() + capman.unset_fixture() diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/compat.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/compat.py new file mode 100644 index 0000000000..7703dee8c5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/compat.py @@ -0,0 +1,417 @@ +"""Python version compatibility code.""" +import enum +import functools +import inspect +import os +import sys +from contextlib import contextmanager +from inspect import Parameter +from inspect import signature +from pathlib import Path +from typing import Any +from typing import Callable +from typing import Generic +from typing import Optional +from typing import Tuple +from typing import TYPE_CHECKING +from typing import TypeVar +from typing import Union + +import attr +import py + +if TYPE_CHECKING: + from typing import NoReturn + from typing_extensions import Final + + +_T = TypeVar("_T") +_S = TypeVar("_S") + +#: constant to prepare valuing pylib path replacements/lazy proxies later on +# intended for removal in pytest 8.0 or 9.0 + +# fmt: off +# intentional space to create a fake difference for the verification +LEGACY_PATH = py.path. local +# fmt: on + + +def legacy_path(path: Union[str, "os.PathLike[str]"]) -> LEGACY_PATH: + """Internal wrapper to prepare lazy proxies for legacy_path instances""" + return LEGACY_PATH(path) + + +# fmt: off +# Singleton type for NOTSET, as described in: +# https://www.python.org/dev/peps/pep-0484/#support-for-singleton-types-in-unions +class NotSetType(enum.Enum): + token = 0 +NOTSET: "Final" = NotSetType.token # noqa: E305 +# fmt: on + +if sys.version_info >= (3, 8): + from importlib import metadata as importlib_metadata +else: + import importlib_metadata # noqa: F401 + + +def _format_args(func: Callable[..., Any]) -> str: + return str(signature(func)) + + +def is_generator(func: object) -> bool: + genfunc = inspect.isgeneratorfunction(func) + return genfunc and not iscoroutinefunction(func) + + +def iscoroutinefunction(func: object) -> bool: + """Return True if func is a coroutine function (a function defined with async + def syntax, and doesn't contain yield), or a function decorated with + @asyncio.coroutine. + + Note: copied and modified from Python 3.5's builtin couroutines.py to avoid + importing asyncio directly, which in turns also initializes the "logging" + module as a side-effect (see issue #8). + """ + return inspect.iscoroutinefunction(func) or getattr(func, "_is_coroutine", False) + + +def is_async_function(func: object) -> bool: + """Return True if the given function seems to be an async function or + an async generator.""" + return iscoroutinefunction(func) or inspect.isasyncgenfunction(func) + + +def getlocation(function, curdir: Optional[str] = None) -> str: + function = get_real_func(function) + fn = Path(inspect.getfile(function)) + lineno = function.__code__.co_firstlineno + if curdir is not None: + try: + relfn = fn.relative_to(curdir) + except ValueError: + pass + else: + return "%s:%d" % (relfn, lineno + 1) + return "%s:%d" % (fn, lineno + 1) + + +def num_mock_patch_args(function) -> int: + """Return number of arguments used up by mock arguments (if any).""" + patchings = getattr(function, "patchings", None) + if not patchings: + return 0 + + mock_sentinel = getattr(sys.modules.get("mock"), "DEFAULT", object()) + ut_mock_sentinel = getattr(sys.modules.get("unittest.mock"), "DEFAULT", object()) + + return len( + [ + p + for p in patchings + if not p.attribute_name + and (p.new is mock_sentinel or p.new is ut_mock_sentinel) + ] + ) + + +def getfuncargnames( + function: Callable[..., Any], + *, + name: str = "", + is_method: bool = False, + cls: Optional[type] = None, +) -> Tuple[str, ...]: + """Return the names of a function's mandatory arguments. + + Should return the names of all function arguments that: + * Aren't bound to an instance or type as in instance or class methods. + * Don't have default values. + * Aren't bound with functools.partial. + * Aren't replaced with mocks. + + The is_method and cls arguments indicate that the function should + be treated as a bound method even though it's not unless, only in + the case of cls, the function is a static method. + + The name parameter should be the original name in which the function was collected. + """ + # TODO(RonnyPfannschmidt): This function should be refactored when we + # revisit fixtures. The fixture mechanism should ask the node for + # the fixture names, and not try to obtain directly from the + # function object well after collection has occurred. + + # The parameters attribute of a Signature object contains an + # ordered mapping of parameter names to Parameter instances. This + # creates a tuple of the names of the parameters that don't have + # defaults. + try: + parameters = signature(function).parameters + except (ValueError, TypeError) as e: + from _pytest.outcomes import fail + + fail( + f"Could not determine arguments of {function!r}: {e}", + pytrace=False, + ) + + arg_names = tuple( + p.name + for p in parameters.values() + if ( + p.kind is Parameter.POSITIONAL_OR_KEYWORD + or p.kind is Parameter.KEYWORD_ONLY + ) + and p.default is Parameter.empty + ) + if not name: + name = function.__name__ + + # If this function should be treated as a bound method even though + # it's passed as an unbound method or function, remove the first + # parameter name. + if is_method or ( + # Not using `getattr` because we don't want to resolve the staticmethod. + # Not using `cls.__dict__` because we want to check the entire MRO. + cls + and not isinstance( + inspect.getattr_static(cls, name, default=None), staticmethod + ) + ): + arg_names = arg_names[1:] + # Remove any names that will be replaced with mocks. + if hasattr(function, "__wrapped__"): + arg_names = arg_names[num_mock_patch_args(function) :] + return arg_names + + +if sys.version_info < (3, 7): + + @contextmanager + def nullcontext(): + yield + + +else: + from contextlib import nullcontext as nullcontext # noqa: F401 + + +def get_default_arg_names(function: Callable[..., Any]) -> Tuple[str, ...]: + # Note: this code intentionally mirrors the code at the beginning of + # getfuncargnames, to get the arguments which were excluded from its result + # because they had default values. + return tuple( + p.name + for p in signature(function).parameters.values() + if p.kind in (Parameter.POSITIONAL_OR_KEYWORD, Parameter.KEYWORD_ONLY) + and p.default is not Parameter.empty + ) + + +_non_printable_ascii_translate_table = { + i: f"\\x{i:02x}" for i in range(128) if i not in range(32, 127) +} +_non_printable_ascii_translate_table.update( + {ord("\t"): "\\t", ord("\r"): "\\r", ord("\n"): "\\n"} +) + + +def _translate_non_printable(s: str) -> str: + return s.translate(_non_printable_ascii_translate_table) + + +STRING_TYPES = bytes, str + + +def _bytes_to_ascii(val: bytes) -> str: + return val.decode("ascii", "backslashreplace") + + +def ascii_escaped(val: Union[bytes, str]) -> str: + r"""If val is pure ASCII, return it as an str, otherwise, escape + bytes objects into a sequence of escaped bytes: + + b'\xc3\xb4\xc5\xd6' -> r'\xc3\xb4\xc5\xd6' + + and escapes unicode objects into a sequence of escaped unicode + ids, e.g.: + + r'4\nV\U00043efa\x0eMXWB\x1e\u3028\u15fd\xcd\U0007d944' + + Note: + The obvious "v.decode('unicode-escape')" will return + valid UTF-8 unicode if it finds them in bytes, but we + want to return escaped bytes for any byte, even if they match + a UTF-8 string. + """ + if isinstance(val, bytes): + ret = _bytes_to_ascii(val) + else: + ret = val.encode("unicode_escape").decode("ascii") + return _translate_non_printable(ret) + + +@attr.s +class _PytestWrapper: + """Dummy wrapper around a function object for internal use only. + + Used to correctly unwrap the underlying function object when we are + creating fixtures, because we wrap the function object ourselves with a + decorator to issue warnings when the fixture function is called directly. + """ + + obj = attr.ib() + + +def get_real_func(obj): + """Get the real function object of the (possibly) wrapped object by + functools.wraps or functools.partial.""" + start_obj = obj + for i in range(100): + # __pytest_wrapped__ is set by @pytest.fixture when wrapping the fixture function + # to trigger a warning if it gets called directly instead of by pytest: we don't + # want to unwrap further than this otherwise we lose useful wrappings like @mock.patch (#3774) + new_obj = getattr(obj, "__pytest_wrapped__", None) + if isinstance(new_obj, _PytestWrapper): + obj = new_obj.obj + break + new_obj = getattr(obj, "__wrapped__", None) + if new_obj is None: + break + obj = new_obj + else: + from _pytest._io.saferepr import saferepr + + raise ValueError( + ("could not find real function of {start}\nstopped at {current}").format( + start=saferepr(start_obj), current=saferepr(obj) + ) + ) + if isinstance(obj, functools.partial): + obj = obj.func + return obj + + +def get_real_method(obj, holder): + """Attempt to obtain the real function object that might be wrapping + ``obj``, while at the same time returning a bound method to ``holder`` if + the original object was a bound method.""" + try: + is_method = hasattr(obj, "__func__") + obj = get_real_func(obj) + except Exception: # pragma: no cover + return obj + if is_method and hasattr(obj, "__get__") and callable(obj.__get__): + obj = obj.__get__(holder) + return obj + + +def getimfunc(func): + try: + return func.__func__ + except AttributeError: + return func + + +def safe_getattr(object: Any, name: str, default: Any) -> Any: + """Like getattr but return default upon any Exception or any OutcomeException. + + Attribute access can potentially fail for 'evil' Python objects. + See issue #214. + It catches OutcomeException because of #2490 (issue #580), new outcomes + are derived from BaseException instead of Exception (for more details + check #2707). + """ + from _pytest.outcomes import TEST_OUTCOME + + try: + return getattr(object, name, default) + except TEST_OUTCOME: + return default + + +def safe_isclass(obj: object) -> bool: + """Ignore any exception via isinstance on Python 3.""" + try: + return inspect.isclass(obj) + except Exception: + return False + + +if TYPE_CHECKING: + if sys.version_info >= (3, 8): + from typing import final as final + else: + from typing_extensions import final as final +elif sys.version_info >= (3, 8): + from typing import final as final +else: + + def final(f): + return f + + +if sys.version_info >= (3, 8): + from functools import cached_property as cached_property +else: + from typing import overload + from typing import Type + + class cached_property(Generic[_S, _T]): + __slots__ = ("func", "__doc__") + + def __init__(self, func: Callable[[_S], _T]) -> None: + self.func = func + self.__doc__ = func.__doc__ + + @overload + def __get__( + self, instance: None, owner: Optional[Type[_S]] = ... + ) -> "cached_property[_S, _T]": + ... + + @overload + def __get__(self, instance: _S, owner: Optional[Type[_S]] = ...) -> _T: + ... + + def __get__(self, instance, owner=None): + if instance is None: + return self + value = instance.__dict__[self.func.__name__] = self.func(instance) + return value + + +# Perform exhaustiveness checking. +# +# Consider this example: +# +# MyUnion = Union[int, str] +# +# def handle(x: MyUnion) -> int { +# if isinstance(x, int): +# return 1 +# elif isinstance(x, str): +# return 2 +# else: +# raise Exception('unreachable') +# +# Now suppose we add a new variant: +# +# MyUnion = Union[int, str, bytes] +# +# After doing this, we must remember ourselves to go and update the handle +# function to handle the new variant. +# +# With `assert_never` we can do better: +# +# // raise Exception('unreachable') +# return assert_never(x) +# +# Now, if we forget to handle the new variant, the type-checker will emit a +# compile-time error, instead of the runtime error we would have gotten +# previously. +# +# This also work for Enums (if you use `is` to compare) and Literals. +def assert_never(value: "NoReturn") -> "NoReturn": + assert False, f"Unhandled value: {value} ({type(value).__name__})" diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/__init__.py new file mode 100644 index 0000000000..ebf6e1b950 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/__init__.py @@ -0,0 +1,1697 @@ +"""Command line options, ini-file and conftest.py processing.""" +import argparse +import collections.abc +import contextlib +import copy +import enum +import inspect +import os +import re +import shlex +import sys +import types +import warnings +from functools import lru_cache +from pathlib import Path +from textwrap import dedent +from types import TracebackType +from typing import Any +from typing import Callable +from typing import cast +from typing import Dict +from typing import Generator +from typing import IO +from typing import Iterable +from typing import Iterator +from typing import List +from typing import Optional +from typing import Sequence +from typing import Set +from typing import TextIO +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import Union + +import attr +from pluggy import HookimplMarker +from pluggy import HookspecMarker +from pluggy import PluginManager + +import _pytest._code +import _pytest.deprecated +import _pytest.hookspec +from .exceptions import PrintHelp as PrintHelp +from .exceptions import UsageError as UsageError +from .findpaths import determine_setup +from _pytest._code import ExceptionInfo +from _pytest._code import filter_traceback +from _pytest._io import TerminalWriter +from _pytest.compat import final +from _pytest.compat import importlib_metadata +from _pytest.outcomes import fail +from _pytest.outcomes import Skipped +from _pytest.pathlib import absolutepath +from _pytest.pathlib import bestrelpath +from _pytest.pathlib import import_path +from _pytest.pathlib import ImportMode +from _pytest.pathlib import resolve_package_path +from _pytest.stash import Stash +from _pytest.warning_types import PytestConfigWarning + +if TYPE_CHECKING: + + from _pytest._code.code import _TracebackStyle + from _pytest.terminal import TerminalReporter + from .argparsing import Argument + + +_PluggyPlugin = object +"""A type to represent plugin objects. + +Plugins can be any namespace, so we can't narrow it down much, but we use an +alias to make the intent clear. + +Ideally this type would be provided by pluggy itself. +""" + + +hookimpl = HookimplMarker("pytest") +hookspec = HookspecMarker("pytest") + + +@final +class ExitCode(enum.IntEnum): + """Encodes the valid exit codes by pytest. + + Currently users and plugins may supply other exit codes as well. + + .. versionadded:: 5.0 + """ + + #: Tests passed. + OK = 0 + #: Tests failed. + TESTS_FAILED = 1 + #: pytest was interrupted. + INTERRUPTED = 2 + #: An internal error got in the way. + INTERNAL_ERROR = 3 + #: pytest was misused. + USAGE_ERROR = 4 + #: pytest couldn't find tests. + NO_TESTS_COLLECTED = 5 + + +class ConftestImportFailure(Exception): + def __init__( + self, + path: Path, + excinfo: Tuple[Type[Exception], Exception, TracebackType], + ) -> None: + super().__init__(path, excinfo) + self.path = path + self.excinfo = excinfo + + def __str__(self) -> str: + return "{}: {} (from {})".format( + self.excinfo[0].__name__, self.excinfo[1], self.path + ) + + +def filter_traceback_for_conftest_import_failure( + entry: _pytest._code.TracebackEntry, +) -> bool: + """Filter tracebacks entries which point to pytest internals or importlib. + + Make a special case for importlib because we use it to import test modules and conftest files + in _pytest.pathlib.import_path. + """ + return filter_traceback(entry) and "importlib" not in str(entry.path).split(os.sep) + + +def main( + args: Optional[Union[List[str], "os.PathLike[str]"]] = None, + plugins: Optional[Sequence[Union[str, _PluggyPlugin]]] = None, +) -> Union[int, ExitCode]: + """Perform an in-process test run. + + :param args: List of command line arguments. + :param plugins: List of plugin objects to be auto-registered during initialization. + + :returns: An exit code. + """ + try: + try: + config = _prepareconfig(args, plugins) + except ConftestImportFailure as e: + exc_info = ExceptionInfo.from_exc_info(e.excinfo) + tw = TerminalWriter(sys.stderr) + tw.line(f"ImportError while loading conftest '{e.path}'.", red=True) + exc_info.traceback = exc_info.traceback.filter( + filter_traceback_for_conftest_import_failure + ) + exc_repr = ( + exc_info.getrepr(style="short", chain=False) + if exc_info.traceback + else exc_info.exconly() + ) + formatted_tb = str(exc_repr) + for line in formatted_tb.splitlines(): + tw.line(line.rstrip(), red=True) + return ExitCode.USAGE_ERROR + else: + try: + ret: Union[ExitCode, int] = config.hook.pytest_cmdline_main( + config=config + ) + try: + return ExitCode(ret) + except ValueError: + return ret + finally: + config._ensure_unconfigure() + except UsageError as e: + tw = TerminalWriter(sys.stderr) + for msg in e.args: + tw.line(f"ERROR: {msg}\n", red=True) + return ExitCode.USAGE_ERROR + + +def console_main() -> int: + """The CLI entry point of pytest. + + This function is not meant for programmable use; use `main()` instead. + """ + # https://docs.python.org/3/library/signal.html#note-on-sigpipe + try: + code = main() + sys.stdout.flush() + return code + except BrokenPipeError: + # Python flushes standard streams on exit; redirect remaining output + # to devnull to avoid another BrokenPipeError at shutdown + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + return 1 # Python exits with error code 1 on EPIPE + + +class cmdline: # compatibility namespace + main = staticmethod(main) + + +def filename_arg(path: str, optname: str) -> str: + """Argparse type validator for filename arguments. + + :path: Path of filename. + :optname: Name of the option. + """ + if os.path.isdir(path): + raise UsageError(f"{optname} must be a filename, given: {path}") + return path + + +def directory_arg(path: str, optname: str) -> str: + """Argparse type validator for directory arguments. + + :path: Path of directory. + :optname: Name of the option. + """ + if not os.path.isdir(path): + raise UsageError(f"{optname} must be a directory, given: {path}") + return path + + +# Plugins that cannot be disabled via "-p no:X" currently. +essential_plugins = ( + "mark", + "main", + "runner", + "fixtures", + "helpconfig", # Provides -p. +) + +default_plugins = essential_plugins + ( + "python", + "terminal", + "debugging", + "unittest", + "capture", + "skipping", + "legacypath", + "tmpdir", + "monkeypatch", + "recwarn", + "pastebin", + "nose", + "assertion", + "junitxml", + "doctest", + "cacheprovider", + "freeze_support", + "setuponly", + "setupplan", + "stepwise", + "warnings", + "logging", + "reports", + "python_path", + *(["unraisableexception", "threadexception"] if sys.version_info >= (3, 8) else []), + "faulthandler", +) + +builtin_plugins = set(default_plugins) +builtin_plugins.add("pytester") +builtin_plugins.add("pytester_assertions") + + +def get_config( + args: Optional[List[str]] = None, + plugins: Optional[Sequence[Union[str, _PluggyPlugin]]] = None, +) -> "Config": + # subsequent calls to main will create a fresh instance + pluginmanager = PytestPluginManager() + config = Config( + pluginmanager, + invocation_params=Config.InvocationParams( + args=args or (), + plugins=plugins, + dir=Path.cwd(), + ), + ) + + if args is not None: + # Handle any "-p no:plugin" args. + pluginmanager.consider_preparse(args, exclude_only=True) + + for spec in default_plugins: + pluginmanager.import_plugin(spec) + + return config + + +def get_plugin_manager() -> "PytestPluginManager": + """Obtain a new instance of the + :py:class:`pytest.PytestPluginManager`, with default plugins + already loaded. + + This function can be used by integration with other tools, like hooking + into pytest to run tests into an IDE. + """ + return get_config().pluginmanager + + +def _prepareconfig( + args: Optional[Union[List[str], "os.PathLike[str]"]] = None, + plugins: Optional[Sequence[Union[str, _PluggyPlugin]]] = None, +) -> "Config": + if args is None: + args = sys.argv[1:] + elif isinstance(args, os.PathLike): + args = [os.fspath(args)] + elif not isinstance(args, list): + msg = "`args` parameter expected to be a list of strings, got: {!r} (type: {})" + raise TypeError(msg.format(args, type(args))) + + config = get_config(args, plugins) + pluginmanager = config.pluginmanager + try: + if plugins: + for plugin in plugins: + if isinstance(plugin, str): + pluginmanager.consider_pluginarg(plugin) + else: + pluginmanager.register(plugin) + config = pluginmanager.hook.pytest_cmdline_parse( + pluginmanager=pluginmanager, args=args + ) + return config + except BaseException: + config._ensure_unconfigure() + raise + + +def _get_directory(path: Path) -> Path: + """Get the directory of a path - itself if already a directory.""" + if path.is_file(): + return path.parent + else: + return path + + +@final +class PytestPluginManager(PluginManager): + """A :py:class:`pluggy.PluginManager ` with + additional pytest-specific functionality: + + * Loading plugins from the command line, ``PYTEST_PLUGINS`` env variable and + ``pytest_plugins`` global variables found in plugins being loaded. + * ``conftest.py`` loading during start-up. + """ + + def __init__(self) -> None: + import _pytest.assertion + + super().__init__("pytest") + # The objects are module objects, only used generically. + self._conftest_plugins: Set[types.ModuleType] = set() + + # State related to local conftest plugins. + self._dirpath2confmods: Dict[Path, List[types.ModuleType]] = {} + self._conftestpath2mod: Dict[Path, types.ModuleType] = {} + self._confcutdir: Optional[Path] = None + self._noconftest = False + + # _getconftestmodules()'s call to _get_directory() causes a stat + # storm when it's called potentially thousands of times in a test + # session (#9478), often with the same path, so cache it. + self._get_directory = lru_cache(256)(_get_directory) + + self._duplicatepaths: Set[Path] = set() + + # plugins that were explicitly skipped with pytest.skip + # list of (module name, skip reason) + # previously we would issue a warning when a plugin was skipped, but + # since we refactored warnings as first citizens of Config, they are + # just stored here to be used later. + self.skipped_plugins: List[Tuple[str, str]] = [] + + self.add_hookspecs(_pytest.hookspec) + self.register(self) + if os.environ.get("PYTEST_DEBUG"): + err: IO[str] = sys.stderr + encoding: str = getattr(err, "encoding", "utf8") + try: + err = open( + os.dup(err.fileno()), + mode=err.mode, + buffering=1, + encoding=encoding, + ) + except Exception: + pass + self.trace.root.setwriter(err.write) + self.enable_tracing() + + # Config._consider_importhook will set a real object if required. + self.rewrite_hook = _pytest.assertion.DummyRewriteHook() + # Used to know when we are importing conftests after the pytest_configure stage. + self._configured = False + + def parse_hookimpl_opts(self, plugin: _PluggyPlugin, name: str): + # pytest hooks are always prefixed with "pytest_", + # so we avoid accessing possibly non-readable attributes + # (see issue #1073). + if not name.startswith("pytest_"): + return + # Ignore names which can not be hooks. + if name == "pytest_plugins": + return + + method = getattr(plugin, name) + opts = super().parse_hookimpl_opts(plugin, name) + + # Consider only actual functions for hooks (#3775). + if not inspect.isroutine(method): + return + + # Collect unmarked hooks as long as they have the `pytest_' prefix. + if opts is None and name.startswith("pytest_"): + opts = {} + if opts is not None: + # TODO: DeprecationWarning, people should use hookimpl + # https://github.com/pytest-dev/pytest/issues/4562 + known_marks = {m.name for m in getattr(method, "pytestmark", [])} + + for name in ("tryfirst", "trylast", "optionalhook", "hookwrapper"): + opts.setdefault(name, hasattr(method, name) or name in known_marks) + return opts + + def parse_hookspec_opts(self, module_or_class, name: str): + opts = super().parse_hookspec_opts(module_or_class, name) + if opts is None: + method = getattr(module_or_class, name) + + if name.startswith("pytest_"): + # todo: deprecate hookspec hacks + # https://github.com/pytest-dev/pytest/issues/4562 + known_marks = {m.name for m in getattr(method, "pytestmark", [])} + opts = { + "firstresult": hasattr(method, "firstresult") + or "firstresult" in known_marks, + "historic": hasattr(method, "historic") + or "historic" in known_marks, + } + return opts + + def register( + self, plugin: _PluggyPlugin, name: Optional[str] = None + ) -> Optional[str]: + if name in _pytest.deprecated.DEPRECATED_EXTERNAL_PLUGINS: + warnings.warn( + PytestConfigWarning( + "{} plugin has been merged into the core, " + "please remove it from your requirements.".format( + name.replace("_", "-") + ) + ) + ) + return None + ret: Optional[str] = super().register(plugin, name) + if ret: + self.hook.pytest_plugin_registered.call_historic( + kwargs=dict(plugin=plugin, manager=self) + ) + + if isinstance(plugin, types.ModuleType): + self.consider_module(plugin) + return ret + + def getplugin(self, name: str): + # Support deprecated naming because plugins (xdist e.g.) use it. + plugin: Optional[_PluggyPlugin] = self.get_plugin(name) + return plugin + + def hasplugin(self, name: str) -> bool: + """Return whether a plugin with the given name is registered.""" + return bool(self.get_plugin(name)) + + def pytest_configure(self, config: "Config") -> None: + """:meta private:""" + # XXX now that the pluginmanager exposes hookimpl(tryfirst...) + # we should remove tryfirst/trylast as markers. + config.addinivalue_line( + "markers", + "tryfirst: mark a hook implementation function such that the " + "plugin machinery will try to call it first/as early as possible.", + ) + config.addinivalue_line( + "markers", + "trylast: mark a hook implementation function such that the " + "plugin machinery will try to call it last/as late as possible.", + ) + self._configured = True + + # + # Internal API for local conftest plugin handling. + # + def _set_initial_conftests( + self, namespace: argparse.Namespace, rootpath: Path + ) -> None: + """Load initial conftest files given a preparsed "namespace". + + As conftest files may add their own command line options which have + arguments ('--my-opt somepath') we might get some false positives. + All builtin and 3rd party plugins will have been loaded, however, so + common options will not confuse our logic here. + """ + current = Path.cwd() + self._confcutdir = ( + absolutepath(current / namespace.confcutdir) + if namespace.confcutdir + else None + ) + self._noconftest = namespace.noconftest + self._using_pyargs = namespace.pyargs + testpaths = namespace.file_or_dir + foundanchor = False + for testpath in testpaths: + path = str(testpath) + # remove node-id syntax + i = path.find("::") + if i != -1: + path = path[:i] + anchor = absolutepath(current / path) + if anchor.exists(): # we found some file object + self._try_load_conftest(anchor, namespace.importmode, rootpath) + foundanchor = True + if not foundanchor: + self._try_load_conftest(current, namespace.importmode, rootpath) + + def _try_load_conftest( + self, anchor: Path, importmode: Union[str, ImportMode], rootpath: Path + ) -> None: + self._getconftestmodules(anchor, importmode, rootpath) + # let's also consider test* subdirs + if anchor.is_dir(): + for x in anchor.glob("test*"): + if x.is_dir(): + self._getconftestmodules(x, importmode, rootpath) + + def _getconftestmodules( + self, path: Path, importmode: Union[str, ImportMode], rootpath: Path + ) -> List[types.ModuleType]: + if self._noconftest: + return [] + + directory = self._get_directory(path) + + # Optimization: avoid repeated searches in the same directory. + # Assumes always called with same importmode and rootpath. + existing_clist = self._dirpath2confmods.get(directory) + if existing_clist is not None: + return existing_clist + + # XXX these days we may rather want to use config.rootpath + # and allow users to opt into looking into the rootdir parent + # directories instead of requiring to specify confcutdir. + clist = [] + confcutdir_parents = self._confcutdir.parents if self._confcutdir else [] + for parent in reversed((directory, *directory.parents)): + if parent in confcutdir_parents: + continue + conftestpath = parent / "conftest.py" + if conftestpath.is_file(): + mod = self._importconftest(conftestpath, importmode, rootpath) + clist.append(mod) + self._dirpath2confmods[directory] = clist + return clist + + def _rget_with_confmod( + self, + name: str, + path: Path, + importmode: Union[str, ImportMode], + rootpath: Path, + ) -> Tuple[types.ModuleType, Any]: + modules = self._getconftestmodules(path, importmode, rootpath=rootpath) + for mod in reversed(modules): + try: + return mod, getattr(mod, name) + except AttributeError: + continue + raise KeyError(name) + + def _importconftest( + self, conftestpath: Path, importmode: Union[str, ImportMode], rootpath: Path + ) -> types.ModuleType: + # Use a resolved Path object as key to avoid loading the same conftest + # twice with build systems that create build directories containing + # symlinks to actual files. + # Using Path().resolve() is better than py.path.realpath because + # it resolves to the correct path/drive in case-insensitive file systems (#5792) + key = conftestpath.resolve() + + with contextlib.suppress(KeyError): + return self._conftestpath2mod[key] + + pkgpath = resolve_package_path(conftestpath) + if pkgpath is None: + _ensure_removed_sysmodule(conftestpath.stem) + + try: + mod = import_path(conftestpath, mode=importmode, root=rootpath) + except Exception as e: + assert e.__traceback__ is not None + exc_info = (type(e), e, e.__traceback__) + raise ConftestImportFailure(conftestpath, exc_info) from e + + self._check_non_top_pytest_plugins(mod, conftestpath) + + self._conftest_plugins.add(mod) + self._conftestpath2mod[key] = mod + dirpath = conftestpath.parent + if dirpath in self._dirpath2confmods: + for path, mods in self._dirpath2confmods.items(): + if path and dirpath in path.parents or path == dirpath: + assert mod not in mods + mods.append(mod) + self.trace(f"loading conftestmodule {mod!r}") + self.consider_conftest(mod) + return mod + + def _check_non_top_pytest_plugins( + self, + mod: types.ModuleType, + conftestpath: Path, + ) -> None: + if ( + hasattr(mod, "pytest_plugins") + and self._configured + and not self._using_pyargs + ): + msg = ( + "Defining 'pytest_plugins' in a non-top-level conftest is no longer supported:\n" + "It affects the entire test suite instead of just below the conftest as expected.\n" + " {}\n" + "Please move it to a top level conftest file at the rootdir:\n" + " {}\n" + "For more information, visit:\n" + " https://docs.pytest.org/en/stable/deprecations.html#pytest-plugins-in-non-top-level-conftest-files" + ) + fail(msg.format(conftestpath, self._confcutdir), pytrace=False) + + # + # API for bootstrapping plugin loading + # + # + + def consider_preparse( + self, args: Sequence[str], *, exclude_only: bool = False + ) -> None: + """:meta private:""" + i = 0 + n = len(args) + while i < n: + opt = args[i] + i += 1 + if isinstance(opt, str): + if opt == "-p": + try: + parg = args[i] + except IndexError: + return + i += 1 + elif opt.startswith("-p"): + parg = opt[2:] + else: + continue + if exclude_only and not parg.startswith("no:"): + continue + self.consider_pluginarg(parg) + + def consider_pluginarg(self, arg: str) -> None: + """:meta private:""" + if arg.startswith("no:"): + name = arg[3:] + if name in essential_plugins: + raise UsageError("plugin %s cannot be disabled" % name) + + # PR #4304: remove stepwise if cacheprovider is blocked. + if name == "cacheprovider": + self.set_blocked("stepwise") + self.set_blocked("pytest_stepwise") + + self.set_blocked(name) + if not name.startswith("pytest_"): + self.set_blocked("pytest_" + name) + else: + name = arg + # Unblock the plugin. None indicates that it has been blocked. + # There is no interface with pluggy for this. + if self._name2plugin.get(name, -1) is None: + del self._name2plugin[name] + if not name.startswith("pytest_"): + if self._name2plugin.get("pytest_" + name, -1) is None: + del self._name2plugin["pytest_" + name] + self.import_plugin(arg, consider_entry_points=True) + + def consider_conftest(self, conftestmodule: types.ModuleType) -> None: + """:meta private:""" + self.register(conftestmodule, name=conftestmodule.__file__) + + def consider_env(self) -> None: + """:meta private:""" + self._import_plugin_specs(os.environ.get("PYTEST_PLUGINS")) + + def consider_module(self, mod: types.ModuleType) -> None: + """:meta private:""" + self._import_plugin_specs(getattr(mod, "pytest_plugins", [])) + + def _import_plugin_specs( + self, spec: Union[None, types.ModuleType, str, Sequence[str]] + ) -> None: + plugins = _get_plugin_specs_as_list(spec) + for import_spec in plugins: + self.import_plugin(import_spec) + + def import_plugin(self, modname: str, consider_entry_points: bool = False) -> None: + """Import a plugin with ``modname``. + + If ``consider_entry_points`` is True, entry point names are also + considered to find a plugin. + """ + # Most often modname refers to builtin modules, e.g. "pytester", + # "terminal" or "capture". Those plugins are registered under their + # basename for historic purposes but must be imported with the + # _pytest prefix. + assert isinstance(modname, str), ( + "module name as text required, got %r" % modname + ) + if self.is_blocked(modname) or self.get_plugin(modname) is not None: + return + + importspec = "_pytest." + modname if modname in builtin_plugins else modname + self.rewrite_hook.mark_rewrite(importspec) + + if consider_entry_points: + loaded = self.load_setuptools_entrypoints("pytest11", name=modname) + if loaded: + return + + try: + __import__(importspec) + except ImportError as e: + raise ImportError( + f'Error importing plugin "{modname}": {e.args[0]}' + ).with_traceback(e.__traceback__) from e + + except Skipped as e: + self.skipped_plugins.append((modname, e.msg or "")) + else: + mod = sys.modules[importspec] + self.register(mod, modname) + + +def _get_plugin_specs_as_list( + specs: Union[None, types.ModuleType, str, Sequence[str]] +) -> List[str]: + """Parse a plugins specification into a list of plugin names.""" + # None means empty. + if specs is None: + return [] + # Workaround for #3899 - a submodule which happens to be called "pytest_plugins". + if isinstance(specs, types.ModuleType): + return [] + # Comma-separated list. + if isinstance(specs, str): + return specs.split(",") if specs else [] + # Direct specification. + if isinstance(specs, collections.abc.Sequence): + return list(specs) + raise UsageError( + "Plugins may be specified as a sequence or a ','-separated string of plugin names. Got: %r" + % specs + ) + + +def _ensure_removed_sysmodule(modname: str) -> None: + try: + del sys.modules[modname] + except KeyError: + pass + + +class Notset: + def __repr__(self): + return "" + + +notset = Notset() + + +def _iter_rewritable_modules(package_files: Iterable[str]) -> Iterator[str]: + """Given an iterable of file names in a source distribution, return the "names" that should + be marked for assertion rewrite. + + For example the package "pytest_mock/__init__.py" should be added as "pytest_mock" in + the assertion rewrite mechanism. + + This function has to deal with dist-info based distributions and egg based distributions + (which are still very much in use for "editable" installs). + + Here are the file names as seen in a dist-info based distribution: + + pytest_mock/__init__.py + pytest_mock/_version.py + pytest_mock/plugin.py + pytest_mock.egg-info/PKG-INFO + + Here are the file names as seen in an egg based distribution: + + src/pytest_mock/__init__.py + src/pytest_mock/_version.py + src/pytest_mock/plugin.py + src/pytest_mock.egg-info/PKG-INFO + LICENSE + setup.py + + We have to take in account those two distribution flavors in order to determine which + names should be considered for assertion rewriting. + + More information: + https://github.com/pytest-dev/pytest-mock/issues/167 + """ + package_files = list(package_files) + seen_some = False + for fn in package_files: + is_simple_module = "/" not in fn and fn.endswith(".py") + is_package = fn.count("/") == 1 and fn.endswith("__init__.py") + if is_simple_module: + module_name, _ = os.path.splitext(fn) + # we ignore "setup.py" at the root of the distribution + if module_name != "setup": + seen_some = True + yield module_name + elif is_package: + package_name = os.path.dirname(fn) + seen_some = True + yield package_name + + if not seen_some: + # At this point we did not find any packages or modules suitable for assertion + # rewriting, so we try again by stripping the first path component (to account for + # "src" based source trees for example). + # This approach lets us have the common case continue to be fast, as egg-distributions + # are rarer. + new_package_files = [] + for fn in package_files: + parts = fn.split("/") + new_fn = "/".join(parts[1:]) + if new_fn: + new_package_files.append(new_fn) + if new_package_files: + yield from _iter_rewritable_modules(new_package_files) + + +def _args_converter(args: Iterable[str]) -> Tuple[str, ...]: + return tuple(args) + + +@final +class Config: + """Access to configuration values, pluginmanager and plugin hooks. + + :param PytestPluginManager pluginmanager: + A pytest PluginManager. + + :param InvocationParams invocation_params: + Object containing parameters regarding the :func:`pytest.main` + invocation. + """ + + @final + @attr.s(frozen=True, auto_attribs=True) + class InvocationParams: + """Holds parameters passed during :func:`pytest.main`. + + The object attributes are read-only. + + .. versionadded:: 5.1 + + .. note:: + + Note that the environment variable ``PYTEST_ADDOPTS`` and the ``addopts`` + ini option are handled by pytest, not being included in the ``args`` attribute. + + Plugins accessing ``InvocationParams`` must be aware of that. + """ + + args: Tuple[str, ...] = attr.ib(converter=_args_converter) + """The command-line arguments as passed to :func:`pytest.main`.""" + plugins: Optional[Sequence[Union[str, _PluggyPlugin]]] + """Extra plugins, might be `None`.""" + dir: Path + """The directory from which :func:`pytest.main` was invoked.""" + + def __init__( + self, + pluginmanager: PytestPluginManager, + *, + invocation_params: Optional[InvocationParams] = None, + ) -> None: + from .argparsing import Parser, FILE_OR_DIR + + if invocation_params is None: + invocation_params = self.InvocationParams( + args=(), plugins=None, dir=Path.cwd() + ) + + self.option = argparse.Namespace() + """Access to command line option as attributes. + + :type: argparse.Namespace + """ + + self.invocation_params = invocation_params + """The parameters with which pytest was invoked. + + :type: InvocationParams + """ + + _a = FILE_OR_DIR + self._parser = Parser( + usage=f"%(prog)s [options] [{_a}] [{_a}] [...]", + processopt=self._processopt, + _ispytest=True, + ) + self.pluginmanager = pluginmanager + """The plugin manager handles plugin registration and hook invocation. + + :type: PytestPluginManager + """ + + self.stash = Stash() + """A place where plugins can store information on the config for their + own use. + + :type: Stash + """ + # Deprecated alias. Was never public. Can be removed in a few releases. + self._store = self.stash + + from .compat import PathAwareHookProxy + + self.trace = self.pluginmanager.trace.root.get("config") + self.hook = PathAwareHookProxy(self.pluginmanager.hook) + self._inicache: Dict[str, Any] = {} + self._override_ini: Sequence[str] = () + self._opt2dest: Dict[str, str] = {} + self._cleanup: List[Callable[[], None]] = [] + self.pluginmanager.register(self, "pytestconfig") + self._configured = False + self.hook.pytest_addoption.call_historic( + kwargs=dict(parser=self._parser, pluginmanager=self.pluginmanager) + ) + + if TYPE_CHECKING: + from _pytest.cacheprovider import Cache + + self.cache: Optional[Cache] = None + + @property + def rootpath(self) -> Path: + """The path to the :ref:`rootdir `. + + :type: pathlib.Path + + .. versionadded:: 6.1 + """ + return self._rootpath + + @property + def inipath(self) -> Optional[Path]: + """The path to the :ref:`configfile `. + + :type: Optional[pathlib.Path] + + .. versionadded:: 6.1 + """ + return self._inipath + + def add_cleanup(self, func: Callable[[], None]) -> None: + """Add a function to be called when the config object gets out of + use (usually coinciding with pytest_unconfigure).""" + self._cleanup.append(func) + + def _do_configure(self) -> None: + assert not self._configured + self._configured = True + with warnings.catch_warnings(): + warnings.simplefilter("default") + self.hook.pytest_configure.call_historic(kwargs=dict(config=self)) + + def _ensure_unconfigure(self) -> None: + if self._configured: + self._configured = False + self.hook.pytest_unconfigure(config=self) + self.hook.pytest_configure._call_history = [] + while self._cleanup: + fin = self._cleanup.pop() + fin() + + def get_terminal_writer(self) -> TerminalWriter: + terminalreporter: TerminalReporter = self.pluginmanager.get_plugin( + "terminalreporter" + ) + return terminalreporter._tw + + def pytest_cmdline_parse( + self, pluginmanager: PytestPluginManager, args: List[str] + ) -> "Config": + try: + self.parse(args) + except UsageError: + + # Handle --version and --help here in a minimal fashion. + # This gets done via helpconfig normally, but its + # pytest_cmdline_main is not called in case of errors. + if getattr(self.option, "version", False) or "--version" in args: + from _pytest.helpconfig import showversion + + showversion(self) + elif ( + getattr(self.option, "help", False) or "--help" in args or "-h" in args + ): + self._parser._getparser().print_help() + sys.stdout.write( + "\nNOTE: displaying only minimal help due to UsageError.\n\n" + ) + + raise + + return self + + def notify_exception( + self, + excinfo: ExceptionInfo[BaseException], + option: Optional[argparse.Namespace] = None, + ) -> None: + if option and getattr(option, "fulltrace", False): + style: _TracebackStyle = "long" + else: + style = "native" + excrepr = excinfo.getrepr( + funcargs=True, showlocals=getattr(option, "showlocals", False), style=style + ) + res = self.hook.pytest_internalerror(excrepr=excrepr, excinfo=excinfo) + if not any(res): + for line in str(excrepr).split("\n"): + sys.stderr.write("INTERNALERROR> %s\n" % line) + sys.stderr.flush() + + def cwd_relative_nodeid(self, nodeid: str) -> str: + # nodeid's are relative to the rootpath, compute relative to cwd. + if self.invocation_params.dir != self.rootpath: + fullpath = self.rootpath / nodeid + nodeid = bestrelpath(self.invocation_params.dir, fullpath) + return nodeid + + @classmethod + def fromdictargs(cls, option_dict, args) -> "Config": + """Constructor usable for subprocesses.""" + config = get_config(args) + config.option.__dict__.update(option_dict) + config.parse(args, addopts=False) + for x in config.option.plugins: + config.pluginmanager.consider_pluginarg(x) + return config + + def _processopt(self, opt: "Argument") -> None: + for name in opt._short_opts + opt._long_opts: + self._opt2dest[name] = opt.dest + + if hasattr(opt, "default"): + if not hasattr(self.option, opt.dest): + setattr(self.option, opt.dest, opt.default) + + @hookimpl(trylast=True) + def pytest_load_initial_conftests(self, early_config: "Config") -> None: + self.pluginmanager._set_initial_conftests( + early_config.known_args_namespace, rootpath=early_config.rootpath + ) + + def _initini(self, args: Sequence[str]) -> None: + ns, unknown_args = self._parser.parse_known_and_unknown_args( + args, namespace=copy.copy(self.option) + ) + rootpath, inipath, inicfg = determine_setup( + ns.inifilename, + ns.file_or_dir + unknown_args, + rootdir_cmd_arg=ns.rootdir or None, + config=self, + ) + self._rootpath = rootpath + self._inipath = inipath + self.inicfg = inicfg + self._parser.extra_info["rootdir"] = str(self.rootpath) + self._parser.extra_info["inifile"] = str(self.inipath) + self._parser.addini("addopts", "extra command line options", "args") + self._parser.addini("minversion", "minimally required pytest version") + self._parser.addini( + "required_plugins", + "plugins that must be present for pytest to run", + type="args", + default=[], + ) + self._override_ini = ns.override_ini or () + + def _consider_importhook(self, args: Sequence[str]) -> None: + """Install the PEP 302 import hook if using assertion rewriting. + + Needs to parse the --assert= option from the commandline + and find all the installed plugins to mark them for rewriting + by the importhook. + """ + ns, unknown_args = self._parser.parse_known_and_unknown_args(args) + mode = getattr(ns, "assertmode", "plain") + if mode == "rewrite": + import _pytest.assertion + + try: + hook = _pytest.assertion.install_importhook(self) + except SystemError: + mode = "plain" + else: + self._mark_plugins_for_rewrite(hook) + self._warn_about_missing_assertion(mode) + + def _mark_plugins_for_rewrite(self, hook) -> None: + """Given an importhook, mark for rewrite any top-level + modules or packages in the distribution package for + all pytest plugins.""" + self.pluginmanager.rewrite_hook = hook + + if os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"): + # We don't autoload from setuptools entry points, no need to continue. + return + + package_files = ( + str(file) + for dist in importlib_metadata.distributions() + if any(ep.group == "pytest11" for ep in dist.entry_points) + for file in dist.files or [] + ) + + for name in _iter_rewritable_modules(package_files): + hook.mark_rewrite(name) + + def _validate_args(self, args: List[str], via: str) -> List[str]: + """Validate known args.""" + self._parser._config_source_hint = via # type: ignore + try: + self._parser.parse_known_and_unknown_args( + args, namespace=copy.copy(self.option) + ) + finally: + del self._parser._config_source_hint # type: ignore + + return args + + def _preparse(self, args: List[str], addopts: bool = True) -> None: + if addopts: + env_addopts = os.environ.get("PYTEST_ADDOPTS", "") + if len(env_addopts): + args[:] = ( + self._validate_args(shlex.split(env_addopts), "via PYTEST_ADDOPTS") + + args + ) + self._initini(args) + if addopts: + args[:] = ( + self._validate_args(self.getini("addopts"), "via addopts config") + args + ) + + self.known_args_namespace = self._parser.parse_known_args( + args, namespace=copy.copy(self.option) + ) + self._checkversion() + self._consider_importhook(args) + self.pluginmanager.consider_preparse(args, exclude_only=False) + if not os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"): + # Don't autoload from setuptools entry point. Only explicitly specified + # plugins are going to be loaded. + self.pluginmanager.load_setuptools_entrypoints("pytest11") + self.pluginmanager.consider_env() + + self.known_args_namespace = self._parser.parse_known_args( + args, namespace=copy.copy(self.known_args_namespace) + ) + + self._validate_plugins() + self._warn_about_skipped_plugins() + + if self.known_args_namespace.strict: + self.issue_config_time_warning( + _pytest.deprecated.STRICT_OPTION, stacklevel=2 + ) + + if self.known_args_namespace.confcutdir is None and self.inipath is not None: + confcutdir = str(self.inipath.parent) + self.known_args_namespace.confcutdir = confcutdir + try: + self.hook.pytest_load_initial_conftests( + early_config=self, args=args, parser=self._parser + ) + except ConftestImportFailure as e: + if self.known_args_namespace.help or self.known_args_namespace.version: + # we don't want to prevent --help/--version to work + # so just let is pass and print a warning at the end + self.issue_config_time_warning( + PytestConfigWarning(f"could not load initial conftests: {e.path}"), + stacklevel=2, + ) + else: + raise + + @hookimpl(hookwrapper=True) + def pytest_collection(self) -> Generator[None, None, None]: + # Validate invalid ini keys after collection is done so we take in account + # options added by late-loading conftest files. + yield + self._validate_config_options() + + def _checkversion(self) -> None: + import pytest + + minver = self.inicfg.get("minversion", None) + if minver: + # Imported lazily to improve start-up time. + from packaging.version import Version + + if not isinstance(minver, str): + raise pytest.UsageError( + "%s: 'minversion' must be a single value" % self.inipath + ) + + if Version(minver) > Version(pytest.__version__): + raise pytest.UsageError( + "%s: 'minversion' requires pytest-%s, actual pytest-%s'" + % ( + self.inipath, + minver, + pytest.__version__, + ) + ) + + def _validate_config_options(self) -> None: + for key in sorted(self._get_unknown_ini_keys()): + self._warn_or_fail_if_strict(f"Unknown config option: {key}\n") + + def _validate_plugins(self) -> None: + required_plugins = sorted(self.getini("required_plugins")) + if not required_plugins: + return + + # Imported lazily to improve start-up time. + from packaging.version import Version + from packaging.requirements import InvalidRequirement, Requirement + + plugin_info = self.pluginmanager.list_plugin_distinfo() + plugin_dist_info = {dist.project_name: dist.version for _, dist in plugin_info} + + missing_plugins = [] + for required_plugin in required_plugins: + try: + req = Requirement(required_plugin) + except InvalidRequirement: + missing_plugins.append(required_plugin) + continue + + if req.name not in plugin_dist_info: + missing_plugins.append(required_plugin) + elif not req.specifier.contains( + Version(plugin_dist_info[req.name]), prereleases=True + ): + missing_plugins.append(required_plugin) + + if missing_plugins: + raise UsageError( + "Missing required plugins: {}".format(", ".join(missing_plugins)), + ) + + def _warn_or_fail_if_strict(self, message: str) -> None: + if self.known_args_namespace.strict_config: + raise UsageError(message) + + self.issue_config_time_warning(PytestConfigWarning(message), stacklevel=3) + + def _get_unknown_ini_keys(self) -> List[str]: + parser_inicfg = self._parser._inidict + return [name for name in self.inicfg if name not in parser_inicfg] + + def parse(self, args: List[str], addopts: bool = True) -> None: + # Parse given cmdline arguments into this config object. + assert not hasattr( + self, "args" + ), "can only parse cmdline args at most once per Config object" + self.hook.pytest_addhooks.call_historic( + kwargs=dict(pluginmanager=self.pluginmanager) + ) + self._preparse(args, addopts=addopts) + # XXX deprecated hook: + self.hook.pytest_cmdline_preparse(config=self, args=args) + self._parser.after_preparse = True # type: ignore + try: + args = self._parser.parse_setoption( + args, self.option, namespace=self.option + ) + if not args: + if self.invocation_params.dir == self.rootpath: + args = self.getini("testpaths") + if not args: + args = [str(self.invocation_params.dir)] + self.args = args + except PrintHelp: + pass + + def issue_config_time_warning(self, warning: Warning, stacklevel: int) -> None: + """Issue and handle a warning during the "configure" stage. + + During ``pytest_configure`` we can't capture warnings using the ``catch_warnings_for_item`` + function because it is not possible to have hookwrappers around ``pytest_configure``. + + This function is mainly intended for plugins that need to issue warnings during + ``pytest_configure`` (or similar stages). + + :param warning: The warning instance. + :param stacklevel: stacklevel forwarded to warnings.warn. + """ + if self.pluginmanager.is_blocked("warnings"): + return + + cmdline_filters = self.known_args_namespace.pythonwarnings or [] + config_filters = self.getini("filterwarnings") + + with warnings.catch_warnings(record=True) as records: + warnings.simplefilter("always", type(warning)) + apply_warning_filters(config_filters, cmdline_filters) + warnings.warn(warning, stacklevel=stacklevel) + + if records: + frame = sys._getframe(stacklevel - 1) + location = frame.f_code.co_filename, frame.f_lineno, frame.f_code.co_name + self.hook.pytest_warning_captured.call_historic( + kwargs=dict( + warning_message=records[0], + when="config", + item=None, + location=location, + ) + ) + self.hook.pytest_warning_recorded.call_historic( + kwargs=dict( + warning_message=records[0], + when="config", + nodeid="", + location=location, + ) + ) + + def addinivalue_line(self, name: str, line: str) -> None: + """Add a line to an ini-file option. The option must have been + declared but might not yet be set in which case the line becomes + the first line in its value.""" + x = self.getini(name) + assert isinstance(x, list) + x.append(line) # modifies the cached list inline + + def getini(self, name: str): + """Return configuration value from an :ref:`ini file `. + + If the specified name hasn't been registered through a prior + :func:`parser.addini ` call (usually from a + plugin), a ValueError is raised. + """ + try: + return self._inicache[name] + except KeyError: + self._inicache[name] = val = self._getini(name) + return val + + # Meant for easy monkeypatching by legacypath plugin. + # Can be inlined back (with no cover removed) once legacypath is gone. + def _getini_unknown_type(self, name: str, type: str, value: Union[str, List[str]]): + msg = f"unknown configuration type: {type}" + raise ValueError(msg, value) # pragma: no cover + + def _getini(self, name: str): + try: + description, type, default = self._parser._inidict[name] + except KeyError as e: + raise ValueError(f"unknown configuration value: {name!r}") from e + override_value = self._get_override_ini_value(name) + if override_value is None: + try: + value = self.inicfg[name] + except KeyError: + if default is not None: + return default + if type is None: + return "" + return [] + else: + value = override_value + # Coerce the values based on types. + # + # Note: some coercions are only required if we are reading from .ini files, because + # the file format doesn't contain type information, but when reading from toml we will + # get either str or list of str values (see _parse_ini_config_from_pyproject_toml). + # For example: + # + # ini: + # a_line_list = "tests acceptance" + # in this case, we need to split the string to obtain a list of strings. + # + # toml: + # a_line_list = ["tests", "acceptance"] + # in this case, we already have a list ready to use. + # + if type == "paths": + # TODO: This assert is probably not valid in all cases. + assert self.inipath is not None + dp = self.inipath.parent + input_values = shlex.split(value) if isinstance(value, str) else value + return [dp / x for x in input_values] + elif type == "args": + return shlex.split(value) if isinstance(value, str) else value + elif type == "linelist": + if isinstance(value, str): + return [t for t in map(lambda x: x.strip(), value.split("\n")) if t] + else: + return value + elif type == "bool": + return _strtobool(str(value).strip()) + elif type == "string": + return value + elif type is None: + return value + else: + return self._getini_unknown_type(name, type, value) + + def _getconftest_pathlist( + self, name: str, path: Path, rootpath: Path + ) -> Optional[List[Path]]: + try: + mod, relroots = self.pluginmanager._rget_with_confmod( + name, path, self.getoption("importmode"), rootpath + ) + except KeyError: + return None + modpath = Path(mod.__file__).parent + values: List[Path] = [] + for relroot in relroots: + if isinstance(relroot, os.PathLike): + relroot = Path(relroot) + else: + relroot = relroot.replace("/", os.sep) + relroot = absolutepath(modpath / relroot) + values.append(relroot) + return values + + def _get_override_ini_value(self, name: str) -> Optional[str]: + value = None + # override_ini is a list of "ini=value" options. + # Always use the last item if multiple values are set for same ini-name, + # e.g. -o foo=bar1 -o foo=bar2 will set foo to bar2. + for ini_config in self._override_ini: + try: + key, user_ini_value = ini_config.split("=", 1) + except ValueError as e: + raise UsageError( + "-o/--override-ini expects option=value style (got: {!r}).".format( + ini_config + ) + ) from e + else: + if key == name: + value = user_ini_value + return value + + def getoption(self, name: str, default=notset, skip: bool = False): + """Return command line option value. + + :param name: Name of the option. You may also specify + the literal ``--OPT`` option instead of the "dest" option name. + :param default: Default value if no option of that name exists. + :param skip: If True, raise pytest.skip if option does not exists + or has a None value. + """ + name = self._opt2dest.get(name, name) + try: + val = getattr(self.option, name) + if val is None and skip: + raise AttributeError(name) + return val + except AttributeError as e: + if default is not notset: + return default + if skip: + import pytest + + pytest.skip(f"no {name!r} option found") + raise ValueError(f"no option named {name!r}") from e + + def getvalue(self, name: str, path=None): + """Deprecated, use getoption() instead.""" + return self.getoption(name) + + def getvalueorskip(self, name: str, path=None): + """Deprecated, use getoption(skip=True) instead.""" + return self.getoption(name, skip=True) + + def _warn_about_missing_assertion(self, mode: str) -> None: + if not _assertion_supported(): + if mode == "plain": + warning_text = ( + "ASSERTIONS ARE NOT EXECUTED" + " and FAILING TESTS WILL PASS. Are you" + " using python -O?" + ) + else: + warning_text = ( + "assertions not in test modules or" + " plugins will be ignored" + " because assert statements are not executed " + "by the underlying Python interpreter " + "(are you using python -O?)\n" + ) + self.issue_config_time_warning( + PytestConfigWarning(warning_text), + stacklevel=3, + ) + + def _warn_about_skipped_plugins(self) -> None: + for module_name, msg in self.pluginmanager.skipped_plugins: + self.issue_config_time_warning( + PytestConfigWarning(f"skipped plugin {module_name!r}: {msg}"), + stacklevel=2, + ) + + +def _assertion_supported() -> bool: + try: + assert False + except AssertionError: + return True + else: + return False # type: ignore[unreachable] + + +def create_terminal_writer( + config: Config, file: Optional[TextIO] = None +) -> TerminalWriter: + """Create a TerminalWriter instance configured according to the options + in the config object. + + Every code which requires a TerminalWriter object and has access to a + config object should use this function. + """ + tw = TerminalWriter(file=file) + + if config.option.color == "yes": + tw.hasmarkup = True + elif config.option.color == "no": + tw.hasmarkup = False + + if config.option.code_highlight == "yes": + tw.code_highlight = True + elif config.option.code_highlight == "no": + tw.code_highlight = False + + return tw + + +def _strtobool(val: str) -> bool: + """Convert a string representation of truth to True or False. + + True values are 'y', 'yes', 't', 'true', 'on', and '1'; false values + are 'n', 'no', 'f', 'false', 'off', and '0'. Raises ValueError if + 'val' is anything else. + + .. note:: Copied from distutils.util. + """ + val = val.lower() + if val in ("y", "yes", "t", "true", "on", "1"): + return True + elif val in ("n", "no", "f", "false", "off", "0"): + return False + else: + raise ValueError(f"invalid truth value {val!r}") + + +@lru_cache(maxsize=50) +def parse_warning_filter( + arg: str, *, escape: bool +) -> Tuple[str, str, Type[Warning], str, int]: + """Parse a warnings filter string. + + This is copied from warnings._setoption with the following changes: + + * Does not apply the filter. + * Escaping is optional. + * Raises UsageError so we get nice error messages on failure. + """ + __tracebackhide__ = True + error_template = dedent( + f"""\ + while parsing the following warning configuration: + + {arg} + + This error occurred: + + {{error}} + """ + ) + + parts = arg.split(":") + if len(parts) > 5: + doc_url = ( + "https://docs.python.org/3/library/warnings.html#describing-warning-filters" + ) + error = dedent( + f"""\ + Too many fields ({len(parts)}), expected at most 5 separated by colons: + + action:message:category:module:line + + For more information please consult: {doc_url} + """ + ) + raise UsageError(error_template.format(error=error)) + + while len(parts) < 5: + parts.append("") + action_, message, category_, module, lineno_ = (s.strip() for s in parts) + try: + action: str = warnings._getaction(action_) # type: ignore[attr-defined] + except warnings._OptionError as e: + raise UsageError(error_template.format(error=str(e))) + try: + category: Type[Warning] = _resolve_warning_category(category_) + except Exception: + exc_info = ExceptionInfo.from_current() + exception_text = exc_info.getrepr(style="native") + raise UsageError(error_template.format(error=exception_text)) + if message and escape: + message = re.escape(message) + if module and escape: + module = re.escape(module) + r"\Z" + if lineno_: + try: + lineno = int(lineno_) + if lineno < 0: + raise ValueError("number is negative") + except ValueError as e: + raise UsageError( + error_template.format(error=f"invalid lineno {lineno_!r}: {e}") + ) + else: + lineno = 0 + return action, message, category, module, lineno + + +def _resolve_warning_category(category: str) -> Type[Warning]: + """ + Copied from warnings._getcategory, but changed so it lets exceptions (specially ImportErrors) + propagate so we can get access to their tracebacks (#9218). + """ + __tracebackhide__ = True + if not category: + return Warning + + if "." not in category: + import builtins as m + + klass = category + else: + module, _, klass = category.rpartition(".") + m = __import__(module, None, None, [klass]) + cat = getattr(m, klass) + if not issubclass(cat, Warning): + raise UsageError(f"{cat} is not a Warning subclass") + return cast(Type[Warning], cat) + + +def apply_warning_filters( + config_filters: Iterable[str], cmdline_filters: Iterable[str] +) -> None: + """Applies pytest-configured filters to the warnings module""" + # Filters should have this precedence: cmdline options, config. + # Filters should be applied in the inverse order of precedence. + for arg in config_filters: + warnings.filterwarnings(*parse_warning_filter(arg, escape=False)) + + for arg in cmdline_filters: + warnings.filterwarnings(*parse_warning_filter(arg, escape=True)) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/argparsing.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/argparsing.py new file mode 100644 index 0000000000..b0bb3f168f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/argparsing.py @@ -0,0 +1,535 @@ +import argparse +import os +import sys +import warnings +from gettext import gettext +from typing import Any +from typing import Callable +from typing import cast +from typing import Dict +from typing import List +from typing import Mapping +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import TYPE_CHECKING +from typing import Union + +import _pytest._io +from _pytest.compat import final +from _pytest.config.exceptions import UsageError +from _pytest.deprecated import ARGUMENT_PERCENT_DEFAULT +from _pytest.deprecated import ARGUMENT_TYPE_STR +from _pytest.deprecated import ARGUMENT_TYPE_STR_CHOICE +from _pytest.deprecated import check_ispytest + +if TYPE_CHECKING: + from typing import NoReturn + from typing_extensions import Literal + +FILE_OR_DIR = "file_or_dir" + + +@final +class Parser: + """Parser for command line arguments and ini-file values. + + :ivar extra_info: Dict of generic param -> value to display in case + there's an error processing the command line arguments. + """ + + prog: Optional[str] = None + + def __init__( + self, + usage: Optional[str] = None, + processopt: Optional[Callable[["Argument"], None]] = None, + *, + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + self._anonymous = OptionGroup("custom options", parser=self, _ispytest=True) + self._groups: List[OptionGroup] = [] + self._processopt = processopt + self._usage = usage + self._inidict: Dict[str, Tuple[str, Optional[str], Any]] = {} + self._ininames: List[str] = [] + self.extra_info: Dict[str, Any] = {} + + def processoption(self, option: "Argument") -> None: + if self._processopt: + if option.dest: + self._processopt(option) + + def getgroup( + self, name: str, description: str = "", after: Optional[str] = None + ) -> "OptionGroup": + """Get (or create) a named option Group. + + :name: Name of the option group. + :description: Long description for --help output. + :after: Name of another group, used for ordering --help output. + + The returned group object has an ``addoption`` method with the same + signature as :func:`parser.addoption ` but + will be shown in the respective group in the output of + ``pytest. --help``. + """ + for group in self._groups: + if group.name == name: + return group + group = OptionGroup(name, description, parser=self, _ispytest=True) + i = 0 + for i, grp in enumerate(self._groups): + if grp.name == after: + break + self._groups.insert(i + 1, group) + return group + + def addoption(self, *opts: str, **attrs: Any) -> None: + """Register a command line option. + + :opts: Option names, can be short or long options. + :attrs: Same attributes which the ``add_argument()`` function of the + `argparse library `_ + accepts. + + After command line parsing, options are available on the pytest config + object via ``config.option.NAME`` where ``NAME`` is usually set + by passing a ``dest`` attribute, for example + ``addoption("--long", dest="NAME", ...)``. + """ + self._anonymous.addoption(*opts, **attrs) + + def parse( + self, + args: Sequence[Union[str, "os.PathLike[str]"]], + namespace: Optional[argparse.Namespace] = None, + ) -> argparse.Namespace: + from _pytest._argcomplete import try_argcomplete + + self.optparser = self._getparser() + try_argcomplete(self.optparser) + strargs = [os.fspath(x) for x in args] + return self.optparser.parse_args(strargs, namespace=namespace) + + def _getparser(self) -> "MyOptionParser": + from _pytest._argcomplete import filescompleter + + optparser = MyOptionParser(self, self.extra_info, prog=self.prog) + groups = self._groups + [self._anonymous] + for group in groups: + if group.options: + desc = group.description or group.name + arggroup = optparser.add_argument_group(desc) + for option in group.options: + n = option.names() + a = option.attrs() + arggroup.add_argument(*n, **a) + file_or_dir_arg = optparser.add_argument(FILE_OR_DIR, nargs="*") + # bash like autocompletion for dirs (appending '/') + # Type ignored because typeshed doesn't know about argcomplete. + file_or_dir_arg.completer = filescompleter # type: ignore + return optparser + + def parse_setoption( + self, + args: Sequence[Union[str, "os.PathLike[str]"]], + option: argparse.Namespace, + namespace: Optional[argparse.Namespace] = None, + ) -> List[str]: + parsedoption = self.parse(args, namespace=namespace) + for name, value in parsedoption.__dict__.items(): + setattr(option, name, value) + return cast(List[str], getattr(parsedoption, FILE_OR_DIR)) + + def parse_known_args( + self, + args: Sequence[Union[str, "os.PathLike[str]"]], + namespace: Optional[argparse.Namespace] = None, + ) -> argparse.Namespace: + """Parse and return a namespace object with known arguments at this point.""" + return self.parse_known_and_unknown_args(args, namespace=namespace)[0] + + def parse_known_and_unknown_args( + self, + args: Sequence[Union[str, "os.PathLike[str]"]], + namespace: Optional[argparse.Namespace] = None, + ) -> Tuple[argparse.Namespace, List[str]]: + """Parse and return a namespace object with known arguments, and + the remaining arguments unknown at this point.""" + optparser = self._getparser() + strargs = [os.fspath(x) for x in args] + return optparser.parse_known_args(strargs, namespace=namespace) + + def addini( + self, + name: str, + help: str, + type: Optional[ + "Literal['string', 'paths', 'pathlist', 'args', 'linelist', 'bool']" + ] = None, + default=None, + ) -> None: + """Register an ini-file option. + + :name: + Name of the ini-variable. + :type: + Type of the variable. Can be: + + * ``string``: a string + * ``bool``: a boolean + * ``args``: a list of strings, separated as in a shell + * ``linelist``: a list of strings, separated by line breaks + * ``paths``: a list of :class:`pathlib.Path`, separated as in a shell + * ``pathlist``: a list of ``py.path``, separated as in a shell + + .. versionadded:: 7.0 + The ``paths`` variable type. + + Defaults to ``string`` if ``None`` or not passed. + :default: + Default value if no ini-file option exists but is queried. + + The value of ini-variables can be retrieved via a call to + :py:func:`config.getini(name) `. + """ + assert type in (None, "string", "paths", "pathlist", "args", "linelist", "bool") + self._inidict[name] = (help, type, default) + self._ininames.append(name) + + +class ArgumentError(Exception): + """Raised if an Argument instance is created with invalid or + inconsistent arguments.""" + + def __init__(self, msg: str, option: Union["Argument", str]) -> None: + self.msg = msg + self.option_id = str(option) + + def __str__(self) -> str: + if self.option_id: + return f"option {self.option_id}: {self.msg}" + else: + return self.msg + + +class Argument: + """Class that mimics the necessary behaviour of optparse.Option. + + It's currently a least effort implementation and ignoring choices + and integer prefixes. + + https://docs.python.org/3/library/optparse.html#optparse-standard-option-types + """ + + _typ_map = {"int": int, "string": str, "float": float, "complex": complex} + + def __init__(self, *names: str, **attrs: Any) -> None: + """Store parms in private vars for use in add_argument.""" + self._attrs = attrs + self._short_opts: List[str] = [] + self._long_opts: List[str] = [] + if "%default" in (attrs.get("help") or ""): + warnings.warn(ARGUMENT_PERCENT_DEFAULT, stacklevel=3) + try: + typ = attrs["type"] + except KeyError: + pass + else: + # This might raise a keyerror as well, don't want to catch that. + if isinstance(typ, str): + if typ == "choice": + warnings.warn( + ARGUMENT_TYPE_STR_CHOICE.format(typ=typ, names=names), + stacklevel=4, + ) + # argparse expects a type here take it from + # the type of the first element + attrs["type"] = type(attrs["choices"][0]) + else: + warnings.warn( + ARGUMENT_TYPE_STR.format(typ=typ, names=names), stacklevel=4 + ) + attrs["type"] = Argument._typ_map[typ] + # Used in test_parseopt -> test_parse_defaultgetter. + self.type = attrs["type"] + else: + self.type = typ + try: + # Attribute existence is tested in Config._processopt. + self.default = attrs["default"] + except KeyError: + pass + self._set_opt_strings(names) + dest: Optional[str] = attrs.get("dest") + if dest: + self.dest = dest + elif self._long_opts: + self.dest = self._long_opts[0][2:].replace("-", "_") + else: + try: + self.dest = self._short_opts[0][1:] + except IndexError as e: + self.dest = "???" # Needed for the error repr. + raise ArgumentError("need a long or short option", self) from e + + def names(self) -> List[str]: + return self._short_opts + self._long_opts + + def attrs(self) -> Mapping[str, Any]: + # Update any attributes set by processopt. + attrs = "default dest help".split() + attrs.append(self.dest) + for attr in attrs: + try: + self._attrs[attr] = getattr(self, attr) + except AttributeError: + pass + if self._attrs.get("help"): + a = self._attrs["help"] + a = a.replace("%default", "%(default)s") + # a = a.replace('%prog', '%(prog)s') + self._attrs["help"] = a + return self._attrs + + def _set_opt_strings(self, opts: Sequence[str]) -> None: + """Directly from optparse. + + Might not be necessary as this is passed to argparse later on. + """ + for opt in opts: + if len(opt) < 2: + raise ArgumentError( + "invalid option string %r: " + "must be at least two characters long" % opt, + self, + ) + elif len(opt) == 2: + if not (opt[0] == "-" and opt[1] != "-"): + raise ArgumentError( + "invalid short option string %r: " + "must be of the form -x, (x any non-dash char)" % opt, + self, + ) + self._short_opts.append(opt) + else: + if not (opt[0:2] == "--" and opt[2] != "-"): + raise ArgumentError( + "invalid long option string %r: " + "must start with --, followed by non-dash" % opt, + self, + ) + self._long_opts.append(opt) + + def __repr__(self) -> str: + args: List[str] = [] + if self._short_opts: + args += ["_short_opts: " + repr(self._short_opts)] + if self._long_opts: + args += ["_long_opts: " + repr(self._long_opts)] + args += ["dest: " + repr(self.dest)] + if hasattr(self, "type"): + args += ["type: " + repr(self.type)] + if hasattr(self, "default"): + args += ["default: " + repr(self.default)] + return "Argument({})".format(", ".join(args)) + + +class OptionGroup: + """A group of options shown in its own section.""" + + def __init__( + self, + name: str, + description: str = "", + parser: Optional[Parser] = None, + *, + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + self.name = name + self.description = description + self.options: List[Argument] = [] + self.parser = parser + + def addoption(self, *optnames: str, **attrs: Any) -> None: + """Add an option to this group. + + If a shortened version of a long option is specified, it will + be suppressed in the help. ``addoption('--twowords', '--two-words')`` + results in help showing ``--two-words`` only, but ``--twowords`` gets + accepted **and** the automatic destination is in ``args.twowords``. + """ + conflict = set(optnames).intersection( + name for opt in self.options for name in opt.names() + ) + if conflict: + raise ValueError("option names %s already added" % conflict) + option = Argument(*optnames, **attrs) + self._addoption_instance(option, shortupper=False) + + def _addoption(self, *optnames: str, **attrs: Any) -> None: + option = Argument(*optnames, **attrs) + self._addoption_instance(option, shortupper=True) + + def _addoption_instance(self, option: "Argument", shortupper: bool = False) -> None: + if not shortupper: + for opt in option._short_opts: + if opt[0] == "-" and opt[1].islower(): + raise ValueError("lowercase shortoptions reserved") + if self.parser: + self.parser.processoption(option) + self.options.append(option) + + +class MyOptionParser(argparse.ArgumentParser): + def __init__( + self, + parser: Parser, + extra_info: Optional[Dict[str, Any]] = None, + prog: Optional[str] = None, + ) -> None: + self._parser = parser + super().__init__( + prog=prog, + usage=parser._usage, + add_help=False, + formatter_class=DropShorterLongHelpFormatter, + allow_abbrev=False, + ) + # extra_info is a dict of (param -> value) to display if there's + # an usage error to provide more contextual information to the user. + self.extra_info = extra_info if extra_info else {} + + def error(self, message: str) -> "NoReturn": + """Transform argparse error message into UsageError.""" + msg = f"{self.prog}: error: {message}" + + if hasattr(self._parser, "_config_source_hint"): + # Type ignored because the attribute is set dynamically. + msg = f"{msg} ({self._parser._config_source_hint})" # type: ignore + + raise UsageError(self.format_usage() + msg) + + # Type ignored because typeshed has a very complex type in the superclass. + def parse_args( # type: ignore + self, + args: Optional[Sequence[str]] = None, + namespace: Optional[argparse.Namespace] = None, + ) -> argparse.Namespace: + """Allow splitting of positional arguments.""" + parsed, unrecognized = self.parse_known_args(args, namespace) + if unrecognized: + for arg in unrecognized: + if arg and arg[0] == "-": + lines = ["unrecognized arguments: %s" % (" ".join(unrecognized))] + for k, v in sorted(self.extra_info.items()): + lines.append(f" {k}: {v}") + self.error("\n".join(lines)) + getattr(parsed, FILE_OR_DIR).extend(unrecognized) + return parsed + + if sys.version_info[:2] < (3, 9): # pragma: no cover + # Backport of https://github.com/python/cpython/pull/14316 so we can + # disable long --argument abbreviations without breaking short flags. + def _parse_optional( + self, arg_string: str + ) -> Optional[Tuple[Optional[argparse.Action], str, Optional[str]]]: + if not arg_string: + return None + if not arg_string[0] in self.prefix_chars: + return None + if arg_string in self._option_string_actions: + action = self._option_string_actions[arg_string] + return action, arg_string, None + if len(arg_string) == 1: + return None + if "=" in arg_string: + option_string, explicit_arg = arg_string.split("=", 1) + if option_string in self._option_string_actions: + action = self._option_string_actions[option_string] + return action, option_string, explicit_arg + if self.allow_abbrev or not arg_string.startswith("--"): + option_tuples = self._get_option_tuples(arg_string) + if len(option_tuples) > 1: + msg = gettext( + "ambiguous option: %(option)s could match %(matches)s" + ) + options = ", ".join(option for _, option, _ in option_tuples) + self.error(msg % {"option": arg_string, "matches": options}) + elif len(option_tuples) == 1: + (option_tuple,) = option_tuples + return option_tuple + if self._negative_number_matcher.match(arg_string): + if not self._has_negative_number_optionals: + return None + if " " in arg_string: + return None + return None, arg_string, None + + +class DropShorterLongHelpFormatter(argparse.HelpFormatter): + """Shorten help for long options that differ only in extra hyphens. + + - Collapse **long** options that are the same except for extra hyphens. + - Shortcut if there are only two options and one of them is a short one. + - Cache result on the action object as this is called at least 2 times. + """ + + def __init__(self, *args: Any, **kwargs: Any) -> None: + # Use more accurate terminal width. + if "width" not in kwargs: + kwargs["width"] = _pytest._io.get_terminal_width() + super().__init__(*args, **kwargs) + + def _format_action_invocation(self, action: argparse.Action) -> str: + orgstr = super()._format_action_invocation(action) + if orgstr and orgstr[0] != "-": # only optional arguments + return orgstr + res: Optional[str] = getattr(action, "_formatted_action_invocation", None) + if res: + return res + options = orgstr.split(", ") + if len(options) == 2 and (len(options[0]) == 2 or len(options[1]) == 2): + # a shortcut for '-h, --help' or '--abc', '-a' + action._formatted_action_invocation = orgstr # type: ignore + return orgstr + return_list = [] + short_long: Dict[str, str] = {} + for option in options: + if len(option) == 2 or option[2] == " ": + continue + if not option.startswith("--"): + raise ArgumentError( + 'long optional argument without "--": [%s]' % (option), option + ) + xxoption = option[2:] + shortened = xxoption.replace("-", "") + if shortened not in short_long or len(short_long[shortened]) < len( + xxoption + ): + short_long[shortened] = xxoption + # now short_long has been filled out to the longest with dashes + # **and** we keep the right option ordering from add_argument + for option in options: + if len(option) == 2 or option[2] == " ": + return_list.append(option) + if option[2:] == short_long.get(option.replace("-", "")): + return_list.append(option.replace(" ", "=", 1)) + formatted_action_invocation = ", ".join(return_list) + action._formatted_action_invocation = formatted_action_invocation # type: ignore + return formatted_action_invocation + + def _split_lines(self, text, width): + """Wrap lines after splitting on original newlines. + + This allows to have explicit line breaks in the help text. + """ + import textwrap + + lines = [] + for line in text.splitlines(): + lines.extend(textwrap.wrap(line.strip(), width)) + return lines diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/compat.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/compat.py new file mode 100644 index 0000000000..ba267d2150 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/compat.py @@ -0,0 +1,71 @@ +import functools +import warnings +from pathlib import Path +from typing import Optional + +from ..compat import LEGACY_PATH +from ..compat import legacy_path +from ..deprecated import HOOK_LEGACY_PATH_ARG +from _pytest.nodes import _check_path + +# hookname: (Path, LEGACY_PATH) +imply_paths_hooks = { + "pytest_ignore_collect": ("collection_path", "path"), + "pytest_collect_file": ("file_path", "path"), + "pytest_pycollect_makemodule": ("module_path", "path"), + "pytest_report_header": ("start_path", "startdir"), + "pytest_report_collectionfinish": ("start_path", "startdir"), +} + + +class PathAwareHookProxy: + """ + this helper wraps around hook callers + until pluggy supports fixingcalls, this one will do + + it currently doesn't return full hook caller proxies for fixed hooks, + this may have to be changed later depending on bugs + """ + + def __init__(self, hook_caller): + self.__hook_caller = hook_caller + + def __dir__(self): + return dir(self.__hook_caller) + + def __getattr__(self, key, _wraps=functools.wraps): + hook = getattr(self.__hook_caller, key) + if key not in imply_paths_hooks: + self.__dict__[key] = hook + return hook + else: + path_var, fspath_var = imply_paths_hooks[key] + + @_wraps(hook) + def fixed_hook(**kw): + + path_value: Optional[Path] = kw.pop(path_var, None) + fspath_value: Optional[LEGACY_PATH] = kw.pop(fspath_var, None) + if fspath_value is not None: + warnings.warn( + HOOK_LEGACY_PATH_ARG.format( + pylib_path_arg=fspath_var, pathlib_path_arg=path_var + ), + stacklevel=2, + ) + if path_value is not None: + if fspath_value is not None: + _check_path(path_value, fspath_value) + else: + fspath_value = legacy_path(path_value) + else: + assert fspath_value is not None + path_value = Path(fspath_value) + + kw[path_var] = path_value + kw[fspath_var] = fspath_value + return hook(**kw) + + fixed_hook.__name__ = key + self.__dict__[key] = fixed_hook + return fixed_hook diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/exceptions.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/exceptions.py new file mode 100644 index 0000000000..4f1320e758 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/exceptions.py @@ -0,0 +1,11 @@ +from _pytest.compat import final + + +@final +class UsageError(Exception): + """Error in pytest usage or invocation.""" + + +class PrintHelp(Exception): + """Raised when pytest should print its help to skip the rest of the + argument parsing and validation.""" diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/findpaths.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/findpaths.py new file mode 100644 index 0000000000..89ade5f23b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/config/findpaths.py @@ -0,0 +1,213 @@ +import os +from pathlib import Path +from typing import Dict +from typing import Iterable +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import TYPE_CHECKING +from typing import Union + +import iniconfig + +from .exceptions import UsageError +from _pytest.outcomes import fail +from _pytest.pathlib import absolutepath +from _pytest.pathlib import commonpath + +if TYPE_CHECKING: + from . import Config + + +def _parse_ini_config(path: Path) -> iniconfig.IniConfig: + """Parse the given generic '.ini' file using legacy IniConfig parser, returning + the parsed object. + + Raise UsageError if the file cannot be parsed. + """ + try: + return iniconfig.IniConfig(str(path)) + except iniconfig.ParseError as exc: + raise UsageError(str(exc)) from exc + + +def load_config_dict_from_file( + filepath: Path, +) -> Optional[Dict[str, Union[str, List[str]]]]: + """Load pytest configuration from the given file path, if supported. + + Return None if the file does not contain valid pytest configuration. + """ + + # Configuration from ini files are obtained from the [pytest] section, if present. + if filepath.suffix == ".ini": + iniconfig = _parse_ini_config(filepath) + + if "pytest" in iniconfig: + return dict(iniconfig["pytest"].items()) + else: + # "pytest.ini" files are always the source of configuration, even if empty. + if filepath.name == "pytest.ini": + return {} + + # '.cfg' files are considered if they contain a "[tool:pytest]" section. + elif filepath.suffix == ".cfg": + iniconfig = _parse_ini_config(filepath) + + if "tool:pytest" in iniconfig.sections: + return dict(iniconfig["tool:pytest"].items()) + elif "pytest" in iniconfig.sections: + # If a setup.cfg contains a "[pytest]" section, we raise a failure to indicate users that + # plain "[pytest]" sections in setup.cfg files is no longer supported (#3086). + fail(CFG_PYTEST_SECTION.format(filename="setup.cfg"), pytrace=False) + + # '.toml' files are considered if they contain a [tool.pytest.ini_options] table. + elif filepath.suffix == ".toml": + import tomli + + toml_text = filepath.read_text(encoding="utf-8") + try: + config = tomli.loads(toml_text) + except tomli.TOMLDecodeError as exc: + raise UsageError(str(exc)) from exc + + result = config.get("tool", {}).get("pytest", {}).get("ini_options", None) + if result is not None: + # TOML supports richer data types than ini files (strings, arrays, floats, ints, etc), + # however we need to convert all scalar values to str for compatibility with the rest + # of the configuration system, which expects strings only. + def make_scalar(v: object) -> Union[str, List[str]]: + return v if isinstance(v, list) else str(v) + + return {k: make_scalar(v) for k, v in result.items()} + + return None + + +def locate_config( + args: Iterable[Path], +) -> Tuple[Optional[Path], Optional[Path], Dict[str, Union[str, List[str]]]]: + """Search in the list of arguments for a valid ini-file for pytest, + and return a tuple of (rootdir, inifile, cfg-dict).""" + config_names = [ + "pytest.ini", + "pyproject.toml", + "tox.ini", + "setup.cfg", + ] + args = [x for x in args if not str(x).startswith("-")] + if not args: + args = [Path.cwd()] + for arg in args: + argpath = absolutepath(arg) + for base in (argpath, *argpath.parents): + for config_name in config_names: + p = base / config_name + if p.is_file(): + ini_config = load_config_dict_from_file(p) + if ini_config is not None: + return base, p, ini_config + return None, None, {} + + +def get_common_ancestor(paths: Iterable[Path]) -> Path: + common_ancestor: Optional[Path] = None + for path in paths: + if not path.exists(): + continue + if common_ancestor is None: + common_ancestor = path + else: + if common_ancestor in path.parents or path == common_ancestor: + continue + elif path in common_ancestor.parents: + common_ancestor = path + else: + shared = commonpath(path, common_ancestor) + if shared is not None: + common_ancestor = shared + if common_ancestor is None: + common_ancestor = Path.cwd() + elif common_ancestor.is_file(): + common_ancestor = common_ancestor.parent + return common_ancestor + + +def get_dirs_from_args(args: Iterable[str]) -> List[Path]: + def is_option(x: str) -> bool: + return x.startswith("-") + + def get_file_part_from_node_id(x: str) -> str: + return x.split("::")[0] + + def get_dir_from_path(path: Path) -> Path: + if path.is_dir(): + return path + return path.parent + + def safe_exists(path: Path) -> bool: + # This can throw on paths that contain characters unrepresentable at the OS level, + # or with invalid syntax on Windows (https://bugs.python.org/issue35306) + try: + return path.exists() + except OSError: + return False + + # These look like paths but may not exist + possible_paths = ( + absolutepath(get_file_part_from_node_id(arg)) + for arg in args + if not is_option(arg) + ) + + return [get_dir_from_path(path) for path in possible_paths if safe_exists(path)] + + +CFG_PYTEST_SECTION = "[pytest] section in {filename} files is no longer supported, change to [tool:pytest] instead." + + +def determine_setup( + inifile: Optional[str], + args: Sequence[str], + rootdir_cmd_arg: Optional[str] = None, + config: Optional["Config"] = None, +) -> Tuple[Path, Optional[Path], Dict[str, Union[str, List[str]]]]: + rootdir = None + dirs = get_dirs_from_args(args) + if inifile: + inipath_ = absolutepath(inifile) + inipath: Optional[Path] = inipath_ + inicfg = load_config_dict_from_file(inipath_) or {} + if rootdir_cmd_arg is None: + rootdir = inipath_.parent + else: + ancestor = get_common_ancestor(dirs) + rootdir, inipath, inicfg = locate_config([ancestor]) + if rootdir is None and rootdir_cmd_arg is None: + for possible_rootdir in (ancestor, *ancestor.parents): + if (possible_rootdir / "setup.py").is_file(): + rootdir = possible_rootdir + break + else: + if dirs != [ancestor]: + rootdir, inipath, inicfg = locate_config(dirs) + if rootdir is None: + if config is not None: + cwd = config.invocation_params.dir + else: + cwd = Path.cwd() + rootdir = get_common_ancestor([cwd, ancestor]) + is_fs_root = os.path.splitdrive(str(rootdir))[1] == "/" + if is_fs_root: + rootdir = ancestor + if rootdir_cmd_arg: + rootdir = absolutepath(os.path.expandvars(rootdir_cmd_arg)) + if not rootdir.is_dir(): + raise UsageError( + "Directory '{}' not found. Check your '--rootdir' option.".format( + rootdir + ) + ) + assert rootdir is not None + return rootdir, inipath, inicfg or {} diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/debugging.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/debugging.py new file mode 100644 index 0000000000..452fb18ac3 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/debugging.py @@ -0,0 +1,388 @@ +"""Interactive debugging with PDB, the Python Debugger.""" +import argparse +import functools +import sys +import types +from typing import Any +from typing import Callable +from typing import Generator +from typing import List +from typing import Optional +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import Union + +from _pytest import outcomes +from _pytest._code import ExceptionInfo +from _pytest.config import Config +from _pytest.config import ConftestImportFailure +from _pytest.config import hookimpl +from _pytest.config import PytestPluginManager +from _pytest.config.argparsing import Parser +from _pytest.config.exceptions import UsageError +from _pytest.nodes import Node +from _pytest.reports import BaseReport + +if TYPE_CHECKING: + from _pytest.capture import CaptureManager + from _pytest.runner import CallInfo + + +def _validate_usepdb_cls(value: str) -> Tuple[str, str]: + """Validate syntax of --pdbcls option.""" + try: + modname, classname = value.split(":") + except ValueError as e: + raise argparse.ArgumentTypeError( + f"{value!r} is not in the format 'modname:classname'" + ) from e + return (modname, classname) + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("general") + group._addoption( + "--pdb", + dest="usepdb", + action="store_true", + help="start the interactive Python debugger on errors or KeyboardInterrupt.", + ) + group._addoption( + "--pdbcls", + dest="usepdb_cls", + metavar="modulename:classname", + type=_validate_usepdb_cls, + help="specify a custom interactive Python debugger for use with --pdb." + "For example: --pdbcls=IPython.terminal.debugger:TerminalPdb", + ) + group._addoption( + "--trace", + dest="trace", + action="store_true", + help="Immediately break when running each test.", + ) + + +def pytest_configure(config: Config) -> None: + import pdb + + if config.getvalue("trace"): + config.pluginmanager.register(PdbTrace(), "pdbtrace") + if config.getvalue("usepdb"): + config.pluginmanager.register(PdbInvoke(), "pdbinvoke") + + pytestPDB._saved.append( + (pdb.set_trace, pytestPDB._pluginmanager, pytestPDB._config) + ) + pdb.set_trace = pytestPDB.set_trace + pytestPDB._pluginmanager = config.pluginmanager + pytestPDB._config = config + + # NOTE: not using pytest_unconfigure, since it might get called although + # pytest_configure was not (if another plugin raises UsageError). + def fin() -> None: + ( + pdb.set_trace, + pytestPDB._pluginmanager, + pytestPDB._config, + ) = pytestPDB._saved.pop() + + config.add_cleanup(fin) + + +class pytestPDB: + """Pseudo PDB that defers to the real pdb.""" + + _pluginmanager: Optional[PytestPluginManager] = None + _config: Optional[Config] = None + _saved: List[ + Tuple[Callable[..., None], Optional[PytestPluginManager], Optional[Config]] + ] = [] + _recursive_debug = 0 + _wrapped_pdb_cls: Optional[Tuple[Type[Any], Type[Any]]] = None + + @classmethod + def _is_capturing(cls, capman: Optional["CaptureManager"]) -> Union[str, bool]: + if capman: + return capman.is_capturing() + return False + + @classmethod + def _import_pdb_cls(cls, capman: Optional["CaptureManager"]): + if not cls._config: + import pdb + + # Happens when using pytest.set_trace outside of a test. + return pdb.Pdb + + usepdb_cls = cls._config.getvalue("usepdb_cls") + + if cls._wrapped_pdb_cls and cls._wrapped_pdb_cls[0] == usepdb_cls: + return cls._wrapped_pdb_cls[1] + + if usepdb_cls: + modname, classname = usepdb_cls + + try: + __import__(modname) + mod = sys.modules[modname] + + # Handle --pdbcls=pdb:pdb.Pdb (useful e.g. with pdbpp). + parts = classname.split(".") + pdb_cls = getattr(mod, parts[0]) + for part in parts[1:]: + pdb_cls = getattr(pdb_cls, part) + except Exception as exc: + value = ":".join((modname, classname)) + raise UsageError( + f"--pdbcls: could not import {value!r}: {exc}" + ) from exc + else: + import pdb + + pdb_cls = pdb.Pdb + + wrapped_cls = cls._get_pdb_wrapper_class(pdb_cls, capman) + cls._wrapped_pdb_cls = (usepdb_cls, wrapped_cls) + return wrapped_cls + + @classmethod + def _get_pdb_wrapper_class(cls, pdb_cls, capman: Optional["CaptureManager"]): + import _pytest.config + + # Type ignored because mypy doesn't support "dynamic" + # inheritance like this. + class PytestPdbWrapper(pdb_cls): # type: ignore[valid-type,misc] + _pytest_capman = capman + _continued = False + + def do_debug(self, arg): + cls._recursive_debug += 1 + ret = super().do_debug(arg) + cls._recursive_debug -= 1 + return ret + + def do_continue(self, arg): + ret = super().do_continue(arg) + if cls._recursive_debug == 0: + assert cls._config is not None + tw = _pytest.config.create_terminal_writer(cls._config) + tw.line() + + capman = self._pytest_capman + capturing = pytestPDB._is_capturing(capman) + if capturing: + if capturing == "global": + tw.sep(">", "PDB continue (IO-capturing resumed)") + else: + tw.sep( + ">", + "PDB continue (IO-capturing resumed for %s)" + % capturing, + ) + assert capman is not None + capman.resume() + else: + tw.sep(">", "PDB continue") + assert cls._pluginmanager is not None + cls._pluginmanager.hook.pytest_leave_pdb(config=cls._config, pdb=self) + self._continued = True + return ret + + do_c = do_cont = do_continue + + def do_quit(self, arg): + """Raise Exit outcome when quit command is used in pdb. + + This is a bit of a hack - it would be better if BdbQuit + could be handled, but this would require to wrap the + whole pytest run, and adjust the report etc. + """ + ret = super().do_quit(arg) + + if cls._recursive_debug == 0: + outcomes.exit("Quitting debugger") + + return ret + + do_q = do_quit + do_exit = do_quit + + def setup(self, f, tb): + """Suspend on setup(). + + Needed after do_continue resumed, and entering another + breakpoint again. + """ + ret = super().setup(f, tb) + if not ret and self._continued: + # pdb.setup() returns True if the command wants to exit + # from the interaction: do not suspend capturing then. + if self._pytest_capman: + self._pytest_capman.suspend_global_capture(in_=True) + return ret + + def get_stack(self, f, t): + stack, i = super().get_stack(f, t) + if f is None: + # Find last non-hidden frame. + i = max(0, len(stack) - 1) + while i and stack[i][0].f_locals.get("__tracebackhide__", False): + i -= 1 + return stack, i + + return PytestPdbWrapper + + @classmethod + def _init_pdb(cls, method, *args, **kwargs): + """Initialize PDB debugging, dropping any IO capturing.""" + import _pytest.config + + if cls._pluginmanager is None: + capman: Optional[CaptureManager] = None + else: + capman = cls._pluginmanager.getplugin("capturemanager") + if capman: + capman.suspend(in_=True) + + if cls._config: + tw = _pytest.config.create_terminal_writer(cls._config) + tw.line() + + if cls._recursive_debug == 0: + # Handle header similar to pdb.set_trace in py37+. + header = kwargs.pop("header", None) + if header is not None: + tw.sep(">", header) + else: + capturing = cls._is_capturing(capman) + if capturing == "global": + tw.sep(">", f"PDB {method} (IO-capturing turned off)") + elif capturing: + tw.sep( + ">", + "PDB %s (IO-capturing turned off for %s)" + % (method, capturing), + ) + else: + tw.sep(">", f"PDB {method}") + + _pdb = cls._import_pdb_cls(capman)(**kwargs) + + if cls._pluginmanager: + cls._pluginmanager.hook.pytest_enter_pdb(config=cls._config, pdb=_pdb) + return _pdb + + @classmethod + def set_trace(cls, *args, **kwargs) -> None: + """Invoke debugging via ``Pdb.set_trace``, dropping any IO capturing.""" + frame = sys._getframe().f_back + _pdb = cls._init_pdb("set_trace", *args, **kwargs) + _pdb.set_trace(frame) + + +class PdbInvoke: + def pytest_exception_interact( + self, node: Node, call: "CallInfo[Any]", report: BaseReport + ) -> None: + capman = node.config.pluginmanager.getplugin("capturemanager") + if capman: + capman.suspend_global_capture(in_=True) + out, err = capman.read_global_capture() + sys.stdout.write(out) + sys.stdout.write(err) + assert call.excinfo is not None + _enter_pdb(node, call.excinfo, report) + + def pytest_internalerror(self, excinfo: ExceptionInfo[BaseException]) -> None: + tb = _postmortem_traceback(excinfo) + post_mortem(tb) + + +class PdbTrace: + @hookimpl(hookwrapper=True) + def pytest_pyfunc_call(self, pyfuncitem) -> Generator[None, None, None]: + wrap_pytest_function_for_tracing(pyfuncitem) + yield + + +def wrap_pytest_function_for_tracing(pyfuncitem): + """Change the Python function object of the given Function item by a + wrapper which actually enters pdb before calling the python function + itself, effectively leaving the user in the pdb prompt in the first + statement of the function.""" + _pdb = pytestPDB._init_pdb("runcall") + testfunction = pyfuncitem.obj + + # we can't just return `partial(pdb.runcall, testfunction)` because (on + # python < 3.7.4) runcall's first param is `func`, which means we'd get + # an exception if one of the kwargs to testfunction was called `func`. + @functools.wraps(testfunction) + def wrapper(*args, **kwargs): + func = functools.partial(testfunction, *args, **kwargs) + _pdb.runcall(func) + + pyfuncitem.obj = wrapper + + +def maybe_wrap_pytest_function_for_tracing(pyfuncitem): + """Wrap the given pytestfunct item for tracing support if --trace was given in + the command line.""" + if pyfuncitem.config.getvalue("trace"): + wrap_pytest_function_for_tracing(pyfuncitem) + + +def _enter_pdb( + node: Node, excinfo: ExceptionInfo[BaseException], rep: BaseReport +) -> BaseReport: + # XXX we re-use the TerminalReporter's terminalwriter + # because this seems to avoid some encoding related troubles + # for not completely clear reasons. + tw = node.config.pluginmanager.getplugin("terminalreporter")._tw + tw.line() + + showcapture = node.config.option.showcapture + + for sectionname, content in ( + ("stdout", rep.capstdout), + ("stderr", rep.capstderr), + ("log", rep.caplog), + ): + if showcapture in (sectionname, "all") and content: + tw.sep(">", "captured " + sectionname) + if content[-1:] == "\n": + content = content[:-1] + tw.line(content) + + tw.sep(">", "traceback") + rep.toterminal(tw) + tw.sep(">", "entering PDB") + tb = _postmortem_traceback(excinfo) + rep._pdbshown = True # type: ignore[attr-defined] + post_mortem(tb) + return rep + + +def _postmortem_traceback(excinfo: ExceptionInfo[BaseException]) -> types.TracebackType: + from doctest import UnexpectedException + + if isinstance(excinfo.value, UnexpectedException): + # A doctest.UnexpectedException is not useful for post_mortem. + # Use the underlying exception instead: + return excinfo.value.exc_info[2] + elif isinstance(excinfo.value, ConftestImportFailure): + # A config.ConftestImportFailure is not useful for post_mortem. + # Use the underlying exception instead: + return excinfo.value.excinfo[2] + else: + assert excinfo._excinfo is not None + return excinfo._excinfo[2] + + +def post_mortem(t: types.TracebackType) -> None: + p = pytestPDB._init_pdb("post_mortem") + p.reset() + p.interaction(None, t) + if p.quitting: + outcomes.exit("Quitting debugger") diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/deprecated.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/deprecated.py new file mode 100644 index 0000000000..5248927113 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/deprecated.py @@ -0,0 +1,155 @@ +"""Deprecation messages and bits of code used elsewhere in the codebase that +is planned to be removed in the next pytest release. + +Keeping it in a central location makes it easy to track what is deprecated and should +be removed when the time comes. + +All constants defined in this module should be either instances of +:class:`PytestWarning`, or :class:`UnformattedWarning` +in case of warnings which need to format their messages. +""" +from warnings import warn + +from _pytest.warning_types import PytestDeprecationWarning +from _pytest.warning_types import PytestRemovedIn7Warning +from _pytest.warning_types import PytestRemovedIn8Warning +from _pytest.warning_types import UnformattedWarning + +# set of plugins which have been integrated into the core; we use this list to ignore +# them during registration to avoid conflicts +DEPRECATED_EXTERNAL_PLUGINS = { + "pytest_catchlog", + "pytest_capturelog", + "pytest_faulthandler", +} + + +FILLFUNCARGS = UnformattedWarning( + PytestRemovedIn7Warning, + "{name} is deprecated, use " + "function._request._fillfixtures() instead if you cannot avoid reaching into internals.", +) + +PYTEST_COLLECT_MODULE = UnformattedWarning( + PytestRemovedIn7Warning, + "pytest.collect.{name} was moved to pytest.{name}\n" + "Please update to the new name.", +) + +# This can be* removed pytest 8, but it's harmless and common, so no rush to remove. +# * If you're in the future: "could have been". +YIELD_FIXTURE = PytestDeprecationWarning( + "@pytest.yield_fixture is deprecated.\n" + "Use @pytest.fixture instead; they are the same." +) + +MINUS_K_DASH = PytestRemovedIn7Warning( + "The `-k '-expr'` syntax to -k is deprecated.\nUse `-k 'not expr'` instead." +) + +MINUS_K_COLON = PytestRemovedIn7Warning( + "The `-k 'expr:'` syntax to -k is deprecated.\n" + "Please open an issue if you use this and want a replacement." +) + +WARNING_CAPTURED_HOOK = PytestRemovedIn7Warning( + "The pytest_warning_captured is deprecated and will be removed in a future release.\n" + "Please use pytest_warning_recorded instead." +) + +WARNING_CMDLINE_PREPARSE_HOOK = PytestRemovedIn8Warning( + "The pytest_cmdline_preparse hook is deprecated and will be removed in a future release. \n" + "Please use pytest_load_initial_conftests hook instead." +) + +FSCOLLECTOR_GETHOOKPROXY_ISINITPATH = PytestRemovedIn8Warning( + "The gethookproxy() and isinitpath() methods of FSCollector and Package are deprecated; " + "use self.session.gethookproxy() and self.session.isinitpath() instead. " +) + +STRICT_OPTION = PytestRemovedIn8Warning( + "The --strict option is deprecated, use --strict-markers instead." +) + +# This deprecation is never really meant to be removed. +PRIVATE = PytestDeprecationWarning("A private pytest class or function was used.") + +UNITTEST_SKIP_DURING_COLLECTION = PytestRemovedIn8Warning( + "Raising unittest.SkipTest to skip tests during collection is deprecated. " + "Use pytest.skip() instead." +) + +ARGUMENT_PERCENT_DEFAULT = PytestRemovedIn8Warning( + 'pytest now uses argparse. "%default" should be changed to "%(default)s"', +) + +ARGUMENT_TYPE_STR_CHOICE = UnformattedWarning( + PytestRemovedIn8Warning, + "`type` argument to addoption() is the string {typ!r}." + " For choices this is optional and can be omitted, " + " but when supplied should be a type (for example `str` or `int`)." + " (options: {names})", +) + +ARGUMENT_TYPE_STR = UnformattedWarning( + PytestRemovedIn8Warning, + "`type` argument to addoption() is the string {typ!r}, " + " but when supplied should be a type (for example `str` or `int`)." + " (options: {names})", +) + + +HOOK_LEGACY_PATH_ARG = UnformattedWarning( + PytestRemovedIn8Warning, + "The ({pylib_path_arg}: py.path.local) argument is deprecated, please use ({pathlib_path_arg}: pathlib.Path)\n" + "see https://docs.pytest.org/en/latest/deprecations.html" + "#py-path-local-arguments-for-hooks-replaced-with-pathlib-path", +) + +NODE_CTOR_FSPATH_ARG = UnformattedWarning( + PytestRemovedIn8Warning, + "The (fspath: py.path.local) argument to {node_type_name} is deprecated. " + "Please use the (path: pathlib.Path) argument instead.\n" + "See https://docs.pytest.org/en/latest/deprecations.html" + "#fspath-argument-for-node-constructors-replaced-with-pathlib-path", +) + +WARNS_NONE_ARG = PytestRemovedIn8Warning( + "Passing None has been deprecated.\n" + "See https://docs.pytest.org/en/latest/how-to/capture-warnings.html" + "#additional-use-cases-of-warnings-in-tests" + " for alternatives in common use cases." +) + +KEYWORD_MSG_ARG = UnformattedWarning( + PytestRemovedIn8Warning, + "pytest.{func}(msg=...) is now deprecated, use pytest.{func}(reason=...) instead", +) + +INSTANCE_COLLECTOR = PytestRemovedIn8Warning( + "The pytest.Instance collector type is deprecated and is no longer used. " + "See https://docs.pytest.org/en/latest/deprecations.html#the-pytest-instance-collector", +) + +# You want to make some `__init__` or function "private". +# +# def my_private_function(some, args): +# ... +# +# Do this: +# +# def my_private_function(some, args, *, _ispytest: bool = False): +# check_ispytest(_ispytest) +# ... +# +# Change all internal/allowed calls to +# +# my_private_function(some, args, _ispytest=True) +# +# All other calls will get the default _ispytest=False and trigger +# the warning (possibly error in the future). + + +def check_ispytest(ispytest: bool) -> None: + if not ispytest: + warn(PRIVATE, stacklevel=3) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/doctest.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/doctest.py new file mode 100644 index 0000000000..0784f431b8 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/doctest.py @@ -0,0 +1,734 @@ +"""Discover and run doctests in modules and test files.""" +import bdb +import inspect +import os +import platform +import sys +import traceback +import types +import warnings +from contextlib import contextmanager +from pathlib import Path +from typing import Any +from typing import Callable +from typing import Dict +from typing import Generator +from typing import Iterable +from typing import List +from typing import Optional +from typing import Pattern +from typing import Sequence +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import Union + +import pytest +from _pytest import outcomes +from _pytest._code.code import ExceptionInfo +from _pytest._code.code import ReprFileLocation +from _pytest._code.code import TerminalRepr +from _pytest._io import TerminalWriter +from _pytest.compat import safe_getattr +from _pytest.config import Config +from _pytest.config.argparsing import Parser +from _pytest.fixtures import FixtureRequest +from _pytest.nodes import Collector +from _pytest.outcomes import OutcomeException +from _pytest.pathlib import fnmatch_ex +from _pytest.pathlib import import_path +from _pytest.python_api import approx +from _pytest.warning_types import PytestWarning + +if TYPE_CHECKING: + import doctest + +DOCTEST_REPORT_CHOICE_NONE = "none" +DOCTEST_REPORT_CHOICE_CDIFF = "cdiff" +DOCTEST_REPORT_CHOICE_NDIFF = "ndiff" +DOCTEST_REPORT_CHOICE_UDIFF = "udiff" +DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE = "only_first_failure" + +DOCTEST_REPORT_CHOICES = ( + DOCTEST_REPORT_CHOICE_NONE, + DOCTEST_REPORT_CHOICE_CDIFF, + DOCTEST_REPORT_CHOICE_NDIFF, + DOCTEST_REPORT_CHOICE_UDIFF, + DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE, +) + +# Lazy definition of runner class +RUNNER_CLASS = None +# Lazy definition of output checker class +CHECKER_CLASS: Optional[Type["doctest.OutputChecker"]] = None + + +def pytest_addoption(parser: Parser) -> None: + parser.addini( + "doctest_optionflags", + "option flags for doctests", + type="args", + default=["ELLIPSIS"], + ) + parser.addini( + "doctest_encoding", "encoding used for doctest files", default="utf-8" + ) + group = parser.getgroup("collect") + group.addoption( + "--doctest-modules", + action="store_true", + default=False, + help="run doctests in all .py modules", + dest="doctestmodules", + ) + group.addoption( + "--doctest-report", + type=str.lower, + default="udiff", + help="choose another output format for diffs on doctest failure", + choices=DOCTEST_REPORT_CHOICES, + dest="doctestreport", + ) + group.addoption( + "--doctest-glob", + action="append", + default=[], + metavar="pat", + help="doctests file matching pattern, default: test*.txt", + dest="doctestglob", + ) + group.addoption( + "--doctest-ignore-import-errors", + action="store_true", + default=False, + help="ignore doctest ImportErrors", + dest="doctest_ignore_import_errors", + ) + group.addoption( + "--doctest-continue-on-failure", + action="store_true", + default=False, + help="for a given doctest, continue to run after the first failure", + dest="doctest_continue_on_failure", + ) + + +def pytest_unconfigure() -> None: + global RUNNER_CLASS + + RUNNER_CLASS = None + + +def pytest_collect_file( + file_path: Path, + parent: Collector, +) -> Optional[Union["DoctestModule", "DoctestTextfile"]]: + config = parent.config + if file_path.suffix == ".py": + if config.option.doctestmodules and not any( + (_is_setup_py(file_path), _is_main_py(file_path)) + ): + mod: DoctestModule = DoctestModule.from_parent(parent, path=file_path) + return mod + elif _is_doctest(config, file_path, parent): + txt: DoctestTextfile = DoctestTextfile.from_parent(parent, path=file_path) + return txt + return None + + +def _is_setup_py(path: Path) -> bool: + if path.name != "setup.py": + return False + contents = path.read_bytes() + return b"setuptools" in contents or b"distutils" in contents + + +def _is_doctest(config: Config, path: Path, parent: Collector) -> bool: + if path.suffix in (".txt", ".rst") and parent.session.isinitpath(path): + return True + globs = config.getoption("doctestglob") or ["test*.txt"] + return any(fnmatch_ex(glob, path) for glob in globs) + + +def _is_main_py(path: Path) -> bool: + return path.name == "__main__.py" + + +class ReprFailDoctest(TerminalRepr): + def __init__( + self, reprlocation_lines: Sequence[Tuple[ReprFileLocation, Sequence[str]]] + ) -> None: + self.reprlocation_lines = reprlocation_lines + + def toterminal(self, tw: TerminalWriter) -> None: + for reprlocation, lines in self.reprlocation_lines: + for line in lines: + tw.line(line) + reprlocation.toterminal(tw) + + +class MultipleDoctestFailures(Exception): + def __init__(self, failures: Sequence["doctest.DocTestFailure"]) -> None: + super().__init__() + self.failures = failures + + +def _init_runner_class() -> Type["doctest.DocTestRunner"]: + import doctest + + class PytestDoctestRunner(doctest.DebugRunner): + """Runner to collect failures. + + Note that the out variable in this case is a list instead of a + stdout-like object. + """ + + def __init__( + self, + checker: Optional["doctest.OutputChecker"] = None, + verbose: Optional[bool] = None, + optionflags: int = 0, + continue_on_failure: bool = True, + ) -> None: + super().__init__(checker=checker, verbose=verbose, optionflags=optionflags) + self.continue_on_failure = continue_on_failure + + def report_failure( + self, + out, + test: "doctest.DocTest", + example: "doctest.Example", + got: str, + ) -> None: + failure = doctest.DocTestFailure(test, example, got) + if self.continue_on_failure: + out.append(failure) + else: + raise failure + + def report_unexpected_exception( + self, + out, + test: "doctest.DocTest", + example: "doctest.Example", + exc_info: Tuple[Type[BaseException], BaseException, types.TracebackType], + ) -> None: + if isinstance(exc_info[1], OutcomeException): + raise exc_info[1] + if isinstance(exc_info[1], bdb.BdbQuit): + outcomes.exit("Quitting debugger") + failure = doctest.UnexpectedException(test, example, exc_info) + if self.continue_on_failure: + out.append(failure) + else: + raise failure + + return PytestDoctestRunner + + +def _get_runner( + checker: Optional["doctest.OutputChecker"] = None, + verbose: Optional[bool] = None, + optionflags: int = 0, + continue_on_failure: bool = True, +) -> "doctest.DocTestRunner": + # We need this in order to do a lazy import on doctest + global RUNNER_CLASS + if RUNNER_CLASS is None: + RUNNER_CLASS = _init_runner_class() + # Type ignored because the continue_on_failure argument is only defined on + # PytestDoctestRunner, which is lazily defined so can't be used as a type. + return RUNNER_CLASS( # type: ignore + checker=checker, + verbose=verbose, + optionflags=optionflags, + continue_on_failure=continue_on_failure, + ) + + +class DoctestItem(pytest.Item): + def __init__( + self, + name: str, + parent: "Union[DoctestTextfile, DoctestModule]", + runner: Optional["doctest.DocTestRunner"] = None, + dtest: Optional["doctest.DocTest"] = None, + ) -> None: + super().__init__(name, parent) + self.runner = runner + self.dtest = dtest + self.obj = None + self.fixture_request: Optional[FixtureRequest] = None + + @classmethod + def from_parent( # type: ignore + cls, + parent: "Union[DoctestTextfile, DoctestModule]", + *, + name: str, + runner: "doctest.DocTestRunner", + dtest: "doctest.DocTest", + ): + # incompatible signature due to imposed limits on subclass + """The public named constructor.""" + return super().from_parent(name=name, parent=parent, runner=runner, dtest=dtest) + + def setup(self) -> None: + if self.dtest is not None: + self.fixture_request = _setup_fixtures(self) + globs = dict(getfixture=self.fixture_request.getfixturevalue) + for name, value in self.fixture_request.getfixturevalue( + "doctest_namespace" + ).items(): + globs[name] = value + self.dtest.globs.update(globs) + + def runtest(self) -> None: + assert self.dtest is not None + assert self.runner is not None + _check_all_skipped(self.dtest) + self._disable_output_capturing_for_darwin() + failures: List["doctest.DocTestFailure"] = [] + # Type ignored because we change the type of `out` from what + # doctest expects. + self.runner.run(self.dtest, out=failures) # type: ignore[arg-type] + if failures: + raise MultipleDoctestFailures(failures) + + def _disable_output_capturing_for_darwin(self) -> None: + """Disable output capturing. Otherwise, stdout is lost to doctest (#985).""" + if platform.system() != "Darwin": + return + capman = self.config.pluginmanager.getplugin("capturemanager") + if capman: + capman.suspend_global_capture(in_=True) + out, err = capman.read_global_capture() + sys.stdout.write(out) + sys.stderr.write(err) + + # TODO: Type ignored -- breaks Liskov Substitution. + def repr_failure( # type: ignore[override] + self, + excinfo: ExceptionInfo[BaseException], + ) -> Union[str, TerminalRepr]: + import doctest + + failures: Optional[ + Sequence[Union[doctest.DocTestFailure, doctest.UnexpectedException]] + ] = None + if isinstance( + excinfo.value, (doctest.DocTestFailure, doctest.UnexpectedException) + ): + failures = [excinfo.value] + elif isinstance(excinfo.value, MultipleDoctestFailures): + failures = excinfo.value.failures + + if failures is None: + return super().repr_failure(excinfo) + + reprlocation_lines = [] + for failure in failures: + example = failure.example + test = failure.test + filename = test.filename + if test.lineno is None: + lineno = None + else: + lineno = test.lineno + example.lineno + 1 + message = type(failure).__name__ + # TODO: ReprFileLocation doesn't expect a None lineno. + reprlocation = ReprFileLocation(filename, lineno, message) # type: ignore[arg-type] + checker = _get_checker() + report_choice = _get_report_choice(self.config.getoption("doctestreport")) + if lineno is not None: + assert failure.test.docstring is not None + lines = failure.test.docstring.splitlines(False) + # add line numbers to the left of the error message + assert test.lineno is not None + lines = [ + "%03d %s" % (i + test.lineno + 1, x) for (i, x) in enumerate(lines) + ] + # trim docstring error lines to 10 + lines = lines[max(example.lineno - 9, 0) : example.lineno + 1] + else: + lines = [ + "EXAMPLE LOCATION UNKNOWN, not showing all tests of that example" + ] + indent = ">>>" + for line in example.source.splitlines(): + lines.append(f"??? {indent} {line}") + indent = "..." + if isinstance(failure, doctest.DocTestFailure): + lines += checker.output_difference( + example, failure.got, report_choice + ).split("\n") + else: + inner_excinfo = ExceptionInfo.from_exc_info(failure.exc_info) + lines += ["UNEXPECTED EXCEPTION: %s" % repr(inner_excinfo.value)] + lines += [ + x.strip("\n") for x in traceback.format_exception(*failure.exc_info) + ] + reprlocation_lines.append((reprlocation, lines)) + return ReprFailDoctest(reprlocation_lines) + + def reportinfo(self) -> Tuple[Union["os.PathLike[str]", str], Optional[int], str]: + assert self.dtest is not None + return self.path, self.dtest.lineno, "[doctest] %s" % self.name + + +def _get_flag_lookup() -> Dict[str, int]: + import doctest + + return dict( + DONT_ACCEPT_TRUE_FOR_1=doctest.DONT_ACCEPT_TRUE_FOR_1, + DONT_ACCEPT_BLANKLINE=doctest.DONT_ACCEPT_BLANKLINE, + NORMALIZE_WHITESPACE=doctest.NORMALIZE_WHITESPACE, + ELLIPSIS=doctest.ELLIPSIS, + IGNORE_EXCEPTION_DETAIL=doctest.IGNORE_EXCEPTION_DETAIL, + COMPARISON_FLAGS=doctest.COMPARISON_FLAGS, + ALLOW_UNICODE=_get_allow_unicode_flag(), + ALLOW_BYTES=_get_allow_bytes_flag(), + NUMBER=_get_number_flag(), + ) + + +def get_optionflags(parent): + optionflags_str = parent.config.getini("doctest_optionflags") + flag_lookup_table = _get_flag_lookup() + flag_acc = 0 + for flag in optionflags_str: + flag_acc |= flag_lookup_table[flag] + return flag_acc + + +def _get_continue_on_failure(config): + continue_on_failure = config.getvalue("doctest_continue_on_failure") + if continue_on_failure: + # We need to turn off this if we use pdb since we should stop at + # the first failure. + if config.getvalue("usepdb"): + continue_on_failure = False + return continue_on_failure + + +class DoctestTextfile(pytest.Module): + obj = None + + def collect(self) -> Iterable[DoctestItem]: + import doctest + + # Inspired by doctest.testfile; ideally we would use it directly, + # but it doesn't support passing a custom checker. + encoding = self.config.getini("doctest_encoding") + text = self.path.read_text(encoding) + filename = str(self.path) + name = self.path.name + globs = {"__name__": "__main__"} + + optionflags = get_optionflags(self) + + runner = _get_runner( + verbose=False, + optionflags=optionflags, + checker=_get_checker(), + continue_on_failure=_get_continue_on_failure(self.config), + ) + + parser = doctest.DocTestParser() + test = parser.get_doctest(text, globs, name, filename, 0) + if test.examples: + yield DoctestItem.from_parent( + self, name=test.name, runner=runner, dtest=test + ) + + +def _check_all_skipped(test: "doctest.DocTest") -> None: + """Raise pytest.skip() if all examples in the given DocTest have the SKIP + option set.""" + import doctest + + all_skipped = all(x.options.get(doctest.SKIP, False) for x in test.examples) + if all_skipped: + pytest.skip("all tests skipped by +SKIP option") + + +def _is_mocked(obj: object) -> bool: + """Return if an object is possibly a mock object by checking the + existence of a highly improbable attribute.""" + return ( + safe_getattr(obj, "pytest_mock_example_attribute_that_shouldnt_exist", None) + is not None + ) + + +@contextmanager +def _patch_unwrap_mock_aware() -> Generator[None, None, None]: + """Context manager which replaces ``inspect.unwrap`` with a version + that's aware of mock objects and doesn't recurse into them.""" + real_unwrap = inspect.unwrap + + def _mock_aware_unwrap( + func: Callable[..., Any], *, stop: Optional[Callable[[Any], Any]] = None + ) -> Any: + try: + if stop is None or stop is _is_mocked: + return real_unwrap(func, stop=_is_mocked) + _stop = stop + return real_unwrap(func, stop=lambda obj: _is_mocked(obj) or _stop(func)) + except Exception as e: + warnings.warn( + "Got %r when unwrapping %r. This is usually caused " + "by a violation of Python's object protocol; see e.g. " + "https://github.com/pytest-dev/pytest/issues/5080" % (e, func), + PytestWarning, + ) + raise + + inspect.unwrap = _mock_aware_unwrap + try: + yield + finally: + inspect.unwrap = real_unwrap + + +class DoctestModule(pytest.Module): + def collect(self) -> Iterable[DoctestItem]: + import doctest + + class MockAwareDocTestFinder(doctest.DocTestFinder): + """A hackish doctest finder that overrides stdlib internals to fix a stdlib bug. + + https://github.com/pytest-dev/pytest/issues/3456 + https://bugs.python.org/issue25532 + """ + + def _find_lineno(self, obj, source_lines): + """Doctest code does not take into account `@property`, this + is a hackish way to fix it. https://bugs.python.org/issue17446 + + Wrapped Doctests will need to be unwrapped so the correct + line number is returned. This will be reported upstream. #8796 + """ + if isinstance(obj, property): + obj = getattr(obj, "fget", obj) + + if hasattr(obj, "__wrapped__"): + # Get the main obj in case of it being wrapped + obj = inspect.unwrap(obj) + + # Type ignored because this is a private function. + return super()._find_lineno( # type:ignore[misc] + obj, + source_lines, + ) + + def _find( + self, tests, obj, name, module, source_lines, globs, seen + ) -> None: + if _is_mocked(obj): + return + with _patch_unwrap_mock_aware(): + + # Type ignored because this is a private function. + super()._find( # type:ignore[misc] + tests, obj, name, module, source_lines, globs, seen + ) + + if self.path.name == "conftest.py": + module = self.config.pluginmanager._importconftest( + self.path, + self.config.getoption("importmode"), + rootpath=self.config.rootpath, + ) + else: + try: + module = import_path(self.path, root=self.config.rootpath) + except ImportError: + if self.config.getvalue("doctest_ignore_import_errors"): + pytest.skip("unable to import module %r" % self.path) + else: + raise + # Uses internal doctest module parsing mechanism. + finder = MockAwareDocTestFinder() + optionflags = get_optionflags(self) + runner = _get_runner( + verbose=False, + optionflags=optionflags, + checker=_get_checker(), + continue_on_failure=_get_continue_on_failure(self.config), + ) + + for test in finder.find(module, module.__name__): + if test.examples: # skip empty doctests + yield DoctestItem.from_parent( + self, name=test.name, runner=runner, dtest=test + ) + + +def _setup_fixtures(doctest_item: DoctestItem) -> FixtureRequest: + """Used by DoctestTextfile and DoctestItem to setup fixture information.""" + + def func() -> None: + pass + + doctest_item.funcargs = {} # type: ignore[attr-defined] + fm = doctest_item.session._fixturemanager + doctest_item._fixtureinfo = fm.getfixtureinfo( # type: ignore[attr-defined] + node=doctest_item, func=func, cls=None, funcargs=False + ) + fixture_request = FixtureRequest(doctest_item, _ispytest=True) + fixture_request._fillfixtures() + return fixture_request + + +def _init_checker_class() -> Type["doctest.OutputChecker"]: + import doctest + import re + + class LiteralsOutputChecker(doctest.OutputChecker): + # Based on doctest_nose_plugin.py from the nltk project + # (https://github.com/nltk/nltk) and on the "numtest" doctest extension + # by Sebastien Boisgerault (https://github.com/boisgera/numtest). + + _unicode_literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE) + _bytes_literal_re = re.compile(r"(\W|^)[bB]([rR]?[\'\"])", re.UNICODE) + _number_re = re.compile( + r""" + (?P + (?P + (?P [+-]?\d*)\.(?P\d+) + | + (?P [+-]?\d+)\. + ) + (?: + [Ee] + (?P [+-]?\d+) + )? + | + (?P [+-]?\d+) + (?: + [Ee] + (?P [+-]?\d+) + ) + ) + """, + re.VERBOSE, + ) + + def check_output(self, want: str, got: str, optionflags: int) -> bool: + if super().check_output(want, got, optionflags): + return True + + allow_unicode = optionflags & _get_allow_unicode_flag() + allow_bytes = optionflags & _get_allow_bytes_flag() + allow_number = optionflags & _get_number_flag() + + if not allow_unicode and not allow_bytes and not allow_number: + return False + + def remove_prefixes(regex: Pattern[str], txt: str) -> str: + return re.sub(regex, r"\1\2", txt) + + if allow_unicode: + want = remove_prefixes(self._unicode_literal_re, want) + got = remove_prefixes(self._unicode_literal_re, got) + + if allow_bytes: + want = remove_prefixes(self._bytes_literal_re, want) + got = remove_prefixes(self._bytes_literal_re, got) + + if allow_number: + got = self._remove_unwanted_precision(want, got) + + return super().check_output(want, got, optionflags) + + def _remove_unwanted_precision(self, want: str, got: str) -> str: + wants = list(self._number_re.finditer(want)) + gots = list(self._number_re.finditer(got)) + if len(wants) != len(gots): + return got + offset = 0 + for w, g in zip(wants, gots): + fraction: Optional[str] = w.group("fraction") + exponent: Optional[str] = w.group("exponent1") + if exponent is None: + exponent = w.group("exponent2") + precision = 0 if fraction is None else len(fraction) + if exponent is not None: + precision -= int(exponent) + if float(w.group()) == approx(float(g.group()), abs=10 ** -precision): + # They're close enough. Replace the text we actually + # got with the text we want, so that it will match when we + # check the string literally. + got = ( + got[: g.start() + offset] + w.group() + got[g.end() + offset :] + ) + offset += w.end() - w.start() - (g.end() - g.start()) + return got + + return LiteralsOutputChecker + + +def _get_checker() -> "doctest.OutputChecker": + """Return a doctest.OutputChecker subclass that supports some + additional options: + + * ALLOW_UNICODE and ALLOW_BYTES options to ignore u'' and b'' + prefixes (respectively) in string literals. Useful when the same + doctest should run in Python 2 and Python 3. + + * NUMBER to ignore floating-point differences smaller than the + precision of the literal number in the doctest. + + An inner class is used to avoid importing "doctest" at the module + level. + """ + global CHECKER_CLASS + if CHECKER_CLASS is None: + CHECKER_CLASS = _init_checker_class() + return CHECKER_CLASS() + + +def _get_allow_unicode_flag() -> int: + """Register and return the ALLOW_UNICODE flag.""" + import doctest + + return doctest.register_optionflag("ALLOW_UNICODE") + + +def _get_allow_bytes_flag() -> int: + """Register and return the ALLOW_BYTES flag.""" + import doctest + + return doctest.register_optionflag("ALLOW_BYTES") + + +def _get_number_flag() -> int: + """Register and return the NUMBER flag.""" + import doctest + + return doctest.register_optionflag("NUMBER") + + +def _get_report_choice(key: str) -> int: + """Return the actual `doctest` module flag value. + + We want to do it as late as possible to avoid importing `doctest` and all + its dependencies when parsing options, as it adds overhead and breaks tests. + """ + import doctest + + return { + DOCTEST_REPORT_CHOICE_UDIFF: doctest.REPORT_UDIFF, + DOCTEST_REPORT_CHOICE_CDIFF: doctest.REPORT_CDIFF, + DOCTEST_REPORT_CHOICE_NDIFF: doctest.REPORT_NDIFF, + DOCTEST_REPORT_CHOICE_ONLY_FIRST_FAILURE: doctest.REPORT_ONLY_FIRST_FAILURE, + DOCTEST_REPORT_CHOICE_NONE: 0, + }[key] + + +@pytest.fixture(scope="session") +def doctest_namespace() -> Dict[str, Any]: + """Fixture that returns a :py:class:`dict` that will be injected into the + namespace of doctests.""" + return dict() diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/faulthandler.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/faulthandler.py new file mode 100644 index 0000000000..aaee307ff2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/faulthandler.py @@ -0,0 +1,97 @@ +import io +import os +import sys +from typing import Generator +from typing import TextIO + +import pytest +from _pytest.config import Config +from _pytest.config.argparsing import Parser +from _pytest.nodes import Item +from _pytest.stash import StashKey + + +fault_handler_stderr_key = StashKey[TextIO]() +fault_handler_originally_enabled_key = StashKey[bool]() + + +def pytest_addoption(parser: Parser) -> None: + help = ( + "Dump the traceback of all threads if a test takes " + "more than TIMEOUT seconds to finish." + ) + parser.addini("faulthandler_timeout", help, default=0.0) + + +def pytest_configure(config: Config) -> None: + import faulthandler + + stderr_fd_copy = os.dup(get_stderr_fileno()) + config.stash[fault_handler_stderr_key] = open(stderr_fd_copy, "w") + config.stash[fault_handler_originally_enabled_key] = faulthandler.is_enabled() + faulthandler.enable(file=config.stash[fault_handler_stderr_key]) + + +def pytest_unconfigure(config: Config) -> None: + import faulthandler + + faulthandler.disable() + # Close the dup file installed during pytest_configure. + if fault_handler_stderr_key in config.stash: + config.stash[fault_handler_stderr_key].close() + del config.stash[fault_handler_stderr_key] + if config.stash.get(fault_handler_originally_enabled_key, False): + # Re-enable the faulthandler if it was originally enabled. + faulthandler.enable(file=get_stderr_fileno()) + + +def get_stderr_fileno() -> int: + try: + fileno = sys.stderr.fileno() + # The Twisted Logger will return an invalid file descriptor since it is not backed + # by an FD. So, let's also forward this to the same code path as with pytest-xdist. + if fileno == -1: + raise AttributeError() + return fileno + except (AttributeError, io.UnsupportedOperation): + # pytest-xdist monkeypatches sys.stderr with an object that is not an actual file. + # https://docs.python.org/3/library/faulthandler.html#issue-with-file-descriptors + # This is potentially dangerous, but the best we can do. + return sys.__stderr__.fileno() + + +def get_timeout_config_value(config: Config) -> float: + return float(config.getini("faulthandler_timeout") or 0.0) + + +@pytest.hookimpl(hookwrapper=True, trylast=True) +def pytest_runtest_protocol(item: Item) -> Generator[None, None, None]: + timeout = get_timeout_config_value(item.config) + stderr = item.config.stash[fault_handler_stderr_key] + if timeout > 0 and stderr is not None: + import faulthandler + + faulthandler.dump_traceback_later(timeout, file=stderr) + try: + yield + finally: + faulthandler.cancel_dump_traceback_later() + else: + yield + + +@pytest.hookimpl(tryfirst=True) +def pytest_enter_pdb() -> None: + """Cancel any traceback dumping due to timeout before entering pdb.""" + import faulthandler + + faulthandler.cancel_dump_traceback_later() + + +@pytest.hookimpl(tryfirst=True) +def pytest_exception_interact() -> None: + """Cancel any traceback dumping due to an interactive exception being + raised.""" + import faulthandler + + faulthandler.cancel_dump_traceback_later() diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/fixtures.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/fixtures.py new file mode 100644 index 0000000000..fddff931c5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/fixtures.py @@ -0,0 +1,1686 @@ +import functools +import inspect +import os +import sys +import warnings +from collections import defaultdict +from collections import deque +from contextlib import suppress +from pathlib import Path +from types import TracebackType +from typing import Any +from typing import Callable +from typing import cast +from typing import Dict +from typing import Generator +from typing import Generic +from typing import Iterable +from typing import Iterator +from typing import List +from typing import MutableMapping +from typing import Optional +from typing import overload +from typing import Sequence +from typing import Set +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import TypeVar +from typing import Union + +import attr + +import _pytest +from _pytest import nodes +from _pytest._code import getfslineno +from _pytest._code.code import FormattedExcinfo +from _pytest._code.code import TerminalRepr +from _pytest._io import TerminalWriter +from _pytest.compat import _format_args +from _pytest.compat import _PytestWrapper +from _pytest.compat import assert_never +from _pytest.compat import final +from _pytest.compat import get_real_func +from _pytest.compat import get_real_method +from _pytest.compat import getfuncargnames +from _pytest.compat import getimfunc +from _pytest.compat import getlocation +from _pytest.compat import is_generator +from _pytest.compat import NOTSET +from _pytest.compat import safe_getattr +from _pytest.config import _PluggyPlugin +from _pytest.config import Config +from _pytest.config.argparsing import Parser +from _pytest.deprecated import check_ispytest +from _pytest.deprecated import FILLFUNCARGS +from _pytest.deprecated import YIELD_FIXTURE +from _pytest.mark import Mark +from _pytest.mark import ParameterSet +from _pytest.mark.structures import MarkDecorator +from _pytest.outcomes import fail +from _pytest.outcomes import TEST_OUTCOME +from _pytest.pathlib import absolutepath +from _pytest.pathlib import bestrelpath +from _pytest.scope import HIGH_SCOPES +from _pytest.scope import Scope +from _pytest.stash import StashKey + + +if TYPE_CHECKING: + from typing import Deque + from typing import NoReturn + + from _pytest.scope import _ScopeName + from _pytest.main import Session + from _pytest.python import CallSpec2 + from _pytest.python import Function + from _pytest.python import Metafunc + + +# The value of the fixture -- return/yield of the fixture function (type variable). +FixtureValue = TypeVar("FixtureValue") +# The type of the fixture function (type variable). +FixtureFunction = TypeVar("FixtureFunction", bound=Callable[..., object]) +# The type of a fixture function (type alias generic in fixture value). +_FixtureFunc = Union[ + Callable[..., FixtureValue], Callable[..., Generator[FixtureValue, None, None]] +] +# The type of FixtureDef.cached_result (type alias generic in fixture value). +_FixtureCachedResult = Union[ + Tuple[ + # The result. + FixtureValue, + # Cache key. + object, + None, + ], + Tuple[ + None, + # Cache key. + object, + # Exc info if raised. + Tuple[Type[BaseException], BaseException, TracebackType], + ], +] + + +@attr.s(frozen=True, auto_attribs=True) +class PseudoFixtureDef(Generic[FixtureValue]): + cached_result: "_FixtureCachedResult[FixtureValue]" + _scope: Scope + + +def pytest_sessionstart(session: "Session") -> None: + session._fixturemanager = FixtureManager(session) + + +def get_scope_package(node, fixturedef: "FixtureDef[object]"): + import pytest + + cls = pytest.Package + current = node + fixture_package_name = "{}/{}".format(fixturedef.baseid, "__init__.py") + while current and ( + type(current) is not cls or fixture_package_name != current.nodeid + ): + current = current.parent + if current is None: + return node.session + return current + + +def get_scope_node( + node: nodes.Node, scope: Scope +) -> Optional[Union[nodes.Item, nodes.Collector]]: + import _pytest.python + + if scope is Scope.Function: + return node.getparent(nodes.Item) + elif scope is Scope.Class: + return node.getparent(_pytest.python.Class) + elif scope is Scope.Module: + return node.getparent(_pytest.python.Module) + elif scope is Scope.Package: + return node.getparent(_pytest.python.Package) + elif scope is Scope.Session: + return node.getparent(_pytest.main.Session) + else: + assert_never(scope) + + +# Used for storing artificial fixturedefs for direct parametrization. +name2pseudofixturedef_key = StashKey[Dict[str, "FixtureDef[Any]"]]() + + +def add_funcarg_pseudo_fixture_def( + collector: nodes.Collector, metafunc: "Metafunc", fixturemanager: "FixtureManager" +) -> None: + # This function will transform all collected calls to functions + # if they use direct funcargs (i.e. direct parametrization) + # because we want later test execution to be able to rely on + # an existing FixtureDef structure for all arguments. + # XXX we can probably avoid this algorithm if we modify CallSpec2 + # to directly care for creating the fixturedefs within its methods. + if not metafunc._calls[0].funcargs: + # This function call does not have direct parametrization. + return + # Collect funcargs of all callspecs into a list of values. + arg2params: Dict[str, List[object]] = {} + arg2scope: Dict[str, Scope] = {} + for callspec in metafunc._calls: + for argname, argvalue in callspec.funcargs.items(): + assert argname not in callspec.params + callspec.params[argname] = argvalue + arg2params_list = arg2params.setdefault(argname, []) + callspec.indices[argname] = len(arg2params_list) + arg2params_list.append(argvalue) + if argname not in arg2scope: + scope = callspec._arg2scope.get(argname, Scope.Function) + arg2scope[argname] = scope + callspec.funcargs.clear() + + # Register artificial FixtureDef's so that later at test execution + # time we can rely on a proper FixtureDef to exist for fixture setup. + arg2fixturedefs = metafunc._arg2fixturedefs + for argname, valuelist in arg2params.items(): + # If we have a scope that is higher than function, we need + # to make sure we only ever create an according fixturedef on + # a per-scope basis. We thus store and cache the fixturedef on the + # node related to the scope. + scope = arg2scope[argname] + node = None + if scope is not Scope.Function: + node = get_scope_node(collector, scope) + if node is None: + assert scope is Scope.Class and isinstance( + collector, _pytest.python.Module + ) + # Use module-level collector for class-scope (for now). + node = collector + if node is None: + name2pseudofixturedef = None + else: + default: Dict[str, FixtureDef[Any]] = {} + name2pseudofixturedef = node.stash.setdefault( + name2pseudofixturedef_key, default + ) + if name2pseudofixturedef is not None and argname in name2pseudofixturedef: + arg2fixturedefs[argname] = [name2pseudofixturedef[argname]] + else: + fixturedef = FixtureDef( + fixturemanager=fixturemanager, + baseid="", + argname=argname, + func=get_direct_param_fixture_func, + scope=arg2scope[argname], + params=valuelist, + unittest=False, + ids=None, + ) + arg2fixturedefs[argname] = [fixturedef] + if name2pseudofixturedef is not None: + name2pseudofixturedef[argname] = fixturedef + + +def getfixturemarker(obj: object) -> Optional["FixtureFunctionMarker"]: + """Return fixturemarker or None if it doesn't exist or raised + exceptions.""" + try: + fixturemarker: Optional[FixtureFunctionMarker] = getattr( + obj, "_pytestfixturefunction", None + ) + except TEST_OUTCOME: + # some objects raise errors like request (from flask import request) + # we don't expect them to be fixture functions + return None + return fixturemarker + + +# Parametrized fixture key, helper alias for code below. +_Key = Tuple[object, ...] + + +def get_parametrized_fixture_keys(item: nodes.Item, scope: Scope) -> Iterator[_Key]: + """Return list of keys for all parametrized arguments which match + the specified scope.""" + assert scope is not Scope.Function + try: + callspec = item.callspec # type: ignore[attr-defined] + except AttributeError: + pass + else: + cs: CallSpec2 = callspec + # cs.indices.items() is random order of argnames. Need to + # sort this so that different calls to + # get_parametrized_fixture_keys will be deterministic. + for argname, param_index in sorted(cs.indices.items()): + if cs._arg2scope[argname] != scope: + continue + if scope is Scope.Session: + key: _Key = (argname, param_index) + elif scope is Scope.Package: + key = (argname, param_index, item.path.parent) + elif scope is Scope.Module: + key = (argname, param_index, item.path) + elif scope is Scope.Class: + item_cls = item.cls # type: ignore[attr-defined] + key = (argname, param_index, item.path, item_cls) + else: + assert_never(scope) + yield key + + +# Algorithm for sorting on a per-parametrized resource setup basis. +# It is called for Session scope first and performs sorting +# down to the lower scopes such as to minimize number of "high scope" +# setups and teardowns. + + +def reorder_items(items: Sequence[nodes.Item]) -> List[nodes.Item]: + argkeys_cache: Dict[Scope, Dict[nodes.Item, Dict[_Key, None]]] = {} + items_by_argkey: Dict[Scope, Dict[_Key, Deque[nodes.Item]]] = {} + for scope in HIGH_SCOPES: + d: Dict[nodes.Item, Dict[_Key, None]] = {} + argkeys_cache[scope] = d + item_d: Dict[_Key, Deque[nodes.Item]] = defaultdict(deque) + items_by_argkey[scope] = item_d + for item in items: + keys = dict.fromkeys(get_parametrized_fixture_keys(item, scope), None) + if keys: + d[item] = keys + for key in keys: + item_d[key].append(item) + items_dict = dict.fromkeys(items, None) + return list( + reorder_items_atscope(items_dict, argkeys_cache, items_by_argkey, Scope.Session) + ) + + +def fix_cache_order( + item: nodes.Item, + argkeys_cache: Dict[Scope, Dict[nodes.Item, Dict[_Key, None]]], + items_by_argkey: Dict[Scope, Dict[_Key, "Deque[nodes.Item]"]], +) -> None: + for scope in HIGH_SCOPES: + for key in argkeys_cache[scope].get(item, []): + items_by_argkey[scope][key].appendleft(item) + + +def reorder_items_atscope( + items: Dict[nodes.Item, None], + argkeys_cache: Dict[Scope, Dict[nodes.Item, Dict[_Key, None]]], + items_by_argkey: Dict[Scope, Dict[_Key, "Deque[nodes.Item]"]], + scope: Scope, +) -> Dict[nodes.Item, None]: + if scope is Scope.Function or len(items) < 3: + return items + ignore: Set[Optional[_Key]] = set() + items_deque = deque(items) + items_done: Dict[nodes.Item, None] = {} + scoped_items_by_argkey = items_by_argkey[scope] + scoped_argkeys_cache = argkeys_cache[scope] + while items_deque: + no_argkey_group: Dict[nodes.Item, None] = {} + slicing_argkey = None + while items_deque: + item = items_deque.popleft() + if item in items_done or item in no_argkey_group: + continue + argkeys = dict.fromkeys( + (k for k in scoped_argkeys_cache.get(item, []) if k not in ignore), None + ) + if not argkeys: + no_argkey_group[item] = None + else: + slicing_argkey, _ = argkeys.popitem() + # We don't have to remove relevant items from later in the + # deque because they'll just be ignored. + matching_items = [ + i for i in scoped_items_by_argkey[slicing_argkey] if i in items + ] + for i in reversed(matching_items): + fix_cache_order(i, argkeys_cache, items_by_argkey) + items_deque.appendleft(i) + break + if no_argkey_group: + no_argkey_group = reorder_items_atscope( + no_argkey_group, argkeys_cache, items_by_argkey, scope.next_lower() + ) + for item in no_argkey_group: + items_done[item] = None + ignore.add(slicing_argkey) + return items_done + + +def _fillfuncargs(function: "Function") -> None: + """Fill missing fixtures for a test function, old public API (deprecated).""" + warnings.warn(FILLFUNCARGS.format(name="pytest._fillfuncargs()"), stacklevel=2) + _fill_fixtures_impl(function) + + +def fillfixtures(function: "Function") -> None: + """Fill missing fixtures for a test function (deprecated).""" + warnings.warn( + FILLFUNCARGS.format(name="_pytest.fixtures.fillfixtures()"), stacklevel=2 + ) + _fill_fixtures_impl(function) + + +def _fill_fixtures_impl(function: "Function") -> None: + """Internal implementation to fill fixtures on the given function object.""" + try: + request = function._request + except AttributeError: + # XXX this special code path is only expected to execute + # with the oejskit plugin. It uses classes with funcargs + # and we thus have to work a bit to allow this. + fm = function.session._fixturemanager + assert function.parent is not None + fi = fm.getfixtureinfo(function.parent, function.obj, None) + function._fixtureinfo = fi + request = function._request = FixtureRequest(function, _ispytest=True) + fm.session._setupstate.setup(function) + request._fillfixtures() + # Prune out funcargs for jstests. + function.funcargs = {name: function.funcargs[name] for name in fi.argnames} + else: + request._fillfixtures() + + +def get_direct_param_fixture_func(request): + return request.param + + +@attr.s(slots=True, auto_attribs=True) +class FuncFixtureInfo: + # Original function argument names. + argnames: Tuple[str, ...] + # Argnames that function immediately requires. These include argnames + + # fixture names specified via usefixtures and via autouse=True in fixture + # definitions. + initialnames: Tuple[str, ...] + names_closure: List[str] + name2fixturedefs: Dict[str, Sequence["FixtureDef[Any]"]] + + def prune_dependency_tree(self) -> None: + """Recompute names_closure from initialnames and name2fixturedefs. + + Can only reduce names_closure, which means that the new closure will + always be a subset of the old one. The order is preserved. + + This method is needed because direct parametrization may shadow some + of the fixtures that were included in the originally built dependency + tree. In this way the dependency tree can get pruned, and the closure + of argnames may get reduced. + """ + closure: Set[str] = set() + working_set = set(self.initialnames) + while working_set: + argname = working_set.pop() + # Argname may be smth not included in the original names_closure, + # in which case we ignore it. This currently happens with pseudo + # FixtureDefs which wrap 'get_direct_param_fixture_func(request)'. + # So they introduce the new dependency 'request' which might have + # been missing in the original tree (closure). + if argname not in closure and argname in self.names_closure: + closure.add(argname) + if argname in self.name2fixturedefs: + working_set.update(self.name2fixturedefs[argname][-1].argnames) + + self.names_closure[:] = sorted(closure, key=self.names_closure.index) + + +class FixtureRequest: + """A request for a fixture from a test or fixture function. + + A request object gives access to the requesting test context and has + an optional ``param`` attribute in case the fixture is parametrized + indirectly. + """ + + def __init__(self, pyfuncitem, *, _ispytest: bool = False) -> None: + check_ispytest(_ispytest) + self._pyfuncitem = pyfuncitem + #: Fixture for which this request is being performed. + self.fixturename: Optional[str] = None + self._scope = Scope.Function + self._fixture_defs: Dict[str, FixtureDef[Any]] = {} + fixtureinfo: FuncFixtureInfo = pyfuncitem._fixtureinfo + self._arg2fixturedefs = fixtureinfo.name2fixturedefs.copy() + self._arg2index: Dict[str, int] = {} + self._fixturemanager: FixtureManager = pyfuncitem.session._fixturemanager + + @property + def scope(self) -> "_ScopeName": + """Scope string, one of "function", "class", "module", "package", "session".""" + return self._scope.value + + @property + def fixturenames(self) -> List[str]: + """Names of all active fixtures in this request.""" + result = list(self._pyfuncitem._fixtureinfo.names_closure) + result.extend(set(self._fixture_defs).difference(result)) + return result + + @property + def node(self): + """Underlying collection node (depends on current request scope).""" + return self._getscopeitem(self._scope) + + def _getnextfixturedef(self, argname: str) -> "FixtureDef[Any]": + fixturedefs = self._arg2fixturedefs.get(argname, None) + if fixturedefs is None: + # We arrive here because of a dynamic call to + # getfixturevalue(argname) usage which was naturally + # not known at parsing/collection time. + assert self._pyfuncitem.parent is not None + parentid = self._pyfuncitem.parent.nodeid + fixturedefs = self._fixturemanager.getfixturedefs(argname, parentid) + # TODO: Fix this type ignore. Either add assert or adjust types. + # Can this be None here? + self._arg2fixturedefs[argname] = fixturedefs # type: ignore[assignment] + # fixturedefs list is immutable so we maintain a decreasing index. + index = self._arg2index.get(argname, 0) - 1 + if fixturedefs is None or (-index > len(fixturedefs)): + raise FixtureLookupError(argname, self) + self._arg2index[argname] = index + return fixturedefs[index] + + @property + def config(self) -> Config: + """The pytest config object associated with this request.""" + return self._pyfuncitem.config # type: ignore[no-any-return] + + @property + def function(self): + """Test function object if the request has a per-function scope.""" + if self.scope != "function": + raise AttributeError( + f"function not available in {self.scope}-scoped context" + ) + return self._pyfuncitem.obj + + @property + def cls(self): + """Class (can be None) where the test function was collected.""" + if self.scope not in ("class", "function"): + raise AttributeError(f"cls not available in {self.scope}-scoped context") + clscol = self._pyfuncitem.getparent(_pytest.python.Class) + if clscol: + return clscol.obj + + @property + def instance(self): + """Instance (can be None) on which test function was collected.""" + # unittest support hack, see _pytest.unittest.TestCaseFunction. + try: + return self._pyfuncitem._testcase + except AttributeError: + function = getattr(self, "function", None) + return getattr(function, "__self__", None) + + @property + def module(self): + """Python module object where the test function was collected.""" + if self.scope not in ("function", "class", "module"): + raise AttributeError(f"module not available in {self.scope}-scoped context") + return self._pyfuncitem.getparent(_pytest.python.Module).obj + + @property + def path(self) -> Path: + if self.scope not in ("function", "class", "module", "package"): + raise AttributeError(f"path not available in {self.scope}-scoped context") + # TODO: Remove ignore once _pyfuncitem is properly typed. + return self._pyfuncitem.path # type: ignore + + @property + def keywords(self) -> MutableMapping[str, Any]: + """Keywords/markers dictionary for the underlying node.""" + node: nodes.Node = self.node + return node.keywords + + @property + def session(self) -> "Session": + """Pytest session object.""" + return self._pyfuncitem.session # type: ignore[no-any-return] + + def addfinalizer(self, finalizer: Callable[[], object]) -> None: + """Add finalizer/teardown function to be called after the last test + within the requesting test context finished execution.""" + # XXX usually this method is shadowed by fixturedef specific ones. + self._addfinalizer(finalizer, scope=self.scope) + + def _addfinalizer(self, finalizer: Callable[[], object], scope) -> None: + node = self._getscopeitem(scope) + node.addfinalizer(finalizer) + + def applymarker(self, marker: Union[str, MarkDecorator]) -> None: + """Apply a marker to a single test function invocation. + + This method is useful if you don't want to have a keyword/marker + on all function invocations. + + :param marker: + A :class:`pytest.MarkDecorator` object created by a call + to ``pytest.mark.NAME(...)``. + """ + self.node.add_marker(marker) + + def raiseerror(self, msg: Optional[str]) -> "NoReturn": + """Raise a FixtureLookupError with the given message.""" + raise self._fixturemanager.FixtureLookupError(None, self, msg) + + def _fillfixtures(self) -> None: + item = self._pyfuncitem + fixturenames = getattr(item, "fixturenames", self.fixturenames) + for argname in fixturenames: + if argname not in item.funcargs: + item.funcargs[argname] = self.getfixturevalue(argname) + + def getfixturevalue(self, argname: str) -> Any: + """Dynamically run a named fixture function. + + Declaring fixtures via function argument is recommended where possible. + But if you can only decide whether to use another fixture at test + setup time, you may use this function to retrieve it inside a fixture + or test function body. + + :raises pytest.FixtureLookupError: + If the given fixture could not be found. + """ + fixturedef = self._get_active_fixturedef(argname) + assert fixturedef.cached_result is not None + return fixturedef.cached_result[0] + + def _get_active_fixturedef( + self, argname: str + ) -> Union["FixtureDef[object]", PseudoFixtureDef[object]]: + try: + return self._fixture_defs[argname] + except KeyError: + try: + fixturedef = self._getnextfixturedef(argname) + except FixtureLookupError: + if argname == "request": + cached_result = (self, [0], None) + return PseudoFixtureDef(cached_result, Scope.Function) + raise + # Remove indent to prevent the python3 exception + # from leaking into the call. + self._compute_fixture_value(fixturedef) + self._fixture_defs[argname] = fixturedef + return fixturedef + + def _get_fixturestack(self) -> List["FixtureDef[Any]"]: + current = self + values: List[FixtureDef[Any]] = [] + while isinstance(current, SubRequest): + values.append(current._fixturedef) # type: ignore[has-type] + current = current._parent_request + values.reverse() + return values + + def _compute_fixture_value(self, fixturedef: "FixtureDef[object]") -> None: + """Create a SubRequest based on "self" and call the execute method + of the given FixtureDef object. + + This will force the FixtureDef object to throw away any previous + results and compute a new fixture value, which will be stored into + the FixtureDef object itself. + """ + # prepare a subrequest object before calling fixture function + # (latter managed by fixturedef) + argname = fixturedef.argname + funcitem = self._pyfuncitem + scope = fixturedef._scope + try: + param = funcitem.callspec.getparam(argname) + except (AttributeError, ValueError): + param = NOTSET + param_index = 0 + has_params = fixturedef.params is not None + fixtures_not_supported = getattr(funcitem, "nofuncargs", False) + if has_params and fixtures_not_supported: + msg = ( + "{name} does not support fixtures, maybe unittest.TestCase subclass?\n" + "Node id: {nodeid}\n" + "Function type: {typename}" + ).format( + name=funcitem.name, + nodeid=funcitem.nodeid, + typename=type(funcitem).__name__, + ) + fail(msg, pytrace=False) + if has_params: + frame = inspect.stack()[3] + frameinfo = inspect.getframeinfo(frame[0]) + source_path = absolutepath(frameinfo.filename) + source_lineno = frameinfo.lineno + try: + source_path_str = str( + source_path.relative_to(funcitem.config.rootpath) + ) + except ValueError: + source_path_str = str(source_path) + msg = ( + "The requested fixture has no parameter defined for test:\n" + " {}\n\n" + "Requested fixture '{}' defined in:\n{}" + "\n\nRequested here:\n{}:{}".format( + funcitem.nodeid, + fixturedef.argname, + getlocation(fixturedef.func, funcitem.config.rootpath), + source_path_str, + source_lineno, + ) + ) + fail(msg, pytrace=False) + else: + param_index = funcitem.callspec.indices[argname] + # If a parametrize invocation set a scope it will override + # the static scope defined with the fixture function. + with suppress(KeyError): + scope = funcitem.callspec._arg2scope[argname] + + subrequest = SubRequest( + self, scope, param, param_index, fixturedef, _ispytest=True + ) + + # Check if a higher-level scoped fixture accesses a lower level one. + subrequest._check_scope(argname, self._scope, scope) + try: + # Call the fixture function. + fixturedef.execute(request=subrequest) + finally: + self._schedule_finalizers(fixturedef, subrequest) + + def _schedule_finalizers( + self, fixturedef: "FixtureDef[object]", subrequest: "SubRequest" + ) -> None: + # If fixture function failed it might have registered finalizers. + subrequest.node.addfinalizer(lambda: fixturedef.finish(request=subrequest)) + + def _check_scope( + self, + argname: str, + invoking_scope: Scope, + requested_scope: Scope, + ) -> None: + if argname == "request": + return + if invoking_scope > requested_scope: + # Try to report something helpful. + text = "\n".join(self._factorytraceback()) + fail( + f"ScopeMismatch: You tried to access the {requested_scope.value} scoped " + f"fixture {argname} with a {invoking_scope.value} scoped request object, " + f"involved factories:\n{text}", + pytrace=False, + ) + + def _factorytraceback(self) -> List[str]: + lines = [] + for fixturedef in self._get_fixturestack(): + factory = fixturedef.func + fs, lineno = getfslineno(factory) + if isinstance(fs, Path): + session: Session = self._pyfuncitem.session + p = bestrelpath(session.path, fs) + else: + p = fs + args = _format_args(factory) + lines.append("%s:%d: def %s%s" % (p, lineno + 1, factory.__name__, args)) + return lines + + def _getscopeitem( + self, scope: Union[Scope, "_ScopeName"] + ) -> Union[nodes.Item, nodes.Collector]: + if isinstance(scope, str): + scope = Scope(scope) + if scope is Scope.Function: + # This might also be a non-function Item despite its attribute name. + node: Optional[Union[nodes.Item, nodes.Collector]] = self._pyfuncitem + elif scope is Scope.Package: + # FIXME: _fixturedef is not defined on FixtureRequest (this class), + # but on FixtureRequest (a subclass). + node = get_scope_package(self._pyfuncitem, self._fixturedef) # type: ignore[attr-defined] + else: + node = get_scope_node(self._pyfuncitem, scope) + if node is None and scope is Scope.Class: + # Fallback to function item itself. + node = self._pyfuncitem + assert node, 'Could not obtain a node for scope "{}" for function {!r}'.format( + scope, self._pyfuncitem + ) + return node + + def __repr__(self) -> str: + return "" % (self.node) + + +@final +class SubRequest(FixtureRequest): + """A sub request for handling getting a fixture from a test function/fixture.""" + + def __init__( + self, + request: "FixtureRequest", + scope: Scope, + param: Any, + param_index: int, + fixturedef: "FixtureDef[object]", + *, + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + self._parent_request = request + self.fixturename = fixturedef.argname + if param is not NOTSET: + self.param = param + self.param_index = param_index + self._scope = scope + self._fixturedef = fixturedef + self._pyfuncitem = request._pyfuncitem + self._fixture_defs = request._fixture_defs + self._arg2fixturedefs = request._arg2fixturedefs + self._arg2index = request._arg2index + self._fixturemanager = request._fixturemanager + + def __repr__(self) -> str: + return f"" + + def addfinalizer(self, finalizer: Callable[[], object]) -> None: + """Add finalizer/teardown function to be called after the last test + within the requesting test context finished execution.""" + self._fixturedef.addfinalizer(finalizer) + + def _schedule_finalizers( + self, fixturedef: "FixtureDef[object]", subrequest: "SubRequest" + ) -> None: + # If the executing fixturedef was not explicitly requested in the argument list (via + # getfixturevalue inside the fixture call) then ensure this fixture def will be finished + # first. + if fixturedef.argname not in self.fixturenames: + fixturedef.addfinalizer( + functools.partial(self._fixturedef.finish, request=self) + ) + super()._schedule_finalizers(fixturedef, subrequest) + + +@final +class FixtureLookupError(LookupError): + """Could not return a requested fixture (missing or invalid).""" + + def __init__( + self, argname: Optional[str], request: FixtureRequest, msg: Optional[str] = None + ) -> None: + self.argname = argname + self.request = request + self.fixturestack = request._get_fixturestack() + self.msg = msg + + def formatrepr(self) -> "FixtureLookupErrorRepr": + tblines: List[str] = [] + addline = tblines.append + stack = [self.request._pyfuncitem.obj] + stack.extend(map(lambda x: x.func, self.fixturestack)) + msg = self.msg + if msg is not None: + # The last fixture raise an error, let's present + # it at the requesting side. + stack = stack[:-1] + for function in stack: + fspath, lineno = getfslineno(function) + try: + lines, _ = inspect.getsourcelines(get_real_func(function)) + except (OSError, IndexError, TypeError): + error_msg = "file %s, line %s: source code not available" + addline(error_msg % (fspath, lineno + 1)) + else: + addline(f"file {fspath}, line {lineno + 1}") + for i, line in enumerate(lines): + line = line.rstrip() + addline(" " + line) + if line.lstrip().startswith("def"): + break + + if msg is None: + fm = self.request._fixturemanager + available = set() + parentid = self.request._pyfuncitem.parent.nodeid + for name, fixturedefs in fm._arg2fixturedefs.items(): + faclist = list(fm._matchfactories(fixturedefs, parentid)) + if faclist: + available.add(name) + if self.argname in available: + msg = " recursive dependency involving fixture '{}' detected".format( + self.argname + ) + else: + msg = f"fixture '{self.argname}' not found" + msg += "\n available fixtures: {}".format(", ".join(sorted(available))) + msg += "\n use 'pytest --fixtures [testpath]' for help on them." + + return FixtureLookupErrorRepr(fspath, lineno, tblines, msg, self.argname) + + +class FixtureLookupErrorRepr(TerminalRepr): + def __init__( + self, + filename: Union[str, "os.PathLike[str]"], + firstlineno: int, + tblines: Sequence[str], + errorstring: str, + argname: Optional[str], + ) -> None: + self.tblines = tblines + self.errorstring = errorstring + self.filename = filename + self.firstlineno = firstlineno + self.argname = argname + + def toterminal(self, tw: TerminalWriter) -> None: + # tw.line("FixtureLookupError: %s" %(self.argname), red=True) + for tbline in self.tblines: + tw.line(tbline.rstrip()) + lines = self.errorstring.split("\n") + if lines: + tw.line( + f"{FormattedExcinfo.fail_marker} {lines[0].strip()}", + red=True, + ) + for line in lines[1:]: + tw.line( + f"{FormattedExcinfo.flow_marker} {line.strip()}", + red=True, + ) + tw.line() + tw.line("%s:%d" % (os.fspath(self.filename), self.firstlineno + 1)) + + +def fail_fixturefunc(fixturefunc, msg: str) -> "NoReturn": + fs, lineno = getfslineno(fixturefunc) + location = f"{fs}:{lineno + 1}" + source = _pytest._code.Source(fixturefunc) + fail(msg + ":\n\n" + str(source.indent()) + "\n" + location, pytrace=False) + + +def call_fixture_func( + fixturefunc: "_FixtureFunc[FixtureValue]", request: FixtureRequest, kwargs +) -> FixtureValue: + if is_generator(fixturefunc): + fixturefunc = cast( + Callable[..., Generator[FixtureValue, None, None]], fixturefunc + ) + generator = fixturefunc(**kwargs) + try: + fixture_result = next(generator) + except StopIteration: + raise ValueError(f"{request.fixturename} did not yield a value") from None + finalizer = functools.partial(_teardown_yield_fixture, fixturefunc, generator) + request.addfinalizer(finalizer) + else: + fixturefunc = cast(Callable[..., FixtureValue], fixturefunc) + fixture_result = fixturefunc(**kwargs) + return fixture_result + + +def _teardown_yield_fixture(fixturefunc, it) -> None: + """Execute the teardown of a fixture function by advancing the iterator + after the yield and ensure the iteration ends (if not it means there is + more than one yield in the function).""" + try: + next(it) + except StopIteration: + pass + else: + fail_fixturefunc(fixturefunc, "fixture function has more than one 'yield'") + + +def _eval_scope_callable( + scope_callable: "Callable[[str, Config], _ScopeName]", + fixture_name: str, + config: Config, +) -> "_ScopeName": + try: + # Type ignored because there is no typing mechanism to specify + # keyword arguments, currently. + result = scope_callable(fixture_name=fixture_name, config=config) # type: ignore[call-arg] + except Exception as e: + raise TypeError( + "Error evaluating {} while defining fixture '{}'.\n" + "Expected a function with the signature (*, fixture_name, config)".format( + scope_callable, fixture_name + ) + ) from e + if not isinstance(result, str): + fail( + "Expected {} to return a 'str' while defining fixture '{}', but it returned:\n" + "{!r}".format(scope_callable, fixture_name, result), + pytrace=False, + ) + return result + + +@final +class FixtureDef(Generic[FixtureValue]): + """A container for a factory definition.""" + + def __init__( + self, + fixturemanager: "FixtureManager", + baseid: Optional[str], + argname: str, + func: "_FixtureFunc[FixtureValue]", + scope: Union[Scope, "_ScopeName", Callable[[str, Config], "_ScopeName"], None], + params: Optional[Sequence[object]], + unittest: bool = False, + ids: Optional[ + Union[ + Tuple[Union[None, str, float, int, bool], ...], + Callable[[Any], Optional[object]], + ] + ] = None, + ) -> None: + self._fixturemanager = fixturemanager + self.baseid = baseid or "" + self.has_location = baseid is not None + self.func = func + self.argname = argname + if scope is None: + scope = Scope.Function + elif callable(scope): + scope = _eval_scope_callable(scope, argname, fixturemanager.config) + + if isinstance(scope, str): + scope = Scope.from_user( + scope, descr=f"Fixture '{func.__name__}'", where=baseid + ) + self._scope = scope + self.params: Optional[Sequence[object]] = params + self.argnames: Tuple[str, ...] = getfuncargnames( + func, name=argname, is_method=unittest + ) + self.unittest = unittest + self.ids = ids + self.cached_result: Optional[_FixtureCachedResult[FixtureValue]] = None + self._finalizers: List[Callable[[], object]] = [] + + @property + def scope(self) -> "_ScopeName": + """Scope string, one of "function", "class", "module", "package", "session".""" + return self._scope.value + + def addfinalizer(self, finalizer: Callable[[], object]) -> None: + self._finalizers.append(finalizer) + + def finish(self, request: SubRequest) -> None: + exc = None + try: + while self._finalizers: + try: + func = self._finalizers.pop() + func() + except BaseException as e: + # XXX Only first exception will be seen by user, + # ideally all should be reported. + if exc is None: + exc = e + if exc: + raise exc + finally: + hook = self._fixturemanager.session.gethookproxy(request.node.path) + hook.pytest_fixture_post_finalizer(fixturedef=self, request=request) + # Even if finalization fails, we invalidate the cached fixture + # value and remove all finalizers because they may be bound methods + # which will keep instances alive. + self.cached_result = None + self._finalizers = [] + + def execute(self, request: SubRequest) -> FixtureValue: + # Get required arguments and register our own finish() + # with their finalization. + for argname in self.argnames: + fixturedef = request._get_active_fixturedef(argname) + if argname != "request": + # PseudoFixtureDef is only for "request". + assert isinstance(fixturedef, FixtureDef) + fixturedef.addfinalizer(functools.partial(self.finish, request=request)) + + my_cache_key = self.cache_key(request) + if self.cached_result is not None: + # note: comparison with `==` can fail (or be expensive) for e.g. + # numpy arrays (#6497). + cache_key = self.cached_result[1] + if my_cache_key is cache_key: + if self.cached_result[2] is not None: + _, val, tb = self.cached_result[2] + raise val.with_traceback(tb) + else: + result = self.cached_result[0] + return result + # We have a previous but differently parametrized fixture instance + # so we need to tear it down before creating a new one. + self.finish(request) + assert self.cached_result is None + + hook = self._fixturemanager.session.gethookproxy(request.node.path) + result = hook.pytest_fixture_setup(fixturedef=self, request=request) + return result + + def cache_key(self, request: SubRequest) -> object: + return request.param_index if not hasattr(request, "param") else request.param + + def __repr__(self) -> str: + return "".format( + self.argname, self.scope, self.baseid + ) + + +def resolve_fixture_function( + fixturedef: FixtureDef[FixtureValue], request: FixtureRequest +) -> "_FixtureFunc[FixtureValue]": + """Get the actual callable that can be called to obtain the fixture + value, dealing with unittest-specific instances and bound methods.""" + fixturefunc = fixturedef.func + if fixturedef.unittest: + if request.instance is not None: + # Bind the unbound method to the TestCase instance. + fixturefunc = fixturedef.func.__get__(request.instance) # type: ignore[union-attr] + else: + # The fixture function needs to be bound to the actual + # request.instance so that code working with "fixturedef" behaves + # as expected. + if request.instance is not None: + # Handle the case where fixture is defined not in a test class, but some other class + # (for example a plugin class with a fixture), see #2270. + if hasattr(fixturefunc, "__self__") and not isinstance( + request.instance, fixturefunc.__self__.__class__ # type: ignore[union-attr] + ): + return fixturefunc + fixturefunc = getimfunc(fixturedef.func) + if fixturefunc != fixturedef.func: + fixturefunc = fixturefunc.__get__(request.instance) # type: ignore[union-attr] + return fixturefunc + + +def pytest_fixture_setup( + fixturedef: FixtureDef[FixtureValue], request: SubRequest +) -> FixtureValue: + """Execution of fixture setup.""" + kwargs = {} + for argname in fixturedef.argnames: + fixdef = request._get_active_fixturedef(argname) + assert fixdef.cached_result is not None + result, arg_cache_key, exc = fixdef.cached_result + request._check_scope(argname, request._scope, fixdef._scope) + kwargs[argname] = result + + fixturefunc = resolve_fixture_function(fixturedef, request) + my_cache_key = fixturedef.cache_key(request) + try: + result = call_fixture_func(fixturefunc, request, kwargs) + except TEST_OUTCOME: + exc_info = sys.exc_info() + assert exc_info[0] is not None + fixturedef.cached_result = (None, my_cache_key, exc_info) + raise + fixturedef.cached_result = (result, my_cache_key, None) + return result + + +def _ensure_immutable_ids( + ids: Optional[ + Union[ + Iterable[Union[None, str, float, int, bool]], + Callable[[Any], Optional[object]], + ] + ], +) -> Optional[ + Union[ + Tuple[Union[None, str, float, int, bool], ...], + Callable[[Any], Optional[object]], + ] +]: + if ids is None: + return None + if callable(ids): + return ids + return tuple(ids) + + +def _params_converter( + params: Optional[Iterable[object]], +) -> Optional[Tuple[object, ...]]: + return tuple(params) if params is not None else None + + +def wrap_function_to_error_out_if_called_directly( + function: FixtureFunction, + fixture_marker: "FixtureFunctionMarker", +) -> FixtureFunction: + """Wrap the given fixture function so we can raise an error about it being called directly, + instead of used as an argument in a test function.""" + message = ( + 'Fixture "{name}" called directly. Fixtures are not meant to be called directly,\n' + "but are created automatically when test functions request them as parameters.\n" + "See https://docs.pytest.org/en/stable/explanation/fixtures.html for more information about fixtures, and\n" + "https://docs.pytest.org/en/stable/deprecations.html#calling-fixtures-directly about how to update your code." + ).format(name=fixture_marker.name or function.__name__) + + @functools.wraps(function) + def result(*args, **kwargs): + fail(message, pytrace=False) + + # Keep reference to the original function in our own custom attribute so we don't unwrap + # further than this point and lose useful wrappings like @mock.patch (#3774). + result.__pytest_wrapped__ = _PytestWrapper(function) # type: ignore[attr-defined] + + return cast(FixtureFunction, result) + + +@final +@attr.s(frozen=True, auto_attribs=True) +class FixtureFunctionMarker: + scope: "Union[_ScopeName, Callable[[str, Config], _ScopeName]]" + params: Optional[Tuple[object, ...]] = attr.ib(converter=_params_converter) + autouse: bool = False + ids: Union[ + Tuple[Union[None, str, float, int, bool], ...], + Callable[[Any], Optional[object]], + ] = attr.ib( + default=None, + converter=_ensure_immutable_ids, + ) + name: Optional[str] = None + + def __call__(self, function: FixtureFunction) -> FixtureFunction: + if inspect.isclass(function): + raise ValueError("class fixtures not supported (maybe in the future)") + + if getattr(function, "_pytestfixturefunction", False): + raise ValueError( + "fixture is being applied more than once to the same function" + ) + + function = wrap_function_to_error_out_if_called_directly(function, self) + + name = self.name or function.__name__ + if name == "request": + location = getlocation(function) + fail( + "'request' is a reserved word for fixtures, use another name:\n {}".format( + location + ), + pytrace=False, + ) + + # Type ignored because https://github.com/python/mypy/issues/2087. + function._pytestfixturefunction = self # type: ignore[attr-defined] + return function + + +@overload +def fixture( + fixture_function: FixtureFunction, + *, + scope: "Union[_ScopeName, Callable[[str, Config], _ScopeName]]" = ..., + params: Optional[Iterable[object]] = ..., + autouse: bool = ..., + ids: Optional[ + Union[ + Iterable[Union[None, str, float, int, bool]], + Callable[[Any], Optional[object]], + ] + ] = ..., + name: Optional[str] = ..., +) -> FixtureFunction: + ... + + +@overload +def fixture( + fixture_function: None = ..., + *, + scope: "Union[_ScopeName, Callable[[str, Config], _ScopeName]]" = ..., + params: Optional[Iterable[object]] = ..., + autouse: bool = ..., + ids: Optional[ + Union[ + Iterable[Union[None, str, float, int, bool]], + Callable[[Any], Optional[object]], + ] + ] = ..., + name: Optional[str] = None, +) -> FixtureFunctionMarker: + ... + + +def fixture( + fixture_function: Optional[FixtureFunction] = None, + *, + scope: "Union[_ScopeName, Callable[[str, Config], _ScopeName]]" = "function", + params: Optional[Iterable[object]] = None, + autouse: bool = False, + ids: Optional[ + Union[ + Iterable[Union[None, str, float, int, bool]], + Callable[[Any], Optional[object]], + ] + ] = None, + name: Optional[str] = None, +) -> Union[FixtureFunctionMarker, FixtureFunction]: + """Decorator to mark a fixture factory function. + + This decorator can be used, with or without parameters, to define a + fixture function. + + The name of the fixture function can later be referenced to cause its + invocation ahead of running tests: test modules or classes can use the + ``pytest.mark.usefixtures(fixturename)`` marker. + + Test functions can directly use fixture names as input arguments in which + case the fixture instance returned from the fixture function will be + injected. + + Fixtures can provide their values to test functions using ``return`` or + ``yield`` statements. When using ``yield`` the code block after the + ``yield`` statement is executed as teardown code regardless of the test + outcome, and must yield exactly once. + + :param scope: + The scope for which this fixture is shared; one of ``"function"`` + (default), ``"class"``, ``"module"``, ``"package"`` or ``"session"``. + + This parameter may also be a callable which receives ``(fixture_name, config)`` + as parameters, and must return a ``str`` with one of the values mentioned above. + + See :ref:`dynamic scope` in the docs for more information. + + :param params: + An optional list of parameters which will cause multiple invocations + of the fixture function and all of the tests using it. The current + parameter is available in ``request.param``. + + :param autouse: + If True, the fixture func is activated for all tests that can see it. + If False (the default), an explicit reference is needed to activate + the fixture. + + :param ids: + List of string ids each corresponding to the params so that they are + part of the test id. If no ids are provided they will be generated + automatically from the params. + + :param name: + The name of the fixture. This defaults to the name of the decorated + function. If a fixture is used in the same module in which it is + defined, the function name of the fixture will be shadowed by the + function arg that requests the fixture; one way to resolve this is to + name the decorated function ``fixture_`` and then use + ``@pytest.fixture(name='')``. + """ + fixture_marker = FixtureFunctionMarker( + scope=scope, + params=params, + autouse=autouse, + ids=ids, + name=name, + ) + + # Direct decoration. + if fixture_function: + return fixture_marker(fixture_function) + + return fixture_marker + + +def yield_fixture( + fixture_function=None, + *args, + scope="function", + params=None, + autouse=False, + ids=None, + name=None, +): + """(Return a) decorator to mark a yield-fixture factory function. + + .. deprecated:: 3.0 + Use :py:func:`pytest.fixture` directly instead. + """ + warnings.warn(YIELD_FIXTURE, stacklevel=2) + return fixture( + fixture_function, + *args, + scope=scope, + params=params, + autouse=autouse, + ids=ids, + name=name, + ) + + +@fixture(scope="session") +def pytestconfig(request: FixtureRequest) -> Config: + """Session-scoped fixture that returns the session's :class:`pytest.Config` + object. + + Example:: + + def test_foo(pytestconfig): + if pytestconfig.getoption("verbose") > 0: + ... + + """ + return request.config + + +def pytest_addoption(parser: Parser) -> None: + parser.addini( + "usefixtures", + type="args", + default=[], + help="list of default fixtures to be used with this project", + ) + + +class FixtureManager: + """pytest fixture definitions and information is stored and managed + from this class. + + During collection fm.parsefactories() is called multiple times to parse + fixture function definitions into FixtureDef objects and internal + data structures. + + During collection of test functions, metafunc-mechanics instantiate + a FuncFixtureInfo object which is cached per node/func-name. + This FuncFixtureInfo object is later retrieved by Function nodes + which themselves offer a fixturenames attribute. + + The FuncFixtureInfo object holds information about fixtures and FixtureDefs + relevant for a particular function. An initial list of fixtures is + assembled like this: + + - ini-defined usefixtures + - autouse-marked fixtures along the collection chain up from the function + - usefixtures markers at module/class/function level + - test function funcargs + + Subsequently the funcfixtureinfo.fixturenames attribute is computed + as the closure of the fixtures needed to setup the initial fixtures, + i.e. fixtures needed by fixture functions themselves are appended + to the fixturenames list. + + Upon the test-setup phases all fixturenames are instantiated, retrieved + by a lookup of their FuncFixtureInfo. + """ + + FixtureLookupError = FixtureLookupError + FixtureLookupErrorRepr = FixtureLookupErrorRepr + + def __init__(self, session: "Session") -> None: + self.session = session + self.config: Config = session.config + self._arg2fixturedefs: Dict[str, List[FixtureDef[Any]]] = {} + self._holderobjseen: Set[object] = set() + # A mapping from a nodeid to a list of autouse fixtures it defines. + self._nodeid_autousenames: Dict[str, List[str]] = { + "": self.config.getini("usefixtures"), + } + session.config.pluginmanager.register(self, "funcmanage") + + def _get_direct_parametrize_args(self, node: nodes.Node) -> List[str]: + """Return all direct parametrization arguments of a node, so we don't + mistake them for fixtures. + + Check https://github.com/pytest-dev/pytest/issues/5036. + + These things are done later as well when dealing with parametrization + so this could be improved. + """ + parametrize_argnames: List[str] = [] + for marker in node.iter_markers(name="parametrize"): + if not marker.kwargs.get("indirect", False): + p_argnames, _ = ParameterSet._parse_parametrize_args( + *marker.args, **marker.kwargs + ) + parametrize_argnames.extend(p_argnames) + + return parametrize_argnames + + def getfixtureinfo( + self, node: nodes.Node, func, cls, funcargs: bool = True + ) -> FuncFixtureInfo: + if funcargs and not getattr(node, "nofuncargs", False): + argnames = getfuncargnames(func, name=node.name, cls=cls) + else: + argnames = () + + usefixtures = tuple( + arg for mark in node.iter_markers(name="usefixtures") for arg in mark.args + ) + initialnames = usefixtures + argnames + fm = node.session._fixturemanager + initialnames, names_closure, arg2fixturedefs = fm.getfixtureclosure( + initialnames, node, ignore_args=self._get_direct_parametrize_args(node) + ) + return FuncFixtureInfo(argnames, initialnames, names_closure, arg2fixturedefs) + + def pytest_plugin_registered(self, plugin: _PluggyPlugin) -> None: + nodeid = None + try: + p = absolutepath(plugin.__file__) # type: ignore[attr-defined] + except AttributeError: + pass + else: + # Construct the base nodeid which is later used to check + # what fixtures are visible for particular tests (as denoted + # by their test id). + if p.name.startswith("conftest.py"): + try: + nodeid = str(p.parent.relative_to(self.config.rootpath)) + except ValueError: + nodeid = "" + if nodeid == ".": + nodeid = "" + if os.sep != nodes.SEP: + nodeid = nodeid.replace(os.sep, nodes.SEP) + + self.parsefactories(plugin, nodeid) + + def _getautousenames(self, nodeid: str) -> Iterator[str]: + """Return the names of autouse fixtures applicable to nodeid.""" + for parentnodeid in nodes.iterparentnodeids(nodeid): + basenames = self._nodeid_autousenames.get(parentnodeid) + if basenames: + yield from basenames + + def getfixtureclosure( + self, + fixturenames: Tuple[str, ...], + parentnode: nodes.Node, + ignore_args: Sequence[str] = (), + ) -> Tuple[Tuple[str, ...], List[str], Dict[str, Sequence[FixtureDef[Any]]]]: + # Collect the closure of all fixtures, starting with the given + # fixturenames as the initial set. As we have to visit all + # factory definitions anyway, we also return an arg2fixturedefs + # mapping so that the caller can reuse it and does not have + # to re-discover fixturedefs again for each fixturename + # (discovering matching fixtures for a given name/node is expensive). + + parentid = parentnode.nodeid + fixturenames_closure = list(self._getautousenames(parentid)) + + def merge(otherlist: Iterable[str]) -> None: + for arg in otherlist: + if arg not in fixturenames_closure: + fixturenames_closure.append(arg) + + merge(fixturenames) + + # At this point, fixturenames_closure contains what we call "initialnames", + # which is a set of fixturenames the function immediately requests. We + # need to return it as well, so save this. + initialnames = tuple(fixturenames_closure) + + arg2fixturedefs: Dict[str, Sequence[FixtureDef[Any]]] = {} + lastlen = -1 + while lastlen != len(fixturenames_closure): + lastlen = len(fixturenames_closure) + for argname in fixturenames_closure: + if argname in ignore_args: + continue + if argname in arg2fixturedefs: + continue + fixturedefs = self.getfixturedefs(argname, parentid) + if fixturedefs: + arg2fixturedefs[argname] = fixturedefs + merge(fixturedefs[-1].argnames) + + def sort_by_scope(arg_name: str) -> Scope: + try: + fixturedefs = arg2fixturedefs[arg_name] + except KeyError: + return Scope.Function + else: + return fixturedefs[-1]._scope + + fixturenames_closure.sort(key=sort_by_scope, reverse=True) + return initialnames, fixturenames_closure, arg2fixturedefs + + def pytest_generate_tests(self, metafunc: "Metafunc") -> None: + """Generate new tests based on parametrized fixtures used by the given metafunc""" + + def get_parametrize_mark_argnames(mark: Mark) -> Sequence[str]: + args, _ = ParameterSet._parse_parametrize_args(*mark.args, **mark.kwargs) + return args + + for argname in metafunc.fixturenames: + # Get the FixtureDefs for the argname. + fixture_defs = metafunc._arg2fixturedefs.get(argname) + if not fixture_defs: + # Will raise FixtureLookupError at setup time if not parametrized somewhere + # else (e.g @pytest.mark.parametrize) + continue + + # If the test itself parametrizes using this argname, give it + # precedence. + if any( + argname in get_parametrize_mark_argnames(mark) + for mark in metafunc.definition.iter_markers("parametrize") + ): + continue + + # In the common case we only look at the fixture def with the + # closest scope (last in the list). But if the fixture overrides + # another fixture, while requesting the super fixture, keep going + # in case the super fixture is parametrized (#1953). + for fixturedef in reversed(fixture_defs): + # Fixture is parametrized, apply it and stop. + if fixturedef.params is not None: + metafunc.parametrize( + argname, + fixturedef.params, + indirect=True, + scope=fixturedef.scope, + ids=fixturedef.ids, + ) + break + + # Not requesting the overridden super fixture, stop. + if argname not in fixturedef.argnames: + break + + # Try next super fixture, if any. + + def pytest_collection_modifyitems(self, items: List[nodes.Item]) -> None: + # Separate parametrized setups. + items[:] = reorder_items(items) + + def parsefactories( + self, node_or_obj, nodeid=NOTSET, unittest: bool = False + ) -> None: + if nodeid is not NOTSET: + holderobj = node_or_obj + else: + holderobj = node_or_obj.obj + nodeid = node_or_obj.nodeid + if holderobj in self._holderobjseen: + return + + self._holderobjseen.add(holderobj) + autousenames = [] + for name in dir(holderobj): + # ugly workaround for one of the fspath deprecated property of node + # todo: safely generalize + if isinstance(holderobj, nodes.Node) and name == "fspath": + continue + + # The attribute can be an arbitrary descriptor, so the attribute + # access below can raise. safe_getatt() ignores such exceptions. + obj = safe_getattr(holderobj, name, None) + marker = getfixturemarker(obj) + if not isinstance(marker, FixtureFunctionMarker): + # Magic globals with __getattr__ might have got us a wrong + # fixture attribute. + continue + + if marker.name: + name = marker.name + + # During fixture definition we wrap the original fixture function + # to issue a warning if called directly, so here we unwrap it in + # order to not emit the warning when pytest itself calls the + # fixture function. + obj = get_real_method(obj, holderobj) + + fixture_def = FixtureDef( + fixturemanager=self, + baseid=nodeid, + argname=name, + func=obj, + scope=marker.scope, + params=marker.params, + unittest=unittest, + ids=marker.ids, + ) + + faclist = self._arg2fixturedefs.setdefault(name, []) + if fixture_def.has_location: + faclist.append(fixture_def) + else: + # fixturedefs with no location are at the front + # so this inserts the current fixturedef after the + # existing fixturedefs from external plugins but + # before the fixturedefs provided in conftests. + i = len([f for f in faclist if not f.has_location]) + faclist.insert(i, fixture_def) + if marker.autouse: + autousenames.append(name) + + if autousenames: + self._nodeid_autousenames.setdefault(nodeid or "", []).extend(autousenames) + + def getfixturedefs( + self, argname: str, nodeid: str + ) -> Optional[Sequence[FixtureDef[Any]]]: + """Get a list of fixtures which are applicable to the given node id. + + :param str argname: Name of the fixture to search for. + :param str nodeid: Full node id of the requesting test. + :rtype: Sequence[FixtureDef] + """ + try: + fixturedefs = self._arg2fixturedefs[argname] + except KeyError: + return None + return tuple(self._matchfactories(fixturedefs, nodeid)) + + def _matchfactories( + self, fixturedefs: Iterable[FixtureDef[Any]], nodeid: str + ) -> Iterator[FixtureDef[Any]]: + parentnodeids = set(nodes.iterparentnodeids(nodeid)) + for fixturedef in fixturedefs: + if fixturedef.baseid in parentnodeids: + yield fixturedef diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/freeze_support.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/freeze_support.py new file mode 100644 index 0000000000..9f8ea231fe --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/freeze_support.py @@ -0,0 +1,44 @@ +"""Provides a function to report all internal modules for using freezing +tools.""" +import types +from typing import Iterator +from typing import List +from typing import Union + + +def freeze_includes() -> List[str]: + """Return a list of module names used by pytest that should be + included by cx_freeze.""" + import _pytest + + result = list(_iter_all_modules(_pytest)) + return result + + +def _iter_all_modules( + package: Union[str, types.ModuleType], + prefix: str = "", +) -> Iterator[str]: + """Iterate over the names of all modules that can be found in the given + package, recursively. + + >>> import _pytest + >>> list(_iter_all_modules(_pytest)) + ['_pytest._argcomplete', '_pytest._code.code', ...] + """ + import os + import pkgutil + + if isinstance(package, str): + path = package + else: + # Type ignored because typeshed doesn't define ModuleType.__path__ + # (only defined on packages). + package_path = package.__path__ # type: ignore[attr-defined] + path, prefix = package_path[0], package.__name__ + "." + for _, name, is_package in pkgutil.iter_modules([path]): + if is_package: + for m in _iter_all_modules(os.path.join(path, name), prefix=name + "."): + yield prefix + m + else: + yield prefix + name diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/helpconfig.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/helpconfig.py new file mode 100644 index 0000000000..aca2cd391e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/helpconfig.py @@ -0,0 +1,264 @@ +"""Version info, help messages, tracing configuration.""" +import os +import sys +from argparse import Action +from typing import List +from typing import Optional +from typing import Union + +import pytest +from _pytest.config import Config +from _pytest.config import ExitCode +from _pytest.config import PrintHelp +from _pytest.config.argparsing import Parser + + +class HelpAction(Action): + """An argparse Action that will raise an exception in order to skip the + rest of the argument parsing when --help is passed. + + This prevents argparse from quitting due to missing required arguments + when any are defined, for example by ``pytest_addoption``. + This is similar to the way that the builtin argparse --help option is + implemented by raising SystemExit. + """ + + def __init__(self, option_strings, dest=None, default=False, help=None): + super().__init__( + option_strings=option_strings, + dest=dest, + const=True, + default=default, + nargs=0, + help=help, + ) + + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, self.const) + + # We should only skip the rest of the parsing after preparse is done. + if getattr(parser._parser, "after_preparse", False): + raise PrintHelp + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("debugconfig") + group.addoption( + "--version", + "-V", + action="count", + default=0, + dest="version", + help="display pytest version and information about plugins. " + "When given twice, also display information about plugins.", + ) + group._addoption( + "-h", + "--help", + action=HelpAction, + dest="help", + help="show help message and configuration info", + ) + group._addoption( + "-p", + action="append", + dest="plugins", + default=[], + metavar="name", + help="early-load given plugin module name or entry point (multi-allowed).\n" + "To avoid loading of plugins, use the `no:` prefix, e.g. " + "`no:doctest`.", + ) + group.addoption( + "--traceconfig", + "--trace-config", + action="store_true", + default=False, + help="trace considerations of conftest.py files.", + ) + group.addoption( + "--debug", + action="store", + nargs="?", + const="pytestdebug.log", + dest="debug", + metavar="DEBUG_FILE_NAME", + help="store internal tracing debug information in this log file.\n" + "This file is opened with 'w' and truncated as a result, care advised.\n" + "Defaults to 'pytestdebug.log'.", + ) + group._addoption( + "-o", + "--override-ini", + dest="override_ini", + action="append", + help='override ini option with "option=value" style, e.g. `-o xfail_strict=True -o cache_dir=cache`.', + ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_cmdline_parse(): + outcome = yield + config: Config = outcome.get_result() + + if config.option.debug: + # --debug | --debug was provided. + path = config.option.debug + debugfile = open(path, "w") + debugfile.write( + "versions pytest-%s, " + "python-%s\ncwd=%s\nargs=%s\n\n" + % ( + pytest.__version__, + ".".join(map(str, sys.version_info)), + os.getcwd(), + config.invocation_params.args, + ) + ) + config.trace.root.setwriter(debugfile.write) + undo_tracing = config.pluginmanager.enable_tracing() + sys.stderr.write("writing pytest debug information to %s\n" % path) + + def unset_tracing() -> None: + debugfile.close() + sys.stderr.write("wrote pytest debug information to %s\n" % debugfile.name) + config.trace.root.setwriter(None) + undo_tracing() + + config.add_cleanup(unset_tracing) + + +def showversion(config: Config) -> None: + if config.option.version > 1: + sys.stdout.write( + "This is pytest version {}, imported from {}\n".format( + pytest.__version__, pytest.__file__ + ) + ) + plugininfo = getpluginversioninfo(config) + if plugininfo: + for line in plugininfo: + sys.stdout.write(line + "\n") + else: + sys.stdout.write(f"pytest {pytest.__version__}\n") + + +def pytest_cmdline_main(config: Config) -> Optional[Union[int, ExitCode]]: + if config.option.version > 0: + showversion(config) + return 0 + elif config.option.help: + config._do_configure() + showhelp(config) + config._ensure_unconfigure() + return 0 + return None + + +def showhelp(config: Config) -> None: + import textwrap + + reporter = config.pluginmanager.get_plugin("terminalreporter") + tw = reporter._tw + tw.write(config._parser.optparser.format_help()) + tw.line() + tw.line( + "[pytest] ini-options in the first pytest.ini|tox.ini|setup.cfg file found:" + ) + tw.line() + + columns = tw.fullwidth # costly call + indent_len = 24 # based on argparse's max_help_position=24 + indent = " " * indent_len + for name in config._parser._ininames: + help, type, default = config._parser._inidict[name] + if type is None: + type = "string" + if help is None: + raise TypeError(f"help argument cannot be None for {name}") + spec = f"{name} ({type}):" + tw.write(" %s" % spec) + spec_len = len(spec) + if spec_len > (indent_len - 3): + # Display help starting at a new line. + tw.line() + helplines = textwrap.wrap( + help, + columns, + initial_indent=indent, + subsequent_indent=indent, + break_on_hyphens=False, + ) + + for line in helplines: + tw.line(line) + else: + # Display help starting after the spec, following lines indented. + tw.write(" " * (indent_len - spec_len - 2)) + wrapped = textwrap.wrap(help, columns - indent_len, break_on_hyphens=False) + + if wrapped: + tw.line(wrapped[0]) + for line in wrapped[1:]: + tw.line(indent + line) + + tw.line() + tw.line("environment variables:") + vars = [ + ("PYTEST_ADDOPTS", "extra command line options"), + ("PYTEST_PLUGINS", "comma-separated plugins to load during startup"), + ("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "set to disable plugin auto-loading"), + ("PYTEST_DEBUG", "set to enable debug tracing of pytest's internals"), + ] + for name, help in vars: + tw.line(f" {name:<24} {help}") + tw.line() + tw.line() + + tw.line("to see available markers type: pytest --markers") + tw.line("to see available fixtures type: pytest --fixtures") + tw.line( + "(shown according to specified file_or_dir or current dir " + "if not specified; fixtures with leading '_' are only shown " + "with the '-v' option" + ) + + for warningreport in reporter.stats.get("warnings", []): + tw.line("warning : " + warningreport.message, red=True) + return + + +conftest_options = [("pytest_plugins", "list of plugin names to load")] + + +def getpluginversioninfo(config: Config) -> List[str]: + lines = [] + plugininfo = config.pluginmanager.list_plugin_distinfo() + if plugininfo: + lines.append("setuptools registered plugins:") + for plugin, dist in plugininfo: + loc = getattr(plugin, "__file__", repr(plugin)) + content = f"{dist.project_name}-{dist.version} at {loc}" + lines.append(" " + content) + return lines + + +def pytest_report_header(config: Config) -> List[str]: + lines = [] + if config.option.debug or config.option.traceconfig: + lines.append(f"using: pytest-{pytest.__version__}") + + verinfo = getpluginversioninfo(config) + if verinfo: + lines.extend(verinfo) + + if config.option.traceconfig: + lines.append("active plugins:") + items = config.pluginmanager.list_name_plugin() + for name, plugin in items: + if hasattr(plugin, "__file__"): + r = plugin.__file__ + else: + r = repr(plugin) + lines.append(f" {name:<20}: {r}") + return lines diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/hookspec.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/hookspec.py new file mode 100644 index 0000000000..79251315d8 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/hookspec.py @@ -0,0 +1,928 @@ +"""Hook specifications for pytest plugins which are invoked by pytest itself +and by builtin plugins.""" +from pathlib import Path +from typing import Any +from typing import Dict +from typing import List +from typing import Mapping +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import TYPE_CHECKING +from typing import Union + +from pluggy import HookspecMarker + +from _pytest.deprecated import WARNING_CAPTURED_HOOK +from _pytest.deprecated import WARNING_CMDLINE_PREPARSE_HOOK + +if TYPE_CHECKING: + import pdb + import warnings + from typing_extensions import Literal + + from _pytest._code.code import ExceptionRepr + from _pytest.code import ExceptionInfo + from _pytest.config import Config + from _pytest.config import ExitCode + from _pytest.config import PytestPluginManager + from _pytest.config import _PluggyPlugin + from _pytest.config.argparsing import Parser + from _pytest.fixtures import FixtureDef + from _pytest.fixtures import SubRequest + from _pytest.main import Session + from _pytest.nodes import Collector + from _pytest.nodes import Item + from _pytest.outcomes import Exit + from _pytest.python import Function + from _pytest.python import Metafunc + from _pytest.python import Module + from _pytest.python import PyCollector + from _pytest.reports import CollectReport + from _pytest.reports import TestReport + from _pytest.runner import CallInfo + from _pytest.terminal import TerminalReporter + from _pytest.compat import LEGACY_PATH + + +hookspec = HookspecMarker("pytest") + +# ------------------------------------------------------------------------- +# Initialization hooks called for every plugin +# ------------------------------------------------------------------------- + + +@hookspec(historic=True) +def pytest_addhooks(pluginmanager: "PytestPluginManager") -> None: + """Called at plugin registration time to allow adding new hooks via a call to + ``pluginmanager.add_hookspecs(module_or_class, prefix)``. + + :param pytest.PytestPluginManager pluginmanager: The pytest plugin manager. + + .. note:: + This hook is incompatible with ``hookwrapper=True``. + """ + + +@hookspec(historic=True) +def pytest_plugin_registered( + plugin: "_PluggyPlugin", manager: "PytestPluginManager" +) -> None: + """A new pytest plugin got registered. + + :param plugin: The plugin module or instance. + :param pytest.PytestPluginManager manager: pytest plugin manager. + + .. note:: + This hook is incompatible with ``hookwrapper=True``. + """ + + +@hookspec(historic=True) +def pytest_addoption(parser: "Parser", pluginmanager: "PytestPluginManager") -> None: + """Register argparse-style options and ini-style config values, + called once at the beginning of a test run. + + .. note:: + + This function should be implemented only in plugins or ``conftest.py`` + files situated at the tests root directory due to how pytest + :ref:`discovers plugins during startup `. + + :param pytest.Parser parser: + To add command line options, call + :py:func:`parser.addoption(...) `. + To add ini-file values call :py:func:`parser.addini(...) + `. + + :param pytest.PytestPluginManager pluginmanager: + The pytest plugin manager, which can be used to install :py:func:`hookspec`'s + or :py:func:`hookimpl`'s and allow one plugin to call another plugin's hooks + to change how command line options are added. + + Options can later be accessed through the + :py:class:`config ` object, respectively: + + - :py:func:`config.getoption(name) ` to + retrieve the value of a command line option. + + - :py:func:`config.getini(name) ` to retrieve + a value read from an ini-style file. + + The config object is passed around on many internal objects via the ``.config`` + attribute or can be retrieved as the ``pytestconfig`` fixture. + + .. note:: + This hook is incompatible with ``hookwrapper=True``. + """ + + +@hookspec(historic=True) +def pytest_configure(config: "Config") -> None: + """Allow plugins and conftest files to perform initial configuration. + + This hook is called for every plugin and initial conftest file + after command line options have been parsed. + + After that, the hook is called for other conftest files as they are + imported. + + .. note:: + This hook is incompatible with ``hookwrapper=True``. + + :param pytest.Config config: The pytest config object. + """ + + +# ------------------------------------------------------------------------- +# Bootstrapping hooks called for plugins registered early enough: +# internal and 3rd party plugins. +# ------------------------------------------------------------------------- + + +@hookspec(firstresult=True) +def pytest_cmdline_parse( + pluginmanager: "PytestPluginManager", args: List[str] +) -> Optional["Config"]: + """Return an initialized config object, parsing the specified args. + + Stops at first non-None result, see :ref:`firstresult`. + + .. note:: + This hook will only be called for plugin classes passed to the + ``plugins`` arg when using `pytest.main`_ to perform an in-process + test run. + + :param pytest.PytestPluginManager pluginmanager: The pytest plugin manager. + :param List[str] args: List of arguments passed on the command line. + """ + + +@hookspec(warn_on_impl=WARNING_CMDLINE_PREPARSE_HOOK) +def pytest_cmdline_preparse(config: "Config", args: List[str]) -> None: + """(**Deprecated**) modify command line arguments before option parsing. + + This hook is considered deprecated and will be removed in a future pytest version. Consider + using :hook:`pytest_load_initial_conftests` instead. + + .. note:: + This hook will not be called for ``conftest.py`` files, only for setuptools plugins. + + :param pytest.Config config: The pytest config object. + :param List[str] args: Arguments passed on the command line. + """ + + +@hookspec(firstresult=True) +def pytest_cmdline_main(config: "Config") -> Optional[Union["ExitCode", int]]: + """Called for performing the main command line action. The default + implementation will invoke the configure hooks and runtest_mainloop. + + Stops at first non-None result, see :ref:`firstresult`. + + :param pytest.Config config: The pytest config object. + """ + + +def pytest_load_initial_conftests( + early_config: "Config", parser: "Parser", args: List[str] +) -> None: + """Called to implement the loading of initial conftest files ahead + of command line option parsing. + + .. note:: + This hook will not be called for ``conftest.py`` files, only for setuptools plugins. + + :param pytest.Config early_config: The pytest config object. + :param List[str] args: Arguments passed on the command line. + :param pytest.Parser parser: To add command line options. + """ + + +# ------------------------------------------------------------------------- +# collection hooks +# ------------------------------------------------------------------------- + + +@hookspec(firstresult=True) +def pytest_collection(session: "Session") -> Optional[object]: + """Perform the collection phase for the given session. + + Stops at first non-None result, see :ref:`firstresult`. + The return value is not used, but only stops further processing. + + The default collection phase is this (see individual hooks for full details): + + 1. Starting from ``session`` as the initial collector: + + 1. ``pytest_collectstart(collector)`` + 2. ``report = pytest_make_collect_report(collector)`` + 3. ``pytest_exception_interact(collector, call, report)`` if an interactive exception occurred + 4. For each collected node: + + 1. If an item, ``pytest_itemcollected(item)`` + 2. If a collector, recurse into it. + + 5. ``pytest_collectreport(report)`` + + 2. ``pytest_collection_modifyitems(session, config, items)`` + + 1. ``pytest_deselected(items)`` for any deselected items (may be called multiple times) + + 3. ``pytest_collection_finish(session)`` + 4. Set ``session.items`` to the list of collected items + 5. Set ``session.testscollected`` to the number of collected items + + You can implement this hook to only perform some action before collection, + for example the terminal plugin uses it to start displaying the collection + counter (and returns `None`). + + :param pytest.Session session: The pytest session object. + """ + + +def pytest_collection_modifyitems( + session: "Session", config: "Config", items: List["Item"] +) -> None: + """Called after collection has been performed. May filter or re-order + the items in-place. + + :param pytest.Session session: The pytest session object. + :param pytest.Config config: The pytest config object. + :param List[pytest.Item] items: List of item objects. + """ + + +def pytest_collection_finish(session: "Session") -> None: + """Called after collection has been performed and modified. + + :param pytest.Session session: The pytest session object. + """ + + +@hookspec(firstresult=True) +def pytest_ignore_collect( + collection_path: Path, path: "LEGACY_PATH", config: "Config" +) -> Optional[bool]: + """Return True to prevent considering this path for collection. + + This hook is consulted for all files and directories prior to calling + more specific hooks. + + Stops at first non-None result, see :ref:`firstresult`. + + :param pathlib.Path collection_path : The path to analyze. + :param LEGACY_PATH path: The path to analyze (deprecated). + :param pytest.Config config: The pytest config object. + + .. versionchanged:: 7.0.0 + The ``collection_path`` parameter was added as a :class:`pathlib.Path` + equivalent of the ``path`` parameter. The ``path`` parameter + has been deprecated. + """ + + +def pytest_collect_file( + file_path: Path, path: "LEGACY_PATH", parent: "Collector" +) -> "Optional[Collector]": + """Create a Collector for the given path, or None if not relevant. + + The new node needs to have the specified ``parent`` as a parent. + + :param pathlib.Path file_path: The path to analyze. + :param LEGACY_PATH path: The path to collect (deprecated). + + .. versionchanged:: 7.0.0 + The ``file_path`` parameter was added as a :class:`pathlib.Path` + equivalent of the ``path`` parameter. The ``path`` parameter + has been deprecated. + """ + + +# logging hooks for collection + + +def pytest_collectstart(collector: "Collector") -> None: + """Collector starts collecting.""" + + +def pytest_itemcollected(item: "Item") -> None: + """We just collected a test item.""" + + +def pytest_collectreport(report: "CollectReport") -> None: + """Collector finished collecting.""" + + +def pytest_deselected(items: Sequence["Item"]) -> None: + """Called for deselected test items, e.g. by keyword. + + May be called multiple times. + """ + + +@hookspec(firstresult=True) +def pytest_make_collect_report(collector: "Collector") -> "Optional[CollectReport]": + """Perform :func:`collector.collect() ` and return + a :class:`~pytest.CollectReport`. + + Stops at first non-None result, see :ref:`firstresult`. + """ + + +# ------------------------------------------------------------------------- +# Python test function related hooks +# ------------------------------------------------------------------------- + + +@hookspec(firstresult=True) +def pytest_pycollect_makemodule( + module_path: Path, path: "LEGACY_PATH", parent +) -> Optional["Module"]: + """Return a Module collector or None for the given path. + + This hook will be called for each matching test module path. + The pytest_collect_file hook needs to be used if you want to + create test modules for files that do not match as a test module. + + Stops at first non-None result, see :ref:`firstresult`. + + :param pathlib.Path module_path: The path of the module to collect. + :param LEGACY_PATH path: The path of the module to collect (deprecated). + + .. versionchanged:: 7.0.0 + The ``module_path`` parameter was added as a :class:`pathlib.Path` + equivalent of the ``path`` parameter. + + The ``path`` parameter has been deprecated in favor of ``fspath``. + """ + + +@hookspec(firstresult=True) +def pytest_pycollect_makeitem( + collector: "PyCollector", name: str, obj: object +) -> Union[None, "Item", "Collector", List[Union["Item", "Collector"]]]: + """Return a custom item/collector for a Python object in a module, or None. + + Stops at first non-None result, see :ref:`firstresult`. + """ + + +@hookspec(firstresult=True) +def pytest_pyfunc_call(pyfuncitem: "Function") -> Optional[object]: + """Call underlying test function. + + Stops at first non-None result, see :ref:`firstresult`. + """ + + +def pytest_generate_tests(metafunc: "Metafunc") -> None: + """Generate (multiple) parametrized calls to a test function.""" + + +@hookspec(firstresult=True) +def pytest_make_parametrize_id( + config: "Config", val: object, argname: str +) -> Optional[str]: + """Return a user-friendly string representation of the given ``val`` + that will be used by @pytest.mark.parametrize calls, or None if the hook + doesn't know about ``val``. + + The parameter name is available as ``argname``, if required. + + Stops at first non-None result, see :ref:`firstresult`. + + :param pytest.Config config: The pytest config object. + :param val: The parametrized value. + :param str argname: The automatic parameter name produced by pytest. + """ + + +# ------------------------------------------------------------------------- +# runtest related hooks +# ------------------------------------------------------------------------- + + +@hookspec(firstresult=True) +def pytest_runtestloop(session: "Session") -> Optional[object]: + """Perform the main runtest loop (after collection finished). + + The default hook implementation performs the runtest protocol for all items + collected in the session (``session.items``), unless the collection failed + or the ``collectonly`` pytest option is set. + + If at any point :py:func:`pytest.exit` is called, the loop is + terminated immediately. + + If at any point ``session.shouldfail`` or ``session.shouldstop`` are set, the + loop is terminated after the runtest protocol for the current item is finished. + + :param pytest.Session session: The pytest session object. + + Stops at first non-None result, see :ref:`firstresult`. + The return value is not used, but only stops further processing. + """ + + +@hookspec(firstresult=True) +def pytest_runtest_protocol( + item: "Item", nextitem: "Optional[Item]" +) -> Optional[object]: + """Perform the runtest protocol for a single test item. + + The default runtest protocol is this (see individual hooks for full details): + + - ``pytest_runtest_logstart(nodeid, location)`` + + - Setup phase: + - ``call = pytest_runtest_setup(item)`` (wrapped in ``CallInfo(when="setup")``) + - ``report = pytest_runtest_makereport(item, call)`` + - ``pytest_runtest_logreport(report)`` + - ``pytest_exception_interact(call, report)`` if an interactive exception occurred + + - Call phase, if the the setup passed and the ``setuponly`` pytest option is not set: + - ``call = pytest_runtest_call(item)`` (wrapped in ``CallInfo(when="call")``) + - ``report = pytest_runtest_makereport(item, call)`` + - ``pytest_runtest_logreport(report)`` + - ``pytest_exception_interact(call, report)`` if an interactive exception occurred + + - Teardown phase: + - ``call = pytest_runtest_teardown(item, nextitem)`` (wrapped in ``CallInfo(when="teardown")``) + - ``report = pytest_runtest_makereport(item, call)`` + - ``pytest_runtest_logreport(report)`` + - ``pytest_exception_interact(call, report)`` if an interactive exception occurred + + - ``pytest_runtest_logfinish(nodeid, location)`` + + :param item: Test item for which the runtest protocol is performed. + :param nextitem: The scheduled-to-be-next test item (or None if this is the end my friend). + + Stops at first non-None result, see :ref:`firstresult`. + The return value is not used, but only stops further processing. + """ + + +def pytest_runtest_logstart( + nodeid: str, location: Tuple[str, Optional[int], str] +) -> None: + """Called at the start of running the runtest protocol for a single item. + + See :hook:`pytest_runtest_protocol` for a description of the runtest protocol. + + :param str nodeid: Full node ID of the item. + :param location: A tuple of ``(filename, lineno, testname)``. + """ + + +def pytest_runtest_logfinish( + nodeid: str, location: Tuple[str, Optional[int], str] +) -> None: + """Called at the end of running the runtest protocol for a single item. + + See :hook:`pytest_runtest_protocol` for a description of the runtest protocol. + + :param str nodeid: Full node ID of the item. + :param location: A tuple of ``(filename, lineno, testname)``. + """ + + +def pytest_runtest_setup(item: "Item") -> None: + """Called to perform the setup phase for a test item. + + The default implementation runs ``setup()`` on ``item`` and all of its + parents (which haven't been setup yet). This includes obtaining the + values of fixtures required by the item (which haven't been obtained + yet). + """ + + +def pytest_runtest_call(item: "Item") -> None: + """Called to run the test for test item (the call phase). + + The default implementation calls ``item.runtest()``. + """ + + +def pytest_runtest_teardown(item: "Item", nextitem: Optional["Item"]) -> None: + """Called to perform the teardown phase for a test item. + + The default implementation runs the finalizers and calls ``teardown()`` + on ``item`` and all of its parents (which need to be torn down). This + includes running the teardown phase of fixtures required by the item (if + they go out of scope). + + :param nextitem: + The scheduled-to-be-next test item (None if no further test item is + scheduled). This argument is used to perform exact teardowns, i.e. + calling just enough finalizers so that nextitem only needs to call + setup functions. + """ + + +@hookspec(firstresult=True) +def pytest_runtest_makereport( + item: "Item", call: "CallInfo[None]" +) -> Optional["TestReport"]: + """Called to create a :class:`~pytest.TestReport` for each of + the setup, call and teardown runtest phases of a test item. + + See :hook:`pytest_runtest_protocol` for a description of the runtest protocol. + + :param call: The :class:`~pytest.CallInfo` for the phase. + + Stops at first non-None result, see :ref:`firstresult`. + """ + + +def pytest_runtest_logreport(report: "TestReport") -> None: + """Process the :class:`~pytest.TestReport` produced for each + of the setup, call and teardown runtest phases of an item. + + See :hook:`pytest_runtest_protocol` for a description of the runtest protocol. + """ + + +@hookspec(firstresult=True) +def pytest_report_to_serializable( + config: "Config", + report: Union["CollectReport", "TestReport"], +) -> Optional[Dict[str, Any]]: + """Serialize the given report object into a data structure suitable for + sending over the wire, e.g. converted to JSON.""" + + +@hookspec(firstresult=True) +def pytest_report_from_serializable( + config: "Config", + data: Dict[str, Any], +) -> Optional[Union["CollectReport", "TestReport"]]: + """Restore a report object previously serialized with + :hook:`pytest_report_to_serializable`.""" + + +# ------------------------------------------------------------------------- +# Fixture related hooks +# ------------------------------------------------------------------------- + + +@hookspec(firstresult=True) +def pytest_fixture_setup( + fixturedef: "FixtureDef[Any]", request: "SubRequest" +) -> Optional[object]: + """Perform fixture setup execution. + + :returns: The return value of the call to the fixture function. + + Stops at first non-None result, see :ref:`firstresult`. + + .. note:: + If the fixture function returns None, other implementations of + this hook function will continue to be called, according to the + behavior of the :ref:`firstresult` option. + """ + + +def pytest_fixture_post_finalizer( + fixturedef: "FixtureDef[Any]", request: "SubRequest" +) -> None: + """Called after fixture teardown, but before the cache is cleared, so + the fixture result ``fixturedef.cached_result`` is still available (not + ``None``).""" + + +# ------------------------------------------------------------------------- +# test session related hooks +# ------------------------------------------------------------------------- + + +def pytest_sessionstart(session: "Session") -> None: + """Called after the ``Session`` object has been created and before performing collection + and entering the run test loop. + + :param pytest.Session session: The pytest session object. + """ + + +def pytest_sessionfinish( + session: "Session", + exitstatus: Union[int, "ExitCode"], +) -> None: + """Called after whole test run finished, right before returning the exit status to the system. + + :param pytest.Session session: The pytest session object. + :param int exitstatus: The status which pytest will return to the system. + """ + + +def pytest_unconfigure(config: "Config") -> None: + """Called before test process is exited. + + :param pytest.Config config: The pytest config object. + """ + + +# ------------------------------------------------------------------------- +# hooks for customizing the assert methods +# ------------------------------------------------------------------------- + + +def pytest_assertrepr_compare( + config: "Config", op: str, left: object, right: object +) -> Optional[List[str]]: + """Return explanation for comparisons in failing assert expressions. + + Return None for no custom explanation, otherwise return a list + of strings. The strings will be joined by newlines but any newlines + *in* a string will be escaped. Note that all but the first line will + be indented slightly, the intention is for the first line to be a summary. + + :param pytest.Config config: The pytest config object. + """ + + +def pytest_assertion_pass(item: "Item", lineno: int, orig: str, expl: str) -> None: + """Called whenever an assertion passes. + + .. versionadded:: 5.0 + + Use this hook to do some processing after a passing assertion. + The original assertion information is available in the `orig` string + and the pytest introspected assertion information is available in the + `expl` string. + + This hook must be explicitly enabled by the ``enable_assertion_pass_hook`` + ini-file option: + + .. code-block:: ini + + [pytest] + enable_assertion_pass_hook=true + + You need to **clean the .pyc** files in your project directory and interpreter libraries + when enabling this option, as assertions will require to be re-written. + + :param pytest.Item item: pytest item object of current test. + :param int lineno: Line number of the assert statement. + :param str orig: String with the original assertion. + :param str expl: String with the assert explanation. + """ + + +# ------------------------------------------------------------------------- +# Hooks for influencing reporting (invoked from _pytest_terminal). +# ------------------------------------------------------------------------- + + +def pytest_report_header( + config: "Config", start_path: Path, startdir: "LEGACY_PATH" +) -> Union[str, List[str]]: + """Return a string or list of strings to be displayed as header info for terminal reporting. + + :param pytest.Config config: The pytest config object. + :param Path start_path: The starting dir. + :param LEGACY_PATH startdir: The starting dir (deprecated). + + .. note:: + + Lines returned by a plugin are displayed before those of plugins which + ran before it. + If you want to have your line(s) displayed first, use + :ref:`trylast=True `. + + .. note:: + + This function should be implemented only in plugins or ``conftest.py`` + files situated at the tests root directory due to how pytest + :ref:`discovers plugins during startup `. + + .. versionchanged:: 7.0.0 + The ``start_path`` parameter was added as a :class:`pathlib.Path` + equivalent of the ``startdir`` parameter. The ``startdir`` parameter + has been deprecated. + """ + + +def pytest_report_collectionfinish( + config: "Config", + start_path: Path, + startdir: "LEGACY_PATH", + items: Sequence["Item"], +) -> Union[str, List[str]]: + """Return a string or list of strings to be displayed after collection + has finished successfully. + + These strings will be displayed after the standard "collected X items" message. + + .. versionadded:: 3.2 + + :param pytest.Config config: The pytest config object. + :param Path start_path: The starting dir. + :param LEGACY_PATH startdir: The starting dir (deprecated). + :param items: List of pytest items that are going to be executed; this list should not be modified. + + .. note:: + + Lines returned by a plugin are displayed before those of plugins which + ran before it. + If you want to have your line(s) displayed first, use + :ref:`trylast=True `. + + .. versionchanged:: 7.0.0 + The ``start_path`` parameter was added as a :class:`pathlib.Path` + equivalent of the ``startdir`` parameter. The ``startdir`` parameter + has been deprecated. + """ + + +@hookspec(firstresult=True) +def pytest_report_teststatus( + report: Union["CollectReport", "TestReport"], config: "Config" +) -> Tuple[str, str, Union[str, Mapping[str, bool]]]: + """Return result-category, shortletter and verbose word for status + reporting. + + The result-category is a category in which to count the result, for + example "passed", "skipped", "error" or the empty string. + + The shortletter is shown as testing progresses, for example ".", "s", + "E" or the empty string. + + The verbose word is shown as testing progresses in verbose mode, for + example "PASSED", "SKIPPED", "ERROR" or the empty string. + + pytest may style these implicitly according to the report outcome. + To provide explicit styling, return a tuple for the verbose word, + for example ``"rerun", "R", ("RERUN", {"yellow": True})``. + + :param report: The report object whose status is to be returned. + :param config: The pytest config object. + + Stops at first non-None result, see :ref:`firstresult`. + """ + + +def pytest_terminal_summary( + terminalreporter: "TerminalReporter", + exitstatus: "ExitCode", + config: "Config", +) -> None: + """Add a section to terminal summary reporting. + + :param _pytest.terminal.TerminalReporter terminalreporter: The internal terminal reporter object. + :param int exitstatus: The exit status that will be reported back to the OS. + :param pytest.Config config: The pytest config object. + + .. versionadded:: 4.2 + The ``config`` parameter. + """ + + +@hookspec(historic=True, warn_on_impl=WARNING_CAPTURED_HOOK) +def pytest_warning_captured( + warning_message: "warnings.WarningMessage", + when: "Literal['config', 'collect', 'runtest']", + item: Optional["Item"], + location: Optional[Tuple[str, int, str]], +) -> None: + """(**Deprecated**) Process a warning captured by the internal pytest warnings plugin. + + .. deprecated:: 6.0 + + This hook is considered deprecated and will be removed in a future pytest version. + Use :func:`pytest_warning_recorded` instead. + + :param warnings.WarningMessage warning_message: + The captured warning. This is the same object produced by :py:func:`warnings.catch_warnings`, and contains + the same attributes as the parameters of :py:func:`warnings.showwarning`. + + :param str when: + Indicates when the warning was captured. Possible values: + + * ``"config"``: during pytest configuration/initialization stage. + * ``"collect"``: during test collection. + * ``"runtest"``: during test execution. + + :param pytest.Item|None item: + The item being executed if ``when`` is ``"runtest"``, otherwise ``None``. + + :param tuple location: + When available, holds information about the execution context of the captured + warning (filename, linenumber, function). ``function`` evaluates to + when the execution context is at the module level. + """ + + +@hookspec(historic=True) +def pytest_warning_recorded( + warning_message: "warnings.WarningMessage", + when: "Literal['config', 'collect', 'runtest']", + nodeid: str, + location: Optional[Tuple[str, int, str]], +) -> None: + """Process a warning captured by the internal pytest warnings plugin. + + :param warnings.WarningMessage warning_message: + The captured warning. This is the same object produced by :py:func:`warnings.catch_warnings`, and contains + the same attributes as the parameters of :py:func:`warnings.showwarning`. + + :param str when: + Indicates when the warning was captured. Possible values: + + * ``"config"``: during pytest configuration/initialization stage. + * ``"collect"``: during test collection. + * ``"runtest"``: during test execution. + + :param str nodeid: + Full id of the item. + + :param tuple|None location: + When available, holds information about the execution context of the captured + warning (filename, linenumber, function). ``function`` evaluates to + when the execution context is at the module level. + + .. versionadded:: 6.0 + """ + + +# ------------------------------------------------------------------------- +# Hooks for influencing skipping +# ------------------------------------------------------------------------- + + +def pytest_markeval_namespace(config: "Config") -> Dict[str, Any]: + """Called when constructing the globals dictionary used for + evaluating string conditions in xfail/skipif markers. + + This is useful when the condition for a marker requires + objects that are expensive or impossible to obtain during + collection time, which is required by normal boolean + conditions. + + .. versionadded:: 6.2 + + :param pytest.Config config: The pytest config object. + :returns: A dictionary of additional globals to add. + """ + + +# ------------------------------------------------------------------------- +# error handling and internal debugging hooks +# ------------------------------------------------------------------------- + + +def pytest_internalerror( + excrepr: "ExceptionRepr", + excinfo: "ExceptionInfo[BaseException]", +) -> Optional[bool]: + """Called for internal errors. + + Return True to suppress the fallback handling of printing an + INTERNALERROR message directly to sys.stderr. + """ + + +def pytest_keyboard_interrupt( + excinfo: "ExceptionInfo[Union[KeyboardInterrupt, Exit]]", +) -> None: + """Called for keyboard interrupt.""" + + +def pytest_exception_interact( + node: Union["Item", "Collector"], + call: "CallInfo[Any]", + report: Union["CollectReport", "TestReport"], +) -> None: + """Called when an exception was raised which can potentially be + interactively handled. + + May be called during collection (see :hook:`pytest_make_collect_report`), + in which case ``report`` is a :class:`CollectReport`. + + May be called during runtest of an item (see :hook:`pytest_runtest_protocol`), + in which case ``report`` is a :class:`TestReport`. + + This hook is not called if the exception that was raised is an internal + exception like ``skip.Exception``. + """ + + +def pytest_enter_pdb(config: "Config", pdb: "pdb.Pdb") -> None: + """Called upon pdb.set_trace(). + + Can be used by plugins to take special action just before the python + debugger enters interactive mode. + + :param pytest.Config config: The pytest config object. + :param pdb.Pdb pdb: The Pdb instance. + """ + + +def pytest_leave_pdb(config: "Config", pdb: "pdb.Pdb") -> None: + """Called when leaving pdb (e.g. with continue after pdb.set_trace()). + + Can be used by plugins to take special action just after the python + debugger leaves interactive mode. + + :param pytest.Config config: The pytest config object. + :param pdb.Pdb pdb: The Pdb instance. + """ diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/junitxml.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/junitxml.py new file mode 100644 index 0000000000..4af5fbab0c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/junitxml.py @@ -0,0 +1,696 @@ +"""Report test results in JUnit-XML format, for use with Jenkins and build +integration servers. + +Based on initial code from Ross Lawley. + +Output conforms to +https://github.com/jenkinsci/xunit-plugin/blob/master/src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd +""" +import functools +import os +import platform +import re +import xml.etree.ElementTree as ET +from datetime import datetime +from typing import Callable +from typing import Dict +from typing import List +from typing import Match +from typing import Optional +from typing import Tuple +from typing import Union + +import pytest +from _pytest import nodes +from _pytest import timing +from _pytest._code.code import ExceptionRepr +from _pytest._code.code import ReprFileLocation +from _pytest.config import Config +from _pytest.config import filename_arg +from _pytest.config.argparsing import Parser +from _pytest.fixtures import FixtureRequest +from _pytest.reports import TestReport +from _pytest.stash import StashKey +from _pytest.terminal import TerminalReporter + + +xml_key = StashKey["LogXML"]() + + +def bin_xml_escape(arg: object) -> str: + r"""Visually escape invalid XML characters. + + For example, transforms + 'hello\aworld\b' + into + 'hello#x07world#x08' + Note that the #xABs are *not* XML escapes - missing the ampersand «. + The idea is to escape visually for the user rather than for XML itself. + """ + + def repl(matchobj: Match[str]) -> str: + i = ord(matchobj.group()) + if i <= 0xFF: + return "#x%02X" % i + else: + return "#x%04X" % i + + # The spec range of valid chars is: + # Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF] + # For an unknown(?) reason, we disallow #x7F (DEL) as well. + illegal_xml_re = ( + "[^\u0009\u000A\u000D\u0020-\u007E\u0080-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF]" + ) + return re.sub(illegal_xml_re, repl, str(arg)) + + +def merge_family(left, right) -> None: + result = {} + for kl, vl in left.items(): + for kr, vr in right.items(): + if not isinstance(vl, list): + raise TypeError(type(vl)) + result[kl] = vl + vr + left.update(result) + + +families = {} +families["_base"] = {"testcase": ["classname", "name"]} +families["_base_legacy"] = {"testcase": ["file", "line", "url"]} + +# xUnit 1.x inherits legacy attributes. +families["xunit1"] = families["_base"].copy() +merge_family(families["xunit1"], families["_base_legacy"]) + +# xUnit 2.x uses strict base attributes. +families["xunit2"] = families["_base"] + + +class _NodeReporter: + def __init__(self, nodeid: Union[str, TestReport], xml: "LogXML") -> None: + self.id = nodeid + self.xml = xml + self.add_stats = self.xml.add_stats + self.family = self.xml.family + self.duration = 0 + self.properties: List[Tuple[str, str]] = [] + self.nodes: List[ET.Element] = [] + self.attrs: Dict[str, str] = {} + + def append(self, node: ET.Element) -> None: + self.xml.add_stats(node.tag) + self.nodes.append(node) + + def add_property(self, name: str, value: object) -> None: + self.properties.append((str(name), bin_xml_escape(value))) + + def add_attribute(self, name: str, value: object) -> None: + self.attrs[str(name)] = bin_xml_escape(value) + + def make_properties_node(self) -> Optional[ET.Element]: + """Return a Junit node containing custom properties, if any.""" + if self.properties: + properties = ET.Element("properties") + for name, value in self.properties: + properties.append(ET.Element("property", name=name, value=value)) + return properties + return None + + def record_testreport(self, testreport: TestReport) -> None: + names = mangle_test_address(testreport.nodeid) + existing_attrs = self.attrs + classnames = names[:-1] + if self.xml.prefix: + classnames.insert(0, self.xml.prefix) + attrs: Dict[str, str] = { + "classname": ".".join(classnames), + "name": bin_xml_escape(names[-1]), + "file": testreport.location[0], + } + if testreport.location[1] is not None: + attrs["line"] = str(testreport.location[1]) + if hasattr(testreport, "url"): + attrs["url"] = testreport.url + self.attrs = attrs + self.attrs.update(existing_attrs) # Restore any user-defined attributes. + + # Preserve legacy testcase behavior. + if self.family == "xunit1": + return + + # Filter out attributes not permitted by this test family. + # Including custom attributes because they are not valid here. + temp_attrs = {} + for key in self.attrs.keys(): + if key in families[self.family]["testcase"]: + temp_attrs[key] = self.attrs[key] + self.attrs = temp_attrs + + def to_xml(self) -> ET.Element: + testcase = ET.Element("testcase", self.attrs, time="%.3f" % self.duration) + properties = self.make_properties_node() + if properties is not None: + testcase.append(properties) + testcase.extend(self.nodes) + return testcase + + def _add_simple(self, tag: str, message: str, data: Optional[str] = None) -> None: + node = ET.Element(tag, message=message) + node.text = bin_xml_escape(data) + self.append(node) + + def write_captured_output(self, report: TestReport) -> None: + if not self.xml.log_passing_tests and report.passed: + return + + content_out = report.capstdout + content_log = report.caplog + content_err = report.capstderr + if self.xml.logging == "no": + return + content_all = "" + if self.xml.logging in ["log", "all"]: + content_all = self._prepare_content(content_log, " Captured Log ") + if self.xml.logging in ["system-out", "out-err", "all"]: + content_all += self._prepare_content(content_out, " Captured Out ") + self._write_content(report, content_all, "system-out") + content_all = "" + if self.xml.logging in ["system-err", "out-err", "all"]: + content_all += self._prepare_content(content_err, " Captured Err ") + self._write_content(report, content_all, "system-err") + content_all = "" + if content_all: + self._write_content(report, content_all, "system-out") + + def _prepare_content(self, content: str, header: str) -> str: + return "\n".join([header.center(80, "-"), content, ""]) + + def _write_content(self, report: TestReport, content: str, jheader: str) -> None: + tag = ET.Element(jheader) + tag.text = bin_xml_escape(content) + self.append(tag) + + def append_pass(self, report: TestReport) -> None: + self.add_stats("passed") + + def append_failure(self, report: TestReport) -> None: + # msg = str(report.longrepr.reprtraceback.extraline) + if hasattr(report, "wasxfail"): + self._add_simple("skipped", "xfail-marked test passes unexpectedly") + else: + assert report.longrepr is not None + reprcrash: Optional[ReprFileLocation] = getattr( + report.longrepr, "reprcrash", None + ) + if reprcrash is not None: + message = reprcrash.message + else: + message = str(report.longrepr) + message = bin_xml_escape(message) + self._add_simple("failure", message, str(report.longrepr)) + + def append_collect_error(self, report: TestReport) -> None: + # msg = str(report.longrepr.reprtraceback.extraline) + assert report.longrepr is not None + self._add_simple("error", "collection failure", str(report.longrepr)) + + def append_collect_skipped(self, report: TestReport) -> None: + self._add_simple("skipped", "collection skipped", str(report.longrepr)) + + def append_error(self, report: TestReport) -> None: + assert report.longrepr is not None + reprcrash: Optional[ReprFileLocation] = getattr( + report.longrepr, "reprcrash", None + ) + if reprcrash is not None: + reason = reprcrash.message + else: + reason = str(report.longrepr) + + if report.when == "teardown": + msg = f'failed on teardown with "{reason}"' + else: + msg = f'failed on setup with "{reason}"' + self._add_simple("error", msg, str(report.longrepr)) + + def append_skipped(self, report: TestReport) -> None: + if hasattr(report, "wasxfail"): + xfailreason = report.wasxfail + if xfailreason.startswith("reason: "): + xfailreason = xfailreason[8:] + xfailreason = bin_xml_escape(xfailreason) + skipped = ET.Element("skipped", type="pytest.xfail", message=xfailreason) + self.append(skipped) + else: + assert isinstance(report.longrepr, tuple) + filename, lineno, skipreason = report.longrepr + if skipreason.startswith("Skipped: "): + skipreason = skipreason[9:] + details = f"{filename}:{lineno}: {skipreason}" + + skipped = ET.Element("skipped", type="pytest.skip", message=skipreason) + skipped.text = bin_xml_escape(details) + self.append(skipped) + self.write_captured_output(report) + + def finalize(self) -> None: + data = self.to_xml() + self.__dict__.clear() + # Type ignored because mypy doesn't like overriding a method. + # Also the return value doesn't match... + self.to_xml = lambda: data # type: ignore[assignment] + + +def _warn_incompatibility_with_xunit2( + request: FixtureRequest, fixture_name: str +) -> None: + """Emit a PytestWarning about the given fixture being incompatible with newer xunit revisions.""" + from _pytest.warning_types import PytestWarning + + xml = request.config.stash.get(xml_key, None) + if xml is not None and xml.family not in ("xunit1", "legacy"): + request.node.warn( + PytestWarning( + "{fixture_name} is incompatible with junit_family '{family}' (use 'legacy' or 'xunit1')".format( + fixture_name=fixture_name, family=xml.family + ) + ) + ) + + +@pytest.fixture +def record_property(request: FixtureRequest) -> Callable[[str, object], None]: + """Add extra properties to the calling test. + + User properties become part of the test report and are available to the + configured reporters, like JUnit XML. + + The fixture is callable with ``name, value``. The value is automatically + XML-encoded. + + Example:: + + def test_function(record_property): + record_property("example_key", 1) + """ + _warn_incompatibility_with_xunit2(request, "record_property") + + def append_property(name: str, value: object) -> None: + request.node.user_properties.append((name, value)) + + return append_property + + +@pytest.fixture +def record_xml_attribute(request: FixtureRequest) -> Callable[[str, object], None]: + """Add extra xml attributes to the tag for the calling test. + + The fixture is callable with ``name, value``. The value is + automatically XML-encoded. + """ + from _pytest.warning_types import PytestExperimentalApiWarning + + request.node.warn( + PytestExperimentalApiWarning("record_xml_attribute is an experimental feature") + ) + + _warn_incompatibility_with_xunit2(request, "record_xml_attribute") + + # Declare noop + def add_attr_noop(name: str, value: object) -> None: + pass + + attr_func = add_attr_noop + + xml = request.config.stash.get(xml_key, None) + if xml is not None: + node_reporter = xml.node_reporter(request.node.nodeid) + attr_func = node_reporter.add_attribute + + return attr_func + + +def _check_record_param_type(param: str, v: str) -> None: + """Used by record_testsuite_property to check that the given parameter name is of the proper + type.""" + __tracebackhide__ = True + if not isinstance(v, str): + msg = "{param} parameter needs to be a string, but {g} given" # type: ignore[unreachable] + raise TypeError(msg.format(param=param, g=type(v).__name__)) + + +@pytest.fixture(scope="session") +def record_testsuite_property(request: FixtureRequest) -> Callable[[str, object], None]: + """Record a new ```` tag as child of the root ````. + + This is suitable to writing global information regarding the entire test + suite, and is compatible with ``xunit2`` JUnit family. + + This is a ``session``-scoped fixture which is called with ``(name, value)``. Example: + + .. code-block:: python + + def test_foo(record_testsuite_property): + record_testsuite_property("ARCH", "PPC") + record_testsuite_property("STORAGE_TYPE", "CEPH") + + ``name`` must be a string, ``value`` will be converted to a string and properly xml-escaped. + + .. warning:: + + Currently this fixture **does not work** with the + `pytest-xdist `__ plugin. See + :issue:`7767` for details. + """ + + __tracebackhide__ = True + + def record_func(name: str, value: object) -> None: + """No-op function in case --junitxml was not passed in the command-line.""" + __tracebackhide__ = True + _check_record_param_type("name", name) + + xml = request.config.stash.get(xml_key, None) + if xml is not None: + record_func = xml.add_global_property # noqa + return record_func + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("terminal reporting") + group.addoption( + "--junitxml", + "--junit-xml", + action="store", + dest="xmlpath", + metavar="path", + type=functools.partial(filename_arg, optname="--junitxml"), + default=None, + help="create junit-xml style report file at given path.", + ) + group.addoption( + "--junitprefix", + "--junit-prefix", + action="store", + metavar="str", + default=None, + help="prepend prefix to classnames in junit-xml output", + ) + parser.addini( + "junit_suite_name", "Test suite name for JUnit report", default="pytest" + ) + parser.addini( + "junit_logging", + "Write captured log messages to JUnit report: " + "one of no|log|system-out|system-err|out-err|all", + default="no", + ) + parser.addini( + "junit_log_passing_tests", + "Capture log information for passing tests to JUnit report: ", + type="bool", + default=True, + ) + parser.addini( + "junit_duration_report", + "Duration time to report: one of total|call", + default="total", + ) # choices=['total', 'call']) + parser.addini( + "junit_family", + "Emit XML for schema: one of legacy|xunit1|xunit2", + default="xunit2", + ) + + +def pytest_configure(config: Config) -> None: + xmlpath = config.option.xmlpath + # Prevent opening xmllog on worker nodes (xdist). + if xmlpath and not hasattr(config, "workerinput"): + junit_family = config.getini("junit_family") + config.stash[xml_key] = LogXML( + xmlpath, + config.option.junitprefix, + config.getini("junit_suite_name"), + config.getini("junit_logging"), + config.getini("junit_duration_report"), + junit_family, + config.getini("junit_log_passing_tests"), + ) + config.pluginmanager.register(config.stash[xml_key]) + + +def pytest_unconfigure(config: Config) -> None: + xml = config.stash.get(xml_key, None) + if xml: + del config.stash[xml_key] + config.pluginmanager.unregister(xml) + + +def mangle_test_address(address: str) -> List[str]: + path, possible_open_bracket, params = address.partition("[") + names = path.split("::") + # Convert file path to dotted path. + names[0] = names[0].replace(nodes.SEP, ".") + names[0] = re.sub(r"\.py$", "", names[0]) + # Put any params back. + names[-1] += possible_open_bracket + params + return names + + +class LogXML: + def __init__( + self, + logfile, + prefix: Optional[str], + suite_name: str = "pytest", + logging: str = "no", + report_duration: str = "total", + family="xunit1", + log_passing_tests: bool = True, + ) -> None: + logfile = os.path.expanduser(os.path.expandvars(logfile)) + self.logfile = os.path.normpath(os.path.abspath(logfile)) + self.prefix = prefix + self.suite_name = suite_name + self.logging = logging + self.log_passing_tests = log_passing_tests + self.report_duration = report_duration + self.family = family + self.stats: Dict[str, int] = dict.fromkeys( + ["error", "passed", "failure", "skipped"], 0 + ) + self.node_reporters: Dict[ + Tuple[Union[str, TestReport], object], _NodeReporter + ] = {} + self.node_reporters_ordered: List[_NodeReporter] = [] + self.global_properties: List[Tuple[str, str]] = [] + + # List of reports that failed on call but teardown is pending. + self.open_reports: List[TestReport] = [] + self.cnt_double_fail_tests = 0 + + # Replaces convenience family with real family. + if self.family == "legacy": + self.family = "xunit1" + + def finalize(self, report: TestReport) -> None: + nodeid = getattr(report, "nodeid", report) + # Local hack to handle xdist report order. + workernode = getattr(report, "node", None) + reporter = self.node_reporters.pop((nodeid, workernode)) + if reporter is not None: + reporter.finalize() + + def node_reporter(self, report: Union[TestReport, str]) -> _NodeReporter: + nodeid: Union[str, TestReport] = getattr(report, "nodeid", report) + # Local hack to handle xdist report order. + workernode = getattr(report, "node", None) + + key = nodeid, workernode + + if key in self.node_reporters: + # TODO: breaks for --dist=each + return self.node_reporters[key] + + reporter = _NodeReporter(nodeid, self) + + self.node_reporters[key] = reporter + self.node_reporters_ordered.append(reporter) + + return reporter + + def add_stats(self, key: str) -> None: + if key in self.stats: + self.stats[key] += 1 + + def _opentestcase(self, report: TestReport) -> _NodeReporter: + reporter = self.node_reporter(report) + reporter.record_testreport(report) + return reporter + + def pytest_runtest_logreport(self, report: TestReport) -> None: + """Handle a setup/call/teardown report, generating the appropriate + XML tags as necessary. + + Note: due to plugins like xdist, this hook may be called in interlaced + order with reports from other nodes. For example: + + Usual call order: + -> setup node1 + -> call node1 + -> teardown node1 + -> setup node2 + -> call node2 + -> teardown node2 + + Possible call order in xdist: + -> setup node1 + -> call node1 + -> setup node2 + -> call node2 + -> teardown node2 + -> teardown node1 + """ + close_report = None + if report.passed: + if report.when == "call": # ignore setup/teardown + reporter = self._opentestcase(report) + reporter.append_pass(report) + elif report.failed: + if report.when == "teardown": + # The following vars are needed when xdist plugin is used. + report_wid = getattr(report, "worker_id", None) + report_ii = getattr(report, "item_index", None) + close_report = next( + ( + rep + for rep in self.open_reports + if ( + rep.nodeid == report.nodeid + and getattr(rep, "item_index", None) == report_ii + and getattr(rep, "worker_id", None) == report_wid + ) + ), + None, + ) + if close_report: + # We need to open new testcase in case we have failure in + # call and error in teardown in order to follow junit + # schema. + self.finalize(close_report) + self.cnt_double_fail_tests += 1 + reporter = self._opentestcase(report) + if report.when == "call": + reporter.append_failure(report) + self.open_reports.append(report) + if not self.log_passing_tests: + reporter.write_captured_output(report) + else: + reporter.append_error(report) + elif report.skipped: + reporter = self._opentestcase(report) + reporter.append_skipped(report) + self.update_testcase_duration(report) + if report.when == "teardown": + reporter = self._opentestcase(report) + reporter.write_captured_output(report) + + for propname, propvalue in report.user_properties: + reporter.add_property(propname, str(propvalue)) + + self.finalize(report) + report_wid = getattr(report, "worker_id", None) + report_ii = getattr(report, "item_index", None) + close_report = next( + ( + rep + for rep in self.open_reports + if ( + rep.nodeid == report.nodeid + and getattr(rep, "item_index", None) == report_ii + and getattr(rep, "worker_id", None) == report_wid + ) + ), + None, + ) + if close_report: + self.open_reports.remove(close_report) + + def update_testcase_duration(self, report: TestReport) -> None: + """Accumulate total duration for nodeid from given report and update + the Junit.testcase with the new total if already created.""" + if self.report_duration == "total" or report.when == self.report_duration: + reporter = self.node_reporter(report) + reporter.duration += getattr(report, "duration", 0.0) + + def pytest_collectreport(self, report: TestReport) -> None: + if not report.passed: + reporter = self._opentestcase(report) + if report.failed: + reporter.append_collect_error(report) + else: + reporter.append_collect_skipped(report) + + def pytest_internalerror(self, excrepr: ExceptionRepr) -> None: + reporter = self.node_reporter("internal") + reporter.attrs.update(classname="pytest", name="internal") + reporter._add_simple("error", "internal error", str(excrepr)) + + def pytest_sessionstart(self) -> None: + self.suite_start_time = timing.time() + + def pytest_sessionfinish(self) -> None: + dirname = os.path.dirname(os.path.abspath(self.logfile)) + if not os.path.isdir(dirname): + os.makedirs(dirname) + + with open(self.logfile, "w", encoding="utf-8") as logfile: + suite_stop_time = timing.time() + suite_time_delta = suite_stop_time - self.suite_start_time + + numtests = ( + self.stats["passed"] + + self.stats["failure"] + + self.stats["skipped"] + + self.stats["error"] + - self.cnt_double_fail_tests + ) + logfile.write('') + + suite_node = ET.Element( + "testsuite", + name=self.suite_name, + errors=str(self.stats["error"]), + failures=str(self.stats["failure"]), + skipped=str(self.stats["skipped"]), + tests=str(numtests), + time="%.3f" % suite_time_delta, + timestamp=datetime.fromtimestamp(self.suite_start_time).isoformat(), + hostname=platform.node(), + ) + global_properties = self._get_global_properties_node() + if global_properties is not None: + suite_node.append(global_properties) + for node_reporter in self.node_reporters_ordered: + suite_node.append(node_reporter.to_xml()) + testsuites = ET.Element("testsuites") + testsuites.append(suite_node) + logfile.write(ET.tostring(testsuites, encoding="unicode")) + + def pytest_terminal_summary(self, terminalreporter: TerminalReporter) -> None: + terminalreporter.write_sep("-", f"generated xml file: {self.logfile}") + + def add_global_property(self, name: str, value: object) -> None: + __tracebackhide__ = True + _check_record_param_type("name", name) + self.global_properties.append((name, bin_xml_escape(value))) + + def _get_global_properties_node(self) -> Optional[ET.Element]: + """Return a Junit node containing custom properties, if any.""" + if self.global_properties: + properties = ET.Element("properties") + for name, value in self.global_properties: + properties.append(ET.Element("property", name=name, value=value)) + return properties + return None diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/legacypath.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/legacypath.py new file mode 100644 index 0000000000..37e8c24220 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/legacypath.py @@ -0,0 +1,467 @@ +"""Add backward compatibility support for the legacy py path type.""" +import shlex +import subprocess +from pathlib import Path +from typing import List +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union + +import attr +from iniconfig import SectionWrapper + +from _pytest.cacheprovider import Cache +from _pytest.compat import final +from _pytest.compat import LEGACY_PATH +from _pytest.compat import legacy_path +from _pytest.config import Config +from _pytest.config import hookimpl +from _pytest.config import PytestPluginManager +from _pytest.deprecated import check_ispytest +from _pytest.fixtures import fixture +from _pytest.fixtures import FixtureRequest +from _pytest.main import Session +from _pytest.monkeypatch import MonkeyPatch +from _pytest.nodes import Collector +from _pytest.nodes import Item +from _pytest.nodes import Node +from _pytest.pytester import HookRecorder +from _pytest.pytester import Pytester +from _pytest.pytester import RunResult +from _pytest.terminal import TerminalReporter +from _pytest.tmpdir import TempPathFactory + +if TYPE_CHECKING: + from typing_extensions import Final + + import pexpect + + +@final +class Testdir: + """ + Similar to :class:`Pytester`, but this class works with legacy legacy_path objects instead. + + All methods just forward to an internal :class:`Pytester` instance, converting results + to `legacy_path` objects as necessary. + """ + + __test__ = False + + CLOSE_STDIN: "Final" = Pytester.CLOSE_STDIN + TimeoutExpired: "Final" = Pytester.TimeoutExpired + + def __init__(self, pytester: Pytester, *, _ispytest: bool = False) -> None: + check_ispytest(_ispytest) + self._pytester = pytester + + @property + def tmpdir(self) -> LEGACY_PATH: + """Temporary directory where tests are executed.""" + return legacy_path(self._pytester.path) + + @property + def test_tmproot(self) -> LEGACY_PATH: + return legacy_path(self._pytester._test_tmproot) + + @property + def request(self): + return self._pytester._request + + @property + def plugins(self): + return self._pytester.plugins + + @plugins.setter + def plugins(self, plugins): + self._pytester.plugins = plugins + + @property + def monkeypatch(self) -> MonkeyPatch: + return self._pytester._monkeypatch + + def make_hook_recorder(self, pluginmanager) -> HookRecorder: + """See :meth:`Pytester.make_hook_recorder`.""" + return self._pytester.make_hook_recorder(pluginmanager) + + def chdir(self) -> None: + """See :meth:`Pytester.chdir`.""" + return self._pytester.chdir() + + def finalize(self) -> None: + """See :meth:`Pytester._finalize`.""" + return self._pytester._finalize() + + def makefile(self, ext, *args, **kwargs) -> LEGACY_PATH: + """See :meth:`Pytester.makefile`.""" + if ext and not ext.startswith("."): + # pytester.makefile is going to throw a ValueError in a way that + # testdir.makefile did not, because + # pathlib.Path is stricter suffixes than py.path + # This ext arguments is likely user error, but since testdir has + # allowed this, we will prepend "." as a workaround to avoid breaking + # testdir usage that worked before + ext = "." + ext + return legacy_path(self._pytester.makefile(ext, *args, **kwargs)) + + def makeconftest(self, source) -> LEGACY_PATH: + """See :meth:`Pytester.makeconftest`.""" + return legacy_path(self._pytester.makeconftest(source)) + + def makeini(self, source) -> LEGACY_PATH: + """See :meth:`Pytester.makeini`.""" + return legacy_path(self._pytester.makeini(source)) + + def getinicfg(self, source: str) -> SectionWrapper: + """See :meth:`Pytester.getinicfg`.""" + return self._pytester.getinicfg(source) + + def makepyprojecttoml(self, source) -> LEGACY_PATH: + """See :meth:`Pytester.makepyprojecttoml`.""" + return legacy_path(self._pytester.makepyprojecttoml(source)) + + def makepyfile(self, *args, **kwargs) -> LEGACY_PATH: + """See :meth:`Pytester.makepyfile`.""" + return legacy_path(self._pytester.makepyfile(*args, **kwargs)) + + def maketxtfile(self, *args, **kwargs) -> LEGACY_PATH: + """See :meth:`Pytester.maketxtfile`.""" + return legacy_path(self._pytester.maketxtfile(*args, **kwargs)) + + def syspathinsert(self, path=None) -> None: + """See :meth:`Pytester.syspathinsert`.""" + return self._pytester.syspathinsert(path) + + def mkdir(self, name) -> LEGACY_PATH: + """See :meth:`Pytester.mkdir`.""" + return legacy_path(self._pytester.mkdir(name)) + + def mkpydir(self, name) -> LEGACY_PATH: + """See :meth:`Pytester.mkpydir`.""" + return legacy_path(self._pytester.mkpydir(name)) + + def copy_example(self, name=None) -> LEGACY_PATH: + """See :meth:`Pytester.copy_example`.""" + return legacy_path(self._pytester.copy_example(name)) + + def getnode(self, config: Config, arg) -> Optional[Union[Item, Collector]]: + """See :meth:`Pytester.getnode`.""" + return self._pytester.getnode(config, arg) + + def getpathnode(self, path): + """See :meth:`Pytester.getpathnode`.""" + return self._pytester.getpathnode(path) + + def genitems(self, colitems: List[Union[Item, Collector]]) -> List[Item]: + """See :meth:`Pytester.genitems`.""" + return self._pytester.genitems(colitems) + + def runitem(self, source): + """See :meth:`Pytester.runitem`.""" + return self._pytester.runitem(source) + + def inline_runsource(self, source, *cmdlineargs): + """See :meth:`Pytester.inline_runsource`.""" + return self._pytester.inline_runsource(source, *cmdlineargs) + + def inline_genitems(self, *args): + """See :meth:`Pytester.inline_genitems`.""" + return self._pytester.inline_genitems(*args) + + def inline_run(self, *args, plugins=(), no_reraise_ctrlc: bool = False): + """See :meth:`Pytester.inline_run`.""" + return self._pytester.inline_run( + *args, plugins=plugins, no_reraise_ctrlc=no_reraise_ctrlc + ) + + def runpytest_inprocess(self, *args, **kwargs) -> RunResult: + """See :meth:`Pytester.runpytest_inprocess`.""" + return self._pytester.runpytest_inprocess(*args, **kwargs) + + def runpytest(self, *args, **kwargs) -> RunResult: + """See :meth:`Pytester.runpytest`.""" + return self._pytester.runpytest(*args, **kwargs) + + def parseconfig(self, *args) -> Config: + """See :meth:`Pytester.parseconfig`.""" + return self._pytester.parseconfig(*args) + + def parseconfigure(self, *args) -> Config: + """See :meth:`Pytester.parseconfigure`.""" + return self._pytester.parseconfigure(*args) + + def getitem(self, source, funcname="test_func"): + """See :meth:`Pytester.getitem`.""" + return self._pytester.getitem(source, funcname) + + def getitems(self, source): + """See :meth:`Pytester.getitems`.""" + return self._pytester.getitems(source) + + def getmodulecol(self, source, configargs=(), withinit=False): + """See :meth:`Pytester.getmodulecol`.""" + return self._pytester.getmodulecol( + source, configargs=configargs, withinit=withinit + ) + + def collect_by_name( + self, modcol: Collector, name: str + ) -> Optional[Union[Item, Collector]]: + """See :meth:`Pytester.collect_by_name`.""" + return self._pytester.collect_by_name(modcol, name) + + def popen( + self, + cmdargs, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=CLOSE_STDIN, + **kw, + ): + """See :meth:`Pytester.popen`.""" + return self._pytester.popen(cmdargs, stdout, stderr, stdin, **kw) + + def run(self, *cmdargs, timeout=None, stdin=CLOSE_STDIN) -> RunResult: + """See :meth:`Pytester.run`.""" + return self._pytester.run(*cmdargs, timeout=timeout, stdin=stdin) + + def runpython(self, script) -> RunResult: + """See :meth:`Pytester.runpython`.""" + return self._pytester.runpython(script) + + def runpython_c(self, command): + """See :meth:`Pytester.runpython_c`.""" + return self._pytester.runpython_c(command) + + def runpytest_subprocess(self, *args, timeout=None) -> RunResult: + """See :meth:`Pytester.runpytest_subprocess`.""" + return self._pytester.runpytest_subprocess(*args, timeout=timeout) + + def spawn_pytest( + self, string: str, expect_timeout: float = 10.0 + ) -> "pexpect.spawn": + """See :meth:`Pytester.spawn_pytest`.""" + return self._pytester.spawn_pytest(string, expect_timeout=expect_timeout) + + def spawn(self, cmd: str, expect_timeout: float = 10.0) -> "pexpect.spawn": + """See :meth:`Pytester.spawn`.""" + return self._pytester.spawn(cmd, expect_timeout=expect_timeout) + + def __repr__(self) -> str: + return f"" + + def __str__(self) -> str: + return str(self.tmpdir) + + +class LegacyTestdirPlugin: + @staticmethod + @fixture + def testdir(pytester: Pytester) -> Testdir: + """ + Identical to :fixture:`pytester`, and provides an instance whose methods return + legacy ``LEGACY_PATH`` objects instead when applicable. + + New code should avoid using :fixture:`testdir` in favor of :fixture:`pytester`. + """ + return Testdir(pytester, _ispytest=True) + + +@final +@attr.s(init=False, auto_attribs=True) +class TempdirFactory: + """Backward compatibility wrapper that implements :class:``_pytest.compat.LEGACY_PATH`` + for :class:``TempPathFactory``.""" + + _tmppath_factory: TempPathFactory + + def __init__( + self, tmppath_factory: TempPathFactory, *, _ispytest: bool = False + ) -> None: + check_ispytest(_ispytest) + self._tmppath_factory = tmppath_factory + + def mktemp(self, basename: str, numbered: bool = True) -> LEGACY_PATH: + """Same as :meth:`TempPathFactory.mktemp`, but returns a ``_pytest.compat.LEGACY_PATH`` object.""" + return legacy_path(self._tmppath_factory.mktemp(basename, numbered).resolve()) + + def getbasetemp(self) -> LEGACY_PATH: + """Backward compat wrapper for ``_tmppath_factory.getbasetemp``.""" + return legacy_path(self._tmppath_factory.getbasetemp().resolve()) + + +class LegacyTmpdirPlugin: + @staticmethod + @fixture(scope="session") + def tmpdir_factory(request: FixtureRequest) -> TempdirFactory: + """Return a :class:`pytest.TempdirFactory` instance for the test session.""" + # Set dynamically by pytest_configure(). + return request.config._tmpdirhandler # type: ignore + + @staticmethod + @fixture + def tmpdir(tmp_path: Path) -> LEGACY_PATH: + """Return a temporary directory path object which is unique to each test + function invocation, created as a sub directory of the base temporary + directory. + + By default, a new base temporary directory is created each test session, + and old bases are removed after 3 sessions, to aid in debugging. If + ``--basetemp`` is used then it is cleared each session. See :ref:`base + temporary directory`. + + The returned object is a `legacy_path`_ object. + + .. _legacy_path: https://py.readthedocs.io/en/latest/path.html + """ + return legacy_path(tmp_path) + + +def Cache_makedir(self: Cache, name: str) -> LEGACY_PATH: + """Return a directory path object with the given name. + + Same as :func:`mkdir`, but returns a legacy py path instance. + """ + return legacy_path(self.mkdir(name)) + + +def FixtureRequest_fspath(self: FixtureRequest) -> LEGACY_PATH: + """(deprecated) The file system path of the test module which collected this test.""" + return legacy_path(self.path) + + +def TerminalReporter_startdir(self: TerminalReporter) -> LEGACY_PATH: + """The directory from which pytest was invoked. + + Prefer to use ``startpath`` which is a :class:`pathlib.Path`. + + :type: LEGACY_PATH + """ + return legacy_path(self.startpath) + + +def Config_invocation_dir(self: Config) -> LEGACY_PATH: + """The directory from which pytest was invoked. + + Prefer to use :attr:`invocation_params.dir `, + which is a :class:`pathlib.Path`. + + :type: LEGACY_PATH + """ + return legacy_path(str(self.invocation_params.dir)) + + +def Config_rootdir(self: Config) -> LEGACY_PATH: + """The path to the :ref:`rootdir `. + + Prefer to use :attr:`rootpath`, which is a :class:`pathlib.Path`. + + :type: LEGACY_PATH + """ + return legacy_path(str(self.rootpath)) + + +def Config_inifile(self: Config) -> Optional[LEGACY_PATH]: + """The path to the :ref:`configfile `. + + Prefer to use :attr:`inipath`, which is a :class:`pathlib.Path`. + + :type: Optional[LEGACY_PATH] + """ + return legacy_path(str(self.inipath)) if self.inipath else None + + +def Session_stardir(self: Session) -> LEGACY_PATH: + """The path from which pytest was invoked. + + Prefer to use ``startpath`` which is a :class:`pathlib.Path`. + + :type: LEGACY_PATH + """ + return legacy_path(self.startpath) + + +def Config__getini_unknown_type( + self, name: str, type: str, value: Union[str, List[str]] +): + if type == "pathlist": + # TODO: This assert is probably not valid in all cases. + assert self.inipath is not None + dp = self.inipath.parent + input_values = shlex.split(value) if isinstance(value, str) else value + return [legacy_path(str(dp / x)) for x in input_values] + else: + raise ValueError(f"unknown configuration type: {type}", value) + + +def Node_fspath(self: Node) -> LEGACY_PATH: + """(deprecated) returns a legacy_path copy of self.path""" + return legacy_path(self.path) + + +def Node_fspath_set(self: Node, value: LEGACY_PATH) -> None: + self.path = Path(value) + + +@hookimpl(tryfirst=True) +def pytest_load_initial_conftests(early_config: Config) -> None: + """Monkeypatch legacy path attributes in several classes, as early as possible.""" + mp = MonkeyPatch() + early_config.add_cleanup(mp.undo) + + # Add Cache.makedir(). + mp.setattr(Cache, "makedir", Cache_makedir, raising=False) + + # Add FixtureRequest.fspath property. + mp.setattr(FixtureRequest, "fspath", property(FixtureRequest_fspath), raising=False) + + # Add TerminalReporter.startdir property. + mp.setattr( + TerminalReporter, "startdir", property(TerminalReporter_startdir), raising=False + ) + + # Add Config.{invocation_dir,rootdir,inifile} properties. + mp.setattr(Config, "invocation_dir", property(Config_invocation_dir), raising=False) + mp.setattr(Config, "rootdir", property(Config_rootdir), raising=False) + mp.setattr(Config, "inifile", property(Config_inifile), raising=False) + + # Add Session.startdir property. + mp.setattr(Session, "startdir", property(Session_stardir), raising=False) + + # Add pathlist configuration type. + mp.setattr(Config, "_getini_unknown_type", Config__getini_unknown_type) + + # Add Node.fspath property. + mp.setattr(Node, "fspath", property(Node_fspath, Node_fspath_set), raising=False) + + +@hookimpl +def pytest_configure(config: Config) -> None: + """Installs the LegacyTmpdirPlugin if the ``tmpdir`` plugin is also installed.""" + if config.pluginmanager.has_plugin("tmpdir"): + mp = MonkeyPatch() + config.add_cleanup(mp.undo) + # Create TmpdirFactory and attach it to the config object. + # + # This is to comply with existing plugins which expect the handler to be + # available at pytest_configure time, but ideally should be moved entirely + # to the tmpdir_factory session fixture. + try: + tmp_path_factory = config._tmp_path_factory # type: ignore[attr-defined] + except AttributeError: + # tmpdir plugin is blocked. + pass + else: + _tmpdirhandler = TempdirFactory(tmp_path_factory, _ispytest=True) + mp.setattr(config, "_tmpdirhandler", _tmpdirhandler, raising=False) + + config.pluginmanager.register(LegacyTmpdirPlugin, "legacypath-tmpdir") + + +@hookimpl +def pytest_plugin_registered(plugin: object, manager: PytestPluginManager) -> None: + # pytester is not loaded by default and is commonly loaded from a conftest, + # so checking for it in `pytest_configure` is not enough. + is_pytester = plugin is manager.get_plugin("pytester") + if is_pytester and not manager.is_registered(LegacyTestdirPlugin): + manager.register(LegacyTestdirPlugin, "legacypath-pytester") diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/logging.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/logging.py new file mode 100644 index 0000000000..31ad830107 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/logging.py @@ -0,0 +1,831 @@ +"""Access and control log capturing.""" +import logging +import os +import re +import sys +from contextlib import contextmanager +from io import StringIO +from pathlib import Path +from typing import AbstractSet +from typing import Dict +from typing import Generator +from typing import List +from typing import Mapping +from typing import Optional +from typing import Tuple +from typing import TypeVar +from typing import Union + +from _pytest import nodes +from _pytest._io import TerminalWriter +from _pytest.capture import CaptureManager +from _pytest.compat import final +from _pytest.compat import nullcontext +from _pytest.config import _strtobool +from _pytest.config import Config +from _pytest.config import create_terminal_writer +from _pytest.config import hookimpl +from _pytest.config import UsageError +from _pytest.config.argparsing import Parser +from _pytest.deprecated import check_ispytest +from _pytest.fixtures import fixture +from _pytest.fixtures import FixtureRequest +from _pytest.main import Session +from _pytest.stash import StashKey +from _pytest.terminal import TerminalReporter + + +DEFAULT_LOG_FORMAT = "%(levelname)-8s %(name)s:%(filename)s:%(lineno)d %(message)s" +DEFAULT_LOG_DATE_FORMAT = "%H:%M:%S" +_ANSI_ESCAPE_SEQ = re.compile(r"\x1b\[[\d;]+m") +caplog_handler_key = StashKey["LogCaptureHandler"]() +caplog_records_key = StashKey[Dict[str, List[logging.LogRecord]]]() + + +def _remove_ansi_escape_sequences(text: str) -> str: + return _ANSI_ESCAPE_SEQ.sub("", text) + + +class ColoredLevelFormatter(logging.Formatter): + """A logging formatter which colorizes the %(levelname)..s part of the + log format passed to __init__.""" + + LOGLEVEL_COLOROPTS: Mapping[int, AbstractSet[str]] = { + logging.CRITICAL: {"red"}, + logging.ERROR: {"red", "bold"}, + logging.WARNING: {"yellow"}, + logging.WARN: {"yellow"}, + logging.INFO: {"green"}, + logging.DEBUG: {"purple"}, + logging.NOTSET: set(), + } + LEVELNAME_FMT_REGEX = re.compile(r"%\(levelname\)([+-.]?\d*(?:\.\d+)?s)") + + def __init__(self, terminalwriter: TerminalWriter, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._terminalwriter = terminalwriter + self._original_fmt = self._style._fmt + self._level_to_fmt_mapping: Dict[int, str] = {} + + for level, color_opts in self.LOGLEVEL_COLOROPTS.items(): + self.add_color_level(level, *color_opts) + + def add_color_level(self, level: int, *color_opts: str) -> None: + """Add or update color opts for a log level. + + :param level: + Log level to apply a style to, e.g. ``logging.INFO``. + :param color_opts: + ANSI escape sequence color options. Capitalized colors indicates + background color, i.e. ``'green', 'Yellow', 'bold'`` will give bold + green text on yellow background. + + .. warning:: + This is an experimental API. + """ + + assert self._fmt is not None + levelname_fmt_match = self.LEVELNAME_FMT_REGEX.search(self._fmt) + if not levelname_fmt_match: + return + levelname_fmt = levelname_fmt_match.group() + + formatted_levelname = levelname_fmt % {"levelname": logging.getLevelName(level)} + + # add ANSI escape sequences around the formatted levelname + color_kwargs = {name: True for name in color_opts} + colorized_formatted_levelname = self._terminalwriter.markup( + formatted_levelname, **color_kwargs + ) + self._level_to_fmt_mapping[level] = self.LEVELNAME_FMT_REGEX.sub( + colorized_formatted_levelname, self._fmt + ) + + def format(self, record: logging.LogRecord) -> str: + fmt = self._level_to_fmt_mapping.get(record.levelno, self._original_fmt) + self._style._fmt = fmt + return super().format(record) + + +class PercentStyleMultiline(logging.PercentStyle): + """A logging style with special support for multiline messages. + + If the message of a record consists of multiple lines, this style + formats the message as if each line were logged separately. + """ + + def __init__(self, fmt: str, auto_indent: Union[int, str, bool, None]) -> None: + super().__init__(fmt) + self._auto_indent = self._get_auto_indent(auto_indent) + + @staticmethod + def _get_auto_indent(auto_indent_option: Union[int, str, bool, None]) -> int: + """Determine the current auto indentation setting. + + Specify auto indent behavior (on/off/fixed) by passing in + extra={"auto_indent": [value]} to the call to logging.log() or + using a --log-auto-indent [value] command line or the + log_auto_indent [value] config option. + + Default behavior is auto-indent off. + + Using the string "True" or "on" or the boolean True as the value + turns auto indent on, using the string "False" or "off" or the + boolean False or the int 0 turns it off, and specifying a + positive integer fixes the indentation position to the value + specified. + + Any other values for the option are invalid, and will silently be + converted to the default. + + :param None|bool|int|str auto_indent_option: + User specified option for indentation from command line, config + or extra kwarg. Accepts int, bool or str. str option accepts the + same range of values as boolean config options, as well as + positive integers represented in str form. + + :returns: + Indentation value, which can be + -1 (automatically determine indentation) or + 0 (auto-indent turned off) or + >0 (explicitly set indentation position). + """ + + if auto_indent_option is None: + return 0 + elif isinstance(auto_indent_option, bool): + if auto_indent_option: + return -1 + else: + return 0 + elif isinstance(auto_indent_option, int): + return int(auto_indent_option) + elif isinstance(auto_indent_option, str): + try: + return int(auto_indent_option) + except ValueError: + pass + try: + if _strtobool(auto_indent_option): + return -1 + except ValueError: + return 0 + + return 0 + + def format(self, record: logging.LogRecord) -> str: + if "\n" in record.message: + if hasattr(record, "auto_indent"): + # Passed in from the "extra={}" kwarg on the call to logging.log(). + auto_indent = self._get_auto_indent(record.auto_indent) # type: ignore[attr-defined] + else: + auto_indent = self._auto_indent + + if auto_indent: + lines = record.message.splitlines() + formatted = self._fmt % {**record.__dict__, "message": lines[0]} + + if auto_indent < 0: + indentation = _remove_ansi_escape_sequences(formatted).find( + lines[0] + ) + else: + # Optimizes logging by allowing a fixed indentation. + indentation = auto_indent + lines[0] = formatted + return ("\n" + " " * indentation).join(lines) + return self._fmt % record.__dict__ + + +def get_option_ini(config: Config, *names: str): + for name in names: + ret = config.getoption(name) # 'default' arg won't work as expected + if ret is None: + ret = config.getini(name) + if ret: + return ret + + +def pytest_addoption(parser: Parser) -> None: + """Add options to control log capturing.""" + group = parser.getgroup("logging") + + def add_option_ini(option, dest, default=None, type=None, **kwargs): + parser.addini( + dest, default=default, type=type, help="default value for " + option + ) + group.addoption(option, dest=dest, **kwargs) + + add_option_ini( + "--log-level", + dest="log_level", + default=None, + metavar="LEVEL", + help=( + "level of messages to catch/display.\n" + "Not set by default, so it depends on the root/parent log handler's" + ' effective level, where it is "WARNING" by default.' + ), + ) + add_option_ini( + "--log-format", + dest="log_format", + default=DEFAULT_LOG_FORMAT, + help="log format as used by the logging module.", + ) + add_option_ini( + "--log-date-format", + dest="log_date_format", + default=DEFAULT_LOG_DATE_FORMAT, + help="log date format as used by the logging module.", + ) + parser.addini( + "log_cli", + default=False, + type="bool", + help='enable log display during test run (also known as "live logging").', + ) + add_option_ini( + "--log-cli-level", dest="log_cli_level", default=None, help="cli logging level." + ) + add_option_ini( + "--log-cli-format", + dest="log_cli_format", + default=None, + help="log format as used by the logging module.", + ) + add_option_ini( + "--log-cli-date-format", + dest="log_cli_date_format", + default=None, + help="log date format as used by the logging module.", + ) + add_option_ini( + "--log-file", + dest="log_file", + default=None, + help="path to a file when logging will be written to.", + ) + add_option_ini( + "--log-file-level", + dest="log_file_level", + default=None, + help="log file logging level.", + ) + add_option_ini( + "--log-file-format", + dest="log_file_format", + default=DEFAULT_LOG_FORMAT, + help="log format as used by the logging module.", + ) + add_option_ini( + "--log-file-date-format", + dest="log_file_date_format", + default=DEFAULT_LOG_DATE_FORMAT, + help="log date format as used by the logging module.", + ) + add_option_ini( + "--log-auto-indent", + dest="log_auto_indent", + default=None, + help="Auto-indent multiline messages passed to the logging module. Accepts true|on, false|off or an integer.", + ) + + +_HandlerType = TypeVar("_HandlerType", bound=logging.Handler) + + +# Not using @contextmanager for performance reasons. +class catching_logs: + """Context manager that prepares the whole logging machinery properly.""" + + __slots__ = ("handler", "level", "orig_level") + + def __init__(self, handler: _HandlerType, level: Optional[int] = None) -> None: + self.handler = handler + self.level = level + + def __enter__(self): + root_logger = logging.getLogger() + if self.level is not None: + self.handler.setLevel(self.level) + root_logger.addHandler(self.handler) + if self.level is not None: + self.orig_level = root_logger.level + root_logger.setLevel(min(self.orig_level, self.level)) + return self.handler + + def __exit__(self, type, value, traceback): + root_logger = logging.getLogger() + if self.level is not None: + root_logger.setLevel(self.orig_level) + root_logger.removeHandler(self.handler) + + +class LogCaptureHandler(logging.StreamHandler): + """A logging handler that stores log records and the log text.""" + + stream: StringIO + + def __init__(self) -> None: + """Create a new log handler.""" + super().__init__(StringIO()) + self.records: List[logging.LogRecord] = [] + + def emit(self, record: logging.LogRecord) -> None: + """Keep the log records in a list in addition to the log text.""" + self.records.append(record) + super().emit(record) + + def reset(self) -> None: + self.records = [] + self.stream = StringIO() + + def handleError(self, record: logging.LogRecord) -> None: + if logging.raiseExceptions: + # Fail the test if the log message is bad (emit failed). + # The default behavior of logging is to print "Logging error" + # to stderr with the call stack and some extra details. + # pytest wants to make such mistakes visible during testing. + raise + + +@final +class LogCaptureFixture: + """Provides access and control of log capturing.""" + + def __init__(self, item: nodes.Node, *, _ispytest: bool = False) -> None: + check_ispytest(_ispytest) + self._item = item + self._initial_handler_level: Optional[int] = None + # Dict of log name -> log level. + self._initial_logger_levels: Dict[Optional[str], int] = {} + + def _finalize(self) -> None: + """Finalize the fixture. + + This restores the log levels changed by :meth:`set_level`. + """ + # Restore log levels. + if self._initial_handler_level is not None: + self.handler.setLevel(self._initial_handler_level) + for logger_name, level in self._initial_logger_levels.items(): + logger = logging.getLogger(logger_name) + logger.setLevel(level) + + @property + def handler(self) -> LogCaptureHandler: + """Get the logging handler used by the fixture. + + :rtype: LogCaptureHandler + """ + return self._item.stash[caplog_handler_key] + + def get_records(self, when: str) -> List[logging.LogRecord]: + """Get the logging records for one of the possible test phases. + + :param str when: + Which test phase to obtain the records from. Valid values are: "setup", "call" and "teardown". + + :returns: The list of captured records at the given stage. + :rtype: List[logging.LogRecord] + + .. versionadded:: 3.4 + """ + return self._item.stash[caplog_records_key].get(when, []) + + @property + def text(self) -> str: + """The formatted log text.""" + return _remove_ansi_escape_sequences(self.handler.stream.getvalue()) + + @property + def records(self) -> List[logging.LogRecord]: + """The list of log records.""" + return self.handler.records + + @property + def record_tuples(self) -> List[Tuple[str, int, str]]: + """A list of a stripped down version of log records intended + for use in assertion comparison. + + The format of the tuple is: + + (logger_name, log_level, message) + """ + return [(r.name, r.levelno, r.getMessage()) for r in self.records] + + @property + def messages(self) -> List[str]: + """A list of format-interpolated log messages. + + Unlike 'records', which contains the format string and parameters for + interpolation, log messages in this list are all interpolated. + + Unlike 'text', which contains the output from the handler, log + messages in this list are unadorned with levels, timestamps, etc, + making exact comparisons more reliable. + + Note that traceback or stack info (from :func:`logging.exception` or + the `exc_info` or `stack_info` arguments to the logging functions) is + not included, as this is added by the formatter in the handler. + + .. versionadded:: 3.7 + """ + return [r.getMessage() for r in self.records] + + def clear(self) -> None: + """Reset the list of log records and the captured log text.""" + self.handler.reset() + + def set_level(self, level: Union[int, str], logger: Optional[str] = None) -> None: + """Set the level of a logger for the duration of a test. + + .. versionchanged:: 3.4 + The levels of the loggers changed by this function will be + restored to their initial values at the end of the test. + + :param int level: The level. + :param str logger: The logger to update. If not given, the root logger. + """ + logger_obj = logging.getLogger(logger) + # Save the original log-level to restore it during teardown. + self._initial_logger_levels.setdefault(logger, logger_obj.level) + logger_obj.setLevel(level) + if self._initial_handler_level is None: + self._initial_handler_level = self.handler.level + self.handler.setLevel(level) + + @contextmanager + def at_level( + self, level: Union[int, str], logger: Optional[str] = None + ) -> Generator[None, None, None]: + """Context manager that sets the level for capturing of logs. After + the end of the 'with' statement the level is restored to its original + value. + + :param int level: The level. + :param str logger: The logger to update. If not given, the root logger. + """ + logger_obj = logging.getLogger(logger) + orig_level = logger_obj.level + logger_obj.setLevel(level) + handler_orig_level = self.handler.level + self.handler.setLevel(level) + try: + yield + finally: + logger_obj.setLevel(orig_level) + self.handler.setLevel(handler_orig_level) + + +@fixture +def caplog(request: FixtureRequest) -> Generator[LogCaptureFixture, None, None]: + """Access and control log capturing. + + Captured logs are available through the following properties/methods:: + + * caplog.messages -> list of format-interpolated log messages + * caplog.text -> string containing formatted log output + * caplog.records -> list of logging.LogRecord instances + * caplog.record_tuples -> list of (logger_name, level, message) tuples + * caplog.clear() -> clear captured records and formatted log output string + """ + result = LogCaptureFixture(request.node, _ispytest=True) + yield result + result._finalize() + + +def get_log_level_for_setting(config: Config, *setting_names: str) -> Optional[int]: + for setting_name in setting_names: + log_level = config.getoption(setting_name) + if log_level is None: + log_level = config.getini(setting_name) + if log_level: + break + else: + return None + + if isinstance(log_level, str): + log_level = log_level.upper() + try: + return int(getattr(logging, log_level, log_level)) + except ValueError as e: + # Python logging does not recognise this as a logging level + raise UsageError( + "'{}' is not recognized as a logging level name for " + "'{}'. Please consider passing the " + "logging level num instead.".format(log_level, setting_name) + ) from e + + +# run after terminalreporter/capturemanager are configured +@hookimpl(trylast=True) +def pytest_configure(config: Config) -> None: + config.pluginmanager.register(LoggingPlugin(config), "logging-plugin") + + +class LoggingPlugin: + """Attaches to the logging module and captures log messages for each test.""" + + def __init__(self, config: Config) -> None: + """Create a new plugin to capture log messages. + + The formatter can be safely shared across all handlers so + create a single one for the entire test session here. + """ + self._config = config + + # Report logging. + self.formatter = self._create_formatter( + get_option_ini(config, "log_format"), + get_option_ini(config, "log_date_format"), + get_option_ini(config, "log_auto_indent"), + ) + self.log_level = get_log_level_for_setting(config, "log_level") + self.caplog_handler = LogCaptureHandler() + self.caplog_handler.setFormatter(self.formatter) + self.report_handler = LogCaptureHandler() + self.report_handler.setFormatter(self.formatter) + + # File logging. + self.log_file_level = get_log_level_for_setting(config, "log_file_level") + log_file = get_option_ini(config, "log_file") or os.devnull + if log_file != os.devnull: + directory = os.path.dirname(os.path.abspath(log_file)) + if not os.path.isdir(directory): + os.makedirs(directory) + + self.log_file_handler = _FileHandler(log_file, mode="w", encoding="UTF-8") + log_file_format = get_option_ini(config, "log_file_format", "log_format") + log_file_date_format = get_option_ini( + config, "log_file_date_format", "log_date_format" + ) + + log_file_formatter = logging.Formatter( + log_file_format, datefmt=log_file_date_format + ) + self.log_file_handler.setFormatter(log_file_formatter) + + # CLI/live logging. + self.log_cli_level = get_log_level_for_setting( + config, "log_cli_level", "log_level" + ) + if self._log_cli_enabled(): + terminal_reporter = config.pluginmanager.get_plugin("terminalreporter") + capture_manager = config.pluginmanager.get_plugin("capturemanager") + # if capturemanager plugin is disabled, live logging still works. + self.log_cli_handler: Union[ + _LiveLoggingStreamHandler, _LiveLoggingNullHandler + ] = _LiveLoggingStreamHandler(terminal_reporter, capture_manager) + else: + self.log_cli_handler = _LiveLoggingNullHandler() + log_cli_formatter = self._create_formatter( + get_option_ini(config, "log_cli_format", "log_format"), + get_option_ini(config, "log_cli_date_format", "log_date_format"), + get_option_ini(config, "log_auto_indent"), + ) + self.log_cli_handler.setFormatter(log_cli_formatter) + + def _create_formatter(self, log_format, log_date_format, auto_indent): + # Color option doesn't exist if terminal plugin is disabled. + color = getattr(self._config.option, "color", "no") + if color != "no" and ColoredLevelFormatter.LEVELNAME_FMT_REGEX.search( + log_format + ): + formatter: logging.Formatter = ColoredLevelFormatter( + create_terminal_writer(self._config), log_format, log_date_format + ) + else: + formatter = logging.Formatter(log_format, log_date_format) + + formatter._style = PercentStyleMultiline( + formatter._style._fmt, auto_indent=auto_indent + ) + + return formatter + + def set_log_path(self, fname: str) -> None: + """Set the filename parameter for Logging.FileHandler(). + + Creates parent directory if it does not exist. + + .. warning:: + This is an experimental API. + """ + fpath = Path(fname) + + if not fpath.is_absolute(): + fpath = self._config.rootpath / fpath + + if not fpath.parent.exists(): + fpath.parent.mkdir(exist_ok=True, parents=True) + + stream = fpath.open(mode="w", encoding="UTF-8") + if sys.version_info >= (3, 7): + old_stream = self.log_file_handler.setStream(stream) + else: + old_stream = self.log_file_handler.stream + self.log_file_handler.acquire() + try: + self.log_file_handler.flush() + self.log_file_handler.stream = stream + finally: + self.log_file_handler.release() + if old_stream: + # https://github.com/python/typeshed/pull/5663 + old_stream.close() # type:ignore[attr-defined] + + def _log_cli_enabled(self): + """Return whether live logging is enabled.""" + enabled = self._config.getoption( + "--log-cli-level" + ) is not None or self._config.getini("log_cli") + if not enabled: + return False + + terminal_reporter = self._config.pluginmanager.get_plugin("terminalreporter") + if terminal_reporter is None: + # terminal reporter is disabled e.g. by pytest-xdist. + return False + + return True + + @hookimpl(hookwrapper=True, tryfirst=True) + def pytest_sessionstart(self) -> Generator[None, None, None]: + self.log_cli_handler.set_when("sessionstart") + + with catching_logs(self.log_cli_handler, level=self.log_cli_level): + with catching_logs(self.log_file_handler, level=self.log_file_level): + yield + + @hookimpl(hookwrapper=True, tryfirst=True) + def pytest_collection(self) -> Generator[None, None, None]: + self.log_cli_handler.set_when("collection") + + with catching_logs(self.log_cli_handler, level=self.log_cli_level): + with catching_logs(self.log_file_handler, level=self.log_file_level): + yield + + @hookimpl(hookwrapper=True) + def pytest_runtestloop(self, session: Session) -> Generator[None, None, None]: + if session.config.option.collectonly: + yield + return + + if self._log_cli_enabled() and self._config.getoption("verbose") < 1: + # The verbose flag is needed to avoid messy test progress output. + self._config.option.verbose = 1 + + with catching_logs(self.log_cli_handler, level=self.log_cli_level): + with catching_logs(self.log_file_handler, level=self.log_file_level): + yield # Run all the tests. + + @hookimpl + def pytest_runtest_logstart(self) -> None: + self.log_cli_handler.reset() + self.log_cli_handler.set_when("start") + + @hookimpl + def pytest_runtest_logreport(self) -> None: + self.log_cli_handler.set_when("logreport") + + def _runtest_for(self, item: nodes.Item, when: str) -> Generator[None, None, None]: + """Implement the internals of the pytest_runtest_xxx() hooks.""" + with catching_logs( + self.caplog_handler, + level=self.log_level, + ) as caplog_handler, catching_logs( + self.report_handler, + level=self.log_level, + ) as report_handler: + caplog_handler.reset() + report_handler.reset() + item.stash[caplog_records_key][when] = caplog_handler.records + item.stash[caplog_handler_key] = caplog_handler + + yield + + log = report_handler.stream.getvalue().strip() + item.add_report_section(when, "log", log) + + @hookimpl(hookwrapper=True) + def pytest_runtest_setup(self, item: nodes.Item) -> Generator[None, None, None]: + self.log_cli_handler.set_when("setup") + + empty: Dict[str, List[logging.LogRecord]] = {} + item.stash[caplog_records_key] = empty + yield from self._runtest_for(item, "setup") + + @hookimpl(hookwrapper=True) + def pytest_runtest_call(self, item: nodes.Item) -> Generator[None, None, None]: + self.log_cli_handler.set_when("call") + + yield from self._runtest_for(item, "call") + + @hookimpl(hookwrapper=True) + def pytest_runtest_teardown(self, item: nodes.Item) -> Generator[None, None, None]: + self.log_cli_handler.set_when("teardown") + + yield from self._runtest_for(item, "teardown") + del item.stash[caplog_records_key] + del item.stash[caplog_handler_key] + + @hookimpl + def pytest_runtest_logfinish(self) -> None: + self.log_cli_handler.set_when("finish") + + @hookimpl(hookwrapper=True, tryfirst=True) + def pytest_sessionfinish(self) -> Generator[None, None, None]: + self.log_cli_handler.set_when("sessionfinish") + + with catching_logs(self.log_cli_handler, level=self.log_cli_level): + with catching_logs(self.log_file_handler, level=self.log_file_level): + yield + + @hookimpl + def pytest_unconfigure(self) -> None: + # Close the FileHandler explicitly. + # (logging.shutdown might have lost the weakref?!) + self.log_file_handler.close() + + +class _FileHandler(logging.FileHandler): + """A logging FileHandler with pytest tweaks.""" + + def handleError(self, record: logging.LogRecord) -> None: + # Handled by LogCaptureHandler. + pass + + +class _LiveLoggingStreamHandler(logging.StreamHandler): + """A logging StreamHandler used by the live logging feature: it will + write a newline before the first log message in each test. + + During live logging we must also explicitly disable stdout/stderr + capturing otherwise it will get captured and won't appear in the + terminal. + """ + + # Officially stream needs to be a IO[str], but TerminalReporter + # isn't. So force it. + stream: TerminalReporter = None # type: ignore + + def __init__( + self, + terminal_reporter: TerminalReporter, + capture_manager: Optional[CaptureManager], + ) -> None: + super().__init__(stream=terminal_reporter) # type: ignore[arg-type] + self.capture_manager = capture_manager + self.reset() + self.set_when(None) + self._test_outcome_written = False + + def reset(self) -> None: + """Reset the handler; should be called before the start of each test.""" + self._first_record_emitted = False + + def set_when(self, when: Optional[str]) -> None: + """Prepare for the given test phase (setup/call/teardown).""" + self._when = when + self._section_name_shown = False + if when == "start": + self._test_outcome_written = False + + def emit(self, record: logging.LogRecord) -> None: + ctx_manager = ( + self.capture_manager.global_and_fixture_disabled() + if self.capture_manager + else nullcontext() + ) + with ctx_manager: + if not self._first_record_emitted: + self.stream.write("\n") + self._first_record_emitted = True + elif self._when in ("teardown", "finish"): + if not self._test_outcome_written: + self._test_outcome_written = True + self.stream.write("\n") + if not self._section_name_shown and self._when: + self.stream.section("live log " + self._when, sep="-", bold=True) + self._section_name_shown = True + super().emit(record) + + def handleError(self, record: logging.LogRecord) -> None: + # Handled by LogCaptureHandler. + pass + + +class _LiveLoggingNullHandler(logging.NullHandler): + """A logging handler used when live logging is disabled.""" + + def reset(self) -> None: + pass + + def set_when(self, when: str) -> None: + pass + + def handleError(self, record: logging.LogRecord) -> None: + # Handled by LogCaptureHandler. + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/main.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/main.py new file mode 100644 index 0000000000..fea8179ca7 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/main.py @@ -0,0 +1,896 @@ +"""Core implementation of the testing process: init, session, runtest loop.""" +import argparse +import fnmatch +import functools +import importlib +import os +import sys +from pathlib import Path +from typing import Callable +from typing import Dict +from typing import FrozenSet +from typing import Iterator +from typing import List +from typing import Optional +from typing import overload +from typing import Sequence +from typing import Set +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import Union + +import attr + +import _pytest._code +from _pytest import nodes +from _pytest.compat import final +from _pytest.config import Config +from _pytest.config import directory_arg +from _pytest.config import ExitCode +from _pytest.config import hookimpl +from _pytest.config import PytestPluginManager +from _pytest.config import UsageError +from _pytest.config.argparsing import Parser +from _pytest.fixtures import FixtureManager +from _pytest.outcomes import exit +from _pytest.pathlib import absolutepath +from _pytest.pathlib import bestrelpath +from _pytest.pathlib import fnmatch_ex +from _pytest.pathlib import visit +from _pytest.reports import CollectReport +from _pytest.reports import TestReport +from _pytest.runner import collect_one_node +from _pytest.runner import SetupState + + +if TYPE_CHECKING: + from typing_extensions import Literal + + +def pytest_addoption(parser: Parser) -> None: + parser.addini( + "norecursedirs", + "directory patterns to avoid for recursion", + type="args", + default=[ + "*.egg", + ".*", + "_darcs", + "build", + "CVS", + "dist", + "node_modules", + "venv", + "{arch}", + ], + ) + parser.addini( + "testpaths", + "directories to search for tests when no files or directories are given in the " + "command line.", + type="args", + default=[], + ) + group = parser.getgroup("general", "running and selection options") + group._addoption( + "-x", + "--exitfirst", + action="store_const", + dest="maxfail", + const=1, + help="exit instantly on first error or failed test.", + ) + group = parser.getgroup("pytest-warnings") + group.addoption( + "-W", + "--pythonwarnings", + action="append", + help="set which warnings to report, see -W option of python itself.", + ) + parser.addini( + "filterwarnings", + type="linelist", + help="Each line specifies a pattern for " + "warnings.filterwarnings. " + "Processed after -W/--pythonwarnings.", + ) + group._addoption( + "--maxfail", + metavar="num", + action="store", + type=int, + dest="maxfail", + default=0, + help="exit after first num failures or errors.", + ) + group._addoption( + "--strict-config", + action="store_true", + help="any warnings encountered while parsing the `pytest` section of the configuration file raise errors.", + ) + group._addoption( + "--strict-markers", + action="store_true", + help="markers not registered in the `markers` section of the configuration file raise errors.", + ) + group._addoption( + "--strict", + action="store_true", + help="(deprecated) alias to --strict-markers.", + ) + group._addoption( + "-c", + metavar="file", + type=str, + dest="inifilename", + help="load configuration from `file` instead of trying to locate one of the implicit " + "configuration files.", + ) + group._addoption( + "--continue-on-collection-errors", + action="store_true", + default=False, + dest="continue_on_collection_errors", + help="Force test execution even if collection errors occur.", + ) + group._addoption( + "--rootdir", + action="store", + dest="rootdir", + help="Define root directory for tests. Can be relative path: 'root_dir', './root_dir', " + "'root_dir/another_dir/'; absolute path: '/home/user/root_dir'; path with variables: " + "'$HOME/root_dir'.", + ) + + group = parser.getgroup("collect", "collection") + group.addoption( + "--collectonly", + "--collect-only", + "--co", + action="store_true", + help="only collect tests, don't execute them.", + ) + group.addoption( + "--pyargs", + action="store_true", + help="try to interpret all arguments as python packages.", + ) + group.addoption( + "--ignore", + action="append", + metavar="path", + help="ignore path during collection (multi-allowed).", + ) + group.addoption( + "--ignore-glob", + action="append", + metavar="path", + help="ignore path pattern during collection (multi-allowed).", + ) + group.addoption( + "--deselect", + action="append", + metavar="nodeid_prefix", + help="deselect item (via node id prefix) during collection (multi-allowed).", + ) + group.addoption( + "--confcutdir", + dest="confcutdir", + default=None, + metavar="dir", + type=functools.partial(directory_arg, optname="--confcutdir"), + help="only load conftest.py's relative to specified dir.", + ) + group.addoption( + "--noconftest", + action="store_true", + dest="noconftest", + default=False, + help="Don't load any conftest.py files.", + ) + group.addoption( + "--keepduplicates", + "--keep-duplicates", + action="store_true", + dest="keepduplicates", + default=False, + help="Keep duplicate tests.", + ) + group.addoption( + "--collect-in-virtualenv", + action="store_true", + dest="collect_in_virtualenv", + default=False, + help="Don't ignore tests in a local virtualenv directory", + ) + group.addoption( + "--import-mode", + default="prepend", + choices=["prepend", "append", "importlib"], + dest="importmode", + help="prepend/append to sys.path when importing test modules and conftest files, " + "default is to prepend.", + ) + + group = parser.getgroup("debugconfig", "test session debugging and configuration") + group.addoption( + "--basetemp", + dest="basetemp", + default=None, + type=validate_basetemp, + metavar="dir", + help=( + "base temporary directory for this test run." + "(warning: this directory is removed if it exists)" + ), + ) + + +def validate_basetemp(path: str) -> str: + # GH 7119 + msg = "basetemp must not be empty, the current working directory or any parent directory of it" + + # empty path + if not path: + raise argparse.ArgumentTypeError(msg) + + def is_ancestor(base: Path, query: Path) -> bool: + """Return whether query is an ancestor of base.""" + if base == query: + return True + return query in base.parents + + # check if path is an ancestor of cwd + if is_ancestor(Path.cwd(), Path(path).absolute()): + raise argparse.ArgumentTypeError(msg) + + # check symlinks for ancestors + if is_ancestor(Path.cwd().resolve(), Path(path).resolve()): + raise argparse.ArgumentTypeError(msg) + + return path + + +def wrap_session( + config: Config, doit: Callable[[Config, "Session"], Optional[Union[int, ExitCode]]] +) -> Union[int, ExitCode]: + """Skeleton command line program.""" + session = Session.from_config(config) + session.exitstatus = ExitCode.OK + initstate = 0 + try: + try: + config._do_configure() + initstate = 1 + config.hook.pytest_sessionstart(session=session) + initstate = 2 + session.exitstatus = doit(config, session) or 0 + except UsageError: + session.exitstatus = ExitCode.USAGE_ERROR + raise + except Failed: + session.exitstatus = ExitCode.TESTS_FAILED + except (KeyboardInterrupt, exit.Exception): + excinfo = _pytest._code.ExceptionInfo.from_current() + exitstatus: Union[int, ExitCode] = ExitCode.INTERRUPTED + if isinstance(excinfo.value, exit.Exception): + if excinfo.value.returncode is not None: + exitstatus = excinfo.value.returncode + if initstate < 2: + sys.stderr.write(f"{excinfo.typename}: {excinfo.value.msg}\n") + config.hook.pytest_keyboard_interrupt(excinfo=excinfo) + session.exitstatus = exitstatus + except BaseException: + session.exitstatus = ExitCode.INTERNAL_ERROR + excinfo = _pytest._code.ExceptionInfo.from_current() + try: + config.notify_exception(excinfo, config.option) + except exit.Exception as exc: + if exc.returncode is not None: + session.exitstatus = exc.returncode + sys.stderr.write(f"{type(exc).__name__}: {exc}\n") + else: + if isinstance(excinfo.value, SystemExit): + sys.stderr.write("mainloop: caught unexpected SystemExit!\n") + + finally: + # Explicitly break reference cycle. + excinfo = None # type: ignore + os.chdir(session.startpath) + if initstate >= 2: + try: + config.hook.pytest_sessionfinish( + session=session, exitstatus=session.exitstatus + ) + except exit.Exception as exc: + if exc.returncode is not None: + session.exitstatus = exc.returncode + sys.stderr.write(f"{type(exc).__name__}: {exc}\n") + config._ensure_unconfigure() + return session.exitstatus + + +def pytest_cmdline_main(config: Config) -> Union[int, ExitCode]: + return wrap_session(config, _main) + + +def _main(config: Config, session: "Session") -> Optional[Union[int, ExitCode]]: + """Default command line protocol for initialization, session, + running tests and reporting.""" + config.hook.pytest_collection(session=session) + config.hook.pytest_runtestloop(session=session) + + if session.testsfailed: + return ExitCode.TESTS_FAILED + elif session.testscollected == 0: + return ExitCode.NO_TESTS_COLLECTED + return None + + +def pytest_collection(session: "Session") -> None: + session.perform_collect() + + +def pytest_runtestloop(session: "Session") -> bool: + if session.testsfailed and not session.config.option.continue_on_collection_errors: + raise session.Interrupted( + "%d error%s during collection" + % (session.testsfailed, "s" if session.testsfailed != 1 else "") + ) + + if session.config.option.collectonly: + return True + + for i, item in enumerate(session.items): + nextitem = session.items[i + 1] if i + 1 < len(session.items) else None + item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem) + if session.shouldfail: + raise session.Failed(session.shouldfail) + if session.shouldstop: + raise session.Interrupted(session.shouldstop) + return True + + +def _in_venv(path: Path) -> bool: + """Attempt to detect if ``path`` is the root of a Virtual Environment by + checking for the existence of the appropriate activate script.""" + bindir = path.joinpath("Scripts" if sys.platform.startswith("win") else "bin") + try: + if not bindir.is_dir(): + return False + except OSError: + return False + activates = ( + "activate", + "activate.csh", + "activate.fish", + "Activate", + "Activate.bat", + "Activate.ps1", + ) + return any(fname.name in activates for fname in bindir.iterdir()) + + +def pytest_ignore_collect(collection_path: Path, config: Config) -> Optional[bool]: + ignore_paths = config._getconftest_pathlist( + "collect_ignore", path=collection_path.parent, rootpath=config.rootpath + ) + ignore_paths = ignore_paths or [] + excludeopt = config.getoption("ignore") + if excludeopt: + ignore_paths.extend(absolutepath(x) for x in excludeopt) + + if collection_path in ignore_paths: + return True + + ignore_globs = config._getconftest_pathlist( + "collect_ignore_glob", path=collection_path.parent, rootpath=config.rootpath + ) + ignore_globs = ignore_globs or [] + excludeglobopt = config.getoption("ignore_glob") + if excludeglobopt: + ignore_globs.extend(absolutepath(x) for x in excludeglobopt) + + if any(fnmatch.fnmatch(str(collection_path), str(glob)) for glob in ignore_globs): + return True + + allow_in_venv = config.getoption("collect_in_virtualenv") + if not allow_in_venv and _in_venv(collection_path): + return True + return None + + +def pytest_collection_modifyitems(items: List[nodes.Item], config: Config) -> None: + deselect_prefixes = tuple(config.getoption("deselect") or []) + if not deselect_prefixes: + return + + remaining = [] + deselected = [] + for colitem in items: + if colitem.nodeid.startswith(deselect_prefixes): + deselected.append(colitem) + else: + remaining.append(colitem) + + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = remaining + + +class FSHookProxy: + def __init__(self, pm: PytestPluginManager, remove_mods) -> None: + self.pm = pm + self.remove_mods = remove_mods + + def __getattr__(self, name: str): + x = self.pm.subset_hook_caller(name, remove_plugins=self.remove_mods) + self.__dict__[name] = x + return x + + +class Interrupted(KeyboardInterrupt): + """Signals that the test run was interrupted.""" + + __module__ = "builtins" # For py3. + + +class Failed(Exception): + """Signals a stop as failed test run.""" + + +@attr.s(slots=True, auto_attribs=True) +class _bestrelpath_cache(Dict[Path, str]): + path: Path + + def __missing__(self, path: Path) -> str: + r = bestrelpath(self.path, path) + self[path] = r + return r + + +@final +class Session(nodes.FSCollector): + Interrupted = Interrupted + Failed = Failed + # Set on the session by runner.pytest_sessionstart. + _setupstate: SetupState + # Set on the session by fixtures.pytest_sessionstart. + _fixturemanager: FixtureManager + exitstatus: Union[int, ExitCode] + + def __init__(self, config: Config) -> None: + super().__init__( + path=config.rootpath, + fspath=None, + parent=None, + config=config, + session=self, + nodeid="", + ) + self.testsfailed = 0 + self.testscollected = 0 + self.shouldstop: Union[bool, str] = False + self.shouldfail: Union[bool, str] = False + self.trace = config.trace.root.get("collection") + self._initialpaths: FrozenSet[Path] = frozenset() + + self._bestrelpathcache: Dict[Path, str] = _bestrelpath_cache(config.rootpath) + + self.config.pluginmanager.register(self, name="session") + + @classmethod + def from_config(cls, config: Config) -> "Session": + session: Session = cls._create(config=config) + return session + + def __repr__(self) -> str: + return "<%s %s exitstatus=%r testsfailed=%d testscollected=%d>" % ( + self.__class__.__name__, + self.name, + getattr(self, "exitstatus", ""), + self.testsfailed, + self.testscollected, + ) + + @property + def startpath(self) -> Path: + """The path from which pytest was invoked. + + .. versionadded:: 7.0.0 + """ + return self.config.invocation_params.dir + + def _node_location_to_relpath(self, node_path: Path) -> str: + # bestrelpath is a quite slow function. + return self._bestrelpathcache[node_path] + + @hookimpl(tryfirst=True) + def pytest_collectstart(self) -> None: + if self.shouldfail: + raise self.Failed(self.shouldfail) + if self.shouldstop: + raise self.Interrupted(self.shouldstop) + + @hookimpl(tryfirst=True) + def pytest_runtest_logreport( + self, report: Union[TestReport, CollectReport] + ) -> None: + if report.failed and not hasattr(report, "wasxfail"): + self.testsfailed += 1 + maxfail = self.config.getvalue("maxfail") + if maxfail and self.testsfailed >= maxfail: + self.shouldfail = "stopping after %d failures" % (self.testsfailed) + + pytest_collectreport = pytest_runtest_logreport + + def isinitpath(self, path: Union[str, "os.PathLike[str]"]) -> bool: + # Optimization: Path(Path(...)) is much slower than isinstance. + path_ = path if isinstance(path, Path) else Path(path) + return path_ in self._initialpaths + + def gethookproxy(self, fspath: "os.PathLike[str]"): + # Optimization: Path(Path(...)) is much slower than isinstance. + path = fspath if isinstance(fspath, Path) else Path(fspath) + pm = self.config.pluginmanager + # Check if we have the common case of running + # hooks with all conftest.py files. + my_conftestmodules = pm._getconftestmodules( + path, + self.config.getoption("importmode"), + rootpath=self.config.rootpath, + ) + remove_mods = pm._conftest_plugins.difference(my_conftestmodules) + if remove_mods: + # One or more conftests are not in use at this fspath. + from .config.compat import PathAwareHookProxy + + proxy = PathAwareHookProxy(FSHookProxy(pm, remove_mods)) + else: + # All plugins are active for this fspath. + proxy = self.config.hook + return proxy + + def _recurse(self, direntry: "os.DirEntry[str]") -> bool: + if direntry.name == "__pycache__": + return False + fspath = Path(direntry.path) + ihook = self.gethookproxy(fspath.parent) + if ihook.pytest_ignore_collect(collection_path=fspath, config=self.config): + return False + norecursepatterns = self.config.getini("norecursedirs") + if any(fnmatch_ex(pat, fspath) for pat in norecursepatterns): + return False + return True + + def _collectfile( + self, fspath: Path, handle_dupes: bool = True + ) -> Sequence[nodes.Collector]: + assert ( + fspath.is_file() + ), "{!r} is not a file (isdir={!r}, exists={!r}, islink={!r})".format( + fspath, fspath.is_dir(), fspath.exists(), fspath.is_symlink() + ) + ihook = self.gethookproxy(fspath) + if not self.isinitpath(fspath): + if ihook.pytest_ignore_collect(collection_path=fspath, config=self.config): + return () + + if handle_dupes: + keepduplicates = self.config.getoption("keepduplicates") + if not keepduplicates: + duplicate_paths = self.config.pluginmanager._duplicatepaths + if fspath in duplicate_paths: + return () + else: + duplicate_paths.add(fspath) + + return ihook.pytest_collect_file(file_path=fspath, parent=self) # type: ignore[no-any-return] + + @overload + def perform_collect( + self, args: Optional[Sequence[str]] = ..., genitems: "Literal[True]" = ... + ) -> Sequence[nodes.Item]: + ... + + @overload + def perform_collect( + self, args: Optional[Sequence[str]] = ..., genitems: bool = ... + ) -> Sequence[Union[nodes.Item, nodes.Collector]]: + ... + + def perform_collect( + self, args: Optional[Sequence[str]] = None, genitems: bool = True + ) -> Sequence[Union[nodes.Item, nodes.Collector]]: + """Perform the collection phase for this session. + + This is called by the default :hook:`pytest_collection` hook + implementation; see the documentation of this hook for more details. + For testing purposes, it may also be called directly on a fresh + ``Session``. + + This function normally recursively expands any collectors collected + from the session to their items, and only items are returned. For + testing purposes, this may be suppressed by passing ``genitems=False``, + in which case the return value contains these collectors unexpanded, + and ``session.items`` is empty. + """ + if args is None: + args = self.config.args + + self.trace("perform_collect", self, args) + self.trace.root.indent += 1 + + self._notfound: List[Tuple[str, Sequence[nodes.Collector]]] = [] + self._initial_parts: List[Tuple[Path, List[str]]] = [] + self.items: List[nodes.Item] = [] + + hook = self.config.hook + + items: Sequence[Union[nodes.Item, nodes.Collector]] = self.items + try: + initialpaths: List[Path] = [] + for arg in args: + fspath, parts = resolve_collection_argument( + self.config.invocation_params.dir, + arg, + as_pypath=self.config.option.pyargs, + ) + self._initial_parts.append((fspath, parts)) + initialpaths.append(fspath) + self._initialpaths = frozenset(initialpaths) + rep = collect_one_node(self) + self.ihook.pytest_collectreport(report=rep) + self.trace.root.indent -= 1 + if self._notfound: + errors = [] + for arg, cols in self._notfound: + line = f"(no name {arg!r} in any of {cols!r})" + errors.append(f"not found: {arg}\n{line}") + raise UsageError(*errors) + if not genitems: + items = rep.result + else: + if rep.passed: + for node in rep.result: + self.items.extend(self.genitems(node)) + + self.config.pluginmanager.check_pending() + hook.pytest_collection_modifyitems( + session=self, config=self.config, items=items + ) + finally: + hook.pytest_collection_finish(session=self) + + self.testscollected = len(items) + return items + + def collect(self) -> Iterator[Union[nodes.Item, nodes.Collector]]: + from _pytest.python import Package + + # Keep track of any collected nodes in here, so we don't duplicate fixtures. + node_cache1: Dict[Path, Sequence[nodes.Collector]] = {} + node_cache2: Dict[Tuple[Type[nodes.Collector], Path], nodes.Collector] = {} + + # Keep track of any collected collectors in matchnodes paths, so they + # are not collected more than once. + matchnodes_cache: Dict[Tuple[Type[nodes.Collector], str], CollectReport] = {} + + # Dirnames of pkgs with dunder-init files. + pkg_roots: Dict[str, Package] = {} + + for argpath, names in self._initial_parts: + self.trace("processing argument", (argpath, names)) + self.trace.root.indent += 1 + + # Start with a Session root, and delve to argpath item (dir or file) + # and stack all Packages found on the way. + # No point in finding packages when collecting doctests. + if not self.config.getoption("doctestmodules", False): + pm = self.config.pluginmanager + confcutdir = pm._confcutdir + for parent in (argpath, *argpath.parents): + if confcutdir and parent in confcutdir.parents: + break + + if parent.is_dir(): + pkginit = parent / "__init__.py" + if pkginit.is_file() and pkginit not in node_cache1: + col = self._collectfile(pkginit, handle_dupes=False) + if col: + if isinstance(col[0], Package): + pkg_roots[str(parent)] = col[0] + node_cache1[col[0].path] = [col[0]] + + # If it's a directory argument, recurse and look for any Subpackages. + # Let the Package collector deal with subnodes, don't collect here. + if argpath.is_dir(): + assert not names, f"invalid arg {(argpath, names)!r}" + + seen_dirs: Set[Path] = set() + for direntry in visit(str(argpath), self._recurse): + if not direntry.is_file(): + continue + + path = Path(direntry.path) + dirpath = path.parent + + if dirpath not in seen_dirs: + # Collect packages first. + seen_dirs.add(dirpath) + pkginit = dirpath / "__init__.py" + if pkginit.exists(): + for x in self._collectfile(pkginit): + yield x + if isinstance(x, Package): + pkg_roots[str(dirpath)] = x + if str(dirpath) in pkg_roots: + # Do not collect packages here. + continue + + for x in self._collectfile(path): + key2 = (type(x), x.path) + if key2 in node_cache2: + yield node_cache2[key2] + else: + node_cache2[key2] = x + yield x + else: + assert argpath.is_file() + + if argpath in node_cache1: + col = node_cache1[argpath] + else: + collect_root = pkg_roots.get(str(argpath.parent), self) + col = collect_root._collectfile(argpath, handle_dupes=False) + if col: + node_cache1[argpath] = col + + matching = [] + work: List[ + Tuple[Sequence[Union[nodes.Item, nodes.Collector]], Sequence[str]] + ] = [(col, names)] + while work: + self.trace("matchnodes", col, names) + self.trace.root.indent += 1 + + matchnodes, matchnames = work.pop() + for node in matchnodes: + if not matchnames: + matching.append(node) + continue + if not isinstance(node, nodes.Collector): + continue + key = (type(node), node.nodeid) + if key in matchnodes_cache: + rep = matchnodes_cache[key] + else: + rep = collect_one_node(node) + matchnodes_cache[key] = rep + if rep.passed: + submatchnodes = [] + for r in rep.result: + # TODO: Remove parametrized workaround once collection structure contains + # parametrization. + if ( + r.name == matchnames[0] + or r.name.split("[")[0] == matchnames[0] + ): + submatchnodes.append(r) + if submatchnodes: + work.append((submatchnodes, matchnames[1:])) + else: + # Report collection failures here to avoid failing to run some test + # specified in the command line because the module could not be + # imported (#134). + node.ihook.pytest_collectreport(report=rep) + + self.trace("matchnodes finished -> ", len(matching), "nodes") + self.trace.root.indent -= 1 + + if not matching: + report_arg = "::".join((str(argpath), *names)) + self._notfound.append((report_arg, col)) + continue + + # If __init__.py was the only file requested, then the matched + # node will be the corresponding Package (by default), and the + # first yielded item will be the __init__ Module itself, so + # just use that. If this special case isn't taken, then all the + # files in the package will be yielded. + if argpath.name == "__init__.py" and isinstance(matching[0], Package): + try: + yield next(iter(matching[0].collect())) + except StopIteration: + # The package collects nothing with only an __init__.py + # file in it, which gets ignored by the default + # "python_files" option. + pass + continue + + yield from matching + + self.trace.root.indent -= 1 + + def genitems( + self, node: Union[nodes.Item, nodes.Collector] + ) -> Iterator[nodes.Item]: + self.trace("genitems", node) + if isinstance(node, nodes.Item): + node.ihook.pytest_itemcollected(item=node) + yield node + else: + assert isinstance(node, nodes.Collector) + rep = collect_one_node(node) + if rep.passed: + for subnode in rep.result: + yield from self.genitems(subnode) + node.ihook.pytest_collectreport(report=rep) + + +def search_pypath(module_name: str) -> str: + """Search sys.path for the given a dotted module name, and return its file system path.""" + try: + spec = importlib.util.find_spec(module_name) + # AttributeError: looks like package module, but actually filename + # ImportError: module does not exist + # ValueError: not a module name + except (AttributeError, ImportError, ValueError): + return module_name + if spec is None or spec.origin is None or spec.origin == "namespace": + return module_name + elif spec.submodule_search_locations: + return os.path.dirname(spec.origin) + else: + return spec.origin + + +def resolve_collection_argument( + invocation_path: Path, arg: str, *, as_pypath: bool = False +) -> Tuple[Path, List[str]]: + """Parse path arguments optionally containing selection parts and return (fspath, names). + + Command-line arguments can point to files and/or directories, and optionally contain + parts for specific tests selection, for example: + + "pkg/tests/test_foo.py::TestClass::test_foo" + + This function ensures the path exists, and returns a tuple: + + (Path("/full/path/to/pkg/tests/test_foo.py"), ["TestClass", "test_foo"]) + + When as_pypath is True, expects that the command-line argument actually contains + module paths instead of file-system paths: + + "pkg.tests.test_foo::TestClass::test_foo" + + In which case we search sys.path for a matching module, and then return the *path* to the + found module. + + If the path doesn't exist, raise UsageError. + If the path is a directory and selection parts are present, raise UsageError. + """ + base, squacket, rest = str(arg).partition("[") + strpath, *parts = base.split("::") + if parts: + parts[-1] = f"{parts[-1]}{squacket}{rest}" + if as_pypath: + strpath = search_pypath(strpath) + fspath = invocation_path / strpath + fspath = absolutepath(fspath) + if not fspath.exists(): + msg = ( + "module or package not found: {arg} (missing __init__.py?)" + if as_pypath + else "file or directory not found: {arg}" + ) + raise UsageError(msg.format(arg=arg)) + if parts and fspath.is_dir(): + msg = ( + "package argument cannot contain :: selection parts: {arg}" + if as_pypath + else "directory argument cannot contain :: selection parts: {arg}" + ) + raise UsageError(msg.format(arg=arg)) + return fspath, parts diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/mark/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/mark/__init__.py new file mode 100644 index 0000000000..7e082f2e6e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/mark/__init__.py @@ -0,0 +1,282 @@ +"""Generic mechanism for marking and selecting python functions.""" +import warnings +from typing import AbstractSet +from typing import Collection +from typing import List +from typing import Optional +from typing import TYPE_CHECKING +from typing import Union + +import attr + +from .expression import Expression +from .expression import ParseError +from .structures import EMPTY_PARAMETERSET_OPTION +from .structures import get_empty_parameterset_mark +from .structures import Mark +from .structures import MARK_GEN +from .structures import MarkDecorator +from .structures import MarkGenerator +from .structures import ParameterSet +from _pytest.config import Config +from _pytest.config import ExitCode +from _pytest.config import hookimpl +from _pytest.config import UsageError +from _pytest.config.argparsing import Parser +from _pytest.deprecated import MINUS_K_COLON +from _pytest.deprecated import MINUS_K_DASH +from _pytest.stash import StashKey + +if TYPE_CHECKING: + from _pytest.nodes import Item + + +__all__ = [ + "MARK_GEN", + "Mark", + "MarkDecorator", + "MarkGenerator", + "ParameterSet", + "get_empty_parameterset_mark", +] + + +old_mark_config_key = StashKey[Optional[Config]]() + + +def param( + *values: object, + marks: Union[MarkDecorator, Collection[Union[MarkDecorator, Mark]]] = (), + id: Optional[str] = None, +) -> ParameterSet: + """Specify a parameter in `pytest.mark.parametrize`_ calls or + :ref:`parametrized fixtures `. + + .. code-block:: python + + @pytest.mark.parametrize( + "test_input,expected", + [ + ("3+5", 8), + pytest.param("6*9", 42, marks=pytest.mark.xfail), + ], + ) + def test_eval(test_input, expected): + assert eval(test_input) == expected + + :param values: Variable args of the values of the parameter set, in order. + :keyword marks: A single mark or a list of marks to be applied to this parameter set. + :keyword str id: The id to attribute to this parameter set. + """ + return ParameterSet.param(*values, marks=marks, id=id) + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("general") + group._addoption( + "-k", + action="store", + dest="keyword", + default="", + metavar="EXPRESSION", + help="only run tests which match the given substring expression. " + "An expression is a python evaluatable expression " + "where all names are substring-matched against test names " + "and their parent classes. Example: -k 'test_method or test_" + "other' matches all test functions and classes whose name " + "contains 'test_method' or 'test_other', while -k 'not test_method' " + "matches those that don't contain 'test_method' in their names. " + "-k 'not test_method and not test_other' will eliminate the matches. " + "Additionally keywords are matched to classes and functions " + "containing extra names in their 'extra_keyword_matches' set, " + "as well as functions which have names assigned directly to them. " + "The matching is case-insensitive.", + ) + + group._addoption( + "-m", + action="store", + dest="markexpr", + default="", + metavar="MARKEXPR", + help="only run tests matching given mark expression.\n" + "For example: -m 'mark1 and not mark2'.", + ) + + group.addoption( + "--markers", + action="store_true", + help="show markers (builtin, plugin and per-project ones).", + ) + + parser.addini("markers", "markers for test functions", "linelist") + parser.addini(EMPTY_PARAMETERSET_OPTION, "default marker for empty parametersets") + + +@hookimpl(tryfirst=True) +def pytest_cmdline_main(config: Config) -> Optional[Union[int, ExitCode]]: + import _pytest.config + + if config.option.markers: + config._do_configure() + tw = _pytest.config.create_terminal_writer(config) + for line in config.getini("markers"): + parts = line.split(":", 1) + name = parts[0] + rest = parts[1] if len(parts) == 2 else "" + tw.write("@pytest.mark.%s:" % name, bold=True) + tw.line(rest) + tw.line() + config._ensure_unconfigure() + return 0 + + return None + + +@attr.s(slots=True, auto_attribs=True) +class KeywordMatcher: + """A matcher for keywords. + + Given a list of names, matches any substring of one of these names. The + string inclusion check is case-insensitive. + + Will match on the name of colitem, including the names of its parents. + Only matches names of items which are either a :class:`Class` or a + :class:`Function`. + + Additionally, matches on names in the 'extra_keyword_matches' set of + any item, as well as names directly assigned to test functions. + """ + + _names: AbstractSet[str] + + @classmethod + def from_item(cls, item: "Item") -> "KeywordMatcher": + mapped_names = set() + + # Add the names of the current item and any parent items. + import pytest + + for node in item.listchain(): + if not isinstance(node, pytest.Session): + mapped_names.add(node.name) + + # Add the names added as extra keywords to current or parent items. + mapped_names.update(item.listextrakeywords()) + + # Add the names attached to the current function through direct assignment. + function_obj = getattr(item, "function", None) + if function_obj: + mapped_names.update(function_obj.__dict__) + + # Add the markers to the keywords as we no longer handle them correctly. + mapped_names.update(mark.name for mark in item.iter_markers()) + + return cls(mapped_names) + + def __call__(self, subname: str) -> bool: + subname = subname.lower() + names = (name.lower() for name in self._names) + + for name in names: + if subname in name: + return True + return False + + +def deselect_by_keyword(items: "List[Item]", config: Config) -> None: + keywordexpr = config.option.keyword.lstrip() + if not keywordexpr: + return + + if keywordexpr.startswith("-"): + # To be removed in pytest 8.0.0. + warnings.warn(MINUS_K_DASH, stacklevel=2) + keywordexpr = "not " + keywordexpr[1:] + selectuntil = False + if keywordexpr[-1:] == ":": + # To be removed in pytest 8.0.0. + warnings.warn(MINUS_K_COLON, stacklevel=2) + selectuntil = True + keywordexpr = keywordexpr[:-1] + + expr = _parse_expression(keywordexpr, "Wrong expression passed to '-k'") + + remaining = [] + deselected = [] + for colitem in items: + if keywordexpr and not expr.evaluate(KeywordMatcher.from_item(colitem)): + deselected.append(colitem) + else: + if selectuntil: + keywordexpr = None + remaining.append(colitem) + + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = remaining + + +@attr.s(slots=True, auto_attribs=True) +class MarkMatcher: + """A matcher for markers which are present. + + Tries to match on any marker names, attached to the given colitem. + """ + + own_mark_names: AbstractSet[str] + + @classmethod + def from_item(cls, item: "Item") -> "MarkMatcher": + mark_names = {mark.name for mark in item.iter_markers()} + return cls(mark_names) + + def __call__(self, name: str) -> bool: + return name in self.own_mark_names + + +def deselect_by_mark(items: "List[Item]", config: Config) -> None: + matchexpr = config.option.markexpr + if not matchexpr: + return + + expr = _parse_expression(matchexpr, "Wrong expression passed to '-m'") + remaining: List[Item] = [] + deselected: List[Item] = [] + for item in items: + if expr.evaluate(MarkMatcher.from_item(item)): + remaining.append(item) + else: + deselected.append(item) + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = remaining + + +def _parse_expression(expr: str, exc_message: str) -> Expression: + try: + return Expression.compile(expr) + except ParseError as e: + raise UsageError(f"{exc_message}: {expr}: {e}") from None + + +def pytest_collection_modifyitems(items: "List[Item]", config: Config) -> None: + deselect_by_keyword(items, config) + deselect_by_mark(items, config) + + +def pytest_configure(config: Config) -> None: + config.stash[old_mark_config_key] = MARK_GEN._config + MARK_GEN._config = config + + empty_parameterset = config.getini(EMPTY_PARAMETERSET_OPTION) + + if empty_parameterset not in ("skip", "xfail", "fail_at_collect", None, ""): + raise UsageError( + "{!s} must be one of skip, xfail or fail_at_collect" + " but it is {!r}".format(EMPTY_PARAMETERSET_OPTION, empty_parameterset) + ) + + +def pytest_unconfigure(config: Config) -> None: + MARK_GEN._config = config.stash.get(old_mark_config_key, None) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/mark/expression.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/mark/expression.py new file mode 100644 index 0000000000..92220d7723 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/mark/expression.py @@ -0,0 +1,225 @@ +r"""Evaluate match expressions, as used by `-k` and `-m`. + +The grammar is: + +expression: expr? EOF +expr: and_expr ('or' and_expr)* +and_expr: not_expr ('and' not_expr)* +not_expr: 'not' not_expr | '(' expr ')' | ident +ident: (\w|:|\+|-|\.|\[|\]|\\|/)+ + +The semantics are: + +- Empty expression evaluates to False. +- ident evaluates to True of False according to a provided matcher function. +- or/and/not evaluate according to the usual boolean semantics. +""" +import ast +import enum +import re +import types +from typing import Callable +from typing import Iterator +from typing import Mapping +from typing import Optional +from typing import Sequence +from typing import TYPE_CHECKING + +import attr + +if TYPE_CHECKING: + from typing import NoReturn + + +__all__ = [ + "Expression", + "ParseError", +] + + +class TokenType(enum.Enum): + LPAREN = "left parenthesis" + RPAREN = "right parenthesis" + OR = "or" + AND = "and" + NOT = "not" + IDENT = "identifier" + EOF = "end of input" + + +@attr.s(frozen=True, slots=True, auto_attribs=True) +class Token: + type: TokenType + value: str + pos: int + + +class ParseError(Exception): + """The expression contains invalid syntax. + + :param column: The column in the line where the error occurred (1-based). + :param message: A description of the error. + """ + + def __init__(self, column: int, message: str) -> None: + self.column = column + self.message = message + + def __str__(self) -> str: + return f"at column {self.column}: {self.message}" + + +class Scanner: + __slots__ = ("tokens", "current") + + def __init__(self, input: str) -> None: + self.tokens = self.lex(input) + self.current = next(self.tokens) + + def lex(self, input: str) -> Iterator[Token]: + pos = 0 + while pos < len(input): + if input[pos] in (" ", "\t"): + pos += 1 + elif input[pos] == "(": + yield Token(TokenType.LPAREN, "(", pos) + pos += 1 + elif input[pos] == ")": + yield Token(TokenType.RPAREN, ")", pos) + pos += 1 + else: + match = re.match(r"(:?\w|:|\+|-|\.|\[|\]|\\|/)+", input[pos:]) + if match: + value = match.group(0) + if value == "or": + yield Token(TokenType.OR, value, pos) + elif value == "and": + yield Token(TokenType.AND, value, pos) + elif value == "not": + yield Token(TokenType.NOT, value, pos) + else: + yield Token(TokenType.IDENT, value, pos) + pos += len(value) + else: + raise ParseError( + pos + 1, + f'unexpected character "{input[pos]}"', + ) + yield Token(TokenType.EOF, "", pos) + + def accept(self, type: TokenType, *, reject: bool = False) -> Optional[Token]: + if self.current.type is type: + token = self.current + if token.type is not TokenType.EOF: + self.current = next(self.tokens) + return token + if reject: + self.reject((type,)) + return None + + def reject(self, expected: Sequence[TokenType]) -> "NoReturn": + raise ParseError( + self.current.pos + 1, + "expected {}; got {}".format( + " OR ".join(type.value for type in expected), + self.current.type.value, + ), + ) + + +# True, False and None are legal match expression identifiers, +# but illegal as Python identifiers. To fix this, this prefix +# is added to identifiers in the conversion to Python AST. +IDENT_PREFIX = "$" + + +def expression(s: Scanner) -> ast.Expression: + if s.accept(TokenType.EOF): + ret: ast.expr = ast.NameConstant(False) + else: + ret = expr(s) + s.accept(TokenType.EOF, reject=True) + return ast.fix_missing_locations(ast.Expression(ret)) + + +def expr(s: Scanner) -> ast.expr: + ret = and_expr(s) + while s.accept(TokenType.OR): + rhs = and_expr(s) + ret = ast.BoolOp(ast.Or(), [ret, rhs]) + return ret + + +def and_expr(s: Scanner) -> ast.expr: + ret = not_expr(s) + while s.accept(TokenType.AND): + rhs = not_expr(s) + ret = ast.BoolOp(ast.And(), [ret, rhs]) + return ret + + +def not_expr(s: Scanner) -> ast.expr: + if s.accept(TokenType.NOT): + return ast.UnaryOp(ast.Not(), not_expr(s)) + if s.accept(TokenType.LPAREN): + ret = expr(s) + s.accept(TokenType.RPAREN, reject=True) + return ret + ident = s.accept(TokenType.IDENT) + if ident: + return ast.Name(IDENT_PREFIX + ident.value, ast.Load()) + s.reject((TokenType.NOT, TokenType.LPAREN, TokenType.IDENT)) + + +class MatcherAdapter(Mapping[str, bool]): + """Adapts a matcher function to a locals mapping as required by eval().""" + + def __init__(self, matcher: Callable[[str], bool]) -> None: + self.matcher = matcher + + def __getitem__(self, key: str) -> bool: + return self.matcher(key[len(IDENT_PREFIX) :]) + + def __iter__(self) -> Iterator[str]: + raise NotImplementedError() + + def __len__(self) -> int: + raise NotImplementedError() + + +class Expression: + """A compiled match expression as used by -k and -m. + + The expression can be evaluated against different matchers. + """ + + __slots__ = ("code",) + + def __init__(self, code: types.CodeType) -> None: + self.code = code + + @classmethod + def compile(self, input: str) -> "Expression": + """Compile a match expression. + + :param input: The input expression - one line. + """ + astexpr = expression(Scanner(input)) + code: types.CodeType = compile( + astexpr, + filename="", + mode="eval", + ) + return Expression(code) + + def evaluate(self, matcher: Callable[[str], bool]) -> bool: + """Evaluate the match expression. + + :param matcher: + Given an identifier, should return whether it matches or not. + Should be prepared to handle arbitrary strings as input. + + :returns: Whether the expression matches or not. + """ + ret: bool = eval(self.code, {"__builtins__": {}}, MatcherAdapter(matcher)) + return ret diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/mark/structures.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/mark/structures.py new file mode 100644 index 0000000000..0e42cd8de5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/mark/structures.py @@ -0,0 +1,595 @@ +import collections.abc +import inspect +import warnings +from typing import Any +from typing import Callable +from typing import Collection +from typing import Iterable +from typing import Iterator +from typing import List +from typing import Mapping +from typing import MutableMapping +from typing import NamedTuple +from typing import Optional +from typing import overload +from typing import Sequence +from typing import Set +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import TypeVar +from typing import Union + +import attr + +from .._code import getfslineno +from ..compat import ascii_escaped +from ..compat import final +from ..compat import NOTSET +from ..compat import NotSetType +from _pytest.config import Config +from _pytest.deprecated import check_ispytest +from _pytest.outcomes import fail +from _pytest.warning_types import PytestUnknownMarkWarning + +if TYPE_CHECKING: + from ..nodes import Node + + +EMPTY_PARAMETERSET_OPTION = "empty_parameter_set_mark" + + +def istestfunc(func) -> bool: + return callable(func) and getattr(func, "__name__", "") != "" + + +def get_empty_parameterset_mark( + config: Config, argnames: Sequence[str], func +) -> "MarkDecorator": + from ..nodes import Collector + + fs, lineno = getfslineno(func) + reason = "got empty parameter set %r, function %s at %s:%d" % ( + argnames, + func.__name__, + fs, + lineno, + ) + + requested_mark = config.getini(EMPTY_PARAMETERSET_OPTION) + if requested_mark in ("", None, "skip"): + mark = MARK_GEN.skip(reason=reason) + elif requested_mark == "xfail": + mark = MARK_GEN.xfail(reason=reason, run=False) + elif requested_mark == "fail_at_collect": + f_name = func.__name__ + _, lineno = getfslineno(func) + raise Collector.CollectError( + "Empty parameter set in '%s' at line %d" % (f_name, lineno + 1) + ) + else: + raise LookupError(requested_mark) + return mark + + +class ParameterSet( + NamedTuple( + "ParameterSet", + [ + ("values", Sequence[Union[object, NotSetType]]), + ("marks", Collection[Union["MarkDecorator", "Mark"]]), + ("id", Optional[str]), + ], + ) +): + @classmethod + def param( + cls, + *values: object, + marks: Union["MarkDecorator", Collection[Union["MarkDecorator", "Mark"]]] = (), + id: Optional[str] = None, + ) -> "ParameterSet": + if isinstance(marks, MarkDecorator): + marks = (marks,) + else: + assert isinstance(marks, collections.abc.Collection) + + if id is not None: + if not isinstance(id, str): + raise TypeError(f"Expected id to be a string, got {type(id)}: {id!r}") + id = ascii_escaped(id) + return cls(values, marks, id) + + @classmethod + def extract_from( + cls, + parameterset: Union["ParameterSet", Sequence[object], object], + force_tuple: bool = False, + ) -> "ParameterSet": + """Extract from an object or objects. + + :param parameterset: + A legacy style parameterset that may or may not be a tuple, + and may or may not be wrapped into a mess of mark objects. + + :param force_tuple: + Enforce tuple wrapping so single argument tuple values + don't get decomposed and break tests. + """ + + if isinstance(parameterset, cls): + return parameterset + if force_tuple: + return cls.param(parameterset) + else: + # TODO: Refactor to fix this type-ignore. Currently the following + # passes type-checking but crashes: + # + # @pytest.mark.parametrize(('x', 'y'), [1, 2]) + # def test_foo(x, y): pass + return cls(parameterset, marks=[], id=None) # type: ignore[arg-type] + + @staticmethod + def _parse_parametrize_args( + argnames: Union[str, List[str], Tuple[str, ...]], + argvalues: Iterable[Union["ParameterSet", Sequence[object], object]], + *args, + **kwargs, + ) -> Tuple[Union[List[str], Tuple[str, ...]], bool]: + if not isinstance(argnames, (tuple, list)): + argnames = [x.strip() for x in argnames.split(",") if x.strip()] + force_tuple = len(argnames) == 1 + else: + force_tuple = False + return argnames, force_tuple + + @staticmethod + def _parse_parametrize_parameters( + argvalues: Iterable[Union["ParameterSet", Sequence[object], object]], + force_tuple: bool, + ) -> List["ParameterSet"]: + return [ + ParameterSet.extract_from(x, force_tuple=force_tuple) for x in argvalues + ] + + @classmethod + def _for_parametrize( + cls, + argnames: Union[str, List[str], Tuple[str, ...]], + argvalues: Iterable[Union["ParameterSet", Sequence[object], object]], + func, + config: Config, + nodeid: str, + ) -> Tuple[Union[List[str], Tuple[str, ...]], List["ParameterSet"]]: + argnames, force_tuple = cls._parse_parametrize_args(argnames, argvalues) + parameters = cls._parse_parametrize_parameters(argvalues, force_tuple) + del argvalues + + if parameters: + # Check all parameter sets have the correct number of values. + for param in parameters: + if len(param.values) != len(argnames): + msg = ( + '{nodeid}: in "parametrize" the number of names ({names_len}):\n' + " {names}\n" + "must be equal to the number of values ({values_len}):\n" + " {values}" + ) + fail( + msg.format( + nodeid=nodeid, + values=param.values, + names=argnames, + names_len=len(argnames), + values_len=len(param.values), + ), + pytrace=False, + ) + else: + # Empty parameter set (likely computed at runtime): create a single + # parameter set with NOTSET values, with the "empty parameter set" mark applied to it. + mark = get_empty_parameterset_mark(config, argnames, func) + parameters.append( + ParameterSet(values=(NOTSET,) * len(argnames), marks=[mark], id=None) + ) + return argnames, parameters + + +@final +@attr.s(frozen=True, init=False, auto_attribs=True) +class Mark: + #: Name of the mark. + name: str + #: Positional arguments of the mark decorator. + args: Tuple[Any, ...] + #: Keyword arguments of the mark decorator. + kwargs: Mapping[str, Any] + + #: Source Mark for ids with parametrize Marks. + _param_ids_from: Optional["Mark"] = attr.ib(default=None, repr=False) + #: Resolved/generated ids with parametrize Marks. + _param_ids_generated: Optional[Sequence[str]] = attr.ib(default=None, repr=False) + + def __init__( + self, + name: str, + args: Tuple[Any, ...], + kwargs: Mapping[str, Any], + param_ids_from: Optional["Mark"] = None, + param_ids_generated: Optional[Sequence[str]] = None, + *, + _ispytest: bool = False, + ) -> None: + """:meta private:""" + check_ispytest(_ispytest) + # Weirdness to bypass frozen=True. + object.__setattr__(self, "name", name) + object.__setattr__(self, "args", args) + object.__setattr__(self, "kwargs", kwargs) + object.__setattr__(self, "_param_ids_from", param_ids_from) + object.__setattr__(self, "_param_ids_generated", param_ids_generated) + + def _has_param_ids(self) -> bool: + return "ids" in self.kwargs or len(self.args) >= 4 + + def combined_with(self, other: "Mark") -> "Mark": + """Return a new Mark which is a combination of this + Mark and another Mark. + + Combines by appending args and merging kwargs. + + :param Mark other: The mark to combine with. + :rtype: Mark + """ + assert self.name == other.name + + # Remember source of ids with parametrize Marks. + param_ids_from: Optional[Mark] = None + if self.name == "parametrize": + if other._has_param_ids(): + param_ids_from = other + elif self._has_param_ids(): + param_ids_from = self + + return Mark( + self.name, + self.args + other.args, + dict(self.kwargs, **other.kwargs), + param_ids_from=param_ids_from, + _ispytest=True, + ) + + +# A generic parameter designating an object to which a Mark may +# be applied -- a test function (callable) or class. +# Note: a lambda is not allowed, but this can't be represented. +Markable = TypeVar("Markable", bound=Union[Callable[..., object], type]) + + +@attr.s(init=False, auto_attribs=True) +class MarkDecorator: + """A decorator for applying a mark on test functions and classes. + + ``MarkDecorators`` are created with ``pytest.mark``:: + + mark1 = pytest.mark.NAME # Simple MarkDecorator + mark2 = pytest.mark.NAME(name1=value) # Parametrized MarkDecorator + + and can then be applied as decorators to test functions:: + + @mark2 + def test_function(): + pass + + When a ``MarkDecorator`` is called, it does the following: + + 1. If called with a single class as its only positional argument and no + additional keyword arguments, it attaches the mark to the class so it + gets applied automatically to all test cases found in that class. + + 2. If called with a single function as its only positional argument and + no additional keyword arguments, it attaches the mark to the function, + containing all the arguments already stored internally in the + ``MarkDecorator``. + + 3. When called in any other case, it returns a new ``MarkDecorator`` + instance with the original ``MarkDecorator``'s content updated with + the arguments passed to this call. + + Note: The rules above prevent a ``MarkDecorator`` from storing only a + single function or class reference as its positional argument with no + additional keyword or positional arguments. You can work around this by + using `with_args()`. + """ + + mark: Mark + + def __init__(self, mark: Mark, *, _ispytest: bool = False) -> None: + """:meta private:""" + check_ispytest(_ispytest) + self.mark = mark + + @property + def name(self) -> str: + """Alias for mark.name.""" + return self.mark.name + + @property + def args(self) -> Tuple[Any, ...]: + """Alias for mark.args.""" + return self.mark.args + + @property + def kwargs(self) -> Mapping[str, Any]: + """Alias for mark.kwargs.""" + return self.mark.kwargs + + @property + def markname(self) -> str: + """:meta private:""" + return self.name # for backward-compat (2.4.1 had this attr) + + def with_args(self, *args: object, **kwargs: object) -> "MarkDecorator": + """Return a MarkDecorator with extra arguments added. + + Unlike calling the MarkDecorator, with_args() can be used even + if the sole argument is a callable/class. + """ + mark = Mark(self.name, args, kwargs, _ispytest=True) + return MarkDecorator(self.mark.combined_with(mark), _ispytest=True) + + # Type ignored because the overloads overlap with an incompatible + # return type. Not much we can do about that. Thankfully mypy picks + # the first match so it works out even if we break the rules. + @overload + def __call__(self, arg: Markable) -> Markable: # type: ignore[misc] + pass + + @overload + def __call__(self, *args: object, **kwargs: object) -> "MarkDecorator": + pass + + def __call__(self, *args: object, **kwargs: object): + """Call the MarkDecorator.""" + if args and not kwargs: + func = args[0] + is_class = inspect.isclass(func) + if len(args) == 1 and (istestfunc(func) or is_class): + store_mark(func, self.mark) + return func + return self.with_args(*args, **kwargs) + + +def get_unpacked_marks(obj: object) -> Iterable[Mark]: + """Obtain the unpacked marks that are stored on an object.""" + mark_list = getattr(obj, "pytestmark", []) + if not isinstance(mark_list, list): + mark_list = [mark_list] + return normalize_mark_list(mark_list) + + +def normalize_mark_list( + mark_list: Iterable[Union[Mark, MarkDecorator]] +) -> Iterable[Mark]: + """ + Normalize an iterable of Mark or MarkDecorator objects into a list of marks + by retrieving the `mark` attribute on MarkDecorator instances. + + :param mark_list: marks to normalize + :returns: A new list of the extracted Mark objects + """ + for mark in mark_list: + mark_obj = getattr(mark, "mark", mark) + if not isinstance(mark_obj, Mark): + raise TypeError(f"got {repr(mark_obj)} instead of Mark") + yield mark_obj + + +def store_mark(obj, mark: Mark) -> None: + """Store a Mark on an object. + + This is used to implement the Mark declarations/decorators correctly. + """ + assert isinstance(mark, Mark), mark + # Always reassign name to avoid updating pytestmark in a reference that + # was only borrowed. + obj.pytestmark = [*get_unpacked_marks(obj), mark] + + +# Typing for builtin pytest marks. This is cheating; it gives builtin marks +# special privilege, and breaks modularity. But practicality beats purity... +if TYPE_CHECKING: + from _pytest.scope import _ScopeName + + class _SkipMarkDecorator(MarkDecorator): + @overload # type: ignore[override,misc] + def __call__(self, arg: Markable) -> Markable: + ... + + @overload + def __call__(self, reason: str = ...) -> "MarkDecorator": + ... + + class _SkipifMarkDecorator(MarkDecorator): + def __call__( # type: ignore[override] + self, + condition: Union[str, bool] = ..., + *conditions: Union[str, bool], + reason: str = ..., + ) -> MarkDecorator: + ... + + class _XfailMarkDecorator(MarkDecorator): + @overload # type: ignore[override,misc] + def __call__(self, arg: Markable) -> Markable: + ... + + @overload + def __call__( + self, + condition: Union[str, bool] = ..., + *conditions: Union[str, bool], + reason: str = ..., + run: bool = ..., + raises: Union[Type[BaseException], Tuple[Type[BaseException], ...]] = ..., + strict: bool = ..., + ) -> MarkDecorator: + ... + + class _ParametrizeMarkDecorator(MarkDecorator): + def __call__( # type: ignore[override] + self, + argnames: Union[str, List[str], Tuple[str, ...]], + argvalues: Iterable[Union[ParameterSet, Sequence[object], object]], + *, + indirect: Union[bool, Sequence[str]] = ..., + ids: Optional[ + Union[ + Iterable[Union[None, str, float, int, bool]], + Callable[[Any], Optional[object]], + ] + ] = ..., + scope: Optional[_ScopeName] = ..., + ) -> MarkDecorator: + ... + + class _UsefixturesMarkDecorator(MarkDecorator): + def __call__(self, *fixtures: str) -> MarkDecorator: # type: ignore[override] + ... + + class _FilterwarningsMarkDecorator(MarkDecorator): + def __call__(self, *filters: str) -> MarkDecorator: # type: ignore[override] + ... + + +@final +class MarkGenerator: + """Factory for :class:`MarkDecorator` objects - exposed as + a ``pytest.mark`` singleton instance. + + Example:: + + import pytest + + @pytest.mark.slowtest + def test_function(): + pass + + applies a 'slowtest' :class:`Mark` on ``test_function``. + """ + + # See TYPE_CHECKING above. + if TYPE_CHECKING: + skip: _SkipMarkDecorator + skipif: _SkipifMarkDecorator + xfail: _XfailMarkDecorator + parametrize: _ParametrizeMarkDecorator + usefixtures: _UsefixturesMarkDecorator + filterwarnings: _FilterwarningsMarkDecorator + + def __init__(self, *, _ispytest: bool = False) -> None: + check_ispytest(_ispytest) + self._config: Optional[Config] = None + self._markers: Set[str] = set() + + def __getattr__(self, name: str) -> MarkDecorator: + """Generate a new :class:`MarkDecorator` with the given name.""" + if name[0] == "_": + raise AttributeError("Marker name must NOT start with underscore") + + if self._config is not None: + # We store a set of markers as a performance optimisation - if a mark + # name is in the set we definitely know it, but a mark may be known and + # not in the set. We therefore start by updating the set! + if name not in self._markers: + for line in self._config.getini("markers"): + # example lines: "skipif(condition): skip the given test if..." + # or "hypothesis: tests which use Hypothesis", so to get the + # marker name we split on both `:` and `(`. + marker = line.split(":")[0].split("(")[0].strip() + self._markers.add(marker) + + # If the name is not in the set of known marks after updating, + # then it really is time to issue a warning or an error. + if name not in self._markers: + if self._config.option.strict_markers or self._config.option.strict: + fail( + f"{name!r} not found in `markers` configuration option", + pytrace=False, + ) + + # Raise a specific error for common misspellings of "parametrize". + if name in ["parameterize", "parametrise", "parameterise"]: + __tracebackhide__ = True + fail(f"Unknown '{name}' mark, did you mean 'parametrize'?") + + warnings.warn( + "Unknown pytest.mark.%s - is this a typo? You can register " + "custom marks to avoid this warning - for details, see " + "https://docs.pytest.org/en/stable/how-to/mark.html" % name, + PytestUnknownMarkWarning, + 2, + ) + + return MarkDecorator(Mark(name, (), {}, _ispytest=True), _ispytest=True) + + +MARK_GEN = MarkGenerator(_ispytest=True) + + +@final +class NodeKeywords(MutableMapping[str, Any]): + __slots__ = ("node", "parent", "_markers") + + def __init__(self, node: "Node") -> None: + self.node = node + self.parent = node.parent + self._markers = {node.name: True} + + def __getitem__(self, key: str) -> Any: + try: + return self._markers[key] + except KeyError: + if self.parent is None: + raise + return self.parent.keywords[key] + + def __setitem__(self, key: str, value: Any) -> None: + self._markers[key] = value + + # Note: we could've avoided explicitly implementing some of the methods + # below and use the collections.abc fallback, but that would be slow. + + def __contains__(self, key: object) -> bool: + return ( + key in self._markers + or self.parent is not None + and key in self.parent.keywords + ) + + def update( # type: ignore[override] + self, + other: Union[Mapping[str, Any], Iterable[Tuple[str, Any]]] = (), + **kwds: Any, + ) -> None: + self._markers.update(other) + self._markers.update(kwds) + + def __delitem__(self, key: str) -> None: + raise ValueError("cannot delete key in keywords dict") + + def __iter__(self) -> Iterator[str]: + # Doesn't need to be fast. + yield from self._markers + if self.parent is not None: + for keyword in self.parent.keywords: + # self._marks and self.parent.keywords can have duplicates. + if keyword not in self._markers: + yield keyword + + def __len__(self) -> int: + # Doesn't need to be fast. + return sum(1 for keyword in self) + + def __repr__(self) -> str: + return f"" diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/monkeypatch.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/monkeypatch.py new file mode 100644 index 0000000000..31f95a95ab --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/monkeypatch.py @@ -0,0 +1,383 @@ +"""Monkeypatching and mocking functionality.""" +import os +import re +import sys +import warnings +from contextlib import contextmanager +from typing import Any +from typing import Generator +from typing import List +from typing import MutableMapping +from typing import Optional +from typing import overload +from typing import Tuple +from typing import TypeVar +from typing import Union + +from _pytest.compat import final +from _pytest.fixtures import fixture +from _pytest.warning_types import PytestWarning + +RE_IMPORT_ERROR_NAME = re.compile(r"^No module named (.*)$") + + +K = TypeVar("K") +V = TypeVar("V") + + +@fixture +def monkeypatch() -> Generator["MonkeyPatch", None, None]: + """A convenient fixture for monkey-patching. + + The fixture provides these methods to modify objects, dictionaries or + os.environ:: + + monkeypatch.setattr(obj, name, value, raising=True) + monkeypatch.delattr(obj, name, raising=True) + monkeypatch.setitem(mapping, name, value) + monkeypatch.delitem(obj, name, raising=True) + monkeypatch.setenv(name, value, prepend=None) + monkeypatch.delenv(name, raising=True) + monkeypatch.syspath_prepend(path) + monkeypatch.chdir(path) + + All modifications will be undone after the requesting test function or + fixture has finished. The ``raising`` parameter determines if a KeyError + or AttributeError will be raised if the set/deletion operation has no target. + """ + mpatch = MonkeyPatch() + yield mpatch + mpatch.undo() + + +def resolve(name: str) -> object: + # Simplified from zope.dottedname. + parts = name.split(".") + + used = parts.pop(0) + found = __import__(used) + for part in parts: + used += "." + part + try: + found = getattr(found, part) + except AttributeError: + pass + else: + continue + # We use explicit un-nesting of the handling block in order + # to avoid nested exceptions. + try: + __import__(used) + except ImportError as ex: + expected = str(ex).split()[-1] + if expected == used: + raise + else: + raise ImportError(f"import error in {used}: {ex}") from ex + found = annotated_getattr(found, part, used) + return found + + +def annotated_getattr(obj: object, name: str, ann: str) -> object: + try: + obj = getattr(obj, name) + except AttributeError as e: + raise AttributeError( + "{!r} object at {} has no attribute {!r}".format( + type(obj).__name__, ann, name + ) + ) from e + return obj + + +def derive_importpath(import_path: str, raising: bool) -> Tuple[str, object]: + if not isinstance(import_path, str) or "." not in import_path: + raise TypeError(f"must be absolute import path string, not {import_path!r}") + module, attr = import_path.rsplit(".", 1) + target = resolve(module) + if raising: + annotated_getattr(target, attr, ann=module) + return attr, target + + +class Notset: + def __repr__(self) -> str: + return "" + + +notset = Notset() + + +@final +class MonkeyPatch: + """Helper to conveniently monkeypatch attributes/items/environment + variables/syspath. + + Returned by the :fixture:`monkeypatch` fixture. + + :versionchanged:: 6.2 + Can now also be used directly as `pytest.MonkeyPatch()`, for when + the fixture is not available. In this case, use + :meth:`with MonkeyPatch.context() as mp: ` or remember to call + :meth:`undo` explicitly. + """ + + def __init__(self) -> None: + self._setattr: List[Tuple[object, str, object]] = [] + self._setitem: List[Tuple[MutableMapping[Any, Any], object, object]] = [] + self._cwd: Optional[str] = None + self._savesyspath: Optional[List[str]] = None + + @classmethod + @contextmanager + def context(cls) -> Generator["MonkeyPatch", None, None]: + """Context manager that returns a new :class:`MonkeyPatch` object + which undoes any patching done inside the ``with`` block upon exit. + + Example: + + .. code-block:: python + + import functools + + + def test_partial(monkeypatch): + with monkeypatch.context() as m: + m.setattr(functools, "partial", 3) + + Useful in situations where it is desired to undo some patches before the test ends, + such as mocking ``stdlib`` functions that might break pytest itself if mocked (for examples + of this see :issue:`3290`). + """ + m = cls() + try: + yield m + finally: + m.undo() + + @overload + def setattr( + self, + target: str, + name: object, + value: Notset = ..., + raising: bool = ..., + ) -> None: + ... + + @overload + def setattr( + self, + target: object, + name: str, + value: object, + raising: bool = ..., + ) -> None: + ... + + def setattr( + self, + target: Union[str, object], + name: Union[object, str], + value: object = notset, + raising: bool = True, + ) -> None: + """Set attribute value on target, memorizing the old value. + + For convenience you can specify a string as ``target`` which + will be interpreted as a dotted import path, with the last part + being the attribute name. For example, + ``monkeypatch.setattr("os.getcwd", lambda: "/")`` + would set the ``getcwd`` function of the ``os`` module. + + Raises AttributeError if the attribute does not exist, unless + ``raising`` is set to False. + """ + __tracebackhide__ = True + import inspect + + if isinstance(value, Notset): + if not isinstance(target, str): + raise TypeError( + "use setattr(target, name, value) or " + "setattr(target, value) with target being a dotted " + "import string" + ) + value = name + name, target = derive_importpath(target, raising) + else: + if not isinstance(name, str): + raise TypeError( + "use setattr(target, name, value) with name being a string or " + "setattr(target, value) with target being a dotted " + "import string" + ) + + oldval = getattr(target, name, notset) + if raising and oldval is notset: + raise AttributeError(f"{target!r} has no attribute {name!r}") + + # avoid class descriptors like staticmethod/classmethod + if inspect.isclass(target): + oldval = target.__dict__.get(name, notset) + self._setattr.append((target, name, oldval)) + setattr(target, name, value) + + def delattr( + self, + target: Union[object, str], + name: Union[str, Notset] = notset, + raising: bool = True, + ) -> None: + """Delete attribute ``name`` from ``target``. + + If no ``name`` is specified and ``target`` is a string + it will be interpreted as a dotted import path with the + last part being the attribute name. + + Raises AttributeError it the attribute does not exist, unless + ``raising`` is set to False. + """ + __tracebackhide__ = True + import inspect + + if isinstance(name, Notset): + if not isinstance(target, str): + raise TypeError( + "use delattr(target, name) or " + "delattr(target) with target being a dotted " + "import string" + ) + name, target = derive_importpath(target, raising) + + if not hasattr(target, name): + if raising: + raise AttributeError(name) + else: + oldval = getattr(target, name, notset) + # Avoid class descriptors like staticmethod/classmethod. + if inspect.isclass(target): + oldval = target.__dict__.get(name, notset) + self._setattr.append((target, name, oldval)) + delattr(target, name) + + def setitem(self, dic: MutableMapping[K, V], name: K, value: V) -> None: + """Set dictionary entry ``name`` to value.""" + self._setitem.append((dic, name, dic.get(name, notset))) + dic[name] = value + + def delitem(self, dic: MutableMapping[K, V], name: K, raising: bool = True) -> None: + """Delete ``name`` from dict. + + Raises ``KeyError`` if it doesn't exist, unless ``raising`` is set to + False. + """ + if name not in dic: + if raising: + raise KeyError(name) + else: + self._setitem.append((dic, name, dic.get(name, notset))) + del dic[name] + + def setenv(self, name: str, value: str, prepend: Optional[str] = None) -> None: + """Set environment variable ``name`` to ``value``. + + If ``prepend`` is a character, read the current environment variable + value and prepend the ``value`` adjoined with the ``prepend`` + character. + """ + if not isinstance(value, str): + warnings.warn( # type: ignore[unreachable] + PytestWarning( + "Value of environment variable {name} type should be str, but got " + "{value!r} (type: {type}); converted to str implicitly".format( + name=name, value=value, type=type(value).__name__ + ) + ), + stacklevel=2, + ) + value = str(value) + if prepend and name in os.environ: + value = value + prepend + os.environ[name] + self.setitem(os.environ, name, value) + + def delenv(self, name: str, raising: bool = True) -> None: + """Delete ``name`` from the environment. + + Raises ``KeyError`` if it does not exist, unless ``raising`` is set to + False. + """ + environ: MutableMapping[str, str] = os.environ + self.delitem(environ, name, raising=raising) + + def syspath_prepend(self, path) -> None: + """Prepend ``path`` to ``sys.path`` list of import locations.""" + + if self._savesyspath is None: + self._savesyspath = sys.path[:] + sys.path.insert(0, str(path)) + + # https://github.com/pypa/setuptools/blob/d8b901bc/docs/pkg_resources.txt#L162-L171 + # this is only needed when pkg_resources was already loaded by the namespace package + if "pkg_resources" in sys.modules: + from pkg_resources import fixup_namespace_packages + + fixup_namespace_packages(str(path)) + + # A call to syspathinsert() usually means that the caller wants to + # import some dynamically created files, thus with python3 we + # invalidate its import caches. + # This is especially important when any namespace package is in use, + # since then the mtime based FileFinder cache (that gets created in + # this case already) gets not invalidated when writing the new files + # quickly afterwards. + from importlib import invalidate_caches + + invalidate_caches() + + def chdir(self, path: Union[str, "os.PathLike[str]"]) -> None: + """Change the current working directory to the specified path. + + Path can be a string or a path object. + """ + if self._cwd is None: + self._cwd = os.getcwd() + os.chdir(path) + + def undo(self) -> None: + """Undo previous changes. + + This call consumes the undo stack. Calling it a second time has no + effect unless you do more monkeypatching after the undo call. + + There is generally no need to call `undo()`, since it is + called automatically during tear-down. + + Note that the same `monkeypatch` fixture is used across a + single test function invocation. If `monkeypatch` is used both by + the test function itself and one of the test fixtures, + calling `undo()` will undo all of the changes made in + both functions. + """ + for obj, name, value in reversed(self._setattr): + if value is not notset: + setattr(obj, name, value) + else: + delattr(obj, name) + self._setattr[:] = [] + for dictionary, key, value in reversed(self._setitem): + if value is notset: + try: + del dictionary[key] + except KeyError: + pass # Was already deleted, so we have the desired state. + else: + dictionary[key] = value + self._setitem[:] = [] + if self._savesyspath is not None: + sys.path[:] = self._savesyspath + self._savesyspath = None + + if self._cwd is not None: + os.chdir(self._cwd) + self._cwd = None diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/nodes.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/nodes.py new file mode 100644 index 0000000000..e49c1b003e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/nodes.py @@ -0,0 +1,762 @@ +import os +import warnings +from inspect import signature +from pathlib import Path +from typing import Any +from typing import Callable +from typing import cast +from typing import Iterable +from typing import Iterator +from typing import List +from typing import MutableMapping +from typing import Optional +from typing import overload +from typing import Set +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import TypeVar +from typing import Union + +import _pytest._code +from _pytest._code import getfslineno +from _pytest._code.code import ExceptionInfo +from _pytest._code.code import TerminalRepr +from _pytest.compat import cached_property +from _pytest.compat import LEGACY_PATH +from _pytest.config import Config +from _pytest.config import ConftestImportFailure +from _pytest.deprecated import FSCOLLECTOR_GETHOOKPROXY_ISINITPATH +from _pytest.deprecated import NODE_CTOR_FSPATH_ARG +from _pytest.mark.structures import Mark +from _pytest.mark.structures import MarkDecorator +from _pytest.mark.structures import NodeKeywords +from _pytest.outcomes import fail +from _pytest.pathlib import absolutepath +from _pytest.pathlib import commonpath +from _pytest.stash import Stash +from _pytest.warning_types import PytestWarning + +if TYPE_CHECKING: + # Imported here due to circular import. + from _pytest.main import Session + from _pytest._code.code import _TracebackStyle + + +SEP = "/" + +tracebackcutdir = Path(_pytest.__file__).parent + + +def iterparentnodeids(nodeid: str) -> Iterator[str]: + """Return the parent node IDs of a given node ID, inclusive. + + For the node ID + + "testing/code/test_excinfo.py::TestFormattedExcinfo::test_repr_source" + + the result would be + + "" + "testing" + "testing/code" + "testing/code/test_excinfo.py" + "testing/code/test_excinfo.py::TestFormattedExcinfo" + "testing/code/test_excinfo.py::TestFormattedExcinfo::test_repr_source" + + Note that / components are only considered until the first ::. + """ + pos = 0 + first_colons: Optional[int] = nodeid.find("::") + if first_colons == -1: + first_colons = None + # The root Session node - always present. + yield "" + # Eagerly consume SEP parts until first colons. + while True: + at = nodeid.find(SEP, pos, first_colons) + if at == -1: + break + if at > 0: + yield nodeid[:at] + pos = at + len(SEP) + # Eagerly consume :: parts. + while True: + at = nodeid.find("::", pos) + if at == -1: + break + if at > 0: + yield nodeid[:at] + pos = at + len("::") + # The node ID itself. + if nodeid: + yield nodeid + + +def _check_path(path: Path, fspath: LEGACY_PATH) -> None: + if Path(fspath) != path: + raise ValueError( + f"Path({fspath!r}) != {path!r}\n" + "if both path and fspath are given they need to be equal" + ) + + +def _imply_path( + node_type: Type["Node"], + path: Optional[Path], + fspath: Optional[LEGACY_PATH], +) -> Path: + if fspath is not None: + warnings.warn( + NODE_CTOR_FSPATH_ARG.format( + node_type_name=node_type.__name__, + ), + stacklevel=3, + ) + if path is not None: + if fspath is not None: + _check_path(path, fspath) + return path + else: + assert fspath is not None + return Path(fspath) + + +_NodeType = TypeVar("_NodeType", bound="Node") + + +class NodeMeta(type): + def __call__(self, *k, **kw): + msg = ( + "Direct construction of {name} has been deprecated, please use {name}.from_parent.\n" + "See " + "https://docs.pytest.org/en/stable/deprecations.html#node-construction-changed-to-node-from-parent" + " for more details." + ).format(name=f"{self.__module__}.{self.__name__}") + fail(msg, pytrace=False) + + def _create(self, *k, **kw): + try: + return super().__call__(*k, **kw) + except TypeError: + sig = signature(getattr(self, "__init__")) + known_kw = {k: v for k, v in kw.items() if k in sig.parameters} + from .warning_types import PytestDeprecationWarning + + warnings.warn( + PytestDeprecationWarning( + f"{self} is not using a cooperative constructor and only takes {set(known_kw)}.\n" + "See https://docs.pytest.org/en/stable/deprecations.html" + "#constructors-of-custom-pytest-node-subclasses-should-take-kwargs " + "for more details." + ) + ) + + return super().__call__(*k, **known_kw) + + +class Node(metaclass=NodeMeta): + """Base class for Collector and Item, the components of the test + collection tree. + + Collector subclasses have children; Items are leaf nodes. + """ + + # Implemented in the legacypath plugin. + #: A ``LEGACY_PATH`` copy of the :attr:`path` attribute. Intended for usage + #: for methods not migrated to ``pathlib.Path`` yet, such as + #: :meth:`Item.reportinfo`. Will be deprecated in a future release, prefer + #: using :attr:`path` instead. + fspath: LEGACY_PATH + + # Use __slots__ to make attribute access faster. + # Note that __dict__ is still available. + __slots__ = ( + "name", + "parent", + "config", + "session", + "path", + "_nodeid", + "_store", + "__dict__", + ) + + def __init__( + self, + name: str, + parent: "Optional[Node]" = None, + config: Optional[Config] = None, + session: "Optional[Session]" = None, + fspath: Optional[LEGACY_PATH] = None, + path: Optional[Path] = None, + nodeid: Optional[str] = None, + ) -> None: + #: A unique name within the scope of the parent node. + self.name = name + + #: The parent collector node. + self.parent = parent + + if config: + #: The pytest config object. + self.config: Config = config + else: + if not parent: + raise TypeError("config or parent must be provided") + self.config = parent.config + + if session: + #: The pytest session this node is part of. + self.session = session + else: + if not parent: + raise TypeError("session or parent must be provided") + self.session = parent.session + + if path is None and fspath is None: + path = getattr(parent, "path", None) + #: Filesystem path where this node was collected from (can be None). + self.path: Path = _imply_path(type(self), path, fspath=fspath) + + # The explicit annotation is to avoid publicly exposing NodeKeywords. + #: Keywords/markers collected from all scopes. + self.keywords: MutableMapping[str, Any] = NodeKeywords(self) + + #: The marker objects belonging to this node. + self.own_markers: List[Mark] = [] + + #: Allow adding of extra keywords to use for matching. + self.extra_keyword_matches: Set[str] = set() + + if nodeid is not None: + assert "::()" not in nodeid + self._nodeid = nodeid + else: + if not self.parent: + raise TypeError("nodeid or parent must be provided") + self._nodeid = self.parent.nodeid + "::" + self.name + + #: A place where plugins can store information on the node for their + #: own use. + #: + #: :type: Stash + self.stash = Stash() + # Deprecated alias. Was never public. Can be removed in a few releases. + self._store = self.stash + + @classmethod + def from_parent(cls, parent: "Node", **kw): + """Public constructor for Nodes. + + This indirection got introduced in order to enable removing + the fragile logic from the node constructors. + + Subclasses can use ``super().from_parent(...)`` when overriding the + construction. + + :param parent: The parent node of this Node. + """ + if "config" in kw: + raise TypeError("config is not a valid argument for from_parent") + if "session" in kw: + raise TypeError("session is not a valid argument for from_parent") + return cls._create(parent=parent, **kw) + + @property + def ihook(self): + """fspath-sensitive hook proxy used to call pytest hooks.""" + return self.session.gethookproxy(self.path) + + def __repr__(self) -> str: + return "<{} {}>".format(self.__class__.__name__, getattr(self, "name", None)) + + def warn(self, warning: Warning) -> None: + """Issue a warning for this Node. + + Warnings will be displayed after the test session, unless explicitly suppressed. + + :param Warning warning: + The warning instance to issue. + + :raises ValueError: If ``warning`` instance is not a subclass of Warning. + + Example usage: + + .. code-block:: python + + node.warn(PytestWarning("some message")) + node.warn(UserWarning("some message")) + + .. versionchanged:: 6.2 + Any subclass of :class:`Warning` is now accepted, rather than only + :class:`PytestWarning ` subclasses. + """ + # enforce type checks here to avoid getting a generic type error later otherwise. + if not isinstance(warning, Warning): + raise ValueError( + "warning must be an instance of Warning or subclass, got {!r}".format( + warning + ) + ) + path, lineno = get_fslocation_from_item(self) + assert lineno is not None + warnings.warn_explicit( + warning, + category=None, + filename=str(path), + lineno=lineno + 1, + ) + + # Methods for ordering nodes. + + @property + def nodeid(self) -> str: + """A ::-separated string denoting its collection tree address.""" + return self._nodeid + + def __hash__(self) -> int: + return hash(self._nodeid) + + def setup(self) -> None: + pass + + def teardown(self) -> None: + pass + + def listchain(self) -> List["Node"]: + """Return list of all parent collectors up to self, starting from + the root of collection tree.""" + chain = [] + item: Optional[Node] = self + while item is not None: + chain.append(item) + item = item.parent + chain.reverse() + return chain + + def add_marker( + self, marker: Union[str, MarkDecorator], append: bool = True + ) -> None: + """Dynamically add a marker object to the node. + + :param append: + Whether to append the marker, or prepend it. + """ + from _pytest.mark import MARK_GEN + + if isinstance(marker, MarkDecorator): + marker_ = marker + elif isinstance(marker, str): + marker_ = getattr(MARK_GEN, marker) + else: + raise ValueError("is not a string or pytest.mark.* Marker") + self.keywords[marker_.name] = marker_ + if append: + self.own_markers.append(marker_.mark) + else: + self.own_markers.insert(0, marker_.mark) + + def iter_markers(self, name: Optional[str] = None) -> Iterator[Mark]: + """Iterate over all markers of the node. + + :param name: If given, filter the results by the name attribute. + """ + return (x[1] for x in self.iter_markers_with_node(name=name)) + + def iter_markers_with_node( + self, name: Optional[str] = None + ) -> Iterator[Tuple["Node", Mark]]: + """Iterate over all markers of the node. + + :param name: If given, filter the results by the name attribute. + :returns: An iterator of (node, mark) tuples. + """ + for node in reversed(self.listchain()): + for mark in node.own_markers: + if name is None or getattr(mark, "name", None) == name: + yield node, mark + + @overload + def get_closest_marker(self, name: str) -> Optional[Mark]: + ... + + @overload + def get_closest_marker(self, name: str, default: Mark) -> Mark: + ... + + def get_closest_marker( + self, name: str, default: Optional[Mark] = None + ) -> Optional[Mark]: + """Return the first marker matching the name, from closest (for + example function) to farther level (for example module level). + + :param default: Fallback return value if no marker was found. + :param name: Name to filter by. + """ + return next(self.iter_markers(name=name), default) + + def listextrakeywords(self) -> Set[str]: + """Return a set of all extra keywords in self and any parents.""" + extra_keywords: Set[str] = set() + for item in self.listchain(): + extra_keywords.update(item.extra_keyword_matches) + return extra_keywords + + def listnames(self) -> List[str]: + return [x.name for x in self.listchain()] + + def addfinalizer(self, fin: Callable[[], object]) -> None: + """Register a function to be called when this node is finalized. + + This method can only be called when this node is active + in a setup chain, for example during self.setup(). + """ + self.session._setupstate.addfinalizer(fin, self) + + def getparent(self, cls: Type[_NodeType]) -> Optional[_NodeType]: + """Get the next parent node (including self) which is an instance of + the given class.""" + current: Optional[Node] = self + while current and not isinstance(current, cls): + current = current.parent + assert current is None or isinstance(current, cls) + return current + + def _prunetraceback(self, excinfo: ExceptionInfo[BaseException]) -> None: + pass + + def _repr_failure_py( + self, + excinfo: ExceptionInfo[BaseException], + style: "Optional[_TracebackStyle]" = None, + ) -> TerminalRepr: + from _pytest.fixtures import FixtureLookupError + + if isinstance(excinfo.value, ConftestImportFailure): + excinfo = ExceptionInfo.from_exc_info(excinfo.value.excinfo) + if isinstance(excinfo.value, fail.Exception): + if not excinfo.value.pytrace: + style = "value" + if isinstance(excinfo.value, FixtureLookupError): + return excinfo.value.formatrepr() + if self.config.getoption("fulltrace", False): + style = "long" + else: + tb = _pytest._code.Traceback([excinfo.traceback[-1]]) + self._prunetraceback(excinfo) + if len(excinfo.traceback) == 0: + excinfo.traceback = tb + if style == "auto": + style = "long" + # XXX should excinfo.getrepr record all data and toterminal() process it? + if style is None: + if self.config.getoption("tbstyle", "auto") == "short": + style = "short" + else: + style = "long" + + if self.config.getoption("verbose", 0) > 1: + truncate_locals = False + else: + truncate_locals = True + + # excinfo.getrepr() formats paths relative to the CWD if `abspath` is False. + # It is possible for a fixture/test to change the CWD while this code runs, which + # would then result in the user seeing confusing paths in the failure message. + # To fix this, if the CWD changed, always display the full absolute path. + # It will be better to just always display paths relative to invocation_dir, but + # this requires a lot of plumbing (#6428). + try: + abspath = Path(os.getcwd()) != self.config.invocation_params.dir + except OSError: + abspath = True + + return excinfo.getrepr( + funcargs=True, + abspath=abspath, + showlocals=self.config.getoption("showlocals", False), + style=style, + tbfilter=False, # pruned already, or in --fulltrace mode. + truncate_locals=truncate_locals, + ) + + def repr_failure( + self, + excinfo: ExceptionInfo[BaseException], + style: "Optional[_TracebackStyle]" = None, + ) -> Union[str, TerminalRepr]: + """Return a representation of a collection or test failure. + + .. seealso:: :ref:`non-python tests` + + :param excinfo: Exception information for the failure. + """ + return self._repr_failure_py(excinfo, style) + + +def get_fslocation_from_item(node: "Node") -> Tuple[Union[str, Path], Optional[int]]: + """Try to extract the actual location from a node, depending on available attributes: + + * "location": a pair (path, lineno) + * "obj": a Python object that the node wraps. + * "fspath": just a path + + :rtype: A tuple of (str|Path, int) with filename and line number. + """ + # See Item.location. + location: Optional[Tuple[str, Optional[int], str]] = getattr(node, "location", None) + if location is not None: + return location[:2] + obj = getattr(node, "obj", None) + if obj is not None: + return getfslineno(obj) + return getattr(node, "fspath", "unknown location"), -1 + + +class Collector(Node): + """Collector instances create children through collect() and thus + iteratively build a tree.""" + + class CollectError(Exception): + """An error during collection, contains a custom message.""" + + def collect(self) -> Iterable[Union["Item", "Collector"]]: + """Return a list of children (items and collectors) for this + collection node.""" + raise NotImplementedError("abstract") + + # TODO: This omits the style= parameter which breaks Liskov Substitution. + def repr_failure( # type: ignore[override] + self, excinfo: ExceptionInfo[BaseException] + ) -> Union[str, TerminalRepr]: + """Return a representation of a collection failure. + + :param excinfo: Exception information for the failure. + """ + if isinstance(excinfo.value, self.CollectError) and not self.config.getoption( + "fulltrace", False + ): + exc = excinfo.value + return str(exc.args[0]) + + # Respect explicit tbstyle option, but default to "short" + # (_repr_failure_py uses "long" with "fulltrace" option always). + tbstyle = self.config.getoption("tbstyle", "auto") + if tbstyle == "auto": + tbstyle = "short" + + return self._repr_failure_py(excinfo, style=tbstyle) + + def _prunetraceback(self, excinfo: ExceptionInfo[BaseException]) -> None: + if hasattr(self, "path"): + traceback = excinfo.traceback + ntraceback = traceback.cut(path=self.path) + if ntraceback == traceback: + ntraceback = ntraceback.cut(excludepath=tracebackcutdir) + excinfo.traceback = ntraceback.filter() + + +def _check_initialpaths_for_relpath(session: "Session", path: Path) -> Optional[str]: + for initial_path in session._initialpaths: + if commonpath(path, initial_path) == initial_path: + rel = str(path.relative_to(initial_path)) + return "" if rel == "." else rel + return None + + +class FSCollector(Collector): + def __init__( + self, + fspath: Optional[LEGACY_PATH] = None, + path_or_parent: Optional[Union[Path, Node]] = None, + path: Optional[Path] = None, + name: Optional[str] = None, + parent: Optional[Node] = None, + config: Optional[Config] = None, + session: Optional["Session"] = None, + nodeid: Optional[str] = None, + ) -> None: + if path_or_parent: + if isinstance(path_or_parent, Node): + assert parent is None + parent = cast(FSCollector, path_or_parent) + elif isinstance(path_or_parent, Path): + assert path is None + path = path_or_parent + + path = _imply_path(type(self), path, fspath=fspath) + if name is None: + name = path.name + if parent is not None and parent.path != path: + try: + rel = path.relative_to(parent.path) + except ValueError: + pass + else: + name = str(rel) + name = name.replace(os.sep, SEP) + self.path = path + + if session is None: + assert parent is not None + session = parent.session + + if nodeid is None: + try: + nodeid = str(self.path.relative_to(session.config.rootpath)) + except ValueError: + nodeid = _check_initialpaths_for_relpath(session, path) + + if nodeid and os.sep != SEP: + nodeid = nodeid.replace(os.sep, SEP) + + super().__init__( + name=name, + parent=parent, + config=config, + session=session, + nodeid=nodeid, + path=path, + ) + + @classmethod + def from_parent( + cls, + parent, + *, + fspath: Optional[LEGACY_PATH] = None, + path: Optional[Path] = None, + **kw, + ): + """The public constructor.""" + return super().from_parent(parent=parent, fspath=fspath, path=path, **kw) + + def gethookproxy(self, fspath: "os.PathLike[str]"): + warnings.warn(FSCOLLECTOR_GETHOOKPROXY_ISINITPATH, stacklevel=2) + return self.session.gethookproxy(fspath) + + def isinitpath(self, path: Union[str, "os.PathLike[str]"]) -> bool: + warnings.warn(FSCOLLECTOR_GETHOOKPROXY_ISINITPATH, stacklevel=2) + return self.session.isinitpath(path) + + +class File(FSCollector): + """Base class for collecting tests from a file. + + :ref:`non-python tests`. + """ + + +class Item(Node): + """A basic test invocation item. + + Note that for a single function there might be multiple test invocation items. + """ + + nextitem = None + + def __init__( + self, + name, + parent=None, + config: Optional[Config] = None, + session: Optional["Session"] = None, + nodeid: Optional[str] = None, + **kw, + ) -> None: + # The first two arguments are intentionally passed positionally, + # to keep plugins who define a node type which inherits from + # (pytest.Item, pytest.File) working (see issue #8435). + # They can be made kwargs when the deprecation above is done. + super().__init__( + name, + parent, + config=config, + session=session, + nodeid=nodeid, + **kw, + ) + self._report_sections: List[Tuple[str, str, str]] = [] + + #: A list of tuples (name, value) that holds user defined properties + #: for this test. + self.user_properties: List[Tuple[str, object]] = [] + + self._check_item_and_collector_diamond_inheritance() + + def _check_item_and_collector_diamond_inheritance(self) -> None: + """ + Check if the current type inherits from both File and Collector + at the same time, emitting a warning accordingly (#8447). + """ + cls = type(self) + + # We inject an attribute in the type to avoid issuing this warning + # for the same class more than once, which is not helpful. + # It is a hack, but was deemed acceptable in order to avoid + # flooding the user in the common case. + attr_name = "_pytest_diamond_inheritance_warning_shown" + if getattr(cls, attr_name, False): + return + setattr(cls, attr_name, True) + + problems = ", ".join( + base.__name__ for base in cls.__bases__ if issubclass(base, Collector) + ) + if problems: + warnings.warn( + f"{cls.__name__} is an Item subclass and should not be a collector, " + f"however its bases {problems} are collectors.\n" + "Please split the Collectors and the Item into separate node types.\n" + "Pytest Doc example: https://docs.pytest.org/en/latest/example/nonpython.html\n" + "example pull request on a plugin: https://github.com/asmeurer/pytest-flakes/pull/40/", + PytestWarning, + ) + + def runtest(self) -> None: + """Run the test case for this item. + + Must be implemented by subclasses. + + .. seealso:: :ref:`non-python tests` + """ + raise NotImplementedError("runtest must be implemented by Item subclass") + + def add_report_section(self, when: str, key: str, content: str) -> None: + """Add a new report section, similar to what's done internally to add + stdout and stderr captured output:: + + item.add_report_section("call", "stdout", "report section contents") + + :param str when: + One of the possible capture states, ``"setup"``, ``"call"``, ``"teardown"``. + :param str key: + Name of the section, can be customized at will. Pytest uses ``"stdout"`` and + ``"stderr"`` internally. + :param str content: + The full contents as a string. + """ + if content: + self._report_sections.append((when, key, content)) + + def reportinfo(self) -> Tuple[Union["os.PathLike[str]", str], Optional[int], str]: + """Get location information for this item for test reports. + + Returns a tuple with three elements: + + - The path of the test (default ``self.path``) + - The line number of the test (default ``None``) + - A name of the test to be shown (default ``""``) + + .. seealso:: :ref:`non-python tests` + """ + return self.path, None, "" + + @cached_property + def location(self) -> Tuple[str, Optional[int], str]: + location = self.reportinfo() + path = absolutepath(os.fspath(location[0])) + relfspath = self.session._node_location_to_relpath(path) + assert type(location[2]) is str + return (relfspath, location[1], location[2]) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/nose.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/nose.py new file mode 100644 index 0000000000..b0699d22bd --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/nose.py @@ -0,0 +1,42 @@ +"""Run testsuites written for nose.""" +from _pytest.config import hookimpl +from _pytest.fixtures import getfixturemarker +from _pytest.nodes import Item +from _pytest.python import Function +from _pytest.unittest import TestCaseFunction + + +@hookimpl(trylast=True) +def pytest_runtest_setup(item: Item) -> None: + if not isinstance(item, Function): + return + # Don't do nose style setup/teardown on direct unittest style classes. + if isinstance(item, TestCaseFunction): + return + + # Capture the narrowed type of item for the teardown closure, + # see https://github.com/python/mypy/issues/2608 + func = item + + call_optional(func.obj, "setup") + func.addfinalizer(lambda: call_optional(func.obj, "teardown")) + + # NOTE: Module- and class-level fixtures are handled in python.py + # with `pluginmanager.has_plugin("nose")` checks. + # It would have been nicer to implement them outside of core, but + # it's not straightforward. + + +def call_optional(obj: object, name: str) -> bool: + method = getattr(obj, name, None) + if method is None: + return False + is_fixture = getfixturemarker(method) is not None + if is_fixture: + return False + if not callable(method): + return False + # If there are any problems allow the exception to raise rather than + # silently ignoring it. + method() + return True diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/outcomes.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/outcomes.py new file mode 100644 index 0000000000..25206fe0e8 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/outcomes.py @@ -0,0 +1,307 @@ +"""Exception classes and constants handling test outcomes as well as +functions creating them.""" +import sys +import warnings +from typing import Any +from typing import Callable +from typing import cast +from typing import Optional +from typing import Type +from typing import TypeVar + +from _pytest.deprecated import KEYWORD_MSG_ARG + +TYPE_CHECKING = False # Avoid circular import through compat. + +if TYPE_CHECKING: + from typing import NoReturn + from typing_extensions import Protocol +else: + # typing.Protocol is only available starting from Python 3.8. It is also + # available from typing_extensions, but we don't want a runtime dependency + # on that. So use a dummy runtime implementation. + from typing import Generic + + Protocol = Generic + + +class OutcomeException(BaseException): + """OutcomeException and its subclass instances indicate and contain info + about test and collection outcomes.""" + + def __init__(self, msg: Optional[str] = None, pytrace: bool = True) -> None: + if msg is not None and not isinstance(msg, str): + error_msg = ( # type: ignore[unreachable] + "{} expected string as 'msg' parameter, got '{}' instead.\n" + "Perhaps you meant to use a mark?" + ) + raise TypeError(error_msg.format(type(self).__name__, type(msg).__name__)) + super().__init__(msg) + self.msg = msg + self.pytrace = pytrace + + def __repr__(self) -> str: + if self.msg is not None: + return self.msg + return f"<{self.__class__.__name__} instance>" + + __str__ = __repr__ + + +TEST_OUTCOME = (OutcomeException, Exception) + + +class Skipped(OutcomeException): + # XXX hackish: on 3k we fake to live in the builtins + # in order to have Skipped exception printing shorter/nicer + __module__ = "builtins" + + def __init__( + self, + msg: Optional[str] = None, + pytrace: bool = True, + allow_module_level: bool = False, + *, + _use_item_location: bool = False, + ) -> None: + super().__init__(msg=msg, pytrace=pytrace) + self.allow_module_level = allow_module_level + # If true, the skip location is reported as the item's location, + # instead of the place that raises the exception/calls skip(). + self._use_item_location = _use_item_location + + +class Failed(OutcomeException): + """Raised from an explicit call to pytest.fail().""" + + __module__ = "builtins" + + +class Exit(Exception): + """Raised for immediate program exits (no tracebacks/summaries).""" + + def __init__( + self, msg: str = "unknown reason", returncode: Optional[int] = None + ) -> None: + self.msg = msg + self.returncode = returncode + super().__init__(msg) + + +# Elaborate hack to work around https://github.com/python/mypy/issues/2087. +# Ideally would just be `exit.Exception = Exit` etc. + +_F = TypeVar("_F", bound=Callable[..., object]) +_ET = TypeVar("_ET", bound=Type[BaseException]) + + +class _WithException(Protocol[_F, _ET]): + Exception: _ET + __call__: _F + + +def _with_exception(exception_type: _ET) -> Callable[[_F], _WithException[_F, _ET]]: + def decorate(func: _F) -> _WithException[_F, _ET]: + func_with_exception = cast(_WithException[_F, _ET], func) + func_with_exception.Exception = exception_type + return func_with_exception + + return decorate + + +# Exposed helper methods. + + +@_with_exception(Exit) +def exit( + reason: str = "", returncode: Optional[int] = None, *, msg: Optional[str] = None +) -> "NoReturn": + """Exit testing process. + + :param reason: + The message to show as the reason for exiting pytest. reason has a default value + only because `msg` is deprecated. + + :param returncode: + Return code to be used when exiting pytest. + + :param msg: + Same as ``reason``, but deprecated. Will be removed in a future version, use ``reason`` instead. + """ + __tracebackhide__ = True + from _pytest.config import UsageError + + if reason and msg: + raise UsageError( + "cannot pass reason and msg to exit(), `msg` is deprecated, use `reason`." + ) + if not reason: + if msg is None: + raise UsageError("exit() requires a reason argument") + warnings.warn(KEYWORD_MSG_ARG.format(func="exit"), stacklevel=2) + reason = msg + raise Exit(reason, returncode) + + +@_with_exception(Skipped) +def skip( + reason: str = "", *, allow_module_level: bool = False, msg: Optional[str] = None +) -> "NoReturn": + """Skip an executing test with the given message. + + This function should be called only during testing (setup, call or teardown) or + during collection by using the ``allow_module_level`` flag. This function can + be called in doctests as well. + + :param reason: + The message to show the user as reason for the skip. + + :param allow_module_level: + Allows this function to be called at module level, skipping the rest + of the module. Defaults to False. + + :param msg: + Same as ``reason``, but deprecated. Will be removed in a future version, use ``reason`` instead. + + .. note:: + It is better to use the :ref:`pytest.mark.skipif ref` marker when + possible to declare a test to be skipped under certain conditions + like mismatching platforms or dependencies. + Similarly, use the ``# doctest: +SKIP`` directive (see :py:data:`doctest.SKIP`) + to skip a doctest statically. + """ + __tracebackhide__ = True + reason = _resolve_msg_to_reason("skip", reason, msg) + raise Skipped(msg=reason, allow_module_level=allow_module_level) + + +@_with_exception(Failed) +def fail( + reason: str = "", pytrace: bool = True, msg: Optional[str] = None +) -> "NoReturn": + """Explicitly fail an executing test with the given message. + + :param reason: + The message to show the user as reason for the failure. + + :param pytrace: + If False, msg represents the full failure information and no + python traceback will be reported. + + :param msg: + Same as ``reason``, but deprecated. Will be removed in a future version, use ``reason`` instead. + """ + __tracebackhide__ = True + reason = _resolve_msg_to_reason("fail", reason, msg) + raise Failed(msg=reason, pytrace=pytrace) + + +def _resolve_msg_to_reason( + func_name: str, reason: str, msg: Optional[str] = None +) -> str: + """ + Handles converting the deprecated msg parameter if provided into + reason, raising a deprecation warning. This function will be removed + when the optional msg argument is removed from here in future. + + :param str func_name: + The name of the offending function, this is formatted into the deprecation message. + + :param str reason: + The reason= passed into either pytest.fail() or pytest.skip() + + :param str msg: + The msg= passed into either pytest.fail() or pytest.skip(). This will + be converted into reason if it is provided to allow pytest.skip(msg=) or + pytest.fail(msg=) to continue working in the interim period. + + :returns: + The value to use as reason. + + """ + __tracebackhide__ = True + if msg is not None: + + if reason: + from pytest import UsageError + + raise UsageError( + f"Passing both ``reason`` and ``msg`` to pytest.{func_name}(...) is not permitted." + ) + warnings.warn(KEYWORD_MSG_ARG.format(func=func_name), stacklevel=3) + reason = msg + return reason + + +class XFailed(Failed): + """Raised from an explicit call to pytest.xfail().""" + + +@_with_exception(XFailed) +def xfail(reason: str = "") -> "NoReturn": + """Imperatively xfail an executing test or setup function with the given reason. + + This function should be called only during testing (setup, call or teardown). + + .. note:: + It is better to use the :ref:`pytest.mark.xfail ref` marker when + possible to declare a test to be xfailed under certain conditions + like known bugs or missing features. + """ + __tracebackhide__ = True + raise XFailed(reason) + + +def importorskip( + modname: str, minversion: Optional[str] = None, reason: Optional[str] = None +) -> Any: + """Import and return the requested module ``modname``, or skip the + current test if the module cannot be imported. + + :param str modname: + The name of the module to import. + :param str minversion: + If given, the imported module's ``__version__`` attribute must be at + least this minimal version, otherwise the test is still skipped. + :param str reason: + If given, this reason is shown as the message when the module cannot + be imported. + + :returns: + The imported module. This should be assigned to its canonical name. + + Example:: + + docutils = pytest.importorskip("docutils") + """ + import warnings + + __tracebackhide__ = True + compile(modname, "", "eval") # to catch syntaxerrors + + with warnings.catch_warnings(): + # Make sure to ignore ImportWarnings that might happen because + # of existing directories with the same name we're trying to + # import but without a __init__.py file. + warnings.simplefilter("ignore") + try: + __import__(modname) + except ImportError as exc: + if reason is None: + reason = f"could not import {modname!r}: {exc}" + raise Skipped(reason, allow_module_level=True) from None + mod = sys.modules[modname] + if minversion is None: + return mod + verattr = getattr(mod, "__version__", None) + if minversion is not None: + # Imported lazily to improve start-up time. + from packaging.version import Version + + if verattr is None or Version(verattr) < Version(minversion): + raise Skipped( + "module %r has __version__ %r, required is: %r" + % (modname, verattr, minversion), + allow_module_level=True, + ) + return mod diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pastebin.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pastebin.py new file mode 100644 index 0000000000..385b3022cc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pastebin.py @@ -0,0 +1,110 @@ +"""Submit failure or test session information to a pastebin service.""" +import tempfile +from io import StringIO +from typing import IO +from typing import Union + +import pytest +from _pytest.config import Config +from _pytest.config import create_terminal_writer +from _pytest.config.argparsing import Parser +from _pytest.stash import StashKey +from _pytest.terminal import TerminalReporter + + +pastebinfile_key = StashKey[IO[bytes]]() + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("terminal reporting") + group._addoption( + "--pastebin", + metavar="mode", + action="store", + dest="pastebin", + default=None, + choices=["failed", "all"], + help="send failed|all info to bpaste.net pastebin service.", + ) + + +@pytest.hookimpl(trylast=True) +def pytest_configure(config: Config) -> None: + if config.option.pastebin == "all": + tr = config.pluginmanager.getplugin("terminalreporter") + # If no terminal reporter plugin is present, nothing we can do here; + # this can happen when this function executes in a worker node + # when using pytest-xdist, for example. + if tr is not None: + # pastebin file will be UTF-8 encoded binary file. + config.stash[pastebinfile_key] = tempfile.TemporaryFile("w+b") + oldwrite = tr._tw.write + + def tee_write(s, **kwargs): + oldwrite(s, **kwargs) + if isinstance(s, str): + s = s.encode("utf-8") + config.stash[pastebinfile_key].write(s) + + tr._tw.write = tee_write + + +def pytest_unconfigure(config: Config) -> None: + if pastebinfile_key in config.stash: + pastebinfile = config.stash[pastebinfile_key] + # Get terminal contents and delete file. + pastebinfile.seek(0) + sessionlog = pastebinfile.read() + pastebinfile.close() + del config.stash[pastebinfile_key] + # Undo our patching in the terminal reporter. + tr = config.pluginmanager.getplugin("terminalreporter") + del tr._tw.__dict__["write"] + # Write summary. + tr.write_sep("=", "Sending information to Paste Service") + pastebinurl = create_new_paste(sessionlog) + tr.write_line("pastebin session-log: %s\n" % pastebinurl) + + +def create_new_paste(contents: Union[str, bytes]) -> str: + """Create a new paste using the bpaste.net service. + + :contents: Paste contents string. + :returns: URL to the pasted contents, or an error message. + """ + import re + from urllib.request import urlopen + from urllib.parse import urlencode + + params = {"code": contents, "lexer": "text", "expiry": "1week"} + url = "https://bpa.st" + try: + response: str = ( + urlopen(url, data=urlencode(params).encode("ascii")).read().decode("utf-8") + ) + except OSError as exc_info: # urllib errors + return "bad response: %s" % exc_info + m = re.search(r'href="/raw/(\w+)"', response) + if m: + return f"{url}/show/{m.group(1)}" + else: + return "bad response: invalid format ('" + response + "')" + + +def pytest_terminal_summary(terminalreporter: TerminalReporter) -> None: + if terminalreporter.config.option.pastebin != "failed": + return + if "failed" in terminalreporter.stats: + terminalreporter.write_sep("=", "Sending information to Paste Service") + for rep in terminalreporter.stats["failed"]: + try: + msg = rep.longrepr.reprtraceback.reprentries[-1].reprfileloc + except AttributeError: + msg = terminalreporter._getfailureheadline(rep) + file = StringIO() + tw = create_terminal_writer(terminalreporter.config, file) + rep.toterminal(tw) + s = file.getvalue() + assert len(s) + pastebinurl = create_new_paste(s) + terminalreporter.write_line(f"{msg} --> {pastebinurl}") diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pathlib.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pathlib.py new file mode 100644 index 0000000000..b44753e1a4 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pathlib.py @@ -0,0 +1,724 @@ +import atexit +import contextlib +import fnmatch +import importlib.util +import itertools +import os +import shutil +import sys +import uuid +import warnings +from enum import Enum +from errno import EBADF +from errno import ELOOP +from errno import ENOENT +from errno import ENOTDIR +from functools import partial +from os.path import expanduser +from os.path import expandvars +from os.path import isabs +from os.path import sep +from pathlib import Path +from pathlib import PurePath +from posixpath import sep as posix_sep +from types import ModuleType +from typing import Callable +from typing import Dict +from typing import Iterable +from typing import Iterator +from typing import Optional +from typing import Set +from typing import TypeVar +from typing import Union + +from _pytest.compat import assert_never +from _pytest.outcomes import skip +from _pytest.warning_types import PytestWarning + +LOCK_TIMEOUT = 60 * 60 * 24 * 3 + + +_AnyPurePath = TypeVar("_AnyPurePath", bound=PurePath) + +# The following function, variables and comments were +# copied from cpython 3.9 Lib/pathlib.py file. + +# EBADF - guard against macOS `stat` throwing EBADF +_IGNORED_ERRORS = (ENOENT, ENOTDIR, EBADF, ELOOP) + +_IGNORED_WINERRORS = ( + 21, # ERROR_NOT_READY - drive exists but is not accessible + 1921, # ERROR_CANT_RESOLVE_FILENAME - fix for broken symlink pointing to itself +) + + +def _ignore_error(exception): + return ( + getattr(exception, "errno", None) in _IGNORED_ERRORS + or getattr(exception, "winerror", None) in _IGNORED_WINERRORS + ) + + +def get_lock_path(path: _AnyPurePath) -> _AnyPurePath: + return path.joinpath(".lock") + + +def on_rm_rf_error(func, path: str, exc, *, start_path: Path) -> bool: + """Handle known read-only errors during rmtree. + + The returned value is used only by our own tests. + """ + exctype, excvalue = exc[:2] + + # Another process removed the file in the middle of the "rm_rf" (xdist for example). + # More context: https://github.com/pytest-dev/pytest/issues/5974#issuecomment-543799018 + if isinstance(excvalue, FileNotFoundError): + return False + + if not isinstance(excvalue, PermissionError): + warnings.warn( + PytestWarning(f"(rm_rf) error removing {path}\n{exctype}: {excvalue}") + ) + return False + + if func not in (os.rmdir, os.remove, os.unlink): + if func not in (os.open,): + warnings.warn( + PytestWarning( + "(rm_rf) unknown function {} when removing {}:\n{}: {}".format( + func, path, exctype, excvalue + ) + ) + ) + return False + + # Chmod + retry. + import stat + + def chmod_rw(p: str) -> None: + mode = os.stat(p).st_mode + os.chmod(p, mode | stat.S_IRUSR | stat.S_IWUSR) + + # For files, we need to recursively go upwards in the directories to + # ensure they all are also writable. + p = Path(path) + if p.is_file(): + for parent in p.parents: + chmod_rw(str(parent)) + # Stop when we reach the original path passed to rm_rf. + if parent == start_path: + break + chmod_rw(str(path)) + + func(path) + return True + + +def ensure_extended_length_path(path: Path) -> Path: + """Get the extended-length version of a path (Windows). + + On Windows, by default, the maximum length of a path (MAX_PATH) is 260 + characters, and operations on paths longer than that fail. But it is possible + to overcome this by converting the path to "extended-length" form before + performing the operation: + https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#maximum-path-length-limitation + + On Windows, this function returns the extended-length absolute version of path. + On other platforms it returns path unchanged. + """ + if sys.platform.startswith("win32"): + path = path.resolve() + path = Path(get_extended_length_path_str(str(path))) + return path + + +def get_extended_length_path_str(path: str) -> str: + """Convert a path to a Windows extended length path.""" + long_path_prefix = "\\\\?\\" + unc_long_path_prefix = "\\\\?\\UNC\\" + if path.startswith((long_path_prefix, unc_long_path_prefix)): + return path + # UNC + if path.startswith("\\\\"): + return unc_long_path_prefix + path[2:] + return long_path_prefix + path + + +def rm_rf(path: Path) -> None: + """Remove the path contents recursively, even if some elements + are read-only.""" + path = ensure_extended_length_path(path) + onerror = partial(on_rm_rf_error, start_path=path) + shutil.rmtree(str(path), onerror=onerror) + + +def find_prefixed(root: Path, prefix: str) -> Iterator[Path]: + """Find all elements in root that begin with the prefix, case insensitive.""" + l_prefix = prefix.lower() + for x in root.iterdir(): + if x.name.lower().startswith(l_prefix): + yield x + + +def extract_suffixes(iter: Iterable[PurePath], prefix: str) -> Iterator[str]: + """Return the parts of the paths following the prefix. + + :param iter: Iterator over path names. + :param prefix: Expected prefix of the path names. + """ + p_len = len(prefix) + for p in iter: + yield p.name[p_len:] + + +def find_suffixes(root: Path, prefix: str) -> Iterator[str]: + """Combine find_prefixes and extract_suffixes.""" + return extract_suffixes(find_prefixed(root, prefix), prefix) + + +def parse_num(maybe_num) -> int: + """Parse number path suffixes, returns -1 on error.""" + try: + return int(maybe_num) + except ValueError: + return -1 + + +def _force_symlink( + root: Path, target: Union[str, PurePath], link_to: Union[str, Path] +) -> None: + """Helper to create the current symlink. + + It's full of race conditions that are reasonably OK to ignore + for the context of best effort linking to the latest test run. + + The presumption being that in case of much parallelism + the inaccuracy is going to be acceptable. + """ + current_symlink = root.joinpath(target) + try: + current_symlink.unlink() + except OSError: + pass + try: + current_symlink.symlink_to(link_to) + except Exception: + pass + + +def make_numbered_dir(root: Path, prefix: str, mode: int = 0o700) -> Path: + """Create a directory with an increased number as suffix for the given prefix.""" + for i in range(10): + # try up to 10 times to create the folder + max_existing = max(map(parse_num, find_suffixes(root, prefix)), default=-1) + new_number = max_existing + 1 + new_path = root.joinpath(f"{prefix}{new_number}") + try: + new_path.mkdir(mode=mode) + except Exception: + pass + else: + _force_symlink(root, prefix + "current", new_path) + return new_path + else: + raise OSError( + "could not create numbered dir with prefix " + "{prefix} in {root} after 10 tries".format(prefix=prefix, root=root) + ) + + +def create_cleanup_lock(p: Path) -> Path: + """Create a lock to prevent premature folder cleanup.""" + lock_path = get_lock_path(p) + try: + fd = os.open(str(lock_path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + except FileExistsError as e: + raise OSError(f"cannot create lockfile in {p}") from e + else: + pid = os.getpid() + spid = str(pid).encode() + os.write(fd, spid) + os.close(fd) + if not lock_path.is_file(): + raise OSError("lock path got renamed after successful creation") + return lock_path + + +def register_cleanup_lock_removal(lock_path: Path, register=atexit.register): + """Register a cleanup function for removing a lock, by default on atexit.""" + pid = os.getpid() + + def cleanup_on_exit(lock_path: Path = lock_path, original_pid: int = pid) -> None: + current_pid = os.getpid() + if current_pid != original_pid: + # fork + return + try: + lock_path.unlink() + except OSError: + pass + + return register(cleanup_on_exit) + + +def maybe_delete_a_numbered_dir(path: Path) -> None: + """Remove a numbered directory if its lock can be obtained and it does + not seem to be in use.""" + path = ensure_extended_length_path(path) + lock_path = None + try: + lock_path = create_cleanup_lock(path) + parent = path.parent + + garbage = parent.joinpath(f"garbage-{uuid.uuid4()}") + path.rename(garbage) + rm_rf(garbage) + except OSError: + # known races: + # * other process did a cleanup at the same time + # * deletable folder was found + # * process cwd (Windows) + return + finally: + # If we created the lock, ensure we remove it even if we failed + # to properly remove the numbered dir. + if lock_path is not None: + try: + lock_path.unlink() + except OSError: + pass + + +def ensure_deletable(path: Path, consider_lock_dead_if_created_before: float) -> bool: + """Check if `path` is deletable based on whether the lock file is expired.""" + if path.is_symlink(): + return False + lock = get_lock_path(path) + try: + if not lock.is_file(): + return True + except OSError: + # we might not have access to the lock file at all, in this case assume + # we don't have access to the entire directory (#7491). + return False + try: + lock_time = lock.stat().st_mtime + except Exception: + return False + else: + if lock_time < consider_lock_dead_if_created_before: + # We want to ignore any errors while trying to remove the lock such as: + # - PermissionDenied, like the file permissions have changed since the lock creation; + # - FileNotFoundError, in case another pytest process got here first; + # and any other cause of failure. + with contextlib.suppress(OSError): + lock.unlink() + return True + return False + + +def try_cleanup(path: Path, consider_lock_dead_if_created_before: float) -> None: + """Try to cleanup a folder if we can ensure it's deletable.""" + if ensure_deletable(path, consider_lock_dead_if_created_before): + maybe_delete_a_numbered_dir(path) + + +def cleanup_candidates(root: Path, prefix: str, keep: int) -> Iterator[Path]: + """List candidates for numbered directories to be removed - follows py.path.""" + max_existing = max(map(parse_num, find_suffixes(root, prefix)), default=-1) + max_delete = max_existing - keep + paths = find_prefixed(root, prefix) + paths, paths2 = itertools.tee(paths) + numbers = map(parse_num, extract_suffixes(paths2, prefix)) + for path, number in zip(paths, numbers): + if number <= max_delete: + yield path + + +def cleanup_numbered_dir( + root: Path, prefix: str, keep: int, consider_lock_dead_if_created_before: float +) -> None: + """Cleanup for lock driven numbered directories.""" + for path in cleanup_candidates(root, prefix, keep): + try_cleanup(path, consider_lock_dead_if_created_before) + for path in root.glob("garbage-*"): + try_cleanup(path, consider_lock_dead_if_created_before) + + +def make_numbered_dir_with_cleanup( + root: Path, + prefix: str, + keep: int, + lock_timeout: float, + mode: int, +) -> Path: + """Create a numbered dir with a cleanup lock and remove old ones.""" + e = None + for i in range(10): + try: + p = make_numbered_dir(root, prefix, mode) + lock_path = create_cleanup_lock(p) + register_cleanup_lock_removal(lock_path) + except Exception as exc: + e = exc + else: + consider_lock_dead_if_created_before = p.stat().st_mtime - lock_timeout + # Register a cleanup for program exit + atexit.register( + cleanup_numbered_dir, + root, + prefix, + keep, + consider_lock_dead_if_created_before, + ) + return p + assert e is not None + raise e + + +def resolve_from_str(input: str, rootpath: Path) -> Path: + input = expanduser(input) + input = expandvars(input) + if isabs(input): + return Path(input) + else: + return rootpath.joinpath(input) + + +def fnmatch_ex(pattern: str, path: Union[str, "os.PathLike[str]"]) -> bool: + """A port of FNMatcher from py.path.common which works with PurePath() instances. + + The difference between this algorithm and PurePath.match() is that the + latter matches "**" glob expressions for each part of the path, while + this algorithm uses the whole path instead. + + For example: + "tests/foo/bar/doc/test_foo.py" matches pattern "tests/**/doc/test*.py" + with this algorithm, but not with PurePath.match(). + + This algorithm was ported to keep backward-compatibility with existing + settings which assume paths match according this logic. + + References: + * https://bugs.python.org/issue29249 + * https://bugs.python.org/issue34731 + """ + path = PurePath(path) + iswin32 = sys.platform.startswith("win") + + if iswin32 and sep not in pattern and posix_sep in pattern: + # Running on Windows, the pattern has no Windows path separators, + # and the pattern has one or more Posix path separators. Replace + # the Posix path separators with the Windows path separator. + pattern = pattern.replace(posix_sep, sep) + + if sep not in pattern: + name = path.name + else: + name = str(path) + if path.is_absolute() and not os.path.isabs(pattern): + pattern = f"*{os.sep}{pattern}" + return fnmatch.fnmatch(name, pattern) + + +def parts(s: str) -> Set[str]: + parts = s.split(sep) + return {sep.join(parts[: i + 1]) or sep for i in range(len(parts))} + + +def symlink_or_skip(src, dst, **kwargs): + """Make a symlink, or skip the test in case symlinks are not supported.""" + try: + os.symlink(str(src), str(dst), **kwargs) + except OSError as e: + skip(f"symlinks not supported: {e}") + + +class ImportMode(Enum): + """Possible values for `mode` parameter of `import_path`.""" + + prepend = "prepend" + append = "append" + importlib = "importlib" + + +class ImportPathMismatchError(ImportError): + """Raised on import_path() if there is a mismatch of __file__'s. + + This can happen when `import_path` is called multiple times with different filenames that has + the same basename but reside in packages + (for example "/tests1/test_foo.py" and "/tests2/test_foo.py"). + """ + + +def import_path( + p: Union[str, "os.PathLike[str]"], + *, + mode: Union[str, ImportMode] = ImportMode.prepend, + root: Path, +) -> ModuleType: + """Import and return a module from the given path, which can be a file (a module) or + a directory (a package). + + The import mechanism used is controlled by the `mode` parameter: + + * `mode == ImportMode.prepend`: the directory containing the module (or package, taking + `__init__.py` files into account) will be put at the *start* of `sys.path` before + being imported with `__import__. + + * `mode == ImportMode.append`: same as `prepend`, but the directory will be appended + to the end of `sys.path`, if not already in `sys.path`. + + * `mode == ImportMode.importlib`: uses more fine control mechanisms provided by `importlib` + to import the module, which avoids having to use `__import__` and muck with `sys.path` + at all. It effectively allows having same-named test modules in different places. + + :param root: + Used as an anchor when mode == ImportMode.importlib to obtain + a unique name for the module being imported so it can safely be stored + into ``sys.modules``. + + :raises ImportPathMismatchError: + If after importing the given `path` and the module `__file__` + are different. Only raised in `prepend` and `append` modes. + """ + mode = ImportMode(mode) + + path = Path(p) + + if not path.exists(): + raise ImportError(path) + + if mode is ImportMode.importlib: + module_name = module_name_from_path(path, root) + + for meta_importer in sys.meta_path: + spec = meta_importer.find_spec(module_name, [str(path.parent)]) + if spec is not None: + break + else: + spec = importlib.util.spec_from_file_location(module_name, str(path)) + + if spec is None: + raise ImportError(f"Can't find module {module_name} at location {path}") + mod = importlib.util.module_from_spec(spec) + sys.modules[module_name] = mod + spec.loader.exec_module(mod) # type: ignore[union-attr] + insert_missing_modules(sys.modules, module_name) + return mod + + pkg_path = resolve_package_path(path) + if pkg_path is not None: + pkg_root = pkg_path.parent + names = list(path.with_suffix("").relative_to(pkg_root).parts) + if names[-1] == "__init__": + names.pop() + module_name = ".".join(names) + else: + pkg_root = path.parent + module_name = path.stem + + # Change sys.path permanently: restoring it at the end of this function would cause surprising + # problems because of delayed imports: for example, a conftest.py file imported by this function + # might have local imports, which would fail at runtime if we restored sys.path. + if mode is ImportMode.append: + if str(pkg_root) not in sys.path: + sys.path.append(str(pkg_root)) + elif mode is ImportMode.prepend: + if str(pkg_root) != sys.path[0]: + sys.path.insert(0, str(pkg_root)) + else: + assert_never(mode) + + importlib.import_module(module_name) + + mod = sys.modules[module_name] + if path.name == "__init__.py": + return mod + + ignore = os.environ.get("PY_IGNORE_IMPORTMISMATCH", "") + if ignore != "1": + module_file = mod.__file__ + if module_file.endswith((".pyc", ".pyo")): + module_file = module_file[:-1] + if module_file.endswith(os.path.sep + "__init__.py"): + module_file = module_file[: -(len(os.path.sep + "__init__.py"))] + + try: + is_same = _is_same(str(path), module_file) + except FileNotFoundError: + is_same = False + + if not is_same: + raise ImportPathMismatchError(module_name, module_file, path) + + return mod + + +# Implement a special _is_same function on Windows which returns True if the two filenames +# compare equal, to circumvent os.path.samefile returning False for mounts in UNC (#7678). +if sys.platform.startswith("win"): + + def _is_same(f1: str, f2: str) -> bool: + return Path(f1) == Path(f2) or os.path.samefile(f1, f2) + + +else: + + def _is_same(f1: str, f2: str) -> bool: + return os.path.samefile(f1, f2) + + +def module_name_from_path(path: Path, root: Path) -> str: + """ + Return a dotted module name based on the given path, anchored on root. + + For example: path="projects/src/tests/test_foo.py" and root="/projects", the + resulting module name will be "src.tests.test_foo". + """ + path = path.with_suffix("") + try: + relative_path = path.relative_to(root) + except ValueError: + # If we can't get a relative path to root, use the full path, except + # for the first part ("d:\\" or "/" depending on the platform, for example). + path_parts = path.parts[1:] + else: + # Use the parts for the relative path to the root path. + path_parts = relative_path.parts + + return ".".join(path_parts) + + +def insert_missing_modules(modules: Dict[str, ModuleType], module_name: str) -> None: + """ + Used by ``import_path`` to create intermediate modules when using mode=importlib. + + When we want to import a module as "src.tests.test_foo" for example, we need + to create empty modules "src" and "src.tests" after inserting "src.tests.test_foo", + otherwise "src.tests.test_foo" is not importable by ``__import__``. + """ + module_parts = module_name.split(".") + while module_name: + if module_name not in modules: + module = ModuleType( + module_name, + doc="Empty module created by pytest's importmode=importlib.", + ) + modules[module_name] = module + module_parts.pop(-1) + module_name = ".".join(module_parts) + + +def resolve_package_path(path: Path) -> Optional[Path]: + """Return the Python package path by looking for the last + directory upwards which still contains an __init__.py. + + Returns None if it can not be determined. + """ + result = None + for parent in itertools.chain((path,), path.parents): + if parent.is_dir(): + if not parent.joinpath("__init__.py").is_file(): + break + if not parent.name.isidentifier(): + break + result = parent + return result + + +def visit( + path: Union[str, "os.PathLike[str]"], recurse: Callable[["os.DirEntry[str]"], bool] +) -> Iterator["os.DirEntry[str]"]: + """Walk a directory recursively, in breadth-first order. + + Entries at each directory level are sorted. + """ + + # Skip entries with symlink loops and other brokenness, so the caller doesn't + # have to deal with it. + entries = [] + for entry in os.scandir(path): + try: + entry.is_file() + except OSError as err: + if _ignore_error(err): + continue + raise + entries.append(entry) + + entries.sort(key=lambda entry: entry.name) + + yield from entries + + for entry in entries: + if entry.is_dir() and recurse(entry): + yield from visit(entry.path, recurse) + + +def absolutepath(path: Union[Path, str]) -> Path: + """Convert a path to an absolute path using os.path.abspath. + + Prefer this over Path.resolve() (see #6523). + Prefer this over Path.absolute() (not public, doesn't normalize). + """ + return Path(os.path.abspath(str(path))) + + +def commonpath(path1: Path, path2: Path) -> Optional[Path]: + """Return the common part shared with the other path, or None if there is + no common part. + + If one path is relative and one is absolute, returns None. + """ + try: + return Path(os.path.commonpath((str(path1), str(path2)))) + except ValueError: + return None + + +def bestrelpath(directory: Path, dest: Path) -> str: + """Return a string which is a relative path from directory to dest such + that directory/bestrelpath == dest. + + The paths must be either both absolute or both relative. + + If no such path can be determined, returns dest. + """ + assert isinstance(directory, Path) + assert isinstance(dest, Path) + if dest == directory: + return os.curdir + # Find the longest common directory. + base = commonpath(directory, dest) + # Can be the case on Windows for two absolute paths on different drives. + # Can be the case for two relative paths without common prefix. + # Can be the case for a relative path and an absolute path. + if not base: + return str(dest) + reldirectory = directory.relative_to(base) + reldest = dest.relative_to(base) + return os.path.join( + # Back from directory to base. + *([os.pardir] * len(reldirectory.parts)), + # Forward from base to dest. + *reldest.parts, + ) + + +# Originates from py. path.local.copy(), with siginficant trims and adjustments. +# TODO(py38): Replace with shutil.copytree(..., symlinks=True, dirs_exist_ok=True) +def copytree(source: Path, target: Path) -> None: + """Recursively copy a source directory to target.""" + assert source.is_dir() + for entry in visit(source, recurse=lambda entry: not entry.is_symlink()): + x = Path(entry) + relpath = x.relative_to(source) + newx = target / relpath + newx.parent.mkdir(exist_ok=True) + if x.is_symlink(): + newx.symlink_to(os.readlink(x)) + elif x.is_file(): + shutil.copyfile(x, newx) + elif x.is_dir(): + newx.mkdir(exist_ok=True) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/py.typed b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pytester.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pytester.py new file mode 100644 index 0000000000..363a372744 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pytester.py @@ -0,0 +1,1748 @@ +"""(Disabled by default) support for testing pytest and pytest plugins. + +PYTEST_DONT_REWRITE +""" +import collections.abc +import contextlib +import gc +import importlib +import os +import platform +import re +import shutil +import subprocess +import sys +import traceback +from fnmatch import fnmatch +from io import StringIO +from pathlib import Path +from typing import Any +from typing import Callable +from typing import Dict +from typing import Generator +from typing import IO +from typing import Iterable +from typing import List +from typing import Optional +from typing import overload +from typing import Sequence +from typing import TextIO +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import Union +from weakref import WeakKeyDictionary + +from iniconfig import IniConfig +from iniconfig import SectionWrapper + +from _pytest import timing +from _pytest._code import Source +from _pytest.capture import _get_multicapture +from _pytest.compat import final +from _pytest.compat import NOTSET +from _pytest.compat import NotSetType +from _pytest.config import _PluggyPlugin +from _pytest.config import Config +from _pytest.config import ExitCode +from _pytest.config import hookimpl +from _pytest.config import main +from _pytest.config import PytestPluginManager +from _pytest.config.argparsing import Parser +from _pytest.deprecated import check_ispytest +from _pytest.fixtures import fixture +from _pytest.fixtures import FixtureRequest +from _pytest.main import Session +from _pytest.monkeypatch import MonkeyPatch +from _pytest.nodes import Collector +from _pytest.nodes import Item +from _pytest.outcomes import fail +from _pytest.outcomes import importorskip +from _pytest.outcomes import skip +from _pytest.pathlib import bestrelpath +from _pytest.pathlib import copytree +from _pytest.pathlib import make_numbered_dir +from _pytest.reports import CollectReport +from _pytest.reports import TestReport +from _pytest.tmpdir import TempPathFactory +from _pytest.warning_types import PytestWarning + + +if TYPE_CHECKING: + from typing_extensions import Final + from typing_extensions import Literal + + import pexpect + + +pytest_plugins = ["pytester_assertions"] + + +IGNORE_PAM = [ # filenames added when obtaining details about the current user + "/var/lib/sss/mc/passwd" +] + + +def pytest_addoption(parser: Parser) -> None: + parser.addoption( + "--lsof", + action="store_true", + dest="lsof", + default=False, + help="run FD checks if lsof is available", + ) + + parser.addoption( + "--runpytest", + default="inprocess", + dest="runpytest", + choices=("inprocess", "subprocess"), + help=( + "run pytest sub runs in tests using an 'inprocess' " + "or 'subprocess' (python -m main) method" + ), + ) + + parser.addini( + "pytester_example_dir", help="directory to take the pytester example files from" + ) + + +def pytest_configure(config: Config) -> None: + if config.getvalue("lsof"): + checker = LsofFdLeakChecker() + if checker.matching_platform(): + config.pluginmanager.register(checker) + + config.addinivalue_line( + "markers", + "pytester_example_path(*path_segments): join the given path " + "segments to `pytester_example_dir` for this test.", + ) + + +class LsofFdLeakChecker: + def get_open_files(self) -> List[Tuple[str, str]]: + out = subprocess.run( + ("lsof", "-Ffn0", "-p", str(os.getpid())), + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=True, + universal_newlines=True, + ).stdout + + def isopen(line: str) -> bool: + return line.startswith("f") and ( + "deleted" not in line + and "mem" not in line + and "txt" not in line + and "cwd" not in line + ) + + open_files = [] + + for line in out.split("\n"): + if isopen(line): + fields = line.split("\0") + fd = fields[0][1:] + filename = fields[1][1:] + if filename in IGNORE_PAM: + continue + if filename.startswith("/"): + open_files.append((fd, filename)) + + return open_files + + def matching_platform(self) -> bool: + try: + subprocess.run(("lsof", "-v"), check=True) + except (OSError, subprocess.CalledProcessError): + return False + else: + return True + + @hookimpl(hookwrapper=True, tryfirst=True) + def pytest_runtest_protocol(self, item: Item) -> Generator[None, None, None]: + lines1 = self.get_open_files() + yield + if hasattr(sys, "pypy_version_info"): + gc.collect() + lines2 = self.get_open_files() + + new_fds = {t[0] for t in lines2} - {t[0] for t in lines1} + leaked_files = [t for t in lines2 if t[0] in new_fds] + if leaked_files: + error = [ + "***** %s FD leakage detected" % len(leaked_files), + *(str(f) for f in leaked_files), + "*** Before:", + *(str(f) for f in lines1), + "*** After:", + *(str(f) for f in lines2), + "***** %s FD leakage detected" % len(leaked_files), + "*** function %s:%s: %s " % item.location, + "See issue #2366", + ] + item.warn(PytestWarning("\n".join(error))) + + +# used at least by pytest-xdist plugin + + +@fixture +def _pytest(request: FixtureRequest) -> "PytestArg": + """Return a helper which offers a gethookrecorder(hook) method which + returns a HookRecorder instance which helps to make assertions about called + hooks.""" + return PytestArg(request) + + +class PytestArg: + def __init__(self, request: FixtureRequest) -> None: + self._request = request + + def gethookrecorder(self, hook) -> "HookRecorder": + hookrecorder = HookRecorder(hook._pm) + self._request.addfinalizer(hookrecorder.finish_recording) + return hookrecorder + + +def get_public_names(values: Iterable[str]) -> List[str]: + """Only return names from iterator values without a leading underscore.""" + return [x for x in values if x[0] != "_"] + + +@final +class RecordedHookCall: + """A recorded call to a hook. + + The arguments to the hook call are set as attributes. + For example: + + .. code-block:: python + + calls = hook_recorder.getcalls("pytest_runtest_setup") + # Suppose pytest_runtest_setup was called once with `item=an_item`. + assert calls[0].item is an_item + """ + + def __init__(self, name: str, kwargs) -> None: + self.__dict__.update(kwargs) + self._name = name + + def __repr__(self) -> str: + d = self.__dict__.copy() + del d["_name"] + return f"" + + if TYPE_CHECKING: + # The class has undetermined attributes, this tells mypy about it. + def __getattr__(self, key: str): + ... + + +@final +class HookRecorder: + """Record all hooks called in a plugin manager. + + Hook recorders are created by :class:`Pytester`. + + This wraps all the hook calls in the plugin manager, recording each call + before propagating the normal calls. + """ + + def __init__( + self, pluginmanager: PytestPluginManager, *, _ispytest: bool = False + ) -> None: + check_ispytest(_ispytest) + + self._pluginmanager = pluginmanager + self.calls: List[RecordedHookCall] = [] + self.ret: Optional[Union[int, ExitCode]] = None + + def before(hook_name: str, hook_impls, kwargs) -> None: + self.calls.append(RecordedHookCall(hook_name, kwargs)) + + def after(outcome, hook_name: str, hook_impls, kwargs) -> None: + pass + + self._undo_wrapping = pluginmanager.add_hookcall_monitoring(before, after) + + def finish_recording(self) -> None: + self._undo_wrapping() + + def getcalls(self, names: Union[str, Iterable[str]]) -> List[RecordedHookCall]: + """Get all recorded calls to hooks with the given names (or name).""" + if isinstance(names, str): + names = names.split() + return [call for call in self.calls if call._name in names] + + def assert_contains(self, entries: Sequence[Tuple[str, str]]) -> None: + __tracebackhide__ = True + i = 0 + entries = list(entries) + backlocals = sys._getframe(1).f_locals + while entries: + name, check = entries.pop(0) + for ind, call in enumerate(self.calls[i:]): + if call._name == name: + print("NAMEMATCH", name, call) + if eval(check, backlocals, call.__dict__): + print("CHECKERMATCH", repr(check), "->", call) + else: + print("NOCHECKERMATCH", repr(check), "-", call) + continue + i += ind + 1 + break + print("NONAMEMATCH", name, "with", call) + else: + fail(f"could not find {name!r} check {check!r}") + + def popcall(self, name: str) -> RecordedHookCall: + __tracebackhide__ = True + for i, call in enumerate(self.calls): + if call._name == name: + del self.calls[i] + return call + lines = [f"could not find call {name!r}, in:"] + lines.extend([" %s" % x for x in self.calls]) + fail("\n".join(lines)) + + def getcall(self, name: str) -> RecordedHookCall: + values = self.getcalls(name) + assert len(values) == 1, (name, values) + return values[0] + + # functionality for test reports + + @overload + def getreports( + self, + names: "Literal['pytest_collectreport']", + ) -> Sequence[CollectReport]: + ... + + @overload + def getreports( + self, + names: "Literal['pytest_runtest_logreport']", + ) -> Sequence[TestReport]: + ... + + @overload + def getreports( + self, + names: Union[str, Iterable[str]] = ( + "pytest_collectreport", + "pytest_runtest_logreport", + ), + ) -> Sequence[Union[CollectReport, TestReport]]: + ... + + def getreports( + self, + names: Union[str, Iterable[str]] = ( + "pytest_collectreport", + "pytest_runtest_logreport", + ), + ) -> Sequence[Union[CollectReport, TestReport]]: + return [x.report for x in self.getcalls(names)] + + def matchreport( + self, + inamepart: str = "", + names: Union[str, Iterable[str]] = ( + "pytest_runtest_logreport", + "pytest_collectreport", + ), + when: Optional[str] = None, + ) -> Union[CollectReport, TestReport]: + """Return a testreport whose dotted import path matches.""" + values = [] + for rep in self.getreports(names=names): + if not when and rep.when != "call" and rep.passed: + # setup/teardown passing reports - let's ignore those + continue + if when and rep.when != when: + continue + if not inamepart or inamepart in rep.nodeid.split("::"): + values.append(rep) + if not values: + raise ValueError( + "could not find test report matching %r: " + "no test reports at all!" % (inamepart,) + ) + if len(values) > 1: + raise ValueError( + "found 2 or more testreports matching {!r}: {}".format( + inamepart, values + ) + ) + return values[0] + + @overload + def getfailures( + self, + names: "Literal['pytest_collectreport']", + ) -> Sequence[CollectReport]: + ... + + @overload + def getfailures( + self, + names: "Literal['pytest_runtest_logreport']", + ) -> Sequence[TestReport]: + ... + + @overload + def getfailures( + self, + names: Union[str, Iterable[str]] = ( + "pytest_collectreport", + "pytest_runtest_logreport", + ), + ) -> Sequence[Union[CollectReport, TestReport]]: + ... + + def getfailures( + self, + names: Union[str, Iterable[str]] = ( + "pytest_collectreport", + "pytest_runtest_logreport", + ), + ) -> Sequence[Union[CollectReport, TestReport]]: + return [rep for rep in self.getreports(names) if rep.failed] + + def getfailedcollections(self) -> Sequence[CollectReport]: + return self.getfailures("pytest_collectreport") + + def listoutcomes( + self, + ) -> Tuple[ + Sequence[TestReport], + Sequence[Union[CollectReport, TestReport]], + Sequence[Union[CollectReport, TestReport]], + ]: + passed = [] + skipped = [] + failed = [] + for rep in self.getreports( + ("pytest_collectreport", "pytest_runtest_logreport") + ): + if rep.passed: + if rep.when == "call": + assert isinstance(rep, TestReport) + passed.append(rep) + elif rep.skipped: + skipped.append(rep) + else: + assert rep.failed, f"Unexpected outcome: {rep!r}" + failed.append(rep) + return passed, skipped, failed + + def countoutcomes(self) -> List[int]: + return [len(x) for x in self.listoutcomes()] + + def assertoutcome(self, passed: int = 0, skipped: int = 0, failed: int = 0) -> None: + __tracebackhide__ = True + from _pytest.pytester_assertions import assertoutcome + + outcomes = self.listoutcomes() + assertoutcome( + outcomes, + passed=passed, + skipped=skipped, + failed=failed, + ) + + def clear(self) -> None: + self.calls[:] = [] + + +@fixture +def linecomp() -> "LineComp": + """A :class: `LineComp` instance for checking that an input linearly + contains a sequence of strings.""" + return LineComp() + + +@fixture(name="LineMatcher") +def LineMatcher_fixture(request: FixtureRequest) -> Type["LineMatcher"]: + """A reference to the :class: `LineMatcher`. + + This is instantiable with a list of lines (without their trailing newlines). + This is useful for testing large texts, such as the output of commands. + """ + return LineMatcher + + +@fixture +def pytester(request: FixtureRequest, tmp_path_factory: TempPathFactory) -> "Pytester": + """ + Facilities to write tests/configuration files, execute pytest in isolation, and match + against expected output, perfect for black-box testing of pytest plugins. + + It attempts to isolate the test run from external factors as much as possible, modifying + the current working directory to ``path`` and environment variables during initialization. + + It is particularly useful for testing plugins. It is similar to the :fixture:`tmp_path` + fixture but provides methods which aid in testing pytest itself. + """ + return Pytester(request, tmp_path_factory, _ispytest=True) + + +@fixture +def _sys_snapshot() -> Generator[None, None, None]: + snappaths = SysPathsSnapshot() + snapmods = SysModulesSnapshot() + yield + snapmods.restore() + snappaths.restore() + + +@fixture +def _config_for_test() -> Generator[Config, None, None]: + from _pytest.config import get_config + + config = get_config() + yield config + config._ensure_unconfigure() # cleanup, e.g. capman closing tmpfiles. + + +# Regex to match the session duration string in the summary: "74.34s". +rex_session_duration = re.compile(r"\d+\.\d\ds") +# Regex to match all the counts and phrases in the summary line: "34 passed, 111 skipped". +rex_outcome = re.compile(r"(\d+) (\w+)") + + +@final +class RunResult: + """The result of running a command from :class:`~pytest.Pytester`.""" + + def __init__( + self, + ret: Union[int, ExitCode], + outlines: List[str], + errlines: List[str], + duration: float, + ) -> None: + try: + self.ret: Union[int, ExitCode] = ExitCode(ret) + """The return value.""" + except ValueError: + self.ret = ret + self.outlines = outlines + """List of lines captured from stdout.""" + self.errlines = errlines + """List of lines captured from stderr.""" + self.stdout = LineMatcher(outlines) + """:class:`~pytest.LineMatcher` of stdout. + + Use e.g. :func:`str(stdout) ` to reconstruct stdout, or the commonly used + :func:`stdout.fnmatch_lines() ` method. + """ + self.stderr = LineMatcher(errlines) + """:class:`~pytest.LineMatcher` of stderr.""" + self.duration = duration + """Duration in seconds.""" + + def __repr__(self) -> str: + return ( + "" + % (self.ret, len(self.stdout.lines), len(self.stderr.lines), self.duration) + ) + + def parseoutcomes(self) -> Dict[str, int]: + """Return a dictionary of outcome noun -> count from parsing the terminal + output that the test process produced. + + The returned nouns will always be in plural form:: + + ======= 1 failed, 1 passed, 1 warning, 1 error in 0.13s ==== + + Will return ``{"failed": 1, "passed": 1, "warnings": 1, "errors": 1}``. + """ + return self.parse_summary_nouns(self.outlines) + + @classmethod + def parse_summary_nouns(cls, lines) -> Dict[str, int]: + """Extract the nouns from a pytest terminal summary line. + + It always returns the plural noun for consistency:: + + ======= 1 failed, 1 passed, 1 warning, 1 error in 0.13s ==== + + Will return ``{"failed": 1, "passed": 1, "warnings": 1, "errors": 1}``. + """ + for line in reversed(lines): + if rex_session_duration.search(line): + outcomes = rex_outcome.findall(line) + ret = {noun: int(count) for (count, noun) in outcomes} + break + else: + raise ValueError("Pytest terminal summary report not found") + + to_plural = { + "warning": "warnings", + "error": "errors", + } + return {to_plural.get(k, k): v for k, v in ret.items()} + + def assert_outcomes( + self, + passed: int = 0, + skipped: int = 0, + failed: int = 0, + errors: int = 0, + xpassed: int = 0, + xfailed: int = 0, + warnings: Optional[int] = None, + deselected: Optional[int] = None, + ) -> None: + """ + Assert that the specified outcomes appear with the respective + numbers (0 means it didn't occur) in the text output from a test run. + + ``warnings`` and ``deselected`` are only checked if not None. + """ + __tracebackhide__ = True + from _pytest.pytester_assertions import assert_outcomes + + outcomes = self.parseoutcomes() + assert_outcomes( + outcomes, + passed=passed, + skipped=skipped, + failed=failed, + errors=errors, + xpassed=xpassed, + xfailed=xfailed, + warnings=warnings, + deselected=deselected, + ) + + +class CwdSnapshot: + def __init__(self) -> None: + self.__saved = os.getcwd() + + def restore(self) -> None: + os.chdir(self.__saved) + + +class SysModulesSnapshot: + def __init__(self, preserve: Optional[Callable[[str], bool]] = None) -> None: + self.__preserve = preserve + self.__saved = dict(sys.modules) + + def restore(self) -> None: + if self.__preserve: + self.__saved.update( + (k, m) for k, m in sys.modules.items() if self.__preserve(k) + ) + sys.modules.clear() + sys.modules.update(self.__saved) + + +class SysPathsSnapshot: + def __init__(self) -> None: + self.__saved = list(sys.path), list(sys.meta_path) + + def restore(self) -> None: + sys.path[:], sys.meta_path[:] = self.__saved + + +@final +class Pytester: + """ + Facilities to write tests/configuration files, execute pytest in isolation, and match + against expected output, perfect for black-box testing of pytest plugins. + + It attempts to isolate the test run from external factors as much as possible, modifying + the current working directory to ``path`` and environment variables during initialization. + + Attributes: + + :ivar Path path: temporary directory path used to create files/run tests from, etc. + + :ivar plugins: + A list of plugins to use with :py:meth:`parseconfig` and + :py:meth:`runpytest`. Initially this is an empty list but plugins can + be added to the list. The type of items to add to the list depends on + the method using them so refer to them for details. + """ + + __test__ = False + + CLOSE_STDIN: "Final" = NOTSET + + class TimeoutExpired(Exception): + pass + + def __init__( + self, + request: FixtureRequest, + tmp_path_factory: TempPathFactory, + *, + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + self._request = request + self._mod_collections: WeakKeyDictionary[ + Collector, List[Union[Item, Collector]] + ] = WeakKeyDictionary() + if request.function: + name: str = request.function.__name__ + else: + name = request.node.name + self._name = name + self._path: Path = tmp_path_factory.mktemp(name, numbered=True) + self.plugins: List[Union[str, _PluggyPlugin]] = [] + self._cwd_snapshot = CwdSnapshot() + self._sys_path_snapshot = SysPathsSnapshot() + self._sys_modules_snapshot = self.__take_sys_modules_snapshot() + self.chdir() + self._request.addfinalizer(self._finalize) + self._method = self._request.config.getoption("--runpytest") + self._test_tmproot = tmp_path_factory.mktemp(f"tmp-{name}", numbered=True) + + self._monkeypatch = mp = MonkeyPatch() + mp.setenv("PYTEST_DEBUG_TEMPROOT", str(self._test_tmproot)) + # Ensure no unexpected caching via tox. + mp.delenv("TOX_ENV_DIR", raising=False) + # Discard outer pytest options. + mp.delenv("PYTEST_ADDOPTS", raising=False) + # Ensure no user config is used. + tmphome = str(self.path) + mp.setenv("HOME", tmphome) + mp.setenv("USERPROFILE", tmphome) + # Do not use colors for inner runs by default. + mp.setenv("PY_COLORS", "0") + + @property + def path(self) -> Path: + """Temporary directory where files are created and pytest is executed.""" + return self._path + + def __repr__(self) -> str: + return f"" + + def _finalize(self) -> None: + """ + Clean up global state artifacts. + + Some methods modify the global interpreter state and this tries to + clean this up. It does not remove the temporary directory however so + it can be looked at after the test run has finished. + """ + self._sys_modules_snapshot.restore() + self._sys_path_snapshot.restore() + self._cwd_snapshot.restore() + self._monkeypatch.undo() + + def __take_sys_modules_snapshot(self) -> SysModulesSnapshot: + # Some zope modules used by twisted-related tests keep internal state + # and can't be deleted; we had some trouble in the past with + # `zope.interface` for example. + # + # Preserve readline due to https://bugs.python.org/issue41033. + # pexpect issues a SIGWINCH. + def preserve_module(name): + return name.startswith(("zope", "readline")) + + return SysModulesSnapshot(preserve=preserve_module) + + def make_hook_recorder(self, pluginmanager: PytestPluginManager) -> HookRecorder: + """Create a new :py:class:`HookRecorder` for a PluginManager.""" + pluginmanager.reprec = reprec = HookRecorder(pluginmanager, _ispytest=True) + self._request.addfinalizer(reprec.finish_recording) + return reprec + + def chdir(self) -> None: + """Cd into the temporary directory. + + This is done automatically upon instantiation. + """ + os.chdir(self.path) + + def _makefile( + self, + ext: str, + lines: Sequence[Union[Any, bytes]], + files: Dict[str, str], + encoding: str = "utf-8", + ) -> Path: + items = list(files.items()) + + if ext and not ext.startswith("."): + raise ValueError( + f"pytester.makefile expects a file extension, try .{ext} instead of {ext}" + ) + + def to_text(s: Union[Any, bytes]) -> str: + return s.decode(encoding) if isinstance(s, bytes) else str(s) + + if lines: + source = "\n".join(to_text(x) for x in lines) + basename = self._name + items.insert(0, (basename, source)) + + ret = None + for basename, value in items: + p = self.path.joinpath(basename).with_suffix(ext) + p.parent.mkdir(parents=True, exist_ok=True) + source_ = Source(value) + source = "\n".join(to_text(line) for line in source_.lines) + p.write_text(source.strip(), encoding=encoding) + if ret is None: + ret = p + assert ret is not None + return ret + + def makefile(self, ext: str, *args: str, **kwargs: str) -> Path: + r"""Create new text file(s) in the test directory. + + :param str ext: + The extension the file(s) should use, including the dot, e.g. `.py`. + :param args: + All args are treated as strings and joined using newlines. + The result is written as contents to the file. The name of the + file is based on the test function requesting this fixture. + :param kwargs: + Each keyword is the name of a file, while the value of it will + be written as contents of the file. + + Examples: + + .. code-block:: python + + pytester.makefile(".txt", "line1", "line2") + + pytester.makefile(".ini", pytest="[pytest]\naddopts=-rs\n") + + To create binary files, use :meth:`pathlib.Path.write_bytes` directly: + + .. code-block:: python + + filename = pytester.path.joinpath("foo.bin") + filename.write_bytes(b"...") + """ + return self._makefile(ext, args, kwargs) + + def makeconftest(self, source: str) -> Path: + """Write a contest.py file with 'source' as contents.""" + return self.makepyfile(conftest=source) + + def makeini(self, source: str) -> Path: + """Write a tox.ini file with 'source' as contents.""" + return self.makefile(".ini", tox=source) + + def getinicfg(self, source: str) -> SectionWrapper: + """Return the pytest section from the tox.ini config file.""" + p = self.makeini(source) + return IniConfig(str(p))["pytest"] + + def makepyprojecttoml(self, source: str) -> Path: + """Write a pyproject.toml file with 'source' as contents. + + .. versionadded:: 6.0 + """ + return self.makefile(".toml", pyproject=source) + + def makepyfile(self, *args, **kwargs) -> Path: + r"""Shortcut for .makefile() with a .py extension. + + Defaults to the test name with a '.py' extension, e.g test_foobar.py, overwriting + existing files. + + Examples: + + .. code-block:: python + + def test_something(pytester): + # Initial file is created test_something.py. + pytester.makepyfile("foobar") + # To create multiple files, pass kwargs accordingly. + pytester.makepyfile(custom="foobar") + # At this point, both 'test_something.py' & 'custom.py' exist in the test directory. + + """ + return self._makefile(".py", args, kwargs) + + def maketxtfile(self, *args, **kwargs) -> Path: + r"""Shortcut for .makefile() with a .txt extension. + + Defaults to the test name with a '.txt' extension, e.g test_foobar.txt, overwriting + existing files. + + Examples: + + .. code-block:: python + + def test_something(pytester): + # Initial file is created test_something.txt. + pytester.maketxtfile("foobar") + # To create multiple files, pass kwargs accordingly. + pytester.maketxtfile(custom="foobar") + # At this point, both 'test_something.txt' & 'custom.txt' exist in the test directory. + + """ + return self._makefile(".txt", args, kwargs) + + def syspathinsert( + self, path: Optional[Union[str, "os.PathLike[str]"]] = None + ) -> None: + """Prepend a directory to sys.path, defaults to :attr:`path`. + + This is undone automatically when this object dies at the end of each + test. + """ + if path is None: + path = self.path + + self._monkeypatch.syspath_prepend(str(path)) + + def mkdir(self, name: str) -> Path: + """Create a new (sub)directory.""" + p = self.path / name + p.mkdir() + return p + + def mkpydir(self, name: str) -> Path: + """Create a new python package. + + This creates a (sub)directory with an empty ``__init__.py`` file so it + gets recognised as a Python package. + """ + p = self.path / name + p.mkdir() + p.joinpath("__init__.py").touch() + return p + + def copy_example(self, name: Optional[str] = None) -> Path: + """Copy file from project's directory into the testdir. + + :param str name: The name of the file to copy. + :return: path to the copied directory (inside ``self.path``). + + """ + example_dir = self._request.config.getini("pytester_example_dir") + if example_dir is None: + raise ValueError("pytester_example_dir is unset, can't copy examples") + example_dir = self._request.config.rootpath / example_dir + + for extra_element in self._request.node.iter_markers("pytester_example_path"): + assert extra_element.args + example_dir = example_dir.joinpath(*extra_element.args) + + if name is None: + func_name = self._name + maybe_dir = example_dir / func_name + maybe_file = example_dir / (func_name + ".py") + + if maybe_dir.is_dir(): + example_path = maybe_dir + elif maybe_file.is_file(): + example_path = maybe_file + else: + raise LookupError( + f"{func_name} can't be found as module or package in {example_dir}" + ) + else: + example_path = example_dir.joinpath(name) + + if example_path.is_dir() and not example_path.joinpath("__init__.py").is_file(): + copytree(example_path, self.path) + return self.path + elif example_path.is_file(): + result = self.path.joinpath(example_path.name) + shutil.copy(example_path, result) + return result + else: + raise LookupError( + f'example "{example_path}" is not found as a file or directory' + ) + + def getnode( + self, config: Config, arg: Union[str, "os.PathLike[str]"] + ) -> Optional[Union[Collector, Item]]: + """Return the collection node of a file. + + :param pytest.Config config: + A pytest config. + See :py:meth:`parseconfig` and :py:meth:`parseconfigure` for creating it. + :param os.PathLike[str] arg: + Path to the file. + """ + session = Session.from_config(config) + assert "::" not in str(arg) + p = Path(os.path.abspath(arg)) + config.hook.pytest_sessionstart(session=session) + res = session.perform_collect([str(p)], genitems=False)[0] + config.hook.pytest_sessionfinish(session=session, exitstatus=ExitCode.OK) + return res + + def getpathnode(self, path: Union[str, "os.PathLike[str]"]): + """Return the collection node of a file. + + This is like :py:meth:`getnode` but uses :py:meth:`parseconfigure` to + create the (configured) pytest Config instance. + + :param os.PathLike[str] path: Path to the file. + """ + path = Path(path) + config = self.parseconfigure(path) + session = Session.from_config(config) + x = bestrelpath(session.path, path) + config.hook.pytest_sessionstart(session=session) + res = session.perform_collect([x], genitems=False)[0] + config.hook.pytest_sessionfinish(session=session, exitstatus=ExitCode.OK) + return res + + def genitems(self, colitems: Sequence[Union[Item, Collector]]) -> List[Item]: + """Generate all test items from a collection node. + + This recurses into the collection node and returns a list of all the + test items contained within. + """ + session = colitems[0].session + result: List[Item] = [] + for colitem in colitems: + result.extend(session.genitems(colitem)) + return result + + def runitem(self, source: str) -> Any: + """Run the "test_func" Item. + + The calling test instance (class containing the test method) must + provide a ``.getrunner()`` method which should return a runner which + can run the test protocol for a single item, e.g. + :py:func:`_pytest.runner.runtestprotocol`. + """ + # used from runner functional tests + item = self.getitem(source) + # the test class where we are called from wants to provide the runner + testclassinstance = self._request.instance + runner = testclassinstance.getrunner() + return runner(item) + + def inline_runsource(self, source: str, *cmdlineargs) -> HookRecorder: + """Run a test module in process using ``pytest.main()``. + + This run writes "source" into a temporary file and runs + ``pytest.main()`` on it, returning a :py:class:`HookRecorder` instance + for the result. + + :param source: The source code of the test module. + :param cmdlineargs: Any extra command line arguments to use. + """ + p = self.makepyfile(source) + values = list(cmdlineargs) + [p] + return self.inline_run(*values) + + def inline_genitems(self, *args) -> Tuple[List[Item], HookRecorder]: + """Run ``pytest.main(['--collectonly'])`` in-process. + + Runs the :py:func:`pytest.main` function to run all of pytest inside + the test process itself like :py:meth:`inline_run`, but returns a + tuple of the collected items and a :py:class:`HookRecorder` instance. + """ + rec = self.inline_run("--collect-only", *args) + items = [x.item for x in rec.getcalls("pytest_itemcollected")] + return items, rec + + def inline_run( + self, + *args: Union[str, "os.PathLike[str]"], + plugins=(), + no_reraise_ctrlc: bool = False, + ) -> HookRecorder: + """Run ``pytest.main()`` in-process, returning a HookRecorder. + + Runs the :py:func:`pytest.main` function to run all of pytest inside + the test process itself. This means it can return a + :py:class:`HookRecorder` instance which gives more detailed results + from that run than can be done by matching stdout/stderr from + :py:meth:`runpytest`. + + :param args: + Command line arguments to pass to :py:func:`pytest.main`. + :param plugins: + Extra plugin instances the ``pytest.main()`` instance should use. + :param no_reraise_ctrlc: + Typically we reraise keyboard interrupts from the child run. If + True, the KeyboardInterrupt exception is captured. + """ + # (maybe a cpython bug?) the importlib cache sometimes isn't updated + # properly between file creation and inline_run (especially if imports + # are interspersed with file creation) + importlib.invalidate_caches() + + plugins = list(plugins) + finalizers = [] + try: + # Any sys.module or sys.path changes done while running pytest + # inline should be reverted after the test run completes to avoid + # clashing with later inline tests run within the same pytest test, + # e.g. just because they use matching test module names. + finalizers.append(self.__take_sys_modules_snapshot().restore) + finalizers.append(SysPathsSnapshot().restore) + + # Important note: + # - our tests should not leave any other references/registrations + # laying around other than possibly loaded test modules + # referenced from sys.modules, as nothing will clean those up + # automatically + + rec = [] + + class Collect: + def pytest_configure(x, config: Config) -> None: + rec.append(self.make_hook_recorder(config.pluginmanager)) + + plugins.append(Collect()) + ret = main([str(x) for x in args], plugins=plugins) + if len(rec) == 1: + reprec = rec.pop() + else: + + class reprec: # type: ignore + pass + + reprec.ret = ret + + # Typically we reraise keyboard interrupts from the child run + # because it's our user requesting interruption of the testing. + if ret == ExitCode.INTERRUPTED and not no_reraise_ctrlc: + calls = reprec.getcalls("pytest_keyboard_interrupt") + if calls and calls[-1].excinfo.type == KeyboardInterrupt: + raise KeyboardInterrupt() + return reprec + finally: + for finalizer in finalizers: + finalizer() + + def runpytest_inprocess( + self, *args: Union[str, "os.PathLike[str]"], **kwargs: Any + ) -> RunResult: + """Return result of running pytest in-process, providing a similar + interface to what self.runpytest() provides.""" + syspathinsert = kwargs.pop("syspathinsert", False) + + if syspathinsert: + self.syspathinsert() + now = timing.time() + capture = _get_multicapture("sys") + capture.start_capturing() + try: + try: + reprec = self.inline_run(*args, **kwargs) + except SystemExit as e: + ret = e.args[0] + try: + ret = ExitCode(e.args[0]) + except ValueError: + pass + + class reprec: # type: ignore + ret = ret + + except Exception: + traceback.print_exc() + + class reprec: # type: ignore + ret = ExitCode(3) + + finally: + out, err = capture.readouterr() + capture.stop_capturing() + sys.stdout.write(out) + sys.stderr.write(err) + + assert reprec.ret is not None + res = RunResult( + reprec.ret, out.splitlines(), err.splitlines(), timing.time() - now + ) + res.reprec = reprec # type: ignore + return res + + def runpytest( + self, *args: Union[str, "os.PathLike[str]"], **kwargs: Any + ) -> RunResult: + """Run pytest inline or in a subprocess, depending on the command line + option "--runpytest" and return a :py:class:`~pytest.RunResult`.""" + new_args = self._ensure_basetemp(args) + if self._method == "inprocess": + return self.runpytest_inprocess(*new_args, **kwargs) + elif self._method == "subprocess": + return self.runpytest_subprocess(*new_args, **kwargs) + raise RuntimeError(f"Unrecognized runpytest option: {self._method}") + + def _ensure_basetemp( + self, args: Sequence[Union[str, "os.PathLike[str]"]] + ) -> List[Union[str, "os.PathLike[str]"]]: + new_args = list(args) + for x in new_args: + if str(x).startswith("--basetemp"): + break + else: + new_args.append("--basetemp=%s" % self.path.parent.joinpath("basetemp")) + return new_args + + def parseconfig(self, *args: Union[str, "os.PathLike[str]"]) -> Config: + """Return a new pytest Config instance from given commandline args. + + This invokes the pytest bootstrapping code in _pytest.config to create + a new :py:class:`_pytest.core.PluginManager` and call the + pytest_cmdline_parse hook to create a new + :py:class:`pytest.Config` instance. + + If :py:attr:`plugins` has been populated they should be plugin modules + to be registered with the PluginManager. + """ + import _pytest.config + + new_args = self._ensure_basetemp(args) + new_args = [str(x) for x in new_args] + + config = _pytest.config._prepareconfig(new_args, self.plugins) # type: ignore[arg-type] + # we don't know what the test will do with this half-setup config + # object and thus we make sure it gets unconfigured properly in any + # case (otherwise capturing could still be active, for example) + self._request.addfinalizer(config._ensure_unconfigure) + return config + + def parseconfigure(self, *args: Union[str, "os.PathLike[str]"]) -> Config: + """Return a new pytest configured Config instance. + + Returns a new :py:class:`pytest.Config` instance like + :py:meth:`parseconfig`, but also calls the pytest_configure hook. + """ + config = self.parseconfig(*args) + config._do_configure() + return config + + def getitem( + self, source: Union[str, "os.PathLike[str]"], funcname: str = "test_func" + ) -> Item: + """Return the test item for a test function. + + Writes the source to a python file and runs pytest's collection on + the resulting module, returning the test item for the requested + function name. + + :param source: + The module source. + :param funcname: + The name of the test function for which to return a test item. + """ + items = self.getitems(source) + for item in items: + if item.name == funcname: + return item + assert 0, "{!r} item not found in module:\n{}\nitems: {}".format( + funcname, source, items + ) + + def getitems(self, source: Union[str, "os.PathLike[str]"]) -> List[Item]: + """Return all test items collected from the module. + + Writes the source to a Python file and runs pytest's collection on + the resulting module, returning all test items contained within. + """ + modcol = self.getmodulecol(source) + return self.genitems([modcol]) + + def getmodulecol( + self, + source: Union[str, "os.PathLike[str]"], + configargs=(), + *, + withinit: bool = False, + ): + """Return the module collection node for ``source``. + + Writes ``source`` to a file using :py:meth:`makepyfile` and then + runs the pytest collection on it, returning the collection node for the + test module. + + :param source: + The source code of the module to collect. + + :param configargs: + Any extra arguments to pass to :py:meth:`parseconfigure`. + + :param withinit: + Whether to also write an ``__init__.py`` file to the same + directory to ensure it is a package. + """ + if isinstance(source, os.PathLike): + path = self.path.joinpath(source) + assert not withinit, "not supported for paths" + else: + kw = {self._name: str(source)} + path = self.makepyfile(**kw) + if withinit: + self.makepyfile(__init__="#") + self.config = config = self.parseconfigure(path, *configargs) + return self.getnode(config, path) + + def collect_by_name( + self, modcol: Collector, name: str + ) -> Optional[Union[Item, Collector]]: + """Return the collection node for name from the module collection. + + Searches a module collection node for a collection node matching the + given name. + + :param modcol: A module collection node; see :py:meth:`getmodulecol`. + :param name: The name of the node to return. + """ + if modcol not in self._mod_collections: + self._mod_collections[modcol] = list(modcol.collect()) + for colitem in self._mod_collections[modcol]: + if colitem.name == name: + return colitem + return None + + def popen( + self, + cmdargs: Sequence[Union[str, "os.PathLike[str]"]], + stdout: Union[int, TextIO] = subprocess.PIPE, + stderr: Union[int, TextIO] = subprocess.PIPE, + stdin: Union[NotSetType, bytes, IO[Any], int] = CLOSE_STDIN, + **kw, + ): + """Invoke :py:class:`subprocess.Popen`. + + Calls :py:class:`subprocess.Popen` making sure the current working + directory is in ``PYTHONPATH``. + + You probably want to use :py:meth:`run` instead. + """ + env = os.environ.copy() + env["PYTHONPATH"] = os.pathsep.join( + filter(None, [os.getcwd(), env.get("PYTHONPATH", "")]) + ) + kw["env"] = env + + if stdin is self.CLOSE_STDIN: + kw["stdin"] = subprocess.PIPE + elif isinstance(stdin, bytes): + kw["stdin"] = subprocess.PIPE + else: + kw["stdin"] = stdin + + popen = subprocess.Popen(cmdargs, stdout=stdout, stderr=stderr, **kw) + if stdin is self.CLOSE_STDIN: + assert popen.stdin is not None + popen.stdin.close() + elif isinstance(stdin, bytes): + assert popen.stdin is not None + popen.stdin.write(stdin) + + return popen + + def run( + self, + *cmdargs: Union[str, "os.PathLike[str]"], + timeout: Optional[float] = None, + stdin: Union[NotSetType, bytes, IO[Any], int] = CLOSE_STDIN, + ) -> RunResult: + """Run a command with arguments. + + Run a process using :py:class:`subprocess.Popen` saving the stdout and + stderr. + + :param cmdargs: + The sequence of arguments to pass to :py:class:`subprocess.Popen`, + with path-like objects being converted to :py:class:`str` + automatically. + :param timeout: + The period in seconds after which to timeout and raise + :py:class:`Pytester.TimeoutExpired`. + :param stdin: + Optional standard input. + + - If it is :py:attr:`CLOSE_STDIN` (Default), then this method calls + :py:class:`subprocess.Popen` with ``stdin=subprocess.PIPE``, and + the standard input is closed immediately after the new command is + started. + + - If it is of type :py:class:`bytes`, these bytes are sent to the + standard input of the command. + + - Otherwise, it is passed through to :py:class:`subprocess.Popen`. + For further information in this case, consult the document of the + ``stdin`` parameter in :py:class:`subprocess.Popen`. + """ + __tracebackhide__ = True + + cmdargs = tuple(os.fspath(arg) for arg in cmdargs) + p1 = self.path.joinpath("stdout") + p2 = self.path.joinpath("stderr") + print("running:", *cmdargs) + print(" in:", Path.cwd()) + + with p1.open("w", encoding="utf8") as f1, p2.open("w", encoding="utf8") as f2: + now = timing.time() + popen = self.popen( + cmdargs, + stdin=stdin, + stdout=f1, + stderr=f2, + close_fds=(sys.platform != "win32"), + ) + if popen.stdin is not None: + popen.stdin.close() + + def handle_timeout() -> None: + __tracebackhide__ = True + + timeout_message = ( + "{seconds} second timeout expired running:" + " {command}".format(seconds=timeout, command=cmdargs) + ) + + popen.kill() + popen.wait() + raise self.TimeoutExpired(timeout_message) + + if timeout is None: + ret = popen.wait() + else: + try: + ret = popen.wait(timeout) + except subprocess.TimeoutExpired: + handle_timeout() + + with p1.open(encoding="utf8") as f1, p2.open(encoding="utf8") as f2: + out = f1.read().splitlines() + err = f2.read().splitlines() + + self._dump_lines(out, sys.stdout) + self._dump_lines(err, sys.stderr) + + with contextlib.suppress(ValueError): + ret = ExitCode(ret) + return RunResult(ret, out, err, timing.time() - now) + + def _dump_lines(self, lines, fp): + try: + for line in lines: + print(line, file=fp) + except UnicodeEncodeError: + print(f"couldn't print to {fp} because of encoding") + + def _getpytestargs(self) -> Tuple[str, ...]: + return sys.executable, "-mpytest" + + def runpython(self, script: "os.PathLike[str]") -> RunResult: + """Run a python script using sys.executable as interpreter.""" + return self.run(sys.executable, script) + + def runpython_c(self, command: str) -> RunResult: + """Run ``python -c "command"``.""" + return self.run(sys.executable, "-c", command) + + def runpytest_subprocess( + self, *args: Union[str, "os.PathLike[str]"], timeout: Optional[float] = None + ) -> RunResult: + """Run pytest as a subprocess with given arguments. + + Any plugins added to the :py:attr:`plugins` list will be added using the + ``-p`` command line option. Additionally ``--basetemp`` is used to put + any temporary files and directories in a numbered directory prefixed + with "runpytest-" to not conflict with the normal numbered pytest + location for temporary files and directories. + + :param args: + The sequence of arguments to pass to the pytest subprocess. + :param timeout: + The period in seconds after which to timeout and raise + :py:class:`Pytester.TimeoutExpired`. + """ + __tracebackhide__ = True + p = make_numbered_dir(root=self.path, prefix="runpytest-", mode=0o700) + args = ("--basetemp=%s" % p,) + args + plugins = [x for x in self.plugins if isinstance(x, str)] + if plugins: + args = ("-p", plugins[0]) + args + args = self._getpytestargs() + args + return self.run(*args, timeout=timeout) + + def spawn_pytest( + self, string: str, expect_timeout: float = 10.0 + ) -> "pexpect.spawn": + """Run pytest using pexpect. + + This makes sure to use the right pytest and sets up the temporary + directory locations. + + The pexpect child is returned. + """ + basetemp = self.path / "temp-pexpect" + basetemp.mkdir(mode=0o700) + invoke = " ".join(map(str, self._getpytestargs())) + cmd = f"{invoke} --basetemp={basetemp} {string}" + return self.spawn(cmd, expect_timeout=expect_timeout) + + def spawn(self, cmd: str, expect_timeout: float = 10.0) -> "pexpect.spawn": + """Run a command using pexpect. + + The pexpect child is returned. + """ + pexpect = importorskip("pexpect", "3.0") + if hasattr(sys, "pypy_version_info") and "64" in platform.machine(): + skip("pypy-64 bit not supported") + if not hasattr(pexpect, "spawn"): + skip("pexpect.spawn not available") + logfile = self.path.joinpath("spawn.out").open("wb") + + child = pexpect.spawn(cmd, logfile=logfile, timeout=expect_timeout) + self._request.addfinalizer(logfile.close) + return child + + +class LineComp: + def __init__(self) -> None: + self.stringio = StringIO() + """:class:`python:io.StringIO()` instance used for input.""" + + def assert_contains_lines(self, lines2: Sequence[str]) -> None: + """Assert that ``lines2`` are contained (linearly) in :attr:`stringio`'s value. + + Lines are matched using :func:`LineMatcher.fnmatch_lines `. + """ + __tracebackhide__ = True + val = self.stringio.getvalue() + self.stringio.truncate(0) + self.stringio.seek(0) + lines1 = val.split("\n") + LineMatcher(lines1).fnmatch_lines(lines2) + + +class LineMatcher: + """Flexible matching of text. + + This is a convenience class to test large texts like the output of + commands. + + The constructor takes a list of lines without their trailing newlines, i.e. + ``text.splitlines()``. + """ + + def __init__(self, lines: List[str]) -> None: + self.lines = lines + self._log_output: List[str] = [] + + def __str__(self) -> str: + """Return the entire original text. + + .. versionadded:: 6.2 + You can use :meth:`str` in older versions. + """ + return "\n".join(self.lines) + + def _getlines(self, lines2: Union[str, Sequence[str], Source]) -> Sequence[str]: + if isinstance(lines2, str): + lines2 = Source(lines2) + if isinstance(lines2, Source): + lines2 = lines2.strip().lines + return lines2 + + def fnmatch_lines_random(self, lines2: Sequence[str]) -> None: + """Check lines exist in the output in any order (using :func:`python:fnmatch.fnmatch`).""" + __tracebackhide__ = True + self._match_lines_random(lines2, fnmatch) + + def re_match_lines_random(self, lines2: Sequence[str]) -> None: + """Check lines exist in the output in any order (using :func:`python:re.match`).""" + __tracebackhide__ = True + self._match_lines_random(lines2, lambda name, pat: bool(re.match(pat, name))) + + def _match_lines_random( + self, lines2: Sequence[str], match_func: Callable[[str, str], bool] + ) -> None: + __tracebackhide__ = True + lines2 = self._getlines(lines2) + for line in lines2: + for x in self.lines: + if line == x or match_func(x, line): + self._log("matched: ", repr(line)) + break + else: + msg = "line %r not found in output" % line + self._log(msg) + self._fail(msg) + + def get_lines_after(self, fnline: str) -> Sequence[str]: + """Return all lines following the given line in the text. + + The given line can contain glob wildcards. + """ + for i, line in enumerate(self.lines): + if fnline == line or fnmatch(line, fnline): + return self.lines[i + 1 :] + raise ValueError("line %r not found in output" % fnline) + + def _log(self, *args) -> None: + self._log_output.append(" ".join(str(x) for x in args)) + + @property + def _log_text(self) -> str: + return "\n".join(self._log_output) + + def fnmatch_lines( + self, lines2: Sequence[str], *, consecutive: bool = False + ) -> None: + """Check lines exist in the output (using :func:`python:fnmatch.fnmatch`). + + The argument is a list of lines which have to match and can use glob + wildcards. If they do not match a pytest.fail() is called. The + matches and non-matches are also shown as part of the error message. + + :param lines2: String patterns to match. + :param consecutive: Match lines consecutively? + """ + __tracebackhide__ = True + self._match_lines(lines2, fnmatch, "fnmatch", consecutive=consecutive) + + def re_match_lines( + self, lines2: Sequence[str], *, consecutive: bool = False + ) -> None: + """Check lines exist in the output (using :func:`python:re.match`). + + The argument is a list of lines which have to match using ``re.match``. + If they do not match a pytest.fail() is called. + + The matches and non-matches are also shown as part of the error message. + + :param lines2: string patterns to match. + :param consecutive: match lines consecutively? + """ + __tracebackhide__ = True + self._match_lines( + lines2, + lambda name, pat: bool(re.match(pat, name)), + "re.match", + consecutive=consecutive, + ) + + def _match_lines( + self, + lines2: Sequence[str], + match_func: Callable[[str, str], bool], + match_nickname: str, + *, + consecutive: bool = False, + ) -> None: + """Underlying implementation of ``fnmatch_lines`` and ``re_match_lines``. + + :param Sequence[str] lines2: + List of string patterns to match. The actual format depends on + ``match_func``. + :param match_func: + A callable ``match_func(line, pattern)`` where line is the + captured line from stdout/stderr and pattern is the matching + pattern. + :param str match_nickname: + The nickname for the match function that will be logged to stdout + when a match occurs. + :param consecutive: + Match lines consecutively? + """ + if not isinstance(lines2, collections.abc.Sequence): + raise TypeError(f"invalid type for lines2: {type(lines2).__name__}") + lines2 = self._getlines(lines2) + lines1 = self.lines[:] + extralines = [] + __tracebackhide__ = True + wnick = len(match_nickname) + 1 + started = False + for line in lines2: + nomatchprinted = False + while lines1: + nextline = lines1.pop(0) + if line == nextline: + self._log("exact match:", repr(line)) + started = True + break + elif match_func(nextline, line): + self._log("%s:" % match_nickname, repr(line)) + self._log( + "{:>{width}}".format("with:", width=wnick), repr(nextline) + ) + started = True + break + else: + if consecutive and started: + msg = f"no consecutive match: {line!r}" + self._log(msg) + self._log( + "{:>{width}}".format("with:", width=wnick), repr(nextline) + ) + self._fail(msg) + if not nomatchprinted: + self._log( + "{:>{width}}".format("nomatch:", width=wnick), repr(line) + ) + nomatchprinted = True + self._log("{:>{width}}".format("and:", width=wnick), repr(nextline)) + extralines.append(nextline) + else: + msg = f"remains unmatched: {line!r}" + self._log(msg) + self._fail(msg) + self._log_output = [] + + def no_fnmatch_line(self, pat: str) -> None: + """Ensure captured lines do not match the given pattern, using ``fnmatch.fnmatch``. + + :param str pat: The pattern to match lines. + """ + __tracebackhide__ = True + self._no_match_line(pat, fnmatch, "fnmatch") + + def no_re_match_line(self, pat: str) -> None: + """Ensure captured lines do not match the given pattern, using ``re.match``. + + :param str pat: The regular expression to match lines. + """ + __tracebackhide__ = True + self._no_match_line( + pat, lambda name, pat: bool(re.match(pat, name)), "re.match" + ) + + def _no_match_line( + self, pat: str, match_func: Callable[[str, str], bool], match_nickname: str + ) -> None: + """Ensure captured lines does not have a the given pattern, using ``fnmatch.fnmatch``. + + :param str pat: The pattern to match lines. + """ + __tracebackhide__ = True + nomatch_printed = False + wnick = len(match_nickname) + 1 + for line in self.lines: + if match_func(line, pat): + msg = f"{match_nickname}: {pat!r}" + self._log(msg) + self._log("{:>{width}}".format("with:", width=wnick), repr(line)) + self._fail(msg) + else: + if not nomatch_printed: + self._log("{:>{width}}".format("nomatch:", width=wnick), repr(pat)) + nomatch_printed = True + self._log("{:>{width}}".format("and:", width=wnick), repr(line)) + self._log_output = [] + + def _fail(self, msg: str) -> None: + __tracebackhide__ = True + log_text = self._log_text + self._log_output = [] + fail(log_text) + + def str(self) -> str: + """Return the entire original text.""" + return str(self) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pytester_assertions.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pytester_assertions.py new file mode 100644 index 0000000000..657e4db5fc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/pytester_assertions.py @@ -0,0 +1,75 @@ +"""Helper plugin for pytester; should not be loaded on its own.""" +# This plugin contains assertions used by pytester. pytester cannot +# contain them itself, since it is imported by the `pytest` module, +# hence cannot be subject to assertion rewriting, which requires a +# module to not be already imported. +from typing import Dict +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Union + +from _pytest.reports import CollectReport +from _pytest.reports import TestReport + + +def assertoutcome( + outcomes: Tuple[ + Sequence[TestReport], + Sequence[Union[CollectReport, TestReport]], + Sequence[Union[CollectReport, TestReport]], + ], + passed: int = 0, + skipped: int = 0, + failed: int = 0, +) -> None: + __tracebackhide__ = True + + realpassed, realskipped, realfailed = outcomes + obtained = { + "passed": len(realpassed), + "skipped": len(realskipped), + "failed": len(realfailed), + } + expected = {"passed": passed, "skipped": skipped, "failed": failed} + assert obtained == expected, outcomes + + +def assert_outcomes( + outcomes: Dict[str, int], + passed: int = 0, + skipped: int = 0, + failed: int = 0, + errors: int = 0, + xpassed: int = 0, + xfailed: int = 0, + warnings: Optional[int] = None, + deselected: Optional[int] = None, +) -> None: + """Assert that the specified outcomes appear with the respective + numbers (0 means it didn't occur) in the text output from a test run.""" + __tracebackhide__ = True + + obtained = { + "passed": outcomes.get("passed", 0), + "skipped": outcomes.get("skipped", 0), + "failed": outcomes.get("failed", 0), + "errors": outcomes.get("errors", 0), + "xpassed": outcomes.get("xpassed", 0), + "xfailed": outcomes.get("xfailed", 0), + } + expected = { + "passed": passed, + "skipped": skipped, + "failed": failed, + "errors": errors, + "xpassed": xpassed, + "xfailed": xfailed, + } + if warnings is not None: + obtained["warnings"] = outcomes.get("warnings", 0) + expected["warnings"] = warnings + if deselected is not None: + obtained["deselected"] = outcomes.get("deselected", 0) + expected["deselected"] = deselected + assert obtained == expected diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/python.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/python.py new file mode 100644 index 0000000000..0fd5702a5c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/python.py @@ -0,0 +1,1764 @@ +"""Python test discovery, setup and run of test functions.""" +import enum +import fnmatch +import inspect +import itertools +import os +import sys +import types +import warnings +from collections import Counter +from collections import defaultdict +from functools import partial +from pathlib import Path +from typing import Any +from typing import Callable +from typing import Dict +from typing import Generator +from typing import Iterable +from typing import Iterator +from typing import List +from typing import Mapping +from typing import Optional +from typing import Pattern +from typing import Sequence +from typing import Set +from typing import Tuple +from typing import TYPE_CHECKING +from typing import Union + +import attr + +import _pytest +from _pytest import fixtures +from _pytest import nodes +from _pytest._code import filter_traceback +from _pytest._code import getfslineno +from _pytest._code.code import ExceptionInfo +from _pytest._code.code import TerminalRepr +from _pytest._io import TerminalWriter +from _pytest._io.saferepr import saferepr +from _pytest.compat import ascii_escaped +from _pytest.compat import assert_never +from _pytest.compat import final +from _pytest.compat import get_default_arg_names +from _pytest.compat import get_real_func +from _pytest.compat import getimfunc +from _pytest.compat import getlocation +from _pytest.compat import is_async_function +from _pytest.compat import is_generator +from _pytest.compat import LEGACY_PATH +from _pytest.compat import NOTSET +from _pytest.compat import safe_getattr +from _pytest.compat import safe_isclass +from _pytest.compat import STRING_TYPES +from _pytest.config import Config +from _pytest.config import ExitCode +from _pytest.config import hookimpl +from _pytest.config.argparsing import Parser +from _pytest.deprecated import check_ispytest +from _pytest.deprecated import FSCOLLECTOR_GETHOOKPROXY_ISINITPATH +from _pytest.deprecated import INSTANCE_COLLECTOR +from _pytest.fixtures import FuncFixtureInfo +from _pytest.main import Session +from _pytest.mark import MARK_GEN +from _pytest.mark import ParameterSet +from _pytest.mark.structures import get_unpacked_marks +from _pytest.mark.structures import Mark +from _pytest.mark.structures import MarkDecorator +from _pytest.mark.structures import normalize_mark_list +from _pytest.outcomes import fail +from _pytest.outcomes import skip +from _pytest.pathlib import bestrelpath +from _pytest.pathlib import fnmatch_ex +from _pytest.pathlib import import_path +from _pytest.pathlib import ImportPathMismatchError +from _pytest.pathlib import parts +from _pytest.pathlib import visit +from _pytest.scope import Scope +from _pytest.warning_types import PytestCollectionWarning +from _pytest.warning_types import PytestUnhandledCoroutineWarning + +if TYPE_CHECKING: + from typing_extensions import Literal + from _pytest.scope import _ScopeName + + +_PYTEST_DIR = Path(_pytest.__file__).parent + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("general") + group.addoption( + "--fixtures", + "--funcargs", + action="store_true", + dest="showfixtures", + default=False, + help="show available fixtures, sorted by plugin appearance " + "(fixtures with leading '_' are only shown with '-v')", + ) + group.addoption( + "--fixtures-per-test", + action="store_true", + dest="show_fixtures_per_test", + default=False, + help="show fixtures per test", + ) + parser.addini( + "python_files", + type="args", + # NOTE: default is also used in AssertionRewritingHook. + default=["test_*.py", "*_test.py"], + help="glob-style file patterns for Python test module discovery", + ) + parser.addini( + "python_classes", + type="args", + default=["Test"], + help="prefixes or glob names for Python test class discovery", + ) + parser.addini( + "python_functions", + type="args", + default=["test"], + help="prefixes or glob names for Python test function and method discovery", + ) + parser.addini( + "disable_test_id_escaping_and_forfeit_all_rights_to_community_support", + type="bool", + default=False, + help="disable string escape non-ascii characters, might cause unwanted " + "side effects(use at your own risk)", + ) + + +def pytest_cmdline_main(config: Config) -> Optional[Union[int, ExitCode]]: + if config.option.showfixtures: + showfixtures(config) + return 0 + if config.option.show_fixtures_per_test: + show_fixtures_per_test(config) + return 0 + return None + + +def pytest_generate_tests(metafunc: "Metafunc") -> None: + for marker in metafunc.definition.iter_markers(name="parametrize"): + metafunc.parametrize(*marker.args, **marker.kwargs, _param_mark=marker) + + +def pytest_configure(config: Config) -> None: + config.addinivalue_line( + "markers", + "parametrize(argnames, argvalues): call a test function multiple " + "times passing in different arguments in turn. argvalues generally " + "needs to be a list of values if argnames specifies only one name " + "or a list of tuples of values if argnames specifies multiple names. " + "Example: @parametrize('arg1', [1,2]) would lead to two calls of the " + "decorated test function, one with arg1=1 and another with arg1=2." + "see https://docs.pytest.org/en/stable/how-to/parametrize.html for more info " + "and examples.", + ) + config.addinivalue_line( + "markers", + "usefixtures(fixturename1, fixturename2, ...): mark tests as needing " + "all of the specified fixtures. see " + "https://docs.pytest.org/en/stable/explanation/fixtures.html#usefixtures ", + ) + + +def async_warn_and_skip(nodeid: str) -> None: + msg = "async def functions are not natively supported and have been skipped.\n" + msg += ( + "You need to install a suitable plugin for your async framework, for example:\n" + ) + msg += " - anyio\n" + msg += " - pytest-asyncio\n" + msg += " - pytest-tornasync\n" + msg += " - pytest-trio\n" + msg += " - pytest-twisted" + warnings.warn(PytestUnhandledCoroutineWarning(msg.format(nodeid))) + skip(reason="async def function and no async plugin installed (see warnings)") + + +@hookimpl(trylast=True) +def pytest_pyfunc_call(pyfuncitem: "Function") -> Optional[object]: + testfunction = pyfuncitem.obj + if is_async_function(testfunction): + async_warn_and_skip(pyfuncitem.nodeid) + funcargs = pyfuncitem.funcargs + testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} + result = testfunction(**testargs) + if hasattr(result, "__await__") or hasattr(result, "__aiter__"): + async_warn_and_skip(pyfuncitem.nodeid) + return True + + +def pytest_collect_file(file_path: Path, parent: nodes.Collector) -> Optional["Module"]: + if file_path.suffix == ".py": + if not parent.session.isinitpath(file_path): + if not path_matches_patterns( + file_path, parent.config.getini("python_files") + ["__init__.py"] + ): + return None + ihook = parent.session.gethookproxy(file_path) + module: Module = ihook.pytest_pycollect_makemodule( + module_path=file_path, parent=parent + ) + return module + return None + + +def path_matches_patterns(path: Path, patterns: Iterable[str]) -> bool: + """Return whether path matches any of the patterns in the list of globs given.""" + return any(fnmatch_ex(pattern, path) for pattern in patterns) + + +def pytest_pycollect_makemodule(module_path: Path, parent) -> "Module": + if module_path.name == "__init__.py": + pkg: Package = Package.from_parent(parent, path=module_path) + return pkg + mod: Module = Module.from_parent(parent, path=module_path) + return mod + + +@hookimpl(trylast=True) +def pytest_pycollect_makeitem(collector: "PyCollector", name: str, obj: object): + # Nothing was collected elsewhere, let's do it here. + if safe_isclass(obj): + if collector.istestclass(obj, name): + return Class.from_parent(collector, name=name, obj=obj) + elif collector.istestfunction(obj, name): + # mock seems to store unbound methods (issue473), normalize it. + obj = getattr(obj, "__func__", obj) + # We need to try and unwrap the function if it's a functools.partial + # or a functools.wrapped. + # We mustn't if it's been wrapped with mock.patch (python 2 only). + if not (inspect.isfunction(obj) or inspect.isfunction(get_real_func(obj))): + filename, lineno = getfslineno(obj) + warnings.warn_explicit( + message=PytestCollectionWarning( + "cannot collect %r because it is not a function." % name + ), + category=None, + filename=str(filename), + lineno=lineno + 1, + ) + elif getattr(obj, "__test__", True): + if is_generator(obj): + res = Function.from_parent(collector, name=name) + reason = "yield tests were removed in pytest 4.0 - {name} will be ignored".format( + name=name + ) + res.add_marker(MARK_GEN.xfail(run=False, reason=reason)) + res.warn(PytestCollectionWarning(reason)) + else: + res = list(collector._genfunctions(name, obj)) + return res + + +class PyobjMixin(nodes.Node): + """this mix-in inherits from Node to carry over the typing information + + as its intended to always mix in before a node + its position in the mro is unaffected""" + + _ALLOW_MARKERS = True + + @property + def module(self): + """Python module object this node was collected from (can be None).""" + node = self.getparent(Module) + return node.obj if node is not None else None + + @property + def cls(self): + """Python class object this node was collected from (can be None).""" + node = self.getparent(Class) + return node.obj if node is not None else None + + @property + def instance(self): + """Python instance object the function is bound to. + + Returns None if not a test method, e.g. for a standalone test function, + a staticmethod, a class or a module. + """ + node = self.getparent(Function) + return getattr(node.obj, "__self__", None) if node is not None else None + + @property + def obj(self): + """Underlying Python object.""" + obj = getattr(self, "_obj", None) + if obj is None: + self._obj = obj = self._getobj() + # XXX evil hack + # used to avoid Function marker duplication + if self._ALLOW_MARKERS: + self.own_markers.extend(get_unpacked_marks(self.obj)) + return obj + + @obj.setter + def obj(self, value): + self._obj = value + + def _getobj(self): + """Get the underlying Python object. May be overwritten by subclasses.""" + # TODO: Improve the type of `parent` such that assert/ignore aren't needed. + assert self.parent is not None + obj = self.parent.obj # type: ignore[attr-defined] + return getattr(obj, self.name) + + def getmodpath(self, stopatmodule: bool = True, includemodule: bool = False) -> str: + """Return Python path relative to the containing module.""" + chain = self.listchain() + chain.reverse() + parts = [] + for node in chain: + name = node.name + if isinstance(node, Module): + name = os.path.splitext(name)[0] + if stopatmodule: + if includemodule: + parts.append(name) + break + parts.append(name) + parts.reverse() + return ".".join(parts) + + def reportinfo(self) -> Tuple[Union["os.PathLike[str]", str], Optional[int], str]: + # XXX caching? + obj = self.obj + compat_co_firstlineno = getattr(obj, "compat_co_firstlineno", None) + if isinstance(compat_co_firstlineno, int): + # nose compatibility + file_path = sys.modules[obj.__module__].__file__ + if file_path.endswith(".pyc"): + file_path = file_path[:-1] + path: Union["os.PathLike[str]", str] = file_path + lineno = compat_co_firstlineno + else: + path, lineno = getfslineno(obj) + modpath = self.getmodpath() + assert isinstance(lineno, int) + return path, lineno, modpath + + +# As an optimization, these builtin attribute names are pre-ignored when +# iterating over an object during collection -- the pytest_pycollect_makeitem +# hook is not called for them. +# fmt: off +class _EmptyClass: pass # noqa: E701 +IGNORED_ATTRIBUTES = frozenset.union( # noqa: E305 + frozenset(), + # Module. + dir(types.ModuleType("empty_module")), + # Some extra module attributes the above doesn't catch. + {"__builtins__", "__file__", "__cached__"}, + # Class. + dir(_EmptyClass), + # Instance. + dir(_EmptyClass()), +) +del _EmptyClass +# fmt: on + + +class PyCollector(PyobjMixin, nodes.Collector): + def funcnamefilter(self, name: str) -> bool: + return self._matches_prefix_or_glob_option("python_functions", name) + + def isnosetest(self, obj: object) -> bool: + """Look for the __test__ attribute, which is applied by the + @nose.tools.istest decorator. + """ + # We explicitly check for "is True" here to not mistakenly treat + # classes with a custom __getattr__ returning something truthy (like a + # function) as test classes. + return safe_getattr(obj, "__test__", False) is True + + def classnamefilter(self, name: str) -> bool: + return self._matches_prefix_or_glob_option("python_classes", name) + + def istestfunction(self, obj: object, name: str) -> bool: + if self.funcnamefilter(name) or self.isnosetest(obj): + if isinstance(obj, staticmethod): + # staticmethods need to be unwrapped. + obj = safe_getattr(obj, "__func__", False) + return callable(obj) and fixtures.getfixturemarker(obj) is None + else: + return False + + def istestclass(self, obj: object, name: str) -> bool: + return self.classnamefilter(name) or self.isnosetest(obj) + + def _matches_prefix_or_glob_option(self, option_name: str, name: str) -> bool: + """Check if the given name matches the prefix or glob-pattern defined + in ini configuration.""" + for option in self.config.getini(option_name): + if name.startswith(option): + return True + # Check that name looks like a glob-string before calling fnmatch + # because this is called for every name in each collected module, + # and fnmatch is somewhat expensive to call. + elif ("*" in option or "?" in option or "[" in option) and fnmatch.fnmatch( + name, option + ): + return True + return False + + def collect(self) -> Iterable[Union[nodes.Item, nodes.Collector]]: + if not getattr(self.obj, "__test__", True): + return [] + + # Avoid random getattrs and peek in the __dict__ instead. + dicts = [getattr(self.obj, "__dict__", {})] + if isinstance(self.obj, type): + for basecls in self.obj.__mro__: + dicts.append(basecls.__dict__) + + # In each class, nodes should be definition ordered. Since Python 3.6, + # __dict__ is definition ordered. + seen: Set[str] = set() + dict_values: List[List[Union[nodes.Item, nodes.Collector]]] = [] + ihook = self.ihook + for dic in dicts: + values: List[Union[nodes.Item, nodes.Collector]] = [] + # Note: seems like the dict can change during iteration - + # be careful not to remove the list() without consideration. + for name, obj in list(dic.items()): + if name in IGNORED_ATTRIBUTES: + continue + if name in seen: + continue + seen.add(name) + res = ihook.pytest_pycollect_makeitem( + collector=self, name=name, obj=obj + ) + if res is None: + continue + elif isinstance(res, list): + values.extend(res) + else: + values.append(res) + dict_values.append(values) + + # Between classes in the class hierarchy, reverse-MRO order -- nodes + # inherited from base classes should come before subclasses. + result = [] + for values in reversed(dict_values): + result.extend(values) + return result + + def _genfunctions(self, name: str, funcobj) -> Iterator["Function"]: + modulecol = self.getparent(Module) + assert modulecol is not None + module = modulecol.obj + clscol = self.getparent(Class) + cls = clscol and clscol.obj or None + + definition = FunctionDefinition.from_parent(self, name=name, callobj=funcobj) + fixtureinfo = definition._fixtureinfo + + # pytest_generate_tests impls call metafunc.parametrize() which fills + # metafunc._calls, the outcome of the hook. + metafunc = Metafunc( + definition=definition, + fixtureinfo=fixtureinfo, + config=self.config, + cls=cls, + module=module, + _ispytest=True, + ) + methods = [] + if hasattr(module, "pytest_generate_tests"): + methods.append(module.pytest_generate_tests) + if cls is not None and hasattr(cls, "pytest_generate_tests"): + methods.append(cls().pytest_generate_tests) + self.ihook.pytest_generate_tests.call_extra(methods, dict(metafunc=metafunc)) + + if not metafunc._calls: + yield Function.from_parent(self, name=name, fixtureinfo=fixtureinfo) + else: + # Add funcargs() as fixturedefs to fixtureinfo.arg2fixturedefs. + fm = self.session._fixturemanager + fixtures.add_funcarg_pseudo_fixture_def(self, metafunc, fm) + + # Add_funcarg_pseudo_fixture_def may have shadowed some fixtures + # with direct parametrization, so make sure we update what the + # function really needs. + fixtureinfo.prune_dependency_tree() + + for callspec in metafunc._calls: + subname = f"{name}[{callspec.id}]" + yield Function.from_parent( + self, + name=subname, + callspec=callspec, + fixtureinfo=fixtureinfo, + keywords={callspec.id: True}, + originalname=name, + ) + + +class Module(nodes.File, PyCollector): + """Collector for test classes and functions.""" + + def _getobj(self): + return self._importtestmodule() + + def collect(self) -> Iterable[Union[nodes.Item, nodes.Collector]]: + self._inject_setup_module_fixture() + self._inject_setup_function_fixture() + self.session._fixturemanager.parsefactories(self) + return super().collect() + + def _inject_setup_module_fixture(self) -> None: + """Inject a hidden autouse, module scoped fixture into the collected module object + that invokes setUpModule/tearDownModule if either or both are available. + + Using a fixture to invoke this methods ensures we play nicely and unsurprisingly with + other fixtures (#517). + """ + has_nose = self.config.pluginmanager.has_plugin("nose") + setup_module = _get_first_non_fixture_func( + self.obj, ("setUpModule", "setup_module") + ) + if setup_module is None and has_nose: + # The name "setup" is too common - only treat as fixture if callable. + setup_module = _get_first_non_fixture_func(self.obj, ("setup",)) + if not callable(setup_module): + setup_module = None + teardown_module = _get_first_non_fixture_func( + self.obj, ("tearDownModule", "teardown_module") + ) + if teardown_module is None and has_nose: + teardown_module = _get_first_non_fixture_func(self.obj, ("teardown",)) + # Same as "setup" above - only treat as fixture if callable. + if not callable(teardown_module): + teardown_module = None + + if setup_module is None and teardown_module is None: + return + + @fixtures.fixture( + autouse=True, + scope="module", + # Use a unique name to speed up lookup. + name=f"_xunit_setup_module_fixture_{self.obj.__name__}", + ) + def xunit_setup_module_fixture(request) -> Generator[None, None, None]: + if setup_module is not None: + _call_with_optional_argument(setup_module, request.module) + yield + if teardown_module is not None: + _call_with_optional_argument(teardown_module, request.module) + + self.obj.__pytest_setup_module = xunit_setup_module_fixture + + def _inject_setup_function_fixture(self) -> None: + """Inject a hidden autouse, function scoped fixture into the collected module object + that invokes setup_function/teardown_function if either or both are available. + + Using a fixture to invoke this methods ensures we play nicely and unsurprisingly with + other fixtures (#517). + """ + setup_function = _get_first_non_fixture_func(self.obj, ("setup_function",)) + teardown_function = _get_first_non_fixture_func( + self.obj, ("teardown_function",) + ) + if setup_function is None and teardown_function is None: + return + + @fixtures.fixture( + autouse=True, + scope="function", + # Use a unique name to speed up lookup. + name=f"_xunit_setup_function_fixture_{self.obj.__name__}", + ) + def xunit_setup_function_fixture(request) -> Generator[None, None, None]: + if request.instance is not None: + # in this case we are bound to an instance, so we need to let + # setup_method handle this + yield + return + if setup_function is not None: + _call_with_optional_argument(setup_function, request.function) + yield + if teardown_function is not None: + _call_with_optional_argument(teardown_function, request.function) + + self.obj.__pytest_setup_function = xunit_setup_function_fixture + + def _importtestmodule(self): + # We assume we are only called once per module. + importmode = self.config.getoption("--import-mode") + try: + mod = import_path(self.path, mode=importmode, root=self.config.rootpath) + except SyntaxError as e: + raise self.CollectError( + ExceptionInfo.from_current().getrepr(style="short") + ) from e + except ImportPathMismatchError as e: + raise self.CollectError( + "import file mismatch:\n" + "imported module %r has this __file__ attribute:\n" + " %s\n" + "which is not the same as the test file we want to collect:\n" + " %s\n" + "HINT: remove __pycache__ / .pyc files and/or use a " + "unique basename for your test file modules" % e.args + ) from e + except ImportError as e: + exc_info = ExceptionInfo.from_current() + if self.config.getoption("verbose") < 2: + exc_info.traceback = exc_info.traceback.filter(filter_traceback) + exc_repr = ( + exc_info.getrepr(style="short") + if exc_info.traceback + else exc_info.exconly() + ) + formatted_tb = str(exc_repr) + raise self.CollectError( + "ImportError while importing test module '{path}'.\n" + "Hint: make sure your test modules/packages have valid Python names.\n" + "Traceback:\n" + "{traceback}".format(path=self.path, traceback=formatted_tb) + ) from e + except skip.Exception as e: + if e.allow_module_level: + raise + raise self.CollectError( + "Using pytest.skip outside of a test will skip the entire module. " + "If that's your intention, pass `allow_module_level=True`. " + "If you want to skip a specific test or an entire class, " + "use the @pytest.mark.skip or @pytest.mark.skipif decorators." + ) from e + self.config.pluginmanager.consider_module(mod) + return mod + + +class Package(Module): + def __init__( + self, + fspath: Optional[LEGACY_PATH], + parent: nodes.Collector, + # NOTE: following args are unused: + config=None, + session=None, + nodeid=None, + path=Optional[Path], + ) -> None: + # NOTE: Could be just the following, but kept as-is for compat. + # nodes.FSCollector.__init__(self, fspath, parent=parent) + session = parent.session + nodes.FSCollector.__init__( + self, + fspath=fspath, + path=path, + parent=parent, + config=config, + session=session, + nodeid=nodeid, + ) + self.name = self.path.parent.name + + def setup(self) -> None: + # Not using fixtures to call setup_module here because autouse fixtures + # from packages are not called automatically (#4085). + setup_module = _get_first_non_fixture_func( + self.obj, ("setUpModule", "setup_module") + ) + if setup_module is not None: + _call_with_optional_argument(setup_module, self.obj) + + teardown_module = _get_first_non_fixture_func( + self.obj, ("tearDownModule", "teardown_module") + ) + if teardown_module is not None: + func = partial(_call_with_optional_argument, teardown_module, self.obj) + self.addfinalizer(func) + + def gethookproxy(self, fspath: "os.PathLike[str]"): + warnings.warn(FSCOLLECTOR_GETHOOKPROXY_ISINITPATH, stacklevel=2) + return self.session.gethookproxy(fspath) + + def isinitpath(self, path: Union[str, "os.PathLike[str]"]) -> bool: + warnings.warn(FSCOLLECTOR_GETHOOKPROXY_ISINITPATH, stacklevel=2) + return self.session.isinitpath(path) + + def _recurse(self, direntry: "os.DirEntry[str]") -> bool: + if direntry.name == "__pycache__": + return False + fspath = Path(direntry.path) + ihook = self.session.gethookproxy(fspath.parent) + if ihook.pytest_ignore_collect(collection_path=fspath, config=self.config): + return False + norecursepatterns = self.config.getini("norecursedirs") + if any(fnmatch_ex(pat, fspath) for pat in norecursepatterns): + return False + return True + + def _collectfile( + self, fspath: Path, handle_dupes: bool = True + ) -> Sequence[nodes.Collector]: + assert ( + fspath.is_file() + ), "{!r} is not a file (isdir={!r}, exists={!r}, islink={!r})".format( + fspath, fspath.is_dir(), fspath.exists(), fspath.is_symlink() + ) + ihook = self.session.gethookproxy(fspath) + if not self.session.isinitpath(fspath): + if ihook.pytest_ignore_collect(collection_path=fspath, config=self.config): + return () + + if handle_dupes: + keepduplicates = self.config.getoption("keepduplicates") + if not keepduplicates: + duplicate_paths = self.config.pluginmanager._duplicatepaths + if fspath in duplicate_paths: + return () + else: + duplicate_paths.add(fspath) + + return ihook.pytest_collect_file(file_path=fspath, parent=self) # type: ignore[no-any-return] + + def collect(self) -> Iterable[Union[nodes.Item, nodes.Collector]]: + this_path = self.path.parent + init_module = this_path / "__init__.py" + if init_module.is_file() and path_matches_patterns( + init_module, self.config.getini("python_files") + ): + yield Module.from_parent(self, path=init_module) + pkg_prefixes: Set[Path] = set() + for direntry in visit(str(this_path), recurse=self._recurse): + path = Path(direntry.path) + + # We will visit our own __init__.py file, in which case we skip it. + if direntry.is_file(): + if direntry.name == "__init__.py" and path.parent == this_path: + continue + + parts_ = parts(direntry.path) + if any( + str(pkg_prefix) in parts_ and pkg_prefix / "__init__.py" != path + for pkg_prefix in pkg_prefixes + ): + continue + + if direntry.is_file(): + yield from self._collectfile(path) + elif not direntry.is_dir(): + # Broken symlink or invalid/missing file. + continue + elif path.joinpath("__init__.py").is_file(): + pkg_prefixes.add(path) + + +def _call_with_optional_argument(func, arg) -> None: + """Call the given function with the given argument if func accepts one argument, otherwise + calls func without arguments.""" + arg_count = func.__code__.co_argcount + if inspect.ismethod(func): + arg_count -= 1 + if arg_count: + func(arg) + else: + func() + + +def _get_first_non_fixture_func(obj: object, names: Iterable[str]) -> Optional[object]: + """Return the attribute from the given object to be used as a setup/teardown + xunit-style function, but only if not marked as a fixture to avoid calling it twice.""" + for name in names: + meth: Optional[object] = getattr(obj, name, None) + if meth is not None and fixtures.getfixturemarker(meth) is None: + return meth + return None + + +class Class(PyCollector): + """Collector for test methods.""" + + @classmethod + def from_parent(cls, parent, *, name, obj=None, **kw): + """The public constructor.""" + return super().from_parent(name=name, parent=parent, **kw) + + def newinstance(self): + return self.obj() + + def collect(self) -> Iterable[Union[nodes.Item, nodes.Collector]]: + if not safe_getattr(self.obj, "__test__", True): + return [] + if hasinit(self.obj): + assert self.parent is not None + self.warn( + PytestCollectionWarning( + "cannot collect test class %r because it has a " + "__init__ constructor (from: %s)" + % (self.obj.__name__, self.parent.nodeid) + ) + ) + return [] + elif hasnew(self.obj): + assert self.parent is not None + self.warn( + PytestCollectionWarning( + "cannot collect test class %r because it has a " + "__new__ constructor (from: %s)" + % (self.obj.__name__, self.parent.nodeid) + ) + ) + return [] + + self._inject_setup_class_fixture() + self._inject_setup_method_fixture() + + self.session._fixturemanager.parsefactories(self.newinstance(), self.nodeid) + + return super().collect() + + def _inject_setup_class_fixture(self) -> None: + """Inject a hidden autouse, class scoped fixture into the collected class object + that invokes setup_class/teardown_class if either or both are available. + + Using a fixture to invoke this methods ensures we play nicely and unsurprisingly with + other fixtures (#517). + """ + setup_class = _get_first_non_fixture_func(self.obj, ("setup_class",)) + teardown_class = getattr(self.obj, "teardown_class", None) + if setup_class is None and teardown_class is None: + return + + @fixtures.fixture( + autouse=True, + scope="class", + # Use a unique name to speed up lookup. + name=f"_xunit_setup_class_fixture_{self.obj.__qualname__}", + ) + def xunit_setup_class_fixture(cls) -> Generator[None, None, None]: + if setup_class is not None: + func = getimfunc(setup_class) + _call_with_optional_argument(func, self.obj) + yield + if teardown_class is not None: + func = getimfunc(teardown_class) + _call_with_optional_argument(func, self.obj) + + self.obj.__pytest_setup_class = xunit_setup_class_fixture + + def _inject_setup_method_fixture(self) -> None: + """Inject a hidden autouse, function scoped fixture into the collected class object + that invokes setup_method/teardown_method if either or both are available. + + Using a fixture to invoke this methods ensures we play nicely and unsurprisingly with + other fixtures (#517). + """ + has_nose = self.config.pluginmanager.has_plugin("nose") + setup_name = "setup_method" + setup_method = _get_first_non_fixture_func(self.obj, (setup_name,)) + if setup_method is None and has_nose: + setup_name = "setup" + setup_method = _get_first_non_fixture_func(self.obj, (setup_name,)) + teardown_name = "teardown_method" + teardown_method = getattr(self.obj, teardown_name, None) + if teardown_method is None and has_nose: + teardown_name = "teardown" + teardown_method = getattr(self.obj, teardown_name, None) + if setup_method is None and teardown_method is None: + return + + @fixtures.fixture( + autouse=True, + scope="function", + # Use a unique name to speed up lookup. + name=f"_xunit_setup_method_fixture_{self.obj.__qualname__}", + ) + def xunit_setup_method_fixture(self, request) -> Generator[None, None, None]: + method = request.function + if setup_method is not None: + func = getattr(self, setup_name) + _call_with_optional_argument(func, method) + yield + if teardown_method is not None: + func = getattr(self, teardown_name) + _call_with_optional_argument(func, method) + + self.obj.__pytest_setup_method = xunit_setup_method_fixture + + +class InstanceDummy: + """Instance used to be a node type between Class and Function. It has been + removed in pytest 7.0. Some plugins exist which reference `pytest.Instance` + only to ignore it; this dummy class keeps them working. This will be removed + in pytest 8.""" + + pass + + +# Note: module __getattr__ only works on Python>=3.7. Unfortunately +# we can't provide this deprecation warning on Python 3.6. +def __getattr__(name: str) -> object: + if name == "Instance": + warnings.warn(INSTANCE_COLLECTOR, 2) + return InstanceDummy + raise AttributeError(f"module {__name__} has no attribute {name}") + + +def hasinit(obj: object) -> bool: + init: object = getattr(obj, "__init__", None) + if init: + return init != object.__init__ + return False + + +def hasnew(obj: object) -> bool: + new: object = getattr(obj, "__new__", None) + if new: + return new != object.__new__ + return False + + +@final +@attr.s(frozen=True, slots=True, auto_attribs=True) +class CallSpec2: + """A planned parameterized invocation of a test function. + + Calculated during collection for a given test function's Metafunc. + Once collection is over, each callspec is turned into a single Item + and stored in item.callspec. + """ + + # arg name -> arg value which will be passed to the parametrized test + # function (direct parameterization). + funcargs: Dict[str, object] = attr.Factory(dict) + # arg name -> arg value which will be passed to a fixture of the same name + # (indirect parametrization). + params: Dict[str, object] = attr.Factory(dict) + # arg name -> arg index. + indices: Dict[str, int] = attr.Factory(dict) + # Used for sorting parametrized resources. + _arg2scope: Dict[str, Scope] = attr.Factory(dict) + # Parts which will be added to the item's name in `[..]` separated by "-". + _idlist: List[str] = attr.Factory(list) + # Marks which will be applied to the item. + marks: List[Mark] = attr.Factory(list) + + def setmulti( + self, + *, + valtypes: Mapping[str, "Literal['params', 'funcargs']"], + argnames: Iterable[str], + valset: Iterable[object], + id: str, + marks: Iterable[Union[Mark, MarkDecorator]], + scope: Scope, + param_index: int, + ) -> "CallSpec2": + funcargs = self.funcargs.copy() + params = self.params.copy() + indices = self.indices.copy() + arg2scope = self._arg2scope.copy() + for arg, val in zip(argnames, valset): + if arg in params or arg in funcargs: + raise ValueError(f"duplicate {arg!r}") + valtype_for_arg = valtypes[arg] + if valtype_for_arg == "params": + params[arg] = val + elif valtype_for_arg == "funcargs": + funcargs[arg] = val + else: + assert_never(valtype_for_arg) + indices[arg] = param_index + arg2scope[arg] = scope + return CallSpec2( + funcargs=funcargs, + params=params, + arg2scope=arg2scope, + indices=indices, + idlist=[*self._idlist, id], + marks=[*self.marks, *normalize_mark_list(marks)], + ) + + def getparam(self, name: str) -> object: + try: + return self.params[name] + except KeyError as e: + raise ValueError(name) from e + + @property + def id(self) -> str: + return "-".join(self._idlist) + + +@final +class Metafunc: + """Objects passed to the :hook:`pytest_generate_tests` hook. + + They help to inspect a test function and to generate tests according to + test configuration or values specified in the class or module where a + test function is defined. + """ + + def __init__( + self, + definition: "FunctionDefinition", + fixtureinfo: fixtures.FuncFixtureInfo, + config: Config, + cls=None, + module=None, + *, + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + + #: Access to the underlying :class:`_pytest.python.FunctionDefinition`. + self.definition = definition + + #: Access to the :class:`pytest.Config` object for the test session. + self.config = config + + #: The module object where the test function is defined in. + self.module = module + + #: Underlying Python test function. + self.function = definition.obj + + #: Set of fixture names required by the test function. + self.fixturenames = fixtureinfo.names_closure + + #: Class object where the test function is defined in or ``None``. + self.cls = cls + + self._arg2fixturedefs = fixtureinfo.name2fixturedefs + + # Result of parametrize(). + self._calls: List[CallSpec2] = [] + + def parametrize( + self, + argnames: Union[str, List[str], Tuple[str, ...]], + argvalues: Iterable[Union[ParameterSet, Sequence[object], object]], + indirect: Union[bool, Sequence[str]] = False, + ids: Optional[ + Union[ + Iterable[Union[None, str, float, int, bool]], + Callable[[Any], Optional[object]], + ] + ] = None, + scope: "Optional[_ScopeName]" = None, + *, + _param_mark: Optional[Mark] = None, + ) -> None: + """Add new invocations to the underlying test function using the list + of argvalues for the given argnames. Parametrization is performed + during the collection phase. If you need to setup expensive resources + see about setting indirect to do it rather than at test setup time. + + Can be called multiple times, in which case each call parametrizes all + previous parametrizations, e.g. + + :: + + unparametrized: t + parametrize ["x", "y"]: t[x], t[y] + parametrize [1, 2]: t[x-1], t[x-2], t[y-1], t[y-2] + + :param argnames: + A comma-separated string denoting one or more argument names, or + a list/tuple of argument strings. + + :param argvalues: + The list of argvalues determines how often a test is invoked with + different argument values. + + If only one argname was specified argvalues is a list of values. + If N argnames were specified, argvalues must be a list of + N-tuples, where each tuple-element specifies a value for its + respective argname. + + :param indirect: + A list of arguments' names (subset of argnames) or a boolean. + If True the list contains all names from the argnames. Each + argvalue corresponding to an argname in this list will + be passed as request.param to its respective argname fixture + function so that it can perform more expensive setups during the + setup phase of a test rather than at collection time. + + :param ids: + Sequence of (or generator for) ids for ``argvalues``, + or a callable to return part of the id for each argvalue. + + With sequences (and generators like ``itertools.count()``) the + returned ids should be of type ``string``, ``int``, ``float``, + ``bool``, or ``None``. + They are mapped to the corresponding index in ``argvalues``. + ``None`` means to use the auto-generated id. + + If it is a callable it will be called for each entry in + ``argvalues``, and the return value is used as part of the + auto-generated id for the whole set (where parts are joined with + dashes ("-")). + This is useful to provide more specific ids for certain items, e.g. + dates. Returning ``None`` will use an auto-generated id. + + If no ids are provided they will be generated automatically from + the argvalues. + + :param scope: + If specified it denotes the scope of the parameters. + The scope is used for grouping tests by parameter instances. + It will also override any fixture-function defined scope, allowing + to set a dynamic scope using test context or configuration. + """ + argnames, parameters = ParameterSet._for_parametrize( + argnames, + argvalues, + self.function, + self.config, + nodeid=self.definition.nodeid, + ) + del argvalues + + if "request" in argnames: + fail( + "'request' is a reserved name and cannot be used in @pytest.mark.parametrize", + pytrace=False, + ) + + if scope is not None: + scope_ = Scope.from_user( + scope, descr=f"parametrize() call in {self.function.__name__}" + ) + else: + scope_ = _find_parametrized_scope(argnames, self._arg2fixturedefs, indirect) + + self._validate_if_using_arg_names(argnames, indirect) + + arg_values_types = self._resolve_arg_value_types(argnames, indirect) + + # Use any already (possibly) generated ids with parametrize Marks. + if _param_mark and _param_mark._param_ids_from: + generated_ids = _param_mark._param_ids_from._param_ids_generated + if generated_ids is not None: + ids = generated_ids + + ids = self._resolve_arg_ids( + argnames, ids, parameters, nodeid=self.definition.nodeid + ) + + # Store used (possibly generated) ids with parametrize Marks. + if _param_mark and _param_mark._param_ids_from and generated_ids is None: + object.__setattr__(_param_mark._param_ids_from, "_param_ids_generated", ids) + + # Create the new calls: if we are parametrize() multiple times (by applying the decorator + # more than once) then we accumulate those calls generating the cartesian product + # of all calls. + newcalls = [] + for callspec in self._calls or [CallSpec2()]: + for param_index, (param_id, param_set) in enumerate(zip(ids, parameters)): + newcallspec = callspec.setmulti( + valtypes=arg_values_types, + argnames=argnames, + valset=param_set.values, + id=param_id, + marks=param_set.marks, + scope=scope_, + param_index=param_index, + ) + newcalls.append(newcallspec) + self._calls = newcalls + + def _resolve_arg_ids( + self, + argnames: Sequence[str], + ids: Optional[ + Union[ + Iterable[Union[None, str, float, int, bool]], + Callable[[Any], Optional[object]], + ] + ], + parameters: Sequence[ParameterSet], + nodeid: str, + ) -> List[str]: + """Resolve the actual ids for the given argnames, based on the ``ids`` parameter given + to ``parametrize``. + + :param List[str] argnames: List of argument names passed to ``parametrize()``. + :param ids: The ids parameter of the parametrized call (see docs). + :param List[ParameterSet] parameters: The list of parameter values, same size as ``argnames``. + :param str str: The nodeid of the item that generated this parametrized call. + :rtype: List[str] + :returns: The list of ids for each argname given. + """ + if ids is None: + idfn = None + ids_ = None + elif callable(ids): + idfn = ids + ids_ = None + else: + idfn = None + ids_ = self._validate_ids(ids, parameters, self.function.__name__) + return idmaker(argnames, parameters, idfn, ids_, self.config, nodeid=nodeid) + + def _validate_ids( + self, + ids: Iterable[Union[None, str, float, int, bool]], + parameters: Sequence[ParameterSet], + func_name: str, + ) -> List[Union[None, str]]: + try: + num_ids = len(ids) # type: ignore[arg-type] + except TypeError: + try: + iter(ids) + except TypeError as e: + raise TypeError("ids must be a callable or an iterable") from e + num_ids = len(parameters) + + # num_ids == 0 is a special case: https://github.com/pytest-dev/pytest/issues/1849 + if num_ids != len(parameters) and num_ids != 0: + msg = "In {}: {} parameter sets specified, with different number of ids: {}" + fail(msg.format(func_name, len(parameters), num_ids), pytrace=False) + + new_ids = [] + for idx, id_value in enumerate(itertools.islice(ids, num_ids)): + if id_value is None or isinstance(id_value, str): + new_ids.append(id_value) + elif isinstance(id_value, (float, int, bool)): + new_ids.append(str(id_value)) + else: + msg = ( # type: ignore[unreachable] + "In {}: ids must be list of string/float/int/bool, " + "found: {} (type: {!r}) at index {}" + ) + fail( + msg.format(func_name, saferepr(id_value), type(id_value), idx), + pytrace=False, + ) + return new_ids + + def _resolve_arg_value_types( + self, + argnames: Sequence[str], + indirect: Union[bool, Sequence[str]], + ) -> Dict[str, "Literal['params', 'funcargs']"]: + """Resolve if each parametrized argument must be considered a + parameter to a fixture or a "funcarg" to the function, based on the + ``indirect`` parameter of the parametrized() call. + + :param List[str] argnames: List of argument names passed to ``parametrize()``. + :param indirect: Same as the ``indirect`` parameter of ``parametrize()``. + :rtype: Dict[str, str] + A dict mapping each arg name to either: + * "params" if the argname should be the parameter of a fixture of the same name. + * "funcargs" if the argname should be a parameter to the parametrized test function. + """ + if isinstance(indirect, bool): + valtypes: Dict[str, Literal["params", "funcargs"]] = dict.fromkeys( + argnames, "params" if indirect else "funcargs" + ) + elif isinstance(indirect, Sequence): + valtypes = dict.fromkeys(argnames, "funcargs") + for arg in indirect: + if arg not in argnames: + fail( + "In {}: indirect fixture '{}' doesn't exist".format( + self.function.__name__, arg + ), + pytrace=False, + ) + valtypes[arg] = "params" + else: + fail( + "In {func}: expected Sequence or boolean for indirect, got {type}".format( + type=type(indirect).__name__, func=self.function.__name__ + ), + pytrace=False, + ) + return valtypes + + def _validate_if_using_arg_names( + self, + argnames: Sequence[str], + indirect: Union[bool, Sequence[str]], + ) -> None: + """Check if all argnames are being used, by default values, or directly/indirectly. + + :param List[str] argnames: List of argument names passed to ``parametrize()``. + :param indirect: Same as the ``indirect`` parameter of ``parametrize()``. + :raises ValueError: If validation fails. + """ + default_arg_names = set(get_default_arg_names(self.function)) + func_name = self.function.__name__ + for arg in argnames: + if arg not in self.fixturenames: + if arg in default_arg_names: + fail( + "In {}: function already takes an argument '{}' with a default value".format( + func_name, arg + ), + pytrace=False, + ) + else: + if isinstance(indirect, Sequence): + name = "fixture" if arg in indirect else "argument" + else: + name = "fixture" if indirect else "argument" + fail( + f"In {func_name}: function uses no {name} '{arg}'", + pytrace=False, + ) + + +def _find_parametrized_scope( + argnames: Sequence[str], + arg2fixturedefs: Mapping[str, Sequence[fixtures.FixtureDef[object]]], + indirect: Union[bool, Sequence[str]], +) -> Scope: + """Find the most appropriate scope for a parametrized call based on its arguments. + + When there's at least one direct argument, always use "function" scope. + + When a test function is parametrized and all its arguments are indirect + (e.g. fixtures), return the most narrow scope based on the fixtures used. + + Related to issue #1832, based on code posted by @Kingdread. + """ + if isinstance(indirect, Sequence): + all_arguments_are_fixtures = len(indirect) == len(argnames) + else: + all_arguments_are_fixtures = bool(indirect) + + if all_arguments_are_fixtures: + fixturedefs = arg2fixturedefs or {} + used_scopes = [ + fixturedef[0]._scope + for name, fixturedef in fixturedefs.items() + if name in argnames + ] + # Takes the most narrow scope from used fixtures. + return min(used_scopes, default=Scope.Function) + + return Scope.Function + + +def _ascii_escaped_by_config(val: Union[str, bytes], config: Optional[Config]) -> str: + if config is None: + escape_option = False + else: + escape_option = config.getini( + "disable_test_id_escaping_and_forfeit_all_rights_to_community_support" + ) + # TODO: If escaping is turned off and the user passes bytes, + # will return a bytes. For now we ignore this but the + # code *probably* doesn't handle this case. + return val if escape_option else ascii_escaped(val) # type: ignore + + +def _idval( + val: object, + argname: str, + idx: int, + idfn: Optional[Callable[[Any], Optional[object]]], + nodeid: Optional[str], + config: Optional[Config], +) -> str: + if idfn: + try: + generated_id = idfn(val) + if generated_id is not None: + val = generated_id + except Exception as e: + prefix = f"{nodeid}: " if nodeid is not None else "" + msg = "error raised while trying to determine id of parameter '{}' at position {}" + msg = prefix + msg.format(argname, idx) + raise ValueError(msg) from e + elif config: + hook_id: Optional[str] = config.hook.pytest_make_parametrize_id( + config=config, val=val, argname=argname + ) + if hook_id: + return hook_id + + if isinstance(val, STRING_TYPES): + return _ascii_escaped_by_config(val, config) + elif val is None or isinstance(val, (float, int, bool, complex)): + return str(val) + elif isinstance(val, Pattern): + return ascii_escaped(val.pattern) + elif val is NOTSET: + # Fallback to default. Note that NOTSET is an enum.Enum. + pass + elif isinstance(val, enum.Enum): + return str(val) + elif isinstance(getattr(val, "__name__", None), str): + # Name of a class, function, module, etc. + name: str = getattr(val, "__name__") + return name + return str(argname) + str(idx) + + +def _idvalset( + idx: int, + parameterset: ParameterSet, + argnames: Iterable[str], + idfn: Optional[Callable[[Any], Optional[object]]], + ids: Optional[List[Union[None, str]]], + nodeid: Optional[str], + config: Optional[Config], +) -> str: + if parameterset.id is not None: + return parameterset.id + id = None if ids is None or idx >= len(ids) else ids[idx] + if id is None: + this_id = [ + _idval(val, argname, idx, idfn, nodeid=nodeid, config=config) + for val, argname in zip(parameterset.values, argnames) + ] + return "-".join(this_id) + else: + return _ascii_escaped_by_config(id, config) + + +def idmaker( + argnames: Iterable[str], + parametersets: Iterable[ParameterSet], + idfn: Optional[Callable[[Any], Optional[object]]] = None, + ids: Optional[List[Union[None, str]]] = None, + config: Optional[Config] = None, + nodeid: Optional[str] = None, +) -> List[str]: + resolved_ids = [ + _idvalset( + valindex, parameterset, argnames, idfn, ids, config=config, nodeid=nodeid + ) + for valindex, parameterset in enumerate(parametersets) + ] + + # All IDs must be unique! + unique_ids = set(resolved_ids) + if len(unique_ids) != len(resolved_ids): + + # Record the number of occurrences of each test ID. + test_id_counts = Counter(resolved_ids) + + # Map the test ID to its next suffix. + test_id_suffixes: Dict[str, int] = defaultdict(int) + + # Suffix non-unique IDs to make them unique. + for index, test_id in enumerate(resolved_ids): + if test_id_counts[test_id] > 1: + resolved_ids[index] = f"{test_id}{test_id_suffixes[test_id]}" + test_id_suffixes[test_id] += 1 + + return resolved_ids + + +def _pretty_fixture_path(func) -> str: + cwd = Path.cwd() + loc = Path(getlocation(func, str(cwd))) + prefix = Path("...", "_pytest") + try: + return str(prefix / loc.relative_to(_PYTEST_DIR)) + except ValueError: + return bestrelpath(cwd, loc) + + +def show_fixtures_per_test(config): + from _pytest.main import wrap_session + + return wrap_session(config, _show_fixtures_per_test) + + +def _show_fixtures_per_test(config: Config, session: Session) -> None: + import _pytest.config + + session.perform_collect() + curdir = Path.cwd() + tw = _pytest.config.create_terminal_writer(config) + verbose = config.getvalue("verbose") + + def get_best_relpath(func) -> str: + loc = getlocation(func, str(curdir)) + return bestrelpath(curdir, Path(loc)) + + def write_fixture(fixture_def: fixtures.FixtureDef[object]) -> None: + argname = fixture_def.argname + if verbose <= 0 and argname.startswith("_"): + return + prettypath = _pretty_fixture_path(fixture_def.func) + tw.write(f"{argname}", green=True) + tw.write(f" -- {prettypath}", yellow=True) + tw.write("\n") + fixture_doc = inspect.getdoc(fixture_def.func) + if fixture_doc: + write_docstring( + tw, fixture_doc.split("\n\n")[0] if verbose <= 0 else fixture_doc + ) + else: + tw.line(" no docstring available", red=True) + + def write_item(item: nodes.Item) -> None: + # Not all items have _fixtureinfo attribute. + info: Optional[FuncFixtureInfo] = getattr(item, "_fixtureinfo", None) + if info is None or not info.name2fixturedefs: + # This test item does not use any fixtures. + return + tw.line() + tw.sep("-", f"fixtures used by {item.name}") + # TODO: Fix this type ignore. + tw.sep("-", f"({get_best_relpath(item.function)})") # type: ignore[attr-defined] + # dict key not used in loop but needed for sorting. + for _, fixturedefs in sorted(info.name2fixturedefs.items()): + assert fixturedefs is not None + if not fixturedefs: + continue + # Last item is expected to be the one used by the test item. + write_fixture(fixturedefs[-1]) + + for session_item in session.items: + write_item(session_item) + + +def showfixtures(config: Config) -> Union[int, ExitCode]: + from _pytest.main import wrap_session + + return wrap_session(config, _showfixtures_main) + + +def _showfixtures_main(config: Config, session: Session) -> None: + import _pytest.config + + session.perform_collect() + curdir = Path.cwd() + tw = _pytest.config.create_terminal_writer(config) + verbose = config.getvalue("verbose") + + fm = session._fixturemanager + + available = [] + seen: Set[Tuple[str, str]] = set() + + for argname, fixturedefs in fm._arg2fixturedefs.items(): + assert fixturedefs is not None + if not fixturedefs: + continue + for fixturedef in fixturedefs: + loc = getlocation(fixturedef.func, str(curdir)) + if (fixturedef.argname, loc) in seen: + continue + seen.add((fixturedef.argname, loc)) + available.append( + ( + len(fixturedef.baseid), + fixturedef.func.__module__, + _pretty_fixture_path(fixturedef.func), + fixturedef.argname, + fixturedef, + ) + ) + + available.sort() + currentmodule = None + for baseid, module, prettypath, argname, fixturedef in available: + if currentmodule != module: + if not module.startswith("_pytest."): + tw.line() + tw.sep("-", f"fixtures defined from {module}") + currentmodule = module + if verbose <= 0 and argname.startswith("_"): + continue + tw.write(f"{argname}", green=True) + if fixturedef.scope != "function": + tw.write(" [%s scope]" % fixturedef.scope, cyan=True) + tw.write(f" -- {prettypath}", yellow=True) + tw.write("\n") + doc = inspect.getdoc(fixturedef.func) + if doc: + write_docstring(tw, doc.split("\n\n")[0] if verbose <= 0 else doc) + else: + tw.line(" no docstring available", red=True) + tw.line() + + +def write_docstring(tw: TerminalWriter, doc: str, indent: str = " ") -> None: + for line in doc.split("\n"): + tw.line(indent + line) + + +class Function(PyobjMixin, nodes.Item): + """An Item responsible for setting up and executing a Python test function. + + :param name: + The full function name, including any decorations like those + added by parametrization (``my_func[my_param]``). + :param parent: + The parent Node. + :param config: + The pytest Config object. + :param callspec: + If given, this is function has been parametrized and the callspec contains + meta information about the parametrization. + :param callobj: + If given, the object which will be called when the Function is invoked, + otherwise the callobj will be obtained from ``parent`` using ``originalname``. + :param keywords: + Keywords bound to the function object for "-k" matching. + :param session: + The pytest Session object. + :param fixtureinfo: + Fixture information already resolved at this fixture node.. + :param originalname: + The attribute name to use for accessing the underlying function object. + Defaults to ``name``. Set this if name is different from the original name, + for example when it contains decorations like those added by parametrization + (``my_func[my_param]``). + """ + + # Disable since functions handle it themselves. + _ALLOW_MARKERS = False + + def __init__( + self, + name: str, + parent, + config: Optional[Config] = None, + callspec: Optional[CallSpec2] = None, + callobj=NOTSET, + keywords=None, + session: Optional[Session] = None, + fixtureinfo: Optional[FuncFixtureInfo] = None, + originalname: Optional[str] = None, + ) -> None: + super().__init__(name, parent, config=config, session=session) + + if callobj is not NOTSET: + self.obj = callobj + + #: Original function name, without any decorations (for example + #: parametrization adds a ``"[...]"`` suffix to function names), used to access + #: the underlying function object from ``parent`` (in case ``callobj`` is not given + #: explicitly). + #: + #: .. versionadded:: 3.0 + self.originalname = originalname or name + + # Note: when FunctionDefinition is introduced, we should change ``originalname`` + # to a readonly property that returns FunctionDefinition.name. + + self.keywords.update(self.obj.__dict__) + self.own_markers.extend(get_unpacked_marks(self.obj)) + if callspec: + self.callspec = callspec + # this is total hostile and a mess + # keywords are broken by design by now + # this will be redeemed later + for mark in callspec.marks: + # feel free to cry, this was broken for years before + # and keywords can't fix it per design + self.keywords[mark.name] = mark + self.own_markers.extend(normalize_mark_list(callspec.marks)) + if keywords: + self.keywords.update(keywords) + + # todo: this is a hell of a hack + # https://github.com/pytest-dev/pytest/issues/4569 + + self.keywords.update( + { + mark.name: True + for mark in self.iter_markers() + if mark.name not in self.keywords + } + ) + + if fixtureinfo is None: + fixtureinfo = self.session._fixturemanager.getfixtureinfo( + self, self.obj, self.cls, funcargs=True + ) + self._fixtureinfo: FuncFixtureInfo = fixtureinfo + self.fixturenames = fixtureinfo.names_closure + self._initrequest() + + @classmethod + def from_parent(cls, parent, **kw): # todo: determine sound type limitations + """The public constructor.""" + return super().from_parent(parent=parent, **kw) + + def _initrequest(self) -> None: + self.funcargs: Dict[str, object] = {} + self._request = fixtures.FixtureRequest(self, _ispytest=True) + + @property + def function(self): + """Underlying python 'function' object.""" + return getimfunc(self.obj) + + def _getobj(self): + assert self.parent is not None + if isinstance(self.parent, Class): + # Each Function gets a fresh class instance. + parent_obj = self.parent.newinstance() + else: + parent_obj = self.parent.obj # type: ignore[attr-defined] + return getattr(parent_obj, self.originalname) + + @property + def _pyfuncitem(self): + """(compatonly) for code expecting pytest-2.2 style request objects.""" + return self + + def runtest(self) -> None: + """Execute the underlying test function.""" + self.ihook.pytest_pyfunc_call(pyfuncitem=self) + + def setup(self) -> None: + self._request._fillfixtures() + + def _prunetraceback(self, excinfo: ExceptionInfo[BaseException]) -> None: + if hasattr(self, "_obj") and not self.config.getoption("fulltrace", False): + code = _pytest._code.Code.from_function(get_real_func(self.obj)) + path, firstlineno = code.path, code.firstlineno + traceback = excinfo.traceback + ntraceback = traceback.cut(path=path, firstlineno=firstlineno) + if ntraceback == traceback: + ntraceback = ntraceback.cut(path=path) + if ntraceback == traceback: + ntraceback = ntraceback.filter(filter_traceback) + if not ntraceback: + ntraceback = traceback + + excinfo.traceback = ntraceback.filter() + # issue364: mark all but first and last frames to + # only show a single-line message for each frame. + if self.config.getoption("tbstyle", "auto") == "auto": + if len(excinfo.traceback) > 2: + for entry in excinfo.traceback[1:-1]: + entry.set_repr_style("short") + + # TODO: Type ignored -- breaks Liskov Substitution. + def repr_failure( # type: ignore[override] + self, + excinfo: ExceptionInfo[BaseException], + ) -> Union[str, TerminalRepr]: + style = self.config.getoption("tbstyle", "auto") + if style == "auto": + style = "long" + return self._repr_failure_py(excinfo, style=style) + + +class FunctionDefinition(Function): + """ + This class is a step gap solution until we evolve to have actual function definition nodes + and manage to get rid of ``metafunc``. + """ + + def runtest(self) -> None: + raise RuntimeError("function definitions are not supposed to be run as tests") + + setup = runtest diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/python_api.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/python_api.py new file mode 100644 index 0000000000..cb72fde1e1 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/python_api.py @@ -0,0 +1,961 @@ +import math +import pprint +from collections.abc import Sized +from decimal import Decimal +from numbers import Complex +from types import TracebackType +from typing import Any +from typing import Callable +from typing import cast +from typing import Generic +from typing import Iterable +from typing import List +from typing import Mapping +from typing import Optional +from typing import overload +from typing import Pattern +from typing import Sequence +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import TypeVar +from typing import Union + +if TYPE_CHECKING: + from numpy import ndarray + + +import _pytest._code +from _pytest.compat import final +from _pytest.compat import STRING_TYPES +from _pytest.outcomes import fail + + +def _non_numeric_type_error(value, at: Optional[str]) -> TypeError: + at_str = f" at {at}" if at else "" + return TypeError( + "cannot make approximate comparisons to non-numeric values: {!r} {}".format( + value, at_str + ) + ) + + +def _compare_approx( + full_object: object, + message_data: Sequence[Tuple[str, str, str]], + number_of_elements: int, + different_ids: Sequence[object], + max_abs_diff: float, + max_rel_diff: float, +) -> List[str]: + message_list = list(message_data) + message_list.insert(0, ("Index", "Obtained", "Expected")) + max_sizes = [0, 0, 0] + for index, obtained, expected in message_list: + max_sizes[0] = max(max_sizes[0], len(index)) + max_sizes[1] = max(max_sizes[1], len(obtained)) + max_sizes[2] = max(max_sizes[2], len(expected)) + explanation = [ + f"comparison failed. Mismatched elements: {len(different_ids)} / {number_of_elements}:", + f"Max absolute difference: {max_abs_diff}", + f"Max relative difference: {max_rel_diff}", + ] + [ + f"{indexes:<{max_sizes[0]}} | {obtained:<{max_sizes[1]}} | {expected:<{max_sizes[2]}}" + for indexes, obtained, expected in message_list + ] + return explanation + + +# builtin pytest.approx helper + + +class ApproxBase: + """Provide shared utilities for making approximate comparisons between + numbers or sequences of numbers.""" + + # Tell numpy to use our `__eq__` operator instead of its. + __array_ufunc__ = None + __array_priority__ = 100 + + def __init__(self, expected, rel=None, abs=None, nan_ok: bool = False) -> None: + __tracebackhide__ = True + self.expected = expected + self.abs = abs + self.rel = rel + self.nan_ok = nan_ok + self._check_type() + + def __repr__(self) -> str: + raise NotImplementedError + + def _repr_compare(self, other_side: Any) -> List[str]: + return [ + "comparison failed", + f"Obtained: {other_side}", + f"Expected: {self}", + ] + + def __eq__(self, actual) -> bool: + return all( + a == self._approx_scalar(x) for a, x in self._yield_comparisons(actual) + ) + + def __bool__(self): + __tracebackhide__ = True + raise AssertionError( + "approx() is not supported in a boolean context.\nDid you mean: `assert a == approx(b)`?" + ) + + # Ignore type because of https://github.com/python/mypy/issues/4266. + __hash__ = None # type: ignore + + def __ne__(self, actual) -> bool: + return not (actual == self) + + def _approx_scalar(self, x) -> "ApproxScalar": + if isinstance(x, Decimal): + return ApproxDecimal(x, rel=self.rel, abs=self.abs, nan_ok=self.nan_ok) + return ApproxScalar(x, rel=self.rel, abs=self.abs, nan_ok=self.nan_ok) + + def _yield_comparisons(self, actual): + """Yield all the pairs of numbers to be compared. + + This is used to implement the `__eq__` method. + """ + raise NotImplementedError + + def _check_type(self) -> None: + """Raise a TypeError if the expected value is not a valid type.""" + # This is only a concern if the expected value is a sequence. In every + # other case, the approx() function ensures that the expected value has + # a numeric type. For this reason, the default is to do nothing. The + # classes that deal with sequences should reimplement this method to + # raise if there are any non-numeric elements in the sequence. + pass + + +def _recursive_list_map(f, x): + if isinstance(x, list): + return [_recursive_list_map(f, xi) for xi in x] + else: + return f(x) + + +class ApproxNumpy(ApproxBase): + """Perform approximate comparisons where the expected value is numpy array.""" + + def __repr__(self) -> str: + list_scalars = _recursive_list_map(self._approx_scalar, self.expected.tolist()) + return f"approx({list_scalars!r})" + + def _repr_compare(self, other_side: "ndarray") -> List[str]: + import itertools + import math + + def get_value_from_nested_list( + nested_list: List[Any], nd_index: Tuple[Any, ...] + ) -> Any: + """ + Helper function to get the value out of a nested list, given an n-dimensional index. + This mimics numpy's indexing, but for raw nested python lists. + """ + value: Any = nested_list + for i in nd_index: + value = value[i] + return value + + np_array_shape = self.expected.shape + approx_side_as_list = _recursive_list_map( + self._approx_scalar, self.expected.tolist() + ) + + if np_array_shape != other_side.shape: + return [ + "Impossible to compare arrays with different shapes.", + f"Shapes: {np_array_shape} and {other_side.shape}", + ] + + number_of_elements = self.expected.size + max_abs_diff = -math.inf + max_rel_diff = -math.inf + different_ids = [] + for index in itertools.product(*(range(i) for i in np_array_shape)): + approx_value = get_value_from_nested_list(approx_side_as_list, index) + other_value = get_value_from_nested_list(other_side, index) + if approx_value != other_value: + abs_diff = abs(approx_value.expected - other_value) + max_abs_diff = max(max_abs_diff, abs_diff) + if other_value == 0.0: + max_rel_diff = math.inf + else: + max_rel_diff = max(max_rel_diff, abs_diff / abs(other_value)) + different_ids.append(index) + + message_data = [ + ( + str(index), + str(get_value_from_nested_list(other_side, index)), + str(get_value_from_nested_list(approx_side_as_list, index)), + ) + for index in different_ids + ] + return _compare_approx( + self.expected, + message_data, + number_of_elements, + different_ids, + max_abs_diff, + max_rel_diff, + ) + + def __eq__(self, actual) -> bool: + import numpy as np + + # self.expected is supposed to always be an array here. + + if not np.isscalar(actual): + try: + actual = np.asarray(actual) + except Exception as e: + raise TypeError(f"cannot compare '{actual}' to numpy.ndarray") from e + + if not np.isscalar(actual) and actual.shape != self.expected.shape: + return False + + return super().__eq__(actual) + + def _yield_comparisons(self, actual): + import numpy as np + + # `actual` can either be a numpy array or a scalar, it is treated in + # `__eq__` before being passed to `ApproxBase.__eq__`, which is the + # only method that calls this one. + + if np.isscalar(actual): + for i in np.ndindex(self.expected.shape): + yield actual, self.expected[i].item() + else: + for i in np.ndindex(self.expected.shape): + yield actual[i].item(), self.expected[i].item() + + +class ApproxMapping(ApproxBase): + """Perform approximate comparisons where the expected value is a mapping + with numeric values (the keys can be anything).""" + + def __repr__(self) -> str: + return "approx({!r})".format( + {k: self._approx_scalar(v) for k, v in self.expected.items()} + ) + + def _repr_compare(self, other_side: Mapping[object, float]) -> List[str]: + import math + + approx_side_as_map = { + k: self._approx_scalar(v) for k, v in self.expected.items() + } + + number_of_elements = len(approx_side_as_map) + max_abs_diff = -math.inf + max_rel_diff = -math.inf + different_ids = [] + for (approx_key, approx_value), other_value in zip( + approx_side_as_map.items(), other_side.values() + ): + if approx_value != other_value: + max_abs_diff = max( + max_abs_diff, abs(approx_value.expected - other_value) + ) + max_rel_diff = max( + max_rel_diff, + abs((approx_value.expected - other_value) / approx_value.expected), + ) + different_ids.append(approx_key) + + message_data = [ + (str(key), str(other_side[key]), str(approx_side_as_map[key])) + for key in different_ids + ] + + return _compare_approx( + self.expected, + message_data, + number_of_elements, + different_ids, + max_abs_diff, + max_rel_diff, + ) + + def __eq__(self, actual) -> bool: + try: + if set(actual.keys()) != set(self.expected.keys()): + return False + except AttributeError: + return False + + return super().__eq__(actual) + + def _yield_comparisons(self, actual): + for k in self.expected.keys(): + yield actual[k], self.expected[k] + + def _check_type(self) -> None: + __tracebackhide__ = True + for key, value in self.expected.items(): + if isinstance(value, type(self.expected)): + msg = "pytest.approx() does not support nested dictionaries: key={!r} value={!r}\n full mapping={}" + raise TypeError(msg.format(key, value, pprint.pformat(self.expected))) + + +class ApproxSequencelike(ApproxBase): + """Perform approximate comparisons where the expected value is a sequence of numbers.""" + + def __repr__(self) -> str: + seq_type = type(self.expected) + if seq_type not in (tuple, list, set): + seq_type = list + return "approx({!r})".format( + seq_type(self._approx_scalar(x) for x in self.expected) + ) + + def _repr_compare(self, other_side: Sequence[float]) -> List[str]: + import math + import numpy as np + + if len(self.expected) != len(other_side): + return [ + "Impossible to compare lists with different sizes.", + f"Lengths: {len(self.expected)} and {len(other_side)}", + ] + + approx_side_as_map = _recursive_list_map(self._approx_scalar, self.expected) + + number_of_elements = len(approx_side_as_map) + max_abs_diff = -math.inf + max_rel_diff = -math.inf + different_ids = [] + for i, (approx_value, other_value) in enumerate( + zip(approx_side_as_map, other_side) + ): + if approx_value != other_value: + abs_diff = abs(approx_value.expected - other_value) + max_abs_diff = max(max_abs_diff, abs_diff) + if other_value == 0.0: + max_rel_diff = np.inf + else: + max_rel_diff = max(max_rel_diff, abs_diff / abs(other_value)) + different_ids.append(i) + + message_data = [ + (str(i), str(other_side[i]), str(approx_side_as_map[i])) + for i in different_ids + ] + + return _compare_approx( + self.expected, + message_data, + number_of_elements, + different_ids, + max_abs_diff, + max_rel_diff, + ) + + def __eq__(self, actual) -> bool: + try: + if len(actual) != len(self.expected): + return False + except TypeError: + return False + return super().__eq__(actual) + + def _yield_comparisons(self, actual): + return zip(actual, self.expected) + + def _check_type(self) -> None: + __tracebackhide__ = True + for index, x in enumerate(self.expected): + if isinstance(x, type(self.expected)): + msg = "pytest.approx() does not support nested data structures: {!r} at index {}\n full sequence: {}" + raise TypeError(msg.format(x, index, pprint.pformat(self.expected))) + + +class ApproxScalar(ApproxBase): + """Perform approximate comparisons where the expected value is a single number.""" + + # Using Real should be better than this Union, but not possible yet: + # https://github.com/python/typeshed/pull/3108 + DEFAULT_ABSOLUTE_TOLERANCE: Union[float, Decimal] = 1e-12 + DEFAULT_RELATIVE_TOLERANCE: Union[float, Decimal] = 1e-6 + + def __repr__(self) -> str: + """Return a string communicating both the expected value and the + tolerance for the comparison being made. + + For example, ``1.0 ± 1e-6``, ``(3+4j) ± 5e-6 ∠ ±180°``. + """ + # Don't show a tolerance for values that aren't compared using + # tolerances, i.e. non-numerics and infinities. Need to call abs to + # handle complex numbers, e.g. (inf + 1j). + if (not isinstance(self.expected, (Complex, Decimal))) or math.isinf( + abs(self.expected) # type: ignore[arg-type] + ): + return str(self.expected) + + # If a sensible tolerance can't be calculated, self.tolerance will + # raise a ValueError. In this case, display '???'. + try: + vetted_tolerance = f"{self.tolerance:.1e}" + if ( + isinstance(self.expected, Complex) + and self.expected.imag + and not math.isinf(self.tolerance) + ): + vetted_tolerance += " ∠ ±180°" + except ValueError: + vetted_tolerance = "???" + + return f"{self.expected} ± {vetted_tolerance}" + + def __eq__(self, actual) -> bool: + """Return whether the given value is equal to the expected value + within the pre-specified tolerance.""" + asarray = _as_numpy_array(actual) + if asarray is not None: + # Call ``__eq__()`` manually to prevent infinite-recursion with + # numpy<1.13. See #3748. + return all(self.__eq__(a) for a in asarray.flat) + + # Short-circuit exact equality. + if actual == self.expected: + return True + + # If either type is non-numeric, fall back to strict equality. + # NB: we need Complex, rather than just Number, to ensure that __abs__, + # __sub__, and __float__ are defined. + if not ( + isinstance(self.expected, (Complex, Decimal)) + and isinstance(actual, (Complex, Decimal)) + ): + return False + + # Allow the user to control whether NaNs are considered equal to each + # other or not. The abs() calls are for compatibility with complex + # numbers. + if math.isnan(abs(self.expected)): # type: ignore[arg-type] + return self.nan_ok and math.isnan(abs(actual)) # type: ignore[arg-type] + + # Infinity shouldn't be approximately equal to anything but itself, but + # if there's a relative tolerance, it will be infinite and infinity + # will seem approximately equal to everything. The equal-to-itself + # case would have been short circuited above, so here we can just + # return false if the expected value is infinite. The abs() call is + # for compatibility with complex numbers. + if math.isinf(abs(self.expected)): # type: ignore[arg-type] + return False + + # Return true if the two numbers are within the tolerance. + result: bool = abs(self.expected - actual) <= self.tolerance + return result + + # Ignore type because of https://github.com/python/mypy/issues/4266. + __hash__ = None # type: ignore + + @property + def tolerance(self): + """Return the tolerance for the comparison. + + This could be either an absolute tolerance or a relative tolerance, + depending on what the user specified or which would be larger. + """ + + def set_default(x, default): + return x if x is not None else default + + # Figure out what the absolute tolerance should be. ``self.abs`` is + # either None or a value specified by the user. + absolute_tolerance = set_default(self.abs, self.DEFAULT_ABSOLUTE_TOLERANCE) + + if absolute_tolerance < 0: + raise ValueError( + f"absolute tolerance can't be negative: {absolute_tolerance}" + ) + if math.isnan(absolute_tolerance): + raise ValueError("absolute tolerance can't be NaN.") + + # If the user specified an absolute tolerance but not a relative one, + # just return the absolute tolerance. + if self.rel is None: + if self.abs is not None: + return absolute_tolerance + + # Figure out what the relative tolerance should be. ``self.rel`` is + # either None or a value specified by the user. This is done after + # we've made sure the user didn't ask for an absolute tolerance only, + # because we don't want to raise errors about the relative tolerance if + # we aren't even going to use it. + relative_tolerance = set_default( + self.rel, self.DEFAULT_RELATIVE_TOLERANCE + ) * abs(self.expected) + + if relative_tolerance < 0: + raise ValueError( + f"relative tolerance can't be negative: {relative_tolerance}" + ) + if math.isnan(relative_tolerance): + raise ValueError("relative tolerance can't be NaN.") + + # Return the larger of the relative and absolute tolerances. + return max(relative_tolerance, absolute_tolerance) + + +class ApproxDecimal(ApproxScalar): + """Perform approximate comparisons where the expected value is a Decimal.""" + + DEFAULT_ABSOLUTE_TOLERANCE = Decimal("1e-12") + DEFAULT_RELATIVE_TOLERANCE = Decimal("1e-6") + + +def approx(expected, rel=None, abs=None, nan_ok: bool = False) -> ApproxBase: + """Assert that two numbers (or two sets of numbers) are equal to each other + within some tolerance. + + Due to the :std:doc:`tutorial/floatingpoint`, numbers that we + would intuitively expect to be equal are not always so:: + + >>> 0.1 + 0.2 == 0.3 + False + + This problem is commonly encountered when writing tests, e.g. when making + sure that floating-point values are what you expect them to be. One way to + deal with this problem is to assert that two floating-point numbers are + equal to within some appropriate tolerance:: + + >>> abs((0.1 + 0.2) - 0.3) < 1e-6 + True + + However, comparisons like this are tedious to write and difficult to + understand. Furthermore, absolute comparisons like the one above are + usually discouraged because there's no tolerance that works well for all + situations. ``1e-6`` is good for numbers around ``1``, but too small for + very big numbers and too big for very small ones. It's better to express + the tolerance as a fraction of the expected value, but relative comparisons + like that are even more difficult to write correctly and concisely. + + The ``approx`` class performs floating-point comparisons using a syntax + that's as intuitive as possible:: + + >>> from pytest import approx + >>> 0.1 + 0.2 == approx(0.3) + True + + The same syntax also works for sequences of numbers:: + + >>> (0.1 + 0.2, 0.2 + 0.4) == approx((0.3, 0.6)) + True + + Dictionary *values*:: + + >>> {'a': 0.1 + 0.2, 'b': 0.2 + 0.4} == approx({'a': 0.3, 'b': 0.6}) + True + + ``numpy`` arrays:: + + >>> import numpy as np # doctest: +SKIP + >>> np.array([0.1, 0.2]) + np.array([0.2, 0.4]) == approx(np.array([0.3, 0.6])) # doctest: +SKIP + True + + And for a ``numpy`` array against a scalar:: + + >>> import numpy as np # doctest: +SKIP + >>> np.array([0.1, 0.2]) + np.array([0.2, 0.1]) == approx(0.3) # doctest: +SKIP + True + + By default, ``approx`` considers numbers within a relative tolerance of + ``1e-6`` (i.e. one part in a million) of its expected value to be equal. + This treatment would lead to surprising results if the expected value was + ``0.0``, because nothing but ``0.0`` itself is relatively close to ``0.0``. + To handle this case less surprisingly, ``approx`` also considers numbers + within an absolute tolerance of ``1e-12`` of its expected value to be + equal. Infinity and NaN are special cases. Infinity is only considered + equal to itself, regardless of the relative tolerance. NaN is not + considered equal to anything by default, but you can make it be equal to + itself by setting the ``nan_ok`` argument to True. (This is meant to + facilitate comparing arrays that use NaN to mean "no data".) + + Both the relative and absolute tolerances can be changed by passing + arguments to the ``approx`` constructor:: + + >>> 1.0001 == approx(1) + False + >>> 1.0001 == approx(1, rel=1e-3) + True + >>> 1.0001 == approx(1, abs=1e-3) + True + + If you specify ``abs`` but not ``rel``, the comparison will not consider + the relative tolerance at all. In other words, two numbers that are within + the default relative tolerance of ``1e-6`` will still be considered unequal + if they exceed the specified absolute tolerance. If you specify both + ``abs`` and ``rel``, the numbers will be considered equal if either + tolerance is met:: + + >>> 1 + 1e-8 == approx(1) + True + >>> 1 + 1e-8 == approx(1, abs=1e-12) + False + >>> 1 + 1e-8 == approx(1, rel=1e-6, abs=1e-12) + True + + You can also use ``approx`` to compare nonnumeric types, or dicts and + sequences containing nonnumeric types, in which case it falls back to + strict equality. This can be useful for comparing dicts and sequences that + can contain optional values:: + + >>> {"required": 1.0000005, "optional": None} == approx({"required": 1, "optional": None}) + True + >>> [None, 1.0000005] == approx([None,1]) + True + >>> ["foo", 1.0000005] == approx([None,1]) + False + + If you're thinking about using ``approx``, then you might want to know how + it compares to other good ways of comparing floating-point numbers. All of + these algorithms are based on relative and absolute tolerances and should + agree for the most part, but they do have meaningful differences: + + - ``math.isclose(a, b, rel_tol=1e-9, abs_tol=0.0)``: True if the relative + tolerance is met w.r.t. either ``a`` or ``b`` or if the absolute + tolerance is met. Because the relative tolerance is calculated w.r.t. + both ``a`` and ``b``, this test is symmetric (i.e. neither ``a`` nor + ``b`` is a "reference value"). You have to specify an absolute tolerance + if you want to compare to ``0.0`` because there is no tolerance by + default. More information: :py:func:`math.isclose`. + + - ``numpy.isclose(a, b, rtol=1e-5, atol=1e-8)``: True if the difference + between ``a`` and ``b`` is less that the sum of the relative tolerance + w.r.t. ``b`` and the absolute tolerance. Because the relative tolerance + is only calculated w.r.t. ``b``, this test is asymmetric and you can + think of ``b`` as the reference value. Support for comparing sequences + is provided by :py:func:`numpy.allclose`. More information: + :std:doc:`numpy:reference/generated/numpy.isclose`. + + - ``unittest.TestCase.assertAlmostEqual(a, b)``: True if ``a`` and ``b`` + are within an absolute tolerance of ``1e-7``. No relative tolerance is + considered , so this function is not appropriate for very large or very + small numbers. Also, it's only available in subclasses of ``unittest.TestCase`` + and it's ugly because it doesn't follow PEP8. More information: + :py:meth:`unittest.TestCase.assertAlmostEqual`. + + - ``a == pytest.approx(b, rel=1e-6, abs=1e-12)``: True if the relative + tolerance is met w.r.t. ``b`` or if the absolute tolerance is met. + Because the relative tolerance is only calculated w.r.t. ``b``, this test + is asymmetric and you can think of ``b`` as the reference value. In the + special case that you explicitly specify an absolute tolerance but not a + relative tolerance, only the absolute tolerance is considered. + + .. note:: + + ``approx`` can handle numpy arrays, but we recommend the + specialised test helpers in :std:doc:`numpy:reference/routines.testing` + if you need support for comparisons, NaNs, or ULP-based tolerances. + + .. warning:: + + .. versionchanged:: 3.2 + + In order to avoid inconsistent behavior, :py:exc:`TypeError` is + raised for ``>``, ``>=``, ``<`` and ``<=`` comparisons. + The example below illustrates the problem:: + + assert approx(0.1) > 0.1 + 1e-10 # calls approx(0.1).__gt__(0.1 + 1e-10) + assert 0.1 + 1e-10 > approx(0.1) # calls approx(0.1).__lt__(0.1 + 1e-10) + + In the second example one expects ``approx(0.1).__le__(0.1 + 1e-10)`` + to be called. But instead, ``approx(0.1).__lt__(0.1 + 1e-10)`` is used to + comparison. This is because the call hierarchy of rich comparisons + follows a fixed behavior. More information: :py:meth:`object.__ge__` + + .. versionchanged:: 3.7.1 + ``approx`` raises ``TypeError`` when it encounters a dict value or + sequence element of nonnumeric type. + + .. versionchanged:: 6.1.0 + ``approx`` falls back to strict equality for nonnumeric types instead + of raising ``TypeError``. + """ + + # Delegate the comparison to a class that knows how to deal with the type + # of the expected value (e.g. int, float, list, dict, numpy.array, etc). + # + # The primary responsibility of these classes is to implement ``__eq__()`` + # and ``__repr__()``. The former is used to actually check if some + # "actual" value is equivalent to the given expected value within the + # allowed tolerance. The latter is used to show the user the expected + # value and tolerance, in the case that a test failed. + # + # The actual logic for making approximate comparisons can be found in + # ApproxScalar, which is used to compare individual numbers. All of the + # other Approx classes eventually delegate to this class. The ApproxBase + # class provides some convenient methods and overloads, but isn't really + # essential. + + __tracebackhide__ = True + + if isinstance(expected, Decimal): + cls: Type[ApproxBase] = ApproxDecimal + elif isinstance(expected, Mapping): + cls = ApproxMapping + elif _is_numpy_array(expected): + expected = _as_numpy_array(expected) + cls = ApproxNumpy + elif ( + isinstance(expected, Iterable) + and isinstance(expected, Sized) + # Type ignored because the error is wrong -- not unreachable. + and not isinstance(expected, STRING_TYPES) # type: ignore[unreachable] + ): + cls = ApproxSequencelike + else: + cls = ApproxScalar + + return cls(expected, rel, abs, nan_ok) + + +def _is_numpy_array(obj: object) -> bool: + """ + Return true if the given object is implicitly convertible to ndarray, + and numpy is already imported. + """ + return _as_numpy_array(obj) is not None + + +def _as_numpy_array(obj: object) -> Optional["ndarray"]: + """ + Return an ndarray if the given object is implicitly convertible to ndarray, + and numpy is already imported, otherwise None. + """ + import sys + + np: Any = sys.modules.get("numpy") + if np is not None: + # avoid infinite recursion on numpy scalars, which have __array__ + if np.isscalar(obj): + return None + elif isinstance(obj, np.ndarray): + return obj + elif hasattr(obj, "__array__") or hasattr("obj", "__array_interface__"): + return np.asarray(obj) + return None + + +# builtin pytest.raises helper + +E = TypeVar("E", bound=BaseException) + + +@overload +def raises( + expected_exception: Union[Type[E], Tuple[Type[E], ...]], + *, + match: Optional[Union[str, Pattern[str]]] = ..., +) -> "RaisesContext[E]": + ... + + +@overload +def raises( + expected_exception: Union[Type[E], Tuple[Type[E], ...]], + func: Callable[..., Any], + *args: Any, + **kwargs: Any, +) -> _pytest._code.ExceptionInfo[E]: + ... + + +def raises( + expected_exception: Union[Type[E], Tuple[Type[E], ...]], *args: Any, **kwargs: Any +) -> Union["RaisesContext[E]", _pytest._code.ExceptionInfo[E]]: + r"""Assert that a code block/function call raises ``expected_exception`` + or raise a failure exception otherwise. + + :kwparam match: + If specified, a string containing a regular expression, + or a regular expression object, that is tested against the string + representation of the exception using :py:func:`re.search`. To match a literal + string that may contain :std:ref:`special characters `, the pattern can + first be escaped with :py:func:`re.escape`. + + (This is only used when :py:func:`pytest.raises` is used as a context manager, + and passed through to the function otherwise. + When using :py:func:`pytest.raises` as a function, you can use: + ``pytest.raises(Exc, func, match="passed on").match("my pattern")``.) + + .. currentmodule:: _pytest._code + + Use ``pytest.raises`` as a context manager, which will capture the exception of the given + type:: + + >>> import pytest + >>> with pytest.raises(ZeroDivisionError): + ... 1/0 + + If the code block does not raise the expected exception (``ZeroDivisionError`` in the example + above), or no exception at all, the check will fail instead. + + You can also use the keyword argument ``match`` to assert that the + exception matches a text or regex:: + + >>> with pytest.raises(ValueError, match='must be 0 or None'): + ... raise ValueError("value must be 0 or None") + + >>> with pytest.raises(ValueError, match=r'must be \d+$'): + ... raise ValueError("value must be 42") + + The context manager produces an :class:`ExceptionInfo` object which can be used to inspect the + details of the captured exception:: + + >>> with pytest.raises(ValueError) as exc_info: + ... raise ValueError("value must be 42") + >>> assert exc_info.type is ValueError + >>> assert exc_info.value.args[0] == "value must be 42" + + .. note:: + + When using ``pytest.raises`` as a context manager, it's worthwhile to + note that normal context manager rules apply and that the exception + raised *must* be the final line in the scope of the context manager. + Lines of code after that, within the scope of the context manager will + not be executed. For example:: + + >>> value = 15 + >>> with pytest.raises(ValueError) as exc_info: + ... if value > 10: + ... raise ValueError("value must be <= 10") + ... assert exc_info.type is ValueError # this will not execute + + Instead, the following approach must be taken (note the difference in + scope):: + + >>> with pytest.raises(ValueError) as exc_info: + ... if value > 10: + ... raise ValueError("value must be <= 10") + ... + >>> assert exc_info.type is ValueError + + **Using with** ``pytest.mark.parametrize`` + + When using :ref:`pytest.mark.parametrize ref` + it is possible to parametrize tests such that + some runs raise an exception and others do not. + + See :ref:`parametrizing_conditional_raising` for an example. + + **Legacy form** + + It is possible to specify a callable by passing a to-be-called lambda:: + + >>> raises(ZeroDivisionError, lambda: 1/0) + + + or you can specify an arbitrary callable with arguments:: + + >>> def f(x): return 1/x + ... + >>> raises(ZeroDivisionError, f, 0) + + >>> raises(ZeroDivisionError, f, x=0) + + + The form above is fully supported but discouraged for new code because the + context manager form is regarded as more readable and less error-prone. + + .. note:: + Similar to caught exception objects in Python, explicitly clearing + local references to returned ``ExceptionInfo`` objects can + help the Python interpreter speed up its garbage collection. + + Clearing those references breaks a reference cycle + (``ExceptionInfo`` --> caught exception --> frame stack raising + the exception --> current frame stack --> local variables --> + ``ExceptionInfo``) which makes Python keep all objects referenced + from that cycle (including all local variables in the current + frame) alive until the next cyclic garbage collection run. + More detailed information can be found in the official Python + documentation for :ref:`the try statement `. + """ + __tracebackhide__ = True + + if isinstance(expected_exception, type): + excepted_exceptions: Tuple[Type[E], ...] = (expected_exception,) + else: + excepted_exceptions = expected_exception + for exc in excepted_exceptions: + if not isinstance(exc, type) or not issubclass(exc, BaseException): + msg = "expected exception must be a BaseException type, not {}" # type: ignore[unreachable] + not_a = exc.__name__ if isinstance(exc, type) else type(exc).__name__ + raise TypeError(msg.format(not_a)) + + message = f"DID NOT RAISE {expected_exception}" + + if not args: + match: Optional[Union[str, Pattern[str]]] = kwargs.pop("match", None) + if kwargs: + msg = "Unexpected keyword arguments passed to pytest.raises: " + msg += ", ".join(sorted(kwargs)) + msg += "\nUse context-manager form instead?" + raise TypeError(msg) + return RaisesContext(expected_exception, message, match) + else: + func = args[0] + if not callable(func): + raise TypeError(f"{func!r} object (type: {type(func)}) must be callable") + try: + func(*args[1:], **kwargs) + except expected_exception as e: + # We just caught the exception - there is a traceback. + assert e.__traceback__ is not None + return _pytest._code.ExceptionInfo.from_exc_info( + (type(e), e, e.__traceback__) + ) + fail(message) + + +# This doesn't work with mypy for now. Use fail.Exception instead. +raises.Exception = fail.Exception # type: ignore + + +@final +class RaisesContext(Generic[E]): + def __init__( + self, + expected_exception: Union[Type[E], Tuple[Type[E], ...]], + message: str, + match_expr: Optional[Union[str, Pattern[str]]] = None, + ) -> None: + self.expected_exception = expected_exception + self.message = message + self.match_expr = match_expr + self.excinfo: Optional[_pytest._code.ExceptionInfo[E]] = None + + def __enter__(self) -> _pytest._code.ExceptionInfo[E]: + self.excinfo = _pytest._code.ExceptionInfo.for_later() + return self.excinfo + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> bool: + __tracebackhide__ = True + if exc_type is None: + fail(self.message) + assert self.excinfo is not None + if not issubclass(exc_type, self.expected_exception): + return False + # Cast to narrow the exception type now that it's verified. + exc_info = cast(Tuple[Type[E], E, TracebackType], (exc_type, exc_val, exc_tb)) + self.excinfo.fill_unfilled(exc_info) + if self.match_expr is not None: + self.excinfo.match(self.match_expr) + return True diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/python_path.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/python_path.py new file mode 100644 index 0000000000..cceabbca12 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/python_path.py @@ -0,0 +1,24 @@ +import sys + +import pytest +from pytest import Config +from pytest import Parser + + +def pytest_addoption(parser: Parser) -> None: + parser.addini("pythonpath", type="paths", help="Add paths to sys.path", default=[]) + + +@pytest.hookimpl(tryfirst=True) +def pytest_load_initial_conftests(early_config: Config) -> None: + # `pythonpath = a b` will set `sys.path` to `[a, b, x, y, z, ...]` + for path in reversed(early_config.getini("pythonpath")): + sys.path.insert(0, str(path)) + + +@pytest.hookimpl(trylast=True) +def pytest_unconfigure(config: Config) -> None: + for path in config.getini("pythonpath"): + path_str = str(path) + if path_str in sys.path: + sys.path.remove(path_str) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/recwarn.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/recwarn.py new file mode 100644 index 0000000000..175b571a80 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/recwarn.py @@ -0,0 +1,296 @@ +"""Record warnings during test function execution.""" +import re +import warnings +from types import TracebackType +from typing import Any +from typing import Callable +from typing import Generator +from typing import Iterator +from typing import List +from typing import Optional +from typing import overload +from typing import Pattern +from typing import Tuple +from typing import Type +from typing import TypeVar +from typing import Union + +from _pytest.compat import final +from _pytest.deprecated import check_ispytest +from _pytest.deprecated import WARNS_NONE_ARG +from _pytest.fixtures import fixture +from _pytest.outcomes import fail + + +T = TypeVar("T") + + +@fixture +def recwarn() -> Generator["WarningsRecorder", None, None]: + """Return a :class:`WarningsRecorder` instance that records all warnings emitted by test functions. + + See https://docs.python.org/library/how-to/capture-warnings.html for information + on warning categories. + """ + wrec = WarningsRecorder(_ispytest=True) + with wrec: + warnings.simplefilter("default") + yield wrec + + +@overload +def deprecated_call( + *, match: Optional[Union[str, Pattern[str]]] = ... +) -> "WarningsRecorder": + ... + + +@overload +def deprecated_call(func: Callable[..., T], *args: Any, **kwargs: Any) -> T: + ... + + +def deprecated_call( + func: Optional[Callable[..., Any]] = None, *args: Any, **kwargs: Any +) -> Union["WarningsRecorder", Any]: + """Assert that code produces a ``DeprecationWarning`` or ``PendingDeprecationWarning``. + + This function can be used as a context manager:: + + >>> import warnings + >>> def api_call_v2(): + ... warnings.warn('use v3 of this api', DeprecationWarning) + ... return 200 + + >>> import pytest + >>> with pytest.deprecated_call(): + ... assert api_call_v2() == 200 + + It can also be used by passing a function and ``*args`` and ``**kwargs``, + in which case it will ensure calling ``func(*args, **kwargs)`` produces one of + the warnings types above. The return value is the return value of the function. + + In the context manager form you may use the keyword argument ``match`` to assert + that the warning matches a text or regex. + + The context manager produces a list of :class:`warnings.WarningMessage` objects, + one for each warning raised. + """ + __tracebackhide__ = True + if func is not None: + args = (func,) + args + return warns((DeprecationWarning, PendingDeprecationWarning), *args, **kwargs) + + +@overload +def warns( + expected_warning: Union[Type[Warning], Tuple[Type[Warning], ...]] = ..., + *, + match: Optional[Union[str, Pattern[str]]] = ..., +) -> "WarningsChecker": + ... + + +@overload +def warns( + expected_warning: Union[Type[Warning], Tuple[Type[Warning], ...]], + func: Callable[..., T], + *args: Any, + **kwargs: Any, +) -> T: + ... + + +def warns( + expected_warning: Union[Type[Warning], Tuple[Type[Warning], ...]] = Warning, + *args: Any, + match: Optional[Union[str, Pattern[str]]] = None, + **kwargs: Any, +) -> Union["WarningsChecker", Any]: + r"""Assert that code raises a particular class of warning. + + Specifically, the parameter ``expected_warning`` can be a warning class or + sequence of warning classes, and the inside the ``with`` block must issue a warning of that class or + classes. + + This helper produces a list of :class:`warnings.WarningMessage` objects, + one for each warning raised. + + This function can be used as a context manager, or any of the other ways + :func:`pytest.raises` can be used:: + + >>> import pytest + >>> with pytest.warns(RuntimeWarning): + ... warnings.warn("my warning", RuntimeWarning) + + In the context manager form you may use the keyword argument ``match`` to assert + that the warning matches a text or regex:: + + >>> with pytest.warns(UserWarning, match='must be 0 or None'): + ... warnings.warn("value must be 0 or None", UserWarning) + + >>> with pytest.warns(UserWarning, match=r'must be \d+$'): + ... warnings.warn("value must be 42", UserWarning) + + >>> with pytest.warns(UserWarning, match=r'must be \d+$'): + ... warnings.warn("this is not here", UserWarning) + Traceback (most recent call last): + ... + Failed: DID NOT WARN. No warnings of type ...UserWarning... were emitted... + + """ + __tracebackhide__ = True + if not args: + if kwargs: + msg = "Unexpected keyword arguments passed to pytest.warns: " + msg += ", ".join(sorted(kwargs)) + msg += "\nUse context-manager form instead?" + raise TypeError(msg) + return WarningsChecker(expected_warning, match_expr=match, _ispytest=True) + else: + func = args[0] + if not callable(func): + raise TypeError(f"{func!r} object (type: {type(func)}) must be callable") + with WarningsChecker(expected_warning, _ispytest=True): + return func(*args[1:], **kwargs) + + +class WarningsRecorder(warnings.catch_warnings): + """A context manager to record raised warnings. + + Adapted from `warnings.catch_warnings`. + """ + + def __init__(self, *, _ispytest: bool = False) -> None: + check_ispytest(_ispytest) + # Type ignored due to the way typeshed handles warnings.catch_warnings. + super().__init__(record=True) # type: ignore[call-arg] + self._entered = False + self._list: List[warnings.WarningMessage] = [] + + @property + def list(self) -> List["warnings.WarningMessage"]: + """The list of recorded warnings.""" + return self._list + + def __getitem__(self, i: int) -> "warnings.WarningMessage": + """Get a recorded warning by index.""" + return self._list[i] + + def __iter__(self) -> Iterator["warnings.WarningMessage"]: + """Iterate through the recorded warnings.""" + return iter(self._list) + + def __len__(self) -> int: + """The number of recorded warnings.""" + return len(self._list) + + def pop(self, cls: Type[Warning] = Warning) -> "warnings.WarningMessage": + """Pop the first recorded warning, raise exception if not exists.""" + for i, w in enumerate(self._list): + if issubclass(w.category, cls): + return self._list.pop(i) + __tracebackhide__ = True + raise AssertionError("%r not found in warning list" % cls) + + def clear(self) -> None: + """Clear the list of recorded warnings.""" + self._list[:] = [] + + # Type ignored because it doesn't exactly warnings.catch_warnings.__enter__ + # -- it returns a List but we only emulate one. + def __enter__(self) -> "WarningsRecorder": # type: ignore + if self._entered: + __tracebackhide__ = True + raise RuntimeError("Cannot enter %r twice" % self) + _list = super().__enter__() + # record=True means it's None. + assert _list is not None + self._list = _list + warnings.simplefilter("always") + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + if not self._entered: + __tracebackhide__ = True + raise RuntimeError("Cannot exit %r without entering first" % self) + + super().__exit__(exc_type, exc_val, exc_tb) + + # Built-in catch_warnings does not reset entered state so we do it + # manually here for this context manager to become reusable. + self._entered = False + + +@final +class WarningsChecker(WarningsRecorder): + def __init__( + self, + expected_warning: Optional[ + Union[Type[Warning], Tuple[Type[Warning], ...]] + ] = Warning, + match_expr: Optional[Union[str, Pattern[str]]] = None, + *, + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + super().__init__(_ispytest=True) + + msg = "exceptions must be derived from Warning, not %s" + if expected_warning is None: + warnings.warn(WARNS_NONE_ARG, stacklevel=4) + expected_warning_tup = None + elif isinstance(expected_warning, tuple): + for exc in expected_warning: + if not issubclass(exc, Warning): + raise TypeError(msg % type(exc)) + expected_warning_tup = expected_warning + elif issubclass(expected_warning, Warning): + expected_warning_tup = (expected_warning,) + else: + raise TypeError(msg % type(expected_warning)) + + self.expected_warning = expected_warning_tup + self.match_expr = match_expr + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + super().__exit__(exc_type, exc_val, exc_tb) + + __tracebackhide__ = True + + # only check if we're not currently handling an exception + if exc_type is None and exc_val is None and exc_tb is None: + if self.expected_warning is not None: + if not any(issubclass(r.category, self.expected_warning) for r in self): + __tracebackhide__ = True + fail( + "DID NOT WARN. No warnings of type {} were emitted. " + "The list of emitted warnings is: {}.".format( + self.expected_warning, [each.message for each in self] + ) + ) + elif self.match_expr is not None: + for r in self: + if issubclass(r.category, self.expected_warning): + if re.compile(self.match_expr).search(str(r.message)): + break + else: + fail( + "DID NOT WARN. No warnings of type {} matching" + " ('{}') were emitted. The list of emitted warnings" + " is: {}.".format( + self.expected_warning, + self.match_expr, + [each.message for each in self], + ) + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/reports.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/reports.py new file mode 100644 index 0000000000..a68e68bc52 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/reports.py @@ -0,0 +1,598 @@ +import os +from io import StringIO +from pprint import pprint +from typing import Any +from typing import cast +from typing import Dict +from typing import Iterable +from typing import Iterator +from typing import List +from typing import Optional +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import TypeVar +from typing import Union + +import attr + +from _pytest._code.code import ExceptionChainRepr +from _pytest._code.code import ExceptionInfo +from _pytest._code.code import ExceptionRepr +from _pytest._code.code import ReprEntry +from _pytest._code.code import ReprEntryNative +from _pytest._code.code import ReprExceptionInfo +from _pytest._code.code import ReprFileLocation +from _pytest._code.code import ReprFuncArgs +from _pytest._code.code import ReprLocals +from _pytest._code.code import ReprTraceback +from _pytest._code.code import TerminalRepr +from _pytest._io import TerminalWriter +from _pytest.compat import final +from _pytest.config import Config +from _pytest.nodes import Collector +from _pytest.nodes import Item +from _pytest.outcomes import skip + +if TYPE_CHECKING: + from typing import NoReturn + from typing_extensions import Literal + + from _pytest.runner import CallInfo + + +def getworkerinfoline(node): + try: + return node._workerinfocache + except AttributeError: + d = node.workerinfo + ver = "%s.%s.%s" % d["version_info"][:3] + node._workerinfocache = s = "[{}] {} -- Python {} {}".format( + d["id"], d["sysplatform"], ver, d["executable"] + ) + return s + + +_R = TypeVar("_R", bound="BaseReport") + + +class BaseReport: + when: Optional[str] + location: Optional[Tuple[str, Optional[int], str]] + longrepr: Union[ + None, ExceptionInfo[BaseException], Tuple[str, int, str], str, TerminalRepr + ] + sections: List[Tuple[str, str]] + nodeid: str + outcome: "Literal['passed', 'failed', 'skipped']" + + def __init__(self, **kw: Any) -> None: + self.__dict__.update(kw) + + if TYPE_CHECKING: + # Can have arbitrary fields given to __init__(). + def __getattr__(self, key: str) -> Any: + ... + + def toterminal(self, out: TerminalWriter) -> None: + if hasattr(self, "node"): + worker_info = getworkerinfoline(self.node) + if worker_info: + out.line(worker_info) + + longrepr = self.longrepr + if longrepr is None: + return + + if hasattr(longrepr, "toterminal"): + longrepr_terminal = cast(TerminalRepr, longrepr) + longrepr_terminal.toterminal(out) + else: + try: + s = str(longrepr) + except UnicodeEncodeError: + s = "" + out.line(s) + + def get_sections(self, prefix: str) -> Iterator[Tuple[str, str]]: + for name, content in self.sections: + if name.startswith(prefix): + yield prefix, content + + @property + def longreprtext(self) -> str: + """Read-only property that returns the full string representation of + ``longrepr``. + + .. versionadded:: 3.0 + """ + file = StringIO() + tw = TerminalWriter(file) + tw.hasmarkup = False + self.toterminal(tw) + exc = file.getvalue() + return exc.strip() + + @property + def caplog(self) -> str: + """Return captured log lines, if log capturing is enabled. + + .. versionadded:: 3.5 + """ + return "\n".join( + content for (prefix, content) in self.get_sections("Captured log") + ) + + @property + def capstdout(self) -> str: + """Return captured text from stdout, if capturing is enabled. + + .. versionadded:: 3.0 + """ + return "".join( + content for (prefix, content) in self.get_sections("Captured stdout") + ) + + @property + def capstderr(self) -> str: + """Return captured text from stderr, if capturing is enabled. + + .. versionadded:: 3.0 + """ + return "".join( + content for (prefix, content) in self.get_sections("Captured stderr") + ) + + @property + def passed(self) -> bool: + """Whether the outcome is passed.""" + return self.outcome == "passed" + + @property + def failed(self) -> bool: + """Whether the outcome is failed.""" + return self.outcome == "failed" + + @property + def skipped(self) -> bool: + """Whether the outcome is skipped.""" + return self.outcome == "skipped" + + @property + def fspath(self) -> str: + """The path portion of the reported node, as a string.""" + return self.nodeid.split("::")[0] + + @property + def count_towards_summary(self) -> bool: + """**Experimental** Whether this report should be counted towards the + totals shown at the end of the test session: "1 passed, 1 failure, etc". + + .. note:: + + This function is considered **experimental**, so beware that it is subject to changes + even in patch releases. + """ + return True + + @property + def head_line(self) -> Optional[str]: + """**Experimental** The head line shown with longrepr output for this + report, more commonly during traceback representation during + failures:: + + ________ Test.foo ________ + + + In the example above, the head_line is "Test.foo". + + .. note:: + + This function is considered **experimental**, so beware that it is subject to changes + even in patch releases. + """ + if self.location is not None: + fspath, lineno, domain = self.location + return domain + return None + + def _get_verbose_word(self, config: Config): + _category, _short, verbose = config.hook.pytest_report_teststatus( + report=self, config=config + ) + return verbose + + def _to_json(self) -> Dict[str, Any]: + """Return the contents of this report as a dict of builtin entries, + suitable for serialization. + + This was originally the serialize_report() function from xdist (ca03269). + + Experimental method. + """ + return _report_to_json(self) + + @classmethod + def _from_json(cls: Type[_R], reportdict: Dict[str, object]) -> _R: + """Create either a TestReport or CollectReport, depending on the calling class. + + It is the callers responsibility to know which class to pass here. + + This was originally the serialize_report() function from xdist (ca03269). + + Experimental method. + """ + kwargs = _report_kwargs_from_json(reportdict) + return cls(**kwargs) + + +def _report_unserialization_failure( + type_name: str, report_class: Type[BaseReport], reportdict +) -> "NoReturn": + url = "https://github.com/pytest-dev/pytest/issues" + stream = StringIO() + pprint("-" * 100, stream=stream) + pprint("INTERNALERROR: Unknown entry type returned: %s" % type_name, stream=stream) + pprint("report_name: %s" % report_class, stream=stream) + pprint(reportdict, stream=stream) + pprint("Please report this bug at %s" % url, stream=stream) + pprint("-" * 100, stream=stream) + raise RuntimeError(stream.getvalue()) + + +@final +class TestReport(BaseReport): + """Basic test report object (also used for setup and teardown calls if + they fail). + + Reports can contain arbitrary extra attributes. + """ + + __test__ = False + + def __init__( + self, + nodeid: str, + location: Tuple[str, Optional[int], str], + keywords, + outcome: "Literal['passed', 'failed', 'skipped']", + longrepr: Union[ + None, ExceptionInfo[BaseException], Tuple[str, int, str], str, TerminalRepr + ], + when: "Literal['setup', 'call', 'teardown']", + sections: Iterable[Tuple[str, str]] = (), + duration: float = 0, + user_properties: Optional[Iterable[Tuple[str, object]]] = None, + **extra, + ) -> None: + #: Normalized collection nodeid. + self.nodeid = nodeid + + #: A (filesystempath, lineno, domaininfo) tuple indicating the + #: actual location of a test item - it might be different from the + #: collected one e.g. if a method is inherited from a different module. + self.location: Tuple[str, Optional[int], str] = location + + #: A name -> value dictionary containing all keywords and + #: markers associated with a test invocation. + self.keywords = keywords + + #: Test outcome, always one of "passed", "failed", "skipped". + self.outcome = outcome + + #: None or a failure representation. + self.longrepr = longrepr + + #: One of 'setup', 'call', 'teardown' to indicate runtest phase. + self.when = when + + #: User properties is a list of tuples (name, value) that holds user + #: defined properties of the test. + self.user_properties = list(user_properties or []) + + #: Tuples of str ``(heading, content)`` with extra information + #: for the test report. Used by pytest to add text captured + #: from ``stdout``, ``stderr``, and intercepted logging events. May + #: be used by other plugins to add arbitrary information to reports. + self.sections = list(sections) + + #: Time it took to run just the test. + self.duration = duration + + self.__dict__.update(extra) + + def __repr__(self) -> str: + return "<{} {!r} when={!r} outcome={!r}>".format( + self.__class__.__name__, self.nodeid, self.when, self.outcome + ) + + @classmethod + def from_item_and_call(cls, item: Item, call: "CallInfo[None]") -> "TestReport": + """Create and fill a TestReport with standard item and call info.""" + when = call.when + # Remove "collect" from the Literal type -- only for collection calls. + assert when != "collect" + duration = call.duration + keywords = {x: 1 for x in item.keywords} + excinfo = call.excinfo + sections = [] + if not call.excinfo: + outcome: Literal["passed", "failed", "skipped"] = "passed" + longrepr: Union[ + None, + ExceptionInfo[BaseException], + Tuple[str, int, str], + str, + TerminalRepr, + ] = None + else: + if not isinstance(excinfo, ExceptionInfo): + outcome = "failed" + longrepr = excinfo + elif isinstance(excinfo.value, skip.Exception): + outcome = "skipped" + r = excinfo._getreprcrash() + if excinfo.value._use_item_location: + path, line = item.reportinfo()[:2] + assert line is not None + longrepr = os.fspath(path), line + 1, r.message + else: + longrepr = (str(r.path), r.lineno, r.message) + else: + outcome = "failed" + if call.when == "call": + longrepr = item.repr_failure(excinfo) + else: # exception in setup or teardown + longrepr = item._repr_failure_py( + excinfo, style=item.config.getoption("tbstyle", "auto") + ) + for rwhen, key, content in item._report_sections: + sections.append((f"Captured {key} {rwhen}", content)) + return cls( + item.nodeid, + item.location, + keywords, + outcome, + longrepr, + when, + sections, + duration, + user_properties=item.user_properties, + ) + + +@final +class CollectReport(BaseReport): + """Collection report object. + + Reports can contain arbitrary extra attributes. + """ + + when = "collect" + + def __init__( + self, + nodeid: str, + outcome: "Literal['passed', 'failed', 'skipped']", + longrepr: Union[ + None, ExceptionInfo[BaseException], Tuple[str, int, str], str, TerminalRepr + ], + result: Optional[List[Union[Item, Collector]]], + sections: Iterable[Tuple[str, str]] = (), + **extra, + ) -> None: + #: Normalized collection nodeid. + self.nodeid = nodeid + + #: Test outcome, always one of "passed", "failed", "skipped". + self.outcome = outcome + + #: None or a failure representation. + self.longrepr = longrepr + + #: The collected items and collection nodes. + self.result = result or [] + + #: Tuples of str ``(heading, content)`` with extra information + #: for the test report. Used by pytest to add text captured + #: from ``stdout``, ``stderr``, and intercepted logging events. May + #: be used by other plugins to add arbitrary information to reports. + self.sections = list(sections) + + self.__dict__.update(extra) + + @property + def location(self): + return (self.fspath, None, self.fspath) + + def __repr__(self) -> str: + return "".format( + self.nodeid, len(self.result), self.outcome + ) + + +class CollectErrorRepr(TerminalRepr): + def __init__(self, msg: str) -> None: + self.longrepr = msg + + def toterminal(self, out: TerminalWriter) -> None: + out.line(self.longrepr, red=True) + + +def pytest_report_to_serializable( + report: Union[CollectReport, TestReport] +) -> Optional[Dict[str, Any]]: + if isinstance(report, (TestReport, CollectReport)): + data = report._to_json() + data["$report_type"] = report.__class__.__name__ + return data + # TODO: Check if this is actually reachable. + return None # type: ignore[unreachable] + + +def pytest_report_from_serializable( + data: Dict[str, Any], +) -> Optional[Union[CollectReport, TestReport]]: + if "$report_type" in data: + if data["$report_type"] == "TestReport": + return TestReport._from_json(data) + elif data["$report_type"] == "CollectReport": + return CollectReport._from_json(data) + assert False, "Unknown report_type unserialize data: {}".format( + data["$report_type"] + ) + return None + + +def _report_to_json(report: BaseReport) -> Dict[str, Any]: + """Return the contents of this report as a dict of builtin entries, + suitable for serialization. + + This was originally the serialize_report() function from xdist (ca03269). + """ + + def serialize_repr_entry( + entry: Union[ReprEntry, ReprEntryNative] + ) -> Dict[str, Any]: + data = attr.asdict(entry) + for key, value in data.items(): + if hasattr(value, "__dict__"): + data[key] = attr.asdict(value) + entry_data = {"type": type(entry).__name__, "data": data} + return entry_data + + def serialize_repr_traceback(reprtraceback: ReprTraceback) -> Dict[str, Any]: + result = attr.asdict(reprtraceback) + result["reprentries"] = [ + serialize_repr_entry(x) for x in reprtraceback.reprentries + ] + return result + + def serialize_repr_crash( + reprcrash: Optional[ReprFileLocation], + ) -> Optional[Dict[str, Any]]: + if reprcrash is not None: + return attr.asdict(reprcrash) + else: + return None + + def serialize_exception_longrepr(rep: BaseReport) -> Dict[str, Any]: + assert rep.longrepr is not None + # TODO: Investigate whether the duck typing is really necessary here. + longrepr = cast(ExceptionRepr, rep.longrepr) + result: Dict[str, Any] = { + "reprcrash": serialize_repr_crash(longrepr.reprcrash), + "reprtraceback": serialize_repr_traceback(longrepr.reprtraceback), + "sections": longrepr.sections, + } + if isinstance(longrepr, ExceptionChainRepr): + result["chain"] = [] + for repr_traceback, repr_crash, description in longrepr.chain: + result["chain"].append( + ( + serialize_repr_traceback(repr_traceback), + serialize_repr_crash(repr_crash), + description, + ) + ) + else: + result["chain"] = None + return result + + d = report.__dict__.copy() + if hasattr(report.longrepr, "toterminal"): + if hasattr(report.longrepr, "reprtraceback") and hasattr( + report.longrepr, "reprcrash" + ): + d["longrepr"] = serialize_exception_longrepr(report) + else: + d["longrepr"] = str(report.longrepr) + else: + d["longrepr"] = report.longrepr + for name in d: + if isinstance(d[name], os.PathLike): + d[name] = os.fspath(d[name]) + elif name == "result": + d[name] = None # for now + return d + + +def _report_kwargs_from_json(reportdict: Dict[str, Any]) -> Dict[str, Any]: + """Return **kwargs that can be used to construct a TestReport or + CollectReport instance. + + This was originally the serialize_report() function from xdist (ca03269). + """ + + def deserialize_repr_entry(entry_data): + data = entry_data["data"] + entry_type = entry_data["type"] + if entry_type == "ReprEntry": + reprfuncargs = None + reprfileloc = None + reprlocals = None + if data["reprfuncargs"]: + reprfuncargs = ReprFuncArgs(**data["reprfuncargs"]) + if data["reprfileloc"]: + reprfileloc = ReprFileLocation(**data["reprfileloc"]) + if data["reprlocals"]: + reprlocals = ReprLocals(data["reprlocals"]["lines"]) + + reprentry: Union[ReprEntry, ReprEntryNative] = ReprEntry( + lines=data["lines"], + reprfuncargs=reprfuncargs, + reprlocals=reprlocals, + reprfileloc=reprfileloc, + style=data["style"], + ) + elif entry_type == "ReprEntryNative": + reprentry = ReprEntryNative(data["lines"]) + else: + _report_unserialization_failure(entry_type, TestReport, reportdict) + return reprentry + + def deserialize_repr_traceback(repr_traceback_dict): + repr_traceback_dict["reprentries"] = [ + deserialize_repr_entry(x) for x in repr_traceback_dict["reprentries"] + ] + return ReprTraceback(**repr_traceback_dict) + + def deserialize_repr_crash(repr_crash_dict: Optional[Dict[str, Any]]): + if repr_crash_dict is not None: + return ReprFileLocation(**repr_crash_dict) + else: + return None + + if ( + reportdict["longrepr"] + and "reprcrash" in reportdict["longrepr"] + and "reprtraceback" in reportdict["longrepr"] + ): + + reprtraceback = deserialize_repr_traceback( + reportdict["longrepr"]["reprtraceback"] + ) + reprcrash = deserialize_repr_crash(reportdict["longrepr"]["reprcrash"]) + if reportdict["longrepr"]["chain"]: + chain = [] + for repr_traceback_data, repr_crash_data, description in reportdict[ + "longrepr" + ]["chain"]: + chain.append( + ( + deserialize_repr_traceback(repr_traceback_data), + deserialize_repr_crash(repr_crash_data), + description, + ) + ) + exception_info: Union[ + ExceptionChainRepr, ReprExceptionInfo + ] = ExceptionChainRepr(chain) + else: + exception_info = ReprExceptionInfo(reprtraceback, reprcrash) + + for section in reportdict["longrepr"]["sections"]: + exception_info.addsection(*section) + reportdict["longrepr"] = exception_info + + return reportdict diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/runner.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/runner.py new file mode 100644 index 0000000000..e43dd2dc81 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/runner.py @@ -0,0 +1,548 @@ +"""Basic collect and runtest protocol implementations.""" +import bdb +import os +import sys +import warnings +from typing import Callable +from typing import cast +from typing import Dict +from typing import Generic +from typing import List +from typing import Optional +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import TypeVar +from typing import Union + +import attr + +from .reports import BaseReport +from .reports import CollectErrorRepr +from .reports import CollectReport +from .reports import TestReport +from _pytest import timing +from _pytest._code.code import ExceptionChainRepr +from _pytest._code.code import ExceptionInfo +from _pytest._code.code import TerminalRepr +from _pytest.compat import final +from _pytest.config.argparsing import Parser +from _pytest.deprecated import check_ispytest +from _pytest.deprecated import UNITTEST_SKIP_DURING_COLLECTION +from _pytest.nodes import Collector +from _pytest.nodes import Item +from _pytest.nodes import Node +from _pytest.outcomes import Exit +from _pytest.outcomes import OutcomeException +from _pytest.outcomes import Skipped +from _pytest.outcomes import TEST_OUTCOME + +if TYPE_CHECKING: + from typing_extensions import Literal + + from _pytest.main import Session + from _pytest.terminal import TerminalReporter + +# +# pytest plugin hooks. + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("terminal reporting", "reporting", after="general") + group.addoption( + "--durations", + action="store", + type=int, + default=None, + metavar="N", + help="show N slowest setup/test durations (N=0 for all).", + ) + group.addoption( + "--durations-min", + action="store", + type=float, + default=0.005, + metavar="N", + help="Minimal duration in seconds for inclusion in slowest list. Default 0.005", + ) + + +def pytest_terminal_summary(terminalreporter: "TerminalReporter") -> None: + durations = terminalreporter.config.option.durations + durations_min = terminalreporter.config.option.durations_min + verbose = terminalreporter.config.getvalue("verbose") + if durations is None: + return + tr = terminalreporter + dlist = [] + for replist in tr.stats.values(): + for rep in replist: + if hasattr(rep, "duration"): + dlist.append(rep) + if not dlist: + return + dlist.sort(key=lambda x: x.duration, reverse=True) # type: ignore[no-any-return] + if not durations: + tr.write_sep("=", "slowest durations") + else: + tr.write_sep("=", "slowest %s durations" % durations) + dlist = dlist[:durations] + + for i, rep in enumerate(dlist): + if verbose < 2 and rep.duration < durations_min: + tr.write_line("") + tr.write_line( + "(%s durations < %gs hidden. Use -vv to show these durations.)" + % (len(dlist) - i, durations_min) + ) + break + tr.write_line(f"{rep.duration:02.2f}s {rep.when:<8} {rep.nodeid}") + + +def pytest_sessionstart(session: "Session") -> None: + session._setupstate = SetupState() + + +def pytest_sessionfinish(session: "Session") -> None: + session._setupstate.teardown_exact(None) + + +def pytest_runtest_protocol(item: Item, nextitem: Optional[Item]) -> bool: + ihook = item.ihook + ihook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location) + runtestprotocol(item, nextitem=nextitem) + ihook.pytest_runtest_logfinish(nodeid=item.nodeid, location=item.location) + return True + + +def runtestprotocol( + item: Item, log: bool = True, nextitem: Optional[Item] = None +) -> List[TestReport]: + hasrequest = hasattr(item, "_request") + if hasrequest and not item._request: # type: ignore[attr-defined] + # This only happens if the item is re-run, as is done by + # pytest-rerunfailures. + item._initrequest() # type: ignore[attr-defined] + rep = call_and_report(item, "setup", log) + reports = [rep] + if rep.passed: + if item.config.getoption("setupshow", False): + show_test_item(item) + if not item.config.getoption("setuponly", False): + reports.append(call_and_report(item, "call", log)) + reports.append(call_and_report(item, "teardown", log, nextitem=nextitem)) + # After all teardown hooks have been called + # want funcargs and request info to go away. + if hasrequest: + item._request = False # type: ignore[attr-defined] + item.funcargs = None # type: ignore[attr-defined] + return reports + + +def show_test_item(item: Item) -> None: + """Show test function, parameters and the fixtures of the test item.""" + tw = item.config.get_terminal_writer() + tw.line() + tw.write(" " * 8) + tw.write(item.nodeid) + used_fixtures = sorted(getattr(item, "fixturenames", [])) + if used_fixtures: + tw.write(" (fixtures used: {})".format(", ".join(used_fixtures))) + tw.flush() + + +def pytest_runtest_setup(item: Item) -> None: + _update_current_test_var(item, "setup") + item.session._setupstate.setup(item) + + +def pytest_runtest_call(item: Item) -> None: + _update_current_test_var(item, "call") + try: + del sys.last_type + del sys.last_value + del sys.last_traceback + except AttributeError: + pass + try: + item.runtest() + except Exception as e: + # Store trace info to allow postmortem debugging + sys.last_type = type(e) + sys.last_value = e + assert e.__traceback__ is not None + # Skip *this* frame + sys.last_traceback = e.__traceback__.tb_next + raise e + + +def pytest_runtest_teardown(item: Item, nextitem: Optional[Item]) -> None: + _update_current_test_var(item, "teardown") + item.session._setupstate.teardown_exact(nextitem) + _update_current_test_var(item, None) + + +def _update_current_test_var( + item: Item, when: Optional["Literal['setup', 'call', 'teardown']"] +) -> None: + """Update :envvar:`PYTEST_CURRENT_TEST` to reflect the current item and stage. + + If ``when`` is None, delete ``PYTEST_CURRENT_TEST`` from the environment. + """ + var_name = "PYTEST_CURRENT_TEST" + if when: + value = f"{item.nodeid} ({when})" + # don't allow null bytes on environment variables (see #2644, #2957) + value = value.replace("\x00", "(null)") + os.environ[var_name] = value + else: + os.environ.pop(var_name) + + +def pytest_report_teststatus(report: BaseReport) -> Optional[Tuple[str, str, str]]: + if report.when in ("setup", "teardown"): + if report.failed: + # category, shortletter, verbose-word + return "error", "E", "ERROR" + elif report.skipped: + return "skipped", "s", "SKIPPED" + else: + return "", "", "" + return None + + +# +# Implementation + + +def call_and_report( + item: Item, when: "Literal['setup', 'call', 'teardown']", log: bool = True, **kwds +) -> TestReport: + call = call_runtest_hook(item, when, **kwds) + hook = item.ihook + report: TestReport = hook.pytest_runtest_makereport(item=item, call=call) + if log: + hook.pytest_runtest_logreport(report=report) + if check_interactive_exception(call, report): + hook.pytest_exception_interact(node=item, call=call, report=report) + return report + + +def check_interactive_exception(call: "CallInfo[object]", report: BaseReport) -> bool: + """Check whether the call raised an exception that should be reported as + interactive.""" + if call.excinfo is None: + # Didn't raise. + return False + if hasattr(report, "wasxfail"): + # Exception was expected. + return False + if isinstance(call.excinfo.value, (Skipped, bdb.BdbQuit)): + # Special control flow exception. + return False + return True + + +def call_runtest_hook( + item: Item, when: "Literal['setup', 'call', 'teardown']", **kwds +) -> "CallInfo[None]": + if when == "setup": + ihook: Callable[..., None] = item.ihook.pytest_runtest_setup + elif when == "call": + ihook = item.ihook.pytest_runtest_call + elif when == "teardown": + ihook = item.ihook.pytest_runtest_teardown + else: + assert False, f"Unhandled runtest hook case: {when}" + reraise: Tuple[Type[BaseException], ...] = (Exit,) + if not item.config.getoption("usepdb", False): + reraise += (KeyboardInterrupt,) + return CallInfo.from_call( + lambda: ihook(item=item, **kwds), when=when, reraise=reraise + ) + + +TResult = TypeVar("TResult", covariant=True) + + +@final +@attr.s(repr=False, init=False, auto_attribs=True) +class CallInfo(Generic[TResult]): + """Result/Exception info of a function invocation.""" + + _result: Optional[TResult] + #: The captured exception of the call, if it raised. + excinfo: Optional[ExceptionInfo[BaseException]] + #: The system time when the call started, in seconds since the epoch. + start: float + #: The system time when the call ended, in seconds since the epoch. + stop: float + #: The call duration, in seconds. + duration: float + #: The context of invocation: "collect", "setup", "call" or "teardown". + when: "Literal['collect', 'setup', 'call', 'teardown']" + + def __init__( + self, + result: Optional[TResult], + excinfo: Optional[ExceptionInfo[BaseException]], + start: float, + stop: float, + duration: float, + when: "Literal['collect', 'setup', 'call', 'teardown']", + *, + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + self._result = result + self.excinfo = excinfo + self.start = start + self.stop = stop + self.duration = duration + self.when = when + + @property + def result(self) -> TResult: + """The return value of the call, if it didn't raise. + + Can only be accessed if excinfo is None. + """ + if self.excinfo is not None: + raise AttributeError(f"{self!r} has no valid result") + # The cast is safe because an exception wasn't raised, hence + # _result has the expected function return type (which may be + # None, that's why a cast and not an assert). + return cast(TResult, self._result) + + @classmethod + def from_call( + cls, + func: "Callable[[], TResult]", + when: "Literal['collect', 'setup', 'call', 'teardown']", + reraise: Optional[ + Union[Type[BaseException], Tuple[Type[BaseException], ...]] + ] = None, + ) -> "CallInfo[TResult]": + """Call func, wrapping the result in a CallInfo. + + :param func: + The function to call. Called without arguments. + :param when: + The phase in which the function is called. + :param reraise: + Exception or exceptions that shall propagate if raised by the + function, instead of being wrapped in the CallInfo. + """ + excinfo = None + start = timing.time() + precise_start = timing.perf_counter() + try: + result: Optional[TResult] = func() + except BaseException: + excinfo = ExceptionInfo.from_current() + if reraise is not None and isinstance(excinfo.value, reraise): + raise + result = None + # use the perf counter + precise_stop = timing.perf_counter() + duration = precise_stop - precise_start + stop = timing.time() + return cls( + start=start, + stop=stop, + duration=duration, + when=when, + result=result, + excinfo=excinfo, + _ispytest=True, + ) + + def __repr__(self) -> str: + if self.excinfo is None: + return f"" + return f"" + + +def pytest_runtest_makereport(item: Item, call: CallInfo[None]) -> TestReport: + return TestReport.from_item_and_call(item, call) + + +def pytest_make_collect_report(collector: Collector) -> CollectReport: + call = CallInfo.from_call(lambda: list(collector.collect()), "collect") + longrepr: Union[None, Tuple[str, int, str], str, TerminalRepr] = None + if not call.excinfo: + outcome: Literal["passed", "skipped", "failed"] = "passed" + else: + skip_exceptions = [Skipped] + unittest = sys.modules.get("unittest") + if unittest is not None: + # Type ignored because unittest is loaded dynamically. + skip_exceptions.append(unittest.SkipTest) # type: ignore + if isinstance(call.excinfo.value, tuple(skip_exceptions)): + if unittest is not None and isinstance( + call.excinfo.value, unittest.SkipTest # type: ignore[attr-defined] + ): + warnings.warn(UNITTEST_SKIP_DURING_COLLECTION, stacklevel=2) + + outcome = "skipped" + r_ = collector._repr_failure_py(call.excinfo, "line") + assert isinstance(r_, ExceptionChainRepr), repr(r_) + r = r_.reprcrash + assert r + longrepr = (str(r.path), r.lineno, r.message) + else: + outcome = "failed" + errorinfo = collector.repr_failure(call.excinfo) + if not hasattr(errorinfo, "toterminal"): + assert isinstance(errorinfo, str) + errorinfo = CollectErrorRepr(errorinfo) + longrepr = errorinfo + result = call.result if not call.excinfo else None + rep = CollectReport(collector.nodeid, outcome, longrepr, result) + rep.call = call # type: ignore # see collect_one_node + return rep + + +class SetupState: + """Shared state for setting up/tearing down test items or collectors + in a session. + + Suppose we have a collection tree as follows: + + + + + + + + The SetupState maintains a stack. The stack starts out empty: + + [] + + During the setup phase of item1, setup(item1) is called. What it does + is: + + push session to stack, run session.setup() + push mod1 to stack, run mod1.setup() + push item1 to stack, run item1.setup() + + The stack is: + + [session, mod1, item1] + + While the stack is in this shape, it is allowed to add finalizers to + each of session, mod1, item1 using addfinalizer(). + + During the teardown phase of item1, teardown_exact(item2) is called, + where item2 is the next item to item1. What it does is: + + pop item1 from stack, run its teardowns + pop mod1 from stack, run its teardowns + + mod1 was popped because it ended its purpose with item1. The stack is: + + [session] + + During the setup phase of item2, setup(item2) is called. What it does + is: + + push mod2 to stack, run mod2.setup() + push item2 to stack, run item2.setup() + + Stack: + + [session, mod2, item2] + + During the teardown phase of item2, teardown_exact(None) is called, + because item2 is the last item. What it does is: + + pop item2 from stack, run its teardowns + pop mod2 from stack, run its teardowns + pop session from stack, run its teardowns + + Stack: + + [] + + The end! + """ + + def __init__(self) -> None: + # The stack is in the dict insertion order. + self.stack: Dict[ + Node, + Tuple[ + # Node's finalizers. + List[Callable[[], object]], + # Node's exception, if its setup raised. + Optional[Union[OutcomeException, Exception]], + ], + ] = {} + + def setup(self, item: Item) -> None: + """Setup objects along the collector chain to the item.""" + needed_collectors = item.listchain() + + # If a collector fails its setup, fail its entire subtree of items. + # The setup is not retried for each item - the same exception is used. + for col, (finalizers, exc) in self.stack.items(): + assert col in needed_collectors, "previous item was not torn down properly" + if exc: + raise exc + + for col in needed_collectors[len(self.stack) :]: + assert col not in self.stack + # Push onto the stack. + self.stack[col] = ([col.teardown], None) + try: + col.setup() + except TEST_OUTCOME as exc: + self.stack[col] = (self.stack[col][0], exc) + raise exc + + def addfinalizer(self, finalizer: Callable[[], object], node: Node) -> None: + """Attach a finalizer to the given node. + + The node must be currently active in the stack. + """ + assert node and not isinstance(node, tuple) + assert callable(finalizer) + assert node in self.stack, (node, self.stack) + self.stack[node][0].append(finalizer) + + def teardown_exact(self, nextitem: Optional[Item]) -> None: + """Teardown the current stack up until reaching nodes that nextitem + also descends from. + + When nextitem is None (meaning we're at the last item), the entire + stack is torn down. + """ + needed_collectors = nextitem and nextitem.listchain() or [] + exc = None + while self.stack: + if list(self.stack.keys()) == needed_collectors[: len(self.stack)]: + break + node, (finalizers, _) = self.stack.popitem() + while finalizers: + fin = finalizers.pop() + try: + fin() + except TEST_OUTCOME as e: + # XXX Only first exception will be seen by user, + # ideally all should be reported. + if exc is None: + exc = e + if exc: + raise exc + if nextitem is None: + assert not self.stack + + +def collect_one_node(collector: Collector) -> CollectReport: + ihook = collector.ihook + ihook.pytest_collectstart(collector=collector) + rep: CollectReport = ihook.pytest_make_collect_report(collector=collector) + call = rep.__dict__.pop("call", None) + if call and check_interactive_exception(call, rep): + ihook.pytest_exception_interact(node=collector, call=call, report=rep) + return rep diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/scope.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/scope.py new file mode 100644 index 0000000000..7a746fb9fa --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/scope.py @@ -0,0 +1,91 @@ +""" +Scope definition and related utilities. + +Those are defined here, instead of in the 'fixtures' module because +their use is spread across many other pytest modules, and centralizing it in 'fixtures' +would cause circular references. + +Also this makes the module light to import, as it should. +""" +from enum import Enum +from functools import total_ordering +from typing import Optional +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing_extensions import Literal + + _ScopeName = Literal["session", "package", "module", "class", "function"] + + +@total_ordering +class Scope(Enum): + """ + Represents one of the possible fixture scopes in pytest. + + Scopes are ordered from lower to higher, that is: + + ->>> higher ->>> + + Function < Class < Module < Package < Session + + <<<- lower <<<- + """ + + # Scopes need to be listed from lower to higher. + Function: "_ScopeName" = "function" + Class: "_ScopeName" = "class" + Module: "_ScopeName" = "module" + Package: "_ScopeName" = "package" + Session: "_ScopeName" = "session" + + def next_lower(self) -> "Scope": + """Return the next lower scope.""" + index = _SCOPE_INDICES[self] + if index == 0: + raise ValueError(f"{self} is the lower-most scope") + return _ALL_SCOPES[index - 1] + + def next_higher(self) -> "Scope": + """Return the next higher scope.""" + index = _SCOPE_INDICES[self] + if index == len(_SCOPE_INDICES) - 1: + raise ValueError(f"{self} is the upper-most scope") + return _ALL_SCOPES[index + 1] + + def __lt__(self, other: "Scope") -> bool: + self_index = _SCOPE_INDICES[self] + other_index = _SCOPE_INDICES[other] + return self_index < other_index + + @classmethod + def from_user( + cls, scope_name: "_ScopeName", descr: str, where: Optional[str] = None + ) -> "Scope": + """ + Given a scope name from the user, return the equivalent Scope enum. Should be used + whenever we want to convert a user provided scope name to its enum object. + + If the scope name is invalid, construct a user friendly message and call pytest.fail. + """ + from _pytest.outcomes import fail + + try: + # Holding this reference is necessary for mypy at the moment. + scope = Scope(scope_name) + except ValueError: + fail( + "{} {}got an unexpected scope value '{}'".format( + descr, f"from {where} " if where else "", scope_name + ), + pytrace=False, + ) + return scope + + +_ALL_SCOPES = list(Scope) +_SCOPE_INDICES = {scope: index for index, scope in enumerate(_ALL_SCOPES)} + + +# Ordered list of scopes which can contain many tests (in practice all except Function). +HIGH_SCOPES = [x for x in Scope if x is not Scope.Function] diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/setuponly.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/setuponly.py new file mode 100644 index 0000000000..531131ce72 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/setuponly.py @@ -0,0 +1,97 @@ +from typing import Generator +from typing import Optional +from typing import Union + +import pytest +from _pytest._io.saferepr import saferepr +from _pytest.config import Config +from _pytest.config import ExitCode +from _pytest.config.argparsing import Parser +from _pytest.fixtures import FixtureDef +from _pytest.fixtures import SubRequest +from _pytest.scope import Scope + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("debugconfig") + group.addoption( + "--setuponly", + "--setup-only", + action="store_true", + help="only setup fixtures, do not execute tests.", + ) + group.addoption( + "--setupshow", + "--setup-show", + action="store_true", + help="show setup of fixtures while executing tests.", + ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_fixture_setup( + fixturedef: FixtureDef[object], request: SubRequest +) -> Generator[None, None, None]: + yield + if request.config.option.setupshow: + if hasattr(request, "param"): + # Save the fixture parameter so ._show_fixture_action() can + # display it now and during the teardown (in .finish()). + if fixturedef.ids: + if callable(fixturedef.ids): + param = fixturedef.ids(request.param) + else: + param = fixturedef.ids[request.param_index] + else: + param = request.param + fixturedef.cached_param = param # type: ignore[attr-defined] + _show_fixture_action(fixturedef, "SETUP") + + +def pytest_fixture_post_finalizer(fixturedef: FixtureDef[object]) -> None: + if fixturedef.cached_result is not None: + config = fixturedef._fixturemanager.config + if config.option.setupshow: + _show_fixture_action(fixturedef, "TEARDOWN") + if hasattr(fixturedef, "cached_param"): + del fixturedef.cached_param # type: ignore[attr-defined] + + +def _show_fixture_action(fixturedef: FixtureDef[object], msg: str) -> None: + config = fixturedef._fixturemanager.config + capman = config.pluginmanager.getplugin("capturemanager") + if capman: + capman.suspend_global_capture() + + tw = config.get_terminal_writer() + tw.line() + # Use smaller indentation the higher the scope: Session = 0, Package = 1, etc. + scope_indent = list(reversed(Scope)).index(fixturedef._scope) + tw.write(" " * 2 * scope_indent) + tw.write( + "{step} {scope} {fixture}".format( + step=msg.ljust(8), # align the output to TEARDOWN + scope=fixturedef.scope[0].upper(), + fixture=fixturedef.argname, + ) + ) + + if msg == "SETUP": + deps = sorted(arg for arg in fixturedef.argnames if arg != "request") + if deps: + tw.write(" (fixtures used: {})".format(", ".join(deps))) + + if hasattr(fixturedef, "cached_param"): + tw.write(f"[{saferepr(fixturedef.cached_param, maxsize=42)}]") # type: ignore[attr-defined] + + tw.flush() + + if capman: + capman.resume_global_capture() + + +@pytest.hookimpl(tryfirst=True) +def pytest_cmdline_main(config: Config) -> Optional[Union[int, ExitCode]]: + if config.option.setuponly: + config.option.setupshow = True + return None diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/setupplan.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/setupplan.py new file mode 100644 index 0000000000..9ba81ccaf0 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/setupplan.py @@ -0,0 +1,40 @@ +from typing import Optional +from typing import Union + +import pytest +from _pytest.config import Config +from _pytest.config import ExitCode +from _pytest.config.argparsing import Parser +from _pytest.fixtures import FixtureDef +from _pytest.fixtures import SubRequest + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("debugconfig") + group.addoption( + "--setupplan", + "--setup-plan", + action="store_true", + help="show what fixtures and tests would be executed but " + "don't execute anything.", + ) + + +@pytest.hookimpl(tryfirst=True) +def pytest_fixture_setup( + fixturedef: FixtureDef[object], request: SubRequest +) -> Optional[object]: + # Will return a dummy fixture if the setuponly option is provided. + if request.config.option.setupplan: + my_cache_key = fixturedef.cache_key(request) + fixturedef.cached_result = (None, my_cache_key, None) + return fixturedef.cached_result + return None + + +@pytest.hookimpl(tryfirst=True) +def pytest_cmdline_main(config: Config) -> Optional[Union[int, ExitCode]]: + if config.option.setupplan: + config.option.setuponly = True + config.option.setupshow = True + return None diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/skipping.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/skipping.py new file mode 100644 index 0000000000..ac7216f838 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/skipping.py @@ -0,0 +1,296 @@ +"""Support for skip/xfail functions and markers.""" +import os +import platform +import sys +import traceback +from collections.abc import Mapping +from typing import Generator +from typing import Optional +from typing import Tuple +from typing import Type + +import attr + +from _pytest.config import Config +from _pytest.config import hookimpl +from _pytest.config.argparsing import Parser +from _pytest.mark.structures import Mark +from _pytest.nodes import Item +from _pytest.outcomes import fail +from _pytest.outcomes import skip +from _pytest.outcomes import xfail +from _pytest.reports import BaseReport +from _pytest.runner import CallInfo +from _pytest.stash import StashKey + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("general") + group.addoption( + "--runxfail", + action="store_true", + dest="runxfail", + default=False, + help="report the results of xfail tests as if they were not marked", + ) + + parser.addini( + "xfail_strict", + "default for the strict parameter of xfail " + "markers when not given explicitly (default: False)", + default=False, + type="bool", + ) + + +def pytest_configure(config: Config) -> None: + if config.option.runxfail: + # yay a hack + import pytest + + old = pytest.xfail + config.add_cleanup(lambda: setattr(pytest, "xfail", old)) + + def nop(*args, **kwargs): + pass + + nop.Exception = xfail.Exception # type: ignore[attr-defined] + setattr(pytest, "xfail", nop) + + config.addinivalue_line( + "markers", + "skip(reason=None): skip the given test function with an optional reason. " + 'Example: skip(reason="no way of currently testing this") skips the ' + "test.", + ) + config.addinivalue_line( + "markers", + "skipif(condition, ..., *, reason=...): " + "skip the given test function if any of the conditions evaluate to True. " + "Example: skipif(sys.platform == 'win32') skips the test if we are on the win32 platform. " + "See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-skipif", + ) + config.addinivalue_line( + "markers", + "xfail(condition, ..., *, reason=..., run=True, raises=None, strict=xfail_strict): " + "mark the test function as an expected failure if any of the conditions " + "evaluate to True. Optionally specify a reason for better reporting " + "and run=False if you don't even want to execute the test function. " + "If only specific exception(s) are expected, you can list them in " + "raises, and if the test fails in other ways, it will be reported as " + "a true failure. See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-xfail", + ) + + +def evaluate_condition(item: Item, mark: Mark, condition: object) -> Tuple[bool, str]: + """Evaluate a single skipif/xfail condition. + + If an old-style string condition is given, it is eval()'d, otherwise the + condition is bool()'d. If this fails, an appropriately formatted pytest.fail + is raised. + + Returns (result, reason). The reason is only relevant if the result is True. + """ + # String condition. + if isinstance(condition, str): + globals_ = { + "os": os, + "sys": sys, + "platform": platform, + "config": item.config, + } + for dictionary in reversed( + item.ihook.pytest_markeval_namespace(config=item.config) + ): + if not isinstance(dictionary, Mapping): + raise ValueError( + "pytest_markeval_namespace() needs to return a dict, got {!r}".format( + dictionary + ) + ) + globals_.update(dictionary) + if hasattr(item, "obj"): + globals_.update(item.obj.__globals__) # type: ignore[attr-defined] + try: + filename = f"<{mark.name} condition>" + condition_code = compile(condition, filename, "eval") + result = eval(condition_code, globals_) + except SyntaxError as exc: + msglines = [ + "Error evaluating %r condition" % mark.name, + " " + condition, + " " + " " * (exc.offset or 0) + "^", + "SyntaxError: invalid syntax", + ] + fail("\n".join(msglines), pytrace=False) + except Exception as exc: + msglines = [ + "Error evaluating %r condition" % mark.name, + " " + condition, + *traceback.format_exception_only(type(exc), exc), + ] + fail("\n".join(msglines), pytrace=False) + + # Boolean condition. + else: + try: + result = bool(condition) + except Exception as exc: + msglines = [ + "Error evaluating %r condition as a boolean" % mark.name, + *traceback.format_exception_only(type(exc), exc), + ] + fail("\n".join(msglines), pytrace=False) + + reason = mark.kwargs.get("reason", None) + if reason is None: + if isinstance(condition, str): + reason = "condition: " + condition + else: + # XXX better be checked at collection time + msg = ( + "Error evaluating %r: " % mark.name + + "you need to specify reason=STRING when using booleans as conditions." + ) + fail(msg, pytrace=False) + + return result, reason + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class Skip: + """The result of evaluate_skip_marks().""" + + reason: str = "unconditional skip" + + +def evaluate_skip_marks(item: Item) -> Optional[Skip]: + """Evaluate skip and skipif marks on item, returning Skip if triggered.""" + for mark in item.iter_markers(name="skipif"): + if "condition" not in mark.kwargs: + conditions = mark.args + else: + conditions = (mark.kwargs["condition"],) + + # Unconditional. + if not conditions: + reason = mark.kwargs.get("reason", "") + return Skip(reason) + + # If any of the conditions are true. + for condition in conditions: + result, reason = evaluate_condition(item, mark, condition) + if result: + return Skip(reason) + + for mark in item.iter_markers(name="skip"): + try: + return Skip(*mark.args, **mark.kwargs) + except TypeError as e: + raise TypeError(str(e) + " - maybe you meant pytest.mark.skipif?") from None + + return None + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class Xfail: + """The result of evaluate_xfail_marks().""" + + reason: str + run: bool + strict: bool + raises: Optional[Tuple[Type[BaseException], ...]] + + +def evaluate_xfail_marks(item: Item) -> Optional[Xfail]: + """Evaluate xfail marks on item, returning Xfail if triggered.""" + for mark in item.iter_markers(name="xfail"): + run = mark.kwargs.get("run", True) + strict = mark.kwargs.get("strict", item.config.getini("xfail_strict")) + raises = mark.kwargs.get("raises", None) + if "condition" not in mark.kwargs: + conditions = mark.args + else: + conditions = (mark.kwargs["condition"],) + + # Unconditional. + if not conditions: + reason = mark.kwargs.get("reason", "") + return Xfail(reason, run, strict, raises) + + # If any of the conditions are true. + for condition in conditions: + result, reason = evaluate_condition(item, mark, condition) + if result: + return Xfail(reason, run, strict, raises) + + return None + + +# Saves the xfail mark evaluation. Can be refreshed during call if None. +xfailed_key = StashKey[Optional[Xfail]]() + + +@hookimpl(tryfirst=True) +def pytest_runtest_setup(item: Item) -> None: + skipped = evaluate_skip_marks(item) + if skipped: + raise skip.Exception(skipped.reason, _use_item_location=True) + + item.stash[xfailed_key] = xfailed = evaluate_xfail_marks(item) + if xfailed and not item.config.option.runxfail and not xfailed.run: + xfail("[NOTRUN] " + xfailed.reason) + + +@hookimpl(hookwrapper=True) +def pytest_runtest_call(item: Item) -> Generator[None, None, None]: + xfailed = item.stash.get(xfailed_key, None) + if xfailed is None: + item.stash[xfailed_key] = xfailed = evaluate_xfail_marks(item) + + if xfailed and not item.config.option.runxfail and not xfailed.run: + xfail("[NOTRUN] " + xfailed.reason) + + yield + + # The test run may have added an xfail mark dynamically. + xfailed = item.stash.get(xfailed_key, None) + if xfailed is None: + item.stash[xfailed_key] = xfailed = evaluate_xfail_marks(item) + + +@hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item: Item, call: CallInfo[None]): + outcome = yield + rep = outcome.get_result() + xfailed = item.stash.get(xfailed_key, None) + if item.config.option.runxfail: + pass # don't interfere + elif call.excinfo and isinstance(call.excinfo.value, xfail.Exception): + assert call.excinfo.value.msg is not None + rep.wasxfail = "reason: " + call.excinfo.value.msg + rep.outcome = "skipped" + elif not rep.skipped and xfailed: + if call.excinfo: + raises = xfailed.raises + if raises is not None and not isinstance(call.excinfo.value, raises): + rep.outcome = "failed" + else: + rep.outcome = "skipped" + rep.wasxfail = xfailed.reason + elif call.when == "call": + if xfailed.strict: + rep.outcome = "failed" + rep.longrepr = "[XPASS(strict)] " + xfailed.reason + else: + rep.outcome = "passed" + rep.wasxfail = xfailed.reason + + +def pytest_report_teststatus(report: BaseReport) -> Optional[Tuple[str, str, str]]: + if hasattr(report, "wasxfail"): + if report.skipped: + return "xfailed", "x", "XFAIL" + elif report.passed: + return "xpassed", "X", "XPASS" + return None diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/stash.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/stash.py new file mode 100644 index 0000000000..e61d75b95f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/stash.py @@ -0,0 +1,112 @@ +from typing import Any +from typing import cast +from typing import Dict +from typing import Generic +from typing import TypeVar +from typing import Union + + +__all__ = ["Stash", "StashKey"] + + +T = TypeVar("T") +D = TypeVar("D") + + +class StashKey(Generic[T]): + """``StashKey`` is an object used as a key to a :class:`Stash`. + + A ``StashKey`` is associated with the type ``T`` of the value of the key. + + A ``StashKey`` is unique and cannot conflict with another key. + """ + + __slots__ = () + + +class Stash: + r"""``Stash`` is a type-safe heterogeneous mutable mapping that + allows keys and value types to be defined separately from + where it (the ``Stash``) is created. + + Usually you will be given an object which has a ``Stash``, for example + :class:`~pytest.Config` or a :class:`~_pytest.nodes.Node`: + + .. code-block:: python + + stash: Stash = some_object.stash + + If a module or plugin wants to store data in this ``Stash``, it creates + :class:`StashKey`\s for its keys (at the module level): + + .. code-block:: python + + # At the top-level of the module + some_str_key = StashKey[str]() + some_bool_key = StashKey[bool]() + + To store information: + + .. code-block:: python + + # Value type must match the key. + stash[some_str_key] = "value" + stash[some_bool_key] = True + + To retrieve the information: + + .. code-block:: python + + # The static type of some_str is str. + some_str = stash[some_str_key] + # The static type of some_bool is bool. + some_bool = stash[some_bool_key] + """ + + __slots__ = ("_storage",) + + def __init__(self) -> None: + self._storage: Dict[StashKey[Any], object] = {} + + def __setitem__(self, key: StashKey[T], value: T) -> None: + """Set a value for key.""" + self._storage[key] = value + + def __getitem__(self, key: StashKey[T]) -> T: + """Get the value for key. + + Raises ``KeyError`` if the key wasn't set before. + """ + return cast(T, self._storage[key]) + + def get(self, key: StashKey[T], default: D) -> Union[T, D]: + """Get the value for key, or return default if the key wasn't set + before.""" + try: + return self[key] + except KeyError: + return default + + def setdefault(self, key: StashKey[T], default: T) -> T: + """Return the value of key if already set, otherwise set the value + of key to default and return default.""" + try: + return self[key] + except KeyError: + self[key] = default + return default + + def __delitem__(self, key: StashKey[T]) -> None: + """Delete the value for key. + + Raises ``KeyError`` if the key wasn't set before. + """ + del self._storage[key] + + def __contains__(self, key: StashKey[T]) -> bool: + """Return whether key was set.""" + return key in self._storage + + def __len__(self) -> int: + """Return how many items exist in the stash.""" + return len(self._storage) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/stepwise.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/stepwise.py new file mode 100644 index 0000000000..4d95a96b87 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/stepwise.py @@ -0,0 +1,122 @@ +from typing import List +from typing import Optional +from typing import TYPE_CHECKING + +import pytest +from _pytest import nodes +from _pytest.config import Config +from _pytest.config.argparsing import Parser +from _pytest.main import Session +from _pytest.reports import TestReport + +if TYPE_CHECKING: + from _pytest.cacheprovider import Cache + +STEPWISE_CACHE_DIR = "cache/stepwise" + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("general") + group.addoption( + "--sw", + "--stepwise", + action="store_true", + default=False, + dest="stepwise", + help="exit on test failure and continue from last failing test next time", + ) + group.addoption( + "--sw-skip", + "--stepwise-skip", + action="store_true", + default=False, + dest="stepwise_skip", + help="ignore the first failing test but stop on the next failing test.\n" + "implicitly enables --stepwise.", + ) + + +@pytest.hookimpl +def pytest_configure(config: Config) -> None: + if config.option.stepwise_skip: + # allow --stepwise-skip to work on it's own merits. + config.option.stepwise = True + if config.getoption("stepwise"): + config.pluginmanager.register(StepwisePlugin(config), "stepwiseplugin") + + +def pytest_sessionfinish(session: Session) -> None: + if not session.config.getoption("stepwise"): + assert session.config.cache is not None + # Clear the list of failing tests if the plugin is not active. + session.config.cache.set(STEPWISE_CACHE_DIR, []) + + +class StepwisePlugin: + def __init__(self, config: Config) -> None: + self.config = config + self.session: Optional[Session] = None + self.report_status = "" + assert config.cache is not None + self.cache: Cache = config.cache + self.lastfailed: Optional[str] = self.cache.get(STEPWISE_CACHE_DIR, None) + self.skip: bool = config.getoption("stepwise_skip") + + def pytest_sessionstart(self, session: Session) -> None: + self.session = session + + def pytest_collection_modifyitems( + self, config: Config, items: List[nodes.Item] + ) -> None: + if not self.lastfailed: + self.report_status = "no previously failed tests, not skipping." + return + + # check all item nodes until we find a match on last failed + failed_index = None + for index, item in enumerate(items): + if item.nodeid == self.lastfailed: + failed_index = index + break + + # If the previously failed test was not found among the test items, + # do not skip any tests. + if failed_index is None: + self.report_status = "previously failed test not found, not skipping." + else: + self.report_status = f"skipping {failed_index} already passed items." + deselected = items[:failed_index] + del items[:failed_index] + config.hook.pytest_deselected(items=deselected) + + def pytest_runtest_logreport(self, report: TestReport) -> None: + if report.failed: + if self.skip: + # Remove test from the failed ones (if it exists) and unset the skip option + # to make sure the following tests will not be skipped. + if report.nodeid == self.lastfailed: + self.lastfailed = None + + self.skip = False + else: + # Mark test as the last failing and interrupt the test session. + self.lastfailed = report.nodeid + assert self.session is not None + self.session.shouldstop = ( + "Test failed, continuing from this test next run." + ) + + else: + # If the test was actually run and did pass. + if report.when == "call": + # Remove test from the failed ones, if exists. + if report.nodeid == self.lastfailed: + self.lastfailed = None + + def pytest_report_collectionfinish(self) -> Optional[str]: + if self.config.getoption("verbose") >= 0 and self.report_status: + return f"stepwise: {self.report_status}" + return None + + def pytest_sessionfinish(self) -> None: + self.cache.set(STEPWISE_CACHE_DIR, self.lastfailed) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/terminal.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/terminal.py new file mode 100644 index 0000000000..ccbd84d7d7 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/terminal.py @@ -0,0 +1,1394 @@ +"""Terminal reporting of the full testing process. + +This is a good source for looking at the various reporting hooks. +""" +import argparse +import datetime +import inspect +import platform +import sys +import warnings +from collections import Counter +from functools import partial +from pathlib import Path +from typing import Any +from typing import Callable +from typing import cast +from typing import ClassVar +from typing import Dict +from typing import Generator +from typing import List +from typing import Mapping +from typing import Optional +from typing import Sequence +from typing import Set +from typing import TextIO +from typing import Tuple +from typing import TYPE_CHECKING +from typing import Union + +import attr +import pluggy + +import _pytest._version +from _pytest import nodes +from _pytest import timing +from _pytest._code import ExceptionInfo +from _pytest._code.code import ExceptionRepr +from _pytest._io.wcwidth import wcswidth +from _pytest.compat import final +from _pytest.config import _PluggyPlugin +from _pytest.config import Config +from _pytest.config import ExitCode +from _pytest.config import hookimpl +from _pytest.config.argparsing import Parser +from _pytest.nodes import Item +from _pytest.nodes import Node +from _pytest.pathlib import absolutepath +from _pytest.pathlib import bestrelpath +from _pytest.reports import BaseReport +from _pytest.reports import CollectReport +from _pytest.reports import TestReport + +if TYPE_CHECKING: + from typing_extensions import Literal + + from _pytest.main import Session + + +REPORT_COLLECTING_RESOLUTION = 0.5 + +KNOWN_TYPES = ( + "failed", + "passed", + "skipped", + "deselected", + "xfailed", + "xpassed", + "warnings", + "error", +) + +_REPORTCHARS_DEFAULT = "fE" + + +class MoreQuietAction(argparse.Action): + """A modified copy of the argparse count action which counts down and updates + the legacy quiet attribute at the same time. + + Used to unify verbosity handling. + """ + + def __init__( + self, + option_strings: Sequence[str], + dest: str, + default: object = None, + required: bool = False, + help: Optional[str] = None, + ) -> None: + super().__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + default=default, + required=required, + help=help, + ) + + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: Union[str, Sequence[object], None], + option_string: Optional[str] = None, + ) -> None: + new_count = getattr(namespace, self.dest, 0) - 1 + setattr(namespace, self.dest, new_count) + # todo Deprecate config.quiet + namespace.quiet = getattr(namespace, "quiet", 0) + 1 + + +def pytest_addoption(parser: Parser) -> None: + group = parser.getgroup("terminal reporting", "reporting", after="general") + group._addoption( + "-v", + "--verbose", + action="count", + default=0, + dest="verbose", + help="increase verbosity.", + ) + group._addoption( + "--no-header", + action="store_true", + default=False, + dest="no_header", + help="disable header", + ) + group._addoption( + "--no-summary", + action="store_true", + default=False, + dest="no_summary", + help="disable summary", + ) + group._addoption( + "-q", + "--quiet", + action=MoreQuietAction, + default=0, + dest="verbose", + help="decrease verbosity.", + ) + group._addoption( + "--verbosity", + dest="verbose", + type=int, + default=0, + help="set verbosity. Default is 0.", + ) + group._addoption( + "-r", + action="store", + dest="reportchars", + default=_REPORTCHARS_DEFAULT, + metavar="chars", + help="show extra test summary info as specified by chars: (f)ailed, " + "(E)rror, (s)kipped, (x)failed, (X)passed, " + "(p)assed, (P)assed with output, (a)ll except passed (p/P), or (A)ll. " + "(w)arnings are enabled by default (see --disable-warnings), " + "'N' can be used to reset the list. (default: 'fE').", + ) + group._addoption( + "--disable-warnings", + "--disable-pytest-warnings", + default=False, + dest="disable_warnings", + action="store_true", + help="disable warnings summary", + ) + group._addoption( + "-l", + "--showlocals", + action="store_true", + dest="showlocals", + default=False, + help="show locals in tracebacks (disabled by default).", + ) + group._addoption( + "--tb", + metavar="style", + action="store", + dest="tbstyle", + default="auto", + choices=["auto", "long", "short", "no", "line", "native"], + help="traceback print mode (auto/long/short/line/native/no).", + ) + group._addoption( + "--show-capture", + action="store", + dest="showcapture", + choices=["no", "stdout", "stderr", "log", "all"], + default="all", + help="Controls how captured stdout/stderr/log is shown on failed tests. " + "Default is 'all'.", + ) + group._addoption( + "--fulltrace", + "--full-trace", + action="store_true", + default=False, + help="don't cut any tracebacks (default is to cut).", + ) + group._addoption( + "--color", + metavar="color", + action="store", + dest="color", + default="auto", + choices=["yes", "no", "auto"], + help="color terminal output (yes/no/auto).", + ) + group._addoption( + "--code-highlight", + default="yes", + choices=["yes", "no"], + help="Whether code should be highlighted (only if --color is also enabled)", + ) + + parser.addini( + "console_output_style", + help='console output: "classic", or with additional progress information ("progress" (percentage) | "count").', + default="progress", + ) + + +def pytest_configure(config: Config) -> None: + reporter = TerminalReporter(config, sys.stdout) + config.pluginmanager.register(reporter, "terminalreporter") + if config.option.debug or config.option.traceconfig: + + def mywriter(tags, args): + msg = " ".join(map(str, args)) + reporter.write_line("[traceconfig] " + msg) + + config.trace.root.setprocessor("pytest:config", mywriter) + + +def getreportopt(config: Config) -> str: + reportchars: str = config.option.reportchars + + old_aliases = {"F", "S"} + reportopts = "" + for char in reportchars: + if char in old_aliases: + char = char.lower() + if char == "a": + reportopts = "sxXEf" + elif char == "A": + reportopts = "PpsxXEf" + elif char == "N": + reportopts = "" + elif char not in reportopts: + reportopts += char + + if not config.option.disable_warnings and "w" not in reportopts: + reportopts = "w" + reportopts + elif config.option.disable_warnings and "w" in reportopts: + reportopts = reportopts.replace("w", "") + + return reportopts + + +@hookimpl(trylast=True) # after _pytest.runner +def pytest_report_teststatus(report: BaseReport) -> Tuple[str, str, str]: + letter = "F" + if report.passed: + letter = "." + elif report.skipped: + letter = "s" + + outcome: str = report.outcome + if report.when in ("collect", "setup", "teardown") and outcome == "failed": + outcome = "error" + letter = "E" + + return outcome, letter, outcome.upper() + + +@attr.s(auto_attribs=True) +class WarningReport: + """Simple structure to hold warnings information captured by ``pytest_warning_recorded``. + + :ivar str message: + User friendly message about the warning. + :ivar str|None nodeid: + nodeid that generated the warning (see ``get_location``). + :ivar tuple fslocation: + File system location of the source of the warning (see ``get_location``). + """ + + message: str + nodeid: Optional[str] = None + fslocation: Optional[Tuple[str, int]] = None + + count_towards_summary: ClassVar = True + + def get_location(self, config: Config) -> Optional[str]: + """Return the more user-friendly information about the location of a warning, or None.""" + if self.nodeid: + return self.nodeid + if self.fslocation: + filename, linenum = self.fslocation + relpath = bestrelpath(config.invocation_params.dir, absolutepath(filename)) + return f"{relpath}:{linenum}" + return None + + +@final +class TerminalReporter: + def __init__(self, config: Config, file: Optional[TextIO] = None) -> None: + import _pytest.config + + self.config = config + self._numcollected = 0 + self._session: Optional[Session] = None + self._showfspath: Optional[bool] = None + + self.stats: Dict[str, List[Any]] = {} + self._main_color: Optional[str] = None + self._known_types: Optional[List[str]] = None + self.startpath = config.invocation_params.dir + if file is None: + file = sys.stdout + self._tw = _pytest.config.create_terminal_writer(config, file) + self._screen_width = self._tw.fullwidth + self.currentfspath: Union[None, Path, str, int] = None + self.reportchars = getreportopt(config) + self.hasmarkup = self._tw.hasmarkup + self.isatty = file.isatty() + self._progress_nodeids_reported: Set[str] = set() + self._show_progress_info = self._determine_show_progress_info() + self._collect_report_last_write: Optional[float] = None + self._already_displayed_warnings: Optional[int] = None + self._keyboardinterrupt_memo: Optional[ExceptionRepr] = None + + def _determine_show_progress_info(self) -> "Literal['progress', 'count', False]": + """Return whether we should display progress information based on the current config.""" + # do not show progress if we are not capturing output (#3038) + if self.config.getoption("capture", "no") == "no": + return False + # do not show progress if we are showing fixture setup/teardown + if self.config.getoption("setupshow", False): + return False + cfg: str = self.config.getini("console_output_style") + if cfg == "progress": + return "progress" + elif cfg == "count": + return "count" + else: + return False + + @property + def verbosity(self) -> int: + verbosity: int = self.config.option.verbose + return verbosity + + @property + def showheader(self) -> bool: + return self.verbosity >= 0 + + @property + def no_header(self) -> bool: + return bool(self.config.option.no_header) + + @property + def no_summary(self) -> bool: + return bool(self.config.option.no_summary) + + @property + def showfspath(self) -> bool: + if self._showfspath is None: + return self.verbosity >= 0 + return self._showfspath + + @showfspath.setter + def showfspath(self, value: Optional[bool]) -> None: + self._showfspath = value + + @property + def showlongtestinfo(self) -> bool: + return self.verbosity > 0 + + def hasopt(self, char: str) -> bool: + char = {"xfailed": "x", "skipped": "s"}.get(char, char) + return char in self.reportchars + + def write_fspath_result(self, nodeid: str, res, **markup: bool) -> None: + fspath = self.config.rootpath / nodeid.split("::")[0] + if self.currentfspath is None or fspath != self.currentfspath: + if self.currentfspath is not None and self._show_progress_info: + self._write_progress_information_filling_space() + self.currentfspath = fspath + relfspath = bestrelpath(self.startpath, fspath) + self._tw.line() + self._tw.write(relfspath + " ") + self._tw.write(res, flush=True, **markup) + + def write_ensure_prefix(self, prefix: str, extra: str = "", **kwargs) -> None: + if self.currentfspath != prefix: + self._tw.line() + self.currentfspath = prefix + self._tw.write(prefix) + if extra: + self._tw.write(extra, **kwargs) + self.currentfspath = -2 + + def ensure_newline(self) -> None: + if self.currentfspath: + self._tw.line() + self.currentfspath = None + + def write(self, content: str, *, flush: bool = False, **markup: bool) -> None: + self._tw.write(content, flush=flush, **markup) + + def flush(self) -> None: + self._tw.flush() + + def write_line(self, line: Union[str, bytes], **markup: bool) -> None: + if not isinstance(line, str): + line = str(line, errors="replace") + self.ensure_newline() + self._tw.line(line, **markup) + + def rewrite(self, line: str, **markup: bool) -> None: + """Rewinds the terminal cursor to the beginning and writes the given line. + + :param erase: + If True, will also add spaces until the full terminal width to ensure + previous lines are properly erased. + + The rest of the keyword arguments are markup instructions. + """ + erase = markup.pop("erase", False) + if erase: + fill_count = self._tw.fullwidth - len(line) - 1 + fill = " " * fill_count + else: + fill = "" + line = str(line) + self._tw.write("\r" + line + fill, **markup) + + def write_sep( + self, + sep: str, + title: Optional[str] = None, + fullwidth: Optional[int] = None, + **markup: bool, + ) -> None: + self.ensure_newline() + self._tw.sep(sep, title, fullwidth, **markup) + + def section(self, title: str, sep: str = "=", **kw: bool) -> None: + self._tw.sep(sep, title, **kw) + + def line(self, msg: str, **kw: bool) -> None: + self._tw.line(msg, **kw) + + def _add_stats(self, category: str, items: Sequence[Any]) -> None: + set_main_color = category not in self.stats + self.stats.setdefault(category, []).extend(items) + if set_main_color: + self._set_main_color() + + def pytest_internalerror(self, excrepr: ExceptionRepr) -> bool: + for line in str(excrepr).split("\n"): + self.write_line("INTERNALERROR> " + line) + return True + + def pytest_warning_recorded( + self, + warning_message: warnings.WarningMessage, + nodeid: str, + ) -> None: + from _pytest.warnings import warning_record_to_str + + fslocation = warning_message.filename, warning_message.lineno + message = warning_record_to_str(warning_message) + + warning_report = WarningReport( + fslocation=fslocation, message=message, nodeid=nodeid + ) + self._add_stats("warnings", [warning_report]) + + def pytest_plugin_registered(self, plugin: _PluggyPlugin) -> None: + if self.config.option.traceconfig: + msg = f"PLUGIN registered: {plugin}" + # XXX This event may happen during setup/teardown time + # which unfortunately captures our output here + # which garbles our output if we use self.write_line. + self.write_line(msg) + + def pytest_deselected(self, items: Sequence[Item]) -> None: + self._add_stats("deselected", items) + + def pytest_runtest_logstart( + self, nodeid: str, location: Tuple[str, Optional[int], str] + ) -> None: + # Ensure that the path is printed before the + # 1st test of a module starts running. + if self.showlongtestinfo: + line = self._locationline(nodeid, *location) + self.write_ensure_prefix(line, "") + self.flush() + elif self.showfspath: + self.write_fspath_result(nodeid, "") + self.flush() + + def pytest_runtest_logreport(self, report: TestReport) -> None: + self._tests_ran = True + rep = report + res: Tuple[ + str, str, Union[str, Tuple[str, Mapping[str, bool]]] + ] = self.config.hook.pytest_report_teststatus(report=rep, config=self.config) + category, letter, word = res + if not isinstance(word, tuple): + markup = None + else: + word, markup = word + self._add_stats(category, [rep]) + if not letter and not word: + # Probably passed setup/teardown. + return + running_xdist = hasattr(rep, "node") + if markup is None: + was_xfail = hasattr(report, "wasxfail") + if rep.passed and not was_xfail: + markup = {"green": True} + elif rep.passed and was_xfail: + markup = {"yellow": True} + elif rep.failed: + markup = {"red": True} + elif rep.skipped: + markup = {"yellow": True} + else: + markup = {} + if self.verbosity <= 0: + self._tw.write(letter, **markup) + else: + self._progress_nodeids_reported.add(rep.nodeid) + line = self._locationline(rep.nodeid, *rep.location) + if not running_xdist: + self.write_ensure_prefix(line, word, **markup) + if rep.skipped or hasattr(report, "wasxfail"): + available_width = ( + (self._tw.fullwidth - self._tw.width_of_current_line) + - len(" [100%]") + - 1 + ) + reason = _get_raw_skip_reason(rep) + reason_ = _format_trimmed(" ({})", reason, available_width) + if reason and reason_ is not None: + self._tw.write(reason_) + if self._show_progress_info: + self._write_progress_information_filling_space() + else: + self.ensure_newline() + self._tw.write("[%s]" % rep.node.gateway.id) + if self._show_progress_info: + self._tw.write( + self._get_progress_information_message() + " ", cyan=True + ) + else: + self._tw.write(" ") + self._tw.write(word, **markup) + self._tw.write(" " + line) + self.currentfspath = -2 + self.flush() + + @property + def _is_last_item(self) -> bool: + assert self._session is not None + return len(self._progress_nodeids_reported) == self._session.testscollected + + def pytest_runtest_logfinish(self, nodeid: str) -> None: + assert self._session + if self.verbosity <= 0 and self._show_progress_info: + if self._show_progress_info == "count": + num_tests = self._session.testscollected + progress_length = len(f" [{num_tests}/{num_tests}]") + else: + progress_length = len(" [100%]") + + self._progress_nodeids_reported.add(nodeid) + + if self._is_last_item: + self._write_progress_information_filling_space() + else: + main_color, _ = self._get_main_color() + w = self._width_of_current_line + past_edge = w + progress_length + 1 >= self._screen_width + if past_edge: + msg = self._get_progress_information_message() + self._tw.write(msg + "\n", **{main_color: True}) + + def _get_progress_information_message(self) -> str: + assert self._session + collected = self._session.testscollected + if self._show_progress_info == "count": + if collected: + progress = self._progress_nodeids_reported + counter_format = f"{{:{len(str(collected))}d}}" + format_string = f" [{counter_format}/{{}}]" + return format_string.format(len(progress), collected) + return f" [ {collected} / {collected} ]" + else: + if collected: + return " [{:3d}%]".format( + len(self._progress_nodeids_reported) * 100 // collected + ) + return " [100%]" + + def _write_progress_information_filling_space(self) -> None: + color, _ = self._get_main_color() + msg = self._get_progress_information_message() + w = self._width_of_current_line + fill = self._tw.fullwidth - w - 1 + self.write(msg.rjust(fill), flush=True, **{color: True}) + + @property + def _width_of_current_line(self) -> int: + """Return the width of the current line.""" + return self._tw.width_of_current_line + + def pytest_collection(self) -> None: + if self.isatty: + if self.config.option.verbose >= 0: + self.write("collecting ... ", flush=True, bold=True) + self._collect_report_last_write = timing.time() + elif self.config.option.verbose >= 1: + self.write("collecting ... ", flush=True, bold=True) + + def pytest_collectreport(self, report: CollectReport) -> None: + if report.failed: + self._add_stats("error", [report]) + elif report.skipped: + self._add_stats("skipped", [report]) + items = [x for x in report.result if isinstance(x, Item)] + self._numcollected += len(items) + if self.isatty: + self.report_collect() + + def report_collect(self, final: bool = False) -> None: + if self.config.option.verbose < 0: + return + + if not final: + # Only write "collecting" report every 0.5s. + t = timing.time() + if ( + self._collect_report_last_write is not None + and self._collect_report_last_write > t - REPORT_COLLECTING_RESOLUTION + ): + return + self._collect_report_last_write = t + + errors = len(self.stats.get("error", [])) + skipped = len(self.stats.get("skipped", [])) + deselected = len(self.stats.get("deselected", [])) + selected = self._numcollected - errors - skipped - deselected + line = "collected " if final else "collecting " + line += ( + str(self._numcollected) + " item" + ("" if self._numcollected == 1 else "s") + ) + if errors: + line += " / %d error%s" % (errors, "s" if errors != 1 else "") + if deselected: + line += " / %d deselected" % deselected + if skipped: + line += " / %d skipped" % skipped + if self._numcollected > selected > 0: + line += " / %d selected" % selected + if self.isatty: + self.rewrite(line, bold=True, erase=True) + if final: + self.write("\n") + else: + self.write_line(line) + + @hookimpl(trylast=True) + def pytest_sessionstart(self, session: "Session") -> None: + self._session = session + self._sessionstarttime = timing.time() + if not self.showheader: + return + self.write_sep("=", "test session starts", bold=True) + verinfo = platform.python_version() + if not self.no_header: + msg = f"platform {sys.platform} -- Python {verinfo}" + pypy_version_info = getattr(sys, "pypy_version_info", None) + if pypy_version_info: + verinfo = ".".join(map(str, pypy_version_info[:3])) + msg += f"[pypy-{verinfo}-{pypy_version_info[3]}]" + msg += ", pytest-{}, pluggy-{}".format( + _pytest._version.version, pluggy.__version__ + ) + if ( + self.verbosity > 0 + or self.config.option.debug + or getattr(self.config.option, "pastebin", None) + ): + msg += " -- " + str(sys.executable) + self.write_line(msg) + lines = self.config.hook.pytest_report_header( + config=self.config, start_path=self.startpath + ) + self._write_report_lines_from_hooks(lines) + + def _write_report_lines_from_hooks( + self, lines: Sequence[Union[str, Sequence[str]]] + ) -> None: + for line_or_lines in reversed(lines): + if isinstance(line_or_lines, str): + self.write_line(line_or_lines) + else: + for line in line_or_lines: + self.write_line(line) + + def pytest_report_header(self, config: Config) -> List[str]: + line = "rootdir: %s" % config.rootpath + + if config.inipath: + line += ", configfile: " + bestrelpath(config.rootpath, config.inipath) + + testpaths: List[str] = config.getini("testpaths") + if config.invocation_params.dir == config.rootpath and config.args == testpaths: + line += ", testpaths: {}".format(", ".join(testpaths)) + + result = [line] + + plugininfo = config.pluginmanager.list_plugin_distinfo() + if plugininfo: + result.append("plugins: %s" % ", ".join(_plugin_nameversions(plugininfo))) + return result + + def pytest_collection_finish(self, session: "Session") -> None: + self.report_collect(True) + + lines = self.config.hook.pytest_report_collectionfinish( + config=self.config, + start_path=self.startpath, + items=session.items, + ) + self._write_report_lines_from_hooks(lines) + + if self.config.getoption("collectonly"): + if session.items: + if self.config.option.verbose > -1: + self._tw.line("") + self._printcollecteditems(session.items) + + failed = self.stats.get("failed") + if failed: + self._tw.sep("!", "collection failures") + for rep in failed: + rep.toterminal(self._tw) + + def _printcollecteditems(self, items: Sequence[Item]) -> None: + if self.config.option.verbose < 0: + if self.config.option.verbose < -1: + counts = Counter(item.nodeid.split("::", 1)[0] for item in items) + for name, count in sorted(counts.items()): + self._tw.line("%s: %d" % (name, count)) + else: + for item in items: + self._tw.line(item.nodeid) + return + stack: List[Node] = [] + indent = "" + for item in items: + needed_collectors = item.listchain()[1:] # strip root node + while stack: + if stack == needed_collectors[: len(stack)]: + break + stack.pop() + for col in needed_collectors[len(stack) :]: + stack.append(col) + indent = (len(stack) - 1) * " " + self._tw.line(f"{indent}{col}") + if self.config.option.verbose >= 1: + obj = getattr(col, "obj", None) + doc = inspect.getdoc(obj) if obj else None + if doc: + for line in doc.splitlines(): + self._tw.line("{}{}".format(indent + " ", line)) + + @hookimpl(hookwrapper=True) + def pytest_sessionfinish( + self, session: "Session", exitstatus: Union[int, ExitCode] + ): + outcome = yield + outcome.get_result() + self._tw.line("") + summary_exit_codes = ( + ExitCode.OK, + ExitCode.TESTS_FAILED, + ExitCode.INTERRUPTED, + ExitCode.USAGE_ERROR, + ExitCode.NO_TESTS_COLLECTED, + ) + if exitstatus in summary_exit_codes and not self.no_summary: + self.config.hook.pytest_terminal_summary( + terminalreporter=self, exitstatus=exitstatus, config=self.config + ) + if session.shouldfail: + self.write_sep("!", str(session.shouldfail), red=True) + if exitstatus == ExitCode.INTERRUPTED: + self._report_keyboardinterrupt() + self._keyboardinterrupt_memo = None + elif session.shouldstop: + self.write_sep("!", str(session.shouldstop), red=True) + self.summary_stats() + + @hookimpl(hookwrapper=True) + def pytest_terminal_summary(self) -> Generator[None, None, None]: + self.summary_errors() + self.summary_failures() + self.summary_warnings() + self.summary_passes() + yield + self.short_test_summary() + # Display any extra warnings from teardown here (if any). + self.summary_warnings() + + def pytest_keyboard_interrupt(self, excinfo: ExceptionInfo[BaseException]) -> None: + self._keyboardinterrupt_memo = excinfo.getrepr(funcargs=True) + + def pytest_unconfigure(self) -> None: + if self._keyboardinterrupt_memo is not None: + self._report_keyboardinterrupt() + + def _report_keyboardinterrupt(self) -> None: + excrepr = self._keyboardinterrupt_memo + assert excrepr is not None + assert excrepr.reprcrash is not None + msg = excrepr.reprcrash.message + self.write_sep("!", msg) + if "KeyboardInterrupt" in msg: + if self.config.option.fulltrace: + excrepr.toterminal(self._tw) + else: + excrepr.reprcrash.toterminal(self._tw) + self._tw.line( + "(to show a full traceback on KeyboardInterrupt use --full-trace)", + yellow=True, + ) + + def _locationline( + self, nodeid: str, fspath: str, lineno: Optional[int], domain: str + ) -> str: + def mkrel(nodeid: str) -> str: + line = self.config.cwd_relative_nodeid(nodeid) + if domain and line.endswith(domain): + line = line[: -len(domain)] + values = domain.split("[") + values[0] = values[0].replace(".", "::") # don't replace '.' in params + line += "[".join(values) + return line + + # collect_fspath comes from testid which has a "/"-normalized path. + if fspath: + res = mkrel(nodeid) + if self.verbosity >= 2 and nodeid.split("::")[0] != fspath.replace( + "\\", nodes.SEP + ): + res += " <- " + bestrelpath(self.startpath, Path(fspath)) + else: + res = "[location]" + return res + " " + + def _getfailureheadline(self, rep): + head_line = rep.head_line + if head_line: + return head_line + return "test session" # XXX? + + def _getcrashline(self, rep): + try: + return str(rep.longrepr.reprcrash) + except AttributeError: + try: + return str(rep.longrepr)[:50] + except AttributeError: + return "" + + # + # Summaries for sessionfinish. + # + def getreports(self, name: str): + return [x for x in self.stats.get(name, ()) if not hasattr(x, "_pdbshown")] + + def summary_warnings(self) -> None: + if self.hasopt("w"): + all_warnings: Optional[List[WarningReport]] = self.stats.get("warnings") + if not all_warnings: + return + + final = self._already_displayed_warnings is not None + if final: + warning_reports = all_warnings[self._already_displayed_warnings :] + else: + warning_reports = all_warnings + self._already_displayed_warnings = len(warning_reports) + if not warning_reports: + return + + reports_grouped_by_message: Dict[str, List[WarningReport]] = {} + for wr in warning_reports: + reports_grouped_by_message.setdefault(wr.message, []).append(wr) + + def collapsed_location_report(reports: List[WarningReport]) -> str: + locations = [] + for w in reports: + location = w.get_location(self.config) + if location: + locations.append(location) + + if len(locations) < 10: + return "\n".join(map(str, locations)) + + counts_by_filename = Counter( + str(loc).split("::", 1)[0] for loc in locations + ) + return "\n".join( + "{}: {} warning{}".format(k, v, "s" if v > 1 else "") + for k, v in counts_by_filename.items() + ) + + title = "warnings summary (final)" if final else "warnings summary" + self.write_sep("=", title, yellow=True, bold=False) + for message, message_reports in reports_grouped_by_message.items(): + maybe_location = collapsed_location_report(message_reports) + if maybe_location: + self._tw.line(maybe_location) + lines = message.splitlines() + indented = "\n".join(" " + x for x in lines) + message = indented.rstrip() + else: + message = message.rstrip() + self._tw.line(message) + self._tw.line() + self._tw.line( + "-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html" + ) + + def summary_passes(self) -> None: + if self.config.option.tbstyle != "no": + if self.hasopt("P"): + reports: List[TestReport] = self.getreports("passed") + if not reports: + return + self.write_sep("=", "PASSES") + for rep in reports: + if rep.sections: + msg = self._getfailureheadline(rep) + self.write_sep("_", msg, green=True, bold=True) + self._outrep_summary(rep) + self._handle_teardown_sections(rep.nodeid) + + def _get_teardown_reports(self, nodeid: str) -> List[TestReport]: + reports = self.getreports("") + return [ + report + for report in reports + if report.when == "teardown" and report.nodeid == nodeid + ] + + def _handle_teardown_sections(self, nodeid: str) -> None: + for report in self._get_teardown_reports(nodeid): + self.print_teardown_sections(report) + + def print_teardown_sections(self, rep: TestReport) -> None: + showcapture = self.config.option.showcapture + if showcapture == "no": + return + for secname, content in rep.sections: + if showcapture != "all" and showcapture not in secname: + continue + if "teardown" in secname: + self._tw.sep("-", secname) + if content[-1:] == "\n": + content = content[:-1] + self._tw.line(content) + + def summary_failures(self) -> None: + if self.config.option.tbstyle != "no": + reports: List[BaseReport] = self.getreports("failed") + if not reports: + return + self.write_sep("=", "FAILURES") + if self.config.option.tbstyle == "line": + for rep in reports: + line = self._getcrashline(rep) + self.write_line(line) + else: + for rep in reports: + msg = self._getfailureheadline(rep) + self.write_sep("_", msg, red=True, bold=True) + self._outrep_summary(rep) + self._handle_teardown_sections(rep.nodeid) + + def summary_errors(self) -> None: + if self.config.option.tbstyle != "no": + reports: List[BaseReport] = self.getreports("error") + if not reports: + return + self.write_sep("=", "ERRORS") + for rep in self.stats["error"]: + msg = self._getfailureheadline(rep) + if rep.when == "collect": + msg = "ERROR collecting " + msg + else: + msg = f"ERROR at {rep.when} of {msg}" + self.write_sep("_", msg, red=True, bold=True) + self._outrep_summary(rep) + + def _outrep_summary(self, rep: BaseReport) -> None: + rep.toterminal(self._tw) + showcapture = self.config.option.showcapture + if showcapture == "no": + return + for secname, content in rep.sections: + if showcapture != "all" and showcapture not in secname: + continue + self._tw.sep("-", secname) + if content[-1:] == "\n": + content = content[:-1] + self._tw.line(content) + + def summary_stats(self) -> None: + if self.verbosity < -1: + return + + session_duration = timing.time() - self._sessionstarttime + (parts, main_color) = self.build_summary_stats_line() + line_parts = [] + + display_sep = self.verbosity >= 0 + if display_sep: + fullwidth = self._tw.fullwidth + for text, markup in parts: + with_markup = self._tw.markup(text, **markup) + if display_sep: + fullwidth += len(with_markup) - len(text) + line_parts.append(with_markup) + msg = ", ".join(line_parts) + + main_markup = {main_color: True} + duration = f" in {format_session_duration(session_duration)}" + duration_with_markup = self._tw.markup(duration, **main_markup) + if display_sep: + fullwidth += len(duration_with_markup) - len(duration) + msg += duration_with_markup + + if display_sep: + markup_for_end_sep = self._tw.markup("", **main_markup) + if markup_for_end_sep.endswith("\x1b[0m"): + markup_for_end_sep = markup_for_end_sep[:-4] + fullwidth += len(markup_for_end_sep) + msg += markup_for_end_sep + + if display_sep: + self.write_sep("=", msg, fullwidth=fullwidth, **main_markup) + else: + self.write_line(msg, **main_markup) + + def short_test_summary(self) -> None: + if not self.reportchars: + return + + def show_simple(stat, lines: List[str]) -> None: + failed = self.stats.get(stat, []) + if not failed: + return + termwidth = self._tw.fullwidth + config = self.config + for rep in failed: + line = _get_line_with_reprcrash_message(config, rep, termwidth) + lines.append(line) + + def show_xfailed(lines: List[str]) -> None: + xfailed = self.stats.get("xfailed", []) + for rep in xfailed: + verbose_word = rep._get_verbose_word(self.config) + pos = _get_pos(self.config, rep) + lines.append(f"{verbose_word} {pos}") + reason = rep.wasxfail + if reason: + lines.append(" " + str(reason)) + + def show_xpassed(lines: List[str]) -> None: + xpassed = self.stats.get("xpassed", []) + for rep in xpassed: + verbose_word = rep._get_verbose_word(self.config) + pos = _get_pos(self.config, rep) + reason = rep.wasxfail + lines.append(f"{verbose_word} {pos} {reason}") + + def show_skipped(lines: List[str]) -> None: + skipped: List[CollectReport] = self.stats.get("skipped", []) + fskips = _folded_skips(self.startpath, skipped) if skipped else [] + if not fskips: + return + verbose_word = skipped[0]._get_verbose_word(self.config) + for num, fspath, lineno, reason in fskips: + if reason.startswith("Skipped: "): + reason = reason[9:] + if lineno is not None: + lines.append( + "%s [%d] %s:%d: %s" + % (verbose_word, num, fspath, lineno, reason) + ) + else: + lines.append("%s [%d] %s: %s" % (verbose_word, num, fspath, reason)) + + REPORTCHAR_ACTIONS: Mapping[str, Callable[[List[str]], None]] = { + "x": show_xfailed, + "X": show_xpassed, + "f": partial(show_simple, "failed"), + "s": show_skipped, + "p": partial(show_simple, "passed"), + "E": partial(show_simple, "error"), + } + + lines: List[str] = [] + for char in self.reportchars: + action = REPORTCHAR_ACTIONS.get(char) + if action: # skipping e.g. "P" (passed with output) here. + action(lines) + + if lines: + self.write_sep("=", "short test summary info") + for line in lines: + self.write_line(line) + + def _get_main_color(self) -> Tuple[str, List[str]]: + if self._main_color is None or self._known_types is None or self._is_last_item: + self._set_main_color() + assert self._main_color + assert self._known_types + return self._main_color, self._known_types + + def _determine_main_color(self, unknown_type_seen: bool) -> str: + stats = self.stats + if "failed" in stats or "error" in stats: + main_color = "red" + elif "warnings" in stats or "xpassed" in stats or unknown_type_seen: + main_color = "yellow" + elif "passed" in stats or not self._is_last_item: + main_color = "green" + else: + main_color = "yellow" + return main_color + + def _set_main_color(self) -> None: + unknown_types: List[str] = [] + for found_type in self.stats.keys(): + if found_type: # setup/teardown reports have an empty key, ignore them + if found_type not in KNOWN_TYPES and found_type not in unknown_types: + unknown_types.append(found_type) + self._known_types = list(KNOWN_TYPES) + unknown_types + self._main_color = self._determine_main_color(bool(unknown_types)) + + def build_summary_stats_line(self) -> Tuple[List[Tuple[str, Dict[str, bool]]], str]: + """ + Build the parts used in the last summary stats line. + + The summary stats line is the line shown at the end, "=== 12 passed, 2 errors in Xs===". + + This function builds a list of the "parts" that make up for the text in that line, in + the example above it would be: + + [ + ("12 passed", {"green": True}), + ("2 errors", {"red": True} + ] + + That last dict for each line is a "markup dictionary", used by TerminalWriter to + color output. + + The final color of the line is also determined by this function, and is the second + element of the returned tuple. + """ + if self.config.getoption("collectonly"): + return self._build_collect_only_summary_stats_line() + else: + return self._build_normal_summary_stats_line() + + def _get_reports_to_display(self, key: str) -> List[Any]: + """Get test/collection reports for the given status key, such as `passed` or `error`.""" + reports = self.stats.get(key, []) + return [x for x in reports if getattr(x, "count_towards_summary", True)] + + def _build_normal_summary_stats_line( + self, + ) -> Tuple[List[Tuple[str, Dict[str, bool]]], str]: + main_color, known_types = self._get_main_color() + parts = [] + + for key in known_types: + reports = self._get_reports_to_display(key) + if reports: + count = len(reports) + color = _color_for_type.get(key, _color_for_type_default) + markup = {color: True, "bold": color == main_color} + parts.append(("%d %s" % pluralize(count, key), markup)) + + if not parts: + parts = [("no tests ran", {_color_for_type_default: True})] + + return parts, main_color + + def _build_collect_only_summary_stats_line( + self, + ) -> Tuple[List[Tuple[str, Dict[str, bool]]], str]: + deselected = len(self._get_reports_to_display("deselected")) + errors = len(self._get_reports_to_display("error")) + + if self._numcollected == 0: + parts = [("no tests collected", {"yellow": True})] + main_color = "yellow" + + elif deselected == 0: + main_color = "green" + collected_output = "%d %s collected" % pluralize(self._numcollected, "test") + parts = [(collected_output, {main_color: True})] + else: + all_tests_were_deselected = self._numcollected == deselected + if all_tests_were_deselected: + main_color = "yellow" + collected_output = f"no tests collected ({deselected} deselected)" + else: + main_color = "green" + selected = self._numcollected - deselected + collected_output = f"{selected}/{self._numcollected} tests collected ({deselected} deselected)" + + parts = [(collected_output, {main_color: True})] + + if errors: + main_color = _color_for_type["error"] + parts += [("%d %s" % pluralize(errors, "error"), {main_color: True})] + + return parts, main_color + + +def _get_pos(config: Config, rep: BaseReport): + nodeid = config.cwd_relative_nodeid(rep.nodeid) + return nodeid + + +def _format_trimmed(format: str, msg: str, available_width: int) -> Optional[str]: + """Format msg into format, ellipsizing it if doesn't fit in available_width. + + Returns None if even the ellipsis can't fit. + """ + # Only use the first line. + i = msg.find("\n") + if i != -1: + msg = msg[:i] + + ellipsis = "..." + format_width = wcswidth(format.format("")) + if format_width + len(ellipsis) > available_width: + return None + + if format_width + wcswidth(msg) > available_width: + available_width -= len(ellipsis) + msg = msg[:available_width] + while format_width + wcswidth(msg) > available_width: + msg = msg[:-1] + msg += ellipsis + + return format.format(msg) + + +def _get_line_with_reprcrash_message( + config: Config, rep: BaseReport, termwidth: int +) -> str: + """Get summary line for a report, trying to add reprcrash message.""" + verbose_word = rep._get_verbose_word(config) + pos = _get_pos(config, rep) + + line = f"{verbose_word} {pos}" + line_width = wcswidth(line) + + try: + # Type ignored intentionally -- possible AttributeError expected. + msg = rep.longrepr.reprcrash.message # type: ignore[union-attr] + except AttributeError: + pass + else: + available_width = termwidth - line_width + msg = _format_trimmed(" - {}", msg, available_width) + if msg is not None: + line += msg + + return line + + +def _folded_skips( + startpath: Path, + skipped: Sequence[CollectReport], +) -> List[Tuple[int, str, Optional[int], str]]: + d: Dict[Tuple[str, Optional[int], str], List[CollectReport]] = {} + for event in skipped: + assert event.longrepr is not None + assert isinstance(event.longrepr, tuple), (event, event.longrepr) + assert len(event.longrepr) == 3, (event, event.longrepr) + fspath, lineno, reason = event.longrepr + # For consistency, report all fspaths in relative form. + fspath = bestrelpath(startpath, Path(fspath)) + keywords = getattr(event, "keywords", {}) + # Folding reports with global pytestmark variable. + # This is a workaround, because for now we cannot identify the scope of a skip marker + # TODO: Revisit after marks scope would be fixed. + if ( + event.when == "setup" + and "skip" in keywords + and "pytestmark" not in keywords + ): + key: Tuple[str, Optional[int], str] = (fspath, None, reason) + else: + key = (fspath, lineno, reason) + d.setdefault(key, []).append(event) + values: List[Tuple[int, str, Optional[int], str]] = [] + for key, events in d.items(): + values.append((len(events), *key)) + return values + + +_color_for_type = { + "failed": "red", + "error": "red", + "warnings": "yellow", + "passed": "green", +} +_color_for_type_default = "yellow" + + +def pluralize(count: int, noun: str) -> Tuple[int, str]: + # No need to pluralize words such as `failed` or `passed`. + if noun not in ["error", "warnings", "test"]: + return count, noun + + # The `warnings` key is plural. To avoid API breakage, we keep it that way but + # set it to singular here so we can determine plurality in the same way as we do + # for `error`. + noun = noun.replace("warnings", "warning") + + return count, noun + "s" if count != 1 else noun + + +def _plugin_nameversions(plugininfo) -> List[str]: + values: List[str] = [] + for plugin, dist in plugininfo: + # Gets us name and version! + name = "{dist.project_name}-{dist.version}".format(dist=dist) + # Questionable convenience, but it keeps things short. + if name.startswith("pytest-"): + name = name[7:] + # We decided to print python package names they can have more than one plugin. + if name not in values: + values.append(name) + return values + + +def format_session_duration(seconds: float) -> str: + """Format the given seconds in a human readable manner to show in the final summary.""" + if seconds < 60: + return f"{seconds:.2f}s" + else: + dt = datetime.timedelta(seconds=int(seconds)) + return f"{seconds:.2f}s ({dt})" + + +def _get_raw_skip_reason(report: TestReport) -> str: + """Get the reason string of a skip/xfail/xpass test report. + + The string is just the part given by the user. + """ + if hasattr(report, "wasxfail"): + reason = cast(str, report.wasxfail) + if reason.startswith("reason: "): + reason = reason[len("reason: ") :] + return reason + else: + assert report.skipped + assert isinstance(report.longrepr, tuple) + _, _, reason = report.longrepr + if reason.startswith("Skipped: "): + reason = reason[len("Skipped: ") :] + elif reason == "Skipped": + reason = "" + return reason diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/threadexception.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/threadexception.py new file mode 100644 index 0000000000..43341e739a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/threadexception.py @@ -0,0 +1,88 @@ +import threading +import traceback +import warnings +from types import TracebackType +from typing import Any +from typing import Callable +from typing import Generator +from typing import Optional +from typing import Type + +import pytest + + +# Copied from cpython/Lib/test/support/threading_helper.py, with modifications. +class catch_threading_exception: + """Context manager catching threading.Thread exception using + threading.excepthook. + + Storing exc_value using a custom hook can create a reference cycle. The + reference cycle is broken explicitly when the context manager exits. + + Storing thread using a custom hook can resurrect it if it is set to an + object which is being finalized. Exiting the context manager clears the + stored object. + + Usage: + with threading_helper.catch_threading_exception() as cm: + # code spawning a thread which raises an exception + ... + # check the thread exception: use cm.args + ... + # cm.args attribute no longer exists at this point + # (to break a reference cycle) + """ + + def __init__(self) -> None: + self.args: Optional["threading.ExceptHookArgs"] = None + self._old_hook: Optional[Callable[["threading.ExceptHookArgs"], Any]] = None + + def _hook(self, args: "threading.ExceptHookArgs") -> None: + self.args = args + + def __enter__(self) -> "catch_threading_exception": + self._old_hook = threading.excepthook + threading.excepthook = self._hook + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + assert self._old_hook is not None + threading.excepthook = self._old_hook + self._old_hook = None + del self.args + + +def thread_exception_runtest_hook() -> Generator[None, None, None]: + with catch_threading_exception() as cm: + yield + if cm.args: + thread_name = "" if cm.args.thread is None else cm.args.thread.name + msg = f"Exception in thread {thread_name}\n\n" + msg += "".join( + traceback.format_exception( + cm.args.exc_type, + cm.args.exc_value, + cm.args.exc_traceback, + ) + ) + warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg)) + + +@pytest.hookimpl(hookwrapper=True, trylast=True) +def pytest_runtest_setup() -> Generator[None, None, None]: + yield from thread_exception_runtest_hook() + + +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_runtest_call() -> Generator[None, None, None]: + yield from thread_exception_runtest_hook() + + +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_runtest_teardown() -> Generator[None, None, None]: + yield from thread_exception_runtest_hook() diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/timing.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/timing.py new file mode 100644 index 0000000000..925163a585 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/timing.py @@ -0,0 +1,12 @@ +"""Indirection for time functions. + +We intentionally grab some "time" functions internally to avoid tests mocking "time" to affect +pytest runtime information (issue #185). + +Fixture "mock_timing" also interacts with this module for pytest's own tests. +""" +from time import perf_counter +from time import sleep +from time import time + +__all__ = ["perf_counter", "sleep", "time"] diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/tmpdir.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/tmpdir.py new file mode 100644 index 0000000000..f901fd5727 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/tmpdir.py @@ -0,0 +1,211 @@ +"""Support for providing temporary directories to test functions.""" +import os +import re +import sys +import tempfile +from pathlib import Path +from typing import Optional + +import attr + +from .pathlib import LOCK_TIMEOUT +from .pathlib import make_numbered_dir +from .pathlib import make_numbered_dir_with_cleanup +from .pathlib import rm_rf +from _pytest.compat import final +from _pytest.config import Config +from _pytest.deprecated import check_ispytest +from _pytest.fixtures import fixture +from _pytest.fixtures import FixtureRequest +from _pytest.monkeypatch import MonkeyPatch + + +@final +@attr.s(init=False) +class TempPathFactory: + """Factory for temporary directories under the common base temp directory. + + The base directory can be configured using the ``--basetemp`` option. + """ + + _given_basetemp = attr.ib(type=Optional[Path]) + _trace = attr.ib() + _basetemp = attr.ib(type=Optional[Path]) + + def __init__( + self, + given_basetemp: Optional[Path], + trace, + basetemp: Optional[Path] = None, + *, + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + if given_basetemp is None: + self._given_basetemp = None + else: + # Use os.path.abspath() to get absolute path instead of resolve() as it + # does not work the same in all platforms (see #4427). + # Path.absolute() exists, but it is not public (see https://bugs.python.org/issue25012). + self._given_basetemp = Path(os.path.abspath(str(given_basetemp))) + self._trace = trace + self._basetemp = basetemp + + @classmethod + def from_config( + cls, + config: Config, + *, + _ispytest: bool = False, + ) -> "TempPathFactory": + """Create a factory according to pytest configuration. + + :meta private: + """ + check_ispytest(_ispytest) + return cls( + given_basetemp=config.option.basetemp, + trace=config.trace.get("tmpdir"), + _ispytest=True, + ) + + def _ensure_relative_to_basetemp(self, basename: str) -> str: + basename = os.path.normpath(basename) + if (self.getbasetemp() / basename).resolve().parent != self.getbasetemp(): + raise ValueError(f"{basename} is not a normalized and relative path") + return basename + + def mktemp(self, basename: str, numbered: bool = True) -> Path: + """Create a new temporary directory managed by the factory. + + :param basename: + Directory base name, must be a relative path. + + :param numbered: + If ``True``, ensure the directory is unique by adding a numbered + suffix greater than any existing one: ``basename="foo-"`` and ``numbered=True`` + means that this function will create directories named ``"foo-0"``, + ``"foo-1"``, ``"foo-2"`` and so on. + + :returns: + The path to the new directory. + """ + basename = self._ensure_relative_to_basetemp(basename) + if not numbered: + p = self.getbasetemp().joinpath(basename) + p.mkdir(mode=0o700) + else: + p = make_numbered_dir(root=self.getbasetemp(), prefix=basename, mode=0o700) + self._trace("mktemp", p) + return p + + def getbasetemp(self) -> Path: + """Return the base temporary directory, creating it if needed.""" + if self._basetemp is not None: + return self._basetemp + + if self._given_basetemp is not None: + basetemp = self._given_basetemp + if basetemp.exists(): + rm_rf(basetemp) + basetemp.mkdir(mode=0o700) + basetemp = basetemp.resolve() + else: + from_env = os.environ.get("PYTEST_DEBUG_TEMPROOT") + temproot = Path(from_env or tempfile.gettempdir()).resolve() + user = get_user() or "unknown" + # use a sub-directory in the temproot to speed-up + # make_numbered_dir() call + rootdir = temproot.joinpath(f"pytest-of-{user}") + try: + rootdir.mkdir(mode=0o700, exist_ok=True) + except OSError: + # getuser() likely returned illegal characters for the platform, use unknown back off mechanism + rootdir = temproot.joinpath("pytest-of-unknown") + rootdir.mkdir(mode=0o700, exist_ok=True) + # Because we use exist_ok=True with a predictable name, make sure + # we are the owners, to prevent any funny business (on unix, where + # temproot is usually shared). + # Also, to keep things private, fixup any world-readable temp + # rootdir's permissions. Historically 0o755 was used, so we can't + # just error out on this, at least for a while. + if sys.platform != "win32": + uid = os.getuid() + rootdir_stat = rootdir.stat() + # getuid shouldn't fail, but cpython defines such a case. + # Let's hope for the best. + if uid != -1: + if rootdir_stat.st_uid != uid: + raise OSError( + f"The temporary directory {rootdir} is not owned by the current user. " + "Fix this and try again." + ) + if (rootdir_stat.st_mode & 0o077) != 0: + os.chmod(rootdir, rootdir_stat.st_mode & ~0o077) + basetemp = make_numbered_dir_with_cleanup( + prefix="pytest-", + root=rootdir, + keep=3, + lock_timeout=LOCK_TIMEOUT, + mode=0o700, + ) + assert basetemp is not None, basetemp + self._basetemp = basetemp + self._trace("new basetemp", basetemp) + return basetemp + + +def get_user() -> Optional[str]: + """Return the current user name, or None if getuser() does not work + in the current environment (see #1010).""" + import getpass + + try: + return getpass.getuser() + except (ImportError, KeyError): + return None + + +def pytest_configure(config: Config) -> None: + """Create a TempPathFactory and attach it to the config object. + + This is to comply with existing plugins which expect the handler to be + available at pytest_configure time, but ideally should be moved entirely + to the tmp_path_factory session fixture. + """ + mp = MonkeyPatch() + config.add_cleanup(mp.undo) + _tmp_path_factory = TempPathFactory.from_config(config, _ispytest=True) + mp.setattr(config, "_tmp_path_factory", _tmp_path_factory, raising=False) + + +@fixture(scope="session") +def tmp_path_factory(request: FixtureRequest) -> TempPathFactory: + """Return a :class:`pytest.TempPathFactory` instance for the test session.""" + # Set dynamically by pytest_configure() above. + return request.config._tmp_path_factory # type: ignore + + +def _mk_tmp(request: FixtureRequest, factory: TempPathFactory) -> Path: + name = request.node.name + name = re.sub(r"[\W]", "_", name) + MAXVAL = 30 + name = name[:MAXVAL] + return factory.mktemp(name, numbered=True) + + +@fixture +def tmp_path(request: FixtureRequest, tmp_path_factory: TempPathFactory) -> Path: + """Return a temporary directory path object which is unique to each test + function invocation, created as a sub directory of the base temporary + directory. + + By default, a new base temporary directory is created each test session, + and old bases are removed after 3 sessions, to aid in debugging. If + ``--basetemp`` is used then it is cleared each session. See :ref:`base + temporary directory`. + + The returned object is a :class:`pathlib.Path` object. + """ + + return _mk_tmp(request, tmp_path_factory) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/unittest.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/unittest.py new file mode 100644 index 0000000000..0315168b04 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/unittest.py @@ -0,0 +1,414 @@ +"""Discover and run std-library "unittest" style tests.""" +import sys +import traceback +import types +from typing import Any +from typing import Callable +from typing import Generator +from typing import Iterable +from typing import List +from typing import Optional +from typing import Tuple +from typing import Type +from typing import TYPE_CHECKING +from typing import Union + +import _pytest._code +import pytest +from _pytest.compat import getimfunc +from _pytest.compat import is_async_function +from _pytest.config import hookimpl +from _pytest.fixtures import FixtureRequest +from _pytest.nodes import Collector +from _pytest.nodes import Item +from _pytest.outcomes import exit +from _pytest.outcomes import fail +from _pytest.outcomes import skip +from _pytest.outcomes import xfail +from _pytest.python import Class +from _pytest.python import Function +from _pytest.python import PyCollector +from _pytest.runner import CallInfo +from _pytest.scope import Scope + +if TYPE_CHECKING: + import unittest + import twisted.trial.unittest + + _SysExcInfoType = Union[ + Tuple[Type[BaseException], BaseException, types.TracebackType], + Tuple[None, None, None], + ] + + +def pytest_pycollect_makeitem( + collector: PyCollector, name: str, obj: object +) -> Optional["UnitTestCase"]: + # Has unittest been imported and is obj a subclass of its TestCase? + try: + ut = sys.modules["unittest"] + # Type ignored because `ut` is an opaque module. + if not issubclass(obj, ut.TestCase): # type: ignore + return None + except Exception: + return None + # Yes, so let's collect it. + item: UnitTestCase = UnitTestCase.from_parent(collector, name=name, obj=obj) + return item + + +class UnitTestCase(Class): + # Marker for fixturemanger.getfixtureinfo() + # to declare that our children do not support funcargs. + nofuncargs = True + + def collect(self) -> Iterable[Union[Item, Collector]]: + from unittest import TestLoader + + cls = self.obj + if not getattr(cls, "__test__", True): + return + + skipped = _is_skipped(cls) + if not skipped: + self._inject_setup_teardown_fixtures(cls) + self._inject_setup_class_fixture() + + self.session._fixturemanager.parsefactories(self, unittest=True) + loader = TestLoader() + foundsomething = False + for name in loader.getTestCaseNames(self.obj): + x = getattr(self.obj, name) + if not getattr(x, "__test__", True): + continue + funcobj = getimfunc(x) + yield TestCaseFunction.from_parent(self, name=name, callobj=funcobj) + foundsomething = True + + if not foundsomething: + runtest = getattr(self.obj, "runTest", None) + if runtest is not None: + ut = sys.modules.get("twisted.trial.unittest", None) + # Type ignored because `ut` is an opaque module. + if ut is None or runtest != ut.TestCase.runTest: # type: ignore + yield TestCaseFunction.from_parent(self, name="runTest") + + def _inject_setup_teardown_fixtures(self, cls: type) -> None: + """Injects a hidden auto-use fixture to invoke setUpClass/setup_method and corresponding + teardown functions (#517).""" + class_fixture = _make_xunit_fixture( + cls, + "setUpClass", + "tearDownClass", + "doClassCleanups", + scope=Scope.Class, + pass_self=False, + ) + if class_fixture: + cls.__pytest_class_setup = class_fixture # type: ignore[attr-defined] + + method_fixture = _make_xunit_fixture( + cls, + "setup_method", + "teardown_method", + None, + scope=Scope.Function, + pass_self=True, + ) + if method_fixture: + cls.__pytest_method_setup = method_fixture # type: ignore[attr-defined] + + +def _make_xunit_fixture( + obj: type, + setup_name: str, + teardown_name: str, + cleanup_name: Optional[str], + scope: Scope, + pass_self: bool, +): + setup = getattr(obj, setup_name, None) + teardown = getattr(obj, teardown_name, None) + if setup is None and teardown is None: + return None + + if cleanup_name: + cleanup = getattr(obj, cleanup_name, lambda *args: None) + else: + + def cleanup(*args): + pass + + @pytest.fixture( + scope=scope.value, + autouse=True, + # Use a unique name to speed up lookup. + name=f"_unittest_{setup_name}_fixture_{obj.__qualname__}", + ) + def fixture(self, request: FixtureRequest) -> Generator[None, None, None]: + if _is_skipped(self): + reason = self.__unittest_skip_why__ + raise pytest.skip.Exception(reason, _use_item_location=True) + if setup is not None: + try: + if pass_self: + setup(self, request.function) + else: + setup() + # unittest does not call the cleanup function for every BaseException, so we + # follow this here. + except Exception: + if pass_self: + cleanup(self) + else: + cleanup() + + raise + yield + try: + if teardown is not None: + if pass_self: + teardown(self, request.function) + else: + teardown() + finally: + if pass_self: + cleanup(self) + else: + cleanup() + + return fixture + + +class TestCaseFunction(Function): + nofuncargs = True + _excinfo: Optional[List[_pytest._code.ExceptionInfo[BaseException]]] = None + _testcase: Optional["unittest.TestCase"] = None + + def _getobj(self): + assert self.parent is not None + # Unlike a regular Function in a Class, where `item.obj` returns + # a *bound* method (attached to an instance), TestCaseFunction's + # `obj` returns an *unbound* method (not attached to an instance). + # This inconsistency is probably not desirable, but needs some + # consideration before changing. + return getattr(self.parent.obj, self.originalname) # type: ignore[attr-defined] + + def setup(self) -> None: + # A bound method to be called during teardown() if set (see 'runtest()'). + self._explicit_tearDown: Optional[Callable[[], None]] = None + assert self.parent is not None + self._testcase = self.parent.obj(self.name) # type: ignore[attr-defined] + self._obj = getattr(self._testcase, self.name) + if hasattr(self, "_request"): + self._request._fillfixtures() + + def teardown(self) -> None: + if self._explicit_tearDown is not None: + self._explicit_tearDown() + self._explicit_tearDown = None + self._testcase = None + self._obj = None + + def startTest(self, testcase: "unittest.TestCase") -> None: + pass + + def _addexcinfo(self, rawexcinfo: "_SysExcInfoType") -> None: + # Unwrap potential exception info (see twisted trial support below). + rawexcinfo = getattr(rawexcinfo, "_rawexcinfo", rawexcinfo) + try: + excinfo = _pytest._code.ExceptionInfo[BaseException].from_exc_info(rawexcinfo) # type: ignore[arg-type] + # Invoke the attributes to trigger storing the traceback + # trial causes some issue there. + excinfo.value + excinfo.traceback + except TypeError: + try: + try: + values = traceback.format_exception(*rawexcinfo) + values.insert( + 0, + "NOTE: Incompatible Exception Representation, " + "displaying natively:\n\n", + ) + fail("".join(values), pytrace=False) + except (fail.Exception, KeyboardInterrupt): + raise + except BaseException: + fail( + "ERROR: Unknown Incompatible Exception " + "representation:\n%r" % (rawexcinfo,), + pytrace=False, + ) + except KeyboardInterrupt: + raise + except fail.Exception: + excinfo = _pytest._code.ExceptionInfo.from_current() + self.__dict__.setdefault("_excinfo", []).append(excinfo) + + def addError( + self, testcase: "unittest.TestCase", rawexcinfo: "_SysExcInfoType" + ) -> None: + try: + if isinstance(rawexcinfo[1], exit.Exception): + exit(rawexcinfo[1].msg) + except TypeError: + pass + self._addexcinfo(rawexcinfo) + + def addFailure( + self, testcase: "unittest.TestCase", rawexcinfo: "_SysExcInfoType" + ) -> None: + self._addexcinfo(rawexcinfo) + + def addSkip(self, testcase: "unittest.TestCase", reason: str) -> None: + try: + raise pytest.skip.Exception(reason, _use_item_location=True) + except skip.Exception: + self._addexcinfo(sys.exc_info()) + + def addExpectedFailure( + self, + testcase: "unittest.TestCase", + rawexcinfo: "_SysExcInfoType", + reason: str = "", + ) -> None: + try: + xfail(str(reason)) + except xfail.Exception: + self._addexcinfo(sys.exc_info()) + + def addUnexpectedSuccess( + self, + testcase: "unittest.TestCase", + reason: Optional["twisted.trial.unittest.Todo"] = None, + ) -> None: + msg = "Unexpected success" + if reason: + msg += f": {reason.reason}" + # Preserve unittest behaviour - fail the test. Explicitly not an XPASS. + try: + fail(msg, pytrace=False) + except fail.Exception: + self._addexcinfo(sys.exc_info()) + + def addSuccess(self, testcase: "unittest.TestCase") -> None: + pass + + def stopTest(self, testcase: "unittest.TestCase") -> None: + pass + + def runtest(self) -> None: + from _pytest.debugging import maybe_wrap_pytest_function_for_tracing + + assert self._testcase is not None + + maybe_wrap_pytest_function_for_tracing(self) + + # Let the unittest framework handle async functions. + if is_async_function(self.obj): + # Type ignored because self acts as the TestResult, but is not actually one. + self._testcase(result=self) # type: ignore[arg-type] + else: + # When --pdb is given, we want to postpone calling tearDown() otherwise + # when entering the pdb prompt, tearDown() would have probably cleaned up + # instance variables, which makes it difficult to debug. + # Arguably we could always postpone tearDown(), but this changes the moment where the + # TestCase instance interacts with the results object, so better to only do it + # when absolutely needed. + if self.config.getoption("usepdb") and not _is_skipped(self.obj): + self._explicit_tearDown = self._testcase.tearDown + setattr(self._testcase, "tearDown", lambda *args: None) + + # We need to update the actual bound method with self.obj, because + # wrap_pytest_function_for_tracing replaces self.obj by a wrapper. + setattr(self._testcase, self.name, self.obj) + try: + self._testcase(result=self) # type: ignore[arg-type] + finally: + delattr(self._testcase, self.name) + + def _prunetraceback( + self, excinfo: _pytest._code.ExceptionInfo[BaseException] + ) -> None: + super()._prunetraceback(excinfo) + traceback = excinfo.traceback.filter( + lambda x: not x.frame.f_globals.get("__unittest") + ) + if traceback: + excinfo.traceback = traceback + + +@hookimpl(tryfirst=True) +def pytest_runtest_makereport(item: Item, call: CallInfo[None]) -> None: + if isinstance(item, TestCaseFunction): + if item._excinfo: + call.excinfo = item._excinfo.pop(0) + try: + del call.result + except AttributeError: + pass + + # Convert unittest.SkipTest to pytest.skip. + # This is actually only needed for nose, which reuses unittest.SkipTest for + # its own nose.SkipTest. For unittest TestCases, SkipTest is already + # handled internally, and doesn't reach here. + unittest = sys.modules.get("unittest") + if ( + unittest + and call.excinfo + and isinstance(call.excinfo.value, unittest.SkipTest) # type: ignore[attr-defined] + ): + excinfo = call.excinfo + call2 = CallInfo[None].from_call( + lambda: pytest.skip(str(excinfo.value)), call.when + ) + call.excinfo = call2.excinfo + + +# Twisted trial support. + + +@hookimpl(hookwrapper=True) +def pytest_runtest_protocol(item: Item) -> Generator[None, None, None]: + if isinstance(item, TestCaseFunction) and "twisted.trial.unittest" in sys.modules: + ut: Any = sys.modules["twisted.python.failure"] + Failure__init__ = ut.Failure.__init__ + check_testcase_implements_trial_reporter() + + def excstore( + self, exc_value=None, exc_type=None, exc_tb=None, captureVars=None + ): + if exc_value is None: + self._rawexcinfo = sys.exc_info() + else: + if exc_type is None: + exc_type = type(exc_value) + self._rawexcinfo = (exc_type, exc_value, exc_tb) + try: + Failure__init__( + self, exc_value, exc_type, exc_tb, captureVars=captureVars + ) + except TypeError: + Failure__init__(self, exc_value, exc_type, exc_tb) + + ut.Failure.__init__ = excstore + yield + ut.Failure.__init__ = Failure__init__ + else: + yield + + +def check_testcase_implements_trial_reporter(done: List[int] = []) -> None: + if done: + return + from zope.interface import classImplements + from twisted.trial.itrial import IReporter + + classImplements(TestCaseFunction, IReporter) + done.append(1) + + +def _is_skipped(obj) -> bool: + """Return True if the given object has been marked with @unittest.skip.""" + return bool(getattr(obj, "__unittest_skip__", False)) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/unraisableexception.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/unraisableexception.py new file mode 100644 index 0000000000..fcb5d8237c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/unraisableexception.py @@ -0,0 +1,93 @@ +import sys +import traceback +import warnings +from types import TracebackType +from typing import Any +from typing import Callable +from typing import Generator +from typing import Optional +from typing import Type + +import pytest + + +# Copied from cpython/Lib/test/support/__init__.py, with modifications. +class catch_unraisable_exception: + """Context manager catching unraisable exception using sys.unraisablehook. + + Storing the exception value (cm.unraisable.exc_value) creates a reference + cycle. The reference cycle is broken explicitly when the context manager + exits. + + Storing the object (cm.unraisable.object) can resurrect it if it is set to + an object which is being finalized. Exiting the context manager clears the + stored object. + + Usage: + with catch_unraisable_exception() as cm: + # code creating an "unraisable exception" + ... + # check the unraisable exception: use cm.unraisable + ... + # cm.unraisable attribute no longer exists at this point + # (to break a reference cycle) + """ + + def __init__(self) -> None: + self.unraisable: Optional["sys.UnraisableHookArgs"] = None + self._old_hook: Optional[Callable[["sys.UnraisableHookArgs"], Any]] = None + + def _hook(self, unraisable: "sys.UnraisableHookArgs") -> None: + # Storing unraisable.object can resurrect an object which is being + # finalized. Storing unraisable.exc_value creates a reference cycle. + self.unraisable = unraisable + + def __enter__(self) -> "catch_unraisable_exception": + self._old_hook = sys.unraisablehook + sys.unraisablehook = self._hook + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + assert self._old_hook is not None + sys.unraisablehook = self._old_hook + self._old_hook = None + del self.unraisable + + +def unraisable_exception_runtest_hook() -> Generator[None, None, None]: + with catch_unraisable_exception() as cm: + yield + if cm.unraisable: + if cm.unraisable.err_msg is not None: + err_msg = cm.unraisable.err_msg + else: + err_msg = "Exception ignored in" + msg = f"{err_msg}: {cm.unraisable.object!r}\n\n" + msg += "".join( + traceback.format_exception( + cm.unraisable.exc_type, + cm.unraisable.exc_value, + cm.unraisable.exc_traceback, + ) + ) + warnings.warn(pytest.PytestUnraisableExceptionWarning(msg)) + + +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_runtest_setup() -> Generator[None, None, None]: + yield from unraisable_exception_runtest_hook() + + +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_runtest_call() -> Generator[None, None, None]: + yield from unraisable_exception_runtest_hook() + + +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_runtest_teardown() -> Generator[None, None, None]: + yield from unraisable_exception_runtest_hook() diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/warning_types.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/warning_types.py new file mode 100644 index 0000000000..2a97a31978 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/warning_types.py @@ -0,0 +1,145 @@ +from typing import Any +from typing import Generic +from typing import Type +from typing import TypeVar + +import attr + +from _pytest.compat import final + + +class PytestWarning(UserWarning): + """Base class for all warnings emitted by pytest.""" + + __module__ = "pytest" + + +@final +class PytestAssertRewriteWarning(PytestWarning): + """Warning emitted by the pytest assert rewrite module.""" + + __module__ = "pytest" + + +@final +class PytestCacheWarning(PytestWarning): + """Warning emitted by the cache plugin in various situations.""" + + __module__ = "pytest" + + +@final +class PytestConfigWarning(PytestWarning): + """Warning emitted for configuration issues.""" + + __module__ = "pytest" + + +@final +class PytestCollectionWarning(PytestWarning): + """Warning emitted when pytest is not able to collect a file or symbol in a module.""" + + __module__ = "pytest" + + +class PytestDeprecationWarning(PytestWarning, DeprecationWarning): + """Warning class for features that will be removed in a future version.""" + + __module__ = "pytest" + + +@final +class PytestRemovedIn7Warning(PytestDeprecationWarning): + """Warning class for features that will be removed in pytest 7.""" + + __module__ = "pytest" + + +@final +class PytestRemovedIn8Warning(PytestDeprecationWarning): + """Warning class for features that will be removed in pytest 8.""" + + __module__ = "pytest" + + +@final +class PytestExperimentalApiWarning(PytestWarning, FutureWarning): + """Warning category used to denote experiments in pytest. + + Use sparingly as the API might change or even be removed completely in a + future version. + """ + + __module__ = "pytest" + + @classmethod + def simple(cls, apiname: str) -> "PytestExperimentalApiWarning": + return cls( + "{apiname} is an experimental api that may change over time".format( + apiname=apiname + ) + ) + + +@final +class PytestUnhandledCoroutineWarning(PytestWarning): + """Warning emitted for an unhandled coroutine. + + A coroutine was encountered when collecting test functions, but was not + handled by any async-aware plugin. + Coroutine test functions are not natively supported. + """ + + __module__ = "pytest" + + +@final +class PytestUnknownMarkWarning(PytestWarning): + """Warning emitted on use of unknown markers. + + See :ref:`mark` for details. + """ + + __module__ = "pytest" + + +@final +class PytestUnraisableExceptionWarning(PytestWarning): + """An unraisable exception was reported. + + Unraisable exceptions are exceptions raised in :meth:`__del__ ` + implementations and similar situations when the exception cannot be raised + as normal. + """ + + __module__ = "pytest" + + +@final +class PytestUnhandledThreadExceptionWarning(PytestWarning): + """An unhandled exception occurred in a :class:`~threading.Thread`. + + Such exceptions don't propagate normally. + """ + + __module__ = "pytest" + + +_W = TypeVar("_W", bound=PytestWarning) + + +@final +@attr.s(auto_attribs=True) +class UnformattedWarning(Generic[_W]): + """A warning meant to be formatted during runtime. + + This is used to hold warnings that need to format their message at runtime, + as opposed to a direct message. + """ + + category: Type["_W"] + template: str + + def format(self, **kwargs: Any) -> _W: + """Return an instance of the warning category, formatted with given kwargs.""" + return self.category(self.template.format(**kwargs)) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/warnings.py b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/warnings.py new file mode 100644 index 0000000000..c0c946cbde --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/_pytest/warnings.py @@ -0,0 +1,141 @@ +import sys +import warnings +from contextlib import contextmanager +from typing import Generator +from typing import Optional +from typing import TYPE_CHECKING + +import pytest +from _pytest.config import apply_warning_filters +from _pytest.config import Config +from _pytest.config import parse_warning_filter +from _pytest.main import Session +from _pytest.nodes import Item +from _pytest.terminal import TerminalReporter + +if TYPE_CHECKING: + from typing_extensions import Literal + + +def pytest_configure(config: Config) -> None: + config.addinivalue_line( + "markers", + "filterwarnings(warning): add a warning filter to the given test. " + "see https://docs.pytest.org/en/stable/how-to/capture-warnings.html#pytest-mark-filterwarnings ", + ) + + +@contextmanager +def catch_warnings_for_item( + config: Config, + ihook, + when: "Literal['config', 'collect', 'runtest']", + item: Optional[Item], +) -> Generator[None, None, None]: + """Context manager that catches warnings generated in the contained execution block. + + ``item`` can be None if we are not in the context of an item execution. + + Each warning captured triggers the ``pytest_warning_recorded`` hook. + """ + config_filters = config.getini("filterwarnings") + cmdline_filters = config.known_args_namespace.pythonwarnings or [] + with warnings.catch_warnings(record=True) as log: + # mypy can't infer that record=True means log is not None; help it. + assert log is not None + + if not sys.warnoptions: + # If user is not explicitly configuring warning filters, show deprecation warnings by default (#2908). + warnings.filterwarnings("always", category=DeprecationWarning) + warnings.filterwarnings("always", category=PendingDeprecationWarning) + + warnings.filterwarnings("error", category=pytest.PytestRemovedIn7Warning) + + apply_warning_filters(config_filters, cmdline_filters) + + # apply filters from "filterwarnings" marks + nodeid = "" if item is None else item.nodeid + if item is not None: + for mark in item.iter_markers(name="filterwarnings"): + for arg in mark.args: + warnings.filterwarnings(*parse_warning_filter(arg, escape=False)) + + yield + + for warning_message in log: + ihook.pytest_warning_captured.call_historic( + kwargs=dict( + warning_message=warning_message, + when=when, + item=item, + location=None, + ) + ) + ihook.pytest_warning_recorded.call_historic( + kwargs=dict( + warning_message=warning_message, + nodeid=nodeid, + when=when, + location=None, + ) + ) + + +def warning_record_to_str(warning_message: warnings.WarningMessage) -> str: + """Convert a warnings.WarningMessage to a string.""" + warn_msg = warning_message.message + msg = warnings.formatwarning( + str(warn_msg), + warning_message.category, + warning_message.filename, + warning_message.lineno, + warning_message.line, + ) + return msg + + +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_runtest_protocol(item: Item) -> Generator[None, None, None]: + with catch_warnings_for_item( + config=item.config, ihook=item.ihook, when="runtest", item=item + ): + yield + + +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_collection(session: Session) -> Generator[None, None, None]: + config = session.config + with catch_warnings_for_item( + config=config, ihook=config.hook, when="collect", item=None + ): + yield + + +@pytest.hookimpl(hookwrapper=True) +def pytest_terminal_summary( + terminalreporter: TerminalReporter, +) -> Generator[None, None, None]: + config = terminalreporter.config + with catch_warnings_for_item( + config=config, ihook=config.hook, when="config", item=None + ): + yield + + +@pytest.hookimpl(hookwrapper=True) +def pytest_sessionfinish(session: Session) -> Generator[None, None, None]: + config = session.config + with catch_warnings_for_item( + config=config, ihook=config.hook, when="config", item=None + ): + yield + + +@pytest.hookimpl(hookwrapper=True) +def pytest_load_initial_conftests( + early_config: "Config", +) -> Generator[None, None, None]: + with catch_warnings_for_item( + config=early_config, ihook=early_config.hook, when="config", item=None + ): + yield diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/pytest/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/src/pytest/__init__.py new file mode 100644 index 0000000000..6050fd1124 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/pytest/__init__.py @@ -0,0 +1,171 @@ +# PYTHON_ARGCOMPLETE_OK +"""pytest: unit and functional testing with Python.""" +from . import collect +from _pytest import __version__ +from _pytest import version_tuple +from _pytest._code import ExceptionInfo +from _pytest.assertion import register_assert_rewrite +from _pytest.cacheprovider import Cache +from _pytest.capture import CaptureFixture +from _pytest.config import cmdline +from _pytest.config import Config +from _pytest.config import console_main +from _pytest.config import ExitCode +from _pytest.config import hookimpl +from _pytest.config import hookspec +from _pytest.config import main +from _pytest.config import PytestPluginManager +from _pytest.config import UsageError +from _pytest.config.argparsing import OptionGroup +from _pytest.config.argparsing import Parser +from _pytest.debugging import pytestPDB as __pytestPDB +from _pytest.fixtures import _fillfuncargs +from _pytest.fixtures import fixture +from _pytest.fixtures import FixtureLookupError +from _pytest.fixtures import FixtureRequest +from _pytest.fixtures import yield_fixture +from _pytest.freeze_support import freeze_includes +from _pytest.legacypath import TempdirFactory +from _pytest.legacypath import Testdir +from _pytest.logging import LogCaptureFixture +from _pytest.main import Session +from _pytest.mark import Mark +from _pytest.mark import MARK_GEN as mark +from _pytest.mark import MarkDecorator +from _pytest.mark import MarkGenerator +from _pytest.mark import param +from _pytest.monkeypatch import MonkeyPatch +from _pytest.nodes import Collector +from _pytest.nodes import File +from _pytest.nodes import Item +from _pytest.outcomes import exit +from _pytest.outcomes import fail +from _pytest.outcomes import importorskip +from _pytest.outcomes import skip +from _pytest.outcomes import xfail +from _pytest.pytester import HookRecorder +from _pytest.pytester import LineMatcher +from _pytest.pytester import Pytester +from _pytest.pytester import RecordedHookCall +from _pytest.pytester import RunResult +from _pytest.python import Class +from _pytest.python import Function +from _pytest.python import Metafunc +from _pytest.python import Module +from _pytest.python import Package +from _pytest.python_api import approx +from _pytest.python_api import raises +from _pytest.recwarn import deprecated_call +from _pytest.recwarn import WarningsRecorder +from _pytest.recwarn import warns +from _pytest.reports import CollectReport +from _pytest.reports import TestReport +from _pytest.runner import CallInfo +from _pytest.stash import Stash +from _pytest.stash import StashKey +from _pytest.tmpdir import TempPathFactory +from _pytest.warning_types import PytestAssertRewriteWarning +from _pytest.warning_types import PytestCacheWarning +from _pytest.warning_types import PytestCollectionWarning +from _pytest.warning_types import PytestConfigWarning +from _pytest.warning_types import PytestDeprecationWarning +from _pytest.warning_types import PytestExperimentalApiWarning +from _pytest.warning_types import PytestRemovedIn7Warning +from _pytest.warning_types import PytestRemovedIn8Warning +from _pytest.warning_types import PytestUnhandledCoroutineWarning +from _pytest.warning_types import PytestUnhandledThreadExceptionWarning +from _pytest.warning_types import PytestUnknownMarkWarning +from _pytest.warning_types import PytestUnraisableExceptionWarning +from _pytest.warning_types import PytestWarning + +set_trace = __pytestPDB.set_trace + + +__all__ = [ + "__version__", + "_fillfuncargs", + "approx", + "Cache", + "CallInfo", + "CaptureFixture", + "Class", + "cmdline", + "collect", + "Collector", + "CollectReport", + "Config", + "console_main", + "deprecated_call", + "exit", + "ExceptionInfo", + "ExitCode", + "fail", + "File", + "fixture", + "FixtureLookupError", + "FixtureRequest", + "freeze_includes", + "Function", + "hookimpl", + "HookRecorder", + "hookspec", + "importorskip", + "Item", + "LineMatcher", + "LogCaptureFixture", + "main", + "mark", + "Mark", + "MarkDecorator", + "MarkGenerator", + "Metafunc", + "Module", + "MonkeyPatch", + "OptionGroup", + "Package", + "param", + "Parser", + "PytestAssertRewriteWarning", + "PytestCacheWarning", + "PytestCollectionWarning", + "PytestConfigWarning", + "PytestDeprecationWarning", + "PytestExperimentalApiWarning", + "PytestRemovedIn7Warning", + "PytestRemovedIn8Warning", + "Pytester", + "PytestPluginManager", + "PytestUnhandledCoroutineWarning", + "PytestUnhandledThreadExceptionWarning", + "PytestUnknownMarkWarning", + "PytestUnraisableExceptionWarning", + "PytestWarning", + "raises", + "RecordedHookCall", + "register_assert_rewrite", + "RunResult", + "Session", + "set_trace", + "skip", + "Stash", + "StashKey", + "version_tuple", + "TempdirFactory", + "TempPathFactory", + "Testdir", + "TestReport", + "UsageError", + "WarningsRecorder", + "warns", + "xfail", + "yield_fixture", +] + + +def __getattr__(name: str) -> object: + if name == "Instance": + # The import emits a deprecation warning. + from _pytest.python import Instance + + return Instance + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/pytest/__main__.py b/testing/web-platform/tests/tools/third_party/pytest/src/pytest/__main__.py new file mode 100644 index 0000000000..b170152937 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/pytest/__main__.py @@ -0,0 +1,5 @@ +"""The pytest entry point.""" +import pytest + +if __name__ == "__main__": + raise SystemExit(pytest.console_main()) diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/pytest/collect.py b/testing/web-platform/tests/tools/third_party/pytest/src/pytest/collect.py new file mode 100644 index 0000000000..4b2b581806 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/src/pytest/collect.py @@ -0,0 +1,38 @@ +import sys +import warnings +from types import ModuleType +from typing import Any +from typing import List + +import pytest +from _pytest.deprecated import PYTEST_COLLECT_MODULE + +COLLECT_FAKEMODULE_ATTRIBUTES = [ + "Collector", + "Module", + "Function", + "Session", + "Item", + "Class", + "File", + "_fillfuncargs", +] + + +class FakeCollectModule(ModuleType): + def __init__(self) -> None: + super().__init__("pytest.collect") + self.__all__ = list(COLLECT_FAKEMODULE_ATTRIBUTES) + self.__pytest = pytest + + def __dir__(self) -> List[str]: + return dir(super()) + self.__all__ + + def __getattr__(self, name: str) -> Any: + if name not in self.__all__: + raise AttributeError(name) + warnings.warn(PYTEST_COLLECT_MODULE.format(name=name), stacklevel=2) + return getattr(pytest, name) + + +sys.modules["pytest.collect"] = FakeCollectModule() diff --git a/testing/web-platform/tests/tools/third_party/pytest/src/pytest/py.typed b/testing/web-platform/tests/tools/third_party/pytest/src/pytest/py.typed new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/acceptance_test.py b/testing/web-platform/tests/tools/third_party/pytest/testing/acceptance_test.py new file mode 100644 index 0000000000..8b8d4a4a6e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/acceptance_test.py @@ -0,0 +1,1297 @@ +import os +import sys +import types + +import attr + +import pytest +from _pytest.compat import importlib_metadata +from _pytest.config import ExitCode +from _pytest.pathlib import symlink_or_skip +from _pytest.pytester import Pytester + + +def prepend_pythonpath(*dirs) -> str: + cur = os.getenv("PYTHONPATH") + if cur: + dirs += (cur,) + return os.pathsep.join(str(p) for p in dirs) + + +class TestGeneralUsage: + def test_config_error(self, pytester: Pytester) -> None: + pytester.copy_example("conftest_usageerror/conftest.py") + result = pytester.runpytest(pytester.path) + assert result.ret == ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["*ERROR: hello"]) + result.stdout.fnmatch_lines(["*pytest_unconfigure_called"]) + + def test_root_conftest_syntax_error(self, pytester: Pytester) -> None: + pytester.makepyfile(conftest="raise SyntaxError\n") + result = pytester.runpytest() + result.stderr.fnmatch_lines(["*raise SyntaxError*"]) + assert result.ret != 0 + + def test_early_hook_error_issue38_1(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_sessionstart(): + 0 / 0 + """ + ) + result = pytester.runpytest(pytester.path) + assert result.ret != 0 + # tracestyle is native by default for hook failures + result.stdout.fnmatch_lines( + ["*INTERNALERROR*File*conftest.py*line 2*", "*0 / 0*"] + ) + result = pytester.runpytest(pytester.path, "--fulltrace") + assert result.ret != 0 + # tracestyle is native by default for hook failures + result.stdout.fnmatch_lines( + ["*INTERNALERROR*def pytest_sessionstart():*", "*INTERNALERROR*0 / 0*"] + ) + + def test_early_hook_configure_error_issue38(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_configure(): + 0 / 0 + """ + ) + result = pytester.runpytest(pytester.path) + assert result.ret != 0 + # here we get it on stderr + result.stderr.fnmatch_lines( + ["*INTERNALERROR*File*conftest.py*line 2*", "*0 / 0*"] + ) + + def test_file_not_found(self, pytester: Pytester) -> None: + result = pytester.runpytest("asd") + assert result.ret != 0 + result.stderr.fnmatch_lines(["ERROR: file or directory not found: asd"]) + + def test_file_not_found_unconfigure_issue143(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_configure(): + print("---configure") + def pytest_unconfigure(): + print("---unconfigure") + """ + ) + result = pytester.runpytest("-s", "asd") + assert result.ret == ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines(["ERROR: file or directory not found: asd"]) + result.stdout.fnmatch_lines(["*---configure", "*---unconfigure"]) + + def test_config_preparse_plugin_option(self, pytester: Pytester) -> None: + pytester.makepyfile( + pytest_xyz=""" + def pytest_addoption(parser): + parser.addoption("--xyz", dest="xyz", action="store") + """ + ) + pytester.makepyfile( + test_one=""" + def test_option(pytestconfig): + assert pytestconfig.option.xyz == "123" + """ + ) + result = pytester.runpytest("-p", "pytest_xyz", "--xyz=123", syspathinsert=True) + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed*"]) + + @pytest.mark.parametrize("load_cov_early", [True, False]) + def test_early_load_setuptools_name( + self, pytester: Pytester, monkeypatch, load_cov_early + ) -> None: + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + + pytester.makepyfile(mytestplugin1_module="") + pytester.makepyfile(mytestplugin2_module="") + pytester.makepyfile(mycov_module="") + pytester.syspathinsert() + + loaded = [] + + @attr.s + class DummyEntryPoint: + name = attr.ib() + module = attr.ib() + group = "pytest11" + + def load(self): + __import__(self.module) + loaded.append(self.name) + return sys.modules[self.module] + + entry_points = [ + DummyEntryPoint("myplugin1", "mytestplugin1_module"), + DummyEntryPoint("myplugin2", "mytestplugin2_module"), + DummyEntryPoint("mycov", "mycov_module"), + ] + + @attr.s + class DummyDist: + entry_points = attr.ib() + files = () + + def my_dists(): + return (DummyDist(entry_points),) + + monkeypatch.setattr(importlib_metadata, "distributions", my_dists) + params = ("-p", "mycov") if load_cov_early else () + pytester.runpytest_inprocess(*params) + if load_cov_early: + assert loaded == ["mycov", "myplugin1", "myplugin2"] + else: + assert loaded == ["myplugin1", "myplugin2", "mycov"] + + @pytest.mark.parametrize("import_mode", ["prepend", "append", "importlib"]) + def test_assertion_rewrite(self, pytester: Pytester, import_mode) -> None: + p = pytester.makepyfile( + """ + def test_this(): + x = 0 + assert x + """ + ) + result = pytester.runpytest(p, f"--import-mode={import_mode}") + result.stdout.fnmatch_lines(["> assert x", "E assert 0"]) + assert result.ret == 1 + + def test_nested_import_error(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import import_fails + def test_this(): + assert import_fails.a == 1 + """ + ) + pytester.makepyfile(import_fails="import does_not_work") + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + [ + "ImportError while importing test module*", + "*No module named *does_not_work*", + ] + ) + assert result.ret == 2 + + def test_not_collectable_arguments(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile("") + p2 = pytester.makefile(".pyc", "123") + result = pytester.runpytest(p1, p2) + assert result.ret == ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines( + [ + f"ERROR: not found: {p2}", + f"(no name {str(p2)!r} in any of [[][]])", + "", + ] + ) + + @pytest.mark.filterwarnings("default") + def test_better_reporting_on_conftest_load_failure( + self, pytester: Pytester + ) -> None: + """Show a user-friendly traceback on conftest import failures (#486, #3332)""" + pytester.makepyfile("") + conftest = pytester.makeconftest( + """ + def foo(): + import qwerty + foo() + """ + ) + result = pytester.runpytest("--help") + result.stdout.fnmatch_lines( + """ + *--version* + *warning*conftest.py* + """ + ) + result = pytester.runpytest() + assert result.stdout.lines == [] + assert result.stderr.lines == [ + f"ImportError while loading conftest '{conftest}'.", + "conftest.py:3: in ", + " foo()", + "conftest.py:2: in foo", + " import qwerty", + "E ModuleNotFoundError: No module named 'qwerty'", + ] + + def test_early_skip(self, pytester: Pytester) -> None: + pytester.mkdir("xyz") + pytester.makeconftest( + """ + import pytest + def pytest_collect_file(): + pytest.skip("early") + """ + ) + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + result.stdout.fnmatch_lines(["*1 skip*"]) + + def test_issue88_initial_file_multinodes(self, pytester: Pytester) -> None: + pytester.copy_example("issue88_initial_file_multinodes") + p = pytester.makepyfile("def test_hello(): pass") + result = pytester.runpytest(p, "--collect-only") + result.stdout.fnmatch_lines(["*MyFile*test_issue88*", "*Module*test_issue88*"]) + + def test_issue93_initialnode_importing_capturing(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import sys + print("should not be seen") + sys.stderr.write("stder42\\n") + """ + ) + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + result.stdout.no_fnmatch_line("*should not be seen*") + assert "stderr42" not in result.stderr.str() + + def test_conftest_printing_shows_if_error(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + print("should be seen") + assert 0 + """ + ) + result = pytester.runpytest() + assert result.ret != 0 + assert "should be seen" in result.stdout.str() + + def test_issue109_sibling_conftests_not_loaded(self, pytester: Pytester) -> None: + sub1 = pytester.mkdir("sub1") + sub2 = pytester.mkdir("sub2") + sub1.joinpath("conftest.py").write_text("assert 0") + result = pytester.runpytest(sub2) + assert result.ret == ExitCode.NO_TESTS_COLLECTED + sub2.joinpath("__init__.py").touch() + p = sub2.joinpath("test_hello.py") + p.touch() + result = pytester.runpytest(p) + assert result.ret == ExitCode.NO_TESTS_COLLECTED + result = pytester.runpytest(sub1) + assert result.ret == ExitCode.USAGE_ERROR + + def test_directory_skipped(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + def pytest_ignore_collect(): + pytest.skip("intentional") + """ + ) + pytester.makepyfile("def test_hello(): pass") + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + result.stdout.fnmatch_lines(["*1 skipped*"]) + + def test_multiple_items_per_collector_byid(self, pytester: Pytester) -> None: + c = pytester.makeconftest( + """ + import pytest + class MyItem(pytest.Item): + def runtest(self): + pass + class MyCollector(pytest.File): + def collect(self): + return [MyItem.from_parent(name="xyz", parent=self)] + def pytest_collect_file(file_path, parent): + if file_path.name.startswith("conftest"): + return MyCollector.from_parent(path=file_path, parent=parent) + """ + ) + result = pytester.runpytest(c.name + "::" + "xyz") + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 pass*"]) + + def test_skip_on_generated_funcarg_id(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + def pytest_generate_tests(metafunc): + metafunc.parametrize('x', [3], ids=['hello-123']) + def pytest_runtest_setup(item): + print(item.keywords) + if 'hello-123' in item.keywords: + pytest.skip("hello") + assert 0 + """ + ) + p = pytester.makepyfile("""def test_func(x): pass""") + res = pytester.runpytest(p) + assert res.ret == 0 + res.stdout.fnmatch_lines(["*1 skipped*"]) + + def test_direct_addressing_selects(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def pytest_generate_tests(metafunc): + metafunc.parametrize('i', [1, 2], ids=["1", "2"]) + def test_func(i): + pass + """ + ) + res = pytester.runpytest(p.name + "::" + "test_func[1]") + assert res.ret == 0 + res.stdout.fnmatch_lines(["*1 passed*"]) + + def test_direct_addressing_notfound(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def test_func(): + pass + """ + ) + res = pytester.runpytest(p.name + "::" + "test_notfound") + assert res.ret + res.stderr.fnmatch_lines(["*ERROR*not found*"]) + + def test_docstring_on_hookspec(self) -> None: + from _pytest import hookspec + + for name, value in vars(hookspec).items(): + if name.startswith("pytest_"): + assert value.__doc__, "no docstring for %s" % name + + def test_initialization_error_issue49(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_configure(): + x + """ + ) + result = pytester.runpytest() + assert result.ret == 3 # internal error + result.stderr.fnmatch_lines(["INTERNAL*pytest_configure*", "INTERNAL*x*"]) + assert "sessionstarttime" not in result.stderr.str() + + @pytest.mark.parametrize("lookfor", ["test_fun.py::test_a"]) + def test_issue134_report_error_when_collecting_member( + self, pytester: Pytester, lookfor + ) -> None: + pytester.makepyfile( + test_fun=""" + def test_a(): + pass + def""" + ) + result = pytester.runpytest(lookfor) + result.stdout.fnmatch_lines(["*SyntaxError*"]) + if "::" in lookfor: + result.stderr.fnmatch_lines(["*ERROR*"]) + assert result.ret == 4 # usage error only if item not found + + def test_report_all_failed_collections_initargs(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + from _pytest.config import ExitCode + + def pytest_sessionfinish(exitstatus): + assert exitstatus == ExitCode.USAGE_ERROR + print("pytest_sessionfinish_called") + """ + ) + pytester.makepyfile(test_a="def", test_b="def") + result = pytester.runpytest("test_a.py::a", "test_b.py::b") + result.stderr.fnmatch_lines(["*ERROR*test_a.py::a*", "*ERROR*test_b.py::b*"]) + result.stdout.fnmatch_lines(["pytest_sessionfinish_called"]) + assert result.ret == ExitCode.USAGE_ERROR + + def test_namespace_import_doesnt_confuse_import_hook( + self, pytester: Pytester + ) -> None: + """Ref #383. + + Python 3.3's namespace package messed with our import hooks. + Importing a module that didn't exist, even if the ImportError was + gracefully handled, would make our test crash. + """ + pytester.mkdir("not_a_package") + p = pytester.makepyfile( + """ + try: + from not_a_package import doesnt_exist + except ImportError: + # We handle the import error gracefully here + pass + + def test_whatever(): + pass + """ + ) + res = pytester.runpytest(p.name) + assert res.ret == 0 + + def test_unknown_option(self, pytester: Pytester) -> None: + result = pytester.runpytest("--qwlkej") + result.stderr.fnmatch_lines( + """ + *unrecognized* + """ + ) + + def test_getsourcelines_error_issue553( + self, pytester: Pytester, monkeypatch + ) -> None: + monkeypatch.setattr("inspect.getsourcelines", None) + p = pytester.makepyfile( + """ + def raise_error(obj): + raise OSError('source code not available') + + import inspect + inspect.getsourcelines = raise_error + + def test_foo(invalid_fixture): + pass + """ + ) + res = pytester.runpytest(p) + res.stdout.fnmatch_lines( + ["*source code not available*", "E*fixture 'invalid_fixture' not found"] + ) + + def test_plugins_given_as_strings( + self, pytester: Pytester, monkeypatch, _sys_snapshot + ) -> None: + """Test that str values passed to main() as `plugins` arg are + interpreted as module names to be imported and registered (#855).""" + with pytest.raises(ImportError) as excinfo: + pytest.main([str(pytester.path)], plugins=["invalid.module"]) + assert "invalid" in str(excinfo.value) + + p = pytester.path.joinpath("test_test_plugins_given_as_strings.py") + p.write_text("def test_foo(): pass") + mod = types.ModuleType("myplugin") + monkeypatch.setitem(sys.modules, "myplugin", mod) + assert pytest.main(args=[str(pytester.path)], plugins=["myplugin"]) == 0 + + def test_parametrized_with_bytes_regex(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import re + import pytest + @pytest.mark.parametrize('r', [re.compile(b'foo')]) + def test_stuff(r): + pass + """ + ) + res = pytester.runpytest(p) + res.stdout.fnmatch_lines(["*1 passed*"]) + + def test_parametrized_with_null_bytes(self, pytester: Pytester) -> None: + """Test parametrization with values that contain null bytes and unicode characters (#2644, #2957)""" + p = pytester.makepyfile( + """\ + import pytest + + @pytest.mark.parametrize("data", [b"\\x00", "\\x00", 'ação']) + def test_foo(data): + assert data + """ + ) + res = pytester.runpytest(p) + res.assert_outcomes(passed=3) + + +class TestInvocationVariants: + def test_earlyinit(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + assert hasattr(pytest, 'mark') + """ + ) + result = pytester.runpython(p) + assert result.ret == 0 + + def test_pydoc(self, pytester: Pytester) -> None: + result = pytester.runpython_c("import pytest;help(pytest)") + assert result.ret == 0 + s = result.stdout.str() + assert "MarkGenerator" in s + + def test_import_star_pytest(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + from pytest import * + #Item + #File + main + skip + xfail + """ + ) + result = pytester.runpython(p) + assert result.ret == 0 + + def test_double_pytestcmdline(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + run=""" + import pytest + pytest.main() + pytest.main() + """ + ) + pytester.makepyfile( + """ + def test_hello(): + pass + """ + ) + result = pytester.runpython(p) + result.stdout.fnmatch_lines(["*1 passed*", "*1 passed*"]) + + def test_python_minus_m_invocation_ok(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile("def test_hello(): pass") + res = pytester.run(sys.executable, "-m", "pytest", str(p1)) + assert res.ret == 0 + + def test_python_minus_m_invocation_fail(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile("def test_fail(): 0/0") + res = pytester.run(sys.executable, "-m", "pytest", str(p1)) + assert res.ret == 1 + + def test_python_pytest_package(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile("def test_pass(): pass") + res = pytester.run(sys.executable, "-m", "pytest", str(p1)) + assert res.ret == 0 + res.stdout.fnmatch_lines(["*1 passed*"]) + + def test_invoke_with_invalid_type(self) -> None: + with pytest.raises( + TypeError, match="expected to be a list of strings, got: '-h'" + ): + pytest.main("-h") # type: ignore[arg-type] + + def test_invoke_with_path(self, pytester: Pytester, capsys) -> None: + retcode = pytest.main([str(pytester.path)]) + assert retcode == ExitCode.NO_TESTS_COLLECTED + out, err = capsys.readouterr() + + def test_invoke_plugin_api(self, capsys) -> None: + class MyPlugin: + def pytest_addoption(self, parser): + parser.addoption("--myopt") + + pytest.main(["-h"], plugins=[MyPlugin()]) + out, err = capsys.readouterr() + assert "--myopt" in out + + def test_pyargs_importerror(self, pytester: Pytester, monkeypatch) -> None: + monkeypatch.delenv("PYTHONDONTWRITEBYTECODE", False) + path = pytester.mkpydir("tpkg") + path.joinpath("test_hello.py").write_text("raise ImportError") + + result = pytester.runpytest("--pyargs", "tpkg.test_hello", syspathinsert=True) + assert result.ret != 0 + + result.stdout.fnmatch_lines(["collected*0*items*/*1*error"]) + + def test_pyargs_only_imported_once(self, pytester: Pytester) -> None: + pkg = pytester.mkpydir("foo") + pkg.joinpath("test_foo.py").write_text( + "print('hello from test_foo')\ndef test(): pass" + ) + pkg.joinpath("conftest.py").write_text( + "def pytest_configure(config): print('configuring')" + ) + + result = pytester.runpytest( + "--pyargs", "foo.test_foo", "-s", syspathinsert=True + ) + # should only import once + assert result.outlines.count("hello from test_foo") == 1 + # should only configure once + assert result.outlines.count("configuring") == 1 + + def test_pyargs_filename_looks_like_module(self, pytester: Pytester) -> None: + pytester.path.joinpath("conftest.py").touch() + pytester.path.joinpath("t.py").write_text("def test(): pass") + result = pytester.runpytest("--pyargs", "t.py") + assert result.ret == ExitCode.OK + + def test_cmdline_python_package(self, pytester: Pytester, monkeypatch) -> None: + import warnings + + monkeypatch.delenv("PYTHONDONTWRITEBYTECODE", False) + path = pytester.mkpydir("tpkg") + path.joinpath("test_hello.py").write_text("def test_hello(): pass") + path.joinpath("test_world.py").write_text("def test_world(): pass") + result = pytester.runpytest("--pyargs", "tpkg") + assert result.ret == 0 + result.stdout.fnmatch_lines(["*2 passed*"]) + result = pytester.runpytest("--pyargs", "tpkg.test_hello", syspathinsert=True) + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed*"]) + + empty_package = pytester.mkpydir("empty_package") + monkeypatch.setenv("PYTHONPATH", str(empty_package), prepend=os.pathsep) + # the path which is not a package raises a warning on pypy; + # no idea why only pypy and not normal python warn about it here + with warnings.catch_warnings(): + warnings.simplefilter("ignore", ImportWarning) + result = pytester.runpytest("--pyargs", ".") + assert result.ret == 0 + result.stdout.fnmatch_lines(["*2 passed*"]) + + monkeypatch.setenv("PYTHONPATH", str(pytester), prepend=os.pathsep) + result = pytester.runpytest("--pyargs", "tpkg.test_missing", syspathinsert=True) + assert result.ret != 0 + result.stderr.fnmatch_lines(["*not*found*test_missing*"]) + + def test_cmdline_python_namespace_package( + self, pytester: Pytester, monkeypatch + ) -> None: + """Test --pyargs option with namespace packages (#1567). + + Ref: https://packaging.python.org/guides/packaging-namespace-packages/ + """ + monkeypatch.delenv("PYTHONDONTWRITEBYTECODE", raising=False) + + search_path = [] + for dirname in "hello", "world": + d = pytester.mkdir(dirname) + search_path.append(d) + ns = d.joinpath("ns_pkg") + ns.mkdir() + ns.joinpath("__init__.py").write_text( + "__import__('pkg_resources').declare_namespace(__name__)" + ) + lib = ns.joinpath(dirname) + lib.mkdir() + lib.joinpath("__init__.py").touch() + lib.joinpath(f"test_{dirname}.py").write_text( + f"def test_{dirname}(): pass\ndef test_other():pass" + ) + + # The structure of the test directory is now: + # . + # ├── hello + # │ └── ns_pkg + # │ ├── __init__.py + # │ └── hello + # │ ├── __init__.py + # │ └── test_hello.py + # └── world + # └── ns_pkg + # ├── __init__.py + # └── world + # ├── __init__.py + # └── test_world.py + + # NOTE: the different/reversed ordering is intentional here. + monkeypatch.setenv("PYTHONPATH", prepend_pythonpath(*search_path)) + for p in search_path: + monkeypatch.syspath_prepend(p) + + # mixed module and filenames: + monkeypatch.chdir("world") + result = pytester.runpytest("--pyargs", "-v", "ns_pkg.hello", "ns_pkg/world") + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + "test_hello.py::test_hello*PASSED*", + "test_hello.py::test_other*PASSED*", + "ns_pkg/world/test_world.py::test_world*PASSED*", + "ns_pkg/world/test_world.py::test_other*PASSED*", + "*4 passed in*", + ] + ) + + # specify tests within a module + pytester.chdir() + result = pytester.runpytest( + "--pyargs", "-v", "ns_pkg.world.test_world::test_other" + ) + assert result.ret == 0 + result.stdout.fnmatch_lines( + ["*test_world.py::test_other*PASSED*", "*1 passed*"] + ) + + def test_invoke_test_and_doctestmodules(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def test(): + pass + """ + ) + result = pytester.runpytest(str(p) + "::test", "--doctest-modules") + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_cmdline_python_package_symlink( + self, pytester: Pytester, monkeypatch + ) -> None: + """ + --pyargs with packages with path containing symlink can have conftest.py in + their package (#2985) + """ + monkeypatch.delenv("PYTHONDONTWRITEBYTECODE", raising=False) + + dirname = "lib" + d = pytester.mkdir(dirname) + foo = d.joinpath("foo") + foo.mkdir() + foo.joinpath("__init__.py").touch() + lib = foo.joinpath("bar") + lib.mkdir() + lib.joinpath("__init__.py").touch() + lib.joinpath("test_bar.py").write_text( + "def test_bar(): pass\ndef test_other(a_fixture):pass" + ) + lib.joinpath("conftest.py").write_text( + "import pytest\n@pytest.fixture\ndef a_fixture():pass" + ) + + d_local = pytester.mkdir("symlink_root") + symlink_location = d_local / "lib" + symlink_or_skip(d, symlink_location, target_is_directory=True) + + # The structure of the test directory is now: + # . + # ├── symlink_root + # │ └── lib -> ../lib + # └── lib + # └── foo + # ├── __init__.py + # └── bar + # ├── __init__.py + # ├── conftest.py + # └── test_bar.py + + # NOTE: the different/reversed ordering is intentional here. + search_path = ["lib", os.path.join("symlink_root", "lib")] + monkeypatch.setenv("PYTHONPATH", prepend_pythonpath(*search_path)) + for p in search_path: + monkeypatch.syspath_prepend(p) + + # module picked up in symlink-ed directory: + # It picks up symlink_root/lib/foo/bar (symlink) via sys.path. + result = pytester.runpytest("--pyargs", "-v", "foo.bar") + pytester.chdir() + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + "symlink_root/lib/foo/bar/test_bar.py::test_bar PASSED*", + "symlink_root/lib/foo/bar/test_bar.py::test_other PASSED*", + "*2 passed*", + ] + ) + + def test_cmdline_python_package_not_exists(self, pytester: Pytester) -> None: + result = pytester.runpytest("--pyargs", "tpkgwhatv") + assert result.ret + result.stderr.fnmatch_lines(["ERROR*module*or*package*not*found*"]) + + @pytest.mark.xfail(reason="decide: feature or bug") + def test_noclass_discovery_if_not_testcase(self, pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + class TestHello(object): + def test_hello(self): + assert self.attr + + class RealTest(unittest.TestCase, TestHello): + attr = 42 + """ + ) + reprec = pytester.inline_run(testpath) + reprec.assertoutcome(passed=1) + + def test_doctest_id(self, pytester: Pytester) -> None: + pytester.makefile( + ".txt", + """ + >>> x=3 + >>> x + 4 + """, + ) + testid = "test_doctest_id.txt::test_doctest_id.txt" + expected_lines = [ + "*= FAILURES =*", + "*_ ?doctest? test_doctest_id.txt _*", + "FAILED test_doctest_id.txt::test_doctest_id.txt", + "*= 1 failed in*", + ] + result = pytester.runpytest(testid, "-rf", "--tb=short") + result.stdout.fnmatch_lines(expected_lines) + + # Ensure that re-running it will still handle it as + # doctest.DocTestFailure, which was not the case before when + # re-importing doctest, but not creating a new RUNNER_CLASS. + result = pytester.runpytest(testid, "-rf", "--tb=short") + result.stdout.fnmatch_lines(expected_lines) + + def test_core_backward_compatibility(self) -> None: + """Test backward compatibility for get_plugin_manager function. See #787.""" + import _pytest.config + + assert ( + type(_pytest.config.get_plugin_manager()) + is _pytest.config.PytestPluginManager + ) + + def test_has_plugin(self, request) -> None: + """Test hasplugin function of the plugin manager (#932).""" + assert request.config.pluginmanager.hasplugin("python") + + +class TestDurations: + source = """ + from _pytest import timing + def test_something(): + pass + def test_2(): + timing.sleep(0.010) + def test_1(): + timing.sleep(0.002) + def test_3(): + timing.sleep(0.020) + """ + + def test_calls(self, pytester: Pytester, mock_timing) -> None: + pytester.makepyfile(self.source) + result = pytester.runpytest_inprocess("--durations=10") + assert result.ret == 0 + + result.stdout.fnmatch_lines_random( + ["*durations*", "*call*test_3*", "*call*test_2*"] + ) + + result.stdout.fnmatch_lines( + ["(8 durations < 0.005s hidden. Use -vv to show these durations.)"] + ) + + def test_calls_show_2(self, pytester: Pytester, mock_timing) -> None: + + pytester.makepyfile(self.source) + result = pytester.runpytest_inprocess("--durations=2") + assert result.ret == 0 + + lines = result.stdout.get_lines_after("*slowest*durations*") + assert "4 passed" in lines[2] + + def test_calls_showall(self, pytester: Pytester, mock_timing) -> None: + pytester.makepyfile(self.source) + result = pytester.runpytest_inprocess("--durations=0") + assert result.ret == 0 + + tested = "3" + for x in tested: + for y in ("call",): # 'setup', 'call', 'teardown': + for line in result.stdout.lines: + if ("test_%s" % x) in line and y in line: + break + else: + raise AssertionError(f"not found {x} {y}") + + def test_calls_showall_verbose(self, pytester: Pytester, mock_timing) -> None: + pytester.makepyfile(self.source) + result = pytester.runpytest_inprocess("--durations=0", "-vv") + assert result.ret == 0 + + for x in "123": + for y in ("call",): # 'setup', 'call', 'teardown': + for line in result.stdout.lines: + if ("test_%s" % x) in line and y in line: + break + else: + raise AssertionError(f"not found {x} {y}") + + def test_with_deselected(self, pytester: Pytester, mock_timing) -> None: + pytester.makepyfile(self.source) + result = pytester.runpytest_inprocess("--durations=2", "-k test_3") + assert result.ret == 0 + + result.stdout.fnmatch_lines(["*durations*", "*call*test_3*"]) + + def test_with_failing_collection(self, pytester: Pytester, mock_timing) -> None: + pytester.makepyfile(self.source) + pytester.makepyfile(test_collecterror="""xyz""") + result = pytester.runpytest_inprocess("--durations=2", "-k test_1") + assert result.ret == 2 + + result.stdout.fnmatch_lines(["*Interrupted: 1 error during collection*"]) + # Collection errors abort test execution, therefore no duration is + # output + result.stdout.no_fnmatch_line("*duration*") + + def test_with_not(self, pytester: Pytester, mock_timing) -> None: + pytester.makepyfile(self.source) + result = pytester.runpytest_inprocess("-k not 1") + assert result.ret == 0 + + +class TestDurationsWithFixture: + source = """ + import pytest + from _pytest import timing + + @pytest.fixture + def setup_fixt(): + timing.sleep(2) + + def test_1(setup_fixt): + timing.sleep(5) + """ + + def test_setup_function(self, pytester: Pytester, mock_timing) -> None: + pytester.makepyfile(self.source) + result = pytester.runpytest_inprocess("--durations=10") + assert result.ret == 0 + + result.stdout.fnmatch_lines_random( + """ + *durations* + 5.00s call *test_1* + 2.00s setup *test_1* + """ + ) + + +def test_zipimport_hook(pytester: Pytester) -> None: + """Test package loader is being used correctly (see #1837).""" + zipapp = pytest.importorskip("zipapp") + pytester.path.joinpath("app").mkdir() + pytester.makepyfile( + **{ + "app/foo.py": """ + import pytest + def main(): + pytest.main(['--pyargs', 'foo']) + """ + } + ) + target = pytester.path.joinpath("foo.zip") + zipapp.create_archive( + str(pytester.path.joinpath("app")), str(target), main="foo:main" + ) + result = pytester.runpython(target) + assert result.ret == 0 + result.stderr.fnmatch_lines(["*not found*foo*"]) + result.stdout.no_fnmatch_line("*INTERNALERROR>*") + + +def test_import_plugin_unicode_name(pytester: Pytester) -> None: + pytester.makepyfile(myplugin="") + pytester.makepyfile("def test(): pass") + pytester.makeconftest("pytest_plugins = ['myplugin']") + r = pytester.runpytest() + assert r.ret == 0 + + +def test_pytest_plugins_as_module(pytester: Pytester) -> None: + """Do not raise an error if pytest_plugins attribute is a module (#3899)""" + pytester.makepyfile( + **{ + "__init__.py": "", + "pytest_plugins.py": "", + "conftest.py": "from . import pytest_plugins", + "test_foo.py": "def test(): pass", + } + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 1 passed in *"]) + + +def test_deferred_hook_checking(pytester: Pytester) -> None: + """Check hooks as late as possible (#1821).""" + pytester.syspathinsert() + pytester.makepyfile( + **{ + "plugin.py": """ + class Hooks(object): + def pytest_my_hook(self, config): + pass + + def pytest_configure(config): + config.pluginmanager.add_hookspecs(Hooks) + """, + "conftest.py": """ + pytest_plugins = ['plugin'] + def pytest_my_hook(config): + return 40 + """, + "test_foo.py": """ + def test(request): + assert request.config.hook.pytest_my_hook(config=request.config) == [40] + """, + } + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 1 passed *"]) + + +def test_fixture_values_leak(pytester: Pytester) -> None: + """Ensure that fixture objects are properly destroyed by the garbage collector at the end of their expected + life-times (#2981). + """ + pytester.makepyfile( + """ + import attr + import gc + import pytest + import weakref + + @attr.s + class SomeObj(object): + name = attr.ib() + + fix_of_test1_ref = None + session_ref = None + + @pytest.fixture(scope='session') + def session_fix(): + global session_ref + obj = SomeObj(name='session-fixture') + session_ref = weakref.ref(obj) + return obj + + @pytest.fixture + def fix(session_fix): + global fix_of_test1_ref + obj = SomeObj(name='local-fixture') + fix_of_test1_ref = weakref.ref(obj) + return obj + + def test1(fix): + assert fix_of_test1_ref() is fix + + def test2(): + gc.collect() + # fixture "fix" created during test1 must have been destroyed by now + assert fix_of_test1_ref() is None + """ + ) + # Running on subprocess does not activate the HookRecorder + # which holds itself a reference to objects in case of the + # pytest_assert_reprcompare hook + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines(["* 2 passed *"]) + + +def test_fixture_order_respects_scope(pytester: Pytester) -> None: + """Ensure that fixtures are created according to scope order (#2405).""" + pytester.makepyfile( + """ + import pytest + + data = {} + + @pytest.fixture(scope='module') + def clean_data(): + data.clear() + + @pytest.fixture(autouse=True) + def add_data(): + data.update(value=True) + + @pytest.mark.usefixtures('clean_data') + def test_value(): + assert data.get('value') + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + + +def test_frame_leak_on_failing_test(pytester: Pytester) -> None: + """Pytest would leak garbage referencing the frames of tests that failed + that could never be reclaimed (#2798). + + Unfortunately it was not possible to remove the actual circles because most of them + are made of traceback objects which cannot be weakly referenced. Those objects at least + can be eventually claimed by the garbage collector. + """ + pytester.makepyfile( + """ + import gc + import weakref + + class Obj: + pass + + ref = None + + def test1(): + obj = Obj() + global ref + ref = weakref.ref(obj) + assert 0 + + def test2(): + gc.collect() + assert ref() is None + """ + ) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines(["*1 failed, 1 passed in*"]) + + +def test_fixture_mock_integration(pytester: Pytester) -> None: + """Test that decorators applied to fixture are left working (#3774)""" + p = pytester.copy_example("acceptance/fixture_mock_integration.py") + result = pytester.runpytest(p) + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_usage_error_code(pytester: Pytester) -> None: + result = pytester.runpytest("-unknown-option-") + assert result.ret == ExitCode.USAGE_ERROR + + +@pytest.mark.filterwarnings("default::pytest.PytestUnhandledCoroutineWarning") +def test_warn_on_async_function(pytester: Pytester) -> None: + # In the below we .close() the coroutine only to avoid + # "RuntimeWarning: coroutine 'test_2' was never awaited" + # which messes with other tests. + pytester.makepyfile( + test_async=""" + async def test_1(): + pass + async def test_2(): + pass + def test_3(): + coro = test_2() + coro.close() + return coro + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "test_async.py::test_1", + "test_async.py::test_2", + "test_async.py::test_3", + "*async def functions are not natively supported*", + "*3 skipped, 3 warnings in*", + ] + ) + # ensure our warning message appears only once + assert ( + result.stdout.str().count("async def functions are not natively supported") == 1 + ) + + +@pytest.mark.filterwarnings("default::pytest.PytestUnhandledCoroutineWarning") +def test_warn_on_async_gen_function(pytester: Pytester) -> None: + pytester.makepyfile( + test_async=""" + async def test_1(): + yield + async def test_2(): + yield + def test_3(): + return test_2() + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "test_async.py::test_1", + "test_async.py::test_2", + "test_async.py::test_3", + "*async def functions are not natively supported*", + "*3 skipped, 3 warnings in*", + ] + ) + # ensure our warning message appears only once + assert ( + result.stdout.str().count("async def functions are not natively supported") == 1 + ) + + +def test_pdb_can_be_rewritten(pytester: Pytester) -> None: + pytester.makepyfile( + **{ + "conftest.py": """ + import pytest + pytest.register_assert_rewrite("pdb") + """, + "__init__.py": "", + "pdb.py": """ + def check(): + assert 1 == 2 + """, + "test_pdb.py": """ + def test(): + import pdb + assert pdb.check() + """, + } + ) + # Disable debugging plugin itself to avoid: + # > INTERNALERROR> AttributeError: module 'pdb' has no attribute 'set_trace' + result = pytester.runpytest_subprocess("-p", "no:debugging", "-vv") + result.stdout.fnmatch_lines( + [ + " def check():", + "> assert 1 == 2", + "E assert 1 == 2", + "E +1", + "E -2", + "", + "pdb.py:2: AssertionError", + "*= 1 failed in *", + ] + ) + assert result.ret == 1 + + +def test_tee_stdio_captures_and_live_prints(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import sys + def test_simple(): + print ("@this is stdout@") + print ("@this is stderr@", file=sys.stderr) + """ + ) + result = pytester.runpytest_subprocess( + testpath, + "--capture=tee-sys", + "--junitxml=output.xml", + "-o", + "junit_logging=all", + ) + + # ensure stdout/stderr were 'live printed' + result.stdout.fnmatch_lines(["*@this is stdout@*"]) + result.stderr.fnmatch_lines(["*@this is stderr@*"]) + + # now ensure the output is in the junitxml + with open(pytester.path.joinpath("output.xml")) as f: + fullXml = f.read() + assert "@this is stdout@\n" in fullXml + assert "@this is stderr@\n" in fullXml + + +@pytest.mark.skipif( + sys.platform == "win32", + reason="Windows raises `OSError: [Errno 22] Invalid argument` instead", +) +def test_no_brokenpipeerror_message(pytester: Pytester) -> None: + """Ensure that the broken pipe error message is suppressed. + + In some Python versions, it reaches sys.unraisablehook, in others + a BrokenPipeError exception is propagated, but either way it prints + to stderr on shutdown, so checking nothing is printed is enough. + """ + popen = pytester.popen((*pytester._getpytestargs(), "--help")) + popen.stdout.close() + ret = popen.wait() + assert popen.stderr.read() == b"" + assert ret == 1 + + # Cleanup. + popen.stderr.close() diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/code/test_code.py b/testing/web-platform/tests/tools/third_party/pytest/testing/code/test_code.py new file mode 100644 index 0000000000..33809528a0 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/code/test_code.py @@ -0,0 +1,212 @@ +import re +import sys +from types import FrameType +from unittest import mock + +import pytest +from _pytest._code import Code +from _pytest._code import ExceptionInfo +from _pytest._code import Frame +from _pytest._code import Source +from _pytest._code.code import ExceptionChainRepr +from _pytest._code.code import ReprFuncArgs + + +def test_ne() -> None: + code1 = Code(compile('foo = "bar"', "", "exec")) + assert code1 == code1 + code2 = Code(compile('foo = "baz"', "", "exec")) + assert code2 != code1 + + +def test_code_gives_back_name_for_not_existing_file() -> None: + name = "abc-123" + co_code = compile("pass\n", name, "exec") + assert co_code.co_filename == name + code = Code(co_code) + assert str(code.path) == name + assert code.fullsource is None + + +def test_code_from_function_with_class() -> None: + class A: + pass + + with pytest.raises(TypeError): + Code.from_function(A) + + +def x() -> None: + raise NotImplementedError() + + +def test_code_fullsource() -> None: + code = Code.from_function(x) + full = code.fullsource + assert "test_code_fullsource()" in str(full) + + +def test_code_source() -> None: + code = Code.from_function(x) + src = code.source() + expected = """def x() -> None: + raise NotImplementedError()""" + assert str(src) == expected + + +def test_frame_getsourcelineno_myself() -> None: + def func() -> FrameType: + return sys._getframe(0) + + f = Frame(func()) + source, lineno = f.code.fullsource, f.lineno + assert source is not None + assert source[lineno].startswith(" return sys._getframe(0)") + + +def test_getstatement_empty_fullsource() -> None: + def func() -> FrameType: + return sys._getframe(0) + + f = Frame(func()) + with mock.patch.object(f.code.__class__, "fullsource", None): + assert f.statement == Source("") + + +def test_code_from_func() -> None: + co = Code.from_function(test_frame_getsourcelineno_myself) + assert co.firstlineno + assert co.path + + +def test_unicode_handling() -> None: + value = "ąć".encode() + + def f() -> None: + raise Exception(value) + + excinfo = pytest.raises(Exception, f) + str(excinfo) + + +def test_code_getargs() -> None: + def f1(x): + raise NotImplementedError() + + c1 = Code.from_function(f1) + assert c1.getargs(var=True) == ("x",) + + def f2(x, *y): + raise NotImplementedError() + + c2 = Code.from_function(f2) + assert c2.getargs(var=True) == ("x", "y") + + def f3(x, **z): + raise NotImplementedError() + + c3 = Code.from_function(f3) + assert c3.getargs(var=True) == ("x", "z") + + def f4(x, *y, **z): + raise NotImplementedError() + + c4 = Code.from_function(f4) + assert c4.getargs(var=True) == ("x", "y", "z") + + +def test_frame_getargs() -> None: + def f1(x) -> FrameType: + return sys._getframe(0) + + fr1 = Frame(f1("a")) + assert fr1.getargs(var=True) == [("x", "a")] + + def f2(x, *y) -> FrameType: + return sys._getframe(0) + + fr2 = Frame(f2("a", "b", "c")) + assert fr2.getargs(var=True) == [("x", "a"), ("y", ("b", "c"))] + + def f3(x, **z) -> FrameType: + return sys._getframe(0) + + fr3 = Frame(f3("a", b="c")) + assert fr3.getargs(var=True) == [("x", "a"), ("z", {"b": "c"})] + + def f4(x, *y, **z) -> FrameType: + return sys._getframe(0) + + fr4 = Frame(f4("a", "b", c="d")) + assert fr4.getargs(var=True) == [("x", "a"), ("y", ("b",)), ("z", {"c": "d"})] + + +class TestExceptionInfo: + def test_bad_getsource(self) -> None: + try: + if False: + pass + else: + assert False + except AssertionError: + exci = ExceptionInfo.from_current() + assert exci.getrepr() + + def test_from_current_with_missing(self) -> None: + with pytest.raises(AssertionError, match="no current exception"): + ExceptionInfo.from_current() + + +class TestTracebackEntry: + def test_getsource(self) -> None: + try: + if False: + pass + else: + assert False + except AssertionError: + exci = ExceptionInfo.from_current() + entry = exci.traceback[0] + source = entry.getsource() + assert source is not None + assert len(source) == 6 + assert "assert False" in source[5] + + def test_tb_entry_str(self): + try: + assert False + except AssertionError: + exci = ExceptionInfo.from_current() + pattern = r" File '.*test_code.py':\d+ in test_tb_entry_str\n assert False" + entry = str(exci.traceback[0]) + assert re.match(pattern, entry) + + +class TestReprFuncArgs: + def test_not_raise_exception_with_mixed_encoding(self, tw_mock) -> None: + args = [("unicode_string", "São Paulo"), ("utf8_string", b"S\xc3\xa3o Paulo")] + + r = ReprFuncArgs(args) + r.toterminal(tw_mock) + + assert ( + tw_mock.lines[0] + == r"unicode_string = São Paulo, utf8_string = b'S\xc3\xa3o Paulo'" + ) + + +def test_ExceptionChainRepr(): + """Test ExceptionChainRepr, especially with regard to being hashable.""" + try: + raise ValueError() + except ValueError: + excinfo1 = ExceptionInfo.from_current() + excinfo2 = ExceptionInfo.from_current() + + repr1 = excinfo1.getrepr() + repr2 = excinfo2.getrepr() + assert repr1 != repr2 + + assert isinstance(repr1, ExceptionChainRepr) + assert hash(repr1) != hash(repr2) + assert repr1 is not excinfo1.getrepr() diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/code/test_excinfo.py b/testing/web-platform/tests/tools/third_party/pytest/testing/code/test_excinfo.py new file mode 100644 index 0000000000..61aa4406ad --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/code/test_excinfo.py @@ -0,0 +1,1470 @@ +import importlib +import io +import operator +import queue +import sys +import textwrap +from pathlib import Path +from typing import Any +from typing import Dict +from typing import Tuple +from typing import TYPE_CHECKING +from typing import Union + +import _pytest +import pytest +from _pytest._code.code import ExceptionChainRepr +from _pytest._code.code import ExceptionInfo +from _pytest._code.code import FormattedExcinfo +from _pytest._io import TerminalWriter +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pathlib import bestrelpath +from _pytest.pathlib import import_path +from _pytest.pytester import LineMatcher +from _pytest.pytester import Pytester + + +if TYPE_CHECKING: + from _pytest._code.code import _TracebackStyle + + +@pytest.fixture +def limited_recursion_depth(): + before = sys.getrecursionlimit() + sys.setrecursionlimit(150) + yield + sys.setrecursionlimit(before) + + +def test_excinfo_simple() -> None: + try: + raise ValueError + except ValueError: + info = _pytest._code.ExceptionInfo.from_current() + assert info.type == ValueError + + +def test_excinfo_from_exc_info_simple() -> None: + try: + raise ValueError + except ValueError as e: + assert e.__traceback__ is not None + info = _pytest._code.ExceptionInfo.from_exc_info((type(e), e, e.__traceback__)) + assert info.type == ValueError + + +def test_excinfo_getstatement(): + def g(): + raise ValueError + + def f(): + g() + + try: + f() + except ValueError: + excinfo = _pytest._code.ExceptionInfo.from_current() + linenumbers = [ + f.__code__.co_firstlineno - 1 + 4, + f.__code__.co_firstlineno - 1 + 1, + g.__code__.co_firstlineno - 1 + 1, + ] + values = list(excinfo.traceback) + foundlinenumbers = [x.lineno for x in values] + assert foundlinenumbers == linenumbers + # for x in info: + # print "%s:%d %s" %(x.path.relto(root), x.lineno, x.statement) + # xxx + + +# testchain for getentries test below + + +def f(): + # + raise ValueError + # + + +def g(): + # + __tracebackhide__ = True + f() + # + + +def h(): + # + g() + # + + +class TestTraceback_f_g_h: + def setup_method(self, method): + try: + h() + except ValueError: + self.excinfo = _pytest._code.ExceptionInfo.from_current() + + def test_traceback_entries(self): + tb = self.excinfo.traceback + entries = list(tb) + assert len(tb) == 4 # maybe fragile test + assert len(entries) == 4 # maybe fragile test + names = ["f", "g", "h"] + for entry in entries: + try: + names.remove(entry.frame.code.name) + except ValueError: + pass + assert not names + + def test_traceback_entry_getsource(self): + tb = self.excinfo.traceback + s = str(tb[-1].getsource()) + assert s.startswith("def f():") + assert s.endswith("raise ValueError") + + def test_traceback_entry_getsource_in_construct(self): + def xyz(): + try: + raise ValueError + except somenoname: # type: ignore[name-defined] # noqa: F821 + pass # pragma: no cover + + try: + xyz() + except NameError: + excinfo = _pytest._code.ExceptionInfo.from_current() + else: + assert False, "did not raise NameError" + + tb = excinfo.traceback + source = tb[-1].getsource() + assert source is not None + assert source.deindent().lines == [ + "def xyz():", + " try:", + " raise ValueError", + " except somenoname: # type: ignore[name-defined] # noqa: F821", + ] + + def test_traceback_cut(self) -> None: + co = _pytest._code.Code.from_function(f) + path, firstlineno = co.path, co.firstlineno + assert isinstance(path, Path) + traceback = self.excinfo.traceback + newtraceback = traceback.cut(path=path, firstlineno=firstlineno) + assert len(newtraceback) == 1 + newtraceback = traceback.cut(path=path, lineno=firstlineno + 2) + assert len(newtraceback) == 1 + + def test_traceback_cut_excludepath(self, pytester: Pytester) -> None: + p = pytester.makepyfile("def f(): raise ValueError") + with pytest.raises(ValueError) as excinfo: + import_path(p, root=pytester.path).f() # type: ignore[attr-defined] + basedir = Path(pytest.__file__).parent + newtraceback = excinfo.traceback.cut(excludepath=basedir) + for x in newtraceback: + assert isinstance(x.path, Path) + assert basedir not in x.path.parents + assert newtraceback[-1].frame.code.path == p + + def test_traceback_filter(self): + traceback = self.excinfo.traceback + ntraceback = traceback.filter() + assert len(ntraceback) == len(traceback) - 1 + + @pytest.mark.parametrize( + "tracebackhide, matching", + [ + (lambda info: True, True), + (lambda info: False, False), + (operator.methodcaller("errisinstance", ValueError), True), + (operator.methodcaller("errisinstance", IndexError), False), + ], + ) + def test_traceback_filter_selective(self, tracebackhide, matching): + def f(): + # + raise ValueError + # + + def g(): + # + __tracebackhide__ = tracebackhide + f() + # + + def h(): + # + g() + # + + excinfo = pytest.raises(ValueError, h) + traceback = excinfo.traceback + ntraceback = traceback.filter() + print(f"old: {traceback!r}") + print(f"new: {ntraceback!r}") + + if matching: + assert len(ntraceback) == len(traceback) - 2 + else: + # -1 because of the __tracebackhide__ in pytest.raises + assert len(ntraceback) == len(traceback) - 1 + + def test_traceback_recursion_index(self): + def f(n): + if n < 10: + n += 1 + f(n) + + excinfo = pytest.raises(RuntimeError, f, 8) + traceback = excinfo.traceback + recindex = traceback.recursionindex() + assert recindex == 3 + + def test_traceback_only_specific_recursion_errors(self, monkeypatch): + def f(n): + if n == 0: + raise RuntimeError("hello") + f(n - 1) + + excinfo = pytest.raises(RuntimeError, f, 25) + monkeypatch.delattr(excinfo.traceback.__class__, "recursionindex") + repr = excinfo.getrepr() + assert "RuntimeError: hello" in str(repr.reprcrash) + + def test_traceback_no_recursion_index(self) -> None: + def do_stuff() -> None: + raise RuntimeError + + def reraise_me() -> None: + import sys + + exc, val, tb = sys.exc_info() + assert val is not None + raise val.with_traceback(tb) + + def f(n: int) -> None: + try: + do_stuff() + except BaseException: + reraise_me() + + excinfo = pytest.raises(RuntimeError, f, 8) + assert excinfo is not None + traceback = excinfo.traceback + recindex = traceback.recursionindex() + assert recindex is None + + def test_traceback_messy_recursion(self): + # XXX: simplified locally testable version + decorator = pytest.importorskip("decorator").decorator + + def log(f, *k, **kw): + print(f"{k} {kw}") + f(*k, **kw) + + log = decorator(log) + + def fail(): + raise ValueError("") + + fail = log(log(fail)) + + excinfo = pytest.raises(ValueError, fail) + assert excinfo.traceback.recursionindex() is None + + def test_traceback_getcrashentry(self): + def i(): + __tracebackhide__ = True + raise ValueError + + def h(): + i() + + def g(): + __tracebackhide__ = True + h() + + def f(): + g() + + excinfo = pytest.raises(ValueError, f) + tb = excinfo.traceback + entry = tb.getcrashentry() + co = _pytest._code.Code.from_function(h) + assert entry.frame.code.path == co.path + assert entry.lineno == co.firstlineno + 1 + assert entry.frame.code.name == "h" + + def test_traceback_getcrashentry_empty(self): + def g(): + __tracebackhide__ = True + raise ValueError + + def f(): + __tracebackhide__ = True + g() + + excinfo = pytest.raises(ValueError, f) + tb = excinfo.traceback + entry = tb.getcrashentry() + co = _pytest._code.Code.from_function(g) + assert entry.frame.code.path == co.path + assert entry.lineno == co.firstlineno + 2 + assert entry.frame.code.name == "g" + + +def test_excinfo_exconly(): + excinfo = pytest.raises(ValueError, h) + assert excinfo.exconly().startswith("ValueError") + with pytest.raises(ValueError) as excinfo: + raise ValueError("hello\nworld") + msg = excinfo.exconly(tryshort=True) + assert msg.startswith("ValueError") + assert msg.endswith("world") + + +def test_excinfo_repr_str() -> None: + excinfo1 = pytest.raises(ValueError, h) + assert repr(excinfo1) == "" + assert str(excinfo1) == "" + + class CustomException(Exception): + def __repr__(self): + return "custom_repr" + + def raises() -> None: + raise CustomException() + + excinfo2 = pytest.raises(CustomException, raises) + assert repr(excinfo2) == "" + assert str(excinfo2) == "" + + +def test_excinfo_for_later() -> None: + e = ExceptionInfo[BaseException].for_later() + assert "for raises" in repr(e) + assert "for raises" in str(e) + + +def test_excinfo_errisinstance(): + excinfo = pytest.raises(ValueError, h) + assert excinfo.errisinstance(ValueError) + + +def test_excinfo_no_sourcecode(): + try: + exec("raise ValueError()") + except ValueError: + excinfo = _pytest._code.ExceptionInfo.from_current() + s = str(excinfo.traceback[-1]) + assert s == " File '':1 in \n ???\n" + + +def test_excinfo_no_python_sourcecode(tmp_path: Path) -> None: + # XXX: simplified locally testable version + tmp_path.joinpath("test.txt").write_text("{{ h()}}:") + + jinja2 = pytest.importorskip("jinja2") + loader = jinja2.FileSystemLoader(str(tmp_path)) + env = jinja2.Environment(loader=loader) + template = env.get_template("test.txt") + excinfo = pytest.raises(ValueError, template.render, h=h) + for item in excinfo.traceback: + print(item) # XXX: for some reason jinja.Template.render is printed in full + item.source # shouldn't fail + if isinstance(item.path, Path) and item.path.name == "test.txt": + assert str(item.source) == "{{ h()}}:" + + +def test_entrysource_Queue_example(): + try: + queue.Queue().get(timeout=0.001) + except queue.Empty: + excinfo = _pytest._code.ExceptionInfo.from_current() + entry = excinfo.traceback[-1] + source = entry.getsource() + assert source is not None + s = str(source).strip() + assert s.startswith("def get") + + +def test_codepath_Queue_example() -> None: + try: + queue.Queue().get(timeout=0.001) + except queue.Empty: + excinfo = _pytest._code.ExceptionInfo.from_current() + entry = excinfo.traceback[-1] + path = entry.path + assert isinstance(path, Path) + assert path.name.lower() == "queue.py" + assert path.exists() + + +def test_match_succeeds(): + with pytest.raises(ZeroDivisionError) as excinfo: + 0 // 0 + excinfo.match(r".*zero.*") + + +def test_match_raises_error(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + def test_division_zero(): + with pytest.raises(ZeroDivisionError) as excinfo: + 0 / 0 + excinfo.match(r'[123]+') + """ + ) + result = pytester.runpytest() + assert result.ret != 0 + + exc_msg = "Regex pattern '[[]123[]]+' does not match 'division by zero'." + result.stdout.fnmatch_lines([f"E * AssertionError: {exc_msg}"]) + result.stdout.no_fnmatch_line("*__tracebackhide__ = True*") + + result = pytester.runpytest("--fulltrace") + assert result.ret != 0 + result.stdout.fnmatch_lines( + ["*__tracebackhide__ = True*", f"E * AssertionError: {exc_msg}"] + ) + + +class TestFormattedExcinfo: + @pytest.fixture + def importasmod(self, tmp_path: Path, _sys_snapshot): + def importasmod(source): + source = textwrap.dedent(source) + modpath = tmp_path.joinpath("mod.py") + tmp_path.joinpath("__init__.py").touch() + modpath.write_text(source) + importlib.invalidate_caches() + return import_path(modpath, root=tmp_path) + + return importasmod + + def test_repr_source(self): + pr = FormattedExcinfo() + source = _pytest._code.Source( + """\ + def f(x): + pass + """ + ).strip() + pr.flow_marker = "|" # type: ignore[misc] + lines = pr.get_source(source, 0) + assert len(lines) == 2 + assert lines[0] == "| def f(x):" + assert lines[1] == " pass" + + def test_repr_source_excinfo(self) -> None: + """Check if indentation is right.""" + try: + + def f(): + 1 / 0 + + f() + + except BaseException: + excinfo = _pytest._code.ExceptionInfo.from_current() + else: + assert False, "did not raise" + + pr = FormattedExcinfo() + source = pr._getentrysource(excinfo.traceback[-1]) + assert source is not None + lines = pr.get_source(source, 1, excinfo) + for line in lines: + print(line) + assert lines == [ + " def f():", + "> 1 / 0", + "E ZeroDivisionError: division by zero", + ] + + def test_repr_source_not_existing(self): + pr = FormattedExcinfo() + co = compile("raise ValueError()", "", "exec") + try: + exec(co) + except ValueError: + excinfo = _pytest._code.ExceptionInfo.from_current() + repr = pr.repr_excinfo(excinfo) + assert repr.reprtraceback.reprentries[1].lines[0] == "> ???" + assert repr.chain[0][0].reprentries[1].lines[0] == "> ???" + + def test_repr_many_line_source_not_existing(self): + pr = FormattedExcinfo() + co = compile( + """ +a = 1 +raise ValueError() +""", + "", + "exec", + ) + try: + exec(co) + except ValueError: + excinfo = _pytest._code.ExceptionInfo.from_current() + repr = pr.repr_excinfo(excinfo) + assert repr.reprtraceback.reprentries[1].lines[0] == "> ???" + assert repr.chain[0][0].reprentries[1].lines[0] == "> ???" + + def test_repr_source_failing_fullsource(self, monkeypatch) -> None: + pr = FormattedExcinfo() + + try: + 1 / 0 + except ZeroDivisionError: + excinfo = ExceptionInfo.from_current() + + with monkeypatch.context() as m: + m.setattr(_pytest._code.Code, "fullsource", property(lambda self: None)) + repr = pr.repr_excinfo(excinfo) + + assert repr.reprtraceback.reprentries[0].lines[0] == "> ???" + assert repr.chain[0][0].reprentries[0].lines[0] == "> ???" + + def test_repr_local(self) -> None: + p = FormattedExcinfo(showlocals=True) + loc = {"y": 5, "z": 7, "x": 3, "@x": 2, "__builtins__": {}} + reprlocals = p.repr_locals(loc) + assert reprlocals is not None + assert reprlocals.lines + assert reprlocals.lines[0] == "__builtins__ = " + assert reprlocals.lines[1] == "x = 3" + assert reprlocals.lines[2] == "y = 5" + assert reprlocals.lines[3] == "z = 7" + + def test_repr_local_with_error(self) -> None: + class ObjWithErrorInRepr: + def __repr__(self): + raise NotImplementedError + + p = FormattedExcinfo(showlocals=True, truncate_locals=False) + loc = {"x": ObjWithErrorInRepr(), "__builtins__": {}} + reprlocals = p.repr_locals(loc) + assert reprlocals is not None + assert reprlocals.lines + assert reprlocals.lines[0] == "__builtins__ = " + assert "[NotImplementedError() raised in repr()]" in reprlocals.lines[1] + + def test_repr_local_with_exception_in_class_property(self) -> None: + class ExceptionWithBrokenClass(Exception): + # Type ignored because it's bypassed intentionally. + @property # type: ignore + def __class__(self): + raise TypeError("boom!") + + class ObjWithErrorInRepr: + def __repr__(self): + raise ExceptionWithBrokenClass() + + p = FormattedExcinfo(showlocals=True, truncate_locals=False) + loc = {"x": ObjWithErrorInRepr(), "__builtins__": {}} + reprlocals = p.repr_locals(loc) + assert reprlocals is not None + assert reprlocals.lines + assert reprlocals.lines[0] == "__builtins__ = " + assert "[ExceptionWithBrokenClass() raised in repr()]" in reprlocals.lines[1] + + def test_repr_local_truncated(self) -> None: + loc = {"l": [i for i in range(10)]} + p = FormattedExcinfo(showlocals=True) + truncated_reprlocals = p.repr_locals(loc) + assert truncated_reprlocals is not None + assert truncated_reprlocals.lines + assert truncated_reprlocals.lines[0] == "l = [0, 1, 2, 3, 4, 5, ...]" + + q = FormattedExcinfo(showlocals=True, truncate_locals=False) + full_reprlocals = q.repr_locals(loc) + assert full_reprlocals is not None + assert full_reprlocals.lines + assert full_reprlocals.lines[0] == "l = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]" + + def test_repr_tracebackentry_lines(self, importasmod) -> None: + mod = importasmod( + """ + def func1(): + raise ValueError("hello\\nworld") + """ + ) + excinfo = pytest.raises(ValueError, mod.func1) + excinfo.traceback = excinfo.traceback.filter() + p = FormattedExcinfo() + reprtb = p.repr_traceback_entry(excinfo.traceback[-1]) + + # test as intermittent entry + lines = reprtb.lines + assert lines[0] == " def func1():" + assert lines[1] == '> raise ValueError("hello\\nworld")' + + # test as last entry + p = FormattedExcinfo(showlocals=True) + repr_entry = p.repr_traceback_entry(excinfo.traceback[-1], excinfo) + lines = repr_entry.lines + assert lines[0] == " def func1():" + assert lines[1] == '> raise ValueError("hello\\nworld")' + assert lines[2] == "E ValueError: hello" + assert lines[3] == "E world" + assert not lines[4:] + + loc = repr_entry.reprfileloc + assert loc is not None + assert loc.path == mod.__file__ + assert loc.lineno == 3 + # assert loc.message == "ValueError: hello" + + def test_repr_tracebackentry_lines2(self, importasmod, tw_mock) -> None: + mod = importasmod( + """ + def func1(m, x, y, z): + raise ValueError("hello\\nworld") + """ + ) + excinfo = pytest.raises(ValueError, mod.func1, "m" * 90, 5, 13, "z" * 120) + excinfo.traceback = excinfo.traceback.filter() + entry = excinfo.traceback[-1] + p = FormattedExcinfo(funcargs=True) + reprfuncargs = p.repr_args(entry) + assert reprfuncargs is not None + assert reprfuncargs.args[0] == ("m", repr("m" * 90)) + assert reprfuncargs.args[1] == ("x", "5") + assert reprfuncargs.args[2] == ("y", "13") + assert reprfuncargs.args[3] == ("z", repr("z" * 120)) + + p = FormattedExcinfo(funcargs=True) + repr_entry = p.repr_traceback_entry(entry) + assert repr_entry.reprfuncargs is not None + assert repr_entry.reprfuncargs.args == reprfuncargs.args + repr_entry.toterminal(tw_mock) + assert tw_mock.lines[0] == "m = " + repr("m" * 90) + assert tw_mock.lines[1] == "x = 5, y = 13" + assert tw_mock.lines[2] == "z = " + repr("z" * 120) + + def test_repr_tracebackentry_lines_var_kw_args(self, importasmod, tw_mock) -> None: + mod = importasmod( + """ + def func1(x, *y, **z): + raise ValueError("hello\\nworld") + """ + ) + excinfo = pytest.raises(ValueError, mod.func1, "a", "b", c="d") + excinfo.traceback = excinfo.traceback.filter() + entry = excinfo.traceback[-1] + p = FormattedExcinfo(funcargs=True) + reprfuncargs = p.repr_args(entry) + assert reprfuncargs is not None + assert reprfuncargs.args[0] == ("x", repr("a")) + assert reprfuncargs.args[1] == ("y", repr(("b",))) + assert reprfuncargs.args[2] == ("z", repr({"c": "d"})) + + p = FormattedExcinfo(funcargs=True) + repr_entry = p.repr_traceback_entry(entry) + assert repr_entry.reprfuncargs + assert repr_entry.reprfuncargs.args == reprfuncargs.args + repr_entry.toterminal(tw_mock) + assert tw_mock.lines[0] == "x = 'a', y = ('b',), z = {'c': 'd'}" + + def test_repr_tracebackentry_short(self, importasmod) -> None: + mod = importasmod( + """ + def func1(): + raise ValueError("hello") + def entry(): + func1() + """ + ) + excinfo = pytest.raises(ValueError, mod.entry) + p = FormattedExcinfo(style="short") + reprtb = p.repr_traceback_entry(excinfo.traceback[-2]) + lines = reprtb.lines + basename = Path(mod.__file__).name + assert lines[0] == " func1()" + assert reprtb.reprfileloc is not None + assert basename in str(reprtb.reprfileloc.path) + assert reprtb.reprfileloc.lineno == 5 + + # test last entry + p = FormattedExcinfo(style="short") + reprtb = p.repr_traceback_entry(excinfo.traceback[-1], excinfo) + lines = reprtb.lines + assert lines[0] == ' raise ValueError("hello")' + assert lines[1] == "E ValueError: hello" + assert reprtb.reprfileloc is not None + assert basename in str(reprtb.reprfileloc.path) + assert reprtb.reprfileloc.lineno == 3 + + def test_repr_tracebackentry_no(self, importasmod): + mod = importasmod( + """ + def func1(): + raise ValueError("hello") + def entry(): + func1() + """ + ) + excinfo = pytest.raises(ValueError, mod.entry) + p = FormattedExcinfo(style="no") + p.repr_traceback_entry(excinfo.traceback[-2]) + + p = FormattedExcinfo(style="no") + reprentry = p.repr_traceback_entry(excinfo.traceback[-1], excinfo) + lines = reprentry.lines + assert lines[0] == "E ValueError: hello" + assert not lines[1:] + + def test_repr_traceback_tbfilter(self, importasmod): + mod = importasmod( + """ + def f(x): + raise ValueError(x) + def entry(): + f(0) + """ + ) + excinfo = pytest.raises(ValueError, mod.entry) + p = FormattedExcinfo(tbfilter=True) + reprtb = p.repr_traceback(excinfo) + assert len(reprtb.reprentries) == 2 + p = FormattedExcinfo(tbfilter=False) + reprtb = p.repr_traceback(excinfo) + assert len(reprtb.reprentries) == 3 + + def test_traceback_short_no_source(self, importasmod, monkeypatch) -> None: + mod = importasmod( + """ + def func1(): + raise ValueError("hello") + def entry(): + func1() + """ + ) + excinfo = pytest.raises(ValueError, mod.entry) + from _pytest._code.code import Code + + monkeypatch.setattr(Code, "path", "bogus") + p = FormattedExcinfo(style="short") + reprtb = p.repr_traceback_entry(excinfo.traceback[-2]) + lines = reprtb.lines + last_p = FormattedExcinfo(style="short") + last_reprtb = last_p.repr_traceback_entry(excinfo.traceback[-1], excinfo) + last_lines = last_reprtb.lines + monkeypatch.undo() + assert lines[0] == " func1()" + + assert last_lines[0] == ' raise ValueError("hello")' + assert last_lines[1] == "E ValueError: hello" + + def test_repr_traceback_and_excinfo(self, importasmod) -> None: + mod = importasmod( + """ + def f(x): + raise ValueError(x) + def entry(): + f(0) + """ + ) + excinfo = pytest.raises(ValueError, mod.entry) + + styles: Tuple[_TracebackStyle, ...] = ("long", "short") + for style in styles: + p = FormattedExcinfo(style=style) + reprtb = p.repr_traceback(excinfo) + assert len(reprtb.reprentries) == 2 + assert reprtb.style == style + assert not reprtb.extraline + repr = p.repr_excinfo(excinfo) + assert repr.reprtraceback + assert len(repr.reprtraceback.reprentries) == len(reprtb.reprentries) + + assert repr.chain[0][0] + assert len(repr.chain[0][0].reprentries) == len(reprtb.reprentries) + assert repr.reprcrash is not None + assert repr.reprcrash.path.endswith("mod.py") + assert repr.reprcrash.message == "ValueError: 0" + + def test_repr_traceback_with_invalid_cwd(self, importasmod, monkeypatch) -> None: + mod = importasmod( + """ + def f(x): + raise ValueError(x) + def entry(): + f(0) + """ + ) + excinfo = pytest.raises(ValueError, mod.entry) + + p = FormattedExcinfo(abspath=False) + + raised = 0 + + orig_path_cwd = Path.cwd + + def raiseos(): + nonlocal raised + upframe = sys._getframe().f_back + assert upframe is not None + if upframe.f_code.co_name == "_makepath": + # Only raise with expected calls, but not via e.g. inspect for + # py38-windows. + raised += 1 + raise OSError(2, "custom_oserror") + return orig_path_cwd() + + monkeypatch.setattr(Path, "cwd", raiseos) + assert p._makepath(Path(__file__)) == __file__ + assert raised == 1 + repr_tb = p.repr_traceback(excinfo) + + matcher = LineMatcher(str(repr_tb).splitlines()) + matcher.fnmatch_lines( + [ + "def entry():", + "> f(0)", + "", + f"{mod.__file__}:5: ", + "_ _ *", + "", + " def f(x):", + "> raise ValueError(x)", + "E ValueError: 0", + "", + f"{mod.__file__}:3: ValueError", + ] + ) + assert raised == 3 + + def test_repr_excinfo_addouterr(self, importasmod, tw_mock): + mod = importasmod( + """ + def entry(): + raise ValueError() + """ + ) + excinfo = pytest.raises(ValueError, mod.entry) + repr = excinfo.getrepr() + repr.addsection("title", "content") + repr.toterminal(tw_mock) + assert tw_mock.lines[-1] == "content" + assert tw_mock.lines[-2] == ("-", "title") + + def test_repr_excinfo_reprcrash(self, importasmod) -> None: + mod = importasmod( + """ + def entry(): + raise ValueError() + """ + ) + excinfo = pytest.raises(ValueError, mod.entry) + repr = excinfo.getrepr() + assert repr.reprcrash is not None + assert repr.reprcrash.path.endswith("mod.py") + assert repr.reprcrash.lineno == 3 + assert repr.reprcrash.message == "ValueError" + assert str(repr.reprcrash).endswith("mod.py:3: ValueError") + + def test_repr_traceback_recursion(self, importasmod): + mod = importasmod( + """ + def rec2(x): + return rec1(x+1) + def rec1(x): + return rec2(x-1) + def entry(): + rec1(42) + """ + ) + excinfo = pytest.raises(RuntimeError, mod.entry) + + for style in ("short", "long", "no"): + p = FormattedExcinfo(style="short") + reprtb = p.repr_traceback(excinfo) + assert reprtb.extraline == "!!! Recursion detected (same locals & position)" + assert str(reprtb) + + def test_reprexcinfo_getrepr(self, importasmod) -> None: + mod = importasmod( + """ + def f(x): + raise ValueError(x) + def entry(): + f(0) + """ + ) + excinfo = pytest.raises(ValueError, mod.entry) + + styles: Tuple[_TracebackStyle, ...] = ("short", "long", "no") + for style in styles: + for showlocals in (True, False): + repr = excinfo.getrepr(style=style, showlocals=showlocals) + assert repr.reprtraceback.style == style + + assert isinstance(repr, ExceptionChainRepr) + for r in repr.chain: + assert r[0].style == style + + def test_reprexcinfo_unicode(self): + from _pytest._code.code import TerminalRepr + + class MyRepr(TerminalRepr): + def toterminal(self, tw: TerminalWriter) -> None: + tw.line("я") + + x = str(MyRepr()) + assert x == "я" + + def test_toterminal_long(self, importasmod, tw_mock): + mod = importasmod( + """ + def g(x): + raise ValueError(x) + def f(): + g(3) + """ + ) + excinfo = pytest.raises(ValueError, mod.f) + excinfo.traceback = excinfo.traceback.filter() + repr = excinfo.getrepr() + repr.toterminal(tw_mock) + assert tw_mock.lines[0] == "" + tw_mock.lines.pop(0) + assert tw_mock.lines[0] == " def f():" + assert tw_mock.lines[1] == "> g(3)" + assert tw_mock.lines[2] == "" + line = tw_mock.get_write_msg(3) + assert line.endswith("mod.py") + assert tw_mock.lines[4] == (":5: ") + assert tw_mock.lines[5] == ("_ ", None) + assert tw_mock.lines[6] == "" + assert tw_mock.lines[7] == " def g(x):" + assert tw_mock.lines[8] == "> raise ValueError(x)" + assert tw_mock.lines[9] == "E ValueError: 3" + assert tw_mock.lines[10] == "" + line = tw_mock.get_write_msg(11) + assert line.endswith("mod.py") + assert tw_mock.lines[12] == ":3: ValueError" + + def test_toterminal_long_missing_source( + self, importasmod, tmp_path: Path, tw_mock + ) -> None: + mod = importasmod( + """ + def g(x): + raise ValueError(x) + def f(): + g(3) + """ + ) + excinfo = pytest.raises(ValueError, mod.f) + tmp_path.joinpath("mod.py").unlink() + excinfo.traceback = excinfo.traceback.filter() + repr = excinfo.getrepr() + repr.toterminal(tw_mock) + assert tw_mock.lines[0] == "" + tw_mock.lines.pop(0) + assert tw_mock.lines[0] == "> ???" + assert tw_mock.lines[1] == "" + line = tw_mock.get_write_msg(2) + assert line.endswith("mod.py") + assert tw_mock.lines[3] == ":5: " + assert tw_mock.lines[4] == ("_ ", None) + assert tw_mock.lines[5] == "" + assert tw_mock.lines[6] == "> ???" + assert tw_mock.lines[7] == "E ValueError: 3" + assert tw_mock.lines[8] == "" + line = tw_mock.get_write_msg(9) + assert line.endswith("mod.py") + assert tw_mock.lines[10] == ":3: ValueError" + + def test_toterminal_long_incomplete_source( + self, importasmod, tmp_path: Path, tw_mock + ) -> None: + mod = importasmod( + """ + def g(x): + raise ValueError(x) + def f(): + g(3) + """ + ) + excinfo = pytest.raises(ValueError, mod.f) + tmp_path.joinpath("mod.py").write_text("asdf") + excinfo.traceback = excinfo.traceback.filter() + repr = excinfo.getrepr() + repr.toterminal(tw_mock) + assert tw_mock.lines[0] == "" + tw_mock.lines.pop(0) + assert tw_mock.lines[0] == "> ???" + assert tw_mock.lines[1] == "" + line = tw_mock.get_write_msg(2) + assert line.endswith("mod.py") + assert tw_mock.lines[3] == ":5: " + assert tw_mock.lines[4] == ("_ ", None) + assert tw_mock.lines[5] == "" + assert tw_mock.lines[6] == "> ???" + assert tw_mock.lines[7] == "E ValueError: 3" + assert tw_mock.lines[8] == "" + line = tw_mock.get_write_msg(9) + assert line.endswith("mod.py") + assert tw_mock.lines[10] == ":3: ValueError" + + def test_toterminal_long_filenames( + self, importasmod, tw_mock, monkeypatch: MonkeyPatch + ) -> None: + mod = importasmod( + """ + def f(): + raise ValueError() + """ + ) + excinfo = pytest.raises(ValueError, mod.f) + path = Path(mod.__file__) + monkeypatch.chdir(path.parent) + repr = excinfo.getrepr(abspath=False) + repr.toterminal(tw_mock) + x = bestrelpath(Path.cwd(), path) + if len(x) < len(str(path)): + msg = tw_mock.get_write_msg(-2) + assert msg == "mod.py" + assert tw_mock.lines[-1] == ":3: ValueError" + + repr = excinfo.getrepr(abspath=True) + repr.toterminal(tw_mock) + msg = tw_mock.get_write_msg(-2) + assert msg == str(path) + line = tw_mock.lines[-1] + assert line == ":3: ValueError" + + @pytest.mark.parametrize( + "reproptions", + [ + pytest.param( + { + "style": style, + "showlocals": showlocals, + "funcargs": funcargs, + "tbfilter": tbfilter, + }, + id="style={},showlocals={},funcargs={},tbfilter={}".format( + style, showlocals, funcargs, tbfilter + ), + ) + for style in ["long", "short", "line", "no", "native", "value", "auto"] + for showlocals in (True, False) + for tbfilter in (True, False) + for funcargs in (True, False) + ], + ) + def test_format_excinfo(self, reproptions: Dict[str, Any]) -> None: + def bar(): + assert False, "some error" + + def foo(): + bar() + + # using inline functions as opposed to importasmod so we get source code lines + # in the tracebacks (otherwise getinspect doesn't find the source code). + with pytest.raises(AssertionError) as excinfo: + foo() + file = io.StringIO() + tw = TerminalWriter(file=file) + repr = excinfo.getrepr(**reproptions) + repr.toterminal(tw) + assert file.getvalue() + + def test_traceback_repr_style(self, importasmod, tw_mock): + mod = importasmod( + """ + def f(): + g() + def g(): + h() + def h(): + i() + def i(): + raise ValueError() + """ + ) + excinfo = pytest.raises(ValueError, mod.f) + excinfo.traceback = excinfo.traceback.filter() + excinfo.traceback[1].set_repr_style("short") + excinfo.traceback[2].set_repr_style("short") + r = excinfo.getrepr(style="long") + r.toterminal(tw_mock) + for line in tw_mock.lines: + print(line) + assert tw_mock.lines[0] == "" + assert tw_mock.lines[1] == " def f():" + assert tw_mock.lines[2] == "> g()" + assert tw_mock.lines[3] == "" + msg = tw_mock.get_write_msg(4) + assert msg.endswith("mod.py") + assert tw_mock.lines[5] == ":3: " + assert tw_mock.lines[6] == ("_ ", None) + tw_mock.get_write_msg(7) + assert tw_mock.lines[8].endswith("in g") + assert tw_mock.lines[9] == " h()" + tw_mock.get_write_msg(10) + assert tw_mock.lines[11].endswith("in h") + assert tw_mock.lines[12] == " i()" + assert tw_mock.lines[13] == ("_ ", None) + assert tw_mock.lines[14] == "" + assert tw_mock.lines[15] == " def i():" + assert tw_mock.lines[16] == "> raise ValueError()" + assert tw_mock.lines[17] == "E ValueError" + assert tw_mock.lines[18] == "" + msg = tw_mock.get_write_msg(19) + msg.endswith("mod.py") + assert tw_mock.lines[20] == ":9: ValueError" + + def test_exc_chain_repr(self, importasmod, tw_mock): + mod = importasmod( + """ + class Err(Exception): + pass + def f(): + try: + g() + except Exception as e: + raise Err() from e + finally: + h() + def g(): + raise ValueError() + + def h(): + raise AttributeError() + """ + ) + excinfo = pytest.raises(AttributeError, mod.f) + r = excinfo.getrepr(style="long") + r.toterminal(tw_mock) + for line in tw_mock.lines: + print(line) + assert tw_mock.lines[0] == "" + assert tw_mock.lines[1] == " def f():" + assert tw_mock.lines[2] == " try:" + assert tw_mock.lines[3] == "> g()" + assert tw_mock.lines[4] == "" + line = tw_mock.get_write_msg(5) + assert line.endswith("mod.py") + assert tw_mock.lines[6] == ":6: " + assert tw_mock.lines[7] == ("_ ", None) + assert tw_mock.lines[8] == "" + assert tw_mock.lines[9] == " def g():" + assert tw_mock.lines[10] == "> raise ValueError()" + assert tw_mock.lines[11] == "E ValueError" + assert tw_mock.lines[12] == "" + line = tw_mock.get_write_msg(13) + assert line.endswith("mod.py") + assert tw_mock.lines[14] == ":12: ValueError" + assert tw_mock.lines[15] == "" + assert ( + tw_mock.lines[16] + == "The above exception was the direct cause of the following exception:" + ) + assert tw_mock.lines[17] == "" + assert tw_mock.lines[18] == " def f():" + assert tw_mock.lines[19] == " try:" + assert tw_mock.lines[20] == " g()" + assert tw_mock.lines[21] == " except Exception as e:" + assert tw_mock.lines[22] == "> raise Err() from e" + assert tw_mock.lines[23] == "E test_exc_chain_repr0.mod.Err" + assert tw_mock.lines[24] == "" + line = tw_mock.get_write_msg(25) + assert line.endswith("mod.py") + assert tw_mock.lines[26] == ":8: Err" + assert tw_mock.lines[27] == "" + assert ( + tw_mock.lines[28] + == "During handling of the above exception, another exception occurred:" + ) + assert tw_mock.lines[29] == "" + assert tw_mock.lines[30] == " def f():" + assert tw_mock.lines[31] == " try:" + assert tw_mock.lines[32] == " g()" + assert tw_mock.lines[33] == " except Exception as e:" + assert tw_mock.lines[34] == " raise Err() from e" + assert tw_mock.lines[35] == " finally:" + assert tw_mock.lines[36] == "> h()" + assert tw_mock.lines[37] == "" + line = tw_mock.get_write_msg(38) + assert line.endswith("mod.py") + assert tw_mock.lines[39] == ":10: " + assert tw_mock.lines[40] == ("_ ", None) + assert tw_mock.lines[41] == "" + assert tw_mock.lines[42] == " def h():" + assert tw_mock.lines[43] == "> raise AttributeError()" + assert tw_mock.lines[44] == "E AttributeError" + assert tw_mock.lines[45] == "" + line = tw_mock.get_write_msg(46) + assert line.endswith("mod.py") + assert tw_mock.lines[47] == ":15: AttributeError" + + @pytest.mark.parametrize("mode", ["from_none", "explicit_suppress"]) + def test_exc_repr_chain_suppression(self, importasmod, mode, tw_mock): + """Check that exc repr does not show chained exceptions in Python 3. + - When the exception is raised with "from None" + - Explicitly suppressed with "chain=False" to ExceptionInfo.getrepr(). + """ + raise_suffix = " from None" if mode == "from_none" else "" + mod = importasmod( + """ + def f(): + try: + g() + except Exception: + raise AttributeError(){raise_suffix} + def g(): + raise ValueError() + """.format( + raise_suffix=raise_suffix + ) + ) + excinfo = pytest.raises(AttributeError, mod.f) + r = excinfo.getrepr(style="long", chain=mode != "explicit_suppress") + r.toterminal(tw_mock) + for line in tw_mock.lines: + print(line) + assert tw_mock.lines[0] == "" + assert tw_mock.lines[1] == " def f():" + assert tw_mock.lines[2] == " try:" + assert tw_mock.lines[3] == " g()" + assert tw_mock.lines[4] == " except Exception:" + assert tw_mock.lines[5] == "> raise AttributeError(){}".format( + raise_suffix + ) + assert tw_mock.lines[6] == "E AttributeError" + assert tw_mock.lines[7] == "" + line = tw_mock.get_write_msg(8) + assert line.endswith("mod.py") + assert tw_mock.lines[9] == ":6: AttributeError" + assert len(tw_mock.lines) == 10 + + @pytest.mark.parametrize( + "reason, description", + [ + pytest.param( + "cause", + "The above exception was the direct cause of the following exception:", + id="cause", + ), + pytest.param( + "context", + "During handling of the above exception, another exception occurred:", + id="context", + ), + ], + ) + def test_exc_chain_repr_without_traceback(self, importasmod, reason, description): + """ + Handle representation of exception chains where one of the exceptions doesn't have a + real traceback, such as those raised in a subprocess submitted by the multiprocessing + module (#1984). + """ + exc_handling_code = " from e" if reason == "cause" else "" + mod = importasmod( + """ + def f(): + try: + g() + except Exception as e: + raise RuntimeError('runtime problem'){exc_handling_code} + def g(): + raise ValueError('invalid value') + """.format( + exc_handling_code=exc_handling_code + ) + ) + + with pytest.raises(RuntimeError) as excinfo: + mod.f() + + # emulate the issue described in #1984 + attr = "__%s__" % reason + getattr(excinfo.value, attr).__traceback__ = None + + r = excinfo.getrepr() + file = io.StringIO() + tw = TerminalWriter(file=file) + tw.hasmarkup = False + r.toterminal(tw) + + matcher = LineMatcher(file.getvalue().splitlines()) + matcher.fnmatch_lines( + [ + "ValueError: invalid value", + description, + "* except Exception as e:", + "> * raise RuntimeError('runtime problem')" + exc_handling_code, + "E *RuntimeError: runtime problem", + ] + ) + + def test_exc_chain_repr_cycle(self, importasmod, tw_mock): + mod = importasmod( + """ + class Err(Exception): + pass + def fail(): + return 0 / 0 + def reraise(): + try: + fail() + except ZeroDivisionError as e: + raise Err() from e + def unreraise(): + try: + reraise() + except Err as e: + raise e.__cause__ + """ + ) + excinfo = pytest.raises(ZeroDivisionError, mod.unreraise) + r = excinfo.getrepr(style="short") + r.toterminal(tw_mock) + out = "\n".join(line for line in tw_mock.lines if isinstance(line, str)) + expected_out = textwrap.dedent( + """\ + :13: in unreraise + reraise() + :10: in reraise + raise Err() from e + E test_exc_chain_repr_cycle0.mod.Err + + During handling of the above exception, another exception occurred: + :15: in unreraise + raise e.__cause__ + :8: in reraise + fail() + :5: in fail + return 0 / 0 + E ZeroDivisionError: division by zero""" + ) + assert out == expected_out + + def test_exec_type_error_filter(self, importasmod): + """See #7742""" + mod = importasmod( + """\ + def f(): + exec("a = 1", {}, []) + """ + ) + with pytest.raises(TypeError) as excinfo: + mod.f() + # previously crashed with `AttributeError: list has no attribute get` + excinfo.traceback.filter() + + +@pytest.mark.parametrize("style", ["short", "long"]) +@pytest.mark.parametrize("encoding", [None, "utf8", "utf16"]) +def test_repr_traceback_with_unicode(style, encoding): + if encoding is None: + msg: Union[str, bytes] = "☹" + else: + msg = "☹".encode(encoding) + try: + raise RuntimeError(msg) + except RuntimeError: + e_info = ExceptionInfo.from_current() + formatter = FormattedExcinfo(style=style) + repr_traceback = formatter.repr_traceback(e_info) + assert repr_traceback is not None + + +def test_cwd_deleted(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import os + + def test(tmp_path): + os.chdir(tmp_path) + tmp_path.unlink() + assert False + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 1 failed in *"]) + result.stdout.no_fnmatch_line("*INTERNALERROR*") + result.stderr.no_fnmatch_line("*INTERNALERROR*") + + +def test_regression_nagative_line_index(pytester: Pytester) -> None: + """ + With Python 3.10 alphas, there was an INTERNALERROR reported in + https://github.com/pytest-dev/pytest/pull/8227 + This test ensures it does not regress. + """ + pytester.makepyfile( + """ + import ast + import pytest + + + def test_literal_eval(): + with pytest.raises(ValueError, match="^$"): + ast.literal_eval("pytest") + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 1 failed in *"]) + result.stdout.no_fnmatch_line("*INTERNALERROR*") + result.stderr.no_fnmatch_line("*INTERNALERROR*") + + +@pytest.mark.usefixtures("limited_recursion_depth") +def test_exception_repr_extraction_error_on_recursion(): + """ + Ensure we can properly detect a recursion error even + if some locals raise error on comparison (#2459). + """ + + class numpy_like: + def __eq__(self, other): + if type(other) is numpy_like: + raise ValueError( + "The truth value of an array " + "with more than one element is ambiguous." + ) + + def a(x): + return b(numpy_like()) + + def b(x): + return a(numpy_like()) + + with pytest.raises(RuntimeError) as excinfo: + a(numpy_like()) + + matcher = LineMatcher(str(excinfo.getrepr()).splitlines()) + matcher.fnmatch_lines( + [ + "!!! Recursion error detected, but an error occurred locating the origin of recursion.", + "*The following exception happened*", + "*ValueError: The truth value of an array*", + ] + ) + + +@pytest.mark.usefixtures("limited_recursion_depth") +def test_no_recursion_index_on_recursion_error(): + """ + Ensure that we don't break in case we can't find the recursion index + during a recursion error (#2486). + """ + + class RecursionDepthError: + def __getattr__(self, attr): + return getattr(self, "_" + attr) + + with pytest.raises(RuntimeError) as excinfo: + RecursionDepthError().trigger + assert "maximum recursion" in str(excinfo.getrepr()) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/code/test_source.py b/testing/web-platform/tests/tools/third_party/pytest/testing/code/test_source.py new file mode 100644 index 0000000000..9f7be5e245 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/code/test_source.py @@ -0,0 +1,656 @@ +# flake8: noqa +# disable flake check on this file because some constructs are strange +# or redundant on purpose and can't be disable on a line-by-line basis +import ast +import inspect +import linecache +import sys +import textwrap +from pathlib import Path +from types import CodeType +from typing import Any +from typing import Dict +from typing import Optional + +import pytest +from _pytest._code import Code +from _pytest._code import Frame +from _pytest._code import getfslineno +from _pytest._code import Source +from _pytest.pathlib import import_path + + +def test_source_str_function() -> None: + x = Source("3") + assert str(x) == "3" + + x = Source(" 3") + assert str(x) == "3" + + x = Source( + """ + 3 + """ + ) + assert str(x) == "\n3" + + +def test_source_from_function() -> None: + source = Source(test_source_str_function) + assert str(source).startswith("def test_source_str_function() -> None:") + + +def test_source_from_method() -> None: + class TestClass: + def test_method(self): + pass + + source = Source(TestClass().test_method) + assert source.lines == ["def test_method(self):", " pass"] + + +def test_source_from_lines() -> None: + lines = ["a \n", "b\n", "c"] + source = Source(lines) + assert source.lines == ["a ", "b", "c"] + + +def test_source_from_inner_function() -> None: + def f(): + raise NotImplementedError() + + source = Source(f) + assert str(source).startswith("def f():") + + +def test_source_strips() -> None: + source = Source("") + assert source == Source() + assert str(source) == "" + assert source.strip() == source + + +def test_source_strip_multiline() -> None: + source = Source() + source.lines = ["", " hello", " "] + source2 = source.strip() + assert source2.lines == [" hello"] + + +class TestAccesses: + def setup_class(self) -> None: + self.source = Source( + """\ + def f(x): + pass + def g(x): + pass + """ + ) + + def test_getrange(self) -> None: + x = self.source[0:2] + assert len(x.lines) == 2 + assert str(x) == "def f(x):\n pass" + + def test_getrange_step_not_supported(self) -> None: + with pytest.raises(IndexError, match=r"step"): + self.source[::2] + + def test_getline(self) -> None: + x = self.source[0] + assert x == "def f(x):" + + def test_len(self) -> None: + assert len(self.source) == 4 + + def test_iter(self) -> None: + values = [x for x in self.source] + assert len(values) == 4 + + +class TestSourceParsing: + def setup_class(self) -> None: + self.source = Source( + """\ + def f(x): + assert (x == + 3 + + 4) + """ + ).strip() + + def test_getstatement(self) -> None: + # print str(self.source) + ass = str(self.source[1:]) + for i in range(1, 4): + # print "trying start in line %r" % self.source[i] + s = self.source.getstatement(i) + # x = s.deindent() + assert str(s) == ass + + def test_getstatementrange_triple_quoted(self) -> None: + # print str(self.source) + source = Source( + """hello(''' + ''')""" + ) + s = source.getstatement(0) + assert s == source + s = source.getstatement(1) + assert s == source + + def test_getstatementrange_within_constructs(self) -> None: + source = Source( + """\ + try: + try: + raise ValueError + except SomeThing: + pass + finally: + 42 + """ + ) + assert len(source) == 7 + # check all lineno's that could occur in a traceback + # assert source.getstatementrange(0) == (0, 7) + # assert source.getstatementrange(1) == (1, 5) + assert source.getstatementrange(2) == (2, 3) + assert source.getstatementrange(3) == (3, 4) + assert source.getstatementrange(4) == (4, 5) + # assert source.getstatementrange(5) == (0, 7) + assert source.getstatementrange(6) == (6, 7) + + def test_getstatementrange_bug(self) -> None: + source = Source( + """\ + try: + x = ( + y + + z) + except: + pass + """ + ) + assert len(source) == 6 + assert source.getstatementrange(2) == (1, 4) + + def test_getstatementrange_bug2(self) -> None: + source = Source( + """\ + assert ( + 33 + == + [ + X(3, + b=1, c=2 + ), + ] + ) + """ + ) + assert len(source) == 9 + assert source.getstatementrange(5) == (0, 9) + + def test_getstatementrange_ast_issue58(self) -> None: + source = Source( + """\ + + def test_some(): + for a in [a for a in + CAUSE_ERROR]: pass + + x = 3 + """ + ) + assert getstatement(2, source).lines == source.lines[2:3] + assert getstatement(3, source).lines == source.lines[3:4] + + def test_getstatementrange_out_of_bounds_py3(self) -> None: + source = Source("if xxx:\n from .collections import something") + r = source.getstatementrange(1) + assert r == (1, 2) + + def test_getstatementrange_with_syntaxerror_issue7(self) -> None: + source = Source(":") + pytest.raises(SyntaxError, lambda: source.getstatementrange(0)) + + +def test_getstartingblock_singleline() -> None: + class A: + def __init__(self, *args) -> None: + frame = sys._getframe(1) + self.source = Frame(frame).statement + + x = A("x", "y") + + values = [i for i in x.source.lines if i.strip()] + assert len(values) == 1 + + +def test_getline_finally() -> None: + def c() -> None: + pass + + with pytest.raises(TypeError) as excinfo: + teardown = None + try: + c(1) # type: ignore + finally: + if teardown: + teardown() # type: ignore[unreachable] + source = excinfo.traceback[-1].statement + assert str(source).strip() == "c(1) # type: ignore" + + +def test_getfuncsource_dynamic() -> None: + def f(): + raise NotImplementedError() + + def g(): + pass # pragma: no cover + + f_source = Source(f) + g_source = Source(g) + assert str(f_source).strip() == "def f():\n raise NotImplementedError()" + assert str(g_source).strip() == "def g():\n pass # pragma: no cover" + + +def test_getfuncsource_with_multine_string() -> None: + def f(): + c = """while True: + pass +""" + + expected = '''\ + def f(): + c = """while True: + pass +""" +''' + assert str(Source(f)) == expected.rstrip() + + +def test_deindent() -> None: + from _pytest._code.source import deindent as deindent + + assert deindent(["\tfoo", "\tbar"]) == ["foo", "bar"] + + source = """\ + def f(): + def g(): + pass + """ + lines = deindent(source.splitlines()) + assert lines == ["def f():", " def g():", " pass"] + + +def test_source_of_class_at_eof_without_newline(_sys_snapshot, tmp_path: Path) -> None: + # this test fails because the implicit inspect.getsource(A) below + # does not return the "x = 1" last line. + source = Source( + """ + class A: + def method(self): + x = 1 + """ + ) + path = tmp_path.joinpath("a.py") + path.write_text(str(source)) + mod: Any = import_path(path, root=tmp_path) + s2 = Source(mod.A) + assert str(source).strip() == str(s2).strip() + + +if True: + + def x(): + pass + + +def test_source_fallback() -> None: + src = Source(x) + expected = """def x(): + pass""" + assert str(src) == expected + + +def test_findsource_fallback() -> None: + from _pytest._code.source import findsource + + src, lineno = findsource(x) + assert src is not None + assert "test_findsource_simple" in str(src) + assert src[lineno] == " def x():" + + +def test_findsource(monkeypatch) -> None: + from _pytest._code.source import findsource + + filename = "" + lines = ["if 1:\n", " def x():\n", " pass\n"] + co = compile("".join(lines), filename, "exec") + + # Type ignored because linecache.cache is private. + monkeypatch.setitem(linecache.cache, filename, (1, None, lines, filename)) # type: ignore[attr-defined] + + src, lineno = findsource(co) + assert src is not None + assert "if 1:" in str(src) + + d: Dict[str, Any] = {} + eval(co, d) + src, lineno = findsource(d["x"]) + assert src is not None + assert "if 1:" in str(src) + assert src[lineno] == " def x():" + + +def test_getfslineno() -> None: + def f(x) -> None: + raise NotImplementedError() + + fspath, lineno = getfslineno(f) + + assert isinstance(fspath, Path) + assert fspath.name == "test_source.py" + assert lineno == f.__code__.co_firstlineno - 1 # see findsource + + class A: + pass + + fspath, lineno = getfslineno(A) + + _, A_lineno = inspect.findsource(A) + assert isinstance(fspath, Path) + assert fspath.name == "test_source.py" + assert lineno == A_lineno + + assert getfslineno(3) == ("", -1) + + class B: + pass + + B.__name__ = B.__qualname__ = "B2" + assert getfslineno(B)[1] == -1 + + +def test_code_of_object_instance_with_call() -> None: + class A: + pass + + pytest.raises(TypeError, lambda: Source(A())) + + class WithCall: + def __call__(self) -> None: + pass + + code = Code.from_function(WithCall()) + assert "pass" in str(code.source()) + + class Hello: + def __call__(self) -> None: + pass + + pytest.raises(TypeError, lambda: Code.from_function(Hello)) + + +def getstatement(lineno: int, source) -> Source: + from _pytest._code.source import getstatementrange_ast + + src = Source(source) + ast, start, end = getstatementrange_ast(lineno, src) + return src[start:end] + + +def test_oneline() -> None: + source = getstatement(0, "raise ValueError") + assert str(source) == "raise ValueError" + + +def test_comment_and_no_newline_at_end() -> None: + from _pytest._code.source import getstatementrange_ast + + source = Source( + [ + "def test_basic_complex():", + " assert 1 == 2", + "# vim: filetype=pyopencl:fdm=marker", + ] + ) + ast, start, end = getstatementrange_ast(1, source) + assert end == 2 + + +def test_oneline_and_comment() -> None: + source = getstatement(0, "raise ValueError\n#hello") + assert str(source) == "raise ValueError" + + +def test_comments() -> None: + source = '''def test(): + "comment 1" + x = 1 + # comment 2 + # comment 3 + + assert False + +""" +comment 4 +""" +''' + for line in range(2, 6): + assert str(getstatement(line, source)) == " x = 1" + if sys.version_info >= (3, 8) or hasattr(sys, "pypy_version_info"): + tqs_start = 8 + else: + tqs_start = 10 + assert str(getstatement(10, source)) == '"""' + for line in range(6, tqs_start): + assert str(getstatement(line, source)) == " assert False" + for line in range(tqs_start, 10): + assert str(getstatement(line, source)) == '"""\ncomment 4\n"""' + + +def test_comment_in_statement() -> None: + source = """test(foo=1, + # comment 1 + bar=2) +""" + for line in range(1, 3): + assert ( + str(getstatement(line, source)) + == "test(foo=1,\n # comment 1\n bar=2)" + ) + + +def test_source_with_decorator() -> None: + """Test behavior with Source / Code().source with regard to decorators.""" + from _pytest.compat import get_real_func + + @pytest.mark.foo + def deco_mark(): + assert False + + src = inspect.getsource(deco_mark) + assert textwrap.indent(str(Source(deco_mark)), " ") + "\n" == src + assert src.startswith(" @pytest.mark.foo") + + @pytest.fixture + def deco_fixture(): + assert False + + src = inspect.getsource(deco_fixture) + assert src == " @pytest.fixture\n def deco_fixture():\n assert False\n" + # currently Source does not unwrap decorators, testing the + # existing behavior here for explicitness, but perhaps we should revisit/change this + # in the future + assert str(Source(deco_fixture)).startswith("@functools.wraps(function)") + assert ( + textwrap.indent(str(Source(get_real_func(deco_fixture))), " ") + "\n" == src + ) + + +def test_single_line_else() -> None: + source = getstatement(1, "if False: 2\nelse: 3") + assert str(source) == "else: 3" + + +def test_single_line_finally() -> None: + source = getstatement(1, "try: 1\nfinally: 3") + assert str(source) == "finally: 3" + + +def test_issue55() -> None: + source = ( + "def round_trip(dinp):\n assert 1 == dinp\n" + 'def test_rt():\n round_trip("""\n""")\n' + ) + s = getstatement(3, source) + assert str(s) == ' round_trip("""\n""")' + + +def test_multiline() -> None: + source = getstatement( + 0, + """\ +raise ValueError( + 23 +) +x = 3 +""", + ) + assert str(source) == "raise ValueError(\n 23\n)" + + +class TestTry: + def setup_class(self) -> None: + self.source = """\ +try: + raise ValueError +except Something: + raise IndexError(1) +else: + raise KeyError() +""" + + def test_body(self) -> None: + source = getstatement(1, self.source) + assert str(source) == " raise ValueError" + + def test_except_line(self) -> None: + source = getstatement(2, self.source) + assert str(source) == "except Something:" + + def test_except_body(self) -> None: + source = getstatement(3, self.source) + assert str(source) == " raise IndexError(1)" + + def test_else(self) -> None: + source = getstatement(5, self.source) + assert str(source) == " raise KeyError()" + + +class TestTryFinally: + def setup_class(self) -> None: + self.source = """\ +try: + raise ValueError +finally: + raise IndexError(1) +""" + + def test_body(self) -> None: + source = getstatement(1, self.source) + assert str(source) == " raise ValueError" + + def test_finally(self) -> None: + source = getstatement(3, self.source) + assert str(source) == " raise IndexError(1)" + + +class TestIf: + def setup_class(self) -> None: + self.source = """\ +if 1: + y = 3 +elif False: + y = 5 +else: + y = 7 +""" + + def test_body(self) -> None: + source = getstatement(1, self.source) + assert str(source) == " y = 3" + + def test_elif_clause(self) -> None: + source = getstatement(2, self.source) + assert str(source) == "elif False:" + + def test_elif(self) -> None: + source = getstatement(3, self.source) + assert str(source) == " y = 5" + + def test_else(self) -> None: + source = getstatement(5, self.source) + assert str(source) == " y = 7" + + +def test_semicolon() -> None: + s = """\ +hello ; pytest.skip() +""" + source = getstatement(0, s) + assert str(source) == s.strip() + + +def test_def_online() -> None: + s = """\ +def func(): raise ValueError(42) + +def something(): + pass +""" + source = getstatement(0, s) + assert str(source) == "def func(): raise ValueError(42)" + + +def test_decorator() -> None: + s = """\ +def foo(f): + pass + +@foo +def bar(): + pass + """ + source = getstatement(3, s) + assert "@foo" in str(source) + + +def XXX_test_expression_multiline() -> None: + source = """\ +something +''' +'''""" + result = getstatement(1, source) + assert str(result) == "'''\n'''" + + +def test_getstartingblock_multiline() -> None: + class A: + def __init__(self, *args): + frame = sys._getframe(1) + self.source = Frame(frame).statement + + # fmt: off + x = A('x', + 'y' + , + 'z') + # fmt: on + values = [i for i in x.source.lines if i.strip()] + assert len(values) == 4 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/conftest.py new file mode 100644 index 0000000000..107aad86b2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/conftest.py @@ -0,0 +1,216 @@ +import re +import sys +from typing import List + +import pytest +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester + +if sys.gettrace(): + + @pytest.fixture(autouse=True) + def restore_tracing(): + """Restore tracing function (when run with Coverage.py). + + https://bugs.python.org/issue37011 + """ + orig_trace = sys.gettrace() + yield + if sys.gettrace() != orig_trace: + sys.settrace(orig_trace) + + +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_collection_modifyitems(items): + """Prefer faster tests. + + Use a hookwrapper to do this in the beginning, so e.g. --ff still works + correctly. + """ + fast_items = [] + slow_items = [] + slowest_items = [] + neutral_items = [] + + spawn_names = {"spawn_pytest", "spawn"} + + for item in items: + try: + fixtures = item.fixturenames + except AttributeError: + # doctest at least + # (https://github.com/pytest-dev/pytest/issues/5070) + neutral_items.append(item) + else: + if "pytester" in fixtures: + co_names = item.function.__code__.co_names + if spawn_names.intersection(co_names): + item.add_marker(pytest.mark.uses_pexpect) + slowest_items.append(item) + elif "runpytest_subprocess" in co_names: + slowest_items.append(item) + else: + slow_items.append(item) + item.add_marker(pytest.mark.slow) + else: + marker = item.get_closest_marker("slow") + if marker: + slowest_items.append(item) + else: + fast_items.append(item) + + items[:] = fast_items + neutral_items + slow_items + slowest_items + + yield + + +@pytest.fixture +def tw_mock(): + """Returns a mock terminal writer""" + + class TWMock: + WRITE = object() + + def __init__(self): + self.lines = [] + self.is_writing = False + + def sep(self, sep, line=None): + self.lines.append((sep, line)) + + def write(self, msg, **kw): + self.lines.append((TWMock.WRITE, msg)) + + def _write_source(self, lines, indents=()): + if not indents: + indents = [""] * len(lines) + for indent, line in zip(indents, lines): + self.line(indent + line) + + def line(self, line, **kw): + self.lines.append(line) + + def markup(self, text, **kw): + return text + + def get_write_msg(self, idx): + flag, msg = self.lines[idx] + assert flag == TWMock.WRITE + return msg + + fullwidth = 80 + + return TWMock() + + +@pytest.fixture +def dummy_yaml_custom_test(pytester: Pytester): + """Writes a conftest file that collects and executes a dummy yaml test. + + Taken from the docs, but stripped down to the bare minimum, useful for + tests which needs custom items collected. + """ + pytester.makeconftest( + """ + import pytest + + def pytest_collect_file(parent, file_path): + if file_path.suffix == ".yaml" and file_path.name.startswith("test"): + return YamlFile.from_parent(path=file_path, parent=parent) + + class YamlFile(pytest.File): + def collect(self): + yield YamlItem.from_parent(name=self.path.name, parent=self) + + class YamlItem(pytest.Item): + def runtest(self): + pass + """ + ) + pytester.makefile(".yaml", test1="") + + +@pytest.fixture +def pytester(pytester: Pytester, monkeypatch: MonkeyPatch) -> Pytester: + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + return pytester + + +@pytest.fixture(scope="session") +def color_mapping(): + """Returns a utility class which can replace keys in strings in the form "{NAME}" + by their equivalent ASCII codes in the terminal. + + Used by tests which check the actual colors output by pytest. + """ + + class ColorMapping: + COLORS = { + "red": "\x1b[31m", + "green": "\x1b[32m", + "yellow": "\x1b[33m", + "bold": "\x1b[1m", + "reset": "\x1b[0m", + "kw": "\x1b[94m", + "hl-reset": "\x1b[39;49;00m", + "function": "\x1b[92m", + "number": "\x1b[94m", + "str": "\x1b[33m", + "print": "\x1b[96m", + } + RE_COLORS = {k: re.escape(v) for k, v in COLORS.items()} + + @classmethod + def format(cls, lines: List[str]) -> List[str]: + """Straightforward replacement of color names to their ASCII codes.""" + return [line.format(**cls.COLORS) for line in lines] + + @classmethod + def format_for_fnmatch(cls, lines: List[str]) -> List[str]: + """Replace color names for use with LineMatcher.fnmatch_lines""" + return [line.format(**cls.COLORS).replace("[", "[[]") for line in lines] + + @classmethod + def format_for_rematch(cls, lines: List[str]) -> List[str]: + """Replace color names for use with LineMatcher.re_match_lines""" + return [line.format(**cls.RE_COLORS) for line in lines] + + return ColorMapping + + +@pytest.fixture +def mock_timing(monkeypatch: MonkeyPatch): + """Mocks _pytest.timing with a known object that can be used to control timing in tests + deterministically. + + pytest itself should always use functions from `_pytest.timing` instead of `time` directly. + + This then allows us more control over time during testing, if testing code also + uses `_pytest.timing` functions. + + Time is static, and only advances through `sleep` calls, thus tests might sleep over large + numbers and obtain accurate time() calls at the end, making tests reliable and instant. + """ + import attr + + @attr.s + class MockTiming: + + _current_time = attr.ib(default=1590150050.0) + + def sleep(self, seconds): + self._current_time += seconds + + def time(self): + return self._current_time + + def patch(self): + from _pytest import timing + + monkeypatch.setattr(timing, "sleep", self.sleep) + monkeypatch.setattr(timing, "time", self.time) + monkeypatch.setattr(timing, "perf_counter", self.time) + + result = MockTiming() + result.patch() + return result diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/deprecated_test.py b/testing/web-platform/tests/tools/third_party/pytest/testing/deprecated_test.py new file mode 100644 index 0000000000..9ac7fe1cac --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/deprecated_test.py @@ -0,0 +1,310 @@ +import re +import sys +import warnings +from pathlib import Path +from unittest import mock + +import pytest +from _pytest import deprecated +from _pytest.compat import legacy_path +from _pytest.pytester import Pytester +from pytest import PytestDeprecationWarning + + +@pytest.mark.parametrize("attribute", pytest.collect.__all__) # type: ignore +# false positive due to dynamic attribute +def test_pytest_collect_module_deprecated(attribute) -> None: + with pytest.warns(DeprecationWarning, match=attribute): + getattr(pytest.collect, attribute) + + +@pytest.mark.parametrize("plugin", sorted(deprecated.DEPRECATED_EXTERNAL_PLUGINS)) +@pytest.mark.filterwarnings("default") +def test_external_plugins_integrated(pytester: Pytester, plugin) -> None: + pytester.syspathinsert() + pytester.makepyfile(**{plugin: ""}) + + with pytest.warns(pytest.PytestConfigWarning): + pytester.parseconfig("-p", plugin) + + +def test_fillfuncargs_is_deprecated() -> None: + with pytest.warns( + pytest.PytestDeprecationWarning, + match=re.escape( + "pytest._fillfuncargs() is deprecated, use " + "function._request._fillfixtures() instead if you cannot avoid reaching into internals." + ), + ): + pytest._fillfuncargs(mock.Mock()) + + +def test_fillfixtures_is_deprecated() -> None: + import _pytest.fixtures + + with pytest.warns( + pytest.PytestDeprecationWarning, + match=re.escape( + "_pytest.fixtures.fillfixtures() is deprecated, use " + "function._request._fillfixtures() instead if you cannot avoid reaching into internals." + ), + ): + _pytest.fixtures.fillfixtures(mock.Mock()) + + +def test_minus_k_dash_is_deprecated(pytester: Pytester) -> None: + threepass = pytester.makepyfile( + test_threepass=""" + def test_one(): assert 1 + def test_two(): assert 1 + def test_three(): assert 1 + """ + ) + result = pytester.runpytest("-k=-test_two", threepass) + result.stdout.fnmatch_lines(["*The `-k '-expr'` syntax*deprecated*"]) + + +def test_minus_k_colon_is_deprecated(pytester: Pytester) -> None: + threepass = pytester.makepyfile( + test_threepass=""" + def test_one(): assert 1 + def test_two(): assert 1 + def test_three(): assert 1 + """ + ) + result = pytester.runpytest("-k", "test_two:", threepass) + result.stdout.fnmatch_lines(["*The `-k 'expr:'` syntax*deprecated*"]) + + +def test_fscollector_gethookproxy_isinitpath(pytester: Pytester) -> None: + module = pytester.getmodulecol( + """ + def test_foo(): pass + """, + withinit=True, + ) + assert isinstance(module, pytest.Module) + package = module.parent + assert isinstance(package, pytest.Package) + + with pytest.warns(pytest.PytestDeprecationWarning, match="gethookproxy"): + package.gethookproxy(pytester.path) + + with pytest.warns(pytest.PytestDeprecationWarning, match="isinitpath"): + package.isinitpath(pytester.path) + + # The methods on Session are *not* deprecated. + session = module.session + with warnings.catch_warnings(record=True) as rec: + session.gethookproxy(pytester.path) + session.isinitpath(pytester.path) + assert len(rec) == 0 + + +def test_strict_option_is_deprecated(pytester: Pytester) -> None: + """--strict is a deprecated alias to --strict-markers (#7530).""" + pytester.makepyfile( + """ + import pytest + + @pytest.mark.unknown + def test_foo(): pass + """ + ) + result = pytester.runpytest("--strict") + result.stdout.fnmatch_lines( + [ + "'unknown' not found in `markers` configuration option", + "*PytestRemovedIn8Warning: The --strict option is deprecated, use --strict-markers instead.", + ] + ) + + +def test_yield_fixture_is_deprecated() -> None: + with pytest.warns(DeprecationWarning, match=r"yield_fixture is deprecated"): + + @pytest.yield_fixture + def fix(): + assert False + + +def test_private_is_deprecated() -> None: + class PrivateInit: + def __init__(self, foo: int, *, _ispytest: bool = False) -> None: + deprecated.check_ispytest(_ispytest) + + with pytest.warns( + pytest.PytestDeprecationWarning, match="private pytest class or function" + ): + PrivateInit(10) + + # Doesn't warn. + PrivateInit(10, _ispytest=True) + + +def test_raising_unittest_skiptest_during_collection_is_deprecated( + pytester: Pytester, +) -> None: + pytester.makepyfile( + """ + import unittest + raise unittest.SkipTest() + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*PytestRemovedIn8Warning: Raising unittest.SkipTest*", + ] + ) + + +@pytest.mark.parametrize("hooktype", ["hook", "ihook"]) +def test_hookproxy_warnings_for_pathlib(tmp_path, hooktype, request): + path = legacy_path(tmp_path) + + PATH_WARN_MATCH = r".*path: py\.path\.local\) argument is deprecated, please use \(collection_path: pathlib\.Path.*" + if hooktype == "ihook": + hooks = request.node.ihook + else: + hooks = request.config.hook + + with pytest.warns(PytestDeprecationWarning, match=PATH_WARN_MATCH) as r: + l1 = sys._getframe().f_lineno + hooks.pytest_ignore_collect( + config=request.config, path=path, collection_path=tmp_path + ) + l2 = sys._getframe().f_lineno + + (record,) = r + assert record.filename == __file__ + assert l1 < record.lineno < l2 + + hooks.pytest_ignore_collect(config=request.config, collection_path=tmp_path) + + # Passing entirely *different* paths is an outright error. + with pytest.raises(ValueError, match=r"path.*fspath.*need to be equal"): + with pytest.warns(PytestDeprecationWarning, match=PATH_WARN_MATCH) as r: + hooks.pytest_ignore_collect( + config=request.config, path=path, collection_path=Path("/bla/bla") + ) + + +def test_warns_none_is_deprecated(): + with pytest.warns( + PytestDeprecationWarning, + match=re.escape( + "Passing None has been deprecated.\n" + "See https://docs.pytest.org/en/latest/how-to/capture-warnings.html" + "#additional-use-cases-of-warnings-in-tests" + " for alternatives in common use cases." + ), + ): + with pytest.warns(None): # type: ignore[call-overload] + pass + + +class TestSkipMsgArgumentDeprecated: + def test_skip_with_msg_is_deprecated(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + def test_skipping_msg(): + pytest.skip(msg="skippedmsg") + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + [ + "*PytestRemovedIn8Warning: pytest.skip(msg=...) is now deprecated, " + "use pytest.skip(reason=...) instead", + '*pytest.skip(msg="skippedmsg")*', + ] + ) + result.assert_outcomes(skipped=1, warnings=1) + + def test_fail_with_msg_is_deprecated(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + def test_failing_msg(): + pytest.fail(msg="failedmsg") + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + [ + "*PytestRemovedIn8Warning: pytest.fail(msg=...) is now deprecated, " + "use pytest.fail(reason=...) instead", + '*pytest.fail(msg="failedmsg")', + ] + ) + result.assert_outcomes(failed=1, warnings=1) + + def test_exit_with_msg_is_deprecated(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + def test_exit_msg(): + pytest.exit(msg="exitmsg") + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + [ + "*PytestRemovedIn8Warning: pytest.exit(msg=...) is now deprecated, " + "use pytest.exit(reason=...) instead", + ] + ) + result.assert_outcomes(warnings=1) + + +def test_deprecation_of_cmdline_preparse(pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_cmdline_preparse(config, args): + ... + + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*PytestRemovedIn8Warning: The pytest_cmdline_preparse hook is deprecated*", + "*Please use pytest_load_initial_conftests hook instead.*", + ] + ) + + +def test_node_ctor_fspath_argument_is_deprecated(pytester: Pytester) -> None: + mod = pytester.getmodulecol("") + + with pytest.warns( + pytest.PytestDeprecationWarning, + match=re.escape("The (fspath: py.path.local) argument to File is deprecated."), + ): + pytest.File.from_parent( + parent=mod.parent, + fspath=legacy_path("bla"), + ) + + +@pytest.mark.skipif( + sys.version_info < (3, 7), + reason="This deprecation can only be emitted on python>=3.7", +) +def test_importing_instance_is_deprecated(pytester: Pytester) -> None: + with pytest.warns( + pytest.PytestDeprecationWarning, + match=re.escape("The pytest.Instance collector type is deprecated"), + ): + pytest.Instance + + with pytest.warns( + pytest.PytestDeprecationWarning, + match=re.escape("The pytest.Instance collector type is deprecated"), + ): + from _pytest.python import Instance # noqa: F401 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/README.rst b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/README.rst new file mode 100644 index 0000000000..97d0fda5c5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/README.rst @@ -0,0 +1,9 @@ +Example test scripts +===================== + + +The files in this folder are not direct tests, but rather example test suites that demonstrate certain issues/behaviours. + +In the future we will move part of the content of the acceptance tests here in order to have directly testable code instead of writing out things and then running them in nested pytest sessions/subprocesses. + +This will aid debugging and comprehension. diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/acceptance/fixture_mock_integration.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/acceptance/fixture_mock_integration.py new file mode 100644 index 0000000000..5b00ac90e1 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/acceptance/fixture_mock_integration.py @@ -0,0 +1,16 @@ +"""Reproduces issue #3774""" +from unittest import mock + +import pytest + +config = {"mykey": "ORIGINAL"} + + +@pytest.fixture(scope="function") +@mock.patch.dict(config, {"mykey": "MOCKED"}) +def my_fixture(): + return config["mykey"] + + +def test_foobar(my_fixture): + assert my_fixture == "MOCKED" diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/collect_init_tests/pytest.ini b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/collect_init_tests/pytest.ini new file mode 100644 index 0000000000..7c47955402 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/collect_init_tests/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +python_files = *.py diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/collect_init_tests/tests/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/collect_init_tests/tests/__init__.py new file mode 100644 index 0000000000..9cd366295e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/collect_init_tests/tests/__init__.py @@ -0,0 +1,2 @@ +def test_init(): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/collect_init_tests/tests/test_foo.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/collect_init_tests/tests/test_foo.py new file mode 100644 index 0000000000..8f2d73cfa4 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/collect_init_tests/tests/test_foo.py @@ -0,0 +1,2 @@ +def test_foo(): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/__init__.pyi b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/__init__.pyi new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/conftest.py new file mode 100644 index 0000000000..973ccc0c03 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/conftest.py @@ -0,0 +1,2 @@ +def pytest_ignore_collect(collection_path): + return False diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/tests/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/tests/test_basic.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/tests/test_basic.py new file mode 100644 index 0000000000..f174823854 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_infinite_recursion/tests/test_basic.py @@ -0,0 +1,2 @@ +def test(): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_init_given_as_arg/pkg/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_init_given_as_arg/pkg/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_init_given_as_arg/pkg/test_foo.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_init_given_as_arg/pkg/test_foo.py new file mode 100644 index 0000000000..f174823854 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/collect/package_init_given_as_arg/pkg/test_foo.py @@ -0,0 +1,2 @@ +def test(): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/config/collect_pytest_prefix/__init__.pyi b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/config/collect_pytest_prefix/__init__.pyi new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/config/collect_pytest_prefix/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/config/collect_pytest_prefix/conftest.py new file mode 100644 index 0000000000..2da4ffe2fe --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/config/collect_pytest_prefix/conftest.py @@ -0,0 +1,2 @@ +class pytest_something: + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/config/collect_pytest_prefix/test_foo.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/config/collect_pytest_prefix/test_foo.py new file mode 100644 index 0000000000..8f2d73cfa4 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/config/collect_pytest_prefix/test_foo.py @@ -0,0 +1,2 @@ +def test_foo(): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/conftest_usageerror/__init__.pyi b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/conftest_usageerror/__init__.pyi new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/conftest_usageerror/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/conftest_usageerror/conftest.py new file mode 100644 index 0000000000..8973e4252d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/conftest_usageerror/conftest.py @@ -0,0 +1,8 @@ +def pytest_configure(config): + import pytest + + raise pytest.UsageError("hello") + + +def pytest_unconfigure(config): + print("pytest_unconfigure_called") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_dataclasses.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_dataclasses.py new file mode 100644 index 0000000000..d96c90a91b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_dataclasses.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass +from dataclasses import field + + +def test_dataclasses() -> None: + @dataclass + class SimpleDataObject: + field_a: int = field() + field_b: str = field() + + left = SimpleDataObject(1, "b") + right = SimpleDataObject(1, "c") + + assert left == right diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_dataclasses_field_comparison_off.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_dataclasses_field_comparison_off.py new file mode 100644 index 0000000000..7479c66c1b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_dataclasses_field_comparison_off.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass +from dataclasses import field + + +def test_dataclasses_with_attribute_comparison_off() -> None: + @dataclass + class SimpleDataObject: + field_a: int = field() + field_b: str = field(compare=False) + + left = SimpleDataObject(1, "b") + right = SimpleDataObject(1, "c") + + assert left == right diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_dataclasses_verbose.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_dataclasses_verbose.py new file mode 100644 index 0000000000..4737ef904e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_dataclasses_verbose.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass +from dataclasses import field + + +def test_dataclasses_verbose() -> None: + @dataclass + class SimpleDataObject: + field_a: int = field() + field_b: str = field() + + left = SimpleDataObject(1, "b") + right = SimpleDataObject(1, "c") + + assert left == right diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_recursive_dataclasses.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_recursive_dataclasses.py new file mode 100644 index 0000000000..0945790f00 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_recursive_dataclasses.py @@ -0,0 +1,44 @@ +from dataclasses import dataclass + + +@dataclass +class S: + a: int + b: str + + +@dataclass +class C: + c: S + d: S + + +@dataclass +class C2: + e: C + f: S + + +@dataclass +class C3: + g: S + h: C2 + i: str + j: str + + +def test_recursive_dataclasses(): + left = C3( + S(10, "ten"), + C2(C(S(1, "one"), S(2, "two")), S(2, "three")), + "equal", + "left", + ) + right = C3( + S(20, "xxx"), + C2(C(S(1, "one"), S(2, "yyy")), S(3, "three")), + "equal", + "right", + ) + + assert left == right diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_two_different_dataclasses.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_two_different_dataclasses.py new file mode 100644 index 0000000000..0a4820c69b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/dataclasses/test_compare_two_different_dataclasses.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass +from dataclasses import field + + +def test_comparing_two_different_data_classes() -> None: + @dataclass + class SimpleDataObjectOne: + field_a: int = field() + field_b: str = field() + + @dataclass + class SimpleDataObjectTwo: + field_a: int = field() + field_b: str = field() + + left = SimpleDataObjectOne(1, "b") + right = SimpleDataObjectTwo(1, "c") + + assert left != right # type: ignore[comparison-overlap] diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/doctest/main_py/__main__.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/doctest/main_py/__main__.py new file mode 100644 index 0000000000..e471d06d64 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/doctest/main_py/__main__.py @@ -0,0 +1,2 @@ +def test_this_is_ignored(): + assert True diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/doctest/main_py/test_normal_module.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/doctest/main_py/test_normal_module.py new file mode 100644 index 0000000000..700cc9750c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/doctest/main_py/test_normal_module.py @@ -0,0 +1,6 @@ +def test_doc(): + """ + >>> 10 > 5 + True + """ + assert False diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/__init__.pyi b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/__init__.pyi new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/conftest.py new file mode 100644 index 0000000000..a7a5e9db80 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/conftest.py @@ -0,0 +1,15 @@ +import pytest + + +class CustomItem(pytest.Item): + def runtest(self): + pass + + +class CustomFile(pytest.File): + def collect(self): + yield CustomItem.from_parent(name="foo", parent=self) + + +def pytest_collect_file(file_path, parent): + return CustomFile.from_parent(path=file_path, parent=parent) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/foo/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/foo/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/foo/test_foo.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/foo/test_foo.py new file mode 100644 index 0000000000..f174823854 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/custom_item/foo/test_foo.py @@ -0,0 +1,2 @@ +def test(): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub1/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub1/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub1/conftest.py new file mode 100644 index 0000000000..be5adbeb6e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub1/conftest.py @@ -0,0 +1,7 @@ +import pytest + + +@pytest.fixture +def arg1(request): + with pytest.raises(pytest.FixtureLookupError): + request.getfixturevalue("arg2") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub1/test_in_sub1.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub1/test_in_sub1.py new file mode 100644 index 0000000000..df36da1369 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub1/test_in_sub1.py @@ -0,0 +1,2 @@ +def test_1(arg1): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/conftest.py new file mode 100644 index 0000000000..00981c5dc1 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/conftest.py @@ -0,0 +1,6 @@ +import pytest + + +@pytest.fixture +def arg2(request): + pytest.raises(Exception, request.getfixturevalue, "arg1") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/test_in_sub2.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/test_in_sub2.py new file mode 100644 index 0000000000..1c34f94acc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/test_in_sub2.py @@ -0,0 +1,2 @@ +def test_2(arg2): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_detect_recursive_dependency_error.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_detect_recursive_dependency_error.py new file mode 100644 index 0000000000..d1efcbb338 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_detect_recursive_dependency_error.py @@ -0,0 +1,15 @@ +import pytest + + +@pytest.fixture +def fix1(fix2): + return 1 + + +@pytest.fixture +def fix2(fix1): + return 1 + + +def test(fix1): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/__init__.pyi b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/__init__.pyi new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/conftest.py new file mode 100644 index 0000000000..5dfd2f7795 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/conftest.py @@ -0,0 +1,6 @@ +import pytest + + +@pytest.fixture +def spam(): + return "spam" diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/pkg/__init__.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/pkg/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/pkg/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/pkg/conftest.py new file mode 100644 index 0000000000..4e22ce5a13 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/pkg/conftest.py @@ -0,0 +1,6 @@ +import pytest + + +@pytest.fixture +def spam(spam): + return spam * 2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/pkg/test_spam.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/pkg/test_spam.py new file mode 100644 index 0000000000..0d891fbb50 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_conftest/pkg/test_spam.py @@ -0,0 +1,2 @@ +def test_spam(spam): + assert spam == "spamspam" diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_module/__init__.pyi b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_module/__init__.pyi new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_module/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_module/conftest.py new file mode 100644 index 0000000000..5dfd2f7795 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_module/conftest.py @@ -0,0 +1,6 @@ +import pytest + + +@pytest.fixture +def spam(): + return "spam" diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_module/test_extend_fixture_conftest_module.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_module/test_extend_fixture_conftest_module.py new file mode 100644 index 0000000000..46d1446f47 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_conftest_module/test_extend_fixture_conftest_module.py @@ -0,0 +1,10 @@ +import pytest + + +@pytest.fixture +def spam(spam): + return spam * 2 + + +def test_spam(spam): + assert spam == "spamspam" diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_module_class.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_module_class.py new file mode 100644 index 0000000000..87a0c89411 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_extend_fixture_module_class.py @@ -0,0 +1,15 @@ +import pytest + + +@pytest.fixture +def spam(): + return "spam" + + +class TestSpam: + @pytest.fixture + def spam(self, spam): + return spam * 2 + + def test_spam(self, spam): + assert spam == "spamspam" diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_basic.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_basic.py new file mode 100644 index 0000000000..0661cb301f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_basic.py @@ -0,0 +1,15 @@ +import pytest + + +@pytest.fixture +def some(request): + return request.function.__name__ + + +@pytest.fixture +def other(request): + return 42 + + +def test_func(some, other): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_lookup_classlevel.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_lookup_classlevel.py new file mode 100644 index 0000000000..256b92a17d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_lookup_classlevel.py @@ -0,0 +1,10 @@ +import pytest + + +class TestClass: + @pytest.fixture + def something(self, request): + return request.instance + + def test_method(self, something): + assert something is self diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_lookup_modulelevel.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_lookup_modulelevel.py new file mode 100644 index 0000000000..e15dbd2ca4 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_lookup_modulelevel.py @@ -0,0 +1,15 @@ +import pytest + + +@pytest.fixture +def something(request): + return request.function.__name__ + + +class TestClass: + def test_method(self, something): + assert something == "test_method" + + +def test_func(something): + assert something == "test_func" diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_lookupfails.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_lookupfails.py new file mode 100644 index 0000000000..b775203231 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/fill_fixtures/test_funcarg_lookupfails.py @@ -0,0 +1,10 @@ +import pytest + + +@pytest.fixture +def xyzsomething(request): + return 42 + + +def test_func(some): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/test_fixture_named_request.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/test_fixture_named_request.py new file mode 100644 index 0000000000..75514bf8b8 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/test_fixture_named_request.py @@ -0,0 +1,10 @@ +import pytest + + +@pytest.fixture +def request(): + pass + + +def test(): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/test_getfixturevalue_dynamic.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/test_getfixturevalue_dynamic.py new file mode 100644 index 0000000000..055a1220b1 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/fixtures/test_getfixturevalue_dynamic.py @@ -0,0 +1,20 @@ +import pytest + + +@pytest.fixture +def dynamic(): + pass + + +@pytest.fixture +def a(request): + request.getfixturevalue("dynamic") + + +@pytest.fixture +def b(a): + pass + + +def test(b, request): + assert request.fixturenames == ["b", "request", "a", "dynamic"] diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue88_initial_file_multinodes/__init__.pyi b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue88_initial_file_multinodes/__init__.pyi new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue88_initial_file_multinodes/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue88_initial_file_multinodes/conftest.py new file mode 100644 index 0000000000..cb8f5d671e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue88_initial_file_multinodes/conftest.py @@ -0,0 +1,14 @@ +import pytest + + +class MyFile(pytest.File): + def collect(self): + return [MyItem.from_parent(name="hello", parent=self)] + + +def pytest_collect_file(file_path, parent): + return MyFile.from_parent(path=file_path, parent=parent) + + +class MyItem(pytest.Item): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue88_initial_file_multinodes/test_hello.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue88_initial_file_multinodes/test_hello.py new file mode 100644 index 0000000000..56444d1474 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue88_initial_file_multinodes/test_hello.py @@ -0,0 +1,2 @@ +def test_hello(): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue_519.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue_519.py new file mode 100644 index 0000000000..e44367fca0 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/issue_519.py @@ -0,0 +1,53 @@ +import pprint +from typing import List +from typing import Tuple + +import pytest + + +def pytest_generate_tests(metafunc): + if "arg1" in metafunc.fixturenames: + metafunc.parametrize("arg1", ["arg1v1", "arg1v2"], scope="module") + + if "arg2" in metafunc.fixturenames: + metafunc.parametrize("arg2", ["arg2v1", "arg2v2"], scope="function") + + +@pytest.fixture(scope="session") +def checked_order(): + order: List[Tuple[str, str, str]] = [] + + yield order + pprint.pprint(order) + assert order == [ + ("issue_519.py", "fix1", "arg1v1"), + ("test_one[arg1v1-arg2v1]", "fix2", "arg2v1"), + ("test_two[arg1v1-arg2v1]", "fix2", "arg2v1"), + ("test_one[arg1v1-arg2v2]", "fix2", "arg2v2"), + ("test_two[arg1v1-arg2v2]", "fix2", "arg2v2"), + ("issue_519.py", "fix1", "arg1v2"), + ("test_one[arg1v2-arg2v1]", "fix2", "arg2v1"), + ("test_two[arg1v2-arg2v1]", "fix2", "arg2v1"), + ("test_one[arg1v2-arg2v2]", "fix2", "arg2v2"), + ("test_two[arg1v2-arg2v2]", "fix2", "arg2v2"), + ] + + +@pytest.fixture(scope="module") +def fix1(request, arg1, checked_order): + checked_order.append((request.node.name, "fix1", arg1)) + yield "fix1-" + arg1 + + +@pytest.fixture(scope="function") +def fix2(request, fix1, arg2, checked_order): + checked_order.append((request.node.name, "fix2", arg2)) + yield "fix2-" + arg2 + fix1 + + +def test_one(fix2): + pass + + +def test_two(fix2): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/junit-10.xsd b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/junit-10.xsd new file mode 100644 index 0000000000..286fbf7c87 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/junit-10.xsd @@ -0,0 +1,147 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/marks/marks_considered_keywords/__init__.pyi b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/marks/marks_considered_keywords/__init__.pyi new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/marks/marks_considered_keywords/conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/marks/marks_considered_keywords/conftest.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/marks/marks_considered_keywords/test_marks_as_keywords.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/marks/marks_considered_keywords/test_marks_as_keywords.py new file mode 100644 index 0000000000..35a2c7b762 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/marks/marks_considered_keywords/test_marks_as_keywords.py @@ -0,0 +1,6 @@ +import pytest + + +@pytest.mark.foo +def test_mark(): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/perf_examples/collect_stats/.gitignore b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/perf_examples/collect_stats/.gitignore new file mode 100644 index 0000000000..1c45c2ea35 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/perf_examples/collect_stats/.gitignore @@ -0,0 +1 @@ +foo_* diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/perf_examples/collect_stats/generate_folders.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/perf_examples/collect_stats/generate_folders.py new file mode 100644 index 0000000000..ff1eaf7d6b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/perf_examples/collect_stats/generate_folders.py @@ -0,0 +1,27 @@ +import argparse +import pathlib + +HERE = pathlib.Path(__file__).parent +TEST_CONTENT = (HERE / "template_test.py").read_bytes() + +parser = argparse.ArgumentParser() +parser.add_argument("numbers", nargs="*", type=int) + + +def generate_folders(root, elements, *more_numbers): + fill_len = len(str(elements)) + if more_numbers: + for i in range(elements): + new_folder = root.joinpath(f"foo_{i:0>{fill_len}}") + new_folder.mkdir() + new_folder.joinpath("__init__.py").write_bytes(TEST_CONTENT) + generate_folders(new_folder, *more_numbers) + else: + for i in range(elements): + new_test = root.joinpath(f"test_{i:0<{fill_len}}.py") + new_test.write_bytes(TEST_CONTENT) + + +if __name__ == "__main__": + args = parser.parse_args() + generate_folders(HERE, *(args.numbers or (10, 100))) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/perf_examples/collect_stats/template_test.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/perf_examples/collect_stats/template_test.py new file mode 100644 index 0000000000..064ade190a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/perf_examples/collect_stats/template_test.py @@ -0,0 +1,2 @@ +def test_x(): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/pytest.ini b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/pytest.ini new file mode 100644 index 0000000000..ec5fe0e83a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +# dummy pytest.ini to ease direct running of example scripts diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/tmpdir/tmp_path_fixture.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/tmpdir/tmp_path_fixture.py new file mode 100644 index 0000000000..8675eb2fa6 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/tmpdir/tmp_path_fixture.py @@ -0,0 +1,7 @@ +import pytest + + +@pytest.mark.parametrize("a", [r"qwe/\abc"]) +def test_fixture(tmp_path, a): + assert tmp_path.is_dir() + assert list(tmp_path.iterdir()) == [] diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_parametrized_fixture_error_message.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_parametrized_fixture_error_message.py new file mode 100644 index 0000000000..d421ce927c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_parametrized_fixture_error_message.py @@ -0,0 +1,14 @@ +import unittest + +import pytest + + +@pytest.fixture(params=[1, 2]) +def two(request): + return request.param + + +@pytest.mark.usefixtures("two") +class TestSomethingElse(unittest.TestCase): + def test_two(self): + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_setup_skip.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_setup_skip.py new file mode 100644 index 0000000000..93f79bb3b2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_setup_skip.py @@ -0,0 +1,13 @@ +"""Skipping an entire subclass with unittest.skip() should *not* call setUp from a base class.""" +import unittest + + +class Base(unittest.TestCase): + def setUp(self): + assert 0 + + +@unittest.skip("skip all tests") +class Test(Base): + def test_foo(self): + assert 0 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_setup_skip_class.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_setup_skip_class.py new file mode 100644 index 0000000000..4f251dcba1 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_setup_skip_class.py @@ -0,0 +1,14 @@ +"""Skipping an entire subclass with unittest.skip() should *not* call setUpClass from a base class.""" +import unittest + + +class Base(unittest.TestCase): + @classmethod + def setUpClass(cls): + assert 0 + + +@unittest.skip("skip all tests") +class Test(Base): + def test_foo(self): + assert 0 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_setup_skip_module.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_setup_skip_module.py new file mode 100644 index 0000000000..98befbe510 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_setup_skip_module.py @@ -0,0 +1,12 @@ +"""setUpModule is always called, even if all tests in the module are skipped""" +import unittest + + +def setUpModule(): + assert 0 + + +@unittest.skip("skip all tests") +class Base(unittest.TestCase): + def test(self): + assert 0 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_unittest_asyncio.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_unittest_asyncio.py new file mode 100644 index 0000000000..1cd2168604 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_unittest_asyncio.py @@ -0,0 +1,25 @@ +from typing import List +from unittest import IsolatedAsyncioTestCase + + +teardowns: List[None] = [] + + +class AsyncArguments(IsolatedAsyncioTestCase): + async def asyncTearDown(self): + teardowns.append(None) + + async def test_something_async(self): + async def addition(x, y): + return x + y + + self.assertEqual(await addition(2, 2), 4) + + async def test_something_async_fails(self): + async def addition(x, y): + return x + y + + self.assertEqual(await addition(2, 2), 3) + + def test_teardowns(self): + assert len(teardowns) == 2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_unittest_asynctest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_unittest_asynctest.py new file mode 100644 index 0000000000..fb26617067 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_unittest_asynctest.py @@ -0,0 +1,23 @@ +"""Issue #7110""" +import asyncio +from typing import List + +import asynctest + + +teardowns: List[None] = [] + + +class Test(asynctest.TestCase): + async def tearDown(self): + teardowns.append(None) + + async def test_error(self): + await asyncio.sleep(0) + self.fail("failing on purpose") + + async def test_ok(self): + await asyncio.sleep(0) + + def test_teardowns(self): + assert len(teardowns) == 2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_unittest_plain_async.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_unittest_plain_async.py new file mode 100644 index 0000000000..78dfece684 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/unittest/test_unittest_plain_async.py @@ -0,0 +1,6 @@ +import unittest + + +class Test(unittest.TestCase): + async def test_foo(self): + assert False diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/warnings/test_group_warnings_by_message.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/warnings/test_group_warnings_by_message.py new file mode 100644 index 0000000000..6985caa440 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/warnings/test_group_warnings_by_message.py @@ -0,0 +1,21 @@ +import warnings + +import pytest + + +def func(msg): + warnings.warn(UserWarning(msg)) + + +@pytest.mark.parametrize("i", range(5)) +def test_foo(i): + func("foo") + + +def test_foo_1(): + func("foo") + + +@pytest.mark.parametrize("i", range(5)) +def test_bar(i): + func("bar") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/warnings/test_group_warnings_by_message_summary/test_1.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/warnings/test_group_warnings_by_message_summary/test_1.py new file mode 100644 index 0000000000..b8c11cb71c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/warnings/test_group_warnings_by_message_summary/test_1.py @@ -0,0 +1,21 @@ +import warnings + +import pytest + + +def func(msg): + warnings.warn(UserWarning(msg)) + + +@pytest.mark.parametrize("i", range(20)) +def test_foo(i): + func("foo") + + +def test_foo_1(): + func("foo") + + +@pytest.mark.parametrize("i", range(20)) +def test_bar(i): + func("bar") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/warnings/test_group_warnings_by_message_summary/test_2.py b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/warnings/test_group_warnings_by_message_summary/test_2.py new file mode 100644 index 0000000000..636d04a550 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/example_scripts/warnings/test_group_warnings_by_message_summary/test_2.py @@ -0,0 +1,5 @@ +from test_1 import func + + +def test_2(): + func("foo") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/examples/test_issue519.py b/testing/web-platform/tests/tools/third_party/pytest/testing/examples/test_issue519.py new file mode 100644 index 0000000000..7b9c109889 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/examples/test_issue519.py @@ -0,0 +1,7 @@ +from _pytest.pytester import Pytester + + +def test_519(pytester: Pytester) -> None: + pytester.copy_example("issue_519.py") + res = pytester.runpytest("issue_519.py") + res.assert_outcomes(passed=8) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/.gitignore b/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/.gitignore new file mode 100644 index 0000000000..b533190872 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/.gitignore @@ -0,0 +1,3 @@ +build/ +dist/ +*.spec diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/create_executable.py b/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/create_executable.py new file mode 100644 index 0000000000..998df7b1ca --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/create_executable.py @@ -0,0 +1,11 @@ +"""Generate an executable with pytest runner embedded using PyInstaller.""" +if __name__ == "__main__": + import pytest + import subprocess + + hidden = [] + for x in pytest.freeze_includes(): + hidden.extend(["--hidden-import", x]) + hidden.extend(["--hidden-import", "distutils"]) + args = ["pyinstaller", "--noconfirm"] + hidden + ["runtests_script.py"] + subprocess.check_call(" ".join(args), shell=True) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/runtests_script.py b/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/runtests_script.py new file mode 100644 index 0000000000..591863016a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/runtests_script.py @@ -0,0 +1,10 @@ +""" +This is the script that is actually frozen into an executable: simply executes +pytest main(). +""" + +if __name__ == "__main__": + import sys + import pytest + + sys.exit(pytest.main()) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/tests/test_doctest.txt b/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/tests/test_doctest.txt new file mode 100644 index 0000000000..e18a4b68cc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/tests/test_doctest.txt @@ -0,0 +1,6 @@ + + +Testing doctest:: + + >>> 1 + 1 + 2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/tests/test_trivial.py b/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/tests/test_trivial.py new file mode 100644 index 0000000000..08a55552ab --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/tests/test_trivial.py @@ -0,0 +1,6 @@ +def test_upper(): + assert "foo".upper() == "FOO" + + +def test_lower(): + assert "FOO".lower() == "foo" diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/tox_run.py b/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/tox_run.py new file mode 100644 index 0000000000..678a69c858 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/freeze/tox_run.py @@ -0,0 +1,12 @@ +""" +Called by tox.ini: uses the generated executable to run the tests in ./tests/ +directory. +""" +if __name__ == "__main__": + import os + import sys + + executable = os.path.join(os.getcwd(), "dist", "runtests_script", "runtests_script") + if sys.platform.startswith("win"): + executable += ".exe" + sys.exit(os.system("%s tests" % executable)) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/io/test_saferepr.py b/testing/web-platform/tests/tools/third_party/pytest/testing/io/test_saferepr.py new file mode 100644 index 0000000000..63d3af822b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/io/test_saferepr.py @@ -0,0 +1,181 @@ +import pytest +from _pytest._io.saferepr import _pformat_dispatch +from _pytest._io.saferepr import DEFAULT_REPR_MAX_SIZE +from _pytest._io.saferepr import saferepr + + +def test_simple_repr(): + assert saferepr(1) == "1" + assert saferepr(None) == "None" + + +def test_maxsize(): + s = saferepr("x" * 50, maxsize=25) + assert len(s) == 25 + expected = repr("x" * 10 + "..." + "x" * 10) + assert s == expected + + +def test_no_maxsize(): + text = "x" * DEFAULT_REPR_MAX_SIZE * 10 + s = saferepr(text, maxsize=None) + expected = repr(text) + assert s == expected + + +def test_maxsize_error_on_instance(): + class A: + def __repr__(self): + raise ValueError("...") + + s = saferepr(("*" * 50, A()), maxsize=25) + assert len(s) == 25 + assert s[0] == "(" and s[-1] == ")" + + +def test_exceptions() -> None: + class BrokenRepr: + def __init__(self, ex): + self.ex = ex + + def __repr__(self): + raise self.ex + + class BrokenReprException(Exception): + __str__ = None # type: ignore[assignment] + __repr__ = None # type: ignore[assignment] + + assert "Exception" in saferepr(BrokenRepr(Exception("broken"))) + s = saferepr(BrokenReprException("really broken")) + assert "TypeError" in s + assert "TypeError" in saferepr(BrokenRepr("string")) + + none = None + try: + none() # type: ignore[misc] + except BaseException as exc: + exp_exc = repr(exc) + obj = BrokenRepr(BrokenReprException("omg even worse")) + s2 = saferepr(obj) + assert s2 == ( + "<[unpresentable exception ({!s}) raised in repr()] BrokenRepr object at 0x{:x}>".format( + exp_exc, id(obj) + ) + ) + + +def test_baseexception(): + """Test saferepr() with BaseExceptions, which includes pytest outcomes.""" + + class RaisingOnStrRepr(BaseException): + def __init__(self, exc_types): + self.exc_types = exc_types + + def raise_exc(self, *args): + try: + self.exc_type = self.exc_types.pop(0) + except IndexError: + pass + if hasattr(self.exc_type, "__call__"): + raise self.exc_type(*args) + raise self.exc_type + + def __str__(self): + self.raise_exc("__str__") + + def __repr__(self): + self.raise_exc("__repr__") + + class BrokenObj: + def __init__(self, exc): + self.exc = exc + + def __repr__(self): + raise self.exc + + __str__ = __repr__ + + baseexc_str = BaseException("__str__") + obj = BrokenObj(RaisingOnStrRepr([BaseException])) + assert saferepr(obj) == ( + "<[unpresentable exception ({!r}) " + "raised in repr()] BrokenObj object at 0x{:x}>".format(baseexc_str, id(obj)) + ) + obj = BrokenObj(RaisingOnStrRepr([RaisingOnStrRepr([BaseException])])) + assert saferepr(obj) == ( + "<[{!r} raised in repr()] BrokenObj object at 0x{:x}>".format( + baseexc_str, id(obj) + ) + ) + + with pytest.raises(KeyboardInterrupt): + saferepr(BrokenObj(KeyboardInterrupt())) + + with pytest.raises(SystemExit): + saferepr(BrokenObj(SystemExit())) + + with pytest.raises(KeyboardInterrupt): + saferepr(BrokenObj(RaisingOnStrRepr([KeyboardInterrupt]))) + + with pytest.raises(SystemExit): + saferepr(BrokenObj(RaisingOnStrRepr([SystemExit]))) + + with pytest.raises(KeyboardInterrupt): + print(saferepr(BrokenObj(RaisingOnStrRepr([BaseException, KeyboardInterrupt])))) + + with pytest.raises(SystemExit): + saferepr(BrokenObj(RaisingOnStrRepr([BaseException, SystemExit]))) + + +def test_buggy_builtin_repr(): + # Simulate a case where a repr for a builtin raises. + # reprlib dispatches by type name, so use "int". + + class int: + def __repr__(self): + raise ValueError("Buggy repr!") + + assert "Buggy" in saferepr(int()) + + +def test_big_repr(): + from _pytest._io.saferepr import SafeRepr + + assert len(saferepr(range(1000))) <= len("[" + SafeRepr(0).maxlist * "1000" + "]") + + +def test_repr_on_newstyle() -> None: + class Function: + def __repr__(self): + return "<%s>" % (self.name) # type: ignore[attr-defined] + + assert saferepr(Function()) + + +def test_unicode(): + val = "£€" + reprval = "'£€'" + assert saferepr(val) == reprval + + +def test_pformat_dispatch(): + assert _pformat_dispatch("a") == "'a'" + assert _pformat_dispatch("a" * 10, width=5) == "'aaaaaaaaaa'" + assert _pformat_dispatch("foo bar", width=5) == "('foo '\n 'bar')" + + +def test_broken_getattribute(): + """saferepr() can create proper representations of classes with + broken __getattribute__ (#7145) + """ + + class SomeClass: + def __getattribute__(self, attr): + raise RuntimeError + + def __repr__(self): + raise RuntimeError + + assert saferepr(SomeClass()).startswith( + "<[RuntimeError() raised in repr()] SomeClass object at 0x" + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/io/test_terminalwriter.py b/testing/web-platform/tests/tools/third_party/pytest/testing/io/test_terminalwriter.py new file mode 100644 index 0000000000..4866c94a55 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/io/test_terminalwriter.py @@ -0,0 +1,293 @@ +import io +import os +import re +import shutil +import sys +from pathlib import Path +from typing import Generator +from unittest import mock + +import pytest +from _pytest._io import terminalwriter +from _pytest.monkeypatch import MonkeyPatch + + +# These tests were initially copied from py 1.8.1. + + +def test_terminal_width_COLUMNS(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setenv("COLUMNS", "42") + assert terminalwriter.get_terminal_width() == 42 + monkeypatch.delenv("COLUMNS", raising=False) + + +def test_terminalwriter_width_bogus(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr(shutil, "get_terminal_size", mock.Mock(return_value=(10, 10))) + monkeypatch.delenv("COLUMNS", raising=False) + tw = terminalwriter.TerminalWriter() + assert tw.fullwidth == 80 + + +def test_terminalwriter_computes_width(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr(terminalwriter, "get_terminal_width", lambda: 42) + tw = terminalwriter.TerminalWriter() + assert tw.fullwidth == 42 + + +def test_terminalwriter_dumb_term_no_markup(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr(os, "environ", {"TERM": "dumb", "PATH": ""}) + + class MyFile: + closed = False + + def isatty(self): + return True + + with monkeypatch.context() as m: + m.setattr(sys, "stdout", MyFile()) + assert sys.stdout.isatty() + tw = terminalwriter.TerminalWriter() + assert not tw.hasmarkup + + +def test_terminalwriter_not_unicode() -> None: + """If the file doesn't support Unicode, the string is unicode-escaped (#7475).""" + buffer = io.BytesIO() + file = io.TextIOWrapper(buffer, encoding="cp1252") + tw = terminalwriter.TerminalWriter(file) + tw.write("hello 🌀 wôrld אבג", flush=True) + assert buffer.getvalue() == br"hello \U0001f300 w\xf4rld \u05d0\u05d1\u05d2" + + +win32 = int(sys.platform == "win32") + + +class TestTerminalWriter: + @pytest.fixture(params=["path", "stringio"]) + def tw( + self, request, tmp_path: Path + ) -> Generator[terminalwriter.TerminalWriter, None, None]: + if request.param == "path": + p = tmp_path.joinpath("tmpfile") + f = open(str(p), "w+", encoding="utf8") + tw = terminalwriter.TerminalWriter(f) + + def getlines(): + f.flush() + with open(str(p), encoding="utf8") as fp: + return fp.readlines() + + elif request.param == "stringio": + f = io.StringIO() + tw = terminalwriter.TerminalWriter(f) + + def getlines(): + f.seek(0) + return f.readlines() + + tw.getlines = getlines # type: ignore + tw.getvalue = lambda: "".join(getlines()) # type: ignore + + with f: + yield tw + + def test_line(self, tw) -> None: + tw.line("hello") + lines = tw.getlines() + assert len(lines) == 1 + assert lines[0] == "hello\n" + + def test_line_unicode(self, tw) -> None: + msg = "b\u00f6y" + tw.line(msg) + lines = tw.getlines() + assert lines[0] == msg + "\n" + + def test_sep_no_title(self, tw) -> None: + tw.sep("-", fullwidth=60) + lines = tw.getlines() + assert len(lines) == 1 + assert lines[0] == "-" * (60 - win32) + "\n" + + def test_sep_with_title(self, tw) -> None: + tw.sep("-", "hello", fullwidth=60) + lines = tw.getlines() + assert len(lines) == 1 + assert lines[0] == "-" * 26 + " hello " + "-" * (27 - win32) + "\n" + + def test_sep_longer_than_width(self, tw) -> None: + tw.sep("-", "a" * 10, fullwidth=5) + (line,) = tw.getlines() + # even though the string is wider than the line, still have a separator + assert line == "- aaaaaaaaaa -\n" + + @pytest.mark.skipif(sys.platform == "win32", reason="win32 has no native ansi") + @pytest.mark.parametrize("bold", (True, False)) + @pytest.mark.parametrize("color", ("red", "green")) + def test_markup(self, tw, bold: bool, color: str) -> None: + text = tw.markup("hello", **{color: True, "bold": bold}) + assert "hello" in text + + def test_markup_bad(self, tw) -> None: + with pytest.raises(ValueError): + tw.markup("x", wronkw=3) + with pytest.raises(ValueError): + tw.markup("x", wronkw=0) + + def test_line_write_markup(self, tw) -> None: + tw.hasmarkup = True + tw.line("x", bold=True) + tw.write("x\n", red=True) + lines = tw.getlines() + if sys.platform != "win32": + assert len(lines[0]) >= 2, lines + assert len(lines[1]) >= 2, lines + + def test_attr_fullwidth(self, tw) -> None: + tw.sep("-", "hello", fullwidth=70) + tw.fullwidth = 70 + tw.sep("-", "hello") + lines = tw.getlines() + assert len(lines[0]) == len(lines[1]) + + +@pytest.mark.skipif(sys.platform == "win32", reason="win32 has no native ansi") +def test_attr_hasmarkup() -> None: + file = io.StringIO() + tw = terminalwriter.TerminalWriter(file) + assert not tw.hasmarkup + tw.hasmarkup = True + tw.line("hello", bold=True) + s = file.getvalue() + assert len(s) > len("hello\n") + assert "\x1b[1m" in s + assert "\x1b[0m" in s + + +def assert_color_set(): + file = io.StringIO() + tw = terminalwriter.TerminalWriter(file) + assert tw.hasmarkup + tw.line("hello", bold=True) + s = file.getvalue() + assert len(s) > len("hello\n") + assert "\x1b[1m" in s + assert "\x1b[0m" in s + + +def assert_color_not_set(): + f = io.StringIO() + f.isatty = lambda: True # type: ignore + tw = terminalwriter.TerminalWriter(file=f) + assert not tw.hasmarkup + tw.line("hello", bold=True) + s = f.getvalue() + assert s == "hello\n" + + +def test_should_do_markup_PY_COLORS_eq_1(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setitem(os.environ, "PY_COLORS", "1") + assert_color_set() + + +def test_should_not_do_markup_PY_COLORS_eq_0(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setitem(os.environ, "PY_COLORS", "0") + assert_color_not_set() + + +def test_should_not_do_markup_NO_COLOR(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setitem(os.environ, "NO_COLOR", "1") + assert_color_not_set() + + +def test_should_do_markup_FORCE_COLOR(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setitem(os.environ, "FORCE_COLOR", "1") + assert_color_set() + + +def test_should_not_do_markup_NO_COLOR_and_FORCE_COLOR( + monkeypatch: MonkeyPatch, +) -> None: + monkeypatch.setitem(os.environ, "NO_COLOR", "1") + monkeypatch.setitem(os.environ, "FORCE_COLOR", "1") + assert_color_not_set() + + +class TestTerminalWriterLineWidth: + def test_init(self) -> None: + tw = terminalwriter.TerminalWriter() + assert tw.width_of_current_line == 0 + + def test_update(self) -> None: + tw = terminalwriter.TerminalWriter() + tw.write("hello world") + assert tw.width_of_current_line == 11 + + def test_update_with_newline(self) -> None: + tw = terminalwriter.TerminalWriter() + tw.write("hello\nworld") + assert tw.width_of_current_line == 5 + + def test_update_with_wide_text(self) -> None: + tw = terminalwriter.TerminalWriter() + tw.write("乇乂ㄒ尺卂 ㄒ卄丨匚匚") + assert tw.width_of_current_line == 21 # 5*2 + 1 + 5*2 + + def test_composed(self) -> None: + tw = terminalwriter.TerminalWriter() + text = "café food" + assert len(text) == 9 + tw.write(text) + assert tw.width_of_current_line == 9 + + def test_combining(self) -> None: + tw = terminalwriter.TerminalWriter() + text = "café food" + assert len(text) == 10 + tw.write(text) + assert tw.width_of_current_line == 9 + + +@pytest.mark.parametrize( + ("has_markup", "code_highlight", "expected"), + [ + pytest.param( + True, + True, + "{kw}assert{hl-reset} {number}0{hl-reset}\n", + id="with markup and code_highlight", + ), + pytest.param( + True, + False, + "assert 0\n", + id="with markup but no code_highlight", + ), + pytest.param( + False, + True, + "assert 0\n", + id="without markup but with code_highlight", + ), + pytest.param( + False, + False, + "assert 0\n", + id="neither markup nor code_highlight", + ), + ], +) +def test_code_highlight(has_markup, code_highlight, expected, color_mapping): + f = io.StringIO() + tw = terminalwriter.TerminalWriter(f) + tw.hasmarkup = has_markup + tw.code_highlight = code_highlight + tw._write_source(["assert 0"]) + + assert f.getvalue().splitlines(keepends=True) == color_mapping.format([expected]) + + with pytest.raises( + ValueError, + match=re.escape("indents size (2) should have same size as lines (1)"), + ): + tw._write_source(["assert 0"], [" ", " "]) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/io/test_wcwidth.py b/testing/web-platform/tests/tools/third_party/pytest/testing/io/test_wcwidth.py new file mode 100644 index 0000000000..7cc74df5d0 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/io/test_wcwidth.py @@ -0,0 +1,38 @@ +import pytest +from _pytest._io.wcwidth import wcswidth +from _pytest._io.wcwidth import wcwidth + + +@pytest.mark.parametrize( + ("c", "expected"), + [ + ("\0", 0), + ("\n", -1), + ("a", 1), + ("1", 1), + ("א", 1), + ("\u200B", 0), + ("\u1ABE", 0), + ("\u0591", 0), + ("🉐", 2), + ("$", 2), + ], +) +def test_wcwidth(c: str, expected: int) -> None: + assert wcwidth(c) == expected + + +@pytest.mark.parametrize( + ("s", "expected"), + [ + ("", 0), + ("hello, world!", 13), + ("hello, world!\n", -1), + ("0123456789", 10), + ("שלום, עולם!", 11), + ("שְבֻעָיים", 6), + ("🉐🉐🉐", 6), + ], +) +def test_wcswidth(s: str, expected: int) -> None: + assert wcswidth(s) == expected diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/logging/test_fixture.py b/testing/web-platform/tests/tools/third_party/pytest/testing/logging/test_fixture.py new file mode 100644 index 0000000000..bcb20de580 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/logging/test_fixture.py @@ -0,0 +1,310 @@ +import logging + +import pytest +from _pytest.logging import caplog_records_key +from _pytest.pytester import Pytester + +logger = logging.getLogger(__name__) +sublogger = logging.getLogger(__name__ + ".baz") + + +def test_fixture_help(pytester: Pytester) -> None: + result = pytester.runpytest("--fixtures") + result.stdout.fnmatch_lines(["*caplog*"]) + + +def test_change_level(caplog): + caplog.set_level(logging.INFO) + logger.debug("handler DEBUG level") + logger.info("handler INFO level") + + caplog.set_level(logging.CRITICAL, logger=sublogger.name) + sublogger.warning("logger WARNING level") + sublogger.critical("logger CRITICAL level") + + assert "DEBUG" not in caplog.text + assert "INFO" in caplog.text + assert "WARNING" not in caplog.text + assert "CRITICAL" in caplog.text + + +def test_change_level_undo(pytester: Pytester) -> None: + """Ensure that 'set_level' is undone after the end of the test. + + Tests the logging output themselves (affacted both by logger and handler levels). + """ + pytester.makepyfile( + """ + import logging + + def test1(caplog): + caplog.set_level(logging.INFO) + # using + operator here so fnmatch_lines doesn't match the code in the traceback + logging.info('log from ' + 'test1') + assert 0 + + def test2(caplog): + # using + operator here so fnmatch_lines doesn't match the code in the traceback + logging.info('log from ' + 'test2') + assert 0 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*log from test1*", "*2 failed in *"]) + result.stdout.no_fnmatch_line("*log from test2*") + + +def test_change_level_undos_handler_level(pytester: Pytester) -> None: + """Ensure that 'set_level' is undone after the end of the test (handler). + + Issue #7569. Tests the handler level specifically. + """ + pytester.makepyfile( + """ + import logging + + def test1(caplog): + assert caplog.handler.level == 0 + caplog.set_level(9999) + caplog.set_level(41) + assert caplog.handler.level == 41 + + def test2(caplog): + assert caplog.handler.level == 0 + + def test3(caplog): + assert caplog.handler.level == 0 + caplog.set_level(43) + assert caplog.handler.level == 43 + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=3) + + +def test_with_statement(caplog): + with caplog.at_level(logging.INFO): + logger.debug("handler DEBUG level") + logger.info("handler INFO level") + + with caplog.at_level(logging.CRITICAL, logger=sublogger.name): + sublogger.warning("logger WARNING level") + sublogger.critical("logger CRITICAL level") + + assert "DEBUG" not in caplog.text + assert "INFO" in caplog.text + assert "WARNING" not in caplog.text + assert "CRITICAL" in caplog.text + + +def test_log_access(caplog): + caplog.set_level(logging.INFO) + logger.info("boo %s", "arg") + assert caplog.records[0].levelname == "INFO" + assert caplog.records[0].msg == "boo %s" + assert "boo arg" in caplog.text + + +def test_messages(caplog): + caplog.set_level(logging.INFO) + logger.info("boo %s", "arg") + logger.info("bar %s\nbaz %s", "arg1", "arg2") + assert "boo arg" == caplog.messages[0] + assert "bar arg1\nbaz arg2" == caplog.messages[1] + assert caplog.text.count("\n") > len(caplog.messages) + assert len(caplog.text.splitlines()) > len(caplog.messages) + + try: + raise Exception("test") + except Exception: + logger.exception("oops") + + assert "oops" in caplog.text + assert "oops" in caplog.messages[-1] + # Tracebacks are stored in the record and not added until the formatter or handler. + assert "Exception" in caplog.text + assert "Exception" not in caplog.messages[-1] + + +def test_record_tuples(caplog): + caplog.set_level(logging.INFO) + logger.info("boo %s", "arg") + + assert caplog.record_tuples == [(__name__, logging.INFO, "boo arg")] + + +def test_unicode(caplog): + caplog.set_level(logging.INFO) + logger.info("bū") + assert caplog.records[0].levelname == "INFO" + assert caplog.records[0].msg == "bū" + assert "bū" in caplog.text + + +def test_clear(caplog): + caplog.set_level(logging.INFO) + logger.info("bū") + assert len(caplog.records) + assert caplog.text + caplog.clear() + assert not len(caplog.records) + assert not caplog.text + + +@pytest.fixture +def logging_during_setup_and_teardown(caplog): + caplog.set_level("INFO") + logger.info("a_setup_log") + yield + logger.info("a_teardown_log") + assert [x.message for x in caplog.get_records("teardown")] == ["a_teardown_log"] + + +def test_caplog_captures_for_all_stages(caplog, logging_during_setup_and_teardown): + assert not caplog.records + assert not caplog.get_records("call") + logger.info("a_call_log") + assert [x.message for x in caplog.get_records("call")] == ["a_call_log"] + + assert [x.message for x in caplog.get_records("setup")] == ["a_setup_log"] + + # This reaches into private API, don't use this type of thing in real tests! + assert set(caplog._item.stash[caplog_records_key]) == {"setup", "call"} + + +def test_ini_controls_global_log_level(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + import logging + def test_log_level_override(request, caplog): + plugin = request.config.pluginmanager.getplugin('logging-plugin') + assert plugin.log_level == logging.ERROR + logger = logging.getLogger('catchlog') + logger.warning("WARNING message won't be shown") + logger.error("ERROR message will be shown") + assert 'WARNING' not in caplog.text + assert 'ERROR' in caplog.text + """ + ) + pytester.makeini( + """ + [pytest] + log_level=ERROR + """ + ) + + result = pytester.runpytest() + # make sure that that we get a '0' exit code for the testsuite + assert result.ret == 0 + + +def test_caplog_can_override_global_log_level(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + import logging + def test_log_level_override(request, caplog): + logger = logging.getLogger('catchlog') + plugin = request.config.pluginmanager.getplugin('logging-plugin') + assert plugin.log_level == logging.WARNING + + logger.info("INFO message won't be shown") + + caplog.set_level(logging.INFO, logger.name) + + with caplog.at_level(logging.DEBUG, logger.name): + logger.debug("DEBUG message will be shown") + + logger.debug("DEBUG message won't be shown") + + with caplog.at_level(logging.CRITICAL, logger.name): + logger.warning("WARNING message won't be shown") + + logger.debug("DEBUG message won't be shown") + logger.info("INFO message will be shown") + + assert "message won't be shown" not in caplog.text + """ + ) + pytester.makeini( + """ + [pytest] + log_level=WARNING + """ + ) + + result = pytester.runpytest() + assert result.ret == 0 + + +def test_caplog_captures_despite_exception(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + import logging + def test_log_level_override(request, caplog): + logger = logging.getLogger('catchlog') + plugin = request.config.pluginmanager.getplugin('logging-plugin') + assert plugin.log_level == logging.WARNING + + logger.error("ERROR message " + "will be shown") + + with caplog.at_level(logging.DEBUG, logger.name): + logger.debug("DEBUG message " + "won't be shown") + raise Exception() + """ + ) + pytester.makeini( + """ + [pytest] + log_level=WARNING + """ + ) + + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*ERROR message will be shown*"]) + result.stdout.no_fnmatch_line("*DEBUG message won't be shown*") + assert result.ret == 1 + + +def test_log_report_captures_according_to_config_option_upon_failure( + pytester: Pytester, +) -> None: + """Test that upon failure: + (1) `caplog` succeeded to capture the DEBUG message and assert on it => No `Exception` is raised. + (2) The `DEBUG` message does NOT appear in the `Captured log call` report. + (3) The stdout, `INFO`, and `WARNING` messages DO appear in the test reports due to `--log-level=INFO`. + """ + pytester.makepyfile( + """ + import pytest + import logging + + def function_that_logs(): + logging.debug('DEBUG log ' + 'message') + logging.info('INFO log ' + 'message') + logging.warning('WARNING log ' + 'message') + print('Print ' + 'message') + + def test_that_fails(request, caplog): + plugin = request.config.pluginmanager.getplugin('logging-plugin') + assert plugin.log_level == logging.INFO + + with caplog.at_level(logging.DEBUG): + function_that_logs() + + if 'DEBUG log ' + 'message' not in caplog.text: + raise Exception('caplog failed to ' + 'capture DEBUG') + + assert False + """ + ) + + result = pytester.runpytest("--log-level=INFO") + result.stdout.no_fnmatch_line("*Exception: caplog failed to capture DEBUG*") + result.stdout.no_fnmatch_line("*DEBUG log message*") + result.stdout.fnmatch_lines( + ["*Print message*", "*INFO log message*", "*WARNING log message*"] + ) + assert result.ret == 1 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/logging/test_formatter.py b/testing/web-platform/tests/tools/third_party/pytest/testing/logging/test_formatter.py new file mode 100644 index 0000000000..3797129372 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/logging/test_formatter.py @@ -0,0 +1,173 @@ +import logging +from typing import Any + +from _pytest._io import TerminalWriter +from _pytest.logging import ColoredLevelFormatter + + +def test_coloredlogformatter() -> None: + logfmt = "%(filename)-25s %(lineno)4d %(levelname)-8s %(message)s" + + record = logging.LogRecord( + name="dummy", + level=logging.INFO, + pathname="dummypath", + lineno=10, + msg="Test Message", + args=(), + exc_info=None, + ) + + tw = TerminalWriter() + tw.hasmarkup = True + formatter = ColoredLevelFormatter(tw, logfmt) + output = formatter.format(record) + assert output == ( + "dummypath 10 \x1b[32mINFO \x1b[0m Test Message" + ) + + tw.hasmarkup = False + formatter = ColoredLevelFormatter(tw, logfmt) + output = formatter.format(record) + assert output == ("dummypath 10 INFO Test Message") + + +def test_coloredlogformatter_with_width_precision() -> None: + logfmt = "%(filename)-25s %(lineno)4d %(levelname)-8.8s %(message)s" + + record = logging.LogRecord( + name="dummy", + level=logging.INFO, + pathname="dummypath", + lineno=10, + msg="Test Message", + args=(), + exc_info=None, + ) + + tw = TerminalWriter() + tw.hasmarkup = True + formatter = ColoredLevelFormatter(tw, logfmt) + output = formatter.format(record) + assert output == ( + "dummypath 10 \x1b[32mINFO \x1b[0m Test Message" + ) + + tw.hasmarkup = False + formatter = ColoredLevelFormatter(tw, logfmt) + output = formatter.format(record) + assert output == ("dummypath 10 INFO Test Message") + + +def test_multiline_message() -> None: + from _pytest.logging import PercentStyleMultiline + + logfmt = "%(filename)-25s %(lineno)4d %(levelname)-8s %(message)s" + + record: Any = logging.LogRecord( + name="dummy", + level=logging.INFO, + pathname="dummypath", + lineno=10, + msg="Test Message line1\nline2", + args=(), + exc_info=None, + ) + # this is called by logging.Formatter.format + record.message = record.getMessage() + + ai_on_style = PercentStyleMultiline(logfmt, True) + output = ai_on_style.format(record) + assert output == ( + "dummypath 10 INFO Test Message line1\n" + " line2" + ) + + ai_off_style = PercentStyleMultiline(logfmt, False) + output = ai_off_style.format(record) + assert output == ( + "dummypath 10 INFO Test Message line1\nline2" + ) + + ai_none_style = PercentStyleMultiline(logfmt, None) + output = ai_none_style.format(record) + assert output == ( + "dummypath 10 INFO Test Message line1\nline2" + ) + + record.auto_indent = False + output = ai_on_style.format(record) + assert output == ( + "dummypath 10 INFO Test Message line1\nline2" + ) + + record.auto_indent = True + output = ai_off_style.format(record) + assert output == ( + "dummypath 10 INFO Test Message line1\n" + " line2" + ) + + record.auto_indent = "False" + output = ai_on_style.format(record) + assert output == ( + "dummypath 10 INFO Test Message line1\nline2" + ) + + record.auto_indent = "True" + output = ai_off_style.format(record) + assert output == ( + "dummypath 10 INFO Test Message line1\n" + " line2" + ) + + # bad string values default to False + record.auto_indent = "junk" + output = ai_off_style.format(record) + assert output == ( + "dummypath 10 INFO Test Message line1\nline2" + ) + + # anything other than string or int will default to False + record.auto_indent = dict() + output = ai_off_style.format(record) + assert output == ( + "dummypath 10 INFO Test Message line1\nline2" + ) + + record.auto_indent = "5" + output = ai_off_style.format(record) + assert output == ( + "dummypath 10 INFO Test Message line1\n line2" + ) + + record.auto_indent = 5 + output = ai_off_style.format(record) + assert output == ( + "dummypath 10 INFO Test Message line1\n line2" + ) + + +def test_colored_short_level() -> None: + logfmt = "%(levelname).1s %(message)s" + + record = logging.LogRecord( + name="dummy", + level=logging.INFO, + pathname="dummypath", + lineno=10, + msg="Test Message", + args=(), + exc_info=None, + ) + + class ColorConfig: + class option: + pass + + tw = TerminalWriter() + tw.hasmarkup = True + formatter = ColoredLevelFormatter(tw, logfmt) + output = formatter.format(record) + # the I (of INFO) is colored + assert output == ("\x1b[32mI\x1b[0m Test Message") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/logging/test_reporting.py b/testing/web-platform/tests/tools/third_party/pytest/testing/logging/test_reporting.py new file mode 100644 index 0000000000..323ff7b244 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/logging/test_reporting.py @@ -0,0 +1,1167 @@ +import io +import os +import re +from typing import cast + +import pytest +from _pytest.capture import CaptureManager +from _pytest.config import ExitCode +from _pytest.fixtures import FixtureRequest +from _pytest.pytester import Pytester +from _pytest.terminal import TerminalReporter + + +def test_nothing_logged(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import sys + + def test_foo(): + sys.stdout.write('text going to stdout') + sys.stderr.write('text going to stderr') + assert False + """ + ) + result = pytester.runpytest() + assert result.ret == 1 + result.stdout.fnmatch_lines(["*- Captured stdout call -*", "text going to stdout"]) + result.stdout.fnmatch_lines(["*- Captured stderr call -*", "text going to stderr"]) + with pytest.raises(pytest.fail.Exception): + result.stdout.fnmatch_lines(["*- Captured *log call -*"]) + + +def test_messages_logged(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import sys + import logging + + logger = logging.getLogger(__name__) + + def test_foo(): + sys.stdout.write('text going to stdout') + sys.stderr.write('text going to stderr') + logger.info('text going to logger') + assert False + """ + ) + result = pytester.runpytest("--log-level=INFO") + assert result.ret == 1 + result.stdout.fnmatch_lines(["*- Captured *log call -*", "*text going to logger*"]) + result.stdout.fnmatch_lines(["*- Captured stdout call -*", "text going to stdout"]) + result.stdout.fnmatch_lines(["*- Captured stderr call -*", "text going to stderr"]) + + +def test_root_logger_affected(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import logging + logger = logging.getLogger() + + def test_foo(): + logger.info('info text ' + 'going to logger') + logger.warning('warning text ' + 'going to logger') + logger.error('error text ' + 'going to logger') + + assert 0 + """ + ) + log_file = str(pytester.path.joinpath("pytest.log")) + result = pytester.runpytest("--log-level=ERROR", "--log-file=pytest.log") + assert result.ret == 1 + + # The capture log calls in the stdout section only contain the + # logger.error msg, because of --log-level=ERROR. + result.stdout.fnmatch_lines(["*error text going to logger*"]) + stdout = result.stdout.str() + assert "warning text going to logger" not in stdout + assert "info text going to logger" not in stdout + + # The log file should contain the warning and the error log messages and + # not the info one, because the default level of the root logger is + # WARNING. + assert os.path.isfile(log_file) + with open(log_file) as rfh: + contents = rfh.read() + assert "info text going to logger" not in contents + assert "warning text going to logger" in contents + assert "error text going to logger" in contents + + +def test_log_cli_level_log_level_interaction(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import logging + logger = logging.getLogger() + + def test_foo(): + logger.debug('debug text ' + 'going to logger') + logger.info('info text ' + 'going to logger') + logger.warning('warning text ' + 'going to logger') + logger.error('error text ' + 'going to logger') + assert 0 + """ + ) + + result = pytester.runpytest("--log-cli-level=INFO", "--log-level=ERROR") + assert result.ret == 1 + + result.stdout.fnmatch_lines( + [ + "*-- live log call --*", + "*INFO*info text going to logger", + "*WARNING*warning text going to logger", + "*ERROR*error text going to logger", + "=* 1 failed in *=", + ] + ) + result.stdout.no_re_match_line("DEBUG") + + +def test_setup_logging(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import logging + + logger = logging.getLogger(__name__) + + def setup_function(function): + logger.info('text going to logger from setup') + + def test_foo(): + logger.info('text going to logger from call') + assert False + """ + ) + result = pytester.runpytest("--log-level=INFO") + assert result.ret == 1 + result.stdout.fnmatch_lines( + [ + "*- Captured *log setup -*", + "*text going to logger from setup*", + "*- Captured *log call -*", + "*text going to logger from call*", + ] + ) + + +def test_teardown_logging(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import logging + + logger = logging.getLogger(__name__) + + def test_foo(): + logger.info('text going to logger from call') + + def teardown_function(function): + logger.info('text going to logger from teardown') + assert False + """ + ) + result = pytester.runpytest("--log-level=INFO") + assert result.ret == 1 + result.stdout.fnmatch_lines( + [ + "*- Captured *log call -*", + "*text going to logger from call*", + "*- Captured *log teardown -*", + "*text going to logger from teardown*", + ] + ) + + +@pytest.mark.parametrize("enabled", [True, False]) +def test_log_cli_enabled_disabled(pytester: Pytester, enabled: bool) -> None: + msg = "critical message logged by test" + pytester.makepyfile( + """ + import logging + def test_log_cli(): + logging.critical("{}") + """.format( + msg + ) + ) + if enabled: + pytester.makeini( + """ + [pytest] + log_cli=true + """ + ) + result = pytester.runpytest() + if enabled: + result.stdout.fnmatch_lines( + [ + "test_log_cli_enabled_disabled.py::test_log_cli ", + "*-- live log call --*", + "CRITICAL *test_log_cli_enabled_disabled.py* critical message logged by test", + "PASSED*", + ] + ) + else: + assert msg not in result.stdout.str() + + +def test_log_cli_default_level(pytester: Pytester) -> None: + # Default log file level + pytester.makepyfile( + """ + import pytest + import logging + def test_log_cli(request): + plugin = request.config.pluginmanager.getplugin('logging-plugin') + assert plugin.log_cli_handler.level == logging.NOTSET + logging.getLogger('catchlog').info("INFO message won't be shown") + logging.getLogger('catchlog').warning("WARNING message will be shown") + """ + ) + pytester.makeini( + """ + [pytest] + log_cli=true + """ + ) + + result = pytester.runpytest() + + # fnmatch_lines does an assertion internally + result.stdout.fnmatch_lines( + [ + "test_log_cli_default_level.py::test_log_cli ", + "WARNING*test_log_cli_default_level.py* message will be shown*", + ] + ) + result.stdout.no_fnmatch_line("*INFO message won't be shown*") + # make sure that we get a '0' exit code for the testsuite + assert result.ret == 0 + + +def test_log_cli_default_level_multiple_tests( + pytester: Pytester, request: FixtureRequest +) -> None: + """Ensure we reset the first newline added by the live logger between tests""" + filename = request.node.name + ".py" + pytester.makepyfile( + """ + import logging + + def test_log_1(): + logging.warning("log message from test_log_1") + + def test_log_2(): + logging.warning("log message from test_log_2") + """ + ) + pytester.makeini( + """ + [pytest] + log_cli=true + """ + ) + + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + f"{filename}::test_log_1 ", + "*WARNING*log message from test_log_1*", + "PASSED *50%*", + f"{filename}::test_log_2 ", + "*WARNING*log message from test_log_2*", + "PASSED *100%*", + "=* 2 passed in *=", + ] + ) + + +def test_log_cli_default_level_sections( + pytester: Pytester, request: FixtureRequest +) -> None: + """Check that with live logging enable we are printing the correct headers during + start/setup/call/teardown/finish.""" + filename = request.node.name + ".py" + pytester.makeconftest( + """ + import pytest + import logging + + def pytest_runtest_logstart(): + logging.warning('>>>>> START >>>>>') + + def pytest_runtest_logfinish(): + logging.warning('<<<<< END <<<<<<<') + """ + ) + + pytester.makepyfile( + """ + import pytest + import logging + + @pytest.fixture + def fix(request): + logging.warning("log message from setup of {}".format(request.node.name)) + yield + logging.warning("log message from teardown of {}".format(request.node.name)) + + def test_log_1(fix): + logging.warning("log message from test_log_1") + + def test_log_2(fix): + logging.warning("log message from test_log_2") + """ + ) + pytester.makeini( + """ + [pytest] + log_cli=true + """ + ) + + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + f"{filename}::test_log_1 ", + "*-- live log start --*", + "*WARNING* >>>>> START >>>>>*", + "*-- live log setup --*", + "*WARNING*log message from setup of test_log_1*", + "*-- live log call --*", + "*WARNING*log message from test_log_1*", + "PASSED *50%*", + "*-- live log teardown --*", + "*WARNING*log message from teardown of test_log_1*", + "*-- live log finish --*", + "*WARNING* <<<<< END <<<<<<<*", + f"{filename}::test_log_2 ", + "*-- live log start --*", + "*WARNING* >>>>> START >>>>>*", + "*-- live log setup --*", + "*WARNING*log message from setup of test_log_2*", + "*-- live log call --*", + "*WARNING*log message from test_log_2*", + "PASSED *100%*", + "*-- live log teardown --*", + "*WARNING*log message from teardown of test_log_2*", + "*-- live log finish --*", + "*WARNING* <<<<< END <<<<<<<*", + "=* 2 passed in *=", + ] + ) + + +def test_live_logs_unknown_sections( + pytester: Pytester, request: FixtureRequest +) -> None: + """Check that with live logging enable we are printing the correct headers during + start/setup/call/teardown/finish.""" + filename = request.node.name + ".py" + pytester.makeconftest( + """ + import pytest + import logging + + def pytest_runtest_protocol(item, nextitem): + logging.warning('Unknown Section!') + + def pytest_runtest_logstart(): + logging.warning('>>>>> START >>>>>') + + def pytest_runtest_logfinish(): + logging.warning('<<<<< END <<<<<<<') + """ + ) + + pytester.makepyfile( + """ + import pytest + import logging + + @pytest.fixture + def fix(request): + logging.warning("log message from setup of {}".format(request.node.name)) + yield + logging.warning("log message from teardown of {}".format(request.node.name)) + + def test_log_1(fix): + logging.warning("log message from test_log_1") + + """ + ) + pytester.makeini( + """ + [pytest] + log_cli=true + """ + ) + + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*WARNING*Unknown Section*", + f"{filename}::test_log_1 ", + "*WARNING* >>>>> START >>>>>*", + "*-- live log setup --*", + "*WARNING*log message from setup of test_log_1*", + "*-- live log call --*", + "*WARNING*log message from test_log_1*", + "PASSED *100%*", + "*-- live log teardown --*", + "*WARNING*log message from teardown of test_log_1*", + "*WARNING* <<<<< END <<<<<<<*", + "=* 1 passed in *=", + ] + ) + + +def test_sections_single_new_line_after_test_outcome( + pytester: Pytester, request: FixtureRequest +) -> None: + """Check that only a single new line is written between log messages during + teardown/finish.""" + filename = request.node.name + ".py" + pytester.makeconftest( + """ + import pytest + import logging + + def pytest_runtest_logstart(): + logging.warning('>>>>> START >>>>>') + + def pytest_runtest_logfinish(): + logging.warning('<<<<< END <<<<<<<') + logging.warning('<<<<< END <<<<<<<') + """ + ) + + pytester.makepyfile( + """ + import pytest + import logging + + @pytest.fixture + def fix(request): + logging.warning("log message from setup of {}".format(request.node.name)) + yield + logging.warning("log message from teardown of {}".format(request.node.name)) + logging.warning("log message from teardown of {}".format(request.node.name)) + + def test_log_1(fix): + logging.warning("log message from test_log_1") + """ + ) + pytester.makeini( + """ + [pytest] + log_cli=true + """ + ) + + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + f"{filename}::test_log_1 ", + "*-- live log start --*", + "*WARNING* >>>>> START >>>>>*", + "*-- live log setup --*", + "*WARNING*log message from setup of test_log_1*", + "*-- live log call --*", + "*WARNING*log message from test_log_1*", + "PASSED *100%*", + "*-- live log teardown --*", + "*WARNING*log message from teardown of test_log_1*", + "*-- live log finish --*", + "*WARNING* <<<<< END <<<<<<<*", + "*WARNING* <<<<< END <<<<<<<*", + "=* 1 passed in *=", + ] + ) + assert ( + re.search( + r"(.+)live log teardown(.+)\nWARNING(.+)\nWARNING(.+)", + result.stdout.str(), + re.MULTILINE, + ) + is not None + ) + assert ( + re.search( + r"(.+)live log finish(.+)\nWARNING(.+)\nWARNING(.+)", + result.stdout.str(), + re.MULTILINE, + ) + is not None + ) + + +def test_log_cli_level(pytester: Pytester) -> None: + # Default log file level + pytester.makepyfile( + """ + import pytest + import logging + def test_log_cli(request): + plugin = request.config.pluginmanager.getplugin('logging-plugin') + assert plugin.log_cli_handler.level == logging.INFO + logging.getLogger('catchlog').debug("This log message won't be shown") + logging.getLogger('catchlog').info("This log message will be shown") + print('PASSED') + """ + ) + pytester.makeini( + """ + [pytest] + log_cli=true + """ + ) + + result = pytester.runpytest("-s", "--log-cli-level=INFO") + + # fnmatch_lines does an assertion internally + result.stdout.fnmatch_lines( + [ + "*test_log_cli_level.py*This log message will be shown", + "PASSED", # 'PASSED' on its own line because the log message prints a new line + ] + ) + result.stdout.no_fnmatch_line("*This log message won't be shown*") + + # make sure that we get a '0' exit code for the testsuite + assert result.ret == 0 + + result = pytester.runpytest("-s", "--log-level=INFO") + + # fnmatch_lines does an assertion internally + result.stdout.fnmatch_lines( + [ + "*test_log_cli_level.py* This log message will be shown", + "PASSED", # 'PASSED' on its own line because the log message prints a new line + ] + ) + result.stdout.no_fnmatch_line("*This log message won't be shown*") + + # make sure that we get a '0' exit code for the testsuite + assert result.ret == 0 + + +def test_log_cli_ini_level(pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + log_cli=true + log_cli_level = INFO + """ + ) + pytester.makepyfile( + """ + import pytest + import logging + def test_log_cli(request): + plugin = request.config.pluginmanager.getplugin('logging-plugin') + assert plugin.log_cli_handler.level == logging.INFO + logging.getLogger('catchlog').debug("This log message won't be shown") + logging.getLogger('catchlog').info("This log message will be shown") + print('PASSED') + """ + ) + + result = pytester.runpytest("-s") + + # fnmatch_lines does an assertion internally + result.stdout.fnmatch_lines( + [ + "*test_log_cli_ini_level.py* This log message will be shown", + "PASSED", # 'PASSED' on its own line because the log message prints a new line + ] + ) + result.stdout.no_fnmatch_line("*This log message won't be shown*") + + # make sure that we get a '0' exit code for the testsuite + assert result.ret == 0 + + +@pytest.mark.parametrize( + "cli_args", + ["", "--log-level=WARNING", "--log-file-level=WARNING", "--log-cli-level=WARNING"], +) +def test_log_cli_auto_enable(pytester: Pytester, cli_args: str) -> None: + """Check that live logs are enabled if --log-level or --log-cli-level is passed on the CLI. + It should not be auto enabled if the same configs are set on the INI file. + """ + pytester.makepyfile( + """ + import logging + + def test_log_1(): + logging.info("log message from test_log_1 not to be shown") + logging.warning("log message from test_log_1") + + """ + ) + pytester.makeini( + """ + [pytest] + log_level=INFO + log_cli_level=INFO + """ + ) + + result = pytester.runpytest(cli_args) + stdout = result.stdout.str() + if cli_args == "--log-cli-level=WARNING": + result.stdout.fnmatch_lines( + [ + "*::test_log_1 ", + "*-- live log call --*", + "*WARNING*log message from test_log_1*", + "PASSED *100%*", + "=* 1 passed in *=", + ] + ) + assert "INFO" not in stdout + else: + result.stdout.fnmatch_lines( + ["*test_log_cli_auto_enable*100%*", "=* 1 passed in *="] + ) + assert "INFO" not in stdout + assert "WARNING" not in stdout + + +def test_log_file_cli(pytester: Pytester) -> None: + # Default log file level + pytester.makepyfile( + """ + import pytest + import logging + def test_log_file(request): + plugin = request.config.pluginmanager.getplugin('logging-plugin') + assert plugin.log_file_handler.level == logging.WARNING + logging.getLogger('catchlog').info("This log message won't be shown") + logging.getLogger('catchlog').warning("This log message will be shown") + print('PASSED') + """ + ) + + log_file = str(pytester.path.joinpath("pytest.log")) + + result = pytester.runpytest( + "-s", f"--log-file={log_file}", "--log-file-level=WARNING" + ) + + # fnmatch_lines does an assertion internally + result.stdout.fnmatch_lines(["test_log_file_cli.py PASSED"]) + + # make sure that we get a '0' exit code for the testsuite + assert result.ret == 0 + assert os.path.isfile(log_file) + with open(log_file) as rfh: + contents = rfh.read() + assert "This log message will be shown" in contents + assert "This log message won't be shown" not in contents + + +def test_log_file_cli_level(pytester: Pytester) -> None: + # Default log file level + pytester.makepyfile( + """ + import pytest + import logging + def test_log_file(request): + plugin = request.config.pluginmanager.getplugin('logging-plugin') + assert plugin.log_file_handler.level == logging.INFO + logging.getLogger('catchlog').debug("This log message won't be shown") + logging.getLogger('catchlog').info("This log message will be shown") + print('PASSED') + """ + ) + + log_file = str(pytester.path.joinpath("pytest.log")) + + result = pytester.runpytest("-s", f"--log-file={log_file}", "--log-file-level=INFO") + + # fnmatch_lines does an assertion internally + result.stdout.fnmatch_lines(["test_log_file_cli_level.py PASSED"]) + + # make sure that we get a '0' exit code for the testsuite + assert result.ret == 0 + assert os.path.isfile(log_file) + with open(log_file) as rfh: + contents = rfh.read() + assert "This log message will be shown" in contents + assert "This log message won't be shown" not in contents + + +def test_log_level_not_changed_by_default(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import logging + def test_log_file(): + assert logging.getLogger().level == logging.WARNING + """ + ) + result = pytester.runpytest("-s") + result.stdout.fnmatch_lines(["* 1 passed in *"]) + + +def test_log_file_ini(pytester: Pytester) -> None: + log_file = str(pytester.path.joinpath("pytest.log")) + + pytester.makeini( + """ + [pytest] + log_file={} + log_file_level=WARNING + """.format( + log_file + ) + ) + pytester.makepyfile( + """ + import pytest + import logging + def test_log_file(request): + plugin = request.config.pluginmanager.getplugin('logging-plugin') + assert plugin.log_file_handler.level == logging.WARNING + logging.getLogger('catchlog').info("This log message won't be shown") + logging.getLogger('catchlog').warning("This log message will be shown") + print('PASSED') + """ + ) + + result = pytester.runpytest("-s") + + # fnmatch_lines does an assertion internally + result.stdout.fnmatch_lines(["test_log_file_ini.py PASSED"]) + + # make sure that we get a '0' exit code for the testsuite + assert result.ret == 0 + assert os.path.isfile(log_file) + with open(log_file) as rfh: + contents = rfh.read() + assert "This log message will be shown" in contents + assert "This log message won't be shown" not in contents + + +def test_log_file_ini_level(pytester: Pytester) -> None: + log_file = str(pytester.path.joinpath("pytest.log")) + + pytester.makeini( + """ + [pytest] + log_file={} + log_file_level = INFO + """.format( + log_file + ) + ) + pytester.makepyfile( + """ + import pytest + import logging + def test_log_file(request): + plugin = request.config.pluginmanager.getplugin('logging-plugin') + assert plugin.log_file_handler.level == logging.INFO + logging.getLogger('catchlog').debug("This log message won't be shown") + logging.getLogger('catchlog').info("This log message will be shown") + print('PASSED') + """ + ) + + result = pytester.runpytest("-s") + + # fnmatch_lines does an assertion internally + result.stdout.fnmatch_lines(["test_log_file_ini_level.py PASSED"]) + + # make sure that we get a '0' exit code for the testsuite + assert result.ret == 0 + assert os.path.isfile(log_file) + with open(log_file) as rfh: + contents = rfh.read() + assert "This log message will be shown" in contents + assert "This log message won't be shown" not in contents + + +def test_log_file_unicode(pytester: Pytester) -> None: + log_file = str(pytester.path.joinpath("pytest.log")) + + pytester.makeini( + """ + [pytest] + log_file={} + log_file_level = INFO + """.format( + log_file + ) + ) + pytester.makepyfile( + """\ + import logging + + def test_log_file(): + logging.getLogger('catchlog').info("Normal message") + logging.getLogger('catchlog').info("├") + logging.getLogger('catchlog').info("Another normal message") + """ + ) + + result = pytester.runpytest() + + # make sure that we get a '0' exit code for the testsuite + assert result.ret == 0 + assert os.path.isfile(log_file) + with open(log_file, encoding="utf-8") as rfh: + contents = rfh.read() + assert "Normal message" in contents + assert "├" in contents + assert "Another normal message" in contents + + +@pytest.mark.parametrize("has_capture_manager", [True, False]) +def test_live_logging_suspends_capture( + has_capture_manager: bool, request: FixtureRequest +) -> None: + """Test that capture manager is suspended when we emitting messages for live logging. + + This tests the implementation calls instead of behavior because it is difficult/impossible to do it using + ``pytester`` facilities because they do their own capturing. + + We parametrize the test to also make sure _LiveLoggingStreamHandler works correctly if no capture manager plugin + is installed. + """ + import logging + import contextlib + from functools import partial + from _pytest.logging import _LiveLoggingStreamHandler + + class MockCaptureManager: + calls = [] + + @contextlib.contextmanager + def global_and_fixture_disabled(self): + self.calls.append("enter disabled") + yield + self.calls.append("exit disabled") + + class DummyTerminal(io.StringIO): + def section(self, *args, **kwargs): + pass + + out_file = cast(TerminalReporter, DummyTerminal()) + capture_manager = ( + cast(CaptureManager, MockCaptureManager()) if has_capture_manager else None + ) + handler = _LiveLoggingStreamHandler(out_file, capture_manager) + handler.set_when("call") + + logger = logging.getLogger(__name__ + ".test_live_logging_suspends_capture") + logger.addHandler(handler) + request.addfinalizer(partial(logger.removeHandler, handler)) + + logger.critical("some message") + if has_capture_manager: + assert MockCaptureManager.calls == ["enter disabled", "exit disabled"] + else: + assert MockCaptureManager.calls == [] + assert cast(io.StringIO, out_file).getvalue() == "\nsome message\n" + + +def test_collection_live_logging(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import logging + + logging.getLogger().info("Normal message") + """ + ) + + result = pytester.runpytest("--log-cli-level=INFO") + result.stdout.fnmatch_lines( + ["*--- live log collection ---*", "*Normal message*", "collected 0 items"] + ) + + +@pytest.mark.parametrize("verbose", ["", "-q", "-qq"]) +def test_collection_collect_only_live_logging(pytester: Pytester, verbose: str) -> None: + pytester.makepyfile( + """ + def test_simple(): + pass + """ + ) + + result = pytester.runpytest("--collect-only", "--log-cli-level=INFO", verbose) + + expected_lines = [] + + if not verbose: + expected_lines.extend( + [ + "*collected 1 item*", + "**", + "*1 test collected*", + ] + ) + elif verbose == "-q": + result.stdout.no_fnmatch_line("*collected 1 item**") + expected_lines.extend( + [ + "*test_collection_collect_only_live_logging.py::test_simple*", + "1 test collected in [0-9].[0-9][0-9]s", + ] + ) + elif verbose == "-qq": + result.stdout.no_fnmatch_line("*collected 1 item**") + expected_lines.extend(["*test_collection_collect_only_live_logging.py: 1*"]) + + result.stdout.fnmatch_lines(expected_lines) + + +def test_collection_logging_to_file(pytester: Pytester) -> None: + log_file = str(pytester.path.joinpath("pytest.log")) + + pytester.makeini( + """ + [pytest] + log_file={} + log_file_level = INFO + """.format( + log_file + ) + ) + + pytester.makepyfile( + """ + import logging + + logging.getLogger().info("Normal message") + + def test_simple(): + logging.getLogger().debug("debug message in test_simple") + logging.getLogger().info("info message in test_simple") + """ + ) + + result = pytester.runpytest() + + result.stdout.no_fnmatch_line("*--- live log collection ---*") + + assert result.ret == 0 + assert os.path.isfile(log_file) + with open(log_file, encoding="utf-8") as rfh: + contents = rfh.read() + assert "Normal message" in contents + assert "debug message in test_simple" not in contents + assert "info message in test_simple" in contents + + +def test_log_in_hooks(pytester: Pytester) -> None: + log_file = str(pytester.path.joinpath("pytest.log")) + + pytester.makeini( + """ + [pytest] + log_file={} + log_file_level = INFO + log_cli=true + """.format( + log_file + ) + ) + pytester.makeconftest( + """ + import logging + + def pytest_runtestloop(session): + logging.info('runtestloop') + + def pytest_sessionstart(session): + logging.info('sessionstart') + + def pytest_sessionfinish(session, exitstatus): + logging.info('sessionfinish') + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*sessionstart*", "*runtestloop*", "*sessionfinish*"]) + with open(log_file) as rfh: + contents = rfh.read() + assert "sessionstart" in contents + assert "runtestloop" in contents + assert "sessionfinish" in contents + + +def test_log_in_runtest_logreport(pytester: Pytester) -> None: + log_file = str(pytester.path.joinpath("pytest.log")) + + pytester.makeini( + """ + [pytest] + log_file={} + log_file_level = INFO + log_cli=true + """.format( + log_file + ) + ) + pytester.makeconftest( + """ + import logging + logger = logging.getLogger(__name__) + + def pytest_runtest_logreport(report): + logger.info("logreport") + """ + ) + pytester.makepyfile( + """ + def test_first(): + assert True + """ + ) + pytester.runpytest() + with open(log_file) as rfh: + contents = rfh.read() + assert contents.count("logreport") == 3 + + +def test_log_set_path(pytester: Pytester) -> None: + report_dir_base = str(pytester.path) + + pytester.makeini( + """ + [pytest] + log_file_level = DEBUG + log_cli=true + """ + ) + pytester.makeconftest( + """ + import os + import pytest + @pytest.hookimpl(hookwrapper=True, tryfirst=True) + def pytest_runtest_setup(item): + config = item.config + logging_plugin = config.pluginmanager.get_plugin("logging-plugin") + report_file = os.path.join({}, item._request.node.name) + logging_plugin.set_log_path(report_file) + yield + """.format( + repr(report_dir_base) + ) + ) + pytester.makepyfile( + """ + import logging + logger = logging.getLogger("testcase-logger") + def test_first(): + logger.info("message from test 1") + assert True + + def test_second(): + logger.debug("message from test 2") + assert True + """ + ) + pytester.runpytest() + with open(os.path.join(report_dir_base, "test_first")) as rfh: + content = rfh.read() + assert "message from test 1" in content + + with open(os.path.join(report_dir_base, "test_second")) as rfh: + content = rfh.read() + assert "message from test 2" in content + + +def test_colored_captured_log(pytester: Pytester) -> None: + """Test that the level names of captured log messages of a failing test + are colored.""" + pytester.makepyfile( + """ + import logging + + logger = logging.getLogger(__name__) + + def test_foo(): + logger.info('text going to logger from call') + assert False + """ + ) + result = pytester.runpytest("--log-level=INFO", "--color=yes") + assert result.ret == 1 + result.stdout.fnmatch_lines( + [ + "*-- Captured log call --*", + "\x1b[32mINFO \x1b[0m*text going to logger from call", + ] + ) + + +def test_colored_ansi_esc_caplogtext(pytester: Pytester) -> None: + """Make sure that caplog.text does not contain ANSI escape sequences.""" + pytester.makepyfile( + """ + import logging + + logger = logging.getLogger(__name__) + + def test_foo(caplog): + logger.info('text going to logger from call') + assert '\x1b' not in caplog.text + """ + ) + result = pytester.runpytest("--log-level=INFO", "--color=yes") + assert result.ret == 0 + + +def test_logging_emit_error(pytester: Pytester) -> None: + """An exception raised during emit() should fail the test. + + The default behavior of logging is to print "Logging error" + to stderr with the call stack and some extra details. + + pytest overrides this behavior to propagate the exception. + """ + pytester.makepyfile( + """ + import logging + + def test_bad_log(): + logging.warning('oops', 'first', 2) + """ + ) + result = pytester.runpytest() + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines( + [ + "====* FAILURES *====", + "*not all arguments converted during string formatting*", + ] + ) + + +def test_logging_emit_error_supressed(pytester: Pytester) -> None: + """If logging is configured to silently ignore errors, pytest + doesn't propagate errors either.""" + pytester.makepyfile( + """ + import logging + + def test_bad_log(monkeypatch): + monkeypatch.setattr(logging, 'raiseExceptions', False) + logging.warning('oops', 'first', 2) + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=1) + + +def test_log_file_cli_subdirectories_are_successfully_created( + pytester: Pytester, +) -> None: + path = pytester.makepyfile(""" def test_logger(): pass """) + expected = os.path.join(os.path.dirname(str(path)), "foo", "bar") + result = pytester.runpytest("--log-file=foo/bar/logf.log") + assert "logf.log" in os.listdir(expected) + assert result.ret == ExitCode.OK diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/.gitignore b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/.gitignore new file mode 100644 index 0000000000..d934447a03 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/.gitignore @@ -0,0 +1,2 @@ +*.html +assets/ diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/README.rst b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/README.rst new file mode 100644 index 0000000000..8f027c3bd3 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/README.rst @@ -0,0 +1,13 @@ +This folder contains tests and support files for smoke testing popular plugins against the current pytest version. + +The objective is to gauge if any intentional or unintentional changes in pytest break plugins. + +As a rule of thumb, we should add plugins here: + +1. That are used at large. This might be subjective in some cases, but if answer is yes to + the question: *if a new release of pytest causes pytest-X to break, will this break a ton of test suites out there?*. +2. That don't have large external dependencies: such as external services. + +Besides adding the plugin as dependency, we should also add a quick test which uses some +minimal part of the plugin, a smoke test. Also consider reusing one of the existing tests if that's +possible. diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/bdd_wallet.feature b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/bdd_wallet.feature new file mode 100644 index 0000000000..e404c4948e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/bdd_wallet.feature @@ -0,0 +1,9 @@ +Feature: Buy things with apple + + Scenario: Buy fruits + Given A wallet with 50 + + When I buy some apples for 1 + And I buy some bananas for 2 + + Then I have 47 left diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/bdd_wallet.py b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/bdd_wallet.py new file mode 100644 index 0000000000..35927ea587 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/bdd_wallet.py @@ -0,0 +1,39 @@ +from pytest_bdd import given +from pytest_bdd import scenario +from pytest_bdd import then +from pytest_bdd import when + +import pytest + + +@scenario("bdd_wallet.feature", "Buy fruits") +def test_publish(): + pass + + +@pytest.fixture +def wallet(): + class Wallet: + amount = 0 + + return Wallet() + + +@given("A wallet with 50") +def fill_wallet(wallet): + wallet.amount = 50 + + +@when("I buy some apples for 1") +def buy_apples(wallet): + wallet.amount -= 1 + + +@when("I buy some bananas for 2") +def buy_bananas(wallet): + wallet.amount -= 2 + + +@then("I have 47 left") +def check(wallet): + assert wallet.amount == 47 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/django_settings.py b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/django_settings.py new file mode 100644 index 0000000000..0715f47653 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/django_settings.py @@ -0,0 +1 @@ +SECRET_KEY = "mysecret" diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest.ini b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest.ini new file mode 100644 index 0000000000..b42b07d145 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +addopts = --strict-markers +filterwarnings = + error::pytest.PytestWarning + ignore:.*.fspath is deprecated and will be replaced by .*.path.*:pytest.PytestDeprecationWarning diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_anyio_integration.py b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_anyio_integration.py new file mode 100644 index 0000000000..65c2f59366 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_anyio_integration.py @@ -0,0 +1,8 @@ +import anyio + +import pytest + + +@pytest.mark.anyio +async def test_sleep(): + await anyio.sleep(0) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_asyncio_integration.py b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_asyncio_integration.py new file mode 100644 index 0000000000..5d2a3faccf --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_asyncio_integration.py @@ -0,0 +1,8 @@ +import asyncio + +import pytest + + +@pytest.mark.asyncio +async def test_sleep(): + await asyncio.sleep(0) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_mock_integration.py b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_mock_integration.py new file mode 100644 index 0000000000..740469d00f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_mock_integration.py @@ -0,0 +1,2 @@ +def test_mocker(mocker): + mocker.MagicMock() diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_trio_integration.py b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_trio_integration.py new file mode 100644 index 0000000000..199f7850bc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_trio_integration.py @@ -0,0 +1,8 @@ +import trio + +import pytest + + +@pytest.mark.trio +async def test_sleep(): + await trio.sleep(0) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_twisted_integration.py b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_twisted_integration.py new file mode 100644 index 0000000000..94748d036e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/pytest_twisted_integration.py @@ -0,0 +1,18 @@ +import pytest_twisted +from twisted.internet.task import deferLater + + +def sleep(): + import twisted.internet.reactor + + return deferLater(clock=twisted.internet.reactor, delay=0) + + +@pytest_twisted.inlineCallbacks +def test_inlineCallbacks(): + yield sleep() + + +@pytest_twisted.ensureDeferred +async def test_inlineCallbacks_async(): + await sleep() diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/requirements.txt b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/requirements.txt new file mode 100644 index 0000000000..90b253cc6d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/requirements.txt @@ -0,0 +1,15 @@ +anyio[curio,trio]==3.4.0 +django==3.2.9 +pytest-asyncio==0.16.0 +pytest-bdd==5.0.0 +pytest-cov==3.0.0 +pytest-django==4.5.1 +pytest-flakes==4.0.5 +pytest-html==3.1.1 +pytest-mock==3.6.1 +pytest-rerunfailures==10.2 +pytest-sugar==0.9.4 +pytest-trio==0.7.0 +pytest-twisted==1.13.4 +twisted==21.7.0 +pytest-xvfb==2.0.0 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/simple_integration.py b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/simple_integration.py new file mode 100644 index 0000000000..20b2fc4b5b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/plugins_integration/simple_integration.py @@ -0,0 +1,10 @@ +import pytest + + +def test_foo(): + assert True + + +@pytest.mark.parametrize("i", range(3)) +def test_bar(i): + assert True diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/python/approx.py b/testing/web-platform/tests/tools/third_party/pytest/testing/python/approx.py new file mode 100644 index 0000000000..0d411d8a6d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/python/approx.py @@ -0,0 +1,872 @@ +import operator +import sys +from contextlib import contextmanager +from decimal import Decimal +from fractions import Fraction +from operator import eq +from operator import ne +from typing import Optional + +import pytest +from _pytest.pytester import Pytester +from pytest import approx + +inf, nan = float("inf"), float("nan") + + +@pytest.fixture +def mocked_doctest_runner(monkeypatch): + import doctest + + class MockedPdb: + def __init__(self, out): + pass + + def set_trace(self): + raise NotImplementedError("not used") + + def reset(self): + pass + + def set_continue(self): + pass + + monkeypatch.setattr("doctest._OutputRedirectingPdb", MockedPdb) + + class MyDocTestRunner(doctest.DocTestRunner): + def report_failure(self, out, test, example, got): + raise AssertionError( + "'{}' evaluates to '{}', not '{}'".format( + example.source.strip(), got.strip(), example.want.strip() + ) + ) + + return MyDocTestRunner() + + +@contextmanager +def temporary_verbosity(config, verbosity=0): + original_verbosity = config.getoption("verbose") + config.option.verbose = verbosity + try: + yield + finally: + config.option.verbose = original_verbosity + + +@pytest.fixture +def assert_approx_raises_regex(pytestconfig): + def do_assert(lhs, rhs, expected_message, verbosity_level=0): + import re + + with temporary_verbosity(pytestconfig, verbosity_level): + with pytest.raises(AssertionError) as e: + assert lhs == approx(rhs) + + nl = "\n" + obtained_message = str(e.value).splitlines()[1:] + assert len(obtained_message) == len(expected_message), ( + "Regex message length doesn't match obtained.\n" + "Obtained:\n" + f"{nl.join(obtained_message)}\n\n" + "Expected regex:\n" + f"{nl.join(expected_message)}\n\n" + ) + + for i, (obtained_line, expected_line) in enumerate( + zip(obtained_message, expected_message) + ): + regex = re.compile(expected_line) + assert regex.match(obtained_line) is not None, ( + "Unexpected error message:\n" + f"{nl.join(obtained_message)}\n\n" + "Did not match regex:\n" + f"{nl.join(expected_message)}\n\n" + f"With verbosity level = {verbosity_level}, on line {i}" + ) + + return do_assert + + +SOME_FLOAT = r"[+-]?([0-9]*[.])?[0-9]+\s*" +SOME_INT = r"[0-9]+\s*" + + +class TestApprox: + def test_error_messages(self, assert_approx_raises_regex): + np = pytest.importorskip("numpy") + + assert_approx_raises_regex( + 2.0, + 1.0, + [ + " comparison failed", + f" Obtained: {SOME_FLOAT}", + f" Expected: {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + assert_approx_raises_regex( + {"a": 1.0, "b": 1000.0, "c": 1000000.0}, + { + "a": 2.0, + "b": 1000.0, + "c": 3000000.0, + }, + [ + r" comparison failed. Mismatched elements: 2 / 3:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected ", + rf" a \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + rf" c \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + assert_approx_raises_regex( + [1.0, 2.0, 3.0, 4.0], + [1.0, 3.0, 3.0, 5.0], + [ + r" comparison failed. Mismatched elements: 2 / 4:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected ", + rf" 1 \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + rf" 3 \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + a = np.linspace(0, 100, 20) + b = np.linspace(0, 100, 20) + a[10] += 0.5 + assert_approx_raises_regex( + a, + b, + [ + r" comparison failed. Mismatched elements: 1 / 20:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected", + rf" \(10,\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + assert_approx_raises_regex( + np.array( + [ + [[1.1987311, 12412342.3], [3.214143244, 1423412423415.677]], + [[1, 2], [3, 219371297321973]], + ] + ), + np.array( + [ + [[1.12313, 12412342.3], [3.214143244, 534523542345.677]], + [[1, 2], [3, 7]], + ] + ), + [ + r" comparison failed. Mismatched elements: 3 / 8:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index\s+\| Obtained\s+\| Expected\s+", + rf" \(0, 0, 0\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + rf" \(0, 1, 1\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + rf" \(1, 1, 1\) \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + # Specific test for comparison with 0.0 (relative diff will be 'inf') + assert_approx_raises_regex( + [0.0], + [1.0], + [ + r" comparison failed. Mismatched elements: 1 / 1:", + rf" Max absolute difference: {SOME_FLOAT}", + r" Max relative difference: inf", + r" Index \| Obtained\s+\| Expected ", + rf"\s*0\s*\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + assert_approx_raises_regex( + np.array([0.0]), + np.array([1.0]), + [ + r" comparison failed. Mismatched elements: 1 / 1:", + rf" Max absolute difference: {SOME_FLOAT}", + r" Max relative difference: inf", + r" Index \| Obtained\s+\| Expected ", + rf"\s*\(0,\)\s*\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + ], + ) + + def test_error_messages_invalid_args(self, assert_approx_raises_regex): + np = pytest.importorskip("numpy") + with pytest.raises(AssertionError) as e: + assert np.array([[1.2, 3.4], [4.0, 5.0]]) == pytest.approx( + np.array([[4.0], [5.0]]) + ) + message = "\n".join(str(e.value).split("\n")[1:]) + assert message == "\n".join( + [ + " Impossible to compare arrays with different shapes.", + " Shapes: (2, 1) and (2, 2)", + ] + ) + + with pytest.raises(AssertionError) as e: + assert [1.0, 2.0, 3.0] == pytest.approx([4.0, 5.0]) + message = "\n".join(str(e.value).split("\n")[1:]) + assert message == "\n".join( + [ + " Impossible to compare lists with different sizes.", + " Lengths: 2 and 3", + ] + ) + + def test_error_messages_with_different_verbosity(self, assert_approx_raises_regex): + np = pytest.importorskip("numpy") + for v in [0, 1, 2]: + # Verbosity level doesn't affect the error message for scalars + assert_approx_raises_regex( + 2.0, + 1.0, + [ + " comparison failed", + f" Obtained: {SOME_FLOAT}", + f" Expected: {SOME_FLOAT} ± {SOME_FLOAT}", + ], + verbosity_level=v, + ) + + a = np.linspace(1, 101, 20) + b = np.linspace(2, 102, 20) + assert_approx_raises_regex( + a, + b, + [ + r" comparison failed. Mismatched elements: 20 / 20:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected", + rf" \(0,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + rf" \(1,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + rf" \(2,\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}...", + "", + rf"\s*...Full output truncated \({SOME_INT} lines hidden\), use '-vv' to show", + ], + verbosity_level=0, + ) + + assert_approx_raises_regex( + a, + b, + [ + r" comparison failed. Mismatched elements: 20 / 20:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected", + ] + + [ + rf" \({i},\)\s+\| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}" + for i in range(20) + ], + verbosity_level=2, + ) + + def test_repr_string(self): + assert repr(approx(1.0)) == "1.0 ± 1.0e-06" + assert repr(approx([1.0, 2.0])) == "approx([1.0 ± 1.0e-06, 2.0 ± 2.0e-06])" + assert repr(approx((1.0, 2.0))) == "approx((1.0 ± 1.0e-06, 2.0 ± 2.0e-06))" + assert repr(approx(inf)) == "inf" + assert repr(approx(1.0, rel=nan)) == "1.0 ± ???" + assert repr(approx(1.0, rel=inf)) == "1.0 ± inf" + + # Dictionaries aren't ordered, so we need to check both orders. + assert repr(approx({"a": 1.0, "b": 2.0})) in ( + "approx({'a': 1.0 ± 1.0e-06, 'b': 2.0 ± 2.0e-06})", + "approx({'b': 2.0 ± 2.0e-06, 'a': 1.0 ± 1.0e-06})", + ) + + def test_repr_complex_numbers(self): + assert repr(approx(inf + 1j)) == "(inf+1j)" + assert repr(approx(1.0j, rel=inf)) == "1j ± inf" + + # can't compute a sensible tolerance + assert repr(approx(nan + 1j)) == "(nan+1j) ± ???" + + assert repr(approx(1.0j)) == "1j ± 1.0e-06 ∠ ±180°" + + # relative tolerance is scaled to |3+4j| = 5 + assert repr(approx(3 + 4 * 1j)) == "(3+4j) ± 5.0e-06 ∠ ±180°" + + # absolute tolerance is not scaled + assert repr(approx(3.3 + 4.4 * 1j, abs=0.02)) == "(3.3+4.4j) ± 2.0e-02 ∠ ±180°" + + @pytest.mark.parametrize( + "value, expected_repr_string", + [ + (5.0, "approx(5.0 ± 5.0e-06)"), + ([5.0], "approx([5.0 ± 5.0e-06])"), + ([[5.0]], "approx([[5.0 ± 5.0e-06]])"), + ([[5.0, 6.0]], "approx([[5.0 ± 5.0e-06, 6.0 ± 6.0e-06]])"), + ([[5.0], [6.0]], "approx([[5.0 ± 5.0e-06], [6.0 ± 6.0e-06]])"), + ], + ) + def test_repr_nd_array(self, value, expected_repr_string): + """Make sure that arrays of all different dimensions are repr'd correctly.""" + np = pytest.importorskip("numpy") + np_array = np.array(value) + assert repr(approx(np_array)) == expected_repr_string + + def test_bool(self): + with pytest.raises(AssertionError) as err: + assert approx(1) + + assert err.match(r"approx\(\) is not supported in a boolean context") + + def test_operator_overloading(self): + assert 1 == approx(1, rel=1e-6, abs=1e-12) + assert not (1 != approx(1, rel=1e-6, abs=1e-12)) + assert 10 != approx(1, rel=1e-6, abs=1e-12) + assert not (10 == approx(1, rel=1e-6, abs=1e-12)) + + def test_exactly_equal(self): + examples = [ + (2.0, 2.0), + (0.1e200, 0.1e200), + (1.123e-300, 1.123e-300), + (12345, 12345.0), + (0.0, -0.0), + (345678, 345678), + (Decimal("1.0001"), Decimal("1.0001")), + (Fraction(1, 3), Fraction(-1, -3)), + ] + for a, x in examples: + assert a == approx(x) + + def test_opposite_sign(self): + examples = [(eq, 1e-100, -1e-100), (ne, 1e100, -1e100)] + for op, a, x in examples: + assert op(a, approx(x)) + + def test_zero_tolerance(self): + within_1e10 = [(1.1e-100, 1e-100), (-1.1e-100, -1e-100)] + for a, x in within_1e10: + assert x == approx(x, rel=0.0, abs=0.0) + assert a != approx(x, rel=0.0, abs=0.0) + assert a == approx(x, rel=0.0, abs=5e-101) + assert a != approx(x, rel=0.0, abs=5e-102) + assert a == approx(x, rel=5e-1, abs=0.0) + assert a != approx(x, rel=5e-2, abs=0.0) + + @pytest.mark.parametrize( + ("rel", "abs"), + [ + (-1e100, None), + (None, -1e100), + (1e100, -1e100), + (-1e100, 1e100), + (-1e100, -1e100), + ], + ) + def test_negative_tolerance( + self, rel: Optional[float], abs: Optional[float] + ) -> None: + # Negative tolerances are not allowed. + with pytest.raises(ValueError): + 1.1 == approx(1, rel, abs) + + def test_negative_tolerance_message(self): + # Error message for negative tolerance should include the value. + with pytest.raises(ValueError, match="-3"): + 0 == approx(1, abs=-3) + with pytest.raises(ValueError, match="-3"): + 0 == approx(1, rel=-3) + + def test_inf_tolerance(self): + # Everything should be equal if the tolerance is infinite. + large_diffs = [(1, 1000), (1e-50, 1e50), (-1.0, -1e300), (0.0, 10)] + for a, x in large_diffs: + assert a != approx(x, rel=0.0, abs=0.0) + assert a == approx(x, rel=inf, abs=0.0) + assert a == approx(x, rel=0.0, abs=inf) + assert a == approx(x, rel=inf, abs=inf) + + def test_inf_tolerance_expecting_zero(self) -> None: + # If the relative tolerance is zero but the expected value is infinite, + # the actual tolerance is a NaN, which should be an error. + with pytest.raises(ValueError): + 1 == approx(0, rel=inf, abs=0.0) + with pytest.raises(ValueError): + 1 == approx(0, rel=inf, abs=inf) + + def test_nan_tolerance(self) -> None: + with pytest.raises(ValueError): + 1.1 == approx(1, rel=nan) + with pytest.raises(ValueError): + 1.1 == approx(1, abs=nan) + with pytest.raises(ValueError): + 1.1 == approx(1, rel=nan, abs=nan) + + def test_reasonable_defaults(self): + # Whatever the defaults are, they should work for numbers close to 1 + # than have a small amount of floating-point error. + assert 0.1 + 0.2 == approx(0.3) + + def test_default_tolerances(self): + # This tests the defaults as they are currently set. If you change the + # defaults, this test will fail but you should feel free to change it. + # None of the other tests (except the doctests) should be affected by + # the choice of defaults. + examples = [ + # Relative tolerance used. + (eq, 1e100 + 1e94, 1e100), + (ne, 1e100 + 2e94, 1e100), + (eq, 1e0 + 1e-6, 1e0), + (ne, 1e0 + 2e-6, 1e0), + # Absolute tolerance used. + (eq, 1e-100, +1e-106), + (eq, 1e-100, +2e-106), + (eq, 1e-100, 0), + ] + for op, a, x in examples: + assert op(a, approx(x)) + + def test_custom_tolerances(self): + assert 1e8 + 1e0 == approx(1e8, rel=5e-8, abs=5e0) + assert 1e8 + 1e0 == approx(1e8, rel=5e-9, abs=5e0) + assert 1e8 + 1e0 == approx(1e8, rel=5e-8, abs=5e-1) + assert 1e8 + 1e0 != approx(1e8, rel=5e-9, abs=5e-1) + + assert 1e0 + 1e-8 == approx(1e0, rel=5e-8, abs=5e-8) + assert 1e0 + 1e-8 == approx(1e0, rel=5e-9, abs=5e-8) + assert 1e0 + 1e-8 == approx(1e0, rel=5e-8, abs=5e-9) + assert 1e0 + 1e-8 != approx(1e0, rel=5e-9, abs=5e-9) + + assert 1e-8 + 1e-16 == approx(1e-8, rel=5e-8, abs=5e-16) + assert 1e-8 + 1e-16 == approx(1e-8, rel=5e-9, abs=5e-16) + assert 1e-8 + 1e-16 == approx(1e-8, rel=5e-8, abs=5e-17) + assert 1e-8 + 1e-16 != approx(1e-8, rel=5e-9, abs=5e-17) + + def test_relative_tolerance(self): + within_1e8_rel = [(1e8 + 1e0, 1e8), (1e0 + 1e-8, 1e0), (1e-8 + 1e-16, 1e-8)] + for a, x in within_1e8_rel: + assert a == approx(x, rel=5e-8, abs=0.0) + assert a != approx(x, rel=5e-9, abs=0.0) + + def test_absolute_tolerance(self): + within_1e8_abs = [(1e8 + 9e-9, 1e8), (1e0 + 9e-9, 1e0), (1e-8 + 9e-9, 1e-8)] + for a, x in within_1e8_abs: + assert a == approx(x, rel=0, abs=5e-8) + assert a != approx(x, rel=0, abs=5e-9) + + def test_expecting_zero(self): + examples = [ + (ne, 1e-6, 0.0), + (ne, -1e-6, 0.0), + (eq, 1e-12, 0.0), + (eq, -1e-12, 0.0), + (ne, 2e-12, 0.0), + (ne, -2e-12, 0.0), + (ne, inf, 0.0), + (ne, nan, 0.0), + ] + for op, a, x in examples: + assert op(a, approx(x, rel=0.0, abs=1e-12)) + assert op(a, approx(x, rel=1e-6, abs=1e-12)) + + def test_expecting_inf(self): + examples = [ + (eq, inf, inf), + (eq, -inf, -inf), + (ne, inf, -inf), + (ne, 0.0, inf), + (ne, nan, inf), + ] + for op, a, x in examples: + assert op(a, approx(x)) + + def test_expecting_nan(self): + examples = [ + (eq, nan, nan), + (eq, -nan, -nan), + (eq, nan, -nan), + (ne, 0.0, nan), + (ne, inf, nan), + ] + for op, a, x in examples: + # Nothing is equal to NaN by default. + assert a != approx(x) + + # If ``nan_ok=True``, then NaN is equal to NaN. + assert op(a, approx(x, nan_ok=True)) + + def test_int(self): + within_1e6 = [(1000001, 1000000), (-1000001, -1000000)] + for a, x in within_1e6: + assert a == approx(x, rel=5e-6, abs=0) + assert a != approx(x, rel=5e-7, abs=0) + assert approx(x, rel=5e-6, abs=0) == a + assert approx(x, rel=5e-7, abs=0) != a + + def test_decimal(self): + within_1e6 = [ + (Decimal("1.000001"), Decimal("1.0")), + (Decimal("-1.000001"), Decimal("-1.0")), + ] + for a, x in within_1e6: + assert a == approx(x) + assert a == approx(x, rel=Decimal("5e-6"), abs=0) + assert a != approx(x, rel=Decimal("5e-7"), abs=0) + assert approx(x, rel=Decimal("5e-6"), abs=0) == a + assert approx(x, rel=Decimal("5e-7"), abs=0) != a + + def test_fraction(self): + within_1e6 = [ + (1 + Fraction(1, 1000000), Fraction(1)), + (-1 - Fraction(-1, 1000000), Fraction(-1)), + ] + for a, x in within_1e6: + assert a == approx(x, rel=5e-6, abs=0) + assert a != approx(x, rel=5e-7, abs=0) + assert approx(x, rel=5e-6, abs=0) == a + assert approx(x, rel=5e-7, abs=0) != a + + def test_complex(self): + within_1e6 = [ + (1.000001 + 1.0j, 1.0 + 1.0j), + (1.0 + 1.000001j, 1.0 + 1.0j), + (-1.000001 + 1.0j, -1.0 + 1.0j), + (1.0 - 1.000001j, 1.0 - 1.0j), + ] + for a, x in within_1e6: + assert a == approx(x, rel=5e-6, abs=0) + assert a != approx(x, rel=5e-7, abs=0) + assert approx(x, rel=5e-6, abs=0) == a + assert approx(x, rel=5e-7, abs=0) != a + + def test_list(self): + actual = [1 + 1e-7, 2 + 1e-8] + expected = [1, 2] + + # Return false if any element is outside the tolerance. + assert actual == approx(expected, rel=5e-7, abs=0) + assert actual != approx(expected, rel=5e-8, abs=0) + assert approx(expected, rel=5e-7, abs=0) == actual + assert approx(expected, rel=5e-8, abs=0) != actual + + def test_list_decimal(self): + actual = [Decimal("1.000001"), Decimal("2.000001")] + expected = [Decimal("1"), Decimal("2")] + + assert actual == approx(expected) + + def test_list_wrong_len(self): + assert [1, 2] != approx([1]) + assert [1, 2] != approx([1, 2, 3]) + + def test_tuple(self): + actual = (1 + 1e-7, 2 + 1e-8) + expected = (1, 2) + + # Return false if any element is outside the tolerance. + assert actual == approx(expected, rel=5e-7, abs=0) + assert actual != approx(expected, rel=5e-8, abs=0) + assert approx(expected, rel=5e-7, abs=0) == actual + assert approx(expected, rel=5e-8, abs=0) != actual + + def test_tuple_wrong_len(self): + assert (1, 2) != approx((1,)) + assert (1, 2) != approx((1, 2, 3)) + + def test_tuple_vs_other(self): + assert 1 != approx((1,)) + + def test_dict(self): + actual = {"a": 1 + 1e-7, "b": 2 + 1e-8} + # Dictionaries became ordered in python3.6, so switch up the order here + # to make sure it doesn't matter. + expected = {"b": 2, "a": 1} + + # Return false if any element is outside the tolerance. + assert actual == approx(expected, rel=5e-7, abs=0) + assert actual != approx(expected, rel=5e-8, abs=0) + assert approx(expected, rel=5e-7, abs=0) == actual + assert approx(expected, rel=5e-8, abs=0) != actual + + def test_dict_decimal(self): + actual = {"a": Decimal("1.000001"), "b": Decimal("2.000001")} + # Dictionaries became ordered in python3.6, so switch up the order here + # to make sure it doesn't matter. + expected = {"b": Decimal("2"), "a": Decimal("1")} + + assert actual == approx(expected) + + def test_dict_wrong_len(self): + assert {"a": 1, "b": 2} != approx({"a": 1}) + assert {"a": 1, "b": 2} != approx({"a": 1, "c": 2}) + assert {"a": 1, "b": 2} != approx({"a": 1, "b": 2, "c": 3}) + + def test_dict_nonnumeric(self): + assert {"a": 1.0, "b": None} == pytest.approx({"a": 1.0, "b": None}) + assert {"a": 1.0, "b": 1} != pytest.approx({"a": 1.0, "b": None}) + + def test_dict_vs_other(self): + assert 1 != approx({"a": 0}) + + def test_numpy_array(self): + np = pytest.importorskip("numpy") + + actual = np.array([1 + 1e-7, 2 + 1e-8]) + expected = np.array([1, 2]) + + # Return false if any element is outside the tolerance. + assert actual == approx(expected, rel=5e-7, abs=0) + assert actual != approx(expected, rel=5e-8, abs=0) + assert approx(expected, rel=5e-7, abs=0) == expected + assert approx(expected, rel=5e-8, abs=0) != actual + + # Should be able to compare lists with numpy arrays. + assert list(actual) == approx(expected, rel=5e-7, abs=0) + assert list(actual) != approx(expected, rel=5e-8, abs=0) + assert actual == approx(list(expected), rel=5e-7, abs=0) + assert actual != approx(list(expected), rel=5e-8, abs=0) + + def test_numpy_tolerance_args(self): + """ + Check that numpy rel/abs args are handled correctly + for comparison against an np.array + Check both sides of the operator, hopefully it doesn't impact things. + Test all permutations of where the approx and np.array() can show up + """ + np = pytest.importorskip("numpy") + expected = 100.0 + actual = 99.0 + abs_diff = expected - actual + rel_diff = (expected - actual) / expected + + tests = [ + (eq, abs_diff, 0), + (eq, 0, rel_diff), + (ne, 0, rel_diff / 2.0), # rel diff fail + (ne, abs_diff / 2.0, 0), # abs diff fail + ] + + for op, _abs, _rel in tests: + assert op(np.array(actual), approx(expected, abs=_abs, rel=_rel)) # a, b + assert op(approx(expected, abs=_abs, rel=_rel), np.array(actual)) # b, a + + assert op(actual, approx(np.array(expected), abs=_abs, rel=_rel)) # a, b + assert op(approx(np.array(expected), abs=_abs, rel=_rel), actual) # b, a + + assert op(np.array(actual), approx(np.array(expected), abs=_abs, rel=_rel)) + assert op(approx(np.array(expected), abs=_abs, rel=_rel), np.array(actual)) + + def test_numpy_expecting_nan(self): + np = pytest.importorskip("numpy") + examples = [ + (eq, nan, nan), + (eq, -nan, -nan), + (eq, nan, -nan), + (ne, 0.0, nan), + (ne, inf, nan), + ] + for op, a, x in examples: + # Nothing is equal to NaN by default. + assert np.array(a) != approx(x) + assert a != approx(np.array(x)) + + # If ``nan_ok=True``, then NaN is equal to NaN. + assert op(np.array(a), approx(x, nan_ok=True)) + assert op(a, approx(np.array(x), nan_ok=True)) + + def test_numpy_expecting_inf(self): + np = pytest.importorskip("numpy") + examples = [ + (eq, inf, inf), + (eq, -inf, -inf), + (ne, inf, -inf), + (ne, 0.0, inf), + (ne, nan, inf), + ] + for op, a, x in examples: + assert op(np.array(a), approx(x)) + assert op(a, approx(np.array(x))) + assert op(np.array(a), approx(np.array(x))) + + def test_numpy_array_wrong_shape(self): + np = pytest.importorskip("numpy") + + a12 = np.array([[1, 2]]) + a21 = np.array([[1], [2]]) + + assert a12 != approx(a21) + assert a21 != approx(a12) + + def test_numpy_array_protocol(self): + """ + array-like objects such as tensorflow's DeviceArray are handled like ndarray. + See issue #8132 + """ + np = pytest.importorskip("numpy") + + class DeviceArray: + def __init__(self, value, size): + self.value = value + self.size = size + + def __array__(self): + return self.value * np.ones(self.size) + + class DeviceScalar: + def __init__(self, value): + self.value = value + + def __array__(self): + return np.array(self.value) + + expected = 1 + actual = 1 + 1e-6 + assert approx(expected) == DeviceArray(actual, size=1) + assert approx(expected) == DeviceArray(actual, size=2) + assert approx(expected) == DeviceScalar(actual) + assert approx(DeviceScalar(expected)) == actual + assert approx(DeviceScalar(expected)) == DeviceScalar(actual) + + def test_doctests(self, mocked_doctest_runner) -> None: + import doctest + + parser = doctest.DocTestParser() + assert approx.__doc__ is not None + test = parser.get_doctest( + approx.__doc__, {"approx": approx}, approx.__name__, None, None + ) + mocked_doctest_runner.run(test) + + def test_unicode_plus_minus(self, pytester: Pytester) -> None: + """ + Comparing approx instances inside lists should not produce an error in the detailed diff. + Integration test for issue #2111. + """ + pytester.makepyfile( + """ + import pytest + def test_foo(): + assert [3] == [pytest.approx(4)] + """ + ) + expected = "4.0e-06" + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [f"*At index 0 diff: 3 != 4 ± {expected}", "=* 1 failed in *="] + ) + + @pytest.mark.parametrize( + "x, name", + [ + pytest.param([[1]], "data structures", id="nested-list"), + pytest.param({"key": {"key": 1}}, "dictionaries", id="nested-dict"), + ], + ) + def test_expected_value_type_error(self, x, name): + with pytest.raises( + TypeError, + match=fr"pytest.approx\(\) does not support nested {name}:", + ): + approx(x) + + @pytest.mark.parametrize( + "x", + [ + pytest.param(None), + pytest.param("string"), + pytest.param(["string"], id="nested-str"), + pytest.param({"key": "string"}, id="dict-with-string"), + ], + ) + def test_nonnumeric_okay_if_equal(self, x): + assert x == approx(x) + + @pytest.mark.parametrize( + "x", + [ + pytest.param("string"), + pytest.param(["string"], id="nested-str"), + pytest.param({"key": "string"}, id="dict-with-string"), + ], + ) + def test_nonnumeric_false_if_unequal(self, x): + """For nonnumeric types, x != pytest.approx(y) reduces to x != y""" + assert "ab" != approx("abc") + assert ["ab"] != approx(["abc"]) + # in particular, both of these should return False + assert {"a": 1.0} != approx({"a": None}) + assert {"a": None} != approx({"a": 1.0}) + + assert 1.0 != approx(None) + assert None != approx(1.0) # noqa: E711 + + assert 1.0 != approx([None]) + assert None != approx([1.0]) # noqa: E711 + + @pytest.mark.skipif(sys.version_info < (3, 7), reason="requires ordered dicts") + def test_nonnumeric_dict_repr(self): + """Dicts with non-numerics and infinites have no tolerances""" + x1 = {"foo": 1.0000005, "bar": None, "foobar": inf} + assert ( + repr(approx(x1)) + == "approx({'foo': 1.0000005 ± 1.0e-06, 'bar': None, 'foobar': inf})" + ) + + def test_nonnumeric_list_repr(self): + """Lists with non-numerics and infinites have no tolerances""" + x1 = [1.0000005, None, inf] + assert repr(approx(x1)) == "approx([1.0000005 ± 1.0e-06, None, inf])" + + @pytest.mark.parametrize( + "op", + [ + pytest.param(operator.le, id="<="), + pytest.param(operator.lt, id="<"), + pytest.param(operator.ge, id=">="), + pytest.param(operator.gt, id=">"), + ], + ) + def test_comparison_operator_type_error(self, op): + """pytest.approx should raise TypeError for operators other than == and != (#2003).""" + with pytest.raises(TypeError): + op(1, approx(1, rel=1e-6, abs=1e-12)) + + def test_numpy_array_with_scalar(self): + np = pytest.importorskip("numpy") + + actual = np.array([1 + 1e-7, 1 - 1e-8]) + expected = 1.0 + + assert actual == approx(expected, rel=5e-7, abs=0) + assert actual != approx(expected, rel=5e-8, abs=0) + assert approx(expected, rel=5e-7, abs=0) == actual + assert approx(expected, rel=5e-8, abs=0) != actual + + def test_numpy_scalar_with_array(self): + np = pytest.importorskip("numpy") + + actual = 1.0 + expected = np.array([1 + 1e-7, 1 - 1e-8]) + + assert actual == approx(expected, rel=5e-7, abs=0) + assert actual != approx(expected, rel=5e-8, abs=0) + assert approx(expected, rel=5e-7, abs=0) == actual + assert approx(expected, rel=5e-8, abs=0) != actual + + def test_generic_sized_iterable_object(self): + class MySizedIterable: + def __iter__(self): + return iter([1, 2, 3, 4]) + + def __len__(self): + return 4 + + expected = MySizedIterable() + assert [1, 2, 3, 4] == approx(expected) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/python/collect.py b/testing/web-platform/tests/tools/third_party/pytest/testing/python/collect.py new file mode 100644 index 0000000000..ac3edd395a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/python/collect.py @@ -0,0 +1,1493 @@ +import os +import sys +import textwrap +from typing import Any +from typing import Dict + +import _pytest._code +import pytest +from _pytest.config import ExitCode +from _pytest.main import Session +from _pytest.monkeypatch import MonkeyPatch +from _pytest.nodes import Collector +from _pytest.pytester import Pytester +from _pytest.python import Class +from _pytest.python import Function + + +class TestModule: + def test_failing_import(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol("import alksdjalskdjalkjals") + pytest.raises(Collector.CollectError, modcol.collect) + + def test_import_duplicate(self, pytester: Pytester) -> None: + a = pytester.mkdir("a") + b = pytester.mkdir("b") + p1 = a.joinpath("test_whatever.py") + p1.touch() + p2 = b.joinpath("test_whatever.py") + p2.touch() + # ensure we don't have it imported already + sys.modules.pop(p1.stem, None) + + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*import*mismatch*", + "*imported*test_whatever*", + "*%s*" % p1, + "*not the same*", + "*%s*" % p2, + "*HINT*", + ] + ) + + def test_import_prepend_append( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + root1 = pytester.mkdir("root1") + root2 = pytester.mkdir("root2") + root1.joinpath("x456.py").touch() + root2.joinpath("x456.py").touch() + p = root2.joinpath("test_x456.py") + monkeypatch.syspath_prepend(str(root1)) + p.write_text( + textwrap.dedent( + """\ + import x456 + def test(): + assert x456.__file__.startswith({!r}) + """.format( + str(root2) + ) + ) + ) + with monkeypatch.context() as mp: + mp.chdir(root2) + reprec = pytester.inline_run("--import-mode=append") + reprec.assertoutcome(passed=0, failed=1) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_syntax_error_in_module(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol("this is a syntax error") + pytest.raises(modcol.CollectError, modcol.collect) + pytest.raises(modcol.CollectError, modcol.collect) + + def test_module_considers_pluginmanager_at_import(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol("pytest_plugins='xasdlkj',") + pytest.raises(ImportError, lambda: modcol.obj) + + def test_invalid_test_module_name(self, pytester: Pytester) -> None: + a = pytester.mkdir("a") + a.joinpath("test_one.part1.py").touch() + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "ImportError while importing test module*test_one.part1*", + "Hint: make sure your test modules/packages have valid Python names.", + ] + ) + + @pytest.mark.parametrize("verbose", [0, 1, 2]) + def test_show_traceback_import_error( + self, pytester: Pytester, verbose: int + ) -> None: + """Import errors when collecting modules should display the traceback (#1976). + + With low verbosity we omit pytest and internal modules, otherwise show all traceback entries. + """ + pytester.makepyfile( + foo_traceback_import_error=""" + from bar_traceback_import_error import NOT_AVAILABLE + """, + bar_traceback_import_error="", + ) + pytester.makepyfile( + """ + import foo_traceback_import_error + """ + ) + args = ("-v",) * verbose + result = pytester.runpytest(*args) + result.stdout.fnmatch_lines( + [ + "ImportError while importing test module*", + "Traceback:", + "*from bar_traceback_import_error import NOT_AVAILABLE", + "*cannot import name *NOT_AVAILABLE*", + ] + ) + assert result.ret == 2 + + stdout = result.stdout.str() + if verbose == 2: + assert "_pytest" in stdout + else: + assert "_pytest" not in stdout + + def test_show_traceback_import_error_unicode(self, pytester: Pytester) -> None: + """Check test modules collected which raise ImportError with unicode messages + are handled properly (#2336). + """ + pytester.makepyfile("raise ImportError('Something bad happened ☺')") + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "ImportError while importing test module*", + "Traceback:", + "*raise ImportError*Something bad happened*", + ] + ) + assert result.ret == 2 + + +class TestClass: + def test_class_with_init_warning(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + class TestClass1(object): + def __init__(self): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*cannot collect test class 'TestClass1' because it has " + "a __init__ constructor (from: test_class_with_init_warning.py)" + ] + ) + + def test_class_with_new_warning(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + class TestClass1(object): + def __new__(self): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*cannot collect test class 'TestClass1' because it has " + "a __new__ constructor (from: test_class_with_new_warning.py)" + ] + ) + + def test_class_subclassobject(self, pytester: Pytester) -> None: + pytester.getmodulecol( + """ + class test(object): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*collected 0*"]) + + def test_static_method(self, pytester: Pytester) -> None: + """Support for collecting staticmethod tests (#2528, #2699)""" + pytester.getmodulecol( + """ + import pytest + class Test(object): + @staticmethod + def test_something(): + pass + + @pytest.fixture + def fix(self): + return 1 + + @staticmethod + def test_fix(fix): + assert fix == 1 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*collected 2 items*", "*2 passed in*"]) + + def test_setup_teardown_class_as_classmethod(self, pytester: Pytester) -> None: + pytester.makepyfile( + test_mod1=""" + class TestClassMethod(object): + @classmethod + def setup_class(cls): + pass + def test_1(self): + pass + @classmethod + def teardown_class(cls): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_issue1035_obj_has_getattr(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol( + """ + class Chameleon(object): + def __getattr__(self, name): + return True + chameleon = Chameleon() + """ + ) + colitems = modcol.collect() + assert len(colitems) == 0 + + def test_issue1579_namedtuple(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import collections + + TestCase = collections.namedtuple('TestCase', ['a']) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + "*cannot collect test class 'TestCase' " + "because it has a __new__ constructor*" + ) + + def test_issue2234_property(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + class TestCase(object): + @property + def prop(self): + raise NotImplementedError() + """ + ) + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +class TestFunction: + def test_getmodulecollector(self, pytester: Pytester) -> None: + item = pytester.getitem("def test_func(): pass") + modcol = item.getparent(pytest.Module) + assert isinstance(modcol, pytest.Module) + assert hasattr(modcol.obj, "test_func") + + @pytest.mark.filterwarnings("default") + def test_function_as_object_instance_ignored(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + class A(object): + def __call__(self, tmp_path): + 0/0 + + test_a = A() + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "collected 0 items", + "*test_function_as_object_instance_ignored.py:2: " + "*cannot collect 'test_a' because it is not a function.", + ] + ) + + @staticmethod + def make_function(pytester: Pytester, **kwargs: Any) -> Any: + from _pytest.fixtures import FixtureManager + + config = pytester.parseconfigure() + session = Session.from_config(config) + session._fixturemanager = FixtureManager(session) + + return pytest.Function.from_parent(parent=session, **kwargs) + + def test_function_equality(self, pytester: Pytester) -> None: + def func1(): + pass + + def func2(): + pass + + f1 = self.make_function(pytester, name="name", callobj=func1) + assert f1 == f1 + f2 = self.make_function( + pytester, name="name", callobj=func2, originalname="foobar" + ) + assert f1 != f2 + + def test_repr_produces_actual_test_id(self, pytester: Pytester) -> None: + f = self.make_function( + pytester, name=r"test[\xe5]", callobj=self.test_repr_produces_actual_test_id + ) + assert repr(f) == r"" + + def test_issue197_parametrize_emptyset(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize('arg', []) + def test_function(arg): + pass + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(skipped=1) + + def test_single_tuple_unwraps_values(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize(('arg',), [(1,)]) + def test_function(arg): + assert arg == 1 + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_issue213_parametrize_value_no_equal(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + class A(object): + def __eq__(self, other): + raise ValueError("not possible") + @pytest.mark.parametrize('arg', [A()]) + def test_function(arg): + assert arg.__class__.__name__ == "A" + """ + ) + reprec = pytester.inline_run("--fulltrace") + reprec.assertoutcome(passed=1) + + def test_parametrize_with_non_hashable_values(self, pytester: Pytester) -> None: + """Test parametrization with non-hashable values.""" + pytester.makepyfile( + """ + archival_mapping = { + '1.0': {'tag': '1.0'}, + '1.2.2a1': {'tag': 'release-1.2.2a1'}, + } + + import pytest + @pytest.mark.parametrize('key value'.split(), + archival_mapping.items()) + def test_archival_to_version(key, value): + assert key in archival_mapping + assert value == archival_mapping[key] + """ + ) + rec = pytester.inline_run() + rec.assertoutcome(passed=2) + + def test_parametrize_with_non_hashable_values_indirect( + self, pytester: Pytester + ) -> None: + """Test parametrization with non-hashable values with indirect parametrization.""" + pytester.makepyfile( + """ + archival_mapping = { + '1.0': {'tag': '1.0'}, + '1.2.2a1': {'tag': 'release-1.2.2a1'}, + } + + import pytest + + @pytest.fixture + def key(request): + return request.param + + @pytest.fixture + def value(request): + return request.param + + @pytest.mark.parametrize('key value'.split(), + archival_mapping.items(), indirect=True) + def test_archival_to_version(key, value): + assert key in archival_mapping + assert value == archival_mapping[key] + """ + ) + rec = pytester.inline_run() + rec.assertoutcome(passed=2) + + def test_parametrize_overrides_fixture(self, pytester: Pytester) -> None: + """Test parametrization when parameter overrides existing fixture with same name.""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def value(): + return 'value' + + @pytest.mark.parametrize('value', + ['overridden']) + def test_overridden_via_param(value): + assert value == 'overridden' + + @pytest.mark.parametrize('somevalue', ['overridden']) + def test_not_overridden(value, somevalue): + assert value == 'value' + assert somevalue == 'overridden' + + @pytest.mark.parametrize('other,value', [('foo', 'overridden')]) + def test_overridden_via_multiparam(other, value): + assert other == 'foo' + assert value == 'overridden' + """ + ) + rec = pytester.inline_run() + rec.assertoutcome(passed=3) + + def test_parametrize_overrides_parametrized_fixture( + self, pytester: Pytester + ) -> None: + """Test parametrization when parameter overrides existing parametrized fixture with same name.""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(params=[1, 2]) + def value(request): + return request.param + + @pytest.mark.parametrize('value', + ['overridden']) + def test_overridden_via_param(value): + assert value == 'overridden' + """ + ) + rec = pytester.inline_run() + rec.assertoutcome(passed=1) + + def test_parametrize_overrides_indirect_dependency_fixture( + self, pytester: Pytester + ) -> None: + """Test parametrization when parameter overrides a fixture that a test indirectly depends on""" + pytester.makepyfile( + """ + import pytest + + fix3_instantiated = False + + @pytest.fixture + def fix1(fix2): + return fix2 + '1' + + @pytest.fixture + def fix2(fix3): + return fix3 + '2' + + @pytest.fixture + def fix3(): + global fix3_instantiated + fix3_instantiated = True + return '3' + + @pytest.mark.parametrize('fix2', ['2']) + def test_it(fix1): + assert fix1 == '21' + assert not fix3_instantiated + """ + ) + rec = pytester.inline_run() + rec.assertoutcome(passed=1) + + def test_parametrize_with_mark(self, pytester: Pytester) -> None: + items = pytester.getitems( + """ + import pytest + @pytest.mark.foo + @pytest.mark.parametrize('arg', [ + 1, + pytest.param(2, marks=[pytest.mark.baz, pytest.mark.bar]) + ]) + def test_function(arg): + pass + """ + ) + keywords = [item.keywords for item in items] + assert ( + "foo" in keywords[0] + and "bar" not in keywords[0] + and "baz" not in keywords[0] + ) + assert "foo" in keywords[1] and "bar" in keywords[1] and "baz" in keywords[1] + + def test_parametrize_with_empty_string_arguments(self, pytester: Pytester) -> None: + items = pytester.getitems( + """\ + import pytest + + @pytest.mark.parametrize('v', ('', ' ')) + @pytest.mark.parametrize('w', ('', ' ')) + def test(v, w): ... + """ + ) + names = {item.name for item in items} + assert names == {"test[-]", "test[ -]", "test[- ]", "test[ - ]"} + + def test_function_equality_with_callspec(self, pytester: Pytester) -> None: + items = pytester.getitems( + """ + import pytest + @pytest.mark.parametrize('arg', [1,2]) + def test_function(arg): + pass + """ + ) + assert items[0] != items[1] + assert not (items[0] == items[1]) + + def test_pyfunc_call(self, pytester: Pytester) -> None: + item = pytester.getitem("def test_func(): raise ValueError") + config = item.config + + class MyPlugin1: + def pytest_pyfunc_call(self): + raise ValueError + + class MyPlugin2: + def pytest_pyfunc_call(self): + return True + + config.pluginmanager.register(MyPlugin1()) + config.pluginmanager.register(MyPlugin2()) + config.hook.pytest_runtest_setup(item=item) + config.hook.pytest_pyfunc_call(pyfuncitem=item) + + def test_multiple_parametrize(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol( + """ + import pytest + @pytest.mark.parametrize('x', [0, 1]) + @pytest.mark.parametrize('y', [2, 3]) + def test1(x, y): + pass + """ + ) + colitems = modcol.collect() + assert colitems[0].name == "test1[2-0]" + assert colitems[1].name == "test1[2-1]" + assert colitems[2].name == "test1[3-0]" + assert colitems[3].name == "test1[3-1]" + + def test_issue751_multiple_parametrize_with_ids(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol( + """ + import pytest + @pytest.mark.parametrize('x', [0], ids=['c']) + @pytest.mark.parametrize('y', [0, 1], ids=['a', 'b']) + class Test(object): + def test1(self, x, y): + pass + def test2(self, x, y): + pass + """ + ) + colitems = modcol.collect()[0].collect() + assert colitems[0].name == "test1[a-c]" + assert colitems[1].name == "test1[b-c]" + assert colitems[2].name == "test2[a-c]" + assert colitems[3].name == "test2[b-c]" + + def test_parametrize_skipif(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + m = pytest.mark.skipif('True') + + @pytest.mark.parametrize('x', [0, 1, pytest.param(2, marks=m)]) + def test_skip_if(x): + assert x < 2 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 2 passed, 1 skipped in *"]) + + def test_parametrize_skip(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + m = pytest.mark.skip('') + + @pytest.mark.parametrize('x', [0, 1, pytest.param(2, marks=m)]) + def test_skip(x): + assert x < 2 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 2 passed, 1 skipped in *"]) + + def test_parametrize_skipif_no_skip(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + m = pytest.mark.skipif('False') + + @pytest.mark.parametrize('x', [0, 1, m(2)]) + def test_skipif_no_skip(x): + assert x < 2 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 1 failed, 2 passed in *"]) + + def test_parametrize_xfail(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + m = pytest.mark.xfail('True') + + @pytest.mark.parametrize('x', [0, 1, pytest.param(2, marks=m)]) + def test_xfail(x): + assert x < 2 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 2 passed, 1 xfailed in *"]) + + def test_parametrize_passed(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + m = pytest.mark.xfail('True') + + @pytest.mark.parametrize('x', [0, 1, pytest.param(2, marks=m)]) + def test_xfail(x): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 2 passed, 1 xpassed in *"]) + + def test_parametrize_xfail_passed(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + m = pytest.mark.xfail('False') + + @pytest.mark.parametrize('x', [0, 1, m(2)]) + def test_passed(x): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 3 passed in *"]) + + def test_function_originalname(self, pytester: Pytester) -> None: + items = pytester.getitems( + """ + import pytest + + @pytest.mark.parametrize('arg', [1,2]) + def test_func(arg): + pass + + def test_no_param(): + pass + """ + ) + originalnames = [] + for x in items: + assert isinstance(x, pytest.Function) + originalnames.append(x.originalname) + assert originalnames == [ + "test_func", + "test_func", + "test_no_param", + ] + + def test_function_with_square_brackets(self, pytester: Pytester) -> None: + """Check that functions with square brackets don't cause trouble.""" + p1 = pytester.makepyfile( + """ + locals()["test_foo[name]"] = lambda: None + """ + ) + result = pytester.runpytest("-v", str(p1)) + result.stdout.fnmatch_lines( + [ + "test_function_with_square_brackets.py::test_foo[[]name[]] PASSED *", + "*= 1 passed in *", + ] + ) + + +class TestSorting: + def test_check_equality(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol( + """ + def test_pass(): pass + def test_fail(): assert 0 + """ + ) + fn1 = pytester.collect_by_name(modcol, "test_pass") + assert isinstance(fn1, pytest.Function) + fn2 = pytester.collect_by_name(modcol, "test_pass") + assert isinstance(fn2, pytest.Function) + + assert fn1 == fn2 + assert fn1 != modcol + assert hash(fn1) == hash(fn2) + + fn3 = pytester.collect_by_name(modcol, "test_fail") + assert isinstance(fn3, pytest.Function) + assert not (fn1 == fn3) + assert fn1 != fn3 + + for fn in fn1, fn2, fn3: + assert fn != 3 # type: ignore[comparison-overlap] + assert fn != modcol + assert fn != [1, 2, 3] # type: ignore[comparison-overlap] + assert [1, 2, 3] != fn # type: ignore[comparison-overlap] + assert modcol != fn + + def test_allow_sane_sorting_for_decorators(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol( + """ + def dec(f): + g = lambda: f(2) + g.place_as = f + return g + + + def test_b(y): + pass + test_b = dec(test_b) + + def test_a(y): + pass + test_a = dec(test_a) + """ + ) + colitems = modcol.collect() + assert len(colitems) == 2 + assert [item.name for item in colitems] == ["test_b", "test_a"] + + def test_ordered_by_definition_order(self, pytester: Pytester) -> None: + pytester.makepyfile( + """\ + class Test1: + def test_foo(): pass + def test_bar(): pass + class Test2: + def test_foo(): pass + test_bar = Test1.test_bar + class Test3(Test2): + def test_baz(): pass + """ + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines( + [ + "*Class Test1*", + "*Function test_foo*", + "*Function test_bar*", + "*Class Test2*", + # previously the order was flipped due to Test1.test_bar reference + "*Function test_foo*", + "*Function test_bar*", + "*Class Test3*", + "*Function test_foo*", + "*Function test_bar*", + "*Function test_baz*", + ] + ) + + +class TestConftestCustomization: + def test_pytest_pycollect_module(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + class MyModule(pytest.Module): + pass + def pytest_pycollect_makemodule(module_path, parent): + if module_path.name == "test_xyz.py": + return MyModule.from_parent(path=module_path, parent=parent) + """ + ) + pytester.makepyfile("def test_some(): pass") + pytester.makepyfile(test_xyz="def test_func(): pass") + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines(["* None: + b = pytester.path.joinpath("a", "b") + b.mkdir(parents=True) + b.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + @pytest.hookimpl(hookwrapper=True) + def pytest_pycollect_makemodule(): + outcome = yield + mod = outcome.get_result() + mod.obj.hello = "world" + """ + ) + ) + b.joinpath("test_module.py").write_text( + textwrap.dedent( + """\ + def test_hello(): + assert hello == "world" + """ + ) + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_customized_pymakeitem(self, pytester: Pytester) -> None: + b = pytester.path.joinpath("a", "b") + b.mkdir(parents=True) + b.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + @pytest.hookimpl(hookwrapper=True) + def pytest_pycollect_makeitem(): + outcome = yield + if outcome.excinfo is None: + result = outcome.get_result() + if result: + for func in result: + func._some123 = "world" + """ + ) + ) + b.joinpath("test_module.py").write_text( + textwrap.dedent( + """\ + import pytest + + @pytest.fixture() + def obj(request): + return request.node._some123 + def test_hello(obj): + assert obj == "world" + """ + ) + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_pytest_pycollect_makeitem(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + class MyFunction(pytest.Function): + pass + def pytest_pycollect_makeitem(collector, name, obj): + if name == "some": + return MyFunction.from_parent(name=name, parent=collector) + """ + ) + pytester.makepyfile("def some(): pass") + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines(["*MyFunction*some*"]) + + def test_issue2369_collect_module_fileext(self, pytester: Pytester) -> None: + """Ensure we can collect files with weird file extensions as Python + modules (#2369)""" + # We'll implement a little finder and loader to import files containing + # Python source code whose file extension is ".narf". + pytester.makeconftest( + """ + import sys, os, imp + from _pytest.python import Module + + class Loader(object): + def load_module(self, name): + return imp.load_source(name, name + ".narf") + class Finder(object): + def find_module(self, name, path=None): + if os.path.exists(name + ".narf"): + return Loader() + sys.meta_path.append(Finder()) + + def pytest_collect_file(file_path, parent): + if file_path.suffix == ".narf": + return Module.from_parent(path=file_path, parent=parent)""" + ) + pytester.makefile( + ".narf", + """\ + def test_something(): + assert 1 + 1 == 2""", + ) + # Use runpytest_subprocess, since we're futzing with sys.meta_path. + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_early_ignored_attributes(self, pytester: Pytester) -> None: + """Builtin attributes should be ignored early on, even if + configuration would otherwise allow them. + + This tests a performance optimization, not correctness, really, + although it tests PytestCollectionWarning is not raised, while + it would have been raised otherwise. + """ + pytester.makeini( + """ + [pytest] + python_classes=* + python_functions=* + """ + ) + pytester.makepyfile( + """ + class TestEmpty: + pass + test_empty = TestEmpty() + def test_real(): + pass + """ + ) + items, rec = pytester.inline_genitems() + assert rec.ret == 0 + assert len(items) == 1 + + +def test_setup_only_available_in_subdir(pytester: Pytester) -> None: + sub1 = pytester.mkpydir("sub1") + sub2 = pytester.mkpydir("sub2") + sub1.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + def pytest_runtest_setup(item): + assert item.path.stem == "test_in_sub1" + def pytest_runtest_call(item): + assert item.path.stem == "test_in_sub1" + def pytest_runtest_teardown(item): + assert item.path.stem == "test_in_sub1" + """ + ) + ) + sub2.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + def pytest_runtest_setup(item): + assert item.path.stem == "test_in_sub2" + def pytest_runtest_call(item): + assert item.path.stem == "test_in_sub2" + def pytest_runtest_teardown(item): + assert item.path.stem == "test_in_sub2" + """ + ) + ) + sub1.joinpath("test_in_sub1.py").write_text("def test_1(): pass") + sub2.joinpath("test_in_sub2.py").write_text("def test_2(): pass") + result = pytester.runpytest("-v", "-s") + result.assert_outcomes(passed=2) + + +def test_modulecol_roundtrip(pytester: Pytester) -> None: + modcol = pytester.getmodulecol("pass", withinit=False) + trail = modcol.nodeid + newcol = modcol.session.perform_collect([trail], genitems=0)[0] + assert modcol.name == newcol.name + + +class TestTracebackCutting: + def test_skip_simple(self): + with pytest.raises(pytest.skip.Exception) as excinfo: + pytest.skip("xxx") + assert excinfo.traceback[-1].frame.code.name == "skip" + assert excinfo.traceback[-1].ishidden() + assert excinfo.traceback[-2].frame.code.name == "test_skip_simple" + assert not excinfo.traceback[-2].ishidden() + + def test_traceback_argsetup(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + + @pytest.fixture + def hello(request): + raise ValueError("xyz") + """ + ) + p = pytester.makepyfile("def test(hello): pass") + result = pytester.runpytest(p) + assert result.ret != 0 + out = result.stdout.str() + assert "xyz" in out + assert "conftest.py:5: ValueError" in out + numentries = out.count("_ _ _") # separator for traceback entries + assert numentries == 0 + + result = pytester.runpytest("--fulltrace", p) + out = result.stdout.str() + assert "conftest.py:5: ValueError" in out + numentries = out.count("_ _ _ _") # separator for traceback entries + assert numentries > 3 + + def test_traceback_error_during_import(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + x = 1 + x = 2 + x = 17 + asd + """ + ) + result = pytester.runpytest() + assert result.ret != 0 + out = result.stdout.str() + assert "x = 1" not in out + assert "x = 2" not in out + result.stdout.fnmatch_lines([" *asd*", "E*NameError*"]) + result = pytester.runpytest("--fulltrace") + out = result.stdout.str() + assert "x = 1" in out + assert "x = 2" in out + result.stdout.fnmatch_lines([">*asd*", "E*NameError*"]) + + def test_traceback_filter_error_during_fixture_collection( + self, pytester: Pytester + ) -> None: + """Integration test for issue #995.""" + pytester.makepyfile( + """ + import pytest + + def fail_me(func): + ns = {} + exec('def w(): raise ValueError("fail me")', ns) + return ns['w'] + + @pytest.fixture(scope='class') + @fail_me + def fail_fixture(): + pass + + def test_failing_fixture(fail_fixture): + pass + """ + ) + result = pytester.runpytest() + assert result.ret != 0 + out = result.stdout.str() + assert "INTERNALERROR>" not in out + result.stdout.fnmatch_lines(["*ValueError: fail me*", "* 1 error in *"]) + + def test_filter_traceback_generated_code(self) -> None: + """Test that filter_traceback() works with the fact that + _pytest._code.code.Code.path attribute might return an str object. + + In this case, one of the entries on the traceback was produced by + dynamically generated code. + See: https://bitbucket.org/pytest-dev/py/issues/71 + This fixes #995. + """ + from _pytest._code import filter_traceback + + tb = None + try: + ns: Dict[str, Any] = {} + exec("def foo(): raise ValueError", ns) + ns["foo"]() + except ValueError: + _, _, tb = sys.exc_info() + + assert tb is not None + traceback = _pytest._code.Traceback(tb) + assert isinstance(traceback[-1].path, str) + assert not filter_traceback(traceback[-1]) + + def test_filter_traceback_path_no_longer_valid(self, pytester: Pytester) -> None: + """Test that filter_traceback() works with the fact that + _pytest._code.code.Code.path attribute might return an str object. + + In this case, one of the files in the traceback no longer exists. + This fixes #1133. + """ + from _pytest._code import filter_traceback + + pytester.syspathinsert() + pytester.makepyfile( + filter_traceback_entry_as_str=""" + def foo(): + raise ValueError + """ + ) + tb = None + try: + import filter_traceback_entry_as_str + + filter_traceback_entry_as_str.foo() + except ValueError: + _, _, tb = sys.exc_info() + + assert tb is not None + pytester.path.joinpath("filter_traceback_entry_as_str.py").unlink() + traceback = _pytest._code.Traceback(tb) + assert isinstance(traceback[-1].path, str) + assert filter_traceback(traceback[-1]) + + +class TestReportInfo: + def test_itemreport_reportinfo(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + class MyFunction(pytest.Function): + def reportinfo(self): + return "ABCDE", 42, "custom" + def pytest_pycollect_makeitem(collector, name, obj): + if name == "test_func": + return MyFunction.from_parent(name=name, parent=collector) + """ + ) + item = pytester.getitem("def test_func(): pass") + item.config.pluginmanager.getplugin("runner") + assert item.location == ("ABCDE", 42, "custom") + + def test_func_reportinfo(self, pytester: Pytester) -> None: + item = pytester.getitem("def test_func(): pass") + path, lineno, modpath = item.reportinfo() + assert os.fspath(path) == str(item.path) + assert lineno == 0 + assert modpath == "test_func" + + def test_class_reportinfo(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol( + """ + # lineno 0 + class TestClass(object): + def test_hello(self): pass + """ + ) + classcol = pytester.collect_by_name(modcol, "TestClass") + assert isinstance(classcol, Class) + path, lineno, msg = classcol.reportinfo() + assert os.fspath(path) == str(modcol.path) + assert lineno == 1 + assert msg == "TestClass" + + @pytest.mark.filterwarnings( + "ignore:usage of Generator.Function is deprecated, please use pytest.Function instead" + ) + def test_reportinfo_with_nasty_getattr(self, pytester: Pytester) -> None: + # https://github.com/pytest-dev/pytest/issues/1204 + modcol = pytester.getmodulecol( + """ + # lineno 0 + class TestClass: + def __getattr__(self, name): + return "this is not an int" + + def __class_getattr__(cls, name): + return "this is not an int" + + def intest_foo(self): + pass + + def test_bar(self): + pass + """ + ) + classcol = pytester.collect_by_name(modcol, "TestClass") + assert isinstance(classcol, Class) + path, lineno, msg = classcol.reportinfo() + func = list(classcol.collect())[0] + assert isinstance(func, Function) + path, lineno, msg = func.reportinfo() + + +def test_customized_python_discovery(pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + python_files=check_*.py + python_classes=Check + python_functions=check + """ + ) + p = pytester.makepyfile( + """ + def check_simple(): + pass + class CheckMyApp(object): + def check_meth(self): + pass + """ + ) + p2 = p.with_name(p.name.replace("test", "check")) + p.rename(p2) + result = pytester.runpytest("--collect-only", "-s") + result.stdout.fnmatch_lines( + ["*check_customized*", "*check_simple*", "*CheckMyApp*", "*check_meth*"] + ) + + result = pytester.runpytest() + assert result.ret == 0 + result.stdout.fnmatch_lines(["*2 passed*"]) + + +def test_customized_python_discovery_functions(pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + python_functions=_test + """ + ) + pytester.makepyfile( + """ + def _test_underscore(): + pass + """ + ) + result = pytester.runpytest("--collect-only", "-s") + result.stdout.fnmatch_lines(["*_test_underscore*"]) + + result = pytester.runpytest() + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_unorderable_types(pytester: Pytester) -> None: + pytester.makepyfile( + """ + class TestJoinEmpty(object): + pass + + def make_test(): + class Test(object): + pass + Test.__name__ = "TestFoo" + return Test + TestFoo = make_test() + """ + ) + result = pytester.runpytest() + result.stdout.no_fnmatch_line("*TypeError*") + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +@pytest.mark.filterwarnings("default::pytest.PytestCollectionWarning") +def test_dont_collect_non_function_callable(pytester: Pytester) -> None: + """Test for issue https://github.com/pytest-dev/pytest/issues/331 + + In this case an INTERNALERROR occurred trying to report the failure of + a test like this one because pytest failed to get the source lines. + """ + pytester.makepyfile( + """ + class Oh(object): + def __call__(self): + pass + + test_a = Oh() + + def test_real(): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*collected 1 item*", + "*test_dont_collect_non_function_callable.py:2: *cannot collect 'test_a' because it is not a function*", + "*1 passed, 1 warning in *", + ] + ) + + +def test_class_injection_does_not_break_collection(pytester: Pytester) -> None: + """Tests whether injection during collection time will terminate testing. + + In this case the error should not occur if the TestClass itself + is modified during collection time, and the original method list + is still used for collection. + """ + pytester.makeconftest( + """ + from test_inject import TestClass + def pytest_generate_tests(metafunc): + TestClass.changed_var = {} + """ + ) + pytester.makepyfile( + test_inject=''' + class TestClass(object): + def test_injection(self): + """Test being parametrized.""" + pass + ''' + ) + result = pytester.runpytest() + assert ( + "RuntimeError: dictionary changed size during iteration" + not in result.stdout.str() + ) + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_syntax_error_with_non_ascii_chars(pytester: Pytester) -> None: + """Fix decoding issue while formatting SyntaxErrors during collection (#578).""" + pytester.makepyfile("☃") + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*ERROR collecting*", "*SyntaxError*", "*1 error in*"]) + + +def test_collect_error_with_fulltrace(pytester: Pytester) -> None: + pytester.makepyfile("assert 0") + result = pytester.runpytest("--fulltrace") + result.stdout.fnmatch_lines( + [ + "collected 0 items / 1 error", + "", + "*= ERRORS =*", + "*_ ERROR collecting test_collect_error_with_fulltrace.py _*", + "", + "> assert 0", + "E assert 0", + "", + "test_collect_error_with_fulltrace.py:1: AssertionError", + "*! Interrupted: 1 error during collection !*", + ] + ) + + +def test_skip_duplicates_by_default(pytester: Pytester) -> None: + """Test for issue https://github.com/pytest-dev/pytest/issues/1609 (#1609) + + Ignore duplicate directories. + """ + a = pytester.mkdir("a") + fh = a.joinpath("test_a.py") + fh.write_text( + textwrap.dedent( + """\ + import pytest + def test_real(): + pass + """ + ) + ) + result = pytester.runpytest(str(a), str(a)) + result.stdout.fnmatch_lines(["*collected 1 item*"]) + + +def test_keep_duplicates(pytester: Pytester) -> None: + """Test for issue https://github.com/pytest-dev/pytest/issues/1609 (#1609) + + Use --keep-duplicates to collect tests from duplicate directories. + """ + a = pytester.mkdir("a") + fh = a.joinpath("test_a.py") + fh.write_text( + textwrap.dedent( + """\ + import pytest + def test_real(): + pass + """ + ) + ) + result = pytester.runpytest("--keep-duplicates", str(a), str(a)) + result.stdout.fnmatch_lines(["*collected 2 item*"]) + + +def test_package_collection_infinite_recursion(pytester: Pytester) -> None: + pytester.copy_example("collect/package_infinite_recursion") + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_package_collection_init_given_as_argument(pytester: Pytester) -> None: + """Regression test for #3749""" + p = pytester.copy_example("collect/package_init_given_as_arg") + result = pytester.runpytest(p / "pkg" / "__init__.py") + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_package_with_modules(pytester: Pytester) -> None: + """ + . + └── root + ├── __init__.py + ├── sub1 + │ ├── __init__.py + │ └── sub1_1 + │ ├── __init__.py + │ └── test_in_sub1.py + └── sub2 + └── test + └── test_in_sub2.py + + """ + root = pytester.mkpydir("root") + sub1 = root.joinpath("sub1") + sub1_test = sub1.joinpath("sub1_1") + sub1_test.mkdir(parents=True) + for d in (sub1, sub1_test): + d.joinpath("__init__.py").touch() + + sub2 = root.joinpath("sub2") + sub2_test = sub2.joinpath("test") + sub2_test.mkdir(parents=True) + + sub1_test.joinpath("test_in_sub1.py").write_text("def test_1(): pass") + sub2_test.joinpath("test_in_sub2.py").write_text("def test_2(): pass") + + # Execute from . + result = pytester.runpytest("-v", "-s") + result.assert_outcomes(passed=2) + + # Execute from . with one argument "root" + result = pytester.runpytest("-v", "-s", "root") + result.assert_outcomes(passed=2) + + # Chdir into package's root and execute with no args + os.chdir(root) + result = pytester.runpytest("-v", "-s") + result.assert_outcomes(passed=2) + + +def test_package_ordering(pytester: Pytester) -> None: + """ + . + └── root + ├── Test_root.py + ├── __init__.py + ├── sub1 + │ ├── Test_sub1.py + │ └── __init__.py + └── sub2 + └── test + └── test_sub2.py + + """ + pytester.makeini( + """ + [pytest] + python_files=*.py + """ + ) + root = pytester.mkpydir("root") + sub1 = root.joinpath("sub1") + sub1.mkdir() + sub1.joinpath("__init__.py").touch() + sub2 = root.joinpath("sub2") + sub2_test = sub2.joinpath("test") + sub2_test.mkdir(parents=True) + + root.joinpath("Test_root.py").write_text("def test_1(): pass") + sub1.joinpath("Test_sub1.py").write_text("def test_2(): pass") + sub2_test.joinpath("test_sub2.py").write_text("def test_3(): pass") + + # Execute from . + result = pytester.runpytest("-v", "-s") + result.assert_outcomes(passed=3) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/python/fixtures.py b/testing/web-platform/tests/tools/third_party/pytest/testing/python/fixtures.py new file mode 100644 index 0000000000..f29ca1dfa5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/python/fixtures.py @@ -0,0 +1,4474 @@ +import os +import sys +import textwrap +from pathlib import Path + +import pytest +from _pytest import fixtures +from _pytest.compat import getfuncargnames +from _pytest.config import ExitCode +from _pytest.fixtures import FixtureRequest +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import get_public_names +from _pytest.pytester import Pytester +from _pytest.python import Function + + +def test_getfuncargnames_functions(): + """Test getfuncargnames for normal functions""" + + def f(): + raise NotImplementedError() + + assert not getfuncargnames(f) + + def g(arg): + raise NotImplementedError() + + assert getfuncargnames(g) == ("arg",) + + def h(arg1, arg2="hello"): + raise NotImplementedError() + + assert getfuncargnames(h) == ("arg1",) + + def j(arg1, arg2, arg3="hello"): + raise NotImplementedError() + + assert getfuncargnames(j) == ("arg1", "arg2") + + +def test_getfuncargnames_methods(): + """Test getfuncargnames for normal methods""" + + class A: + def f(self, arg1, arg2="hello"): + raise NotImplementedError() + + assert getfuncargnames(A().f) == ("arg1",) + + +def test_getfuncargnames_staticmethod(): + """Test getfuncargnames for staticmethods""" + + class A: + @staticmethod + def static(arg1, arg2, x=1): + raise NotImplementedError() + + assert getfuncargnames(A.static, cls=A) == ("arg1", "arg2") + + +def test_getfuncargnames_staticmethod_inherited() -> None: + """Test getfuncargnames for inherited staticmethods (#8061)""" + + class A: + @staticmethod + def static(arg1, arg2, x=1): + raise NotImplementedError() + + class B(A): + pass + + assert getfuncargnames(B.static, cls=B) == ("arg1", "arg2") + + +def test_getfuncargnames_partial(): + """Check getfuncargnames for methods defined with functools.partial (#5701)""" + import functools + + def check(arg1, arg2, i): + raise NotImplementedError() + + class T: + test_ok = functools.partial(check, i=2) + + values = getfuncargnames(T().test_ok, name="test_ok") + assert values == ("arg1", "arg2") + + +def test_getfuncargnames_staticmethod_partial(): + """Check getfuncargnames for staticmethods defined with functools.partial (#5701)""" + import functools + + def check(arg1, arg2, i): + raise NotImplementedError() + + class T: + test_ok = staticmethod(functools.partial(check, i=2)) + + values = getfuncargnames(T().test_ok, name="test_ok") + assert values == ("arg1", "arg2") + + +@pytest.mark.pytester_example_path("fixtures/fill_fixtures") +class TestFillFixtures: + def test_fillfuncargs_exposed(self): + # used by oejskit, kept for compatibility + assert pytest._fillfuncargs == fixtures._fillfuncargs + + def test_funcarg_lookupfails(self, pytester: Pytester) -> None: + pytester.copy_example() + result = pytester.runpytest() # "--collect-only") + assert result.ret != 0 + result.stdout.fnmatch_lines( + """ + *def test_func(some)* + *fixture*some*not found* + *xyzsomething* + """ + ) + + def test_detect_recursive_dependency_error(self, pytester: Pytester) -> None: + pytester.copy_example() + result = pytester.runpytest() + result.stdout.fnmatch_lines( + ["*recursive dependency involving fixture 'fix1' detected*"] + ) + + def test_funcarg_basic(self, pytester: Pytester) -> None: + pytester.copy_example() + item = pytester.getitem(Path("test_funcarg_basic.py")) + assert isinstance(item, Function) + # Execute's item's setup, which fills fixtures. + item.session._setupstate.setup(item) + del item.funcargs["request"] + assert len(get_public_names(item.funcargs)) == 2 + assert item.funcargs["some"] == "test_func" + assert item.funcargs["other"] == 42 + + def test_funcarg_lookup_modulelevel(self, pytester: Pytester) -> None: + pytester.copy_example() + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + def test_funcarg_lookup_classlevel(self, pytester: Pytester) -> None: + p = pytester.copy_example() + result = pytester.runpytest(p) + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_conftest_funcargs_only_available_in_subdir( + self, pytester: Pytester + ) -> None: + pytester.copy_example() + result = pytester.runpytest("-v") + result.assert_outcomes(passed=2) + + def test_extend_fixture_module_class(self, pytester: Pytester) -> None: + testfile = pytester.copy_example() + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + result = pytester.runpytest(testfile) + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_extend_fixture_conftest_module(self, pytester: Pytester) -> None: + p = pytester.copy_example() + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + result = pytester.runpytest(str(next(Path(str(p)).rglob("test_*.py")))) + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_extend_fixture_conftest_conftest(self, pytester: Pytester) -> None: + p = pytester.copy_example() + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + result = pytester.runpytest(str(next(Path(str(p)).rglob("test_*.py")))) + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_extend_fixture_conftest_plugin(self, pytester: Pytester) -> None: + pytester.makepyfile( + testplugin=""" + import pytest + + @pytest.fixture + def foo(): + return 7 + """ + ) + pytester.syspathinsert() + pytester.makeconftest( + """ + import pytest + + pytest_plugins = 'testplugin' + + @pytest.fixture + def foo(foo): + return foo + 7 + """ + ) + pytester.makepyfile( + """ + def test_foo(foo): + assert foo == 14 + """ + ) + result = pytester.runpytest("-s") + assert result.ret == 0 + + def test_extend_fixture_plugin_plugin(self, pytester: Pytester) -> None: + # Two plugins should extend each order in loading order + pytester.makepyfile( + testplugin0=""" + import pytest + + @pytest.fixture + def foo(): + return 7 + """ + ) + pytester.makepyfile( + testplugin1=""" + import pytest + + @pytest.fixture + def foo(foo): + return foo + 7 + """ + ) + pytester.syspathinsert() + pytester.makepyfile( + """ + pytest_plugins = ['testplugin0', 'testplugin1'] + + def test_foo(foo): + assert foo == 14 + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + + def test_override_parametrized_fixture_conftest_module( + self, pytester: Pytester + ) -> None: + """Test override of the parametrized fixture with non-parametrized one on the test module level.""" + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(params=[1, 2, 3]) + def spam(request): + return request.param + """ + ) + testfile = pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def spam(): + return 'spam' + + def test_spam(spam): + assert spam == 'spam' + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + result = pytester.runpytest(testfile) + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_override_parametrized_fixture_conftest_conftest( + self, pytester: Pytester + ) -> None: + """Test override of the parametrized fixture with non-parametrized one on the conftest level.""" + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(params=[1, 2, 3]) + def spam(request): + return request.param + """ + ) + subdir = pytester.mkpydir("subdir") + subdir.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + + @pytest.fixture + def spam(): + return 'spam' + """ + ) + ) + testfile = subdir.joinpath("test_spam.py") + testfile.write_text( + textwrap.dedent( + """\ + def test_spam(spam): + assert spam == "spam" + """ + ) + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + result = pytester.runpytest(testfile) + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_override_non_parametrized_fixture_conftest_module( + self, pytester: Pytester + ) -> None: + """Test override of the non-parametrized fixture with parametrized one on the test module level.""" + pytester.makeconftest( + """ + import pytest + + @pytest.fixture + def spam(): + return 'spam' + """ + ) + testfile = pytester.makepyfile( + """ + import pytest + + @pytest.fixture(params=[1, 2, 3]) + def spam(request): + return request.param + + params = {'spam': 1} + + def test_spam(spam): + assert spam == params['spam'] + params['spam'] += 1 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*3 passed*"]) + result = pytester.runpytest(testfile) + result.stdout.fnmatch_lines(["*3 passed*"]) + + def test_override_non_parametrized_fixture_conftest_conftest( + self, pytester: Pytester + ) -> None: + """Test override of the non-parametrized fixture with parametrized one on the conftest level.""" + pytester.makeconftest( + """ + import pytest + + @pytest.fixture + def spam(): + return 'spam' + """ + ) + subdir = pytester.mkpydir("subdir") + subdir.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + + @pytest.fixture(params=[1, 2, 3]) + def spam(request): + return request.param + """ + ) + ) + testfile = subdir.joinpath("test_spam.py") + testfile.write_text( + textwrap.dedent( + """\ + params = {'spam': 1} + + def test_spam(spam): + assert spam == params['spam'] + params['spam'] += 1 + """ + ) + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*3 passed*"]) + result = pytester.runpytest(testfile) + result.stdout.fnmatch_lines(["*3 passed*"]) + + def test_override_autouse_fixture_with_parametrized_fixture_conftest_conftest( + self, pytester: Pytester + ) -> None: + """Test override of the autouse fixture with parametrized one on the conftest level. + This test covers the issue explained in issue 1601 + """ + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(autouse=True) + def spam(): + return 'spam' + """ + ) + subdir = pytester.mkpydir("subdir") + subdir.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + + @pytest.fixture(params=[1, 2, 3]) + def spam(request): + return request.param + """ + ) + ) + testfile = subdir.joinpath("test_spam.py") + testfile.write_text( + textwrap.dedent( + """\ + params = {'spam': 1} + + def test_spam(spam): + assert spam == params['spam'] + params['spam'] += 1 + """ + ) + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*3 passed*"]) + result = pytester.runpytest(testfile) + result.stdout.fnmatch_lines(["*3 passed*"]) + + def test_override_fixture_reusing_super_fixture_parametrization( + self, pytester: Pytester + ) -> None: + """Override a fixture at a lower level, reusing the higher-level fixture that + is parametrized (#1953). + """ + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(params=[1, 2]) + def foo(request): + return request.param + """ + ) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def foo(foo): + return foo * 2 + + def test_spam(foo): + assert foo in (2, 4) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*2 passed*"]) + + def test_override_parametrize_fixture_and_indirect( + self, pytester: Pytester + ) -> None: + """Override a fixture at a lower level, reusing the higher-level fixture that + is parametrized, while also using indirect parametrization. + """ + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(params=[1, 2]) + def foo(request): + return request.param + """ + ) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def foo(foo): + return foo * 2 + + @pytest.fixture + def bar(request): + return request.param * 100 + + @pytest.mark.parametrize("bar", [42], indirect=True) + def test_spam(bar, foo): + assert bar == 4200 + assert foo in (2, 4) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*2 passed*"]) + + def test_override_top_level_fixture_reusing_super_fixture_parametrization( + self, pytester: Pytester + ) -> None: + """Same as the above test, but with another level of overwriting.""" + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(params=['unused', 'unused']) + def foo(request): + return request.param + """ + ) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(params=[1, 2]) + def foo(request): + return request.param + + class Test: + + @pytest.fixture + def foo(self, foo): + return foo * 2 + + def test_spam(self, foo): + assert foo in (2, 4) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*2 passed*"]) + + def test_override_parametrized_fixture_with_new_parametrized_fixture( + self, pytester: Pytester + ) -> None: + """Overriding a parametrized fixture, while also parametrizing the new fixture and + simultaneously requesting the overwritten fixture as parameter, yields the same value + as ``request.param``. + """ + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(params=['ignored', 'ignored']) + def foo(request): + return request.param + """ + ) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(params=[10, 20]) + def foo(foo, request): + assert request.param == foo + return foo * 2 + + def test_spam(foo): + assert foo in (20, 40) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*2 passed*"]) + + def test_autouse_fixture_plugin(self, pytester: Pytester) -> None: + # A fixture from a plugin has no baseid set, which screwed up + # the autouse fixture handling. + pytester.makepyfile( + testplugin=""" + import pytest + + @pytest.fixture(autouse=True) + def foo(request): + request.function.foo = 7 + """ + ) + pytester.syspathinsert() + pytester.makepyfile( + """ + pytest_plugins = 'testplugin' + + def test_foo(request): + assert request.function.foo == 7 + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + + def test_funcarg_lookup_error(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + + @pytest.fixture + def a_fixture(): pass + + @pytest.fixture + def b_fixture(): pass + + @pytest.fixture + def c_fixture(): pass + + @pytest.fixture + def d_fixture(): pass + """ + ) + pytester.makepyfile( + """ + def test_lookup_error(unknown): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*ERROR at setup of test_lookup_error*", + " def test_lookup_error(unknown):*", + "E fixture 'unknown' not found", + "> available fixtures:*a_fixture,*b_fixture,*c_fixture,*d_fixture*monkeypatch,*", + # sorted + "> use 'py*test --fixtures *' for help on them.", + "*1 error*", + ] + ) + result.stdout.no_fnmatch_line("*INTERNAL*") + + def test_fixture_excinfo_leak(self, pytester: Pytester) -> None: + # on python2 sys.excinfo would leak into fixture executions + pytester.makepyfile( + """ + import sys + import traceback + import pytest + + @pytest.fixture + def leak(): + if sys.exc_info()[0]: # python3 bug :) + traceback.print_exc() + #fails + assert sys.exc_info() == (None, None, None) + + def test_leak(leak): + if sys.exc_info()[0]: # python3 bug :) + traceback.print_exc() + assert sys.exc_info() == (None, None, None) + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + + +class TestRequestBasic: + def test_request_attributes(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + + @pytest.fixture + def something(request): pass + def test_func(something): pass + """ + ) + assert isinstance(item, Function) + req = fixtures.FixtureRequest(item, _ispytest=True) + assert req.function == item.obj + assert req.keywords == item.keywords + assert hasattr(req.module, "test_func") + assert req.cls is None + assert req.function.__name__ == "test_func" + assert req.config == item.config + assert repr(req).find(req.function.__name__) != -1 + + def test_request_attributes_method(self, pytester: Pytester) -> None: + (item,) = pytester.getitems( + """ + import pytest + class TestB(object): + + @pytest.fixture + def something(self, request): + return 1 + def test_func(self, something): + pass + """ + ) + assert isinstance(item, Function) + req = item._request + assert req.cls.__name__ == "TestB" + assert req.instance.__class__ == req.cls + + def test_request_contains_funcarg_arg2fixturedefs(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol( + """ + import pytest + @pytest.fixture + def something(request): + pass + class TestClass(object): + def test_method(self, something): + pass + """ + ) + (item1,) = pytester.genitems([modcol]) + assert item1.name == "test_method" + arg2fixturedefs = fixtures.FixtureRequest( + item1, _ispytest=True + )._arg2fixturedefs + assert len(arg2fixturedefs) == 1 + assert arg2fixturedefs["something"][0].argname == "something" + + @pytest.mark.skipif( + hasattr(sys, "pypy_version_info"), + reason="this method of test doesn't work on pypy", + ) + def test_request_garbage(self, pytester: Pytester) -> None: + try: + import xdist # noqa + except ImportError: + pass + else: + pytest.xfail("this test is flaky when executed with xdist") + pytester.makepyfile( + """ + import sys + import pytest + from _pytest.fixtures import PseudoFixtureDef + import gc + + @pytest.fixture(autouse=True) + def something(request): + original = gc.get_debug() + gc.set_debug(gc.DEBUG_SAVEALL) + gc.collect() + + yield + + try: + gc.collect() + leaked = [x for _ in gc.garbage if isinstance(_, PseudoFixtureDef)] + assert leaked == [] + finally: + gc.set_debug(original) + + def test_func(): + pass + """ + ) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines(["* 1 passed in *"]) + + def test_getfixturevalue_recursive(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + + @pytest.fixture + def something(request): + return 1 + """ + ) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def something(request): + return request.getfixturevalue("something") + 1 + def test_func(something): + assert something == 2 + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_getfixturevalue_teardown(self, pytester: Pytester) -> None: + """ + Issue #1895 + + `test_inner` requests `inner` fixture, which in turn requests `resource` + using `getfixturevalue`. `test_func` then requests `resource`. + + `resource` is teardown before `inner` because the fixture mechanism won't consider + `inner` dependent on `resource` when it is used via `getfixturevalue`: `test_func` + will then cause the `resource`'s finalizer to be called first because of this. + """ + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope='session') + def resource(): + r = ['value'] + yield r + r.pop() + + @pytest.fixture(scope='session') + def inner(request): + resource = request.getfixturevalue('resource') + assert resource == ['value'] + yield + assert resource == ['value'] + + def test_inner(inner): + pass + + def test_func(resource): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 2 passed in *"]) + + def test_getfixturevalue(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + + @pytest.fixture + def something(request): + return 1 + + values = [2] + @pytest.fixture + def other(request): + return values.pop() + + def test_func(something): pass + """ + ) + assert isinstance(item, Function) + req = item._request + + # Execute item's setup. + item.session._setupstate.setup(item) + + with pytest.raises(pytest.FixtureLookupError): + req.getfixturevalue("notexists") + val = req.getfixturevalue("something") + assert val == 1 + val = req.getfixturevalue("something") + assert val == 1 + val2 = req.getfixturevalue("other") + assert val2 == 2 + val2 = req.getfixturevalue("other") # see about caching + assert val2 == 2 + assert item.funcargs["something"] == 1 + assert len(get_public_names(item.funcargs)) == 2 + assert "request" in item.funcargs + + def test_request_addfinalizer(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + teardownlist = [] + @pytest.fixture + def something(request): + request.addfinalizer(lambda: teardownlist.append(1)) + def test_func(something): pass + """ + ) + assert isinstance(item, Function) + item.session._setupstate.setup(item) + item._request._fillfixtures() + # successively check finalization calls + parent = item.getparent(pytest.Module) + assert parent is not None + teardownlist = parent.obj.teardownlist + ss = item.session._setupstate + assert not teardownlist + ss.teardown_exact(None) + print(ss.stack) + assert teardownlist == [1] + + def test_request_addfinalizer_failing_setup(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [1] + @pytest.fixture + def myfix(request): + request.addfinalizer(values.pop) + assert 0 + def test_fix(myfix): + pass + def test_finalizer_ran(): + assert not values + """ + ) + reprec = pytester.inline_run("-s") + reprec.assertoutcome(failed=1, passed=1) + + def test_request_addfinalizer_failing_setup_module( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """ + import pytest + values = [1, 2] + @pytest.fixture(scope="module") + def myfix(request): + request.addfinalizer(values.pop) + request.addfinalizer(values.pop) + assert 0 + def test_fix(myfix): + pass + """ + ) + reprec = pytester.inline_run("-s") + mod = reprec.getcalls("pytest_runtest_setup")[0].item.module + assert not mod.values + + def test_request_addfinalizer_partial_setup_failure( + self, pytester: Pytester + ) -> None: + p = pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture + def something(request): + request.addfinalizer(lambda: values.append(None)) + def test_func(something, missingarg): + pass + def test_second(): + assert len(values) == 1 + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + ["*1 error*"] # XXX the whole module collection fails + ) + + def test_request_subrequest_addfinalizer_exceptions( + self, pytester: Pytester + ) -> None: + """ + Ensure exceptions raised during teardown by a finalizer are suppressed + until all finalizers are called, re-raising the first exception (#2440) + """ + pytester.makepyfile( + """ + import pytest + values = [] + def _excepts(where): + raise Exception('Error in %s fixture' % where) + @pytest.fixture + def subrequest(request): + return request + @pytest.fixture + def something(subrequest): + subrequest.addfinalizer(lambda: values.append(1)) + subrequest.addfinalizer(lambda: values.append(2)) + subrequest.addfinalizer(lambda: _excepts('something')) + @pytest.fixture + def excepts(subrequest): + subrequest.addfinalizer(lambda: _excepts('excepts')) + subrequest.addfinalizer(lambda: values.append(3)) + def test_first(something, excepts): + pass + def test_second(): + assert values == [3, 2, 1] + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + ["*Exception: Error in excepts fixture", "* 2 passed, 1 error in *"] + ) + + def test_request_getmodulepath(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol("def test_somefunc(): pass") + (item,) = pytester.genitems([modcol]) + req = fixtures.FixtureRequest(item, _ispytest=True) + assert req.path == modcol.path + + def test_request_fixturenames(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + from _pytest.pytester import get_public_names + @pytest.fixture() + def arg1(): + pass + @pytest.fixture() + def farg(arg1): + pass + @pytest.fixture(autouse=True) + def sarg(tmp_path): + pass + def test_function(request, farg): + assert set(get_public_names(request.fixturenames)) == \ + set(["sarg", "arg1", "request", "farg", + "tmp_path", "tmp_path_factory"]) + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_request_fixturenames_dynamic_fixture(self, pytester: Pytester) -> None: + """Regression test for #3057""" + pytester.copy_example("fixtures/test_getfixturevalue_dynamic.py") + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_setupdecorator_and_xunit(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture(scope='module', autouse=True) + def setup_module(): + values.append("module") + @pytest.fixture(autouse=True) + def setup_function(): + values.append("function") + + def test_func(): + pass + + class TestClass(object): + @pytest.fixture(scope="class", autouse=True) + def setup_class(self): + values.append("class") + @pytest.fixture(autouse=True) + def setup_method(self): + values.append("method") + def test_method(self): + pass + def test_all(): + assert values == ["module", "function", "class", + "function", "method", "function"] + """ + ) + reprec = pytester.inline_run("-v") + reprec.assertoutcome(passed=3) + + def test_fixtures_sub_subdir_normalize_sep(self, pytester: Pytester) -> None: + # this tests that normalization of nodeids takes place + b = pytester.path.joinpath("tests", "unit") + b.mkdir(parents=True) + b.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + @pytest.fixture + def arg1(): + pass + """ + ) + ) + p = b.joinpath("test_module.py") + p.write_text("def test_func(arg1): pass") + result = pytester.runpytest(p, "--fixtures") + assert result.ret == 0 + result.stdout.fnmatch_lines( + """ + *fixtures defined*conftest* + *arg1* + """ + ) + + def test_show_fixtures_color_yes(self, pytester: Pytester) -> None: + pytester.makepyfile("def test_this(): assert 1") + result = pytester.runpytest("--color=yes", "--fixtures") + assert "\x1b[32mtmp_path" in result.stdout.str() + + def test_newstyle_with_request(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture() + def arg(request): + pass + def test_1(arg): + pass + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_setupcontext_no_param(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(params=[1,2]) + def arg(request): + return request.param + + @pytest.fixture(autouse=True) + def mysetup(request, arg): + assert not hasattr(request, "param") + def test_1(arg): + assert arg in (1,2) + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + +class TestRequestSessionScoped: + @pytest.fixture(scope="session") + def session_request(self, request): + return request + + @pytest.mark.parametrize("name", ["path", "module"]) + def test_session_scoped_unavailable_attributes(self, session_request, name): + with pytest.raises( + AttributeError, + match=f"{name} not available in session-scoped context", + ): + getattr(session_request, name) + + +class TestRequestMarking: + def test_applymarker(self, pytester: Pytester) -> None: + item1, item2 = pytester.getitems( + """ + import pytest + + @pytest.fixture + def something(request): + pass + class TestClass(object): + def test_func1(self, something): + pass + def test_func2(self, something): + pass + """ + ) + req1 = fixtures.FixtureRequest(item1, _ispytest=True) + assert "xfail" not in item1.keywords + req1.applymarker(pytest.mark.xfail) + assert "xfail" in item1.keywords + assert "skipif" not in item1.keywords + req1.applymarker(pytest.mark.skipif) + assert "skipif" in item1.keywords + with pytest.raises(ValueError): + req1.applymarker(42) # type: ignore[arg-type] + + def test_accesskeywords(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture() + def keywords(request): + return request.keywords + @pytest.mark.XYZ + def test_function(keywords): + assert keywords["XYZ"] + assert "abc" not in keywords + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_accessmarker_dynamic(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + @pytest.fixture() + def keywords(request): + return request.keywords + + @pytest.fixture(scope="class", autouse=True) + def marking(request): + request.applymarker(pytest.mark.XYZ("hello")) + """ + ) + pytester.makepyfile( + """ + import pytest + def test_fun1(keywords): + assert keywords["XYZ"] is not None + assert "abc" not in keywords + def test_fun2(keywords): + assert keywords["XYZ"] is not None + assert "abc" not in keywords + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + +class TestFixtureUsages: + def test_noargfixturedec(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture + def arg1(): + return 1 + + def test_func(arg1): + assert arg1 == 1 + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_receives_funcargs(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture() + def arg1(): + return 1 + + @pytest.fixture() + def arg2(arg1): + return arg1 + 1 + + def test_add(arg2): + assert arg2 == 2 + def test_all(arg1, arg2): + assert arg1 == 1 + assert arg2 == 2 + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + def test_receives_funcargs_scope_mismatch(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope="function") + def arg1(): + return 1 + + @pytest.fixture(scope="module") + def arg2(arg1): + return arg1 + 1 + + def test_add(arg2): + assert arg2 == 2 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*ScopeMismatch*involved factories*", + "test_receives_funcargs_scope_mismatch.py:6: def arg2(arg1)", + "test_receives_funcargs_scope_mismatch.py:2: def arg1()", + "*1 error*", + ] + ) + + def test_receives_funcargs_scope_mismatch_issue660( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope="function") + def arg1(): + return 1 + + @pytest.fixture(scope="module") + def arg2(arg1): + return arg1 + 1 + + def test_add(arg1, arg2): + assert arg2 == 2 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + ["*ScopeMismatch*involved factories*", "* def arg2*", "*1 error*"] + ) + + def test_invalid_scope(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope="functions") + def badscope(): + pass + + def test_nothing(badscope): + pass + """ + ) + result = pytester.runpytest_inprocess() + result.stdout.fnmatch_lines( + "*Fixture 'badscope' from test_invalid_scope.py got an unexpected scope value 'functions'" + ) + + @pytest.mark.parametrize("scope", ["function", "session"]) + def test_parameters_without_eq_semantics(self, scope, pytester: Pytester) -> None: + pytester.makepyfile( + """ + class NoEq1: # fails on `a == b` statement + def __eq__(self, _): + raise RuntimeError + + class NoEq2: # fails on `if a == b:` statement + def __eq__(self, _): + class NoBool: + def __bool__(self): + raise RuntimeError + return NoBool() + + import pytest + @pytest.fixture(params=[NoEq1(), NoEq2()], scope={scope!r}) + def no_eq(request): + return request.param + + def test1(no_eq): + pass + + def test2(no_eq): + pass + """.format( + scope=scope + ) + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*4 passed*"]) + + def test_funcarg_parametrized_and_used_twice(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture(params=[1,2]) + def arg1(request): + values.append(1) + return request.param + + @pytest.fixture() + def arg2(arg1): + return arg1 + 1 + + def test_add(arg1, arg2): + assert arg2 == arg1 + 1 + assert len(values) == arg1 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*2 passed*"]) + + def test_factory_uses_unknown_funcarg_as_dependency_error( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture() + def fail(missing): + return + + @pytest.fixture() + def call_fail(fail): + return + + def test_missing(call_fail): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + """ + *pytest.fixture()* + *def call_fail(fail)* + *pytest.fixture()* + *def fail* + *fixture*'missing'*not found* + """ + ) + + def test_factory_setup_as_classes_fails(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + class arg1(object): + def __init__(self, request): + self.x = 1 + arg1 = pytest.fixture()(arg1) + + """ + ) + reprec = pytester.inline_run() + values = reprec.getfailedcollections() + assert len(values) == 1 + + def test_usefixtures_marker(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + values = [] + + @pytest.fixture(scope="class") + def myfix(request): + request.cls.hello = "world" + values.append(1) + + class TestClass(object): + def test_one(self): + assert self.hello == "world" + assert len(values) == 1 + def test_two(self): + assert self.hello == "world" + assert len(values) == 1 + pytest.mark.usefixtures("myfix")(TestClass) + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + def test_usefixtures_ini(self, pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + usefixtures = myfix + """ + ) + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(scope="class") + def myfix(request): + request.cls.hello = "world" + + """ + ) + pytester.makepyfile( + """ + class TestClass(object): + def test_one(self): + assert self.hello == "world" + def test_two(self): + assert self.hello == "world" + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + def test_usefixtures_seen_in_showmarkers(self, pytester: Pytester) -> None: + result = pytester.runpytest("--markers") + result.stdout.fnmatch_lines( + """ + *usefixtures(fixturename1*mark tests*fixtures* + """ + ) + + def test_request_instance_issue203(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + class TestClass(object): + @pytest.fixture + def setup1(self, request): + assert self == request.instance + self.arg1 = 1 + def test_hello(self, setup1): + assert self.arg1 == 1 + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_fixture_parametrized_with_iterator(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + values = [] + def f(): + yield 1 + yield 2 + dec = pytest.fixture(scope="module", params=f()) + + @dec + def arg(request): + return request.param + @dec + def arg2(request): + return request.param + + def test_1(arg): + values.append(arg) + def test_2(arg2): + values.append(arg2*10) + """ + ) + reprec = pytester.inline_run("-v") + reprec.assertoutcome(passed=4) + values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + assert values == [1, 2, 10, 20] + + def test_setup_functions_as_fixtures(self, pytester: Pytester) -> None: + """Ensure setup_* methods obey fixture scope rules (#517, #3094).""" + pytester.makepyfile( + """ + import pytest + + DB_INITIALIZED = None + + @pytest.fixture(scope="session", autouse=True) + def db(): + global DB_INITIALIZED + DB_INITIALIZED = True + yield + DB_INITIALIZED = False + + def setup_module(): + assert DB_INITIALIZED + + def teardown_module(): + assert DB_INITIALIZED + + class TestClass(object): + + def setup_method(self, method): + assert DB_INITIALIZED + + def teardown_method(self, method): + assert DB_INITIALIZED + + def test_printer_1(self): + pass + + def test_printer_2(self): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 2 passed in *"]) + + +class TestFixtureManagerParseFactories: + @pytest.fixture + def pytester(self, pytester: Pytester) -> Pytester: + pytester.makeconftest( + """ + import pytest + + @pytest.fixture + def hello(request): + return "conftest" + + @pytest.fixture + def fm(request): + return request._fixturemanager + + @pytest.fixture + def item(request): + return request._pyfuncitem + """ + ) + return pytester + + def test_parsefactories_evil_objects_issue214(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + class A(object): + def __call__(self): + pass + def __getattr__(self, name): + raise RuntimeError() + a = A() + def test_hello(): + pass + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1, failed=0) + + def test_parsefactories_conftest(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_hello(item, fm): + for name in ("fm", "hello", "item"): + faclist = fm.getfixturedefs(name, item.nodeid) + assert len(faclist) == 1 + fac = faclist[0] + assert fac.func.__name__ == name + """ + ) + reprec = pytester.inline_run("-s") + reprec.assertoutcome(passed=1) + + def test_parsefactories_conftest_and_module_and_class( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """\ + import pytest + + @pytest.fixture + def hello(request): + return "module" + class TestClass(object): + @pytest.fixture + def hello(self, request): + return "class" + def test_hello(self, item, fm): + faclist = fm.getfixturedefs("hello", item.nodeid) + print(faclist) + assert len(faclist) == 3 + + assert faclist[0].func(item._request) == "conftest" + assert faclist[1].func(item._request) == "module" + assert faclist[2].func(item._request) == "class" + """ + ) + reprec = pytester.inline_run("-s") + reprec.assertoutcome(passed=1) + + def test_parsefactories_relative_node_ids( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + # example mostly taken from: + # https://mail.python.org/pipermail/pytest-dev/2014-September/002617.html + runner = pytester.mkdir("runner") + package = pytester.mkdir("package") + package.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + @pytest.fixture + def one(): + return 1 + """ + ) + ) + package.joinpath("test_x.py").write_text( + textwrap.dedent( + """\ + def test_x(one): + assert one == 1 + """ + ) + ) + sub = package.joinpath("sub") + sub.mkdir() + sub.joinpath("__init__.py").touch() + sub.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + @pytest.fixture + def one(): + return 2 + """ + ) + ) + sub.joinpath("test_y.py").write_text( + textwrap.dedent( + """\ + def test_x(one): + assert one == 2 + """ + ) + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + with monkeypatch.context() as mp: + mp.chdir(runner) + reprec = pytester.inline_run("..") + reprec.assertoutcome(passed=2) + + def test_package_xunit_fixture(self, pytester: Pytester) -> None: + pytester.makepyfile( + __init__="""\ + values = [] + """ + ) + package = pytester.mkdir("package") + package.joinpath("__init__.py").write_text( + textwrap.dedent( + """\ + from .. import values + def setup_module(): + values.append("package") + def teardown_module(): + values[:] = [] + """ + ) + ) + package.joinpath("test_x.py").write_text( + textwrap.dedent( + """\ + from .. import values + def test_x(): + assert values == ["package"] + """ + ) + ) + package = pytester.mkdir("package2") + package.joinpath("__init__.py").write_text( + textwrap.dedent( + """\ + from .. import values + def setup_module(): + values.append("package2") + def teardown_module(): + values[:] = [] + """ + ) + ) + package.joinpath("test_x.py").write_text( + textwrap.dedent( + """\ + from .. import values + def test_x(): + assert values == ["package2"] + """ + ) + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + def test_package_fixture_complex(self, pytester: Pytester) -> None: + pytester.makepyfile( + __init__="""\ + values = [] + """ + ) + pytester.syspathinsert(pytester.path.name) + package = pytester.mkdir("package") + package.joinpath("__init__.py").write_text("") + package.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + from .. import values + @pytest.fixture(scope="package") + def one(): + values.append("package") + yield values + values.pop() + @pytest.fixture(scope="package", autouse=True) + def two(): + values.append("package-auto") + yield values + values.pop() + """ + ) + ) + package.joinpath("test_x.py").write_text( + textwrap.dedent( + """\ + from .. import values + def test_package_autouse(): + assert values == ["package-auto"] + def test_package(one): + assert values == ["package-auto", "package"] + """ + ) + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + def test_collect_custom_items(self, pytester: Pytester) -> None: + pytester.copy_example("fixtures/custom_item") + result = pytester.runpytest("foo") + result.stdout.fnmatch_lines(["*passed*"]) + + +class TestAutouseDiscovery: + @pytest.fixture + def pytester(self, pytester: Pytester) -> Pytester: + pytester.makeconftest( + """ + import pytest + @pytest.fixture(autouse=True) + def perfunction(request, tmp_path): + pass + + @pytest.fixture() + def arg1(tmp_path): + pass + @pytest.fixture(autouse=True) + def perfunction2(arg1): + pass + + @pytest.fixture + def fm(request): + return request._fixturemanager + + @pytest.fixture + def item(request): + return request._pyfuncitem + """ + ) + return pytester + + def test_parsefactories_conftest(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + from _pytest.pytester import get_public_names + def test_check_setup(item, fm): + autousenames = list(fm._getautousenames(item.nodeid)) + assert len(get_public_names(autousenames)) == 2 + assert "perfunction2" in autousenames + assert "perfunction" in autousenames + """ + ) + reprec = pytester.inline_run("-s") + reprec.assertoutcome(passed=1) + + def test_two_classes_separated_autouse(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + class TestA(object): + values = [] + @pytest.fixture(autouse=True) + def setup1(self): + self.values.append(1) + def test_setup1(self): + assert self.values == [1] + class TestB(object): + values = [] + @pytest.fixture(autouse=True) + def setup2(self): + self.values.append(1) + def test_setup2(self): + assert self.values == [1] + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + def test_setup_at_classlevel(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + class TestClass(object): + @pytest.fixture(autouse=True) + def permethod(self, request): + request.instance.funcname = request.function.__name__ + def test_method1(self): + assert self.funcname == "test_method1" + def test_method2(self): + assert self.funcname == "test_method2" + """ + ) + reprec = pytester.inline_run("-s") + reprec.assertoutcome(passed=2) + + @pytest.mark.xfail(reason="'enabled' feature not implemented") + def test_setup_enabled_functionnode(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + def enabled(parentnode, markers): + return "needsdb" in markers + + @pytest.fixture(params=[1,2]) + def db(request): + return request.param + + @pytest.fixture(enabled=enabled, autouse=True) + def createdb(db): + pass + + def test_func1(request): + assert "db" not in request.fixturenames + + @pytest.mark.needsdb + def test_func2(request): + assert "db" in request.fixturenames + """ + ) + reprec = pytester.inline_run("-s") + reprec.assertoutcome(passed=2) + + def test_callables_nocode(self, pytester: Pytester) -> None: + """An imported mock.call would break setup/factory discovery due to + it being callable and __code__ not being a code object.""" + pytester.makepyfile( + """ + class _call(tuple): + def __call__(self, *k, **kw): + pass + def __getattr__(self, k): + return self + + call = _call() + """ + ) + reprec = pytester.inline_run("-s") + reprec.assertoutcome(failed=0, passed=0) + + def test_autouse_in_conftests(self, pytester: Pytester) -> None: + a = pytester.mkdir("a") + b = pytester.mkdir("a1") + conftest = pytester.makeconftest( + """ + import pytest + @pytest.fixture(autouse=True) + def hello(): + xxx + """ + ) + conftest.rename(a.joinpath(conftest.name)) + a.joinpath("test_something.py").write_text("def test_func(): pass") + b.joinpath("test_otherthing.py").write_text("def test_func(): pass") + result = pytester.runpytest() + result.stdout.fnmatch_lines( + """ + *1 passed*1 error* + """ + ) + + def test_autouse_in_module_and_two_classes(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture(autouse=True) + def append1(): + values.append("module") + def test_x(): + assert values == ["module"] + + class TestA(object): + @pytest.fixture(autouse=True) + def append2(self): + values.append("A") + def test_hello(self): + assert values == ["module", "module", "A"], values + class TestA2(object): + def test_world(self): + assert values == ["module", "module", "A", "module"], values + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=3) + + +class TestAutouseManagement: + def test_autouse_conftest_mid_directory(self, pytester: Pytester) -> None: + pkgdir = pytester.mkpydir("xyz123") + pkgdir.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + @pytest.fixture(autouse=True) + def app(): + import sys + sys._myapp = "hello" + """ + ) + ) + sub = pkgdir.joinpath("tests") + sub.mkdir() + t = sub.joinpath("test_app.py") + t.touch() + t.write_text( + textwrap.dedent( + """\ + import sys + def test_app(): + assert sys._myapp == "hello" + """ + ) + ) + reprec = pytester.inline_run("-s") + reprec.assertoutcome(passed=1) + + def test_funcarg_and_setup(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture(scope="module") + def arg(): + values.append(1) + return 0 + @pytest.fixture(scope="module", autouse=True) + def something(arg): + values.append(2) + + def test_hello(arg): + assert len(values) == 2 + assert values == [1,2] + assert arg == 0 + + def test_hello2(arg): + assert len(values) == 2 + assert values == [1,2] + assert arg == 0 + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + def test_uses_parametrized_resource(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture(params=[1,2]) + def arg(request): + return request.param + + @pytest.fixture(autouse=True) + def something(arg): + values.append(arg) + + def test_hello(): + if len(values) == 1: + assert values == [1] + elif len(values) == 2: + assert values == [1, 2] + else: + 0/0 + + """ + ) + reprec = pytester.inline_run("-s") + reprec.assertoutcome(passed=2) + + def test_session_parametrized_function(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + values = [] + + @pytest.fixture(scope="session", params=[1,2]) + def arg(request): + return request.param + + @pytest.fixture(scope="function", autouse=True) + def append(request, arg): + if request.function.__name__ == "test_some": + values.append(arg) + + def test_some(): + pass + + def test_result(arg): + assert len(values) == arg + assert values[:arg] == [1,2][:arg] + """ + ) + reprec = pytester.inline_run("-v", "-s") + reprec.assertoutcome(passed=4) + + def test_class_function_parametrization_finalization( + self, pytester: Pytester + ) -> None: + p = pytester.makeconftest( + """ + import pytest + import pprint + + values = [] + + @pytest.fixture(scope="function", params=[1,2]) + def farg(request): + return request.param + + @pytest.fixture(scope="class", params=list("ab")) + def carg(request): + return request.param + + @pytest.fixture(scope="function", autouse=True) + def append(request, farg, carg): + def fin(): + values.append("fin_%s%s" % (carg, farg)) + request.addfinalizer(fin) + """ + ) + pytester.makepyfile( + """ + import pytest + + class TestClass(object): + def test_1(self): + pass + class TestClass2(object): + def test_2(self): + pass + """ + ) + reprec = pytester.inline_run("-v", "-s", "--confcutdir", pytester.path) + reprec.assertoutcome(passed=8) + config = reprec.getcalls("pytest_unconfigure")[0].config + values = config.pluginmanager._getconftestmodules( + p, importmode="prepend", rootpath=pytester.path + )[0].values + assert values == ["fin_a1", "fin_a2", "fin_b1", "fin_b2"] * 2 + + def test_scope_ordering(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture(scope="function", autouse=True) + def fappend2(): + values.append(2) + @pytest.fixture(scope="class", autouse=True) + def classappend3(): + values.append(3) + @pytest.fixture(scope="module", autouse=True) + def mappend(): + values.append(1) + + class TestHallo(object): + def test_method(self): + assert values == [1,3,2] + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_parametrization_setup_teardown_ordering(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + def pytest_generate_tests(metafunc): + if metafunc.cls is None: + assert metafunc.function is test_finish + if metafunc.cls is not None: + metafunc.parametrize("item", [1,2], scope="class") + class TestClass(object): + @pytest.fixture(scope="class", autouse=True) + def addteardown(self, item, request): + values.append("setup-%d" % item) + request.addfinalizer(lambda: values.append("teardown-%d" % item)) + def test_step1(self, item): + values.append("step1-%d" % item) + def test_step2(self, item): + values.append("step2-%d" % item) + + def test_finish(): + print(values) + assert values == ["setup-1", "step1-1", "step2-1", "teardown-1", + "setup-2", "step1-2", "step2-2", "teardown-2",] + """ + ) + reprec = pytester.inline_run("-s") + reprec.assertoutcome(passed=5) + + def test_ordering_autouse_before_explicit(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + values = [] + @pytest.fixture(autouse=True) + def fix1(): + values.append(1) + @pytest.fixture() + def arg1(): + values.append(2) + def test_hello(arg1): + assert values == [1,2] + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + @pytest.mark.parametrize("param1", ["", "params=[1]"], ids=["p00", "p01"]) + @pytest.mark.parametrize("param2", ["", "params=[1]"], ids=["p10", "p11"]) + def test_ordering_dependencies_torndown_first( + self, pytester: Pytester, param1, param2 + ) -> None: + """#226""" + pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture(%(param1)s) + def arg1(request): + request.addfinalizer(lambda: values.append("fin1")) + values.append("new1") + @pytest.fixture(%(param2)s) + def arg2(request, arg1): + request.addfinalizer(lambda: values.append("fin2")) + values.append("new2") + + def test_arg(arg2): + pass + def test_check(): + assert values == ["new1", "new2", "fin2", "fin1"] + """ + % locals() + ) + reprec = pytester.inline_run("-s") + reprec.assertoutcome(passed=2) + + +class TestFixtureMarker: + def test_parametrize(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(params=["a", "b", "c"]) + def arg(request): + return request.param + values = [] + def test_param(arg): + values.append(arg) + def test_result(): + assert values == list("abc") + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=4) + + def test_multiple_parametrization_issue_736(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(params=[1,2,3]) + def foo(request): + return request.param + + @pytest.mark.parametrize('foobar', [4,5,6]) + def test_issue(foo, foobar): + assert foo in [1,2,3] + assert foobar in [4,5,6] + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=9) + + @pytest.mark.parametrize( + "param_args", + ["'fixt, val'", "'fixt,val'", "['fixt', 'val']", "('fixt', 'val')"], + ) + def test_override_parametrized_fixture_issue_979( + self, pytester: Pytester, param_args + ) -> None: + """Make sure a parametrized argument can override a parametrized fixture. + + This was a regression introduced in the fix for #736. + """ + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(params=[1, 2]) + def fixt(request): + return request.param + + @pytest.mark.parametrize(%s, [(3, 'x'), (4, 'x')]) + def test_foo(fixt, val): + pass + """ + % param_args + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + def test_scope_session(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture(scope="module") + def arg(): + values.append(1) + return 1 + + def test_1(arg): + assert arg == 1 + def test_2(arg): + assert arg == 1 + assert len(values) == 1 + class TestClass(object): + def test3(self, arg): + assert arg == 1 + assert len(values) == 1 + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=3) + + def test_scope_session_exc(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture(scope="session") + def fix(): + values.append(1) + pytest.skip('skipping') + + def test_1(fix): + pass + def test_2(fix): + pass + def test_last(): + assert values == [1] + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(skipped=2, passed=1) + + def test_scope_session_exc_two_fix(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + m = [] + @pytest.fixture(scope="session") + def a(): + values.append(1) + pytest.skip('skipping') + @pytest.fixture(scope="session") + def b(a): + m.append(1) + + def test_1(b): + pass + def test_2(b): + pass + def test_last(): + assert values == [1] + assert m == [] + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(skipped=2, passed=1) + + def test_scope_exc(self, pytester: Pytester) -> None: + pytester.makepyfile( + test_foo=""" + def test_foo(fix): + pass + """, + test_bar=""" + def test_bar(fix): + pass + """, + conftest=""" + import pytest + reqs = [] + @pytest.fixture(scope="session") + def fix(request): + reqs.append(1) + pytest.skip() + @pytest.fixture + def req_list(): + return reqs + """, + test_real=""" + def test_last(req_list): + assert req_list == [1] + """, + ) + reprec = pytester.inline_run() + reprec.assertoutcome(skipped=2, passed=1) + + def test_scope_module_uses_session(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture(scope="module") + def arg(): + values.append(1) + return 1 + + def test_1(arg): + assert arg == 1 + def test_2(arg): + assert arg == 1 + assert len(values) == 1 + class TestClass(object): + def test3(self, arg): + assert arg == 1 + assert len(values) == 1 + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=3) + + def test_scope_module_and_finalizer(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + finalized_list = [] + created_list = [] + @pytest.fixture(scope="module") + def arg(request): + created_list.append(1) + assert request.scope == "module" + request.addfinalizer(lambda: finalized_list.append(1)) + @pytest.fixture + def created(request): + return len(created_list) + @pytest.fixture + def finalized(request): + return len(finalized_list) + """ + ) + pytester.makepyfile( + test_mod1=""" + def test_1(arg, created, finalized): + assert created == 1 + assert finalized == 0 + def test_2(arg, created, finalized): + assert created == 1 + assert finalized == 0""", + test_mod2=""" + def test_3(arg, created, finalized): + assert created == 2 + assert finalized == 1""", + test_mode3=""" + def test_4(arg, created, finalized): + assert created == 3 + assert finalized == 2 + """, + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=4) + + def test_scope_mismatch_various(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + finalized = [] + created = [] + @pytest.fixture(scope="function") + def arg(request): + pass + """ + ) + pytester.makepyfile( + test_mod1=""" + import pytest + @pytest.fixture(scope="session") + def arg(request): + request.getfixturevalue("arg") + def test_1(arg): + pass + """ + ) + result = pytester.runpytest() + assert result.ret != 0 + result.stdout.fnmatch_lines( + ["*ScopeMismatch*You tried*function*session*request*"] + ) + + def test_dynamic_scope(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + + + def pytest_addoption(parser): + parser.addoption("--extend-scope", action="store_true", default=False) + + + def dynamic_scope(fixture_name, config): + if config.getoption("--extend-scope"): + return "session" + return "function" + + + @pytest.fixture(scope=dynamic_scope) + def dynamic_fixture(calls=[]): + calls.append("call") + return len(calls) + + """ + ) + + pytester.makepyfile( + """ + def test_first(dynamic_fixture): + assert dynamic_fixture == 1 + + + def test_second(dynamic_fixture): + assert dynamic_fixture == 2 + + """ + ) + + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + reprec = pytester.inline_run("--extend-scope") + reprec.assertoutcome(passed=1, failed=1) + + def test_dynamic_scope_bad_return(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + def dynamic_scope(**_): + return "wrong-scope" + + @pytest.fixture(scope=dynamic_scope) + def fixture(): + pass + + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + "Fixture 'fixture' from test_dynamic_scope_bad_return.py " + "got an unexpected scope value 'wrong-scope'" + ) + + def test_register_only_with_mark(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + @pytest.fixture() + def arg(): + return 1 + """ + ) + pytester.makepyfile( + test_mod1=""" + import pytest + @pytest.fixture() + def arg(arg): + return arg + 1 + def test_1(arg): + assert arg == 2 + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_parametrize_and_scope(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope="module", params=["a", "b", "c"]) + def arg(request): + return request.param + values = [] + def test_param(arg): + values.append(arg) + """ + ) + reprec = pytester.inline_run("-v") + reprec.assertoutcome(passed=3) + values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + assert len(values) == 3 + assert "a" in values + assert "b" in values + assert "c" in values + + def test_scope_mismatch(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + @pytest.fixture(scope="function") + def arg(request): + pass + """ + ) + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope="session") + def arg(arg): + pass + def test_mismatch(arg): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*ScopeMismatch*", "*1 error*"]) + + def test_parametrize_separated_order(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="module", params=[1, 2]) + def arg(request): + return request.param + + values = [] + def test_1(arg): + values.append(arg) + def test_2(arg): + values.append(arg) + """ + ) + reprec = pytester.inline_run("-v") + reprec.assertoutcome(passed=4) + values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + assert values == [1, 1, 2, 2] + + def test_module_parametrized_ordering(self, pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + console_output_style=classic + """ + ) + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(scope="session", params="s1 s2".split()) + def sarg(): + pass + @pytest.fixture(scope="module", params="m1 m2".split()) + def marg(): + pass + """ + ) + pytester.makepyfile( + test_mod1=""" + def test_func(sarg): + pass + def test_func1(marg): + pass + """, + test_mod2=""" + def test_func2(sarg): + pass + def test_func3(sarg, marg): + pass + def test_func3b(sarg, marg): + pass + def test_func4(marg): + pass + """, + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + """ + test_mod1.py::test_func[s1] PASSED + test_mod2.py::test_func2[s1] PASSED + test_mod2.py::test_func3[s1-m1] PASSED + test_mod2.py::test_func3b[s1-m1] PASSED + test_mod2.py::test_func3[s1-m2] PASSED + test_mod2.py::test_func3b[s1-m2] PASSED + test_mod1.py::test_func[s2] PASSED + test_mod2.py::test_func2[s2] PASSED + test_mod2.py::test_func3[s2-m1] PASSED + test_mod2.py::test_func3b[s2-m1] PASSED + test_mod2.py::test_func4[m1] PASSED + test_mod2.py::test_func3[s2-m2] PASSED + test_mod2.py::test_func3b[s2-m2] PASSED + test_mod2.py::test_func4[m2] PASSED + test_mod1.py::test_func1[m1] PASSED + test_mod1.py::test_func1[m2] PASSED + """ + ) + + def test_dynamic_parametrized_ordering(self, pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + console_output_style=classic + """ + ) + pytester.makeconftest( + """ + import pytest + + def pytest_configure(config): + class DynamicFixturePlugin(object): + @pytest.fixture(scope='session', params=['flavor1', 'flavor2']) + def flavor(self, request): + return request.param + config.pluginmanager.register(DynamicFixturePlugin(), 'flavor-fixture') + + @pytest.fixture(scope='session', params=['vxlan', 'vlan']) + def encap(request): + return request.param + + @pytest.fixture(scope='session', autouse='True') + def reprovision(request, flavor, encap): + pass + """ + ) + pytester.makepyfile( + """ + def test(reprovision): + pass + def test2(reprovision): + pass + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + """ + test_dynamic_parametrized_ordering.py::test[flavor1-vxlan] PASSED + test_dynamic_parametrized_ordering.py::test2[flavor1-vxlan] PASSED + test_dynamic_parametrized_ordering.py::test[flavor2-vxlan] PASSED + test_dynamic_parametrized_ordering.py::test2[flavor2-vxlan] PASSED + test_dynamic_parametrized_ordering.py::test[flavor2-vlan] PASSED + test_dynamic_parametrized_ordering.py::test2[flavor2-vlan] PASSED + test_dynamic_parametrized_ordering.py::test[flavor1-vlan] PASSED + test_dynamic_parametrized_ordering.py::test2[flavor1-vlan] PASSED + """ + ) + + def test_class_ordering(self, pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + console_output_style=classic + """ + ) + pytester.makeconftest( + """ + import pytest + + values = [] + + @pytest.fixture(scope="function", params=[1,2]) + def farg(request): + return request.param + + @pytest.fixture(scope="class", params=list("ab")) + def carg(request): + return request.param + + @pytest.fixture(scope="function", autouse=True) + def append(request, farg, carg): + def fin(): + values.append("fin_%s%s" % (carg, farg)) + request.addfinalizer(fin) + """ + ) + pytester.makepyfile( + """ + import pytest + + class TestClass2(object): + def test_1(self): + pass + def test_2(self): + pass + class TestClass(object): + def test_3(self): + pass + """ + ) + result = pytester.runpytest("-vs") + result.stdout.re_match_lines( + r""" + test_class_ordering.py::TestClass2::test_1\[a-1\] PASSED + test_class_ordering.py::TestClass2::test_1\[a-2\] PASSED + test_class_ordering.py::TestClass2::test_2\[a-1\] PASSED + test_class_ordering.py::TestClass2::test_2\[a-2\] PASSED + test_class_ordering.py::TestClass2::test_1\[b-1\] PASSED + test_class_ordering.py::TestClass2::test_1\[b-2\] PASSED + test_class_ordering.py::TestClass2::test_2\[b-1\] PASSED + test_class_ordering.py::TestClass2::test_2\[b-2\] PASSED + test_class_ordering.py::TestClass::test_3\[a-1\] PASSED + test_class_ordering.py::TestClass::test_3\[a-2\] PASSED + test_class_ordering.py::TestClass::test_3\[b-1\] PASSED + test_class_ordering.py::TestClass::test_3\[b-2\] PASSED + """ + ) + + def test_parametrize_separated_order_higher_scope_first( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="function", params=[1, 2]) + def arg(request): + param = request.param + request.addfinalizer(lambda: values.append("fin:%s" % param)) + values.append("create:%s" % param) + return request.param + + @pytest.fixture(scope="module", params=["mod1", "mod2"]) + def modarg(request): + param = request.param + request.addfinalizer(lambda: values.append("fin:%s" % param)) + values.append("create:%s" % param) + return request.param + + values = [] + def test_1(arg): + values.append("test1") + def test_2(modarg): + values.append("test2") + def test_3(arg, modarg): + values.append("test3") + def test_4(modarg, arg): + values.append("test4") + """ + ) + reprec = pytester.inline_run("-v") + reprec.assertoutcome(passed=12) + values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + expected = [ + "create:1", + "test1", + "fin:1", + "create:2", + "test1", + "fin:2", + "create:mod1", + "test2", + "create:1", + "test3", + "fin:1", + "create:2", + "test3", + "fin:2", + "create:1", + "test4", + "fin:1", + "create:2", + "test4", + "fin:2", + "fin:mod1", + "create:mod2", + "test2", + "create:1", + "test3", + "fin:1", + "create:2", + "test3", + "fin:2", + "create:1", + "test4", + "fin:1", + "create:2", + "test4", + "fin:2", + "fin:mod2", + ] + import pprint + + pprint.pprint(list(zip(values, expected))) + assert values == expected + + def test_parametrized_fixture_teardown_order(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(params=[1,2], scope="class") + def param1(request): + return request.param + + values = [] + + class TestClass(object): + @classmethod + @pytest.fixture(scope="class", autouse=True) + def setup1(self, request, param1): + values.append(1) + request.addfinalizer(self.teardown1) + @classmethod + def teardown1(self): + assert values.pop() == 1 + @pytest.fixture(scope="class", autouse=True) + def setup2(self, request, param1): + values.append(2) + request.addfinalizer(self.teardown2) + @classmethod + def teardown2(self): + assert values.pop() == 2 + def test(self): + pass + + def test_finish(): + assert not values + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + """ + *3 passed* + """ + ) + result.stdout.no_fnmatch_line("*error*") + + def test_fixture_finalizer(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + import sys + + @pytest.fixture + def browser(request): + + def finalize(): + sys.stdout.write_text('Finalized') + request.addfinalizer(finalize) + return {} + """ + ) + b = pytester.mkdir("subdir") + b.joinpath("test_overridden_fixture_finalizer.py").write_text( + textwrap.dedent( + """\ + import pytest + @pytest.fixture + def browser(browser): + browser['visited'] = True + return browser + + def test_browser(browser): + assert browser['visited'] is True + """ + ) + ) + reprec = pytester.runpytest("-s") + for test in ["test_browser"]: + reprec.stdout.fnmatch_lines(["*Finalized*"]) + + def test_class_scope_with_normal_tests(self, pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import pytest + + class Box(object): + value = 0 + + @pytest.fixture(scope='class') + def a(request): + Box.value += 1 + return Box.value + + def test_a(a): + assert a == 1 + + class Test1(object): + def test_b(self, a): + assert a == 2 + + class Test2(object): + def test_c(self, a): + assert a == 3""" + ) + reprec = pytester.inline_run(testpath) + for test in ["test_a", "test_b", "test_c"]: + assert reprec.matchreport(test).passed + + def test_request_is_clean(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture(params=[1, 2]) + def fix(request): + request.addfinalizer(lambda: values.append(request.param)) + def test_fix(fix): + pass + """ + ) + reprec = pytester.inline_run("-s") + values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + assert values == [1, 2] + + def test_parametrize_separated_lifecycle(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + values = [] + @pytest.fixture(scope="module", params=[1, 2]) + def arg(request): + x = request.param + request.addfinalizer(lambda: values.append("fin%s" % x)) + return request.param + def test_1(arg): + values.append(arg) + def test_2(arg): + values.append(arg) + """ + ) + reprec = pytester.inline_run("-vs") + reprec.assertoutcome(passed=4) + values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + import pprint + + pprint.pprint(values) + # assert len(values) == 6 + assert values[0] == values[1] == 1 + assert values[2] == "fin1" + assert values[3] == values[4] == 2 + assert values[5] == "fin2" + + def test_parametrize_function_scoped_finalizers_called( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="function", params=[1, 2]) + def arg(request): + x = request.param + request.addfinalizer(lambda: values.append("fin%s" % x)) + return request.param + + values = [] + def test_1(arg): + values.append(arg) + def test_2(arg): + values.append(arg) + def test_3(): + assert len(values) == 8 + assert values == [1, "fin1", 2, "fin2", 1, "fin1", 2, "fin2"] + """ + ) + reprec = pytester.inline_run("-v") + reprec.assertoutcome(passed=5) + + @pytest.mark.parametrize("scope", ["session", "function", "module"]) + def test_finalizer_order_on_parametrization( + self, scope, pytester: Pytester + ) -> None: + """#246""" + pytester.makepyfile( + """ + import pytest + values = [] + + @pytest.fixture(scope=%(scope)r, params=["1"]) + def fix1(request): + return request.param + + @pytest.fixture(scope=%(scope)r) + def fix2(request, base): + def cleanup_fix2(): + assert not values, "base should not have been finalized" + request.addfinalizer(cleanup_fix2) + + @pytest.fixture(scope=%(scope)r) + def base(request, fix1): + def cleanup_base(): + values.append("fin_base") + print("finalizing base") + request.addfinalizer(cleanup_base) + + def test_begin(): + pass + def test_baz(base, fix2): + pass + def test_other(): + pass + """ + % {"scope": scope} + ) + reprec = pytester.inline_run("-lvs") + reprec.assertoutcome(passed=3) + + def test_class_scope_parametrization_ordering(self, pytester: Pytester) -> None: + """#396""" + pytester.makepyfile( + """ + import pytest + values = [] + @pytest.fixture(params=["John", "Doe"], scope="class") + def human(request): + request.addfinalizer(lambda: values.append("fin %s" % request.param)) + return request.param + + class TestGreetings(object): + def test_hello(self, human): + values.append("test_hello") + + class TestMetrics(object): + def test_name(self, human): + values.append("test_name") + + def test_population(self, human): + values.append("test_population") + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=6) + values = reprec.getcalls("pytest_runtest_call")[0].item.module.values + assert values == [ + "test_hello", + "fin John", + "test_hello", + "fin Doe", + "test_name", + "test_population", + "fin John", + "test_name", + "test_population", + "fin Doe", + ] + + def test_parametrize_setup_function(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="module", params=[1, 2]) + def arg(request): + return request.param + + @pytest.fixture(scope="module", autouse=True) + def mysetup(request, arg): + request.addfinalizer(lambda: values.append("fin%s" % arg)) + values.append("setup%s" % arg) + + values = [] + def test_1(arg): + values.append(arg) + def test_2(arg): + values.append(arg) + def test_3(): + import pprint + pprint.pprint(values) + if arg == 1: + assert values == ["setup1", 1, 1, ] + elif arg == 2: + assert values == ["setup1", 1, 1, "fin1", + "setup2", 2, 2, ] + + """ + ) + reprec = pytester.inline_run("-v") + reprec.assertoutcome(passed=6) + + def test_fixture_marked_function_not_collected_as_test( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture + def test_app(): + return 1 + + def test_something(test_app): + assert test_app == 1 + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_params_and_ids(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(params=[object(), object()], + ids=['alpha', 'beta']) + def fix(request): + return request.param + + def test_foo(fix): + assert 1 + """ + ) + res = pytester.runpytest("-v") + res.stdout.fnmatch_lines(["*test_foo*alpha*", "*test_foo*beta*"]) + + def test_params_and_ids_yieldfixture(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(params=[object(), object()], ids=['alpha', 'beta']) + def fix(request): + yield request.param + + def test_foo(fix): + assert 1 + """ + ) + res = pytester.runpytest("-v") + res.stdout.fnmatch_lines(["*test_foo*alpha*", "*test_foo*beta*"]) + + def test_deterministic_fixture_collection( + self, pytester: Pytester, monkeypatch + ) -> None: + """#920""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="module", + params=["A", + "B", + "C"]) + def A(request): + return request.param + + @pytest.fixture(scope="module", + params=["DDDDDDDDD", "EEEEEEEEEEEE", "FFFFFFFFFFF", "banansda"]) + def B(request, A): + return request.param + + def test_foo(B): + # Something funky is going on here. + # Despite specified seeds, on what is collected, + # sometimes we get unexpected passes. hashing B seems + # to help? + assert hash(B) or True + """ + ) + monkeypatch.setenv("PYTHONHASHSEED", "1") + out1 = pytester.runpytest_subprocess("-v") + monkeypatch.setenv("PYTHONHASHSEED", "2") + out2 = pytester.runpytest_subprocess("-v") + output1 = [ + line + for line in out1.outlines + if line.startswith("test_deterministic_fixture_collection.py::test_foo") + ] + output2 = [ + line + for line in out2.outlines + if line.startswith("test_deterministic_fixture_collection.py::test_foo") + ] + assert len(output1) == 12 + assert output1 == output2 + + +class TestRequestScopeAccess: + pytestmark = pytest.mark.parametrize( + ("scope", "ok", "error"), + [ + ["session", "", "path class function module"], + ["module", "module path", "cls function"], + ["class", "module path cls", "function"], + ["function", "module path cls function", ""], + ], + ) + + def test_setup(self, pytester: Pytester, scope, ok, error) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope=%r, autouse=True) + def myscoped(request): + for x in %r: + assert hasattr(request, x) + for x in %r: + pytest.raises(AttributeError, lambda: + getattr(request, x)) + assert request.session + assert request.config + def test_func(): + pass + """ + % (scope, ok.split(), error.split()) + ) + reprec = pytester.inline_run("-l") + reprec.assertoutcome(passed=1) + + def test_funcarg(self, pytester: Pytester, scope, ok, error) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope=%r) + def arg(request): + for x in %r: + assert hasattr(request, x) + for x in %r: + pytest.raises(AttributeError, lambda: + getattr(request, x)) + assert request.session + assert request.config + def test_func(arg): + pass + """ + % (scope, ok.split(), error.split()) + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +class TestErrors: + def test_subfactory_missing_funcarg(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture() + def gen(qwe123): + return 1 + def test_something(gen): + pass + """ + ) + result = pytester.runpytest() + assert result.ret != 0 + result.stdout.fnmatch_lines( + ["*def gen(qwe123):*", "*fixture*qwe123*not found*", "*1 error*"] + ) + + def test_issue498_fixture_finalizer_failing(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture + def fix1(request): + def f(): + raise KeyError + request.addfinalizer(f) + return object() + + values = [] + def test_1(fix1): + values.append(fix1) + def test_2(fix1): + values.append(fix1) + def test_3(): + assert values[0] != values[1] + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + """ + *ERROR*teardown*test_1* + *KeyError* + *ERROR*teardown*test_2* + *KeyError* + *3 pass*2 errors* + """ + ) + + def test_setupfunc_missing_funcarg(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(autouse=True) + def gen(qwe123): + return 1 + def test_something(): + pass + """ + ) + result = pytester.runpytest() + assert result.ret != 0 + result.stdout.fnmatch_lines( + ["*def gen(qwe123):*", "*fixture*qwe123*not found*", "*1 error*"] + ) + + +class TestShowFixtures: + def test_funcarg_compat(self, pytester: Pytester) -> None: + config = pytester.parseconfigure("--funcargs") + assert config.option.showfixtures + + def test_show_fixtures(self, pytester: Pytester) -> None: + result = pytester.runpytest("--fixtures") + result.stdout.fnmatch_lines( + [ + "tmp_path_factory [[]session scope[]] -- .../_pytest/tmpdir.py:*", + "*for the test session*", + "tmp_path -- .../_pytest/tmpdir.py:*", + "*temporary directory*", + ] + ) + + def test_show_fixtures_verbose(self, pytester: Pytester) -> None: + result = pytester.runpytest("--fixtures", "-v") + result.stdout.fnmatch_lines( + [ + "tmp_path_factory [[]session scope[]] -- .../_pytest/tmpdir.py:*", + "*for the test session*", + "tmp_path -- .../_pytest/tmpdir.py:*", + "*temporary directory*", + ] + ) + + def test_show_fixtures_testmodule(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + ''' + import pytest + @pytest.fixture + def _arg0(): + """ hidden """ + @pytest.fixture + def arg1(): + """ hello world """ + ''' + ) + result = pytester.runpytest("--fixtures", p) + result.stdout.fnmatch_lines( + """ + *tmp_path -- * + *fixtures defined from* + *arg1 -- test_show_fixtures_testmodule.py:6* + *hello world* + """ + ) + result.stdout.no_fnmatch_line("*arg0*") + + @pytest.mark.parametrize("testmod", [True, False]) + def test_show_fixtures_conftest(self, pytester: Pytester, testmod) -> None: + pytester.makeconftest( + ''' + import pytest + @pytest.fixture + def arg1(): + """ hello world """ + ''' + ) + if testmod: + pytester.makepyfile( + """ + def test_hello(): + pass + """ + ) + result = pytester.runpytest("--fixtures") + result.stdout.fnmatch_lines( + """ + *tmp_path* + *fixtures defined from*conftest* + *arg1* + *hello world* + """ + ) + + def test_show_fixtures_trimmed_doc(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + textwrap.dedent( + '''\ + import pytest + @pytest.fixture + def arg1(): + """ + line1 + line2 + + """ + @pytest.fixture + def arg2(): + """ + line1 + line2 + + """ + ''' + ) + ) + result = pytester.runpytest("--fixtures", p) + result.stdout.fnmatch_lines( + textwrap.dedent( + """\ + * fixtures defined from test_show_fixtures_trimmed_doc * + arg2 -- test_show_fixtures_trimmed_doc.py:10 + line1 + line2 + arg1 -- test_show_fixtures_trimmed_doc.py:3 + line1 + line2 + """ + ) + ) + + def test_show_fixtures_indented_doc(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + textwrap.dedent( + '''\ + import pytest + @pytest.fixture + def fixture1(): + """ + line1 + indented line + """ + ''' + ) + ) + result = pytester.runpytest("--fixtures", p) + result.stdout.fnmatch_lines( + textwrap.dedent( + """\ + * fixtures defined from test_show_fixtures_indented_doc * + fixture1 -- test_show_fixtures_indented_doc.py:3 + line1 + indented line + """ + ) + ) + + def test_show_fixtures_indented_doc_first_line_unindented( + self, pytester: Pytester + ) -> None: + p = pytester.makepyfile( + textwrap.dedent( + '''\ + import pytest + @pytest.fixture + def fixture1(): + """line1 + line2 + indented line + """ + ''' + ) + ) + result = pytester.runpytest("--fixtures", p) + result.stdout.fnmatch_lines( + textwrap.dedent( + """\ + * fixtures defined from test_show_fixtures_indented_doc_first_line_unindented * + fixture1 -- test_show_fixtures_indented_doc_first_line_unindented.py:3 + line1 + line2 + indented line + """ + ) + ) + + def test_show_fixtures_indented_in_class(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + textwrap.dedent( + '''\ + import pytest + class TestClass(object): + @pytest.fixture + def fixture1(self): + """line1 + line2 + indented line + """ + ''' + ) + ) + result = pytester.runpytest("--fixtures", p) + result.stdout.fnmatch_lines( + textwrap.dedent( + """\ + * fixtures defined from test_show_fixtures_indented_in_class * + fixture1 -- test_show_fixtures_indented_in_class.py:4 + line1 + line2 + indented line + """ + ) + ) + + def test_show_fixtures_different_files(self, pytester: Pytester) -> None: + """`--fixtures` only shows fixtures from first file (#833).""" + pytester.makepyfile( + test_a=''' + import pytest + + @pytest.fixture + def fix_a(): + """Fixture A""" + pass + + def test_a(fix_a): + pass + ''' + ) + pytester.makepyfile( + test_b=''' + import pytest + + @pytest.fixture + def fix_b(): + """Fixture B""" + pass + + def test_b(fix_b): + pass + ''' + ) + result = pytester.runpytest("--fixtures") + result.stdout.fnmatch_lines( + """ + * fixtures defined from test_a * + fix_a -- test_a.py:4 + Fixture A + + * fixtures defined from test_b * + fix_b -- test_b.py:4 + Fixture B + """ + ) + + def test_show_fixtures_with_same_name(self, pytester: Pytester) -> None: + pytester.makeconftest( + ''' + import pytest + @pytest.fixture + def arg1(): + """Hello World in conftest.py""" + return "Hello World" + ''' + ) + pytester.makepyfile( + """ + def test_foo(arg1): + assert arg1 == "Hello World" + """ + ) + pytester.makepyfile( + ''' + import pytest + @pytest.fixture + def arg1(): + """Hi from test module""" + return "Hi" + def test_bar(arg1): + assert arg1 == "Hi" + ''' + ) + result = pytester.runpytest("--fixtures") + result.stdout.fnmatch_lines( + """ + * fixtures defined from conftest * + arg1 -- conftest.py:3 + Hello World in conftest.py + + * fixtures defined from test_show_fixtures_with_same_name * + arg1 -- test_show_fixtures_with_same_name.py:3 + Hi from test module + """ + ) + + def test_fixture_disallow_twice(self): + """Test that applying @pytest.fixture twice generates an error (#2334).""" + with pytest.raises(ValueError): + + @pytest.fixture + @pytest.fixture + def foo(): + raise NotImplementedError() + + +class TestContextManagerFixtureFuncs: + def test_simple(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture + def arg1(): + print("setup") + yield 1 + print("teardown") + def test_1(arg1): + print("test1", arg1) + def test_2(arg1): + print("test2", arg1) + assert 0 + """ + ) + result = pytester.runpytest("-s") + result.stdout.fnmatch_lines( + """ + *setup* + *test1 1* + *teardown* + *setup* + *test2 1* + *teardown* + """ + ) + + def test_scoped(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope="module") + def arg1(): + print("setup") + yield 1 + print("teardown") + def test_1(arg1): + print("test1", arg1) + def test_2(arg1): + print("test2", arg1) + """ + ) + result = pytester.runpytest("-s") + result.stdout.fnmatch_lines( + """ + *setup* + *test1 1* + *test2 1* + *teardown* + """ + ) + + def test_setup_exception(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope="module") + def arg1(): + pytest.fail("setup") + yield 1 + def test_1(arg1): + pass + """ + ) + result = pytester.runpytest("-s") + result.stdout.fnmatch_lines( + """ + *pytest.fail*setup* + *1 error* + """ + ) + + def test_teardown_exception(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope="module") + def arg1(): + yield 1 + pytest.fail("teardown") + def test_1(arg1): + pass + """ + ) + result = pytester.runpytest("-s") + result.stdout.fnmatch_lines( + """ + *pytest.fail*teardown* + *1 passed*1 error* + """ + ) + + def test_yields_more_than_one(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope="module") + def arg1(): + yield 1 + yield 2 + def test_1(arg1): + pass + """ + ) + result = pytester.runpytest("-s") + result.stdout.fnmatch_lines( + """ + *fixture function* + *test_yields*:2* + """ + ) + + def test_custom_name(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(name='meow') + def arg1(): + return 'mew' + def test_1(meow): + print(meow) + """ + ) + result = pytester.runpytest("-s") + result.stdout.fnmatch_lines(["*mew*"]) + + +class TestParameterizedSubRequest: + def test_call_from_fixture(self, pytester: Pytester) -> None: + pytester.makepyfile( + test_call_from_fixture=""" + import pytest + + @pytest.fixture(params=[0, 1, 2]) + def fix_with_param(request): + return request.param + + @pytest.fixture + def get_named_fixture(request): + return request.getfixturevalue('fix_with_param') + + def test_foo(request, get_named_fixture): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "The requested fixture has no parameter defined for test:", + " test_call_from_fixture.py::test_foo", + "Requested fixture 'fix_with_param' defined in:", + "test_call_from_fixture.py:4", + "Requested here:", + "test_call_from_fixture.py:9", + "*1 error in*", + ] + ) + + def test_call_from_test(self, pytester: Pytester) -> None: + pytester.makepyfile( + test_call_from_test=""" + import pytest + + @pytest.fixture(params=[0, 1, 2]) + def fix_with_param(request): + return request.param + + def test_foo(request): + request.getfixturevalue('fix_with_param') + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "The requested fixture has no parameter defined for test:", + " test_call_from_test.py::test_foo", + "Requested fixture 'fix_with_param' defined in:", + "test_call_from_test.py:4", + "Requested here:", + "test_call_from_test.py:8", + "*1 failed*", + ] + ) + + def test_external_fixture(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(params=[0, 1, 2]) + def fix_with_param(request): + return request.param + """ + ) + + pytester.makepyfile( + test_external_fixture=""" + def test_foo(request): + request.getfixturevalue('fix_with_param') + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "The requested fixture has no parameter defined for test:", + " test_external_fixture.py::test_foo", + "", + "Requested fixture 'fix_with_param' defined in:", + "conftest.py:4", + "Requested here:", + "test_external_fixture.py:2", + "*1 failed*", + ] + ) + + def test_non_relative_path(self, pytester: Pytester) -> None: + tests_dir = pytester.mkdir("tests") + fixdir = pytester.mkdir("fixtures") + fixfile = fixdir.joinpath("fix.py") + fixfile.write_text( + textwrap.dedent( + """\ + import pytest + + @pytest.fixture(params=[0, 1, 2]) + def fix_with_param(request): + return request.param + """ + ) + ) + + testfile = tests_dir.joinpath("test_foos.py") + testfile.write_text( + textwrap.dedent( + """\ + from fix import fix_with_param + + def test_foo(request): + request.getfixturevalue('fix_with_param') + """ + ) + ) + + os.chdir(tests_dir) + pytester.syspathinsert(fixdir) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "The requested fixture has no parameter defined for test:", + " test_foos.py::test_foo", + "", + "Requested fixture 'fix_with_param' defined in:", + f"{fixfile}:4", + "Requested here:", + "test_foos.py:4", + "*1 failed*", + ] + ) + + # With non-overlapping rootdir, passing tests_dir. + rootdir = pytester.mkdir("rootdir") + os.chdir(rootdir) + result = pytester.runpytest("--rootdir", rootdir, tests_dir) + result.stdout.fnmatch_lines( + [ + "The requested fixture has no parameter defined for test:", + " test_foos.py::test_foo", + "", + "Requested fixture 'fix_with_param' defined in:", + f"{fixfile}:4", + "Requested here:", + f"{testfile}:4", + "*1 failed*", + ] + ) + + +def test_pytest_fixture_setup_and_post_finalizer_hook(pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_fixture_setup(fixturedef, request): + print('ROOT setup hook called for {0} from {1}'.format(fixturedef.argname, request.node.name)) + def pytest_fixture_post_finalizer(fixturedef, request): + print('ROOT finalizer hook called for {0} from {1}'.format(fixturedef.argname, request.node.name)) + """ + ) + pytester.makepyfile( + **{ + "tests/conftest.py": """ + def pytest_fixture_setup(fixturedef, request): + print('TESTS setup hook called for {0} from {1}'.format(fixturedef.argname, request.node.name)) + def pytest_fixture_post_finalizer(fixturedef, request): + print('TESTS finalizer hook called for {0} from {1}'.format(fixturedef.argname, request.node.name)) + """, + "tests/test_hooks.py": """ + import pytest + + @pytest.fixture() + def my_fixture(): + return 'some' + + def test_func(my_fixture): + print('TEST test_func') + assert my_fixture == 'some' + """, + } + ) + result = pytester.runpytest("-s") + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + "*TESTS setup hook called for my_fixture from test_func*", + "*ROOT setup hook called for my_fixture from test_func*", + "*TEST test_func*", + "*TESTS finalizer hook called for my_fixture from test_func*", + "*ROOT finalizer hook called for my_fixture from test_func*", + ] + ) + + +class TestScopeOrdering: + """Class of tests that ensure fixtures are ordered based on their scopes (#2405)""" + + @pytest.mark.parametrize("variant", ["mark", "autouse"]) + def test_func_closure_module_auto( + self, pytester: Pytester, variant, monkeypatch + ) -> None: + """Semantically identical to the example posted in #2405 when ``use_mark=True``""" + monkeypatch.setenv("FIXTURE_ACTIVATION_VARIANT", variant) + pytester.makepyfile( + """ + import warnings + import os + import pytest + VAR = 'FIXTURE_ACTIVATION_VARIANT' + VALID_VARS = ('autouse', 'mark') + + VARIANT = os.environ.get(VAR) + if VARIANT is None or VARIANT not in VALID_VARS: + warnings.warn("{!r} is not in {}, assuming autouse".format(VARIANT, VALID_VARS) ) + variant = 'mark' + + @pytest.fixture(scope='module', autouse=VARIANT == 'autouse') + def m1(): pass + + if VARIANT=='mark': + pytestmark = pytest.mark.usefixtures('m1') + + @pytest.fixture(scope='function', autouse=True) + def f1(): pass + + def test_func(m1): + pass + """ + ) + items, _ = pytester.inline_genitems() + request = FixtureRequest(items[0], _ispytest=True) + assert request.fixturenames == "m1 f1".split() + + def test_func_closure_with_native_fixtures( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + """Sanity check that verifies the order returned by the closures and the actual fixture execution order: + The execution order may differ because of fixture inter-dependencies. + """ + monkeypatch.setattr(pytest, "FIXTURE_ORDER", [], raising=False) + pytester.makepyfile( + """ + import pytest + + FIXTURE_ORDER = pytest.FIXTURE_ORDER + + @pytest.fixture(scope="session") + def s1(): + FIXTURE_ORDER.append('s1') + + @pytest.fixture(scope="package") + def p1(): + FIXTURE_ORDER.append('p1') + + @pytest.fixture(scope="module") + def m1(): + FIXTURE_ORDER.append('m1') + + @pytest.fixture(scope='session') + def my_tmp_path_factory(): + FIXTURE_ORDER.append('my_tmp_path_factory') + + @pytest.fixture + def my_tmp_path(my_tmp_path_factory): + FIXTURE_ORDER.append('my_tmp_path') + + @pytest.fixture + def f1(my_tmp_path): + FIXTURE_ORDER.append('f1') + + @pytest.fixture + def f2(): + FIXTURE_ORDER.append('f2') + + def test_foo(f1, p1, m1, f2, s1): pass + """ + ) + items, _ = pytester.inline_genitems() + request = FixtureRequest(items[0], _ispytest=True) + # order of fixtures based on their scope and position in the parameter list + assert ( + request.fixturenames + == "s1 my_tmp_path_factory p1 m1 f1 f2 my_tmp_path".split() + ) + pytester.runpytest() + # actual fixture execution differs: dependent fixtures must be created first ("my_tmp_path") + FIXTURE_ORDER = pytest.FIXTURE_ORDER # type: ignore[attr-defined] + assert FIXTURE_ORDER == "s1 my_tmp_path_factory p1 m1 my_tmp_path f1 f2".split() + + def test_func_closure_module(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope='module') + def m1(): pass + + @pytest.fixture(scope='function') + def f1(): pass + + def test_func(f1, m1): + pass + """ + ) + items, _ = pytester.inline_genitems() + request = FixtureRequest(items[0], _ispytest=True) + assert request.fixturenames == "m1 f1".split() + + def test_func_closure_scopes_reordered(self, pytester: Pytester) -> None: + """Test ensures that fixtures are ordered by scope regardless of the order of the parameters, although + fixtures of same scope keep the declared order + """ + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope='session') + def s1(): pass + + @pytest.fixture(scope='module') + def m1(): pass + + @pytest.fixture(scope='function') + def f1(): pass + + @pytest.fixture(scope='function') + def f2(): pass + + class Test: + + @pytest.fixture(scope='class') + def c1(cls): pass + + def test_func(self, f2, f1, c1, m1, s1): + pass + """ + ) + items, _ = pytester.inline_genitems() + request = FixtureRequest(items[0], _ispytest=True) + assert request.fixturenames == "s1 m1 c1 f2 f1".split() + + def test_func_closure_same_scope_closer_root_first( + self, pytester: Pytester + ) -> None: + """Auto-use fixtures of same scope are ordered by closer-to-root first""" + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(scope='module', autouse=True) + def m_conf(): pass + """ + ) + pytester.makepyfile( + **{ + "sub/conftest.py": """ + import pytest + + @pytest.fixture(scope='package', autouse=True) + def p_sub(): pass + + @pytest.fixture(scope='module', autouse=True) + def m_sub(): pass + """, + "sub/__init__.py": "", + "sub/test_func.py": """ + import pytest + + @pytest.fixture(scope='module', autouse=True) + def m_test(): pass + + @pytest.fixture(scope='function') + def f1(): pass + + def test_func(m_test, f1): + pass + """, + } + ) + items, _ = pytester.inline_genitems() + request = FixtureRequest(items[0], _ispytest=True) + assert request.fixturenames == "p_sub m_conf m_sub m_test f1".split() + + def test_func_closure_all_scopes_complex(self, pytester: Pytester) -> None: + """Complex test involving all scopes and mixing autouse with normal fixtures""" + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(scope='session') + def s1(): pass + + @pytest.fixture(scope='package', autouse=True) + def p1(): pass + """ + ) + pytester.makepyfile(**{"__init__.py": ""}) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope='module', autouse=True) + def m1(): pass + + @pytest.fixture(scope='module') + def m2(s1): pass + + @pytest.fixture(scope='function') + def f1(): pass + + @pytest.fixture(scope='function') + def f2(): pass + + class Test: + + @pytest.fixture(scope='class', autouse=True) + def c1(self): + pass + + def test_func(self, f2, f1, m2): + pass + """ + ) + items, _ = pytester.inline_genitems() + request = FixtureRequest(items[0], _ispytest=True) + assert request.fixturenames == "s1 p1 m1 m2 c1 f2 f1".split() + + def test_multiple_packages(self, pytester: Pytester) -> None: + """Complex test involving multiple package fixtures. Make sure teardowns + are executed in order. + . + └── root + ├── __init__.py + ├── sub1 + │ ├── __init__.py + │ ├── conftest.py + │ └── test_1.py + └── sub2 + ├── __init__.py + ├── conftest.py + └── test_2.py + """ + root = pytester.mkdir("root") + root.joinpath("__init__.py").write_text("values = []") + sub1 = root.joinpath("sub1") + sub1.mkdir() + sub1.joinpath("__init__.py").touch() + sub1.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + from .. import values + @pytest.fixture(scope="package") + def fix(): + values.append("pre-sub1") + yield values + assert values.pop() == "pre-sub1" + """ + ) + ) + sub1.joinpath("test_1.py").write_text( + textwrap.dedent( + """\ + from .. import values + def test_1(fix): + assert values == ["pre-sub1"] + """ + ) + ) + sub2 = root.joinpath("sub2") + sub2.mkdir() + sub2.joinpath("__init__.py").touch() + sub2.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + from .. import values + @pytest.fixture(scope="package") + def fix(): + values.append("pre-sub2") + yield values + assert values.pop() == "pre-sub2" + """ + ) + ) + sub2.joinpath("test_2.py").write_text( + textwrap.dedent( + """\ + from .. import values + def test_2(fix): + assert values == ["pre-sub2"] + """ + ) + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + def test_class_fixture_self_instance(self, pytester: Pytester) -> None: + """Check that plugin classes which implement fixtures receive the plugin instance + as self (see #2270). + """ + pytester.makeconftest( + """ + import pytest + + def pytest_configure(config): + config.pluginmanager.register(MyPlugin()) + + class MyPlugin(): + def __init__(self): + self.arg = 1 + + @pytest.fixture(scope='function') + def myfix(self): + assert isinstance(self, MyPlugin) + return self.arg + """ + ) + + pytester.makepyfile( + """ + class TestClass(object): + def test_1(self, myfix): + assert myfix == 1 + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +def test_call_fixture_function_error(): + """Check if an error is raised if a fixture function is called directly (#4545)""" + + @pytest.fixture + def fix(): + raise NotImplementedError() + + with pytest.raises(pytest.fail.Exception): + assert fix() == 1 + + +def test_fixture_param_shadowing(pytester: Pytester) -> None: + """Parametrized arguments would be shadowed if a fixture with the same name also exists (#5036)""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(params=['a', 'b']) + def argroot(request): + return request.param + + @pytest.fixture + def arg(argroot): + return argroot + + # This should only be parametrized directly + @pytest.mark.parametrize("arg", [1]) + def test_direct(arg): + assert arg == 1 + + # This should be parametrized based on the fixtures + def test_normal_fixture(arg): + assert isinstance(arg, str) + + # Indirect should still work: + + @pytest.fixture + def arg2(request): + return 2*request.param + + @pytest.mark.parametrize("arg2", [1], indirect=True) + def test_indirect(arg2): + assert arg2 == 2 + """ + ) + # Only one test should have run + result = pytester.runpytest("-v") + result.assert_outcomes(passed=4) + result.stdout.fnmatch_lines(["*::test_direct[[]1[]]*"]) + result.stdout.fnmatch_lines(["*::test_normal_fixture[[]a[]]*"]) + result.stdout.fnmatch_lines(["*::test_normal_fixture[[]b[]]*"]) + result.stdout.fnmatch_lines(["*::test_indirect[[]1[]]*"]) + + +def test_fixture_named_request(pytester: Pytester) -> None: + pytester.copy_example("fixtures/test_fixture_named_request.py") + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*'request' is a reserved word for fixtures, use another name:", + " *test_fixture_named_request.py:5", + ] + ) + + +def test_indirect_fixture_does_not_break_scope(pytester: Pytester) -> None: + """Ensure that fixture scope is respected when using indirect fixtures (#570)""" + pytester.makepyfile( + """ + import pytest + instantiated = [] + + @pytest.fixture(scope="session") + def fixture_1(request): + instantiated.append(("fixture_1", request.param)) + + + @pytest.fixture(scope="session") + def fixture_2(request): + instantiated.append(("fixture_2", request.param)) + + + scenarios = [ + ("A", "a1"), + ("A", "a2"), + ("B", "b1"), + ("B", "b2"), + ("C", "c1"), + ("C", "c2"), + ] + + @pytest.mark.parametrize( + "fixture_1,fixture_2", scenarios, indirect=["fixture_1", "fixture_2"] + ) + def test_create_fixtures(fixture_1, fixture_2): + pass + + + def test_check_fixture_instantiations(): + assert instantiated == [ + ('fixture_1', 'A'), + ('fixture_2', 'a1'), + ('fixture_2', 'a2'), + ('fixture_1', 'B'), + ('fixture_2', 'b1'), + ('fixture_2', 'b2'), + ('fixture_1', 'C'), + ('fixture_2', 'c1'), + ('fixture_2', 'c2'), + ] + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=7) + + +def test_fixture_parametrization_nparray(pytester: Pytester) -> None: + pytest.importorskip("numpy") + + pytester.makepyfile( + """ + from numpy import linspace + from pytest import fixture + + @fixture(params=linspace(1, 10, 10)) + def value(request): + return request.param + + def test_bug(value): + assert value == value + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=10) + + +def test_fixture_arg_ordering(pytester: Pytester) -> None: + """ + This test describes how fixtures in the same scope but without explicit dependencies + between them are created. While users should make dependencies explicit, often + they rely on this order, so this test exists to catch regressions in this regard. + See #6540 and #6492. + """ + p1 = pytester.makepyfile( + """ + import pytest + + suffixes = [] + + @pytest.fixture + def fix_1(): suffixes.append("fix_1") + @pytest.fixture + def fix_2(): suffixes.append("fix_2") + @pytest.fixture + def fix_3(): suffixes.append("fix_3") + @pytest.fixture + def fix_4(): suffixes.append("fix_4") + @pytest.fixture + def fix_5(): suffixes.append("fix_5") + + @pytest.fixture + def fix_combined(fix_1, fix_2, fix_3, fix_4, fix_5): pass + + def test_suffix(fix_combined): + assert suffixes == ["fix_1", "fix_2", "fix_3", "fix_4", "fix_5"] + """ + ) + result = pytester.runpytest("-vv", str(p1)) + assert result.ret == 0 + + +def test_yield_fixture_with_no_value(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(name='custom') + def empty_yield(): + if False: + yield + + def test_fixt(custom): + pass + """ + ) + expected = "E ValueError: custom did not yield a value" + result = pytester.runpytest() + result.assert_outcomes(errors=1) + result.stdout.fnmatch_lines([expected]) + assert result.ret == ExitCode.TESTS_FAILED diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/python/integration.py b/testing/web-platform/tests/tools/third_party/pytest/testing/python/integration.py new file mode 100644 index 0000000000..d138b72663 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/python/integration.py @@ -0,0 +1,503 @@ +from typing import Any + +import pytest +from _pytest import runner +from _pytest._code import getfslineno +from _pytest.fixtures import getfixturemarker +from _pytest.pytester import Pytester +from _pytest.python import Function + + +class TestOEJSKITSpecials: + def test_funcarg_non_pycollectobj( + self, pytester: Pytester, recwarn + ) -> None: # rough jstests usage + pytester.makeconftest( + """ + import pytest + def pytest_pycollect_makeitem(collector, name, obj): + if name == "MyClass": + return MyCollector.from_parent(collector, name=name) + class MyCollector(pytest.Collector): + def reportinfo(self): + return self.path, 3, "xyz" + """ + ) + modcol = pytester.getmodulecol( + """ + import pytest + @pytest.fixture + def arg1(request): + return 42 + class MyClass(object): + pass + """ + ) + # this hook finds funcarg factories + rep = runner.collect_one_node(collector=modcol) + # TODO: Don't treat as Any. + clscol: Any = rep.result[0] + clscol.obj = lambda arg1: None + clscol.funcargs = {} + pytest._fillfuncargs(clscol) + assert clscol.funcargs["arg1"] == 42 + + def test_autouse_fixture( + self, pytester: Pytester, recwarn + ) -> None: # rough jstests usage + pytester.makeconftest( + """ + import pytest + def pytest_pycollect_makeitem(collector, name, obj): + if name == "MyClass": + return MyCollector.from_parent(collector, name=name) + class MyCollector(pytest.Collector): + def reportinfo(self): + return self.path, 3, "xyz" + """ + ) + modcol = pytester.getmodulecol( + """ + import pytest + @pytest.fixture(autouse=True) + def hello(): + pass + @pytest.fixture + def arg1(request): + return 42 + class MyClass(object): + pass + """ + ) + # this hook finds funcarg factories + rep = runner.collect_one_node(modcol) + # TODO: Don't treat as Any. + clscol: Any = rep.result[0] + clscol.obj = lambda: None + clscol.funcargs = {} + pytest._fillfuncargs(clscol) + assert not clscol.funcargs + + +def test_wrapped_getfslineno() -> None: + def func(): + pass + + def wrap(f): + func.__wrapped__ = f # type: ignore + func.patchings = ["qwe"] # type: ignore + return func + + @wrap + def wrapped_func(x, y, z): + pass + + fs, lineno = getfslineno(wrapped_func) + fs2, lineno2 = getfslineno(wrap) + assert lineno > lineno2, "getfslineno does not unwrap correctly" + + +class TestMockDecoration: + def test_wrapped_getfuncargnames(self) -> None: + from _pytest.compat import getfuncargnames + + def wrap(f): + def func(): + pass + + func.__wrapped__ = f # type: ignore + return func + + @wrap + def f(x): + pass + + values = getfuncargnames(f) + assert values == ("x",) + + def test_getfuncargnames_patching(self): + from _pytest.compat import getfuncargnames + from unittest.mock import patch + + class T: + def original(self, x, y, z): + pass + + @patch.object(T, "original") + def f(x, y, z): + pass + + values = getfuncargnames(f) + assert values == ("y", "z") + + def test_unittest_mock(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import unittest.mock + class T(unittest.TestCase): + @unittest.mock.patch("os.path.abspath") + def test_hello(self, abspath): + import os + os.path.abspath("hello") + abspath.assert_any_call("hello") + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_unittest_mock_and_fixture(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import os.path + import unittest.mock + import pytest + + @pytest.fixture + def inject_me(): + pass + + @unittest.mock.patch.object(os.path, "abspath", + new=unittest.mock.MagicMock) + def test_hello(inject_me): + import os + os.path.abspath("hello") + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_unittest_mock_and_pypi_mock(self, pytester: Pytester) -> None: + pytest.importorskip("mock", "1.0.1") + pytester.makepyfile( + """ + import mock + import unittest.mock + class TestBoth(object): + @unittest.mock.patch("os.path.abspath") + def test_hello(self, abspath): + import os + os.path.abspath("hello") + abspath.assert_any_call("hello") + + @mock.patch("os.path.abspath") + def test_hello_mock(self, abspath): + import os + os.path.abspath("hello") + abspath.assert_any_call("hello") + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + def test_mock_sentinel_check_against_numpy_like(self, pytester: Pytester) -> None: + """Ensure our function that detects mock arguments compares against sentinels using + identity to circumvent objects which can't be compared with equality against others + in a truth context, like with numpy arrays (#5606). + """ + pytester.makepyfile( + dummy=""" + class NumpyLike: + def __init__(self, value): + self.value = value + def __eq__(self, other): + raise ValueError("like numpy, cannot compare against others for truth") + FOO = NumpyLike(10) + """ + ) + pytester.makepyfile( + """ + from unittest.mock import patch + import dummy + class Test(object): + @patch("dummy.FOO", new=dummy.NumpyLike(50)) + def test_hello(self): + assert dummy.FOO.value == 50 + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_mock(self, pytester: Pytester) -> None: + pytest.importorskip("mock", "1.0.1") + pytester.makepyfile( + """ + import os + import unittest + import mock + + class T(unittest.TestCase): + @mock.patch("os.path.abspath") + def test_hello(self, abspath): + os.path.abspath("hello") + abspath.assert_any_call("hello") + def mock_basename(path): + return "mock_basename" + @mock.patch("os.path.abspath") + @mock.patch("os.path.normpath") + @mock.patch("os.path.basename", new=mock_basename) + def test_someting(normpath, abspath, tmp_path): + abspath.return_value = "this" + os.path.normpath(os.path.abspath("hello")) + normpath.assert_any_call("this") + assert os.path.basename("123") == "mock_basename" + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + calls = reprec.getcalls("pytest_runtest_logreport") + funcnames = [ + call.report.location[2] for call in calls if call.report.when == "call" + ] + assert funcnames == ["T.test_hello", "test_someting"] + + def test_mock_sorting(self, pytester: Pytester) -> None: + pytest.importorskip("mock", "1.0.1") + pytester.makepyfile( + """ + import os + import mock + + @mock.patch("os.path.abspath") + def test_one(abspath): + pass + @mock.patch("os.path.abspath") + def test_two(abspath): + pass + @mock.patch("os.path.abspath") + def test_three(abspath): + pass + """ + ) + reprec = pytester.inline_run() + calls = reprec.getreports("pytest_runtest_logreport") + calls = [x for x in calls if x.when == "call"] + names = [x.nodeid.split("::")[-1] for x in calls] + assert names == ["test_one", "test_two", "test_three"] + + def test_mock_double_patch_issue473(self, pytester: Pytester) -> None: + pytest.importorskip("mock", "1.0.1") + pytester.makepyfile( + """ + from mock import patch + from pytest import mark + + @patch('os.getcwd') + @patch('os.path') + @mark.slow + class TestSimple(object): + def test_simple_thing(self, mock_path, mock_getcwd): + pass + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +class TestReRunTests: + def test_rerun(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + from _pytest.runner import runtestprotocol + def pytest_runtest_protocol(item, nextitem): + runtestprotocol(item, log=False, nextitem=nextitem) + runtestprotocol(item, log=True, nextitem=nextitem) + """ + ) + pytester.makepyfile( + """ + import pytest + count = 0 + req = None + @pytest.fixture + def fix(request): + global count, req + assert request != req + req = request + print("fix count %s" % count) + count += 1 + def test_fix(fix): + pass + """ + ) + result = pytester.runpytest("-s") + result.stdout.fnmatch_lines( + """ + *fix count 0* + *fix count 1* + """ + ) + result.stdout.fnmatch_lines( + """ + *2 passed* + """ + ) + + +def test_pytestconfig_is_session_scoped() -> None: + from _pytest.fixtures import pytestconfig + + marker = getfixturemarker(pytestconfig) + assert marker is not None + assert marker.scope == "session" + + +class TestNoselikeTestAttribute: + def test_module_with_global_test(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + __test__ = False + def test_hello(): + pass + """ + ) + reprec = pytester.inline_run() + assert not reprec.getfailedcollections() + calls = reprec.getreports("pytest_runtest_logreport") + assert not calls + + def test_class_and_method(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + __test__ = True + def test_func(): + pass + test_func.__test__ = False + + class TestSome(object): + __test__ = False + def test_method(self): + pass + """ + ) + reprec = pytester.inline_run() + assert not reprec.getfailedcollections() + calls = reprec.getreports("pytest_runtest_logreport") + assert not calls + + def test_unittest_class(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import unittest + class TC(unittest.TestCase): + def test_1(self): + pass + class TC2(unittest.TestCase): + __test__ = False + def test_2(self): + pass + """ + ) + reprec = pytester.inline_run() + assert not reprec.getfailedcollections() + call = reprec.getcalls("pytest_collection_modifyitems")[0] + assert len(call.items) == 1 + assert call.items[0].cls.__name__ == "TC" + + def test_class_with_nasty_getattr(self, pytester: Pytester) -> None: + """Make sure we handle classes with a custom nasty __getattr__ right. + + With a custom __getattr__ which e.g. returns a function (like with a + RPC wrapper), we shouldn't assume this meant "__test__ = True". + """ + # https://github.com/pytest-dev/pytest/issues/1204 + pytester.makepyfile( + """ + class MetaModel(type): + + def __getattr__(cls, key): + return lambda: None + + + BaseModel = MetaModel('Model', (), {}) + + + class Model(BaseModel): + + __metaclass__ = MetaModel + + def test_blah(self): + pass + """ + ) + reprec = pytester.inline_run() + assert not reprec.getfailedcollections() + call = reprec.getcalls("pytest_collection_modifyitems")[0] + assert not call.items + + +class TestParameterize: + """#351""" + + def test_idfn_marker(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + def idfn(param): + if param == 0: + return 'spam' + elif param == 1: + return 'ham' + else: + return None + + @pytest.mark.parametrize('a,b', [(0, 2), (1, 2)], ids=idfn) + def test_params(a, b): + pass + """ + ) + res = pytester.runpytest("--collect-only") + res.stdout.fnmatch_lines(["*spam-2*", "*ham-2*"]) + + def test_idfn_fixture(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + def idfn(param): + if param == 0: + return 'spam' + elif param == 1: + return 'ham' + else: + return None + + @pytest.fixture(params=[0, 1], ids=idfn) + def a(request): + return request.param + + @pytest.fixture(params=[1, 2], ids=idfn) + def b(request): + return request.param + + def test_params(a, b): + pass + """ + ) + res = pytester.runpytest("--collect-only") + res.stdout.fnmatch_lines(["*spam-2*", "*ham-2*"]) + + +def test_function_instance(pytester: Pytester) -> None: + items = pytester.getitems( + """ + def test_func(): pass + class TestIt: + def test_method(self): pass + @classmethod + def test_class(cls): pass + @staticmethod + def test_static(): pass + """ + ) + assert len(items) == 3 + assert isinstance(items[0], Function) + assert items[0].name == "test_func" + assert items[0].instance is None + assert isinstance(items[1], Function) + assert items[1].name == "test_method" + assert items[1].instance is not None + assert items[1].instance.__class__.__name__ == "TestIt" + assert isinstance(items[2], Function) + assert items[2].name == "test_static" + assert items[2].instance is None diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/python/metafunc.py b/testing/web-platform/tests/tools/third_party/pytest/testing/python/metafunc.py new file mode 100644 index 0000000000..fc0082eb6b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/python/metafunc.py @@ -0,0 +1,1907 @@ +import itertools +import re +import sys +import textwrap +from typing import Any +from typing import cast +from typing import Dict +from typing import Iterator +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Union + +import attr +import hypothesis +from hypothesis import strategies + +import pytest +from _pytest import fixtures +from _pytest import python +from _pytest.compat import _format_args +from _pytest.compat import getfuncargnames +from _pytest.compat import NOTSET +from _pytest.outcomes import fail +from _pytest.pytester import Pytester +from _pytest.python import _idval +from _pytest.python import idmaker +from _pytest.scope import Scope + + +class TestMetafunc: + def Metafunc(self, func, config=None) -> python.Metafunc: + # The unit tests of this class check if things work correctly + # on the funcarg level, so we don't need a full blown + # initialization. + class FuncFixtureInfoMock: + name2fixturedefs = None + + def __init__(self, names): + self.names_closure = names + + @attr.s + class DefinitionMock(python.FunctionDefinition): + obj = attr.ib() + _nodeid = attr.ib() + + names = getfuncargnames(func) + fixtureinfo: Any = FuncFixtureInfoMock(names) + definition: Any = DefinitionMock._create(func, "mock::nodeid") + return python.Metafunc(definition, fixtureinfo, config, _ispytest=True) + + def test_no_funcargs(self) -> None: + def function(): + pass + + metafunc = self.Metafunc(function) + assert not metafunc.fixturenames + repr(metafunc._calls) + + def test_function_basic(self) -> None: + def func(arg1, arg2="qwe"): + pass + + metafunc = self.Metafunc(func) + assert len(metafunc.fixturenames) == 1 + assert "arg1" in metafunc.fixturenames + assert metafunc.function is func + assert metafunc.cls is None + + def test_parametrize_error(self) -> None: + def func(x, y): + pass + + metafunc = self.Metafunc(func) + metafunc.parametrize("x", [1, 2]) + pytest.raises(ValueError, lambda: metafunc.parametrize("x", [5, 6])) + pytest.raises(ValueError, lambda: metafunc.parametrize("x", [5, 6])) + metafunc.parametrize("y", [1, 2]) + pytest.raises(ValueError, lambda: metafunc.parametrize("y", [5, 6])) + pytest.raises(ValueError, lambda: metafunc.parametrize("y", [5, 6])) + + with pytest.raises(TypeError, match="^ids must be a callable or an iterable$"): + metafunc.parametrize("y", [5, 6], ids=42) # type: ignore[arg-type] + + def test_parametrize_error_iterator(self) -> None: + def func(x): + raise NotImplementedError() + + class Exc(Exception): + def __repr__(self): + return "Exc(from_gen)" + + def gen() -> Iterator[Union[int, None, Exc]]: + yield 0 + yield None + yield Exc() + + metafunc = self.Metafunc(func) + # When the input is an iterator, only len(args) are taken, + # so the bad Exc isn't reached. + metafunc.parametrize("x", [1, 2], ids=gen()) # type: ignore[arg-type] + assert [(x.funcargs, x.id) for x in metafunc._calls] == [ + ({"x": 1}, "0"), + ({"x": 2}, "2"), + ] + with pytest.raises( + fail.Exception, + match=( + r"In func: ids must be list of string/float/int/bool, found:" + r" Exc\(from_gen\) \(type: \) at index 2" + ), + ): + metafunc.parametrize("x", [1, 2, 3], ids=gen()) # type: ignore[arg-type] + + def test_parametrize_bad_scope(self) -> None: + def func(x): + pass + + metafunc = self.Metafunc(func) + with pytest.raises( + fail.Exception, + match=r"parametrize\(\) call in func got an unexpected scope value 'doggy'", + ): + metafunc.parametrize("x", [1], scope="doggy") # type: ignore[arg-type] + + def test_parametrize_request_name(self, pytester: Pytester) -> None: + """Show proper error when 'request' is used as a parameter name in parametrize (#6183)""" + + def func(request): + raise NotImplementedError() + + metafunc = self.Metafunc(func) + with pytest.raises( + fail.Exception, + match=r"'request' is a reserved name and cannot be used in @pytest.mark.parametrize", + ): + metafunc.parametrize("request", [1]) + + def test_find_parametrized_scope(self) -> None: + """Unit test for _find_parametrized_scope (#3941).""" + from _pytest.python import _find_parametrized_scope + + @attr.s + class DummyFixtureDef: + _scope = attr.ib() + + fixtures_defs = cast( + Dict[str, Sequence[fixtures.FixtureDef[object]]], + dict( + session_fix=[DummyFixtureDef(Scope.Session)], + package_fix=[DummyFixtureDef(Scope.Package)], + module_fix=[DummyFixtureDef(Scope.Module)], + class_fix=[DummyFixtureDef(Scope.Class)], + func_fix=[DummyFixtureDef(Scope.Function)], + ), + ) + + # use arguments to determine narrow scope; the cause of the bug is that it would look on all + # fixture defs given to the method + def find_scope(argnames, indirect): + return _find_parametrized_scope(argnames, fixtures_defs, indirect=indirect) + + assert find_scope(["func_fix"], indirect=True) == Scope.Function + assert find_scope(["class_fix"], indirect=True) == Scope.Class + assert find_scope(["module_fix"], indirect=True) == Scope.Module + assert find_scope(["package_fix"], indirect=True) == Scope.Package + assert find_scope(["session_fix"], indirect=True) == Scope.Session + + assert find_scope(["class_fix", "func_fix"], indirect=True) == Scope.Function + assert find_scope(["func_fix", "session_fix"], indirect=True) == Scope.Function + assert find_scope(["session_fix", "class_fix"], indirect=True) == Scope.Class + assert ( + find_scope(["package_fix", "session_fix"], indirect=True) == Scope.Package + ) + assert find_scope(["module_fix", "session_fix"], indirect=True) == Scope.Module + + # when indirect is False or is not for all scopes, always use function + assert ( + find_scope(["session_fix", "module_fix"], indirect=False) == Scope.Function + ) + assert ( + find_scope(["session_fix", "module_fix"], indirect=["module_fix"]) + == Scope.Function + ) + assert ( + find_scope( + ["session_fix", "module_fix"], indirect=["session_fix", "module_fix"] + ) + == Scope.Module + ) + + def test_parametrize_and_id(self) -> None: + def func(x, y): + pass + + metafunc = self.Metafunc(func) + + metafunc.parametrize("x", [1, 2], ids=["basic", "advanced"]) + metafunc.parametrize("y", ["abc", "def"]) + ids = [x.id for x in metafunc._calls] + assert ids == ["basic-abc", "basic-def", "advanced-abc", "advanced-def"] + + def test_parametrize_and_id_unicode(self) -> None: + """Allow unicode strings for "ids" parameter in Python 2 (##1905)""" + + def func(x): + pass + + metafunc = self.Metafunc(func) + metafunc.parametrize("x", [1, 2], ids=["basic", "advanced"]) + ids = [x.id for x in metafunc._calls] + assert ids == ["basic", "advanced"] + + def test_parametrize_with_wrong_number_of_ids(self) -> None: + def func(x, y): + pass + + metafunc = self.Metafunc(func) + + with pytest.raises(fail.Exception): + metafunc.parametrize("x", [1, 2], ids=["basic"]) + + with pytest.raises(fail.Exception): + metafunc.parametrize( + ("x", "y"), [("abc", "def"), ("ghi", "jkl")], ids=["one"] + ) + + def test_parametrize_ids_iterator_without_mark(self) -> None: + def func(x, y): + pass + + it = itertools.count() + + metafunc = self.Metafunc(func) + metafunc.parametrize("x", [1, 2], ids=it) + metafunc.parametrize("y", [3, 4], ids=it) + ids = [x.id for x in metafunc._calls] + assert ids == ["0-2", "0-3", "1-2", "1-3"] + + metafunc = self.Metafunc(func) + metafunc.parametrize("x", [1, 2], ids=it) + metafunc.parametrize("y", [3, 4], ids=it) + ids = [x.id for x in metafunc._calls] + assert ids == ["4-6", "4-7", "5-6", "5-7"] + + def test_parametrize_empty_list(self) -> None: + """#510""" + + def func(y): + pass + + class MockConfig: + def getini(self, name): + return "" + + @property + def hook(self): + return self + + def pytest_make_parametrize_id(self, **kw): + pass + + metafunc = self.Metafunc(func, MockConfig()) + metafunc.parametrize("y", []) + assert "skip" == metafunc._calls[0].marks[0].name + + def test_parametrize_with_userobjects(self) -> None: + def func(x, y): + pass + + metafunc = self.Metafunc(func) + + class A: + pass + + metafunc.parametrize("x", [A(), A()]) + metafunc.parametrize("y", list("ab")) + assert metafunc._calls[0].id == "x0-a" + assert metafunc._calls[1].id == "x0-b" + assert metafunc._calls[2].id == "x1-a" + assert metafunc._calls[3].id == "x1-b" + + @hypothesis.given(strategies.text() | strategies.binary()) + @hypothesis.settings( + deadline=400.0 + ) # very close to std deadline and CI boxes are not reliable in CPU power + def test_idval_hypothesis(self, value) -> None: + escaped = _idval(value, "a", 6, None, nodeid=None, config=None) + assert isinstance(escaped, str) + escaped.encode("ascii") + + def test_unicode_idval(self) -> None: + """Test that Unicode strings outside the ASCII character set get + escaped, using byte escapes if they're in that range or unicode + escapes if they're not. + + """ + values = [ + ("", r""), + ("ascii", r"ascii"), + ("ação", r"a\xe7\xe3o"), + ("josé@blah.com", r"jos\xe9@blah.com"), + ( + r"δοκ.ιμή@παράδειγμα.δοκιμή", + r"\u03b4\u03bf\u03ba.\u03b9\u03bc\u03ae@\u03c0\u03b1\u03c1\u03ac\u03b4\u03b5\u03b9\u03b3" + r"\u03bc\u03b1.\u03b4\u03bf\u03ba\u03b9\u03bc\u03ae", + ), + ] + for val, expected in values: + assert _idval(val, "a", 6, None, nodeid=None, config=None) == expected + + def test_unicode_idval_with_config(self) -> None: + """Unit test for expected behavior to obtain ids with + disable_test_id_escaping_and_forfeit_all_rights_to_community_support + option (#5294).""" + + class MockConfig: + def __init__(self, config): + self.config = config + + @property + def hook(self): + return self + + def pytest_make_parametrize_id(self, **kw): + pass + + def getini(self, name): + return self.config[name] + + option = "disable_test_id_escaping_and_forfeit_all_rights_to_community_support" + + values: List[Tuple[str, Any, str]] = [ + ("ação", MockConfig({option: True}), "ação"), + ("ação", MockConfig({option: False}), "a\\xe7\\xe3o"), + ] + for val, config, expected in values: + actual = _idval(val, "a", 6, None, nodeid=None, config=config) + assert actual == expected + + def test_bytes_idval(self) -> None: + """Unit test for the expected behavior to obtain ids for parametrized + bytes values: bytes objects are always escaped using "binary escape".""" + values = [ + (b"", r""), + (b"\xc3\xb4\xff\xe4", r"\xc3\xb4\xff\xe4"), + (b"ascii", r"ascii"), + ("αρά".encode(), r"\xce\xb1\xcf\x81\xce\xac"), + ] + for val, expected in values: + assert _idval(val, "a", 6, idfn=None, nodeid=None, config=None) == expected + + def test_class_or_function_idval(self) -> None: + """Unit test for the expected behavior to obtain ids for parametrized + values that are classes or functions: their __name__.""" + + class TestClass: + pass + + def test_function(): + pass + + values = [(TestClass, "TestClass"), (test_function, "test_function")] + for val, expected in values: + assert _idval(val, "a", 6, None, nodeid=None, config=None) == expected + + def test_notset_idval(self) -> None: + """Test that a NOTSET value (used by an empty parameterset) generates + a proper ID. + + Regression test for #7686. + """ + assert _idval(NOTSET, "a", 0, None, nodeid=None, config=None) == "a0" + + def test_idmaker_autoname(self) -> None: + """#250""" + result = idmaker( + ("a", "b"), [pytest.param("string", 1.0), pytest.param("st-ring", 2.0)] + ) + assert result == ["string-1.0", "st-ring-2.0"] + + result = idmaker( + ("a", "b"), [pytest.param(object(), 1.0), pytest.param(object(), object())] + ) + assert result == ["a0-1.0", "a1-b1"] + # unicode mixing, issue250 + result = idmaker(("a", "b"), [pytest.param({}, b"\xc3\xb4")]) + assert result == ["a0-\\xc3\\xb4"] + + def test_idmaker_with_bytes_regex(self) -> None: + result = idmaker(("a"), [pytest.param(re.compile(b"foo"), 1.0)]) + assert result == ["foo"] + + def test_idmaker_native_strings(self) -> None: + result = idmaker( + ("a", "b"), + [ + pytest.param(1.0, -1.1), + pytest.param(2, -202), + pytest.param("three", "three hundred"), + pytest.param(True, False), + pytest.param(None, None), + pytest.param(re.compile("foo"), re.compile("bar")), + pytest.param(str, int), + pytest.param(list("six"), [66, 66]), + pytest.param({7}, set("seven")), + pytest.param(tuple("eight"), (8, -8, 8)), + pytest.param(b"\xc3\xb4", b"name"), + pytest.param(b"\xc3\xb4", "other"), + pytest.param(1.0j, -2.0j), + ], + ) + assert result == [ + "1.0--1.1", + "2--202", + "three-three hundred", + "True-False", + "None-None", + "foo-bar", + "str-int", + "a7-b7", + "a8-b8", + "a9-b9", + "\\xc3\\xb4-name", + "\\xc3\\xb4-other", + "1j-(-0-2j)", + ] + + def test_idmaker_non_printable_characters(self) -> None: + result = idmaker( + ("s", "n"), + [ + pytest.param("\x00", 1), + pytest.param("\x05", 2), + pytest.param(b"\x00", 3), + pytest.param(b"\x05", 4), + pytest.param("\t", 5), + pytest.param(b"\t", 6), + ], + ) + assert result == ["\\x00-1", "\\x05-2", "\\x00-3", "\\x05-4", "\\t-5", "\\t-6"] + + def test_idmaker_manual_ids_must_be_printable(self) -> None: + result = idmaker( + ("s",), + [ + pytest.param("x00", id="hello \x00"), + pytest.param("x05", id="hello \x05"), + ], + ) + assert result == ["hello \\x00", "hello \\x05"] + + def test_idmaker_enum(self) -> None: + enum = pytest.importorskip("enum") + e = enum.Enum("Foo", "one, two") + result = idmaker(("a", "b"), [pytest.param(e.one, e.two)]) + assert result == ["Foo.one-Foo.two"] + + def test_idmaker_idfn(self) -> None: + """#351""" + + def ids(val: object) -> Optional[str]: + if isinstance(val, Exception): + return repr(val) + return None + + result = idmaker( + ("a", "b"), + [ + pytest.param(10.0, IndexError()), + pytest.param(20, KeyError()), + pytest.param("three", [1, 2, 3]), + ], + idfn=ids, + ) + assert result == ["10.0-IndexError()", "20-KeyError()", "three-b2"] + + def test_idmaker_idfn_unique_names(self) -> None: + """#351""" + + def ids(val: object) -> str: + return "a" + + result = idmaker( + ("a", "b"), + [ + pytest.param(10.0, IndexError()), + pytest.param(20, KeyError()), + pytest.param("three", [1, 2, 3]), + ], + idfn=ids, + ) + assert result == ["a-a0", "a-a1", "a-a2"] + + def test_idmaker_with_idfn_and_config(self) -> None: + """Unit test for expected behavior to create ids with idfn and + disable_test_id_escaping_and_forfeit_all_rights_to_community_support + option (#5294). + """ + + class MockConfig: + def __init__(self, config): + self.config = config + + @property + def hook(self): + return self + + def pytest_make_parametrize_id(self, **kw): + pass + + def getini(self, name): + return self.config[name] + + option = "disable_test_id_escaping_and_forfeit_all_rights_to_community_support" + + values: List[Tuple[Any, str]] = [ + (MockConfig({option: True}), "ação"), + (MockConfig({option: False}), "a\\xe7\\xe3o"), + ] + for config, expected in values: + result = idmaker( + ("a",), + [pytest.param("string")], + idfn=lambda _: "ação", + config=config, + ) + assert result == [expected] + + def test_idmaker_with_ids_and_config(self) -> None: + """Unit test for expected behavior to create ids with ids and + disable_test_id_escaping_and_forfeit_all_rights_to_community_support + option (#5294). + """ + + class MockConfig: + def __init__(self, config): + self.config = config + + @property + def hook(self): + return self + + def pytest_make_parametrize_id(self, **kw): + pass + + def getini(self, name): + return self.config[name] + + option = "disable_test_id_escaping_and_forfeit_all_rights_to_community_support" + + values: List[Tuple[Any, str]] = [ + (MockConfig({option: True}), "ação"), + (MockConfig({option: False}), "a\\xe7\\xe3o"), + ] + for config, expected in values: + result = idmaker( + ("a",), + [pytest.param("string")], + ids=["ação"], + config=config, + ) + assert result == [expected] + + def test_parametrize_ids_exception(self, pytester: Pytester) -> None: + """ + :param pytester: the instance of Pytester class, a temporary + test directory. + """ + pytester.makepyfile( + """ + import pytest + + def ids(arg): + raise Exception("bad ids") + + @pytest.mark.parametrize("arg", ["a", "b"], ids=ids) + def test_foo(arg): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*Exception: bad ids", + "*test_foo: error raised while trying to determine id of parameter 'arg' at position 0", + ] + ) + + def test_parametrize_ids_returns_non_string(self, pytester: Pytester) -> None: + pytester.makepyfile( + """\ + import pytest + + def ids(d): + return d + + @pytest.mark.parametrize("arg", ({1: 2}, {3, 4}), ids=ids) + def test(arg): + assert arg + + @pytest.mark.parametrize("arg", (1, 2.0, True), ids=ids) + def test_int(arg): + assert arg + """ + ) + result = pytester.runpytest("-vv", "-s") + result.stdout.fnmatch_lines( + [ + "test_parametrize_ids_returns_non_string.py::test[arg0] PASSED", + "test_parametrize_ids_returns_non_string.py::test[arg1] PASSED", + "test_parametrize_ids_returns_non_string.py::test_int[1] PASSED", + "test_parametrize_ids_returns_non_string.py::test_int[2.0] PASSED", + "test_parametrize_ids_returns_non_string.py::test_int[True] PASSED", + ] + ) + + def test_idmaker_with_ids(self) -> None: + result = idmaker( + ("a", "b"), [pytest.param(1, 2), pytest.param(3, 4)], ids=["a", None] + ) + assert result == ["a", "3-4"] + + def test_idmaker_with_paramset_id(self) -> None: + result = idmaker( + ("a", "b"), + [pytest.param(1, 2, id="me"), pytest.param(3, 4, id="you")], + ids=["a", None], + ) + assert result == ["me", "you"] + + def test_idmaker_with_ids_unique_names(self) -> None: + result = idmaker( + ("a"), map(pytest.param, [1, 2, 3, 4, 5]), ids=["a", "a", "b", "c", "b"] + ) + assert result == ["a0", "a1", "b0", "c", "b1"] + + def test_parametrize_indirect(self) -> None: + """#714""" + + def func(x, y): + pass + + metafunc = self.Metafunc(func) + metafunc.parametrize("x", [1], indirect=True) + metafunc.parametrize("y", [2, 3], indirect=True) + assert len(metafunc._calls) == 2 + assert metafunc._calls[0].funcargs == {} + assert metafunc._calls[1].funcargs == {} + assert metafunc._calls[0].params == dict(x=1, y=2) + assert metafunc._calls[1].params == dict(x=1, y=3) + + def test_parametrize_indirect_list(self) -> None: + """#714""" + + def func(x, y): + pass + + metafunc = self.Metafunc(func) + metafunc.parametrize("x, y", [("a", "b")], indirect=["x"]) + assert metafunc._calls[0].funcargs == dict(y="b") + assert metafunc._calls[0].params == dict(x="a") + + def test_parametrize_indirect_list_all(self) -> None: + """#714""" + + def func(x, y): + pass + + metafunc = self.Metafunc(func) + metafunc.parametrize("x, y", [("a", "b")], indirect=["x", "y"]) + assert metafunc._calls[0].funcargs == {} + assert metafunc._calls[0].params == dict(x="a", y="b") + + def test_parametrize_indirect_list_empty(self) -> None: + """#714""" + + def func(x, y): + pass + + metafunc = self.Metafunc(func) + metafunc.parametrize("x, y", [("a", "b")], indirect=[]) + assert metafunc._calls[0].funcargs == dict(x="a", y="b") + assert metafunc._calls[0].params == {} + + def test_parametrize_indirect_wrong_type(self) -> None: + def func(x, y): + pass + + metafunc = self.Metafunc(func) + with pytest.raises( + fail.Exception, + match="In func: expected Sequence or boolean for indirect, got dict", + ): + metafunc.parametrize("x, y", [("a", "b")], indirect={}) # type: ignore[arg-type] + + def test_parametrize_indirect_list_functional(self, pytester: Pytester) -> None: + """ + #714 + Test parametrization with 'indirect' parameter applied on + particular arguments. As y is direct, its value should + be used directly rather than being passed to the fixture y. + + :param pytester: the instance of Pytester class, a temporary + test directory. + """ + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope='function') + def x(request): + return request.param * 3 + @pytest.fixture(scope='function') + def y(request): + return request.param * 2 + @pytest.mark.parametrize('x, y', [('a', 'b')], indirect=['x']) + def test_simple(x,y): + assert len(x) == 3 + assert len(y) == 1 + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines(["*test_simple*a-b*", "*1 passed*"]) + + def test_parametrize_indirect_list_error(self) -> None: + """#714""" + + def func(x, y): + pass + + metafunc = self.Metafunc(func) + with pytest.raises(fail.Exception): + metafunc.parametrize("x, y", [("a", "b")], indirect=["x", "z"]) + + def test_parametrize_uses_no_fixture_error_indirect_false( + self, pytester: Pytester + ) -> None: + """The 'uses no fixture' error tells the user at collection time + that the parametrize data they've set up doesn't correspond to the + fixtures in their test function, rather than silently ignoring this + and letting the test potentially pass. + + #714 + """ + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize('x, y', [('a', 'b')], indirect=False) + def test_simple(x): + assert len(x) == 3 + """ + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines(["*uses no argument 'y'*"]) + + def test_parametrize_uses_no_fixture_error_indirect_true( + self, pytester: Pytester + ) -> None: + """#714""" + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope='function') + def x(request): + return request.param * 3 + @pytest.fixture(scope='function') + def y(request): + return request.param * 2 + + @pytest.mark.parametrize('x, y', [('a', 'b')], indirect=True) + def test_simple(x): + assert len(x) == 3 + """ + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines(["*uses no fixture 'y'*"]) + + def test_parametrize_indirect_uses_no_fixture_error_indirect_string( + self, pytester: Pytester + ) -> None: + """#714""" + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope='function') + def x(request): + return request.param * 3 + + @pytest.mark.parametrize('x, y', [('a', 'b')], indirect='y') + def test_simple(x): + assert len(x) == 3 + """ + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines(["*uses no fixture 'y'*"]) + + def test_parametrize_indirect_uses_no_fixture_error_indirect_list( + self, pytester: Pytester + ) -> None: + """#714""" + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope='function') + def x(request): + return request.param * 3 + + @pytest.mark.parametrize('x, y', [('a', 'b')], indirect=['y']) + def test_simple(x): + assert len(x) == 3 + """ + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines(["*uses no fixture 'y'*"]) + + def test_parametrize_argument_not_in_indirect_list( + self, pytester: Pytester + ) -> None: + """#714""" + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope='function') + def x(request): + return request.param * 3 + + @pytest.mark.parametrize('x, y', [('a', 'b')], indirect=['x']) + def test_simple(x): + assert len(x) == 3 + """ + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines(["*uses no argument 'y'*"]) + + def test_parametrize_gives_indicative_error_on_function_with_default_argument( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize('x, y', [('a', 'b')]) + def test_simple(x, y=1): + assert len(x) == 1 + """ + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines( + ["*already takes an argument 'y' with a default value"] + ) + + def test_parametrize_functional(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + def pytest_generate_tests(metafunc): + metafunc.parametrize('x', [1,2], indirect=True) + metafunc.parametrize('y', [2]) + @pytest.fixture + def x(request): + return request.param * 10 + + def test_simple(x,y): + assert x in (10,20) + assert y == 2 + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + ["*test_simple*1-2*", "*test_simple*2-2*", "*2 passed*"] + ) + + def test_parametrize_onearg(self) -> None: + metafunc = self.Metafunc(lambda x: None) + metafunc.parametrize("x", [1, 2]) + assert len(metafunc._calls) == 2 + assert metafunc._calls[0].funcargs == dict(x=1) + assert metafunc._calls[0].id == "1" + assert metafunc._calls[1].funcargs == dict(x=2) + assert metafunc._calls[1].id == "2" + + def test_parametrize_onearg_indirect(self) -> None: + metafunc = self.Metafunc(lambda x: None) + metafunc.parametrize("x", [1, 2], indirect=True) + assert metafunc._calls[0].params == dict(x=1) + assert metafunc._calls[0].id == "1" + assert metafunc._calls[1].params == dict(x=2) + assert metafunc._calls[1].id == "2" + + def test_parametrize_twoargs(self) -> None: + metafunc = self.Metafunc(lambda x, y: None) + metafunc.parametrize(("x", "y"), [(1, 2), (3, 4)]) + assert len(metafunc._calls) == 2 + assert metafunc._calls[0].funcargs == dict(x=1, y=2) + assert metafunc._calls[0].id == "1-2" + assert metafunc._calls[1].funcargs == dict(x=3, y=4) + assert metafunc._calls[1].id == "3-4" + + def test_parametrize_multiple_times(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + pytestmark = pytest.mark.parametrize("x", [1,2]) + def test_func(x): + assert 0, x + class TestClass(object): + pytestmark = pytest.mark.parametrize("y", [3,4]) + def test_meth(self, x, y): + assert 0, x + """ + ) + result = pytester.runpytest() + assert result.ret == 1 + result.assert_outcomes(failed=6) + + def test_parametrize_CSV(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize("x, y,", [(1,2), (2,3)]) + def test_func(x, y): + assert x+1 == y + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + def test_parametrize_class_scenarios(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + # same as doc/en/example/parametrize scenario example + def pytest_generate_tests(metafunc): + idlist = [] + argvalues = [] + for scenario in metafunc.cls.scenarios: + idlist.append(scenario[0]) + items = scenario[1].items() + argnames = [x[0] for x in items] + argvalues.append(([x[1] for x in items])) + metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class") + + class Test(object): + scenarios = [['1', {'arg': {1: 2}, "arg2": "value2"}], + ['2', {'arg':'value2', "arg2": "value2"}]] + + def test_1(self, arg, arg2): + pass + + def test_2(self, arg2, arg): + pass + + def test_3(self, arg, arg2): + pass + """ + ) + result = pytester.runpytest("-v") + assert result.ret == 0 + result.stdout.fnmatch_lines( + """ + *test_1*1* + *test_2*1* + *test_3*1* + *test_1*2* + *test_2*2* + *test_3*2* + *6 passed* + """ + ) + + def test_format_args(self) -> None: + def function1(): + pass + + assert _format_args(function1) == "()" + + def function2(arg1): + pass + + assert _format_args(function2) == "(arg1)" + + def function3(arg1, arg2="qwe"): + pass + + assert _format_args(function3) == "(arg1, arg2='qwe')" + + def function4(arg1, *args, **kwargs): + pass + + assert _format_args(function4) == "(arg1, *args, **kwargs)" + + +class TestMetafuncFunctional: + def test_attributes(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + # assumes that generate/provide runs in the same process + import sys, pytest + def pytest_generate_tests(metafunc): + metafunc.parametrize('metafunc', [metafunc]) + + @pytest.fixture + def metafunc(request): + return request.param + + def test_function(metafunc, pytestconfig): + assert metafunc.config == pytestconfig + assert metafunc.module.__name__ == __name__ + assert metafunc.function == test_function + assert metafunc.cls is None + + class TestClass(object): + def test_method(self, metafunc, pytestconfig): + assert metafunc.config == pytestconfig + assert metafunc.module.__name__ == __name__ + unbound = TestClass.test_method + assert metafunc.function == unbound + assert metafunc.cls == TestClass + """ + ) + result = pytester.runpytest(p, "-v") + result.assert_outcomes(passed=2) + + def test_two_functions(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def pytest_generate_tests(metafunc): + metafunc.parametrize('arg1', [10, 20], ids=['0', '1']) + + def test_func1(arg1): + assert arg1 == 10 + + def test_func2(arg1): + assert arg1 in (10, 20) + """ + ) + result = pytester.runpytest("-v", p) + result.stdout.fnmatch_lines( + [ + "*test_func1*0*PASS*", + "*test_func1*1*FAIL*", + "*test_func2*PASS*", + "*test_func2*PASS*", + "*1 failed, 3 passed*", + ] + ) + + def test_noself_in_method(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def pytest_generate_tests(metafunc): + assert 'xyz' not in metafunc.fixturenames + + class TestHello(object): + def test_hello(xyz): + pass + """ + ) + result = pytester.runpytest(p) + result.assert_outcomes(passed=1) + + def test_generate_tests_in_class(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + class TestClass(object): + def pytest_generate_tests(self, metafunc): + metafunc.parametrize('hello', ['world'], ids=['hellow']) + + def test_myfunc(self, hello): + assert hello == "world" + """ + ) + result = pytester.runpytest("-v", p) + result.stdout.fnmatch_lines(["*test_myfunc*hello*PASS*", "*1 passed*"]) + + def test_two_functions_not_same_instance(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def pytest_generate_tests(metafunc): + metafunc.parametrize('arg1', [10, 20], ids=["0", "1"]) + + class TestClass(object): + def test_func(self, arg1): + assert not hasattr(self, 'x') + self.x = 1 + """ + ) + result = pytester.runpytest("-v", p) + result.stdout.fnmatch_lines( + ["*test_func*0*PASS*", "*test_func*1*PASS*", "*2 pass*"] + ) + + def test_issue28_setup_method_in_generate_tests(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def pytest_generate_tests(metafunc): + metafunc.parametrize('arg1', [1]) + + class TestClass(object): + def test_method(self, arg1): + assert arg1 == self.val + def setup_method(self, func): + self.val = 1 + """ + ) + result = pytester.runpytest(p) + result.assert_outcomes(passed=1) + + def test_parametrize_functional2(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def pytest_generate_tests(metafunc): + metafunc.parametrize("arg1", [1,2]) + metafunc.parametrize("arg2", [4,5]) + def test_hello(arg1, arg2): + assert 0, (arg1, arg2) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + ["*(1, 4)*", "*(1, 5)*", "*(2, 4)*", "*(2, 5)*", "*4 failed*"] + ) + + def test_parametrize_and_inner_getfixturevalue(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def pytest_generate_tests(metafunc): + metafunc.parametrize("arg1", [1], indirect=True) + metafunc.parametrize("arg2", [10], indirect=True) + + import pytest + @pytest.fixture + def arg1(request): + x = request.getfixturevalue("arg2") + return x + request.param + + @pytest.fixture + def arg2(request): + return request.param + + def test_func1(arg1, arg2): + assert arg1 == 11 + """ + ) + result = pytester.runpytest("-v", p) + result.stdout.fnmatch_lines(["*test_func1*1*PASS*", "*1 passed*"]) + + def test_parametrize_on_setup_arg(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def pytest_generate_tests(metafunc): + assert "arg1" in metafunc.fixturenames + metafunc.parametrize("arg1", [1], indirect=True) + + import pytest + @pytest.fixture + def arg1(request): + return request.param + + @pytest.fixture + def arg2(request, arg1): + return 10 * arg1 + + def test_func(arg2): + assert arg2 == 10 + """ + ) + result = pytester.runpytest("-v", p) + result.stdout.fnmatch_lines(["*test_func*1*PASS*", "*1 passed*"]) + + def test_parametrize_with_ids(self, pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + console_output_style=classic + """ + ) + pytester.makepyfile( + """ + import pytest + def pytest_generate_tests(metafunc): + metafunc.parametrize(("a", "b"), [(1,1), (1,2)], + ids=["basic", "advanced"]) + + def test_function(a, b): + assert a == b + """ + ) + result = pytester.runpytest("-v") + assert result.ret == 1 + result.stdout.fnmatch_lines_random( + ["*test_function*basic*PASSED", "*test_function*advanced*FAILED"] + ) + + def test_parametrize_without_ids(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + def pytest_generate_tests(metafunc): + metafunc.parametrize(("a", "b"), + [(1,object()), (1.3,object())]) + + def test_function(a, b): + assert 1 + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + """ + *test_function*1-b0* + *test_function*1.3-b1* + """ + ) + + def test_parametrize_with_None_in_ids(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + def pytest_generate_tests(metafunc): + metafunc.parametrize(("a", "b"), [(1,1), (1,1), (1,2)], + ids=["basic", None, "advanced"]) + + def test_function(a, b): + assert a == b + """ + ) + result = pytester.runpytest("-v") + assert result.ret == 1 + result.stdout.fnmatch_lines_random( + [ + "*test_function*basic*PASSED*", + "*test_function*1-1*PASSED*", + "*test_function*advanced*FAILED*", + ] + ) + + def test_fixture_parametrized_empty_ids(self, pytester: Pytester) -> None: + """Fixtures parametrized with empty ids cause an internal error (#1849).""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="module", ids=[], params=[]) + def temp(request): + return request.param + + def test_temp(temp): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 1 skipped *"]) + + def test_parametrized_empty_ids(self, pytester: Pytester) -> None: + """Tests parametrized with empty ids cause an internal error (#1849).""" + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize('temp', [], ids=list()) + def test_temp(temp): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 1 skipped *"]) + + def test_parametrized_ids_invalid_type(self, pytester: Pytester) -> None: + """Test error with non-strings/non-ints, without generator (#1857).""" + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize("x, expected", [(1, 2), (3, 4), (5, 6)], ids=(None, 2, type)) + def test_ids_numbers(x,expected): + assert x * 2 == expected + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "In test_ids_numbers: ids must be list of string/float/int/bool," + " found: (type: ) at index 2" + ] + ) + + def test_parametrize_with_identical_ids_get_unique_names( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """ + import pytest + def pytest_generate_tests(metafunc): + metafunc.parametrize(("a", "b"), [(1,1), (1,2)], + ids=["a", "a"]) + + def test_function(a, b): + assert a == b + """ + ) + result = pytester.runpytest("-v") + assert result.ret == 1 + result.stdout.fnmatch_lines_random( + ["*test_function*a0*PASSED*", "*test_function*a1*FAILED*"] + ) + + @pytest.mark.parametrize(("scope", "length"), [("module", 2), ("function", 4)]) + def test_parametrize_scope_overrides( + self, pytester: Pytester, scope: str, length: int + ) -> None: + pytester.makepyfile( + """ + import pytest + values = [] + def pytest_generate_tests(metafunc): + if "arg" in metafunc.fixturenames: + metafunc.parametrize("arg", [1,2], indirect=True, + scope=%r) + @pytest.fixture + def arg(request): + values.append(request.param) + return request.param + def test_hello(arg): + assert arg in (1,2) + def test_world(arg): + assert arg in (1,2) + def test_checklength(): + assert len(values) == %d + """ + % (scope, length) + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=5) + + def test_parametrize_issue323(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope='module', params=range(966)) + def foo(request): + return request.param + + def test_it(foo): + pass + def test_it2(foo): + pass + """ + ) + reprec = pytester.inline_run("--collect-only") + assert not reprec.getcalls("pytest_internalerror") + + def test_usefixtures_seen_in_generate_tests(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + def pytest_generate_tests(metafunc): + assert "abc" in metafunc.fixturenames + metafunc.parametrize("abc", [1]) + + @pytest.mark.usefixtures("abc") + def test_function(): + pass + """ + ) + reprec = pytester.runpytest() + reprec.assert_outcomes(passed=1) + + def test_generate_tests_only_done_in_subdir(self, pytester: Pytester) -> None: + sub1 = pytester.mkpydir("sub1") + sub2 = pytester.mkpydir("sub2") + sub1.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + def pytest_generate_tests(metafunc): + assert metafunc.function.__name__ == "test_1" + """ + ) + ) + sub2.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + def pytest_generate_tests(metafunc): + assert metafunc.function.__name__ == "test_2" + """ + ) + ) + sub1.joinpath("test_in_sub1.py").write_text("def test_1(): pass") + sub2.joinpath("test_in_sub2.py").write_text("def test_2(): pass") + result = pytester.runpytest("--keep-duplicates", "-v", "-s", sub1, sub2, sub1) + result.assert_outcomes(passed=3) + + def test_generate_same_function_names_issue403(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + def make_tests(): + @pytest.mark.parametrize("x", range(2)) + def test_foo(x): + pass + return test_foo + + test_x = make_tests() + test_y = make_tests() + """ + ) + reprec = pytester.runpytest() + reprec.assert_outcomes(passed=4) + + def test_parametrize_misspelling(self, pytester: Pytester) -> None: + """#463""" + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrise("x", range(2)) + def test_foo(x): + pass + """ + ) + result = pytester.runpytest("--collectonly") + result.stdout.fnmatch_lines( + [ + "collected 0 items / 1 error", + "", + "*= ERRORS =*", + "*_ ERROR collecting test_parametrize_misspelling.py _*", + "test_parametrize_misspelling.py:3: in ", + ' @pytest.mark.parametrise("x", range(2))', + "E Failed: Unknown 'parametrise' mark, did you mean 'parametrize'?", + "*! Interrupted: 1 error during collection !*", + "*= no tests collected, 1 error in *", + ] + ) + + +class TestMetafuncFunctionalAuto: + """Tests related to automatically find out the correct scope for + parametrized tests (#1832).""" + + def test_parametrize_auto_scope(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope='session', autouse=True) + def fixture(): + return 1 + + @pytest.mark.parametrize('animal', ["dog", "cat"]) + def test_1(animal): + assert animal in ('dog', 'cat') + + @pytest.mark.parametrize('animal', ['fish']) + def test_2(animal): + assert animal == 'fish' + + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 3 passed *"]) + + def test_parametrize_auto_scope_indirect(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope='session') + def echo(request): + return request.param + + @pytest.mark.parametrize('animal, echo', [("dog", 1), ("cat", 2)], indirect=['echo']) + def test_1(animal, echo): + assert animal in ('dog', 'cat') + assert echo in (1, 2, 3) + + @pytest.mark.parametrize('animal, echo', [('fish', 3)], indirect=['echo']) + def test_2(animal, echo): + assert animal == 'fish' + assert echo in (1, 2, 3) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 3 passed *"]) + + def test_parametrize_auto_scope_override_fixture(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope='session', autouse=True) + def animal(): + return 'fox' + + @pytest.mark.parametrize('animal', ["dog", "cat"]) + def test_1(animal): + assert animal in ('dog', 'cat') + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 2 passed *"]) + + def test_parametrize_all_indirects(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture() + def animal(request): + return request.param + + @pytest.fixture(scope='session') + def echo(request): + return request.param + + @pytest.mark.parametrize('animal, echo', [("dog", 1), ("cat", 2)], indirect=True) + def test_1(animal, echo): + assert animal in ('dog', 'cat') + assert echo in (1, 2, 3) + + @pytest.mark.parametrize('animal, echo', [("fish", 3)], indirect=True) + def test_2(animal, echo): + assert animal == 'fish' + assert echo in (1, 2, 3) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 3 passed *"]) + + def test_parametrize_some_arguments_auto_scope( + self, pytester: Pytester, monkeypatch + ) -> None: + """Integration test for (#3941)""" + class_fix_setup: List[object] = [] + monkeypatch.setattr(sys, "class_fix_setup", class_fix_setup, raising=False) + func_fix_setup: List[object] = [] + monkeypatch.setattr(sys, "func_fix_setup", func_fix_setup, raising=False) + + pytester.makepyfile( + """ + import pytest + import sys + + @pytest.fixture(scope='class', autouse=True) + def class_fix(request): + sys.class_fix_setup.append(request.param) + + @pytest.fixture(autouse=True) + def func_fix(): + sys.func_fix_setup.append(True) + + @pytest.mark.parametrize('class_fix', [10, 20], indirect=True) + class Test: + def test_foo(self): + pass + def test_bar(self): + pass + """ + ) + result = pytester.runpytest_inprocess() + result.stdout.fnmatch_lines(["* 4 passed in *"]) + assert func_fix_setup == [True] * 4 + assert class_fix_setup == [10, 20] + + def test_parametrize_issue634(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope='module') + def foo(request): + print('preparing foo-%d' % request.param) + return 'foo-%d' % request.param + + def test_one(foo): + pass + + def test_two(foo): + pass + + test_two.test_with = (2, 3) + + def pytest_generate_tests(metafunc): + params = (1, 2, 3, 4) + if not 'foo' in metafunc.fixturenames: + return + + test_with = getattr(metafunc.function, 'test_with', None) + if test_with: + params = test_with + metafunc.parametrize('foo', params, indirect=True) + """ + ) + result = pytester.runpytest("-s") + output = result.stdout.str() + assert output.count("preparing foo-2") == 1 + assert output.count("preparing foo-3") == 1 + + +class TestMarkersWithParametrization: + """#308""" + + def test_simple_mark(self, pytester: Pytester) -> None: + s = """ + import pytest + + @pytest.mark.foo + @pytest.mark.parametrize(("n", "expected"), [ + (1, 2), + pytest.param(1, 3, marks=pytest.mark.bar), + (2, 3), + ]) + def test_increment(n, expected): + assert n + 1 == expected + """ + items = pytester.getitems(s) + assert len(items) == 3 + for item in items: + assert "foo" in item.keywords + assert "bar" not in items[0].keywords + assert "bar" in items[1].keywords + assert "bar" not in items[2].keywords + + def test_select_based_on_mark(self, pytester: Pytester) -> None: + s = """ + import pytest + + @pytest.mark.parametrize(("n", "expected"), [ + (1, 2), + pytest.param(2, 3, marks=pytest.mark.foo), + (3, 4), + ]) + def test_increment(n, expected): + assert n + 1 == expected + """ + pytester.makepyfile(s) + rec = pytester.inline_run("-m", "foo") + passed, skipped, fail = rec.listoutcomes() + assert len(passed) == 1 + assert len(skipped) == 0 + assert len(fail) == 0 + + def test_simple_xfail(self, pytester: Pytester) -> None: + s = """ + import pytest + + @pytest.mark.parametrize(("n", "expected"), [ + (1, 2), + pytest.param(1, 3, marks=pytest.mark.xfail), + (2, 3), + ]) + def test_increment(n, expected): + assert n + 1 == expected + """ + pytester.makepyfile(s) + reprec = pytester.inline_run() + # xfail is skip?? + reprec.assertoutcome(passed=2, skipped=1) + + def test_simple_xfail_single_argname(self, pytester: Pytester) -> None: + s = """ + import pytest + + @pytest.mark.parametrize("n", [ + 2, + pytest.param(3, marks=pytest.mark.xfail), + 4, + ]) + def test_isEven(n): + assert n % 2 == 0 + """ + pytester.makepyfile(s) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2, skipped=1) + + def test_xfail_with_arg(self, pytester: Pytester) -> None: + s = """ + import pytest + + @pytest.mark.parametrize(("n", "expected"), [ + (1, 2), + pytest.param(1, 3, marks=pytest.mark.xfail("True")), + (2, 3), + ]) + def test_increment(n, expected): + assert n + 1 == expected + """ + pytester.makepyfile(s) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2, skipped=1) + + def test_xfail_with_kwarg(self, pytester: Pytester) -> None: + s = """ + import pytest + + @pytest.mark.parametrize(("n", "expected"), [ + (1, 2), + pytest.param(1, 3, marks=pytest.mark.xfail(reason="some bug")), + (2, 3), + ]) + def test_increment(n, expected): + assert n + 1 == expected + """ + pytester.makepyfile(s) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2, skipped=1) + + def test_xfail_with_arg_and_kwarg(self, pytester: Pytester) -> None: + s = """ + import pytest + + @pytest.mark.parametrize(("n", "expected"), [ + (1, 2), + pytest.param(1, 3, marks=pytest.mark.xfail("True", reason="some bug")), + (2, 3), + ]) + def test_increment(n, expected): + assert n + 1 == expected + """ + pytester.makepyfile(s) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2, skipped=1) + + @pytest.mark.parametrize("strict", [True, False]) + def test_xfail_passing_is_xpass(self, pytester: Pytester, strict: bool) -> None: + s = """ + import pytest + + m = pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict}) + + @pytest.mark.parametrize(("n", "expected"), [ + (1, 2), + pytest.param(2, 3, marks=m), + (3, 4), + ]) + def test_increment(n, expected): + assert n + 1 == expected + """.format( + strict=strict + ) + pytester.makepyfile(s) + reprec = pytester.inline_run() + passed, failed = (2, 1) if strict else (3, 0) + reprec.assertoutcome(passed=passed, failed=failed) + + def test_parametrize_called_in_generate_tests(self, pytester: Pytester) -> None: + s = """ + import pytest + + + def pytest_generate_tests(metafunc): + passingTestData = [(1, 2), + (2, 3)] + failingTestData = [(1, 3), + (2, 2)] + + testData = passingTestData + [pytest.param(*d, marks=pytest.mark.xfail) + for d in failingTestData] + metafunc.parametrize(("n", "expected"), testData) + + + def test_increment(n, expected): + assert n + 1 == expected + """ + pytester.makepyfile(s) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2, skipped=2) + + def test_parametrize_ID_generation_string_int_works( + self, pytester: Pytester + ) -> None: + """#290""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def myfixture(): + return 'example' + @pytest.mark.parametrize( + 'limit', (0, '0')) + def test_limit(limit, myfixture): + return + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=2) + + @pytest.mark.parametrize("strict", [True, False]) + def test_parametrize_marked_value(self, pytester: Pytester, strict: bool) -> None: + s = """ + import pytest + + @pytest.mark.parametrize(("n", "expected"), [ + pytest.param( + 2,3, + marks=pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict}), + ), + pytest.param( + 2,3, + marks=[pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict})], + ), + ]) + def test_increment(n, expected): + assert n + 1 == expected + """.format( + strict=strict + ) + pytester.makepyfile(s) + reprec = pytester.inline_run() + passed, failed = (0, 2) if strict else (2, 0) + reprec.assertoutcome(passed=passed, failed=failed) + + def test_pytest_make_parametrize_id(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_make_parametrize_id(config, val): + return str(val * 2) + """ + ) + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize("x", range(2)) + def test_func(x): + pass + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines(["*test_func*0*PASS*", "*test_func*2*PASS*"]) + + def test_pytest_make_parametrize_id_with_argname(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_make_parametrize_id(config, val, argname): + return str(val * 2 if argname == 'x' else val * 10) + """ + ) + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize("x", range(2)) + def test_func_a(x): + pass + + @pytest.mark.parametrize("y", [1]) + def test_func_b(y): + pass + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + ["*test_func_a*0*PASS*", "*test_func_a*2*PASS*", "*test_func_b*10*PASS*"] + ) + + def test_parametrize_positional_args(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize("a", [1], False) + def test_foo(a): + pass + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=1) + + def test_parametrize_iterator(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import itertools + import pytest + + id_parametrize = pytest.mark.parametrize( + ids=("param%d" % i for i in itertools.count()) + ) + + @id_parametrize('y', ['a', 'b']) + def test1(y): + pass + + @id_parametrize('y', ['a', 'b']) + def test2(y): + pass + + @pytest.mark.parametrize("a, b", [(1, 2), (3, 4)], ids=itertools.count()) + def test_converted_to_str(a, b): + pass + """ + ) + result = pytester.runpytest("-vv", "-s") + result.stdout.fnmatch_lines( + [ + "test_parametrize_iterator.py::test1[param0] PASSED", + "test_parametrize_iterator.py::test1[param1] PASSED", + "test_parametrize_iterator.py::test2[param0] PASSED", + "test_parametrize_iterator.py::test2[param1] PASSED", + "test_parametrize_iterator.py::test_converted_to_str[0] PASSED", + "test_parametrize_iterator.py::test_converted_to_str[1] PASSED", + "*= 6 passed in *", + ] + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/python/raises.py b/testing/web-platform/tests/tools/third_party/pytest/testing/python/raises.py new file mode 100644 index 0000000000..2d62e91091 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/python/raises.py @@ -0,0 +1,298 @@ +import re +import sys + +import pytest +from _pytest.outcomes import Failed +from _pytest.pytester import Pytester + + +class TestRaises: + def test_check_callable(self) -> None: + with pytest.raises(TypeError, match=r".* must be callable"): + pytest.raises(RuntimeError, "int('qwe')") # type: ignore[call-overload] + + def test_raises(self): + excinfo = pytest.raises(ValueError, int, "qwe") + assert "invalid literal" in str(excinfo.value) + + def test_raises_function(self): + excinfo = pytest.raises(ValueError, int, "hello") + assert "invalid literal" in str(excinfo.value) + + def test_raises_callable_no_exception(self) -> None: + class A: + def __call__(self): + pass + + try: + pytest.raises(ValueError, A()) + except pytest.fail.Exception: + pass + + def test_raises_falsey_type_error(self) -> None: + with pytest.raises(TypeError): + with pytest.raises(AssertionError, match=0): # type: ignore[call-overload] + raise AssertionError("ohai") + + def test_raises_repr_inflight(self): + """Ensure repr() on an exception info inside a pytest.raises with block works (#4386)""" + + class E(Exception): + pass + + with pytest.raises(E) as excinfo: + # this test prints the inflight uninitialized object + # using repr and str as well as pprint to demonstrate + # it works + print(str(excinfo)) + print(repr(excinfo)) + import pprint + + pprint.pprint(excinfo) + raise E() + + def test_raises_as_contextmanager(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + import _pytest._code + + def test_simple(): + with pytest.raises(ZeroDivisionError) as excinfo: + assert isinstance(excinfo, _pytest._code.ExceptionInfo) + 1/0 + print(excinfo) + assert excinfo.type == ZeroDivisionError + assert isinstance(excinfo.value, ZeroDivisionError) + + def test_noraise(): + with pytest.raises(pytest.raises.Exception): + with pytest.raises(ValueError): + int() + + def test_raise_wrong_exception_passes_by(): + with pytest.raises(ZeroDivisionError): + with pytest.raises(ValueError): + 1/0 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*3 passed*"]) + + def test_does_not_raise(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + from contextlib import contextmanager + import pytest + + @contextmanager + def does_not_raise(): + yield + + @pytest.mark.parametrize('example_input,expectation', [ + (3, does_not_raise()), + (2, does_not_raise()), + (1, does_not_raise()), + (0, pytest.raises(ZeroDivisionError)), + ]) + def test_division(example_input, expectation): + '''Test how much I know division.''' + with expectation: + assert (6 / example_input) is not None + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*4 passed*"]) + + def test_does_not_raise_does_raise(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + from contextlib import contextmanager + import pytest + + @contextmanager + def does_not_raise(): + yield + + @pytest.mark.parametrize('example_input,expectation', [ + (0, does_not_raise()), + (1, pytest.raises(ZeroDivisionError)), + ]) + def test_division(example_input, expectation): + '''Test how much I know division.''' + with expectation: + assert (6 / example_input) is not None + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*2 failed*"]) + + def test_noclass(self) -> None: + with pytest.raises(TypeError): + pytest.raises("wrong", lambda: None) # type: ignore[call-overload] + + def test_invalid_arguments_to_raises(self) -> None: + with pytest.raises(TypeError, match="unknown"): + with pytest.raises(TypeError, unknown="bogus"): # type: ignore[call-overload] + raise ValueError() + + def test_tuple(self): + with pytest.raises((KeyError, ValueError)): + raise KeyError("oops") + + def test_no_raise_message(self) -> None: + try: + pytest.raises(ValueError, int, "0") + except pytest.fail.Exception as e: + assert e.msg == f"DID NOT RAISE {repr(ValueError)}" + else: + assert False, "Expected pytest.raises.Exception" + + try: + with pytest.raises(ValueError): + pass + except pytest.fail.Exception as e: + assert e.msg == f"DID NOT RAISE {repr(ValueError)}" + else: + assert False, "Expected pytest.raises.Exception" + + @pytest.mark.parametrize("method", ["function", "function_match", "with"]) + def test_raises_cyclic_reference(self, method): + """Ensure pytest.raises does not leave a reference cycle (#1965).""" + import gc + + class T: + def __call__(self): + raise ValueError + + t = T() + refcount = len(gc.get_referrers(t)) + + if method == "function": + pytest.raises(ValueError, t) + elif method == "function_match": + pytest.raises(ValueError, t).match("^$") + else: + with pytest.raises(ValueError): + t() + + # ensure both forms of pytest.raises don't leave exceptions in sys.exc_info() + assert sys.exc_info() == (None, None, None) + + assert refcount == len(gc.get_referrers(t)) + + def test_raises_match(self) -> None: + msg = r"with base \d+" + with pytest.raises(ValueError, match=msg): + int("asdf") + + msg = "with base 10" + with pytest.raises(ValueError, match=msg): + int("asdf") + + msg = "with base 16" + expr = "Regex pattern {!r} does not match \"invalid literal for int() with base 10: 'asdf'\".".format( + msg + ) + with pytest.raises(AssertionError, match=re.escape(expr)): + with pytest.raises(ValueError, match=msg): + int("asdf", base=10) + + # "match" without context manager. + pytest.raises(ValueError, int, "asdf").match("invalid literal") + with pytest.raises(AssertionError) as excinfo: + pytest.raises(ValueError, int, "asdf").match(msg) + assert str(excinfo.value) == expr + + pytest.raises(TypeError, int, match="invalid") + + def tfunc(match): + raise ValueError(f"match={match}") + + pytest.raises(ValueError, tfunc, match="asdf").match("match=asdf") + pytest.raises(ValueError, tfunc, match="").match("match=") + + def test_match_failure_string_quoting(self): + with pytest.raises(AssertionError) as excinfo: + with pytest.raises(AssertionError, match="'foo"): + raise AssertionError("'bar") + (msg,) = excinfo.value.args + assert msg == 'Regex pattern "\'foo" does not match "\'bar".' + + def test_match_failure_exact_string_message(self): + message = "Oh here is a message with (42) numbers in parameters" + with pytest.raises(AssertionError) as excinfo: + with pytest.raises(AssertionError, match=message): + raise AssertionError(message) + (msg,) = excinfo.value.args + assert msg == ( + "Regex pattern 'Oh here is a message with (42) numbers in " + "parameters' does not match 'Oh here is a message with (42) " + "numbers in parameters'. Did you mean to `re.escape()` the regex?" + ) + + def test_raises_match_wrong_type(self): + """Raising an exception with the wrong type and match= given. + + pytest should throw the unexpected exception - the pattern match is not + really relevant if we got a different exception. + """ + with pytest.raises(ValueError): + with pytest.raises(IndexError, match="nomatch"): + int("asdf") + + def test_raises_exception_looks_iterable(self): + class Meta(type): + def __getitem__(self, item): + return 1 / 0 + + def __len__(self): + return 1 + + class ClassLooksIterableException(Exception, metaclass=Meta): + pass + + with pytest.raises( + Failed, + match=r"DID NOT RAISE ", + ): + pytest.raises(ClassLooksIterableException, lambda: None) + + def test_raises_with_raising_dunder_class(self) -> None: + """Test current behavior with regard to exceptions via __class__ (#4284).""" + + class CrappyClass(Exception): + # Type ignored because it's bypassed intentionally. + @property # type: ignore + def __class__(self): + assert False, "via __class__" + + with pytest.raises(AssertionError) as excinfo: + with pytest.raises(CrappyClass()): # type: ignore[call-overload] + pass + assert "via __class__" in excinfo.value.args[0] + + def test_raises_context_manager_with_kwargs(self): + with pytest.raises(TypeError) as excinfo: + with pytest.raises(Exception, foo="bar"): # type: ignore[call-overload] + pass + assert "Unexpected keyword arguments" in str(excinfo.value) + + def test_expected_exception_is_not_a_baseexception(self) -> None: + with pytest.raises(TypeError) as excinfo: + with pytest.raises("hello"): # type: ignore[call-overload] + pass # pragma: no cover + assert "must be a BaseException type, not str" in str(excinfo.value) + + class NotAnException: + pass + + with pytest.raises(TypeError) as excinfo: + with pytest.raises(NotAnException): # type: ignore[type-var] + pass # pragma: no cover + assert "must be a BaseException type, not NotAnException" in str(excinfo.value) + + with pytest.raises(TypeError) as excinfo: + with pytest.raises(("hello", NotAnException)): # type: ignore[arg-type] + pass # pragma: no cover + assert "must be a BaseException type, not str" in str(excinfo.value) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/python/show_fixtures_per_test.py b/testing/web-platform/tests/tools/third_party/pytest/testing/python/show_fixtures_per_test.py new file mode 100644 index 0000000000..f756dca41c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/python/show_fixtures_per_test.py @@ -0,0 +1,254 @@ +from _pytest.pytester import Pytester + + +def test_no_items_should_not_show_output(pytester: Pytester) -> None: + result = pytester.runpytest("--fixtures-per-test") + result.stdout.no_fnmatch_line("*fixtures used by*") + assert result.ret == 0 + + +def test_fixtures_in_module(pytester: Pytester) -> None: + p = pytester.makepyfile( + ''' + import pytest + @pytest.fixture + def _arg0(): + """hidden arg0 fixture""" + @pytest.fixture + def arg1(): + """arg1 docstring""" + def test_arg1(arg1): + pass + ''' + ) + + result = pytester.runpytest("--fixtures-per-test", p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + [ + "*fixtures used by test_arg1*", + "*(test_fixtures_in_module.py:9)*", + "arg1 -- test_fixtures_in_module.py:6", + " arg1 docstring", + ] + ) + result.stdout.no_fnmatch_line("*_arg0*") + + +def test_fixtures_in_conftest(pytester: Pytester) -> None: + pytester.makeconftest( + ''' + import pytest + @pytest.fixture + def arg1(): + """arg1 docstring""" + @pytest.fixture + def arg2(): + """arg2 docstring""" + @pytest.fixture + def arg3(arg1, arg2): + """arg3 + docstring + """ + ''' + ) + p = pytester.makepyfile( + """ + def test_arg2(arg2): + pass + def test_arg3(arg3): + pass + """ + ) + result = pytester.runpytest("--fixtures-per-test", p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + [ + "*fixtures used by test_arg2*", + "*(test_fixtures_in_conftest.py:2)*", + "arg2 -- conftest.py:6", + " arg2 docstring", + "*fixtures used by test_arg3*", + "*(test_fixtures_in_conftest.py:4)*", + "arg1 -- conftest.py:3", + " arg1 docstring", + "arg2 -- conftest.py:6", + " arg2 docstring", + "arg3 -- conftest.py:9", + " arg3", + ] + ) + + +def test_should_show_fixtures_used_by_test(pytester: Pytester) -> None: + pytester.makeconftest( + ''' + import pytest + @pytest.fixture + def arg1(): + """arg1 from conftest""" + @pytest.fixture + def arg2(): + """arg2 from conftest""" + ''' + ) + p = pytester.makepyfile( + ''' + import pytest + @pytest.fixture + def arg1(): + """arg1 from testmodule""" + def test_args(arg1, arg2): + pass + ''' + ) + result = pytester.runpytest("--fixtures-per-test", p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + [ + "*fixtures used by test_args*", + "*(test_should_show_fixtures_used_by_test.py:6)*", + "arg1 -- test_should_show_fixtures_used_by_test.py:3", + " arg1 from testmodule", + "arg2 -- conftest.py:6", + " arg2 from conftest", + ] + ) + + +def test_verbose_include_private_fixtures_and_loc(pytester: Pytester) -> None: + pytester.makeconftest( + ''' + import pytest + @pytest.fixture + def _arg1(): + """_arg1 from conftest""" + @pytest.fixture + def arg2(_arg1): + """arg2 from conftest""" + ''' + ) + p = pytester.makepyfile( + ''' + import pytest + @pytest.fixture + def arg3(): + """arg3 from testmodule""" + def test_args(arg2, arg3): + pass + ''' + ) + result = pytester.runpytest("--fixtures-per-test", "-v", p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + [ + "*fixtures used by test_args*", + "*(test_verbose_include_private_fixtures_and_loc.py:6)*", + "_arg1 -- conftest.py:3", + " _arg1 from conftest", + "arg2 -- conftest.py:6", + " arg2 from conftest", + "arg3 -- test_verbose_include_private_fixtures_and_loc.py:3", + " arg3 from testmodule", + ] + ) + + +def test_doctest_items(pytester: Pytester) -> None: + pytester.makepyfile( + ''' + def foo(): + """ + >>> 1 + 1 + 2 + """ + ''' + ) + pytester.maketxtfile( + """ + >>> 1 + 1 + 2 + """ + ) + result = pytester.runpytest( + "--fixtures-per-test", "--doctest-modules", "--doctest-glob=*.txt", "-v" + ) + assert result.ret == 0 + + result.stdout.fnmatch_lines(["*collected 2 items*"]) + + +def test_multiline_docstring_in_module(pytester: Pytester) -> None: + p = pytester.makepyfile( + ''' + import pytest + @pytest.fixture + def arg1(): + """Docstring content that spans across multiple lines, + through second line, + and through third line. + + Docstring content that extends into a second paragraph. + + Docstring content that extends into a third paragraph. + """ + def test_arg1(arg1): + pass + ''' + ) + + result = pytester.runpytest("--fixtures-per-test", p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + [ + "*fixtures used by test_arg1*", + "*(test_multiline_docstring_in_module.py:13)*", + "arg1 -- test_multiline_docstring_in_module.py:3", + " Docstring content that spans across multiple lines,", + " through second line,", + " and through third line.", + ] + ) + + +def test_verbose_include_multiline_docstring(pytester: Pytester) -> None: + p = pytester.makepyfile( + ''' + import pytest + @pytest.fixture + def arg1(): + """Docstring content that spans across multiple lines, + through second line, + and through third line. + + Docstring content that extends into a second paragraph. + + Docstring content that extends into a third paragraph. + """ + def test_arg1(arg1): + pass + ''' + ) + + result = pytester.runpytest("--fixtures-per-test", "-v", p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + [ + "*fixtures used by test_arg1*", + "*(test_verbose_include_multiline_docstring.py:13)*", + "arg1 -- test_verbose_include_multiline_docstring.py:3", + " Docstring content that spans across multiple lines,", + " through second line,", + " and through third line.", + " ", + " Docstring content that extends into a second paragraph.", + " ", + " Docstring content that extends into a third paragraph.", + ] + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_argcomplete.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_argcomplete.py new file mode 100644 index 0000000000..8c10e230b0 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_argcomplete.py @@ -0,0 +1,95 @@ +import subprocess +import sys +from pathlib import Path + +import pytest +from _pytest.monkeypatch import MonkeyPatch + +# Test for _argcomplete but not specific for any application. + + +def equal_with_bash(prefix, ffc, fc, out=None): + res = ffc(prefix) + res_bash = set(fc(prefix)) + retval = set(res) == res_bash + if out: + out.write(f"equal_with_bash({prefix}) {retval} {res}\n") + if not retval: + out.write(" python - bash: %s\n" % (set(res) - res_bash)) + out.write(" bash - python: %s\n" % (res_bash - set(res))) + return retval + + +# Copied from argcomplete.completers as import from there. +# Also pulls in argcomplete.__init__ which opens filedescriptor 9. +# This gives an OSError at the end of testrun. + + +def _wrapcall(*args, **kargs): + try: + return subprocess.check_output(*args, **kargs).decode().splitlines() + except subprocess.CalledProcessError: + return [] + + +class FilesCompleter: + """File completer class, optionally takes a list of allowed extensions.""" + + def __init__(self, allowednames=(), directories=True): + # Fix if someone passes in a string instead of a list + if type(allowednames) is str: + allowednames = [allowednames] + + self.allowednames = [x.lstrip("*").lstrip(".") for x in allowednames] + self.directories = directories + + def __call__(self, prefix, **kwargs): + completion = [] + if self.allowednames: + if self.directories: + files = _wrapcall(["bash", "-c", f"compgen -A directory -- '{prefix}'"]) + completion += [f + "/" for f in files] + for x in self.allowednames: + completion += _wrapcall( + ["bash", "-c", f"compgen -A file -X '!*.{x}' -- '{prefix}'"] + ) + else: + completion += _wrapcall(["bash", "-c", f"compgen -A file -- '{prefix}'"]) + + anticomp = _wrapcall(["bash", "-c", f"compgen -A directory -- '{prefix}'"]) + + completion = list(set(completion) - set(anticomp)) + + if self.directories: + completion += [f + "/" for f in anticomp] + return completion + + +class TestArgComplete: + @pytest.mark.skipif("sys.platform in ('win32', 'darwin')") + def test_compare_with_compgen( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: + from _pytest._argcomplete import FastFilesCompleter + + ffc = FastFilesCompleter() + fc = FilesCompleter() + + monkeypatch.chdir(tmp_path) + + assert equal_with_bash("", ffc, fc, out=sys.stdout) + + tmp_path.cwd().joinpath("data").touch() + + for x in ["d", "data", "doesnotexist", ""]: + assert equal_with_bash(x, ffc, fc, out=sys.stdout) + + @pytest.mark.skipif("sys.platform in ('win32', 'darwin')") + def test_remove_dir_prefix(self): + """This is not compatible with compgen but it is with bash itself: ls /usr/.""" + from _pytest._argcomplete import FastFilesCompleter + + ffc = FastFilesCompleter() + fc = FilesCompleter() + for x in "/usr/".split(): + assert not equal_with_bash(x, ffc, fc, out=sys.stdout) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_assertion.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_assertion.py new file mode 100644 index 0000000000..2516ff1629 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_assertion.py @@ -0,0 +1,1685 @@ +import collections +import sys +import textwrap +from typing import Any +from typing import List +from typing import MutableSequence +from typing import Optional + +import attr + +import _pytest.assertion as plugin +import pytest +from _pytest import outcomes +from _pytest.assertion import truncate +from _pytest.assertion import util +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester + + +def mock_config(verbose=0): + class Config: + def getoption(self, name): + if name == "verbose": + return verbose + raise KeyError("Not mocked out: %s" % name) + + return Config() + + +class TestImportHookInstallation: + @pytest.mark.parametrize("initial_conftest", [True, False]) + @pytest.mark.parametrize("mode", ["plain", "rewrite"]) + def test_conftest_assertion_rewrite( + self, pytester: Pytester, initial_conftest, mode + ) -> None: + """Test that conftest files are using assertion rewrite on import (#1619).""" + pytester.mkdir("foo") + pytester.mkdir("foo/tests") + conftest_path = "conftest.py" if initial_conftest else "foo/conftest.py" + contents = { + conftest_path: """ + import pytest + @pytest.fixture + def check_first(): + def check(values, value): + assert values.pop(0) == value + return check + """, + "foo/tests/test_foo.py": """ + def test(check_first): + check_first([10, 30], 30) + """, + } + pytester.makepyfile(**contents) + result = pytester.runpytest_subprocess("--assert=%s" % mode) + if mode == "plain": + expected = "E AssertionError" + elif mode == "rewrite": + expected = "*assert 10 == 30*" + else: + assert 0 + result.stdout.fnmatch_lines([expected]) + + def test_rewrite_assertions_pytester_plugin(self, pytester: Pytester) -> None: + """ + Assertions in the pytester plugin must also benefit from assertion + rewriting (#1920). + """ + pytester.makepyfile( + """ + pytest_plugins = ['pytester'] + def test_dummy_failure(pytester): # how meta! + pytester.makepyfile('def test(): assert 0') + r = pytester.inline_run() + r.assertoutcome(passed=1) + """ + ) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines( + [ + "> r.assertoutcome(passed=1)", + "E AssertionError: ([[][]], [[][]], [[][]])*", + "E assert {'failed': 1,... 'skipped': 0} == {'failed': 0,... 'skipped': 0}", + "E Omitting 1 identical items, use -vv to show", + "E Differing items:", + "E Use -v to get the full diff", + ] + ) + # XXX: unstable output. + result.stdout.fnmatch_lines_random( + [ + "E {'failed': 1} != {'failed': 0}", + "E {'passed': 0} != {'passed': 1}", + ] + ) + + @pytest.mark.parametrize("mode", ["plain", "rewrite"]) + def test_pytest_plugins_rewrite(self, pytester: Pytester, mode) -> None: + contents = { + "conftest.py": """ + pytest_plugins = ['ham'] + """, + "ham.py": """ + import pytest + @pytest.fixture + def check_first(): + def check(values, value): + assert values.pop(0) == value + return check + """, + "test_foo.py": """ + def test_foo(check_first): + check_first([10, 30], 30) + """, + } + pytester.makepyfile(**contents) + result = pytester.runpytest_subprocess("--assert=%s" % mode) + if mode == "plain": + expected = "E AssertionError" + elif mode == "rewrite": + expected = "*assert 10 == 30*" + else: + assert 0 + result.stdout.fnmatch_lines([expected]) + + @pytest.mark.parametrize("mode", ["str", "list"]) + def test_pytest_plugins_rewrite_module_names( + self, pytester: Pytester, mode + ) -> None: + """Test that pluginmanager correct marks pytest_plugins variables + for assertion rewriting if they are defined as plain strings or + list of strings (#1888). + """ + plugins = '"ham"' if mode == "str" else '["ham"]' + contents = { + "conftest.py": """ + pytest_plugins = {plugins} + """.format( + plugins=plugins + ), + "ham.py": """ + import pytest + """, + "test_foo.py": """ + def test_foo(pytestconfig): + assert 'ham' in pytestconfig.pluginmanager.rewrite_hook._must_rewrite + """, + } + pytester.makepyfile(**contents) + result = pytester.runpytest_subprocess("--assert=rewrite") + assert result.ret == 0 + + def test_pytest_plugins_rewrite_module_names_correctly( + self, pytester: Pytester + ) -> None: + """Test that we match files correctly when they are marked for rewriting (#2939).""" + contents = { + "conftest.py": """\ + pytest_plugins = "ham" + """, + "ham.py": "", + "hamster.py": "", + "test_foo.py": """\ + def test_foo(pytestconfig): + assert pytestconfig.pluginmanager.rewrite_hook.find_spec('ham') is not None + assert pytestconfig.pluginmanager.rewrite_hook.find_spec('hamster') is None + """, + } + pytester.makepyfile(**contents) + result = pytester.runpytest_subprocess("--assert=rewrite") + assert result.ret == 0 + + @pytest.mark.parametrize("mode", ["plain", "rewrite"]) + def test_installed_plugin_rewrite( + self, pytester: Pytester, mode, monkeypatch + ) -> None: + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + # Make sure the hook is installed early enough so that plugins + # installed via setuptools are rewritten. + pytester.mkdir("hampkg") + contents = { + "hampkg/__init__.py": """\ + import pytest + + @pytest.fixture + def check_first2(): + def check(values, value): + assert values.pop(0) == value + return check + """, + "spamplugin.py": """\ + import pytest + from hampkg import check_first2 + + @pytest.fixture + def check_first(): + def check(values, value): + assert values.pop(0) == value + return check + """, + "mainwrapper.py": """\ + import pytest + from _pytest.compat import importlib_metadata + + class DummyEntryPoint(object): + name = 'spam' + module_name = 'spam.py' + group = 'pytest11' + + def load(self): + import spamplugin + return spamplugin + + class DummyDistInfo(object): + version = '1.0' + files = ('spamplugin.py', 'hampkg/__init__.py') + entry_points = (DummyEntryPoint(),) + metadata = {'name': 'foo'} + + def distributions(): + return (DummyDistInfo(),) + + importlib_metadata.distributions = distributions + pytest.main() + """, + "test_foo.py": """\ + def test(check_first): + check_first([10, 30], 30) + + def test2(check_first2): + check_first([10, 30], 30) + """, + } + pytester.makepyfile(**contents) + result = pytester.run( + sys.executable, "mainwrapper.py", "-s", "--assert=%s" % mode + ) + if mode == "plain": + expected = "E AssertionError" + elif mode == "rewrite": + expected = "*assert 10 == 30*" + else: + assert 0 + result.stdout.fnmatch_lines([expected]) + + def test_rewrite_ast(self, pytester: Pytester) -> None: + pytester.mkdir("pkg") + contents = { + "pkg/__init__.py": """ + import pytest + pytest.register_assert_rewrite('pkg.helper') + """, + "pkg/helper.py": """ + def tool(): + a, b = 2, 3 + assert a == b + """, + "pkg/plugin.py": """ + import pytest, pkg.helper + @pytest.fixture + def tool(): + return pkg.helper.tool + """, + "pkg/other.py": """ + values = [3, 2] + def tool(): + assert values.pop() == 3 + """, + "conftest.py": """ + pytest_plugins = ['pkg.plugin'] + """, + "test_pkg.py": """ + import pkg.other + def test_tool(tool): + tool() + def test_other(): + pkg.other.tool() + """, + } + pytester.makepyfile(**contents) + result = pytester.runpytest_subprocess("--assert=rewrite") + result.stdout.fnmatch_lines( + [ + ">*assert a == b*", + "E*assert 2 == 3*", + ">*assert values.pop() == 3*", + "E*AssertionError", + ] + ) + + def test_register_assert_rewrite_checks_types(self) -> None: + with pytest.raises(TypeError): + pytest.register_assert_rewrite(["pytest_tests_internal_non_existing"]) # type: ignore + pytest.register_assert_rewrite( + "pytest_tests_internal_non_existing", "pytest_tests_internal_non_existing2" + ) + + +class TestBinReprIntegration: + def test_pytest_assertrepr_compare_called(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + values = [] + def pytest_assertrepr_compare(op, left, right): + values.append((op, left, right)) + + @pytest.fixture + def list(request): + return values + """ + ) + pytester.makepyfile( + """ + def test_hello(): + assert 0 == 1 + def test_check(list): + assert list == [("==", 0, 1)] + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines(["*test_hello*FAIL*", "*test_check*PASS*"]) + + +def callop(op: str, left: Any, right: Any, verbose: int = 0) -> Optional[List[str]]: + config = mock_config(verbose=verbose) + return plugin.pytest_assertrepr_compare(config, op, left, right) + + +def callequal(left: Any, right: Any, verbose: int = 0) -> Optional[List[str]]: + return callop("==", left, right, verbose) + + +class TestAssert_reprcompare: + def test_different_types(self) -> None: + assert callequal([0, 1], "foo") is None + + def test_summary(self) -> None: + lines = callequal([0, 1], [0, 2]) + assert lines is not None + summary = lines[0] + assert len(summary) < 65 + + def test_text_diff(self) -> None: + assert callequal("spam", "eggs") == [ + "'spam' == 'eggs'", + "- eggs", + "+ spam", + ] + + def test_text_skipping(self) -> None: + lines = callequal("a" * 50 + "spam", "a" * 50 + "eggs") + assert lines is not None + assert "Skipping" in lines[1] + for line in lines: + assert "a" * 50 not in line + + def test_text_skipping_verbose(self) -> None: + lines = callequal("a" * 50 + "spam", "a" * 50 + "eggs", verbose=1) + assert lines is not None + assert "- " + "a" * 50 + "eggs" in lines + assert "+ " + "a" * 50 + "spam" in lines + + def test_multiline_text_diff(self) -> None: + left = "foo\nspam\nbar" + right = "foo\neggs\nbar" + diff = callequal(left, right) + assert diff is not None + assert "- eggs" in diff + assert "+ spam" in diff + + def test_bytes_diff_normal(self) -> None: + """Check special handling for bytes diff (#5260)""" + diff = callequal(b"spam", b"eggs") + + assert diff == [ + "b'spam' == b'eggs'", + "At index 0 diff: b's' != b'e'", + "Use -v to get the full diff", + ] + + def test_bytes_diff_verbose(self) -> None: + """Check special handling for bytes diff (#5260)""" + diff = callequal(b"spam", b"eggs", verbose=1) + assert diff == [ + "b'spam' == b'eggs'", + "At index 0 diff: b's' != b'e'", + "Full diff:", + "- b'eggs'", + "+ b'spam'", + ] + + def test_list(self) -> None: + expl = callequal([0, 1], [0, 2]) + assert expl is not None + assert len(expl) > 1 + + @pytest.mark.parametrize( + ["left", "right", "expected"], + [ + pytest.param( + [0, 1], + [0, 2], + """ + Full diff: + - [0, 2] + ? ^ + + [0, 1] + ? ^ + """, + id="lists", + ), + pytest.param( + {0: 1}, + {0: 2}, + """ + Full diff: + - {0: 2} + ? ^ + + {0: 1} + ? ^ + """, + id="dicts", + ), + pytest.param( + {0, 1}, + {0, 2}, + """ + Full diff: + - {0, 2} + ? ^ + + {0, 1} + ? ^ + """, + id="sets", + ), + ], + ) + def test_iterable_full_diff(self, left, right, expected) -> None: + """Test the full diff assertion failure explanation. + + When verbose is False, then just a -v notice to get the diff is rendered, + when verbose is True, then ndiff of the pprint is returned. + """ + expl = callequal(left, right, verbose=0) + assert expl is not None + assert expl[-1] == "Use -v to get the full diff" + verbose_expl = callequal(left, right, verbose=1) + assert verbose_expl is not None + assert "\n".join(verbose_expl).endswith(textwrap.dedent(expected).strip()) + + def test_iterable_full_diff_ci( + self, monkeypatch: MonkeyPatch, pytester: Pytester + ) -> None: + pytester.makepyfile( + r""" + def test_full_diff(): + left = [0, 1] + right = [0, 2] + assert left == right + """ + ) + monkeypatch.setenv("CI", "true") + result = pytester.runpytest() + result.stdout.fnmatch_lines(["E Full diff:"]) + + monkeypatch.delenv("CI", raising=False) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["E Use -v to get the full diff"]) + + def test_list_different_lengths(self) -> None: + expl = callequal([0, 1], [0, 1, 2]) + assert expl is not None + assert len(expl) > 1 + expl = callequal([0, 1, 2], [0, 1]) + assert expl is not None + assert len(expl) > 1 + + def test_list_wrap_for_multiple_lines(self) -> None: + long_d = "d" * 80 + l1 = ["a", "b", "c"] + l2 = ["a", "b", "c", long_d] + diff = callequal(l1, l2, verbose=True) + assert diff == [ + "['a', 'b', 'c'] == ['a', 'b', 'c...dddddddddddd']", + "Right contains one more item: '" + long_d + "'", + "Full diff:", + " [", + " 'a',", + " 'b',", + " 'c',", + "- '" + long_d + "',", + " ]", + ] + + diff = callequal(l2, l1, verbose=True) + assert diff == [ + "['a', 'b', 'c...dddddddddddd'] == ['a', 'b', 'c']", + "Left contains one more item: '" + long_d + "'", + "Full diff:", + " [", + " 'a',", + " 'b',", + " 'c',", + "+ '" + long_d + "',", + " ]", + ] + + def test_list_wrap_for_width_rewrap_same_length(self) -> None: + long_a = "a" * 30 + long_b = "b" * 30 + long_c = "c" * 30 + l1 = [long_a, long_b, long_c] + l2 = [long_b, long_c, long_a] + diff = callequal(l1, l2, verbose=True) + assert diff == [ + "['aaaaaaaaaaa...cccccccccccc'] == ['bbbbbbbbbbb...aaaaaaaaaaaa']", + "At index 0 diff: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' != 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'", + "Full diff:", + " [", + "+ 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',", + " 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',", + " 'cccccccccccccccccccccccccccccc',", + "- 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',", + " ]", + ] + + def test_list_dont_wrap_strings(self) -> None: + long_a = "a" * 10 + l1 = ["a"] + [long_a for _ in range(0, 7)] + l2 = ["should not get wrapped"] + diff = callequal(l1, l2, verbose=True) + assert diff == [ + "['a', 'aaaaaa...aaaaaaa', ...] == ['should not get wrapped']", + "At index 0 diff: 'a' != 'should not get wrapped'", + "Left contains 7 more items, first extra item: 'aaaaaaaaaa'", + "Full diff:", + " [", + "- 'should not get wrapped',", + "+ 'a',", + "+ 'aaaaaaaaaa',", + "+ 'aaaaaaaaaa',", + "+ 'aaaaaaaaaa',", + "+ 'aaaaaaaaaa',", + "+ 'aaaaaaaaaa',", + "+ 'aaaaaaaaaa',", + "+ 'aaaaaaaaaa',", + " ]", + ] + + def test_dict_wrap(self) -> None: + d1 = {"common": 1, "env": {"env1": 1, "env2": 2}} + d2 = {"common": 1, "env": {"env1": 1}} + + diff = callequal(d1, d2, verbose=True) + assert diff == [ + "{'common': 1,...1, 'env2': 2}} == {'common': 1,...: {'env1': 1}}", + "Omitting 1 identical items, use -vv to show", + "Differing items:", + "{'env': {'env1': 1, 'env2': 2}} != {'env': {'env1': 1}}", + "Full diff:", + "- {'common': 1, 'env': {'env1': 1}}", + "+ {'common': 1, 'env': {'env1': 1, 'env2': 2}}", + "? +++++++++++", + ] + + long_a = "a" * 80 + sub = {"long_a": long_a, "sub1": {"long_a": "substring that gets wrapped " * 2}} + d1 = {"env": {"sub": sub}} + d2 = {"env": {"sub": sub}, "new": 1} + diff = callequal(d1, d2, verbose=True) + assert diff == [ + "{'env': {'sub... wrapped '}}}} == {'env': {'sub...}}}, 'new': 1}", + "Omitting 1 identical items, use -vv to show", + "Right contains 1 more item:", + "{'new': 1}", + "Full diff:", + " {", + " 'env': {'sub': {'long_a': '" + long_a + "',", + " 'sub1': {'long_a': 'substring that gets wrapped substring '", + " 'that gets wrapped '}}},", + "- 'new': 1,", + " }", + ] + + def test_dict(self) -> None: + expl = callequal({"a": 0}, {"a": 1}) + assert expl is not None + assert len(expl) > 1 + + def test_dict_omitting(self) -> None: + lines = callequal({"a": 0, "b": 1}, {"a": 1, "b": 1}) + assert lines is not None + assert lines[1].startswith("Omitting 1 identical item") + assert "Common items" not in lines + for line in lines[1:]: + assert "b" not in line + + def test_dict_omitting_with_verbosity_1(self) -> None: + """Ensure differing items are visible for verbosity=1 (#1512).""" + lines = callequal({"a": 0, "b": 1}, {"a": 1, "b": 1}, verbose=1) + assert lines is not None + assert lines[1].startswith("Omitting 1 identical item") + assert lines[2].startswith("Differing items") + assert lines[3] == "{'a': 0} != {'a': 1}" + assert "Common items" not in lines + + def test_dict_omitting_with_verbosity_2(self) -> None: + lines = callequal({"a": 0, "b": 1}, {"a": 1, "b": 1}, verbose=2) + assert lines is not None + assert lines[1].startswith("Common items:") + assert "Omitting" not in lines[1] + assert lines[2] == "{'b': 1}" + + def test_dict_different_items(self) -> None: + lines = callequal({"a": 0}, {"b": 1, "c": 2}, verbose=2) + assert lines == [ + "{'a': 0} == {'b': 1, 'c': 2}", + "Left contains 1 more item:", + "{'a': 0}", + "Right contains 2 more items:", + "{'b': 1, 'c': 2}", + "Full diff:", + "- {'b': 1, 'c': 2}", + "+ {'a': 0}", + ] + lines = callequal({"b": 1, "c": 2}, {"a": 0}, verbose=2) + assert lines == [ + "{'b': 1, 'c': 2} == {'a': 0}", + "Left contains 2 more items:", + "{'b': 1, 'c': 2}", + "Right contains 1 more item:", + "{'a': 0}", + "Full diff:", + "- {'a': 0}", + "+ {'b': 1, 'c': 2}", + ] + + def test_sequence_different_items(self) -> None: + lines = callequal((1, 2), (3, 4, 5), verbose=2) + assert lines == [ + "(1, 2) == (3, 4, 5)", + "At index 0 diff: 1 != 3", + "Right contains one more item: 5", + "Full diff:", + "- (3, 4, 5)", + "+ (1, 2)", + ] + lines = callequal((1, 2, 3), (4,), verbose=2) + assert lines == [ + "(1, 2, 3) == (4,)", + "At index 0 diff: 1 != 4", + "Left contains 2 more items, first extra item: 2", + "Full diff:", + "- (4,)", + "+ (1, 2, 3)", + ] + + def test_set(self) -> None: + expl = callequal({0, 1}, {0, 2}) + assert expl is not None + assert len(expl) > 1 + + def test_frozenzet(self) -> None: + expl = callequal(frozenset([0, 1]), {0, 2}) + assert expl is not None + assert len(expl) > 1 + + def test_Sequence(self) -> None: + # Test comparing with a Sequence subclass. + class TestSequence(MutableSequence[int]): + def __init__(self, iterable): + self.elements = list(iterable) + + def __getitem__(self, item): + return self.elements[item] + + def __len__(self): + return len(self.elements) + + def __setitem__(self, item, value): + pass + + def __delitem__(self, item): + pass + + def insert(self, item, index): + pass + + expl = callequal(TestSequence([0, 1]), list([0, 2])) + assert expl is not None + assert len(expl) > 1 + + def test_list_tuples(self) -> None: + expl = callequal([], [(1, 2)]) + assert expl is not None + assert len(expl) > 1 + expl = callequal([(1, 2)], []) + assert expl is not None + assert len(expl) > 1 + + def test_repr_verbose(self) -> None: + class Nums: + def __init__(self, nums): + self.nums = nums + + def __repr__(self): + return str(self.nums) + + list_x = list(range(5000)) + list_y = list(range(5000)) + list_y[len(list_y) // 2] = 3 + nums_x = Nums(list_x) + nums_y = Nums(list_y) + + assert callequal(nums_x, nums_y) is None + + expl = callequal(nums_x, nums_y, verbose=1) + assert expl is not None + assert "+" + repr(nums_x) in expl + assert "-" + repr(nums_y) in expl + + expl = callequal(nums_x, nums_y, verbose=2) + assert expl is not None + assert "+" + repr(nums_x) in expl + assert "-" + repr(nums_y) in expl + + def test_list_bad_repr(self) -> None: + class A: + def __repr__(self): + raise ValueError(42) + + expl = callequal([], [A()]) + assert expl is not None + assert "ValueError" in "".join(expl) + expl = callequal({}, {"1": A()}, verbose=2) + assert expl is not None + assert expl[0].startswith("{} == <[ValueError") + assert "raised in repr" in expl[0] + assert expl[1:] == [ + "(pytest_assertion plugin: representation of details failed:" + " {}:{}: ValueError: 42.".format( + __file__, A.__repr__.__code__.co_firstlineno + 1 + ), + " Probably an object has a faulty __repr__.)", + ] + + def test_one_repr_empty(self) -> None: + """The faulty empty string repr did trigger an unbound local error in _diff_text.""" + + class A(str): + def __repr__(self): + return "" + + expl = callequal(A(), "") + assert not expl + + def test_repr_no_exc(self) -> None: + expl = callequal("foo", "bar") + assert expl is not None + assert "raised in repr()" not in " ".join(expl) + + def test_unicode(self) -> None: + assert callequal("£€", "£") == [ + "'£€' == '£'", + "- £", + "+ £€", + ] + + def test_nonascii_text(self) -> None: + """ + :issue: 877 + non ascii python2 str caused a UnicodeDecodeError + """ + + class A(str): + def __repr__(self): + return "\xff" + + expl = callequal(A(), "1") + assert expl == ["ÿ == '1'", "- 1"] + + def test_format_nonascii_explanation(self) -> None: + assert util.format_explanation("λ") + + def test_mojibake(self) -> None: + # issue 429 + left = b"e" + right = b"\xc3\xa9" + expl = callequal(left, right) + assert expl is not None + for line in expl: + assert isinstance(line, str) + msg = "\n".join(expl) + assert msg + + +class TestAssert_reprcompare_dataclass: + @pytest.mark.skipif(sys.version_info < (3, 7), reason="Dataclasses in Python3.7+") + def test_dataclasses(self, pytester: Pytester) -> None: + p = pytester.copy_example("dataclasses/test_compare_dataclasses.py") + result = pytester.runpytest(p) + result.assert_outcomes(failed=1, passed=0) + result.stdout.fnmatch_lines( + [ + "E Omitting 1 identical items, use -vv to show", + "E Differing attributes:", + "E ['field_b']", + "E ", + "E Drill down into differing attribute field_b:", + "E field_b: 'b' != 'c'...", + "E ", + "E ...Full output truncated (3 lines hidden), use '-vv' to show", + ], + consecutive=True, + ) + + @pytest.mark.skipif(sys.version_info < (3, 7), reason="Dataclasses in Python3.7+") + def test_recursive_dataclasses(self, pytester: Pytester) -> None: + p = pytester.copy_example("dataclasses/test_compare_recursive_dataclasses.py") + result = pytester.runpytest(p) + result.assert_outcomes(failed=1, passed=0) + result.stdout.fnmatch_lines( + [ + "E Omitting 1 identical items, use -vv to show", + "E Differing attributes:", + "E ['g', 'h', 'j']", + "E ", + "E Drill down into differing attribute g:", + "E g: S(a=10, b='ten') != S(a=20, b='xxx')...", + "E ", + "E ...Full output truncated (52 lines hidden), use '-vv' to show", + ], + consecutive=True, + ) + + @pytest.mark.skipif(sys.version_info < (3, 7), reason="Dataclasses in Python3.7+") + def test_recursive_dataclasses_verbose(self, pytester: Pytester) -> None: + p = pytester.copy_example("dataclasses/test_compare_recursive_dataclasses.py") + result = pytester.runpytest(p, "-vv") + result.assert_outcomes(failed=1, passed=0) + result.stdout.fnmatch_lines( + [ + "E Matching attributes:", + "E ['i']", + "E Differing attributes:", + "E ['g', 'h', 'j']", + "E ", + "E Drill down into differing attribute g:", + "E g: S(a=10, b='ten') != S(a=20, b='xxx')", + "E ", + "E Differing attributes:", + "E ['a', 'b']", + "E ", + "E Drill down into differing attribute a:", + "E a: 10 != 20", + "E +10", + "E -20", + "E ", + "E Drill down into differing attribute b:", + "E b: 'ten' != 'xxx'", + "E - xxx", + "E + ten", + "E ", + "E Drill down into differing attribute h:", + ], + consecutive=True, + ) + + @pytest.mark.skipif(sys.version_info < (3, 7), reason="Dataclasses in Python3.7+") + def test_dataclasses_verbose(self, pytester: Pytester) -> None: + p = pytester.copy_example("dataclasses/test_compare_dataclasses_verbose.py") + result = pytester.runpytest(p, "-vv") + result.assert_outcomes(failed=1, passed=0) + result.stdout.fnmatch_lines( + [ + "*Matching attributes:*", + "*['field_a']*", + "*Differing attributes:*", + "*field_b: 'b' != 'c'*", + ] + ) + + @pytest.mark.skipif(sys.version_info < (3, 7), reason="Dataclasses in Python3.7+") + def test_dataclasses_with_attribute_comparison_off( + self, pytester: Pytester + ) -> None: + p = pytester.copy_example( + "dataclasses/test_compare_dataclasses_field_comparison_off.py" + ) + result = pytester.runpytest(p, "-vv") + result.assert_outcomes(failed=0, passed=1) + + @pytest.mark.skipif(sys.version_info < (3, 7), reason="Dataclasses in Python3.7+") + def test_comparing_two_different_data_classes(self, pytester: Pytester) -> None: + p = pytester.copy_example( + "dataclasses/test_compare_two_different_dataclasses.py" + ) + result = pytester.runpytest(p, "-vv") + result.assert_outcomes(failed=0, passed=1) + + +class TestAssert_reprcompare_attrsclass: + def test_attrs(self) -> None: + @attr.s + class SimpleDataObject: + field_a = attr.ib() + field_b = attr.ib() + + left = SimpleDataObject(1, "b") + right = SimpleDataObject(1, "c") + + lines = callequal(left, right) + assert lines is not None + assert lines[2].startswith("Omitting 1 identical item") + assert "Matching attributes" not in lines + for line in lines[2:]: + assert "field_a" not in line + + def test_attrs_recursive(self) -> None: + @attr.s + class OtherDataObject: + field_c = attr.ib() + field_d = attr.ib() + + @attr.s + class SimpleDataObject: + field_a = attr.ib() + field_b = attr.ib() + + left = SimpleDataObject(OtherDataObject(1, "a"), "b") + right = SimpleDataObject(OtherDataObject(1, "b"), "b") + + lines = callequal(left, right) + assert lines is not None + assert "Matching attributes" not in lines + for line in lines[1:]: + assert "field_b:" not in line + assert "field_c:" not in line + + def test_attrs_recursive_verbose(self) -> None: + @attr.s + class OtherDataObject: + field_c = attr.ib() + field_d = attr.ib() + + @attr.s + class SimpleDataObject: + field_a = attr.ib() + field_b = attr.ib() + + left = SimpleDataObject(OtherDataObject(1, "a"), "b") + right = SimpleDataObject(OtherDataObject(1, "b"), "b") + + lines = callequal(left, right) + assert lines is not None + # indentation in output because of nested object structure + assert " field_d: 'a' != 'b'" in lines + + def test_attrs_verbose(self) -> None: + @attr.s + class SimpleDataObject: + field_a = attr.ib() + field_b = attr.ib() + + left = SimpleDataObject(1, "b") + right = SimpleDataObject(1, "c") + + lines = callequal(left, right, verbose=2) + assert lines is not None + assert lines[2].startswith("Matching attributes:") + assert "Omitting" not in lines[2] + assert lines[3] == "['field_a']" + + def test_attrs_with_attribute_comparison_off(self) -> None: + @attr.s + class SimpleDataObject: + field_a = attr.ib() + field_b = attr.ib(eq=False) + + left = SimpleDataObject(1, "b") + right = SimpleDataObject(1, "b") + + lines = callequal(left, right, verbose=2) + print(lines) + assert lines is not None + assert lines[2].startswith("Matching attributes:") + assert "Omitting" not in lines[1] + assert lines[3] == "['field_a']" + for line in lines[3:]: + assert "field_b" not in line + + def test_comparing_two_different_attrs_classes(self) -> None: + @attr.s + class SimpleDataObjectOne: + field_a = attr.ib() + field_b = attr.ib() + + @attr.s + class SimpleDataObjectTwo: + field_a = attr.ib() + field_b = attr.ib() + + left = SimpleDataObjectOne(1, "b") + right = SimpleDataObjectTwo(1, "c") + + lines = callequal(left, right) + assert lines is None + + +class TestAssert_reprcompare_namedtuple: + def test_namedtuple(self) -> None: + NT = collections.namedtuple("NT", ["a", "b"]) + + left = NT(1, "b") + right = NT(1, "c") + + lines = callequal(left, right) + assert lines == [ + "NT(a=1, b='b') == NT(a=1, b='c')", + "", + "Omitting 1 identical items, use -vv to show", + "Differing attributes:", + "['b']", + "", + "Drill down into differing attribute b:", + " b: 'b' != 'c'", + " - c", + " + b", + "Use -v to get the full diff", + ] + + def test_comparing_two_different_namedtuple(self) -> None: + NT1 = collections.namedtuple("NT1", ["a", "b"]) + NT2 = collections.namedtuple("NT2", ["a", "b"]) + + left = NT1(1, "b") + right = NT2(2, "b") + + lines = callequal(left, right) + # Because the types are different, uses the generic sequence matcher. + assert lines == [ + "NT1(a=1, b='b') == NT2(a=2, b='b')", + "At index 0 diff: 1 != 2", + "Use -v to get the full diff", + ] + + +class TestFormatExplanation: + def test_special_chars_full(self, pytester: Pytester) -> None: + # Issue 453, for the bug this would raise IndexError + pytester.makepyfile( + """ + def test_foo(): + assert '\\n}' == '' + """ + ) + result = pytester.runpytest() + assert result.ret == 1 + result.stdout.fnmatch_lines(["*AssertionError*"]) + + def test_fmt_simple(self) -> None: + expl = "assert foo" + assert util.format_explanation(expl) == "assert foo" + + def test_fmt_where(self) -> None: + expl = "\n".join(["assert 1", "{1 = foo", "} == 2"]) + res = "\n".join(["assert 1 == 2", " + where 1 = foo"]) + assert util.format_explanation(expl) == res + + def test_fmt_and(self) -> None: + expl = "\n".join(["assert 1", "{1 = foo", "} == 2", "{2 = bar", "}"]) + res = "\n".join(["assert 1 == 2", " + where 1 = foo", " + and 2 = bar"]) + assert util.format_explanation(expl) == res + + def test_fmt_where_nested(self) -> None: + expl = "\n".join(["assert 1", "{1 = foo", "{foo = bar", "}", "} == 2"]) + res = "\n".join(["assert 1 == 2", " + where 1 = foo", " + where foo = bar"]) + assert util.format_explanation(expl) == res + + def test_fmt_newline(self) -> None: + expl = "\n".join(['assert "foo" == "bar"', "~- foo", "~+ bar"]) + res = "\n".join(['assert "foo" == "bar"', " - foo", " + bar"]) + assert util.format_explanation(expl) == res + + def test_fmt_newline_escaped(self) -> None: + expl = "\n".join(["assert foo == bar", "baz"]) + res = "assert foo == bar\\nbaz" + assert util.format_explanation(expl) == res + + def test_fmt_newline_before_where(self) -> None: + expl = "\n".join( + [ + "the assertion message here", + ">assert 1", + "{1 = foo", + "} == 2", + "{2 = bar", + "}", + ] + ) + res = "\n".join( + [ + "the assertion message here", + "assert 1 == 2", + " + where 1 = foo", + " + and 2 = bar", + ] + ) + assert util.format_explanation(expl) == res + + def test_fmt_multi_newline_before_where(self) -> None: + expl = "\n".join( + [ + "the assertion", + "~message here", + ">assert 1", + "{1 = foo", + "} == 2", + "{2 = bar", + "}", + ] + ) + res = "\n".join( + [ + "the assertion", + " message here", + "assert 1 == 2", + " + where 1 = foo", + " + and 2 = bar", + ] + ) + assert util.format_explanation(expl) == res + + +class TestTruncateExplanation: + # The number of lines in the truncation explanation message. Used + # to calculate that results have the expected length. + LINES_IN_TRUNCATION_MSG = 2 + + def test_doesnt_truncate_when_input_is_empty_list(self) -> None: + expl: List[str] = [] + result = truncate._truncate_explanation(expl, max_lines=8, max_chars=100) + assert result == expl + + def test_doesnt_truncate_at_when_input_is_5_lines_and_LT_max_chars(self) -> None: + expl = ["a" * 100 for x in range(5)] + result = truncate._truncate_explanation(expl, max_lines=8, max_chars=8 * 80) + assert result == expl + + def test_truncates_at_8_lines_when_given_list_of_empty_strings(self) -> None: + expl = ["" for x in range(50)] + result = truncate._truncate_explanation(expl, max_lines=8, max_chars=100) + assert result != expl + assert len(result) == 8 + self.LINES_IN_TRUNCATION_MSG + assert "Full output truncated" in result[-1] + assert "43 lines hidden" in result[-1] + last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] + assert last_line_before_trunc_msg.endswith("...") + + def test_truncates_at_8_lines_when_first_8_lines_are_LT_max_chars(self) -> None: + expl = ["a" for x in range(100)] + result = truncate._truncate_explanation(expl, max_lines=8, max_chars=8 * 80) + assert result != expl + assert len(result) == 8 + self.LINES_IN_TRUNCATION_MSG + assert "Full output truncated" in result[-1] + assert "93 lines hidden" in result[-1] + last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] + assert last_line_before_trunc_msg.endswith("...") + + def test_truncates_at_8_lines_when_first_8_lines_are_EQ_max_chars(self) -> None: + expl = ["a" * 80 for x in range(16)] + result = truncate._truncate_explanation(expl, max_lines=8, max_chars=8 * 80) + assert result != expl + assert len(result) == 8 + self.LINES_IN_TRUNCATION_MSG + assert "Full output truncated" in result[-1] + assert "9 lines hidden" in result[-1] + last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] + assert last_line_before_trunc_msg.endswith("...") + + def test_truncates_at_4_lines_when_first_4_lines_are_GT_max_chars(self) -> None: + expl = ["a" * 250 for x in range(10)] + result = truncate._truncate_explanation(expl, max_lines=8, max_chars=999) + assert result != expl + assert len(result) == 4 + self.LINES_IN_TRUNCATION_MSG + assert "Full output truncated" in result[-1] + assert "7 lines hidden" in result[-1] + last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] + assert last_line_before_trunc_msg.endswith("...") + + def test_truncates_at_1_line_when_first_line_is_GT_max_chars(self) -> None: + expl = ["a" * 250 for x in range(1000)] + result = truncate._truncate_explanation(expl, max_lines=8, max_chars=100) + assert result != expl + assert len(result) == 1 + self.LINES_IN_TRUNCATION_MSG + assert "Full output truncated" in result[-1] + assert "1000 lines hidden" in result[-1] + last_line_before_trunc_msg = result[-self.LINES_IN_TRUNCATION_MSG - 1] + assert last_line_before_trunc_msg.endswith("...") + + def test_full_output_truncated(self, monkeypatch, pytester: Pytester) -> None: + """Test against full runpytest() output.""" + + line_count = 7 + line_len = 100 + expected_truncated_lines = 2 + pytester.makepyfile( + r""" + def test_many_lines(): + a = list([str(i)[0] * %d for i in range(%d)]) + b = a[::2] + a = '\n'.join(map(str, a)) + b = '\n'.join(map(str, b)) + assert a == b + """ + % (line_len, line_count) + ) + monkeypatch.delenv("CI", raising=False) + + result = pytester.runpytest() + # without -vv, truncate the message showing a few diff lines only + result.stdout.fnmatch_lines( + [ + "*+ 1*", + "*+ 3*", + "*+ 5*", + "*truncated (%d lines hidden)*use*-vv*" % expected_truncated_lines, + ] + ) + + result = pytester.runpytest("-vv") + result.stdout.fnmatch_lines(["* 6*"]) + + monkeypatch.setenv("CI", "1") + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 6*"]) + + +def test_python25_compile_issue257(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_rewritten(): + assert 1 == 2 + # some comment + """ + ) + result = pytester.runpytest() + assert result.ret == 1 + result.stdout.fnmatch_lines( + """ + *E*assert 1 == 2* + *1 failed* + """ + ) + + +def test_rewritten(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_rewritten(): + assert "@py_builtins" in globals() + """ + ) + assert pytester.runpytest().ret == 0 + + +def test_reprcompare_notin() -> None: + assert callop("not in", "foo", "aaafoobbb") == [ + "'foo' not in 'aaafoobbb'", + "'foo' is contained here:", + " aaafoobbb", + "? +++", + ] + + +def test_reprcompare_whitespaces() -> None: + assert callequal("\r\n", "\n") == [ + r"'\r\n' == '\n'", + r"Strings contain only whitespace, escaping them using repr()", + r"- '\n'", + r"+ '\r\n'", + r"? ++", + ] + + +def test_pytest_assertrepr_compare_integration(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_hello(): + x = set(range(100)) + y = x.copy() + y.remove(50) + assert x == y + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*def test_hello():*", + "*assert x == y*", + "*E*Extra items*left*", + "*E*50*", + "*= 1 failed in*", + ] + ) + + +def test_sequence_comparison_uses_repr(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_hello(): + x = set("hello x") + y = set("hello y") + assert x == y + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*def test_hello():*", + "*assert x == y*", + "*E*Extra items*left*", + "*E*'x'*", + "*E*Extra items*right*", + "*E*'y'*", + ] + ) + + +def test_assertrepr_loaded_per_dir(pytester: Pytester) -> None: + pytester.makepyfile(test_base=["def test_base(): assert 1 == 2"]) + a = pytester.mkdir("a") + a.joinpath("test_a.py").write_text("def test_a(): assert 1 == 2") + a.joinpath("conftest.py").write_text( + 'def pytest_assertrepr_compare(): return ["summary a"]' + ) + b = pytester.mkdir("b") + b.joinpath("test_b.py").write_text("def test_b(): assert 1 == 2") + b.joinpath("conftest.py").write_text( + 'def pytest_assertrepr_compare(): return ["summary b"]' + ) + + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*def test_base():*", + "*E*assert 1 == 2*", + "*def test_a():*", + "*E*assert summary a*", + "*def test_b():*", + "*E*assert summary b*", + ] + ) + + +def test_assertion_options(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_hello(): + x = 3 + assert x == 4 + """ + ) + result = pytester.runpytest() + assert "3 == 4" in result.stdout.str() + result = pytester.runpytest_subprocess("--assert=plain") + result.stdout.no_fnmatch_line("*3 == 4*") + + +def test_triple_quoted_string_issue113(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_hello(): + assert "" == ''' + '''""" + ) + result = pytester.runpytest("--fulltrace") + result.stdout.fnmatch_lines(["*1 failed*"]) + result.stdout.no_fnmatch_line("*SyntaxError*") + + +def test_traceback_failure(pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def g(): + return 2 + def f(x): + assert x == g() + def test_onefails(): + f(3) + """ + ) + result = pytester.runpytest(p1, "--tb=long") + result.stdout.fnmatch_lines( + [ + "*test_traceback_failure.py F*", + "====* FAILURES *====", + "____*____", + "", + " def test_onefails():", + "> f(3)", + "", + "*test_*.py:6: ", + "_ _ _ *", + # "", + " def f(x):", + "> assert x == g()", + "E assert 3 == 2", + "E + where 2 = g()", + "", + "*test_traceback_failure.py:4: AssertionError", + ] + ) + + result = pytester.runpytest(p1) # "auto" + result.stdout.fnmatch_lines( + [ + "*test_traceback_failure.py F*", + "====* FAILURES *====", + "____*____", + "", + " def test_onefails():", + "> f(3)", + "", + "*test_*.py:6: ", + "", + " def f(x):", + "> assert x == g()", + "E assert 3 == 2", + "E + where 2 = g()", + "", + "*test_traceback_failure.py:4: AssertionError", + ] + ) + + +def test_exception_handling_no_traceback(pytester: Pytester) -> None: + """Handle chain exceptions in tasks submitted by the multiprocess module (#1984).""" + p1 = pytester.makepyfile( + """ + from multiprocessing import Pool + + def process_task(n): + assert n == 10 + + def multitask_job(): + tasks = [1] + with Pool(processes=1) as pool: + pool.map(process_task, tasks) + + def test_multitask_job(): + multitask_job() + """ + ) + pytester.syspathinsert() + result = pytester.runpytest(p1, "--tb=long") + result.stdout.fnmatch_lines( + [ + "====* FAILURES *====", + "*multiprocessing.pool.RemoteTraceback:*", + "Traceback (most recent call last):", + "*assert n == 10", + "The above exception was the direct cause of the following exception:", + "> * multitask_job()", + ] + ) + + +@pytest.mark.skipif("'__pypy__' in sys.builtin_module_names") +@pytest.mark.parametrize( + "cmdline_args, warning_output", + [ + ( + ["-OO", "-m", "pytest", "-h"], + ["warning :*PytestConfigWarning:*assert statements are not executed*"], + ), + ( + ["-OO", "-m", "pytest"], + [ + "=*= warnings summary =*=", + "*PytestConfigWarning:*assert statements are not executed*", + ], + ), + ( + ["-OO", "-m", "pytest", "--assert=plain"], + [ + "=*= warnings summary =*=", + "*PytestConfigWarning: ASSERTIONS ARE NOT EXECUTED and FAILING TESTS WILL PASS. " + "Are you using python -O?", + ], + ), + ], +) +def test_warn_missing(pytester: Pytester, cmdline_args, warning_output) -> None: + pytester.makepyfile("") + + result = pytester.run(sys.executable, *cmdline_args) + result.stdout.fnmatch_lines(warning_output) + + +def test_recursion_source_decode(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_something(): + pass + """ + ) + pytester.makeini( + """ + [pytest] + python_files = *.py + """ + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines( + """ + + """ + ) + + +def test_AssertionError_message(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_hello(): + x,y = 1,2 + assert 0, (x,y) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + """ + *def test_hello* + *assert 0, (x,y)* + *AssertionError: (1, 2)* + """ + ) + + +def test_diff_newline_at_end(pytester: Pytester) -> None: + pytester.makepyfile( + r""" + def test_diff(): + assert 'asdf' == 'asdf\n' + """ + ) + + result = pytester.runpytest() + result.stdout.fnmatch_lines( + r""" + *assert 'asdf' == 'asdf\n' + * - asdf + * ? - + * + asdf + """ + ) + + +@pytest.mark.filterwarnings("default") +def test_assert_tuple_warning(pytester: Pytester) -> None: + msg = "assertion is always true" + pytester.makepyfile( + """ + def test_tuple(): + assert(False, 'you shall not pass') + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines([f"*test_assert_tuple_warning.py:2:*{msg}*"]) + + # tuples with size != 2 should not trigger the warning + pytester.makepyfile( + """ + def test_tuple(): + assert () + """ + ) + result = pytester.runpytest() + assert msg not in result.stdout.str() + + +def test_assert_indirect_tuple_no_warning(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_tuple(): + tpl = ('foo', 'bar') + assert tpl + """ + ) + result = pytester.runpytest() + output = "\n".join(result.stdout.lines) + assert "WR1" not in output + + +def test_assert_with_unicode(pytester: Pytester) -> None: + pytester.makepyfile( + """\ + def test_unicode(): + assert '유니코드' == 'Unicode' + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*AssertionError*"]) + + +def test_raise_unprintable_assertion_error(pytester: Pytester) -> None: + pytester.makepyfile( + r""" + def test_raise_assertion_error(): + raise AssertionError('\xff') + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [r"> raise AssertionError('\xff')", "E AssertionError: *"] + ) + + +def test_raise_assertion_error_raising_repr(pytester: Pytester) -> None: + pytester.makepyfile( + """ + class RaisingRepr(object): + def __repr__(self): + raise Exception() + def test_raising_repr(): + raise AssertionError(RaisingRepr()) + """ + ) + result = pytester.runpytest() + if sys.version_info >= (3, 11): + # python 3.11 has native support for un-str-able exceptions + result.stdout.fnmatch_lines( + ["E AssertionError: "] + ) + else: + result.stdout.fnmatch_lines( + ["E AssertionError: "] + ) + + +def test_issue_1944(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def f(): + return + + assert f() == 10 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 error*"]) + assert ( + "AttributeError: 'Module' object has no attribute '_obj'" + not in result.stdout.str() + ) + + +def test_exit_from_assertrepr_compare(monkeypatch) -> None: + def raise_exit(obj): + outcomes.exit("Quitting debugger") + + monkeypatch.setattr(util, "istext", raise_exit) + + with pytest.raises(outcomes.Exit, match="Quitting debugger"): + callequal(1, 1) + + +def test_assertion_location_with_coverage(pytester: Pytester) -> None: + """This used to report the wrong location when run with coverage (#5754).""" + p = pytester.makepyfile( + """ + def test(): + assert False, 1 + assert False, 2 + """ + ) + result = pytester.runpytest(str(p)) + result.stdout.fnmatch_lines( + [ + "> assert False, 1", + "E AssertionError: 1", + "E assert False", + "*= 1 failed in*", + ] + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_assertrewrite.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_assertrewrite.py new file mode 100644 index 0000000000..4417eb4350 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_assertrewrite.py @@ -0,0 +1,1841 @@ +import ast +import errno +import glob +import importlib +import marshal +import os +import py_compile +import stat +import sys +import textwrap +import zipfile +from functools import partial +from pathlib import Path +from typing import cast +from typing import Dict +from typing import List +from typing import Mapping +from typing import Optional +from typing import Set + +import _pytest._code +import pytest +from _pytest._io.saferepr import DEFAULT_REPR_MAX_SIZE +from _pytest.assertion import util +from _pytest.assertion.rewrite import _get_assertion_exprs +from _pytest.assertion.rewrite import _get_maxsize_for_saferepr +from _pytest.assertion.rewrite import AssertionRewritingHook +from _pytest.assertion.rewrite import get_cache_dir +from _pytest.assertion.rewrite import PYC_TAIL +from _pytest.assertion.rewrite import PYTEST_TAG +from _pytest.assertion.rewrite import rewrite_asserts +from _pytest.config import Config +from _pytest.config import ExitCode +from _pytest.pathlib import make_numbered_dir +from _pytest.pytester import Pytester + + +def rewrite(src: str) -> ast.Module: + tree = ast.parse(src) + rewrite_asserts(tree, src.encode()) + return tree + + +def getmsg( + f, extra_ns: Optional[Mapping[str, object]] = None, *, must_pass: bool = False +) -> Optional[str]: + """Rewrite the assertions in f, run it, and get the failure message.""" + src = "\n".join(_pytest._code.Code.from_function(f).source().lines) + mod = rewrite(src) + code = compile(mod, "", "exec") + ns: Dict[str, object] = {} + if extra_ns is not None: + ns.update(extra_ns) + exec(code, ns) + func = ns[f.__name__] + try: + func() # type: ignore[operator] + except AssertionError: + if must_pass: + pytest.fail("shouldn't have raised") + s = str(sys.exc_info()[1]) + if not s.startswith("assert"): + return "AssertionError: " + s + return s + else: + if not must_pass: + pytest.fail("function didn't raise at all") + return None + + +class TestAssertionRewrite: + def test_place_initial_imports(self) -> None: + s = """'Doc string'\nother = stuff""" + m = rewrite(s) + assert isinstance(m.body[0], ast.Expr) + for imp in m.body[1:3]: + assert isinstance(imp, ast.Import) + assert imp.lineno == 2 + assert imp.col_offset == 0 + assert isinstance(m.body[3], ast.Assign) + s = """from __future__ import division\nother_stuff""" + m = rewrite(s) + assert isinstance(m.body[0], ast.ImportFrom) + for imp in m.body[1:3]: + assert isinstance(imp, ast.Import) + assert imp.lineno == 2 + assert imp.col_offset == 0 + assert isinstance(m.body[3], ast.Expr) + s = """'doc string'\nfrom __future__ import division""" + m = rewrite(s) + assert isinstance(m.body[0], ast.Expr) + assert isinstance(m.body[1], ast.ImportFrom) + for imp in m.body[2:4]: + assert isinstance(imp, ast.Import) + assert imp.lineno == 2 + assert imp.col_offset == 0 + s = """'doc string'\nfrom __future__ import division\nother""" + m = rewrite(s) + assert isinstance(m.body[0], ast.Expr) + assert isinstance(m.body[1], ast.ImportFrom) + for imp in m.body[2:4]: + assert isinstance(imp, ast.Import) + assert imp.lineno == 3 + assert imp.col_offset == 0 + assert isinstance(m.body[4], ast.Expr) + s = """from . import relative\nother_stuff""" + m = rewrite(s) + for imp in m.body[:2]: + assert isinstance(imp, ast.Import) + assert imp.lineno == 1 + assert imp.col_offset == 0 + assert isinstance(m.body[3], ast.Expr) + + def test_location_is_set(self) -> None: + s = textwrap.dedent( + """ + + assert False, ( + + "Ouch" + ) + + """ + ) + m = rewrite(s) + for node in m.body: + if isinstance(node, ast.Import): + continue + for n in [node, *ast.iter_child_nodes(node)]: + assert n.lineno == 3 + assert n.col_offset == 0 + if sys.version_info >= (3, 8): + assert n.end_lineno == 6 + assert n.end_col_offset == 3 + + def test_dont_rewrite(self) -> None: + s = """'PYTEST_DONT_REWRITE'\nassert 14""" + m = rewrite(s) + assert len(m.body) == 2 + assert isinstance(m.body[1], ast.Assert) + assert m.body[1].msg is None + + def test_dont_rewrite_plugin(self, pytester: Pytester) -> None: + contents = { + "conftest.py": "pytest_plugins = 'plugin'; import plugin", + "plugin.py": "'PYTEST_DONT_REWRITE'", + "test_foo.py": "def test_foo(): pass", + } + pytester.makepyfile(**contents) + result = pytester.runpytest_subprocess() + assert "warning" not in "".join(result.outlines) + + def test_rewrites_plugin_as_a_package(self, pytester: Pytester) -> None: + pkgdir = pytester.mkpydir("plugin") + pkgdir.joinpath("__init__.py").write_text( + "import pytest\n" + "@pytest.fixture\n" + "def special_asserter():\n" + " def special_assert(x, y):\n" + " assert x == y\n" + " return special_assert\n" + ) + pytester.makeconftest('pytest_plugins = ["plugin"]') + pytester.makepyfile("def test(special_asserter): special_asserter(1, 2)\n") + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*assert 1 == 2*"]) + + def test_honors_pep_235(self, pytester: Pytester, monkeypatch) -> None: + # note: couldn't make it fail on macos with a single `sys.path` entry + # note: these modules are named `test_*` to trigger rewriting + pytester.makepyfile(test_y="x = 1") + xdir = pytester.mkdir("x") + pytester.mkpydir(str(xdir.joinpath("test_Y"))) + xdir.joinpath("test_Y").joinpath("__init__.py").write_text("x = 2") + pytester.makepyfile( + "import test_y\n" + "import test_Y\n" + "def test():\n" + " assert test_y.x == 1\n" + " assert test_Y.x == 2\n" + ) + monkeypatch.syspath_prepend(str(xdir)) + pytester.runpytest().assert_outcomes(passed=1) + + def test_name(self, request) -> None: + def f1() -> None: + assert False + + assert getmsg(f1) == "assert False" + + def f2() -> None: + f = False + assert f + + assert getmsg(f2) == "assert False" + + def f3() -> None: + assert a_global # type: ignore[name-defined] # noqa + + assert getmsg(f3, {"a_global": False}) == "assert False" + + def f4() -> None: + assert sys == 42 # type: ignore[comparison-overlap] + + verbose = request.config.getoption("verbose") + msg = getmsg(f4, {"sys": sys}) + if verbose > 0: + assert msg == ( + "assert == 42\n" + " +\n" + " -42" + ) + else: + assert msg == "assert sys == 42" + + def f5() -> None: + assert cls == 42 # type: ignore[name-defined] # noqa: F821 + + class X: + pass + + msg = getmsg(f5, {"cls": X}) + assert msg is not None + lines = msg.splitlines() + if verbose > 1: + assert lines == [ + f"assert {X!r} == 42", + f" +{X!r}", + " -42", + ] + elif verbose > 0: + assert lines == [ + "assert .X'> == 42", + f" +{X!r}", + " -42", + ] + else: + assert lines == ["assert cls == 42"] + + def test_assertrepr_compare_same_width(self, request) -> None: + """Should use same width/truncation with same initial width.""" + + def f() -> None: + assert "1234567890" * 5 + "A" == "1234567890" * 5 + "B" + + msg = getmsg(f) + assert msg is not None + line = msg.splitlines()[0] + if request.config.getoption("verbose") > 1: + assert line == ( + "assert '12345678901234567890123456789012345678901234567890A' " + "== '12345678901234567890123456789012345678901234567890B'" + ) + else: + assert line == ( + "assert '123456789012...901234567890A' " + "== '123456789012...901234567890B'" + ) + + def test_dont_rewrite_if_hasattr_fails(self, request) -> None: + class Y: + """A class whose getattr fails, but not with `AttributeError`.""" + + def __getattr__(self, attribute_name): + raise KeyError() + + def __repr__(self) -> str: + return "Y" + + def __init__(self) -> None: + self.foo = 3 + + def f() -> None: + assert cls().foo == 2 # type: ignore[name-defined] # noqa: F821 + + # XXX: looks like the "where" should also be there in verbose mode?! + msg = getmsg(f, {"cls": Y}) + assert msg is not None + lines = msg.splitlines() + if request.config.getoption("verbose") > 0: + assert lines == ["assert 3 == 2", " +3", " -2"] + else: + assert lines == [ + "assert 3 == 2", + " + where 3 = Y.foo", + " + where Y = cls()", + ] + + def test_assert_already_has_message(self) -> None: + def f(): + assert False, "something bad!" + + assert getmsg(f) == "AssertionError: something bad!\nassert False" + + def test_assertion_message(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_foo(): + assert 1 == 2, "The failure message" + """ + ) + result = pytester.runpytest() + assert result.ret == 1 + result.stdout.fnmatch_lines( + ["*AssertionError*The failure message*", "*assert 1 == 2*"] + ) + + def test_assertion_message_multiline(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_foo(): + assert 1 == 2, "A multiline\\nfailure message" + """ + ) + result = pytester.runpytest() + assert result.ret == 1 + result.stdout.fnmatch_lines( + ["*AssertionError*A multiline*", "*failure message*", "*assert 1 == 2*"] + ) + + def test_assertion_message_tuple(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_foo(): + assert 1 == 2, (1, 2) + """ + ) + result = pytester.runpytest() + assert result.ret == 1 + result.stdout.fnmatch_lines( + ["*AssertionError*%s*" % repr((1, 2)), "*assert 1 == 2*"] + ) + + def test_assertion_message_expr(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_foo(): + assert 1 == 2, 1 + 2 + """ + ) + result = pytester.runpytest() + assert result.ret == 1 + result.stdout.fnmatch_lines(["*AssertionError*3*", "*assert 1 == 2*"]) + + def test_assertion_message_escape(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_foo(): + assert 1 == 2, 'To be escaped: %' + """ + ) + result = pytester.runpytest() + assert result.ret == 1 + result.stdout.fnmatch_lines( + ["*AssertionError: To be escaped: %", "*assert 1 == 2"] + ) + + def test_assertion_messages_bytes(self, pytester: Pytester) -> None: + pytester.makepyfile("def test_bytes_assertion():\n assert False, b'ohai!'\n") + result = pytester.runpytest() + assert result.ret == 1 + result.stdout.fnmatch_lines(["*AssertionError: b'ohai!'", "*assert False"]) + + def test_boolop(self) -> None: + def f1() -> None: + f = g = False + assert f and g + + assert getmsg(f1) == "assert (False)" + + def f2() -> None: + f = True + g = False + assert f and g + + assert getmsg(f2) == "assert (True and False)" + + def f3() -> None: + f = False + g = True + assert f and g + + assert getmsg(f3) == "assert (False)" + + def f4() -> None: + f = g = False + assert f or g + + assert getmsg(f4) == "assert (False or False)" + + def f5() -> None: + f = g = False + assert not f and not g + + getmsg(f5, must_pass=True) + + def x() -> bool: + return False + + def f6() -> None: + assert x() and x() + + assert ( + getmsg(f6, {"x": x}) + == """assert (False) + + where False = x()""" + ) + + def f7() -> None: + assert False or x() + + assert ( + getmsg(f7, {"x": x}) + == """assert (False or False) + + where False = x()""" + ) + + def f8() -> None: + assert 1 in {} and 2 in {} + + assert getmsg(f8) == "assert (1 in {})" + + def f9() -> None: + x = 1 + y = 2 + assert x in {1: None} and y in {} + + assert getmsg(f9) == "assert (1 in {1: None} and 2 in {})" + + def f10() -> None: + f = True + g = False + assert f or g + + getmsg(f10, must_pass=True) + + def f11() -> None: + f = g = h = lambda: True + assert f() and g() and h() + + getmsg(f11, must_pass=True) + + def test_short_circuit_evaluation(self) -> None: + def f1() -> None: + assert True or explode # type: ignore[name-defined,unreachable] # noqa: F821 + + getmsg(f1, must_pass=True) + + def f2() -> None: + x = 1 + assert x == 1 or x == 2 + + getmsg(f2, must_pass=True) + + def test_unary_op(self) -> None: + def f1() -> None: + x = True + assert not x + + assert getmsg(f1) == "assert not True" + + def f2() -> None: + x = 0 + assert ~x + 1 + + assert getmsg(f2) == "assert (~0 + 1)" + + def f3() -> None: + x = 3 + assert -x + x + + assert getmsg(f3) == "assert (-3 + 3)" + + def f4() -> None: + x = 0 + assert +x + x + + assert getmsg(f4) == "assert (+0 + 0)" + + def test_binary_op(self) -> None: + def f1() -> None: + x = 1 + y = -1 + assert x + y + + assert getmsg(f1) == "assert (1 + -1)" + + def f2() -> None: + assert not 5 % 4 + + assert getmsg(f2) == "assert not (5 % 4)" + + def test_boolop_percent(self) -> None: + def f1() -> None: + assert 3 % 2 and False + + assert getmsg(f1) == "assert ((3 % 2) and False)" + + def f2() -> None: + assert False or 4 % 2 + + assert getmsg(f2) == "assert (False or (4 % 2))" + + def test_at_operator_issue1290(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + class Matrix(object): + def __init__(self, num): + self.num = num + def __matmul__(self, other): + return self.num * other.num + + def test_multmat_operator(): + assert Matrix(2) @ Matrix(3) == 6""" + ) + pytester.runpytest().assert_outcomes(passed=1) + + def test_starred_with_side_effect(self, pytester: Pytester) -> None: + """See #4412""" + pytester.makepyfile( + """\ + def test(): + f = lambda x: x + x = iter([1, 2, 3]) + assert 2 * next(x) == f(*[next(x)]) + """ + ) + pytester.runpytest().assert_outcomes(passed=1) + + def test_call(self) -> None: + def g(a=42, *args, **kwargs) -> bool: + return False + + ns = {"g": g} + + def f1() -> None: + assert g() + + assert ( + getmsg(f1, ns) + == """assert False + + where False = g()""" + ) + + def f2() -> None: + assert g(1) + + assert ( + getmsg(f2, ns) + == """assert False + + where False = g(1)""" + ) + + def f3() -> None: + assert g(1, 2) + + assert ( + getmsg(f3, ns) + == """assert False + + where False = g(1, 2)""" + ) + + def f4() -> None: + assert g(1, g=42) + + assert ( + getmsg(f4, ns) + == """assert False + + where False = g(1, g=42)""" + ) + + def f5() -> None: + assert g(1, 3, g=23) + + assert ( + getmsg(f5, ns) + == """assert False + + where False = g(1, 3, g=23)""" + ) + + def f6() -> None: + seq = [1, 2, 3] + assert g(*seq) + + assert ( + getmsg(f6, ns) + == """assert False + + where False = g(*[1, 2, 3])""" + ) + + def f7() -> None: + x = "a" + assert g(**{x: 2}) + + assert ( + getmsg(f7, ns) + == """assert False + + where False = g(**{'a': 2})""" + ) + + def test_attribute(self) -> None: + class X: + g = 3 + + ns = {"x": X} + + def f1() -> None: + assert not x.g # type: ignore[name-defined] # noqa: F821 + + assert ( + getmsg(f1, ns) + == """assert not 3 + + where 3 = x.g""" + ) + + def f2() -> None: + x.a = False # type: ignore[name-defined] # noqa: F821 + assert x.a # type: ignore[name-defined] # noqa: F821 + + assert ( + getmsg(f2, ns) + == """assert False + + where False = x.a""" + ) + + def test_comparisons(self) -> None: + def f1() -> None: + a, b = range(2) + assert b < a + + assert getmsg(f1) == """assert 1 < 0""" + + def f2() -> None: + a, b, c = range(3) + assert a > b > c + + assert getmsg(f2) == """assert 0 > 1""" + + def f3() -> None: + a, b, c = range(3) + assert a < b > c + + assert getmsg(f3) == """assert 1 > 2""" + + def f4() -> None: + a, b, c = range(3) + assert a < b <= c + + getmsg(f4, must_pass=True) + + def f5() -> None: + a, b, c = range(3) + assert a < b + assert b < c + + getmsg(f5, must_pass=True) + + def test_len(self, request) -> None: + def f(): + values = list(range(10)) + assert len(values) == 11 + + msg = getmsg(f) + if request.config.getoption("verbose") > 0: + assert msg == "assert 10 == 11\n +10\n -11" + else: + assert msg == "assert 10 == 11\n + where 10 = len([0, 1, 2, 3, 4, 5, ...])" + + def test_custom_reprcompare(self, monkeypatch) -> None: + def my_reprcompare1(op, left, right) -> str: + return "42" + + monkeypatch.setattr(util, "_reprcompare", my_reprcompare1) + + def f1() -> None: + assert 42 < 3 + + assert getmsg(f1) == "assert 42" + + def my_reprcompare2(op, left, right) -> str: + return f"{left} {op} {right}" + + monkeypatch.setattr(util, "_reprcompare", my_reprcompare2) + + def f2() -> None: + assert 1 < 3 < 5 <= 4 < 7 + + assert getmsg(f2) == "assert 5 <= 4" + + def test_assert_raising__bool__in_comparison(self) -> None: + def f() -> None: + class A: + def __bool__(self): + raise ValueError(42) + + def __lt__(self, other): + return A() + + def __repr__(self): + return "" + + def myany(x) -> bool: + return False + + assert myany(A() < 0) + + msg = getmsg(f) + assert msg is not None + assert " < 0" in msg + + def test_formatchar(self) -> None: + def f() -> None: + assert "%test" == "test" # type: ignore[comparison-overlap] + + msg = getmsg(f) + assert msg is not None + assert msg.startswith("assert '%test' == 'test'") + + def test_custom_repr(self, request) -> None: + def f() -> None: + class Foo: + a = 1 + + def __repr__(self): + return "\n{ \n~ \n}" + + f = Foo() + assert 0 == f.a + + msg = getmsg(f) + assert msg is not None + lines = util._format_lines([msg]) + if request.config.getoption("verbose") > 0: + assert lines == ["assert 0 == 1\n +0\n -1"] + else: + assert lines == ["assert 0 == 1\n + where 1 = \\n{ \\n~ \\n}.a"] + + def test_custom_repr_non_ascii(self) -> None: + def f() -> None: + class A: + name = "ä" + + def __repr__(self): + return self.name.encode("UTF-8") # only legal in python2 + + a = A() + assert not a.name + + msg = getmsg(f) + assert msg is not None + assert "UnicodeDecodeError" not in msg + assert "UnicodeEncodeError" not in msg + + +class TestRewriteOnImport: + def test_pycache_is_a_file(self, pytester: Pytester) -> None: + pytester.path.joinpath("__pycache__").write_text("Hello") + pytester.makepyfile( + """ + def test_rewritten(): + assert "@py_builtins" in globals()""" + ) + assert pytester.runpytest().ret == 0 + + def test_pycache_is_readonly(self, pytester: Pytester) -> None: + cache = pytester.mkdir("__pycache__") + old_mode = cache.stat().st_mode + cache.chmod(old_mode ^ stat.S_IWRITE) + pytester.makepyfile( + """ + def test_rewritten(): + assert "@py_builtins" in globals()""" + ) + try: + assert pytester.runpytest().ret == 0 + finally: + cache.chmod(old_mode) + + def test_zipfile(self, pytester: Pytester) -> None: + z = pytester.path.joinpath("myzip.zip") + z_fn = str(z) + f = zipfile.ZipFile(z_fn, "w") + try: + f.writestr("test_gum/__init__.py", "") + f.writestr("test_gum/test_lizard.py", "") + finally: + f.close() + z.chmod(256) + pytester.makepyfile( + """ + import sys + sys.path.append(%r) + import test_gum.test_lizard""" + % (z_fn,) + ) + assert pytester.runpytest().ret == ExitCode.NO_TESTS_COLLECTED + + @pytest.mark.skipif( + sys.version_info < (3, 9), + reason="importlib.resources.files was introduced in 3.9", + ) + def test_load_resource_via_files_with_rewrite(self, pytester: Pytester) -> None: + example = pytester.path.joinpath("demo") / "example" + init = pytester.path.joinpath("demo") / "__init__.py" + pytester.makepyfile( + **{ + "demo/__init__.py": """ + from importlib.resources import files + + def load(): + return files(__name__) + """, + "test_load": f""" + pytest_plugins = ["demo"] + + def test_load(): + from demo import load + found = {{str(i) for i in load().iterdir() if i.name != "__pycache__"}} + assert found == {{{str(example)!r}, {str(init)!r}}} + """, + } + ) + example.mkdir() + + assert pytester.runpytest("-vv").ret == ExitCode.OK + + def test_readonly(self, pytester: Pytester) -> None: + sub = pytester.mkdir("testing") + sub.joinpath("test_readonly.py").write_bytes( + b""" +def test_rewritten(): + assert "@py_builtins" in globals() + """, + ) + old_mode = sub.stat().st_mode + sub.chmod(320) + try: + assert pytester.runpytest().ret == 0 + finally: + sub.chmod(old_mode) + + def test_dont_write_bytecode(self, pytester: Pytester, monkeypatch) -> None: + monkeypatch.delenv("PYTHONPYCACHEPREFIX", raising=False) + + pytester.makepyfile( + """ + import os + def test_no_bytecode(): + assert "__pycache__" in __cached__ + assert not os.path.exists(__cached__) + assert not os.path.exists(os.path.dirname(__cached__))""" + ) + monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", "1") + assert pytester.runpytest_subprocess().ret == 0 + + def test_orphaned_pyc_file(self, pytester: Pytester, monkeypatch) -> None: + monkeypatch.delenv("PYTHONPYCACHEPREFIX", raising=False) + monkeypatch.setattr(sys, "pycache_prefix", None, raising=False) + + pytester.makepyfile( + """ + import orphan + def test_it(): + assert orphan.value == 17 + """ + ) + pytester.makepyfile( + orphan=""" + value = 17 + """ + ) + py_compile.compile("orphan.py") + os.remove("orphan.py") + + # Python 3 puts the .pyc files in a __pycache__ directory, and will + # not import from there without source. It will import a .pyc from + # the source location though. + if not os.path.exists("orphan.pyc"): + pycs = glob.glob("__pycache__/orphan.*.pyc") + assert len(pycs) == 1 + os.rename(pycs[0], "orphan.pyc") + + assert pytester.runpytest().ret == 0 + + def test_cached_pyc_includes_pytest_version( + self, pytester: Pytester, monkeypatch + ) -> None: + """Avoid stale caches (#1671)""" + monkeypatch.delenv("PYTHONDONTWRITEBYTECODE", raising=False) + monkeypatch.delenv("PYTHONPYCACHEPREFIX", raising=False) + pytester.makepyfile( + test_foo=""" + def test_foo(): + assert True + """ + ) + result = pytester.runpytest_subprocess() + assert result.ret == 0 + found_names = glob.glob(f"__pycache__/*-pytest-{pytest.__version__}.pyc") + assert found_names, "pyc with expected tag not found in names: {}".format( + glob.glob("__pycache__/*.pyc") + ) + + @pytest.mark.skipif('"__pypy__" in sys.modules') + def test_pyc_vs_pyo(self, pytester: Pytester, monkeypatch) -> None: + pytester.makepyfile( + """ + import pytest + def test_optimized(): + "hello" + assert test_optimized.__doc__ is None""" + ) + p = make_numbered_dir(root=Path(pytester.path), prefix="runpytest-") + tmp = "--basetemp=%s" % p + monkeypatch.setenv("PYTHONOPTIMIZE", "2") + monkeypatch.delenv("PYTHONDONTWRITEBYTECODE", raising=False) + monkeypatch.delenv("PYTHONPYCACHEPREFIX", raising=False) + assert pytester.runpytest_subprocess(tmp).ret == 0 + tagged = "test_pyc_vs_pyo." + PYTEST_TAG + assert tagged + ".pyo" in os.listdir("__pycache__") + monkeypatch.undo() + monkeypatch.delenv("PYTHONDONTWRITEBYTECODE", raising=False) + monkeypatch.delenv("PYTHONPYCACHEPREFIX", raising=False) + assert pytester.runpytest_subprocess(tmp).ret == 1 + assert tagged + ".pyc" in os.listdir("__pycache__") + + def test_package(self, pytester: Pytester) -> None: + pkg = pytester.path.joinpath("pkg") + pkg.mkdir() + pkg.joinpath("__init__.py") + pkg.joinpath("test_blah.py").write_text( + """ +def test_rewritten(): + assert "@py_builtins" in globals()""" + ) + assert pytester.runpytest().ret == 0 + + def test_translate_newlines(self, pytester: Pytester) -> None: + content = "def test_rewritten():\r\n assert '@py_builtins' in globals()" + b = content.encode("utf-8") + pytester.path.joinpath("test_newlines.py").write_bytes(b) + assert pytester.runpytest().ret == 0 + + def test_package_without__init__py(self, pytester: Pytester) -> None: + pkg = pytester.mkdir("a_package_without_init_py") + pkg.joinpath("module.py").touch() + pytester.makepyfile("import a_package_without_init_py.module") + assert pytester.runpytest().ret == ExitCode.NO_TESTS_COLLECTED + + def test_rewrite_warning(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + pytest.register_assert_rewrite("_pytest") + """ + ) + # needs to be a subprocess because pytester explicitly disables this warning + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines(["*Module already imported*: _pytest"]) + + def test_rewrite_module_imported_from_conftest(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import test_rewrite_module_imported + """ + ) + pytester.makepyfile( + test_rewrite_module_imported=""" + def test_rewritten(): + assert "@py_builtins" in globals() + """ + ) + assert pytester.runpytest_subprocess().ret == 0 + + def test_remember_rewritten_modules( + self, pytestconfig, pytester: Pytester, monkeypatch + ) -> None: + """`AssertionRewriteHook` should remember rewritten modules so it + doesn't give false positives (#2005).""" + monkeypatch.syspath_prepend(pytester.path) + pytester.makepyfile(test_remember_rewritten_modules="") + warnings = [] + hook = AssertionRewritingHook(pytestconfig) + monkeypatch.setattr( + hook, "_warn_already_imported", lambda code, msg: warnings.append(msg) + ) + spec = hook.find_spec("test_remember_rewritten_modules") + assert spec is not None + module = importlib.util.module_from_spec(spec) + hook.exec_module(module) + hook.mark_rewrite("test_remember_rewritten_modules") + hook.mark_rewrite("test_remember_rewritten_modules") + assert warnings == [] + + def test_rewrite_warning_using_pytest_plugins(self, pytester: Pytester) -> None: + pytester.makepyfile( + **{ + "conftest.py": "pytest_plugins = ['core', 'gui', 'sci']", + "core.py": "", + "gui.py": "pytest_plugins = ['core', 'sci']", + "sci.py": "pytest_plugins = ['core']", + "test_rewrite_warning_pytest_plugins.py": "def test(): pass", + } + ) + pytester.chdir() + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines(["*= 1 passed in *=*"]) + result.stdout.no_fnmatch_line("*pytest-warning summary*") + + def test_rewrite_warning_using_pytest_plugins_env_var( + self, pytester: Pytester, monkeypatch + ) -> None: + monkeypatch.setenv("PYTEST_PLUGINS", "plugin") + pytester.makepyfile( + **{ + "plugin.py": "", + "test_rewrite_warning_using_pytest_plugins_env_var.py": """ + import plugin + pytest_plugins = ['plugin'] + def test(): + pass + """, + } + ) + pytester.chdir() + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines(["*= 1 passed in *=*"]) + result.stdout.no_fnmatch_line("*pytest-warning summary*") + + +class TestAssertionRewriteHookDetails: + def test_sys_meta_path_munged(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_meta_path(): + import sys; sys.meta_path = []""" + ) + assert pytester.runpytest().ret == 0 + + def test_write_pyc(self, pytester: Pytester, tmp_path, monkeypatch) -> None: + from _pytest.assertion.rewrite import _write_pyc + from _pytest.assertion import AssertionState + + config = pytester.parseconfig() + state = AssertionState(config, "rewrite") + tmp_path.joinpath("source.py").touch() + source_path = str(tmp_path) + pycpath = tmp_path.joinpath("pyc") + co = compile("1", "f.py", "single") + assert _write_pyc(state, co, os.stat(source_path), pycpath) + + if sys.platform == "win32": + from contextlib import contextmanager + + @contextmanager + def atomic_write_failed(fn, mode="r", overwrite=False): + e = OSError() + e.errno = 10 + raise e + yield # type:ignore[unreachable] + + monkeypatch.setattr( + _pytest.assertion.rewrite, "atomic_write", atomic_write_failed + ) + else: + + def raise_oserror(*args): + raise OSError() + + monkeypatch.setattr("os.rename", raise_oserror) + + assert not _write_pyc(state, co, os.stat(source_path), pycpath) + + def test_resources_provider_for_loader(self, pytester: Pytester) -> None: + """ + Attempts to load resources from a package should succeed normally, + even when the AssertionRewriteHook is used to load the modules. + + See #366 for details. + """ + pytest.importorskip("pkg_resources") + + pytester.mkpydir("testpkg") + contents = { + "testpkg/test_pkg": """ + import pkg_resources + + import pytest + from _pytest.assertion.rewrite import AssertionRewritingHook + + def test_load_resource(): + assert isinstance(__loader__, AssertionRewritingHook) + res = pkg_resources.resource_string(__name__, 'resource.txt') + res = res.decode('ascii') + assert res == 'Load me please.' + """ + } + pytester.makepyfile(**contents) + pytester.maketxtfile(**{"testpkg/resource": "Load me please."}) + + result = pytester.runpytest_subprocess() + result.assert_outcomes(passed=1) + + def test_read_pyc(self, tmp_path: Path) -> None: + """ + Ensure that the `_read_pyc` can properly deal with corrupted pyc files. + In those circumstances it should just give up instead of generating + an exception that is propagated to the caller. + """ + import py_compile + from _pytest.assertion.rewrite import _read_pyc + + source = tmp_path / "source.py" + pyc = Path(str(source) + "c") + + source.write_text("def test(): pass") + py_compile.compile(str(source), str(pyc)) + + contents = pyc.read_bytes() + strip_bytes = 20 # header is around 16 bytes, strip a little more + assert len(contents) > strip_bytes + pyc.write_bytes(contents[:strip_bytes]) + + assert _read_pyc(source, pyc) is None # no error + + def test_read_pyc_success(self, tmp_path: Path, pytester: Pytester) -> None: + """ + Ensure that the _rewrite_test() -> _write_pyc() produces a pyc file + that can be properly read with _read_pyc() + """ + from _pytest.assertion import AssertionState + from _pytest.assertion.rewrite import _read_pyc + from _pytest.assertion.rewrite import _rewrite_test + from _pytest.assertion.rewrite import _write_pyc + + config = pytester.parseconfig() + state = AssertionState(config, "rewrite") + + fn = tmp_path / "source.py" + pyc = Path(str(fn) + "c") + + fn.write_text("def test(): assert True") + + source_stat, co = _rewrite_test(fn, config) + _write_pyc(state, co, source_stat, pyc) + assert _read_pyc(fn, pyc, state.trace) is not None + + @pytest.mark.skipif( + sys.version_info < (3, 7), reason="Only the Python 3.7 format for simplicity" + ) + def test_read_pyc_more_invalid(self, tmp_path: Path) -> None: + from _pytest.assertion.rewrite import _read_pyc + + source = tmp_path / "source.py" + pyc = tmp_path / "source.pyc" + + source_bytes = b"def test(): pass\n" + source.write_bytes(source_bytes) + + magic = importlib.util.MAGIC_NUMBER + + flags = b"\x00\x00\x00\x00" + + mtime = b"\x58\x3c\xb0\x5f" + mtime_int = int.from_bytes(mtime, "little") + os.utime(source, (mtime_int, mtime_int)) + + size = len(source_bytes).to_bytes(4, "little") + + code = marshal.dumps(compile(source_bytes, str(source), "exec")) + + # Good header. + pyc.write_bytes(magic + flags + mtime + size + code) + assert _read_pyc(source, pyc, print) is not None + + # Too short. + pyc.write_bytes(magic + flags + mtime) + assert _read_pyc(source, pyc, print) is None + + # Bad magic. + pyc.write_bytes(b"\x12\x34\x56\x78" + flags + mtime + size + code) + assert _read_pyc(source, pyc, print) is None + + # Unsupported flags. + pyc.write_bytes(magic + b"\x00\xff\x00\x00" + mtime + size + code) + assert _read_pyc(source, pyc, print) is None + + # Bad mtime. + pyc.write_bytes(magic + flags + b"\x58\x3d\xb0\x5f" + size + code) + assert _read_pyc(source, pyc, print) is None + + # Bad size. + pyc.write_bytes(magic + flags + mtime + b"\x99\x00\x00\x00" + code) + assert _read_pyc(source, pyc, print) is None + + def test_reload_is_same_and_reloads(self, pytester: Pytester) -> None: + """Reloading a (collected) module after change picks up the change.""" + pytester.makeini( + """ + [pytest] + python_files = *.py + """ + ) + pytester.makepyfile( + file=""" + def reloaded(): + return False + + def rewrite_self(): + with open(__file__, 'w') as self: + self.write('def reloaded(): return True') + """, + test_fun=""" + import sys + from importlib import reload + + def test_loader(): + import file + assert not file.reloaded() + file.rewrite_self() + assert sys.modules["file"] is reload(file) + assert file.reloaded() + """, + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 1 passed*"]) + + def test_get_data_support(self, pytester: Pytester) -> None: + """Implement optional PEP302 api (#808).""" + path = pytester.mkpydir("foo") + path.joinpath("test_foo.py").write_text( + textwrap.dedent( + """\ + class Test(object): + def test_foo(self): + import pkgutil + data = pkgutil.get_data('foo.test_foo', 'data.txt') + assert data == b'Hey' + """ + ) + ) + path.joinpath("data.txt").write_text("Hey") + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_issue731(pytester: Pytester) -> None: + pytester.makepyfile( + """ + class LongReprWithBraces(object): + def __repr__(self): + return 'LongReprWithBraces({' + ('a' * 80) + '}' + ('a' * 120) + ')' + + def some_method(self): + return False + + def test_long_repr(): + obj = LongReprWithBraces() + assert obj.some_method() + """ + ) + result = pytester.runpytest() + result.stdout.no_fnmatch_line("*unbalanced braces*") + + +class TestIssue925: + def test_simple_case(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_ternary_display(): + assert (False == False) == False + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*E*assert (False == False) == False"]) + + def test_long_case(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_ternary_display(): + assert False == (False == True) == True + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*E*assert (False == True) == True"]) + + def test_many_brackets(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_ternary_display(): + assert True == ((False == True) == True) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*E*assert True == ((False == True) == True)"]) + + +class TestIssue2121: + def test_rewrite_python_files_contain_subdirs(self, pytester: Pytester) -> None: + pytester.makepyfile( + **{ + "tests/file.py": """ + def test_simple_failure(): + assert 1 + 1 == 3 + """ + } + ) + pytester.makeini( + """ + [pytest] + python_files = tests/**.py + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*E*assert (1 + 1) == 3"]) + + +@pytest.mark.skipif( + sys.maxsize <= (2 ** 31 - 1), reason="Causes OverflowError on 32bit systems" +) +@pytest.mark.parametrize("offset", [-1, +1]) +def test_source_mtime_long_long(pytester: Pytester, offset) -> None: + """Support modification dates after 2038 in rewritten files (#4903). + + pytest would crash with: + + fp.write(struct.pack(" None: + """Fix infinite recursion when writing pyc files: if an import happens to be triggered when writing the pyc + file, this would cause another call to the hook, which would trigger another pyc writing, which could + trigger another import, and so on. (#3506)""" + from _pytest.assertion import rewrite as rewritemod + + pytester.syspathinsert() + pytester.makepyfile(test_foo="def test_foo(): pass") + pytester.makepyfile(test_bar="def test_bar(): pass") + + original_write_pyc = rewritemod._write_pyc + + write_pyc_called = [] + + def spy_write_pyc(*args, **kwargs): + # make a note that we have called _write_pyc + write_pyc_called.append(True) + # try to import a module at this point: we should not try to rewrite this module + assert hook.find_spec("test_bar") is None + return original_write_pyc(*args, **kwargs) + + monkeypatch.setattr(rewritemod, "_write_pyc", spy_write_pyc) + monkeypatch.setattr(sys, "dont_write_bytecode", False) + + hook = AssertionRewritingHook(pytestconfig) + spec = hook.find_spec("test_foo") + assert spec is not None + module = importlib.util.module_from_spec(spec) + hook.exec_module(module) + assert len(write_pyc_called) == 1 + + +class TestEarlyRewriteBailout: + @pytest.fixture + def hook( + self, pytestconfig, monkeypatch, pytester: Pytester + ) -> AssertionRewritingHook: + """Returns a patched AssertionRewritingHook instance so we can configure its initial paths and track + if PathFinder.find_spec has been called. + """ + import importlib.machinery + + self.find_spec_calls: List[str] = [] + self.initial_paths: Set[Path] = set() + + class StubSession: + _initialpaths = self.initial_paths + + def isinitpath(self, p): + return p in self._initialpaths + + def spy_find_spec(name, path): + self.find_spec_calls.append(name) + return importlib.machinery.PathFinder.find_spec(name, path) + + hook = AssertionRewritingHook(pytestconfig) + # use default patterns, otherwise we inherit pytest's testing config + hook.fnpats[:] = ["test_*.py", "*_test.py"] + monkeypatch.setattr(hook, "_find_spec", spy_find_spec) + hook.set_session(StubSession()) # type: ignore[arg-type] + pytester.syspathinsert() + return hook + + def test_basic(self, pytester: Pytester, hook: AssertionRewritingHook) -> None: + """ + Ensure we avoid calling PathFinder.find_spec when we know for sure a certain + module will not be rewritten to optimize assertion rewriting (#3918). + """ + pytester.makeconftest( + """ + import pytest + @pytest.fixture + def fix(): return 1 + """ + ) + pytester.makepyfile(test_foo="def test_foo(): pass") + pytester.makepyfile(bar="def bar(): pass") + foobar_path = pytester.makepyfile(foobar="def foobar(): pass") + self.initial_paths.add(foobar_path) + + # conftest files should always be rewritten + assert hook.find_spec("conftest") is not None + assert self.find_spec_calls == ["conftest"] + + # files matching "python_files" mask should always be rewritten + assert hook.find_spec("test_foo") is not None + assert self.find_spec_calls == ["conftest", "test_foo"] + + # file does not match "python_files": early bailout + assert hook.find_spec("bar") is None + assert self.find_spec_calls == ["conftest", "test_foo"] + + # file is an initial path (passed on the command-line): should be rewritten + assert hook.find_spec("foobar") is not None + assert self.find_spec_calls == ["conftest", "test_foo", "foobar"] + + def test_pattern_contains_subdirectories( + self, pytester: Pytester, hook: AssertionRewritingHook + ) -> None: + """If one of the python_files patterns contain subdirectories ("tests/**.py") we can't bailout early + because we need to match with the full path, which can only be found by calling PathFinder.find_spec + """ + pytester.makepyfile( + **{ + "tests/file.py": """\ + def test_simple_failure(): + assert 1 + 1 == 3 + """ + } + ) + pytester.syspathinsert("tests") + hook.fnpats[:] = ["tests/**.py"] + assert hook.find_spec("file") is not None + assert self.find_spec_calls == ["file"] + + @pytest.mark.skipif( + sys.platform.startswith("win32"), reason="cannot remove cwd on Windows" + ) + @pytest.mark.skipif( + sys.platform.startswith("sunos5"), reason="cannot remove cwd on Solaris" + ) + def test_cwd_changed(self, pytester: Pytester, monkeypatch) -> None: + # Setup conditions for py's fspath trying to import pathlib on py34 + # always (previously triggered via xdist only). + # Ref: https://github.com/pytest-dev/py/pull/207 + monkeypatch.syspath_prepend("") + monkeypatch.delitem(sys.modules, "pathlib", raising=False) + + pytester.makepyfile( + **{ + "test_setup_nonexisting_cwd.py": """\ + import os + import tempfile + + with tempfile.TemporaryDirectory() as d: + os.chdir(d) + """, + "test_test.py": """\ + def test(): + pass + """, + } + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 1 passed in *"]) + + +class TestAssertionPass: + def test_option_default(self, pytester: Pytester) -> None: + config = pytester.parseconfig() + assert config.getini("enable_assertion_pass_hook") is False + + @pytest.fixture + def flag_on(self, pytester: Pytester): + pytester.makeini("[pytest]\nenable_assertion_pass_hook = True\n") + + @pytest.fixture + def hook_on(self, pytester: Pytester): + pytester.makeconftest( + """\ + def pytest_assertion_pass(item, lineno, orig, expl): + raise Exception("Assertion Passed: {} {} at line {}".format(orig, expl, lineno)) + """ + ) + + def test_hook_call(self, pytester: Pytester, flag_on, hook_on) -> None: + pytester.makepyfile( + """\ + def test_simple(): + a=1 + b=2 + c=3 + d=0 + + assert a+b == c+d + + # cover failing assertions with a message + def test_fails(): + assert False, "assert with message" + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + "*Assertion Passed: a+b == c+d (1 + 2) == (3 + 0) at line 7*" + ) + + def test_hook_call_with_parens(self, pytester: Pytester, flag_on, hook_on) -> None: + pytester.makepyfile( + """\ + def f(): return 1 + def test(): + assert f() + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines("*Assertion Passed: f() 1") + + def test_hook_not_called_without_hookimpl( + self, pytester: Pytester, monkeypatch, flag_on + ) -> None: + """Assertion pass should not be called (and hence formatting should + not occur) if there is no hook declared for pytest_assertion_pass""" + + def raise_on_assertionpass(*_, **__): + raise Exception("Assertion passed called when it shouldn't!") + + monkeypatch.setattr( + _pytest.assertion.rewrite, "_call_assertion_pass", raise_on_assertionpass + ) + + pytester.makepyfile( + """\ + def test_simple(): + a=1 + b=2 + c=3 + d=0 + + assert a+b == c+d + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=1) + + def test_hook_not_called_without_cmd_option( + self, pytester: Pytester, monkeypatch + ) -> None: + """Assertion pass should not be called (and hence formatting should + not occur) if there is no hook declared for pytest_assertion_pass""" + + def raise_on_assertionpass(*_, **__): + raise Exception("Assertion passed called when it shouldn't!") + + monkeypatch.setattr( + _pytest.assertion.rewrite, "_call_assertion_pass", raise_on_assertionpass + ) + + pytester.makeconftest( + """\ + def pytest_assertion_pass(item, lineno, orig, expl): + raise Exception("Assertion Passed: {} {} at line {}".format(orig, expl, lineno)) + """ + ) + + pytester.makepyfile( + """\ + def test_simple(): + a=1 + b=2 + c=3 + d=0 + + assert a+b == c+d + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=1) + + +@pytest.mark.parametrize( + ("src", "expected"), + ( + # fmt: off + pytest.param(b"", {}, id="trivial"), + pytest.param( + b"def x(): assert 1\n", + {1: "1"}, + id="assert statement not on own line", + ), + pytest.param( + b"def x():\n" + b" assert 1\n" + b" assert 1+2\n", + {2: "1", 3: "1+2"}, + id="multiple assertions", + ), + pytest.param( + # changes in encoding cause the byte offsets to be different + "# -*- coding: latin1\n" + "def ÀÀÀÀÀ(): assert 1\n".encode("latin1"), + {2: "1"}, + id="latin1 encoded on first line\n", + ), + pytest.param( + # using the default utf-8 encoding + "def ÀÀÀÀÀ(): assert 1\n".encode(), + {1: "1"}, + id="utf-8 encoded on first line", + ), + pytest.param( + b"def x():\n" + b" assert (\n" + b" 1 + 2 # comment\n" + b" )\n", + {2: "(\n 1 + 2 # comment\n )"}, + id="multi-line assertion", + ), + pytest.param( + b"def x():\n" + b" assert y == [\n" + b" 1, 2, 3\n" + b" ]\n", + {2: "y == [\n 1, 2, 3\n ]"}, + id="multi line assert with list continuation", + ), + pytest.param( + b"def x():\n" + b" assert 1 + \\\n" + b" 2\n", + {2: "1 + \\\n 2"}, + id="backslash continuation", + ), + pytest.param( + b"def x():\n" + b" assert x, y\n", + {2: "x"}, + id="assertion with message", + ), + pytest.param( + b"def x():\n" + b" assert (\n" + b" f(1, 2, 3)\n" + b" ), 'f did not work!'\n", + {2: "(\n f(1, 2, 3)\n )"}, + id="assertion with message, test spanning multiple lines", + ), + pytest.param( + b"def x():\n" + b" assert \\\n" + b" x\\\n" + b" , 'failure message'\n", + {2: "x"}, + id="escaped newlines plus message", + ), + pytest.param( + b"def x(): assert 5", + {1: "5"}, + id="no newline at end of file", + ), + # fmt: on + ), +) +def test_get_assertion_exprs(src, expected) -> None: + assert _get_assertion_exprs(src) == expected + + +def test_try_makedirs(monkeypatch, tmp_path: Path) -> None: + from _pytest.assertion.rewrite import try_makedirs + + p = tmp_path / "foo" + + # create + assert try_makedirs(p) + assert p.is_dir() + + # already exist + assert try_makedirs(p) + + # monkeypatch to simulate all error situations + def fake_mkdir(p, exist_ok=False, *, exc): + assert isinstance(p, Path) + raise exc + + monkeypatch.setattr(os, "makedirs", partial(fake_mkdir, exc=FileNotFoundError())) + assert not try_makedirs(p) + + monkeypatch.setattr(os, "makedirs", partial(fake_mkdir, exc=NotADirectoryError())) + assert not try_makedirs(p) + + monkeypatch.setattr(os, "makedirs", partial(fake_mkdir, exc=PermissionError())) + assert not try_makedirs(p) + + err = OSError() + err.errno = errno.EROFS + monkeypatch.setattr(os, "makedirs", partial(fake_mkdir, exc=err)) + assert not try_makedirs(p) + + # unhandled OSError should raise + err = OSError() + err.errno = errno.ECHILD + monkeypatch.setattr(os, "makedirs", partial(fake_mkdir, exc=err)) + with pytest.raises(OSError) as exc_info: + try_makedirs(p) + assert exc_info.value.errno == errno.ECHILD + + +class TestPyCacheDir: + @pytest.mark.parametrize( + "prefix, source, expected", + [ + ("c:/tmp/pycs", "d:/projects/src/foo.py", "c:/tmp/pycs/projects/src"), + (None, "d:/projects/src/foo.py", "d:/projects/src/__pycache__"), + ("/tmp/pycs", "/home/projects/src/foo.py", "/tmp/pycs/home/projects/src"), + (None, "/home/projects/src/foo.py", "/home/projects/src/__pycache__"), + ], + ) + def test_get_cache_dir(self, monkeypatch, prefix, source, expected) -> None: + monkeypatch.delenv("PYTHONPYCACHEPREFIX", raising=False) + + if prefix is not None and sys.version_info < (3, 8): + pytest.skip("pycache_prefix not available in py<38") + monkeypatch.setattr(sys, "pycache_prefix", prefix, raising=False) + + assert get_cache_dir(Path(source)) == Path(expected) + + @pytest.mark.skipif( + sys.version_info < (3, 8), reason="pycache_prefix not available in py<38" + ) + @pytest.mark.skipif( + sys.version_info[:2] == (3, 9) and sys.platform.startswith("win"), + reason="#9298", + ) + def test_sys_pycache_prefix_integration( + self, tmp_path, monkeypatch, pytester: Pytester + ) -> None: + """Integration test for sys.pycache_prefix (#4730).""" + pycache_prefix = tmp_path / "my/pycs" + monkeypatch.setattr(sys, "pycache_prefix", str(pycache_prefix)) + monkeypatch.setattr(sys, "dont_write_bytecode", False) + + pytester.makepyfile( + **{ + "src/test_foo.py": """ + import bar + def test_foo(): + pass + """, + "src/bar/__init__.py": "", + } + ) + result = pytester.runpytest() + assert result.ret == 0 + + test_foo = pytester.path.joinpath("src/test_foo.py") + bar_init = pytester.path.joinpath("src/bar/__init__.py") + assert test_foo.is_file() + assert bar_init.is_file() + + # test file: rewritten, custom pytest cache tag + test_foo_pyc = get_cache_dir(test_foo) / ("test_foo" + PYC_TAIL) + assert test_foo_pyc.is_file() + + # normal file: not touched by pytest, normal cache tag + bar_init_pyc = get_cache_dir(bar_init) / "__init__.{cache_tag}.pyc".format( + cache_tag=sys.implementation.cache_tag + ) + assert bar_init_pyc.is_file() + + +class TestReprSizeVerbosity: + """ + Check that verbosity also controls the string length threshold to shorten it using + ellipsis. + """ + + @pytest.mark.parametrize( + "verbose, expected_size", + [ + (0, DEFAULT_REPR_MAX_SIZE), + (1, DEFAULT_REPR_MAX_SIZE * 10), + (2, None), + (3, None), + ], + ) + def test_get_maxsize_for_saferepr(self, verbose: int, expected_size) -> None: + class FakeConfig: + def getoption(self, name: str) -> int: + assert name == "verbose" + return verbose + + config = FakeConfig() + assert _get_maxsize_for_saferepr(cast(Config, config)) == expected_size + + def create_test_file(self, pytester: Pytester, size: int) -> None: + pytester.makepyfile( + f""" + def test_very_long_string(): + text = "x" * {size} + assert "hello world" in text + """ + ) + + def test_default_verbosity(self, pytester: Pytester) -> None: + self.create_test_file(pytester, DEFAULT_REPR_MAX_SIZE) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*xxx...xxx*"]) + + def test_increased_verbosity(self, pytester: Pytester) -> None: + self.create_test_file(pytester, DEFAULT_REPR_MAX_SIZE) + result = pytester.runpytest("-v") + result.stdout.no_fnmatch_line("*xxx...xxx*") + + def test_max_increased_verbosity(self, pytester: Pytester) -> None: + self.create_test_file(pytester, DEFAULT_REPR_MAX_SIZE * 10) + result = pytester.runpytest("-vv") + result.stdout.no_fnmatch_line("*xxx...xxx*") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_cacheprovider.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_cacheprovider.py new file mode 100644 index 0000000000..cc6d547dfb --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_cacheprovider.py @@ -0,0 +1,1251 @@ +import os +import shutil +from pathlib import Path +from typing import Generator +from typing import List + +import pytest +from _pytest.config import ExitCode +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester +from _pytest.tmpdir import TempPathFactory + +pytest_plugins = ("pytester",) + + +class TestNewAPI: + def test_config_cache_mkdir(self, pytester: Pytester) -> None: + pytester.makeini("[pytest]") + config = pytester.parseconfigure() + assert config.cache is not None + with pytest.raises(ValueError): + config.cache.mkdir("key/name") + + p = config.cache.mkdir("name") + assert p.is_dir() + + def test_config_cache_dataerror(self, pytester: Pytester) -> None: + pytester.makeini("[pytest]") + config = pytester.parseconfigure() + assert config.cache is not None + cache = config.cache + pytest.raises(TypeError, lambda: cache.set("key/name", cache)) + config.cache.set("key/name", 0) + config.cache._getvaluepath("key/name").write_bytes(b"123invalid") + val = config.cache.get("key/name", -2) + assert val == -2 + + @pytest.mark.filterwarnings("ignore:could not create cache path") + def test_cache_writefail_cachfile_silent(self, pytester: Pytester) -> None: + pytester.makeini("[pytest]") + pytester.path.joinpath(".pytest_cache").write_text("gone wrong") + config = pytester.parseconfigure() + cache = config.cache + assert cache is not None + cache.set("test/broken", []) + + @pytest.fixture + def unwritable_cache_dir(self, pytester: Pytester) -> Generator[Path, None, None]: + cache_dir = pytester.path.joinpath(".pytest_cache") + cache_dir.mkdir() + mode = cache_dir.stat().st_mode + cache_dir.chmod(0) + if os.access(cache_dir, os.W_OK): + pytest.skip("Failed to make cache dir unwritable") + + yield cache_dir + cache_dir.chmod(mode) + + @pytest.mark.filterwarnings( + "ignore:could not create cache path:pytest.PytestWarning" + ) + def test_cache_writefail_permissions( + self, unwritable_cache_dir: Path, pytester: Pytester + ) -> None: + pytester.makeini("[pytest]") + config = pytester.parseconfigure() + cache = config.cache + assert cache is not None + cache.set("test/broken", []) + + @pytest.mark.filterwarnings("default") + def test_cache_failure_warns( + self, + pytester: Pytester, + monkeypatch: MonkeyPatch, + unwritable_cache_dir: Path, + ) -> None: + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + + pytester.makepyfile("def test_error(): raise Exception") + result = pytester.runpytest() + assert result.ret == 1 + # warnings from nodeids, lastfailed, and stepwise + result.stdout.fnmatch_lines( + [ + # Validate location/stacklevel of warning from cacheprovider. + "*= warnings summary =*", + "*/cacheprovider.py:*", + " */cacheprovider.py:*: PytestCacheWarning: could not create cache path " + f"{unwritable_cache_dir}/v/cache/nodeids", + ' config.cache.set("cache/nodeids", sorted(self.cached_nodeids))', + "*1 failed, 3 warnings in*", + ] + ) + + def test_config_cache(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_configure(config): + # see that we get cache information early on + assert hasattr(config, "cache") + """ + ) + pytester.makepyfile( + """ + def test_session(pytestconfig): + assert hasattr(pytestconfig, "cache") + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_cachefuncarg(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + def test_cachefuncarg(cache): + val = cache.get("some/thing", None) + assert val is None + cache.set("some/thing", [1]) + pytest.raises(TypeError, lambda: cache.get("some/thing")) + val = cache.get("some/thing", []) + assert val == [1] + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_custom_rel_cache_dir(self, pytester: Pytester) -> None: + rel_cache_dir = os.path.join("custom_cache_dir", "subdir") + pytester.makeini( + """ + [pytest] + cache_dir = {cache_dir} + """.format( + cache_dir=rel_cache_dir + ) + ) + pytester.makepyfile(test_errored="def test_error():\n assert False") + pytester.runpytest() + assert pytester.path.joinpath(rel_cache_dir).is_dir() + + def test_custom_abs_cache_dir( + self, pytester: Pytester, tmp_path_factory: TempPathFactory + ) -> None: + tmp = tmp_path_factory.mktemp("tmp") + abs_cache_dir = tmp / "custom_cache_dir" + pytester.makeini( + """ + [pytest] + cache_dir = {cache_dir} + """.format( + cache_dir=abs_cache_dir + ) + ) + pytester.makepyfile(test_errored="def test_error():\n assert False") + pytester.runpytest() + assert abs_cache_dir.is_dir() + + def test_custom_cache_dir_with_env_var( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.setenv("env_var", "custom_cache_dir") + pytester.makeini( + """ + [pytest] + cache_dir = {cache_dir} + """.format( + cache_dir="$env_var" + ) + ) + pytester.makepyfile(test_errored="def test_error():\n assert False") + pytester.runpytest() + assert pytester.path.joinpath("custom_cache_dir").is_dir() + + +@pytest.mark.parametrize("env", ((), ("TOX_ENV_DIR", "/tox_env_dir"))) +def test_cache_reportheader(env, pytester: Pytester, monkeypatch: MonkeyPatch) -> None: + pytester.makepyfile("""def test_foo(): pass""") + if env: + monkeypatch.setenv(*env) + expected = os.path.join(env[1], ".pytest_cache") + else: + monkeypatch.delenv("TOX_ENV_DIR", raising=False) + expected = ".pytest_cache" + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines(["cachedir: %s" % expected]) + + +def test_cache_reportheader_external_abspath( + pytester: Pytester, tmp_path_factory: TempPathFactory +) -> None: + external_cache = tmp_path_factory.mktemp( + "test_cache_reportheader_external_abspath_abs" + ) + + pytester.makepyfile("def test_hello(): pass") + pytester.makeini( + """ + [pytest] + cache_dir = {abscache} + """.format( + abscache=external_cache + ) + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines([f"cachedir: {external_cache}"]) + + +def test_cache_show(pytester: Pytester) -> None: + result = pytester.runpytest("--cache-show") + assert result.ret == 0 + result.stdout.fnmatch_lines(["*cache is empty*"]) + pytester.makeconftest( + """ + def pytest_configure(config): + config.cache.set("my/name", [1,2,3]) + config.cache.set("my/hello", "world") + config.cache.set("other/some", {1:2}) + dp = config.cache.mkdir("mydb") + dp.joinpath("hello").touch() + dp.joinpath("world").touch() + """ + ) + result = pytester.runpytest() + assert result.ret == 5 # no tests executed + + result = pytester.runpytest("--cache-show") + result.stdout.fnmatch_lines( + [ + "*cachedir:*", + "*- cache values for '[*]' -*", + "cache/nodeids contains:", + "my/name contains:", + " [1, 2, 3]", + "other/some contains:", + " {*'1': 2}", + "*- cache directories for '[*]' -*", + "*mydb/hello*length 0*", + "*mydb/world*length 0*", + ] + ) + assert result.ret == 0 + + result = pytester.runpytest("--cache-show", "*/hello") + result.stdout.fnmatch_lines( + [ + "*cachedir:*", + "*- cache values for '[*]/hello' -*", + "my/hello contains:", + " *'world'", + "*- cache directories for '[*]/hello' -*", + "d/mydb/hello*length 0*", + ] + ) + stdout = result.stdout.str() + assert "other/some" not in stdout + assert "d/mydb/world" not in stdout + assert result.ret == 0 + + +class TestLastFailed: + def test_lastfailed_usecase( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.setattr("sys.dont_write_bytecode", True) + p = pytester.makepyfile( + """ + def test_1(): assert 0 + def test_2(): assert 0 + def test_3(): assert 1 + """ + ) + result = pytester.runpytest(str(p)) + result.stdout.fnmatch_lines(["*2 failed*"]) + p = pytester.makepyfile( + """ + def test_1(): assert 1 + def test_2(): assert 1 + def test_3(): assert 0 + """ + ) + result = pytester.runpytest(str(p), "--lf") + result.stdout.fnmatch_lines( + [ + "collected 3 items / 1 deselected / 2 selected", + "run-last-failure: rerun previous 2 failures", + "*= 2 passed, 1 deselected in *", + ] + ) + result = pytester.runpytest(str(p), "--lf") + result.stdout.fnmatch_lines( + [ + "collected 3 items", + "run-last-failure: no previously failed tests, not deselecting items.", + "*1 failed*2 passed*", + ] + ) + pytester.path.joinpath(".pytest_cache", ".git").mkdir(parents=True) + result = pytester.runpytest(str(p), "--lf", "--cache-clear") + result.stdout.fnmatch_lines(["*1 failed*2 passed*"]) + assert pytester.path.joinpath(".pytest_cache", "README.md").is_file() + assert pytester.path.joinpath(".pytest_cache", ".git").is_dir() + + # Run this again to make sure clear-cache is robust + if os.path.isdir(".pytest_cache"): + shutil.rmtree(".pytest_cache") + result = pytester.runpytest("--lf", "--cache-clear") + result.stdout.fnmatch_lines(["*1 failed*2 passed*"]) + + def test_failedfirst_order(self, pytester: Pytester) -> None: + pytester.makepyfile( + test_a="def test_always_passes(): pass", + test_b="def test_always_fails(): assert 0", + ) + result = pytester.runpytest() + # Test order will be collection order; alphabetical + result.stdout.fnmatch_lines(["test_a.py*", "test_b.py*"]) + result = pytester.runpytest("--ff") + # Test order will be failing tests first + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "run-last-failure: rerun previous 1 failure first", + "test_b.py*", + "test_a.py*", + ] + ) + + def test_lastfailed_failedfirst_order(self, pytester: Pytester) -> None: + pytester.makepyfile( + test_a="def test_always_passes(): assert 1", + test_b="def test_always_fails(): assert 0", + ) + result = pytester.runpytest() + # Test order will be collection order; alphabetical + result.stdout.fnmatch_lines(["test_a.py*", "test_b.py*"]) + result = pytester.runpytest("--lf", "--ff") + # Test order will be failing tests first + result.stdout.fnmatch_lines(["test_b.py*"]) + result.stdout.no_fnmatch_line("*test_a.py*") + + def test_lastfailed_difference_invocations( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.setattr("sys.dont_write_bytecode", True) + pytester.makepyfile( + test_a=""" + def test_a1(): assert 0 + def test_a2(): assert 1 + """, + test_b="def test_b1(): assert 0", + ) + p = pytester.path.joinpath("test_a.py") + p2 = pytester.path.joinpath("test_b.py") + + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*2 failed*"]) + result = pytester.runpytest("--lf", p2) + result.stdout.fnmatch_lines(["*1 failed*"]) + + pytester.makepyfile(test_b="def test_b1(): assert 1") + result = pytester.runpytest("--lf", p2) + result.stdout.fnmatch_lines(["*1 passed*"]) + result = pytester.runpytest("--lf", p) + result.stdout.fnmatch_lines( + [ + "collected 2 items / 1 deselected / 1 selected", + "run-last-failure: rerun previous 1 failure", + "*= 1 failed, 1 deselected in *", + ] + ) + + def test_lastfailed_usecase_splice( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.setattr("sys.dont_write_bytecode", True) + pytester.makepyfile( + "def test_1(): assert 0", test_something="def test_2(): assert 0" + ) + p2 = pytester.path.joinpath("test_something.py") + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*2 failed*"]) + result = pytester.runpytest("--lf", p2) + result.stdout.fnmatch_lines(["*1 failed*"]) + result = pytester.runpytest("--lf") + result.stdout.fnmatch_lines(["*2 failed*"]) + + def test_lastfailed_xpass(self, pytester: Pytester) -> None: + pytester.inline_runsource( + """ + import pytest + @pytest.mark.xfail + def test_hello(): + assert 1 + """ + ) + config = pytester.parseconfigure() + assert config.cache is not None + lastfailed = config.cache.get("cache/lastfailed", -1) + assert lastfailed == -1 + + def test_non_serializable_parametrize(self, pytester: Pytester) -> None: + """Test that failed parametrized tests with unmarshable parameters + don't break pytest-cache. + """ + pytester.makepyfile( + r""" + import pytest + + @pytest.mark.parametrize('val', [ + b'\xac\x10\x02G', + ]) + def test_fail(val): + assert False + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 failed in*"]) + + def test_terminal_report_lastfailed(self, pytester: Pytester) -> None: + test_a = pytester.makepyfile( + test_a=""" + def test_a1(): pass + def test_a2(): pass + """ + ) + test_b = pytester.makepyfile( + test_b=""" + def test_b1(): assert 0 + def test_b2(): assert 0 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["collected 4 items", "*2 failed, 2 passed in*"]) + + result = pytester.runpytest("--lf") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "run-last-failure: rerun previous 2 failures (skipped 1 file)", + "*2 failed in*", + ] + ) + + result = pytester.runpytest(test_a, "--lf") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "run-last-failure: 2 known failures not in selected tests", + "*2 passed in*", + ] + ) + + result = pytester.runpytest(test_b, "--lf") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "run-last-failure: rerun previous 2 failures", + "*2 failed in*", + ] + ) + + result = pytester.runpytest("test_b.py::test_b1", "--lf") + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "run-last-failure: rerun previous 1 failure", + "*1 failed in*", + ] + ) + + def test_terminal_report_failedfirst(self, pytester: Pytester) -> None: + pytester.makepyfile( + test_a=""" + def test_a1(): assert 0 + def test_a2(): pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["collected 2 items", "*1 failed, 1 passed in*"]) + + result = pytester.runpytest("--ff") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "run-last-failure: rerun previous 1 failure first", + "*1 failed, 1 passed in*", + ] + ) + + def test_lastfailed_collectfailure( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + + pytester.makepyfile( + test_maybe=""" + import os + env = os.environ + if '1' == env['FAILIMPORT']: + raise ImportError('fail') + def test_hello(): + assert '0' == env['FAILTEST'] + """ + ) + + def rlf(fail_import, fail_run): + monkeypatch.setenv("FAILIMPORT", str(fail_import)) + monkeypatch.setenv("FAILTEST", str(fail_run)) + + pytester.runpytest("-q") + config = pytester.parseconfigure() + assert config.cache is not None + lastfailed = config.cache.get("cache/lastfailed", -1) + return lastfailed + + lastfailed = rlf(fail_import=0, fail_run=0) + assert lastfailed == -1 + + lastfailed = rlf(fail_import=1, fail_run=0) + assert list(lastfailed) == ["test_maybe.py"] + + lastfailed = rlf(fail_import=0, fail_run=1) + assert list(lastfailed) == ["test_maybe.py::test_hello"] + + def test_lastfailed_failure_subset( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + pytester.makepyfile( + test_maybe=""" + import os + env = os.environ + if '1' == env['FAILIMPORT']: + raise ImportError('fail') + def test_hello(): + assert '0' == env['FAILTEST'] + """ + ) + + pytester.makepyfile( + test_maybe2=""" + import os + env = os.environ + if '1' == env['FAILIMPORT']: + raise ImportError('fail') + + def test_hello(): + assert '0' == env['FAILTEST'] + + def test_pass(): + pass + """ + ) + + def rlf(fail_import, fail_run, args=()): + monkeypatch.setenv("FAILIMPORT", str(fail_import)) + monkeypatch.setenv("FAILTEST", str(fail_run)) + + result = pytester.runpytest("-q", "--lf", *args) + config = pytester.parseconfigure() + assert config.cache is not None + lastfailed = config.cache.get("cache/lastfailed", -1) + return result, lastfailed + + result, lastfailed = rlf(fail_import=0, fail_run=0) + assert lastfailed == -1 + result.stdout.fnmatch_lines(["*3 passed*"]) + + result, lastfailed = rlf(fail_import=1, fail_run=0) + assert sorted(list(lastfailed)) == ["test_maybe.py", "test_maybe2.py"] + + result, lastfailed = rlf(fail_import=0, fail_run=0, args=("test_maybe2.py",)) + assert list(lastfailed) == ["test_maybe.py"] + + # edge case of test selection - even if we remember failures + # from other tests we still need to run all tests if no test + # matches the failures + result, lastfailed = rlf(fail_import=0, fail_run=0, args=("test_maybe2.py",)) + assert list(lastfailed) == ["test_maybe.py"] + result.stdout.fnmatch_lines(["*2 passed*"]) + + def test_lastfailed_creates_cache_when_needed(self, pytester: Pytester) -> None: + # Issue #1342 + pytester.makepyfile(test_empty="") + pytester.runpytest("-q", "--lf") + assert not os.path.exists(".pytest_cache/v/cache/lastfailed") + + pytester.makepyfile(test_successful="def test_success():\n assert True") + pytester.runpytest("-q", "--lf") + assert not os.path.exists(".pytest_cache/v/cache/lastfailed") + + pytester.makepyfile(test_errored="def test_error():\n assert False") + pytester.runpytest("-q", "--lf") + assert os.path.exists(".pytest_cache/v/cache/lastfailed") + + def test_xfail_not_considered_failure(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail + def test(): assert 0 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 xfailed*"]) + assert self.get_cached_last_failed(pytester) == [] + + def test_xfail_strict_considered_failure(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail(strict=True) + def test(): pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 failed*"]) + assert self.get_cached_last_failed(pytester) == [ + "test_xfail_strict_considered_failure.py::test" + ] + + @pytest.mark.parametrize("mark", ["mark.xfail", "mark.skip"]) + def test_failed_changed_to_xfail_or_skip( + self, pytester: Pytester, mark: str + ) -> None: + pytester.makepyfile( + """ + import pytest + def test(): assert 0 + """ + ) + result = pytester.runpytest() + assert self.get_cached_last_failed(pytester) == [ + "test_failed_changed_to_xfail_or_skip.py::test" + ] + assert result.ret == 1 + + pytester.makepyfile( + """ + import pytest + @pytest.{mark} + def test(): assert 0 + """.format( + mark=mark + ) + ) + result = pytester.runpytest() + assert result.ret == 0 + assert self.get_cached_last_failed(pytester) == [] + assert result.ret == 0 + + @pytest.mark.parametrize("quiet", [True, False]) + @pytest.mark.parametrize("opt", ["--ff", "--lf"]) + def test_lf_and_ff_prints_no_needless_message( + self, quiet: bool, opt: str, pytester: Pytester + ) -> None: + # Issue 3853 + pytester.makepyfile("def test(): assert 0") + args = [opt] + if quiet: + args.append("-q") + result = pytester.runpytest(*args) + result.stdout.no_fnmatch_line("*run all*") + + result = pytester.runpytest(*args) + if quiet: + result.stdout.no_fnmatch_line("*run all*") + else: + assert "rerun previous" in result.stdout.str() + + def get_cached_last_failed(self, pytester: Pytester) -> List[str]: + config = pytester.parseconfigure() + assert config.cache is not None + return sorted(config.cache.get("cache/lastfailed", {})) + + def test_cache_cumulative(self, pytester: Pytester) -> None: + """Test workflow where user fixes errors gradually file by file using --lf.""" + # 1. initial run + test_bar = pytester.makepyfile( + test_bar=""" + def test_bar_1(): pass + def test_bar_2(): assert 0 + """ + ) + test_foo = pytester.makepyfile( + test_foo=""" + def test_foo_3(): pass + def test_foo_4(): assert 0 + """ + ) + pytester.runpytest() + assert self.get_cached_last_failed(pytester) == [ + "test_bar.py::test_bar_2", + "test_foo.py::test_foo_4", + ] + + # 2. fix test_bar_2, run only test_bar.py + pytester.makepyfile( + test_bar=""" + def test_bar_1(): pass + def test_bar_2(): pass + """ + ) + result = pytester.runpytest(test_bar) + result.stdout.fnmatch_lines(["*2 passed*"]) + # ensure cache does not forget that test_foo_4 failed once before + assert self.get_cached_last_failed(pytester) == ["test_foo.py::test_foo_4"] + + result = pytester.runpytest("--last-failed") + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "run-last-failure: rerun previous 1 failure (skipped 1 file)", + "*= 1 failed in *", + ] + ) + assert self.get_cached_last_failed(pytester) == ["test_foo.py::test_foo_4"] + + # 3. fix test_foo_4, run only test_foo.py + test_foo = pytester.makepyfile( + test_foo=""" + def test_foo_3(): pass + def test_foo_4(): pass + """ + ) + result = pytester.runpytest(test_foo, "--last-failed") + result.stdout.fnmatch_lines( + [ + "collected 2 items / 1 deselected / 1 selected", + "run-last-failure: rerun previous 1 failure", + "*= 1 passed, 1 deselected in *", + ] + ) + assert self.get_cached_last_failed(pytester) == [] + + result = pytester.runpytest("--last-failed") + result.stdout.fnmatch_lines(["*4 passed*"]) + assert self.get_cached_last_failed(pytester) == [] + + def test_lastfailed_no_failures_behavior_all_passed( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """ + def test_1(): pass + def test_2(): pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*2 passed*"]) + result = pytester.runpytest("--lf") + result.stdout.fnmatch_lines(["*2 passed*"]) + result = pytester.runpytest("--lf", "--lfnf", "all") + result.stdout.fnmatch_lines(["*2 passed*"]) + + # Ensure the list passed to pytest_deselected is a copy, + # and not a reference which is cleared right after. + pytester.makeconftest( + """ + deselected = [] + + def pytest_deselected(items): + global deselected + deselected = items + + def pytest_sessionfinish(): + print("\\ndeselected={}".format(len(deselected))) + """ + ) + + result = pytester.runpytest("--lf", "--lfnf", "none") + result.stdout.fnmatch_lines( + [ + "collected 2 items / 2 deselected", + "run-last-failure: no previously failed tests, deselecting all items.", + "deselected=2", + "* 2 deselected in *", + ] + ) + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + def test_lastfailed_no_failures_behavior_empty_cache( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """ + def test_1(): pass + def test_2(): assert 0 + """ + ) + result = pytester.runpytest("--lf", "--cache-clear") + result.stdout.fnmatch_lines(["*1 failed*1 passed*"]) + result = pytester.runpytest("--lf", "--cache-clear", "--lfnf", "all") + result.stdout.fnmatch_lines(["*1 failed*1 passed*"]) + result = pytester.runpytest("--lf", "--cache-clear", "--lfnf", "none") + result.stdout.fnmatch_lines(["*2 desel*"]) + + def test_lastfailed_skip_collection(self, pytester: Pytester) -> None: + """ + Test --lf behavior regarding skipping collection of files that are not marked as + failed in the cache (#5172). + """ + pytester.makepyfile( + **{ + "pkg1/test_1.py": """ + import pytest + + @pytest.mark.parametrize('i', range(3)) + def test_1(i): pass + """, + "pkg2/test_2.py": """ + import pytest + + @pytest.mark.parametrize('i', range(5)) + def test_1(i): + assert i not in (1, 3) + """, + } + ) + # first run: collects 8 items (test_1: 3, test_2: 5) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["collected 8 items", "*2 failed*6 passed*"]) + # second run: collects only 5 items from test_2, because all tests from test_1 have passed + result = pytester.runpytest("--lf") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "run-last-failure: rerun previous 2 failures (skipped 1 file)", + "*= 2 failed in *", + ] + ) + + # add another file and check if message is correct when skipping more than 1 file + pytester.makepyfile( + **{ + "pkg1/test_3.py": """ + def test_3(): pass + """ + } + ) + result = pytester.runpytest("--lf") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "run-last-failure: rerun previous 2 failures (skipped 2 files)", + "*= 2 failed in *", + ] + ) + + def test_lastfailed_with_known_failures_not_being_selected( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + **{ + "pkg1/test_1.py": """def test_1(): assert 0""", + "pkg1/test_2.py": """def test_2(): pass""", + } + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["collected 2 items", "* 1 failed, 1 passed in *"]) + + Path("pkg1/test_1.py").unlink() + result = pytester.runpytest("--lf") + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "run-last-failure: 1 known failures not in selected tests", + "* 1 passed in *", + ] + ) + + # Recreate file with known failure. + pytester.makepyfile(**{"pkg1/test_1.py": """def test_1(): assert 0"""}) + result = pytester.runpytest("--lf") + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "run-last-failure: rerun previous 1 failure (skipped 1 file)", + "* 1 failed in *", + ] + ) + + # Remove/rename test: collects the file again. + pytester.makepyfile(**{"pkg1/test_1.py": """def test_renamed(): assert 0"""}) + result = pytester.runpytest("--lf", "-rf") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "run-last-failure: 1 known failures not in selected tests", + "pkg1/test_1.py F *", + "pkg1/test_2.py . *", + "FAILED pkg1/test_1.py::test_renamed - assert 0", + "* 1 failed, 1 passed in *", + ] + ) + + result = pytester.runpytest("--lf", "--co") + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "run-last-failure: rerun previous 1 failure (skipped 1 file)", + "", + "", + " ", + ] + ) + + def test_lastfailed_args_with_deselected(self, pytester: Pytester) -> None: + """Test regression with --lf running into NoMatch error. + + This was caused by it not collecting (non-failed) nodes given as + arguments. + """ + pytester.makepyfile( + **{ + "pkg1/test_1.py": """ + def test_pass(): pass + def test_fail(): assert 0 + """, + } + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["collected 2 items", "* 1 failed, 1 passed in *"]) + assert result.ret == 1 + + result = pytester.runpytest("pkg1/test_1.py::test_pass", "--lf", "--co") + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + "*collected 1 item", + "run-last-failure: 1 known failures not in selected tests", + "", + "", + " ", + ], + consecutive=True, + ) + + result = pytester.runpytest( + "pkg1/test_1.py::test_pass", "pkg1/test_1.py::test_fail", "--lf", "--co" + ) + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + "collected 2 items / 1 deselected / 1 selected", + "run-last-failure: rerun previous 1 failure", + "", + "", + " ", + "*= 1/2 tests collected (1 deselected) in *", + ], + ) + + def test_lastfailed_with_class_items(self, pytester: Pytester) -> None: + """Test regression with --lf deselecting whole classes.""" + pytester.makepyfile( + **{ + "pkg1/test_1.py": """ + class TestFoo: + def test_pass(self): pass + def test_fail(self): assert 0 + + def test_other(): assert 0 + """, + } + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["collected 3 items", "* 2 failed, 1 passed in *"]) + assert result.ret == 1 + + result = pytester.runpytest("--lf", "--co") + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + "collected 3 items / 1 deselected / 2 selected", + "run-last-failure: rerun previous 2 failures", + "", + "", + " ", + " ", + " ", + "", + "*= 2/3 tests collected (1 deselected) in *", + ], + consecutive=True, + ) + + def test_lastfailed_with_all_filtered(self, pytester: Pytester) -> None: + pytester.makepyfile( + **{ + "pkg1/test_1.py": """ + def test_fail(): assert 0 + def test_pass(): pass + """, + } + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["collected 2 items", "* 1 failed, 1 passed in *"]) + assert result.ret == 1 + + # Remove known failure. + pytester.makepyfile( + **{ + "pkg1/test_1.py": """ + def test_pass(): pass + """, + } + ) + result = pytester.runpytest("--lf", "--co") + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "run-last-failure: 1 known failures not in selected tests", + "", + "", + " ", + "", + "*= 1 test collected in*", + ], + consecutive=True, + ) + assert result.ret == 0 + + def test_packages(self, pytester: Pytester) -> None: + """Regression test for #7758. + + The particular issue here was that Package nodes were included in the + filtering, being themselves Modules for the __init__.py, even if they + had failed Modules in them. + + The tests includes a test in an __init__.py file just to make sure the + fix doesn't somehow regress that, it is not critical for the issue. + """ + pytester.makepyfile( + **{ + "__init__.py": "", + "a/__init__.py": "def test_a_init(): assert False", + "a/test_one.py": "def test_1(): assert False", + "b/__init__.py": "", + "b/test_two.py": "def test_2(): assert False", + }, + ) + pytester.makeini( + """ + [pytest] + python_files = *.py + """ + ) + result = pytester.runpytest() + result.assert_outcomes(failed=3) + result = pytester.runpytest("--lf") + result.assert_outcomes(failed=3) + + +class TestNewFirst: + def test_newfirst_usecase(self, pytester: Pytester) -> None: + pytester.makepyfile( + **{ + "test_1/test_1.py": """ + def test_1(): assert 1 + """, + "test_2/test_2.py": """ + def test_1(): assert 1 + """, + } + ) + + p1 = pytester.path.joinpath("test_1/test_1.py") + os.utime(p1, ns=(p1.stat().st_atime_ns, int(1e9))) + + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + ["*test_1/test_1.py::test_1 PASSED*", "*test_2/test_2.py::test_1 PASSED*"] + ) + + result = pytester.runpytest("-v", "--nf") + result.stdout.fnmatch_lines( + ["*test_2/test_2.py::test_1 PASSED*", "*test_1/test_1.py::test_1 PASSED*"] + ) + + p1.write_text("def test_1(): assert 1\n" "def test_2(): assert 1\n") + os.utime(p1, ns=(p1.stat().st_atime_ns, int(1e9))) + + result = pytester.runpytest("--nf", "--collect-only", "-q") + result.stdout.fnmatch_lines( + [ + "test_1/test_1.py::test_2", + "test_2/test_2.py::test_1", + "test_1/test_1.py::test_1", + ] + ) + + # Newest first with (plugin) pytest_collection_modifyitems hook. + pytester.makepyfile( + myplugin=""" + def pytest_collection_modifyitems(items): + items[:] = sorted(items, key=lambda item: item.nodeid) + print("new_items:", [x.nodeid for x in items]) + """ + ) + pytester.syspathinsert() + result = pytester.runpytest("--nf", "-p", "myplugin", "--collect-only", "-q") + result.stdout.fnmatch_lines( + [ + "new_items: *test_1.py*test_1.py*test_2.py*", + "test_1/test_1.py::test_2", + "test_2/test_2.py::test_1", + "test_1/test_1.py::test_1", + ] + ) + + def test_newfirst_parametrize(self, pytester: Pytester) -> None: + pytester.makepyfile( + **{ + "test_1/test_1.py": """ + import pytest + @pytest.mark.parametrize('num', [1, 2]) + def test_1(num): assert num + """, + "test_2/test_2.py": """ + import pytest + @pytest.mark.parametrize('num', [1, 2]) + def test_1(num): assert num + """, + } + ) + + p1 = pytester.path.joinpath("test_1/test_1.py") + os.utime(p1, ns=(p1.stat().st_atime_ns, int(1e9))) + + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "*test_1/test_1.py::test_1[1*", + "*test_1/test_1.py::test_1[2*", + "*test_2/test_2.py::test_1[1*", + "*test_2/test_2.py::test_1[2*", + ] + ) + + result = pytester.runpytest("-v", "--nf") + result.stdout.fnmatch_lines( + [ + "*test_2/test_2.py::test_1[1*", + "*test_2/test_2.py::test_1[2*", + "*test_1/test_1.py::test_1[1*", + "*test_1/test_1.py::test_1[2*", + ] + ) + + p1.write_text( + "import pytest\n" + "@pytest.mark.parametrize('num', [1, 2, 3])\n" + "def test_1(num): assert num\n" + ) + os.utime(p1, ns=(p1.stat().st_atime_ns, int(1e9))) + + # Running only a subset does not forget about existing ones. + result = pytester.runpytest("-v", "--nf", "test_2/test_2.py") + result.stdout.fnmatch_lines( + ["*test_2/test_2.py::test_1[1*", "*test_2/test_2.py::test_1[2*"] + ) + + result = pytester.runpytest("-v", "--nf") + result.stdout.fnmatch_lines( + [ + "*test_1/test_1.py::test_1[3*", + "*test_2/test_2.py::test_1[1*", + "*test_2/test_2.py::test_1[2*", + "*test_1/test_1.py::test_1[1*", + "*test_1/test_1.py::test_1[2*", + ] + ) + + +class TestReadme: + def check_readme(self, pytester: Pytester) -> bool: + config = pytester.parseconfigure() + assert config.cache is not None + readme = config.cache._cachedir.joinpath("README.md") + return readme.is_file() + + def test_readme_passed(self, pytester: Pytester) -> None: + pytester.makepyfile("def test_always_passes(): pass") + pytester.runpytest() + assert self.check_readme(pytester) is True + + def test_readme_failed(self, pytester: Pytester) -> None: + pytester.makepyfile("def test_always_fails(): assert 0") + pytester.runpytest() + assert self.check_readme(pytester) is True + + +def test_gitignore(pytester: Pytester) -> None: + """Ensure we automatically create .gitignore file in the pytest_cache directory (#3286).""" + from _pytest.cacheprovider import Cache + + config = pytester.parseconfig() + cache = Cache.for_config(config, _ispytest=True) + cache.set("foo", "bar") + msg = "# Created by pytest automatically.\n*\n" + gitignore_path = cache._cachedir.joinpath(".gitignore") + assert gitignore_path.read_text(encoding="UTF-8") == msg + + # Does not overwrite existing/custom one. + gitignore_path.write_text("custom") + cache.set("something", "else") + assert gitignore_path.read_text(encoding="UTF-8") == "custom" + + +def test_preserve_keys_order(pytester: Pytester) -> None: + """Ensure keys order is preserved when saving dicts (#9205).""" + from _pytest.cacheprovider import Cache + + config = pytester.parseconfig() + cache = Cache.for_config(config, _ispytest=True) + cache.set("foo", {"z": 1, "b": 2, "a": 3, "d": 10}) + read_back = cache.get("foo", None) + assert list(read_back.items()) == [("z", 1), ("b", 2), ("a", 3), ("d", 10)] + + +def test_does_not_create_boilerplate_in_existing_dirs(pytester: Pytester) -> None: + from _pytest.cacheprovider import Cache + + pytester.makeini( + """ + [pytest] + cache_dir = . + """ + ) + config = pytester.parseconfig() + cache = Cache.for_config(config, _ispytest=True) + cache.set("foo", "bar") + + assert os.path.isdir("v") # cache contents + assert not os.path.exists(".gitignore") + assert not os.path.exists("README.md") + + +def test_cachedir_tag(pytester: Pytester) -> None: + """Ensure we automatically create CACHEDIR.TAG file in the pytest_cache directory (#4278).""" + from _pytest.cacheprovider import Cache + from _pytest.cacheprovider import CACHEDIR_TAG_CONTENT + + config = pytester.parseconfig() + cache = Cache.for_config(config, _ispytest=True) + cache.set("foo", "bar") + cachedir_tag_path = cache._cachedir.joinpath("CACHEDIR.TAG") + assert cachedir_tag_path.read_bytes() == CACHEDIR_TAG_CONTENT diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_capture.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_capture.py new file mode 100644 index 0000000000..1bc1f2f8db --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_capture.py @@ -0,0 +1,1666 @@ +import contextlib +import io +import os +import subprocess +import sys +import textwrap +from io import UnsupportedOperation +from typing import BinaryIO +from typing import cast +from typing import Generator +from typing import TextIO + +import pytest +from _pytest import capture +from _pytest.capture import _get_multicapture +from _pytest.capture import CaptureFixture +from _pytest.capture import CaptureManager +from _pytest.capture import CaptureResult +from _pytest.capture import MultiCapture +from _pytest.config import ExitCode +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester + +# note: py.io capture tests where copied from +# pylib 1.4.20.dev2 (rev 13d9af95547e) + + +def StdCaptureFD( + out: bool = True, err: bool = True, in_: bool = True +) -> MultiCapture[str]: + return capture.MultiCapture( + in_=capture.FDCapture(0) if in_ else None, + out=capture.FDCapture(1) if out else None, + err=capture.FDCapture(2) if err else None, + ) + + +def StdCapture( + out: bool = True, err: bool = True, in_: bool = True +) -> MultiCapture[str]: + return capture.MultiCapture( + in_=capture.SysCapture(0) if in_ else None, + out=capture.SysCapture(1) if out else None, + err=capture.SysCapture(2) if err else None, + ) + + +def TeeStdCapture( + out: bool = True, err: bool = True, in_: bool = True +) -> MultiCapture[str]: + return capture.MultiCapture( + in_=capture.SysCapture(0, tee=True) if in_ else None, + out=capture.SysCapture(1, tee=True) if out else None, + err=capture.SysCapture(2, tee=True) if err else None, + ) + + +class TestCaptureManager: + @pytest.mark.parametrize("method", ["no", "sys", "fd"]) + def test_capturing_basic_api(self, method) -> None: + capouter = StdCaptureFD() + old = sys.stdout, sys.stderr, sys.stdin + try: + capman = CaptureManager(method) + capman.start_global_capturing() + capman.suspend_global_capture() + outerr = capman.read_global_capture() + assert outerr == ("", "") + capman.suspend_global_capture() + outerr = capman.read_global_capture() + assert outerr == ("", "") + print("hello") + capman.suspend_global_capture() + out, err = capman.read_global_capture() + if method == "no": + assert old == (sys.stdout, sys.stderr, sys.stdin) + else: + assert not out + capman.resume_global_capture() + print("hello") + capman.suspend_global_capture() + out, err = capman.read_global_capture() + if method != "no": + assert out == "hello\n" + capman.stop_global_capturing() + finally: + capouter.stop_capturing() + + def test_init_capturing(self): + capouter = StdCaptureFD() + try: + capman = CaptureManager("fd") + capman.start_global_capturing() + pytest.raises(AssertionError, capman.start_global_capturing) + capman.stop_global_capturing() + finally: + capouter.stop_capturing() + + +@pytest.mark.parametrize("method", ["fd", "sys"]) +def test_capturing_unicode(pytester: Pytester, method: str) -> None: + obj = "'b\u00f6y'" + pytester.makepyfile( + """\ + # taken from issue 227 from nosetests + def test_unicode(): + import sys + print(sys.stdout) + print(%s) + """ + % obj + ) + result = pytester.runpytest("--capture=%s" % method) + result.stdout.fnmatch_lines(["*1 passed*"]) + + +@pytest.mark.parametrize("method", ["fd", "sys"]) +def test_capturing_bytes_in_utf8_encoding(pytester: Pytester, method: str) -> None: + pytester.makepyfile( + """\ + def test_unicode(): + print('b\\u00f6y') + """ + ) + result = pytester.runpytest("--capture=%s" % method) + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_collect_capturing(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import sys + + print("collect %s failure" % 13) + sys.stderr.write("collect %s_stderr failure" % 13) + import xyz42123 + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + [ + "*Captured stdout*", + "collect 13 failure", + "*Captured stderr*", + "collect 13_stderr failure", + ] + ) + + +class TestPerTestCapturing: + def test_capture_and_fixtures(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def setup_module(mod): + print("setup module") + def setup_function(function): + print("setup " + function.__name__) + def test_func1(): + print("in func1") + assert 0 + def test_func2(): + print("in func2") + assert 0 + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + [ + "setup module*", + "setup test_func1*", + "in func1*", + "setup test_func2*", + "in func2*", + ] + ) + + @pytest.mark.xfail(reason="unimplemented feature") + def test_capture_scope_cache(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import sys + def setup_module(func): + print("module-setup") + def setup_function(func): + print("function-setup") + def test_func(): + print("in function") + assert 0 + def teardown_function(func): + print("in teardown") + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + [ + "*test_func():*", + "*Captured stdout during setup*", + "module-setup*", + "function-setup*", + "*Captured stdout*", + "in teardown*", + ] + ) + + def test_no_carry_over(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def test_func1(): + print("in func1") + def test_func2(): + print("in func2") + assert 0 + """ + ) + result = pytester.runpytest(p) + s = result.stdout.str() + assert "in func1" not in s + assert "in func2" in s + + def test_teardown_capturing(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def setup_function(function): + print("setup func1") + def teardown_function(function): + print("teardown func1") + assert 0 + def test_func1(): + print("in func1") + pass + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + [ + "*teardown_function*", + "*Captured stdout*", + "setup func1*", + "in func1*", + "teardown func1*", + # "*1 fixture failure*" + ] + ) + + def test_teardown_capturing_final(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def teardown_module(mod): + print("teardown module") + assert 0 + def test_func(): + pass + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + [ + "*def teardown_module(mod):*", + "*Captured stdout*", + "*teardown module*", + "*1 error*", + ] + ) + + def test_capturing_outerr(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """\ + import sys + def test_capturing(): + print(42) + sys.stderr.write(str(23)) + def test_capturing_error(): + print(1) + sys.stderr.write(str(2)) + raise ValueError + """ + ) + result = pytester.runpytest(p1) + result.stdout.fnmatch_lines( + [ + "*test_capturing_outerr.py .F*", + "====* FAILURES *====", + "____*____", + "*test_capturing_outerr.py:8: ValueError", + "*--- Captured stdout *call*", + "1", + "*--- Captured stderr *call*", + "2", + ] + ) + + +class TestLoggingInteraction: + def test_logging_stream_ownership(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """\ + def test_logging(): + import logging + import pytest + stream = capture.CaptureIO() + logging.basicConfig(stream=stream) + stream.close() # to free memory/release resources + """ + ) + result = pytester.runpytest_subprocess(p) + assert result.stderr.str().find("atexit") == -1 + + def test_logging_and_immediate_setupteardown(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """\ + import logging + def setup_function(function): + logging.warning("hello1") + + def test_logging(): + logging.warning("hello2") + assert 0 + + def teardown_function(function): + logging.warning("hello3") + assert 0 + """ + ) + for optargs in (("--capture=sys",), ("--capture=fd",)): + print(optargs) + result = pytester.runpytest_subprocess(p, *optargs) + s = result.stdout.str() + result.stdout.fnmatch_lines( + ["*WARN*hello3", "*WARN*hello1", "*WARN*hello2"] # errors show first! + ) + # verify proper termination + assert "closed" not in s + + def test_logging_and_crossscope_fixtures(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """\ + import logging + def setup_module(function): + logging.warning("hello1") + + def test_logging(): + logging.warning("hello2") + assert 0 + + def teardown_module(function): + logging.warning("hello3") + assert 0 + """ + ) + for optargs in (("--capture=sys",), ("--capture=fd",)): + print(optargs) + result = pytester.runpytest_subprocess(p, *optargs) + s = result.stdout.str() + result.stdout.fnmatch_lines( + ["*WARN*hello3", "*WARN*hello1", "*WARN*hello2"] # errors come first + ) + # verify proper termination + assert "closed" not in s + + def test_conftestlogging_is_shown(self, pytester: Pytester) -> None: + pytester.makeconftest( + """\ + import logging + logging.basicConfig() + logging.warning("hello435") + """ + ) + # make sure that logging is still captured in tests + result = pytester.runpytest_subprocess("-s", "-p", "no:capturelog") + assert result.ret == ExitCode.NO_TESTS_COLLECTED + result.stderr.fnmatch_lines(["WARNING*hello435*"]) + assert "operation on closed file" not in result.stderr.str() + + def test_conftestlogging_and_test_logging(self, pytester: Pytester) -> None: + pytester.makeconftest( + """\ + import logging + logging.basicConfig() + """ + ) + # make sure that logging is still captured in tests + p = pytester.makepyfile( + """\ + def test_hello(): + import logging + logging.warning("hello433") + assert 0 + """ + ) + result = pytester.runpytest_subprocess(p, "-p", "no:capturelog") + assert result.ret != 0 + result.stdout.fnmatch_lines(["WARNING*hello433*"]) + assert "something" not in result.stderr.str() + assert "operation on closed file" not in result.stderr.str() + + def test_logging_after_cap_stopped(self, pytester: Pytester) -> None: + pytester.makeconftest( + """\ + import pytest + import logging + + log = logging.getLogger(__name__) + + @pytest.fixture + def log_on_teardown(): + yield + log.warning('Logging on teardown') + """ + ) + # make sure that logging is still captured in tests + p = pytester.makepyfile( + """\ + def test_hello(log_on_teardown): + import logging + logging.warning("hello433") + assert 1 + raise KeyboardInterrupt() + """ + ) + result = pytester.runpytest_subprocess(p, "--log-cli-level", "info") + assert result.ret != 0 + result.stdout.fnmatch_lines( + ["*WARNING*hello433*", "*WARNING*Logging on teardown*"] + ) + assert ( + "AttributeError: 'NoneType' object has no attribute 'resume_capturing'" + not in result.stderr.str() + ) + + +class TestCaptureFixture: + @pytest.mark.parametrize("opt", [[], ["-s"]]) + def test_std_functional(self, pytester: Pytester, opt) -> None: + reprec = pytester.inline_runsource( + """\ + def test_hello(capsys): + print(42) + out, err = capsys.readouterr() + assert out.startswith("42") + """, + *opt, + ) + reprec.assertoutcome(passed=1) + + def test_capsyscapfd(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """\ + def test_one(capsys, capfd): + pass + def test_two(capfd, capsys): + pass + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + [ + "*ERROR*setup*test_one*", + "E*capfd*capsys*same*time*", + "*ERROR*setup*test_two*", + "E*capsys*capfd*same*time*", + "*2 errors*", + ] + ) + + def test_capturing_getfixturevalue(self, pytester: Pytester) -> None: + """Test that asking for "capfd" and "capsys" using request.getfixturevalue + in the same test is an error. + """ + pytester.makepyfile( + """\ + def test_one(capsys, request): + request.getfixturevalue("capfd") + def test_two(capfd, request): + request.getfixturevalue("capsys") + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*test_one*", + "E * cannot use capfd and capsys at the same time", + "*test_two*", + "E * cannot use capsys and capfd at the same time", + "*2 failed in*", + ] + ) + + def test_capsyscapfdbinary(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """\ + def test_one(capsys, capfdbinary): + pass + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + ["*ERROR*setup*test_one*", "E*capfdbinary*capsys*same*time*", "*1 error*"] + ) + + @pytest.mark.parametrize("method", ["sys", "fd"]) + def test_capture_is_represented_on_failure_issue128( + self, pytester: Pytester, method + ) -> None: + p = pytester.makepyfile( + """\ + def test_hello(cap{}): + print("xxx42xxx") + assert 0 + """.format( + method + ) + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines(["xxx42xxx"]) + + def test_stdfd_functional(self, pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """\ + def test_hello(capfd): + import os + os.write(1, b"42") + out, err = capfd.readouterr() + assert out.startswith("42") + capfd.close() + """ + ) + reprec.assertoutcome(passed=1) + + @pytest.mark.parametrize("nl", ("\n", "\r\n", "\r")) + def test_cafd_preserves_newlines(self, capfd, nl) -> None: + print("test", end=nl) + out, err = capfd.readouterr() + assert out.endswith(nl) + + def test_capfdbinary(self, pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """\ + def test_hello(capfdbinary): + import os + # some likely un-decodable bytes + os.write(1, b'\\xfe\\x98\\x20') + out, err = capfdbinary.readouterr() + assert out == b'\\xfe\\x98\\x20' + assert err == b'' + """ + ) + reprec.assertoutcome(passed=1) + + def test_capsysbinary(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + r""" + def test_hello(capsysbinary): + import sys + + sys.stdout.buffer.write(b'hello') + + # Some likely un-decodable bytes. + sys.stdout.buffer.write(b'\xfe\x98\x20') + + sys.stdout.buffer.flush() + + # Ensure writing in text mode still works and is captured. + # https://github.com/pytest-dev/pytest/issues/6871 + print("world", flush=True) + + out, err = capsysbinary.readouterr() + assert out == b'hello\xfe\x98\x20world\n' + assert err == b'' + + print("stdout after") + print("stderr after", file=sys.stderr) + """ + ) + result = pytester.runpytest(str(p1), "-rA") + result.stdout.fnmatch_lines( + [ + "*- Captured stdout call -*", + "stdout after", + "*- Captured stderr call -*", + "stderr after", + "*= 1 passed in *", + ] + ) + + def test_partial_setup_failure(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """\ + def test_hello(capsys, missingarg): + pass + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines(["*test_partial_setup_failure*", "*1 error*"]) + + def test_keyboardinterrupt_disables_capturing(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """\ + def test_hello(capfd): + import os + os.write(1, b'42') + raise KeyboardInterrupt() + """ + ) + result = pytester.runpytest_subprocess(p) + result.stdout.fnmatch_lines(["*KeyboardInterrupt*"]) + assert result.ret == 2 + + def test_capture_and_logging(self, pytester: Pytester) -> None: + """#14""" + p = pytester.makepyfile( + """\ + import logging + def test_log(capsys): + logging.error('x') + """ + ) + result = pytester.runpytest_subprocess(p) + assert "closed" not in result.stderr.str() + + @pytest.mark.parametrize("fixture", ["capsys", "capfd"]) + @pytest.mark.parametrize("no_capture", [True, False]) + def test_disabled_capture_fixture( + self, pytester: Pytester, fixture: str, no_capture: bool + ) -> None: + pytester.makepyfile( + """\ + def test_disabled({fixture}): + print('captured before') + with {fixture}.disabled(): + print('while capture is disabled') + print('captured after') + assert {fixture}.readouterr() == ('captured before\\ncaptured after\\n', '') + + def test_normal(): + print('test_normal executed') + """.format( + fixture=fixture + ) + ) + args = ("-s",) if no_capture else () + result = pytester.runpytest_subprocess(*args) + result.stdout.fnmatch_lines(["*while capture is disabled*", "*= 2 passed in *"]) + result.stdout.no_fnmatch_line("*captured before*") + result.stdout.no_fnmatch_line("*captured after*") + if no_capture: + assert "test_normal executed" in result.stdout.str() + else: + result.stdout.no_fnmatch_line("*test_normal executed*") + + def test_disabled_capture_fixture_twice(self, pytester: Pytester) -> None: + """Test that an inner disabled() exit doesn't undo an outer disabled(). + + Issue #7148. + """ + pytester.makepyfile( + """ + def test_disabled(capfd): + print('captured before') + with capfd.disabled(): + print('while capture is disabled 1') + with capfd.disabled(): + print('while capture is disabled 2') + print('while capture is disabled 1 after') + print('captured after') + assert capfd.readouterr() == ('captured before\\ncaptured after\\n', '') + """ + ) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines( + [ + "*while capture is disabled 1", + "*while capture is disabled 2", + "*while capture is disabled 1 after", + ], + consecutive=True, + ) + + @pytest.mark.parametrize("fixture", ["capsys", "capfd"]) + def test_fixture_use_by_other_fixtures(self, pytester: Pytester, fixture) -> None: + """Ensure that capsys and capfd can be used by other fixtures during + setup and teardown.""" + pytester.makepyfile( + """\ + import sys + import pytest + + @pytest.fixture + def captured_print({fixture}): + print('stdout contents begin') + print('stderr contents begin', file=sys.stderr) + out, err = {fixture}.readouterr() + + yield out, err + + print('stdout contents end') + print('stderr contents end', file=sys.stderr) + out, err = {fixture}.readouterr() + assert out == 'stdout contents end\\n' + assert err == 'stderr contents end\\n' + + def test_captured_print(captured_print): + out, err = captured_print + assert out == 'stdout contents begin\\n' + assert err == 'stderr contents begin\\n' + """.format( + fixture=fixture + ) + ) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines(["*1 passed*"]) + result.stdout.no_fnmatch_line("*stdout contents begin*") + result.stdout.no_fnmatch_line("*stderr contents begin*") + + @pytest.mark.parametrize("cap", ["capsys", "capfd"]) + def test_fixture_use_by_other_fixtures_teardown( + self, pytester: Pytester, cap + ) -> None: + """Ensure we can access setup and teardown buffers from teardown when using capsys/capfd (##3033)""" + pytester.makepyfile( + """\ + import sys + import pytest + import os + + @pytest.fixture() + def fix({cap}): + print("setup out") + sys.stderr.write("setup err\\n") + yield + out, err = {cap}.readouterr() + assert out == 'setup out\\ncall out\\n' + assert err == 'setup err\\ncall err\\n' + + def test_a(fix): + print("call out") + sys.stderr.write("call err\\n") + """.format( + cap=cap + ) + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +def test_setup_failure_does_not_kill_capturing(pytester: Pytester) -> None: + sub1 = pytester.mkpydir("sub1") + sub1.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + def pytest_runtest_setup(item): + raise ValueError(42) + """ + ) + ) + sub1.joinpath("test_mod.py").write_text("def test_func1(): pass") + result = pytester.runpytest(pytester.path, "--traceconfig") + result.stdout.fnmatch_lines(["*ValueError(42)*", "*1 error*"]) + + +def test_capture_conftest_runtest_setup(pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_runtest_setup(): + print("hello19") + """ + ) + pytester.makepyfile("def test_func(): pass") + result = pytester.runpytest() + assert result.ret == 0 + result.stdout.no_fnmatch_line("*hello19*") + + +def test_capture_badoutput_issue412(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import os + + def test_func(): + omg = bytearray([1,129,1]) + os.write(1, omg) + assert 0 + """ + ) + result = pytester.runpytest("--capture=fd") + result.stdout.fnmatch_lines( + """ + *def test_func* + *assert 0* + *Captured* + *1 failed* + """ + ) + + +def test_capture_early_option_parsing(pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_runtest_setup(): + print("hello19") + """ + ) + pytester.makepyfile("def test_func(): pass") + result = pytester.runpytest("-vs") + assert result.ret == 0 + assert "hello19" in result.stdout.str() + + +def test_capture_binary_output(pytester: Pytester) -> None: + pytester.makepyfile( + r""" + import pytest + + def test_a(): + import sys + import subprocess + subprocess.call([sys.executable, __file__]) + + def test_foo(): + import os;os.write(1, b'\xc3') + + if __name__ == '__main__': + test_foo() + """ + ) + result = pytester.runpytest("--assert=plain") + result.assert_outcomes(passed=2) + + +def test_error_during_readouterr(pytester: Pytester) -> None: + """Make sure we suspend capturing if errors occur during readouterr""" + pytester.makepyfile( + pytest_xyz=""" + from _pytest.capture import FDCapture + + def bad_snap(self): + raise Exception('boom') + + assert FDCapture.snap + FDCapture.snap = bad_snap + """ + ) + result = pytester.runpytest_subprocess("-p", "pytest_xyz", "--version") + result.stderr.fnmatch_lines( + ["*in bad_snap", " raise Exception('boom')", "Exception: boom"] + ) + + +class TestCaptureIO: + def test_text(self) -> None: + f = capture.CaptureIO() + f.write("hello") + s = f.getvalue() + assert s == "hello" + f.close() + + def test_unicode_and_str_mixture(self) -> None: + f = capture.CaptureIO() + f.write("\u00f6") + pytest.raises(TypeError, f.write, b"hello") + + def test_write_bytes_to_buffer(self) -> None: + """In python3, stdout / stderr are text io wrappers (exposing a buffer + property of the underlying bytestream). See issue #1407 + """ + f = capture.CaptureIO() + f.buffer.write(b"foo\r\n") + assert f.getvalue() == "foo\r\n" + + +class TestTeeCaptureIO(TestCaptureIO): + def test_text(self) -> None: + sio = io.StringIO() + f = capture.TeeCaptureIO(sio) + f.write("hello") + s1 = f.getvalue() + assert s1 == "hello" + s2 = sio.getvalue() + assert s2 == s1 + f.close() + sio.close() + + def test_unicode_and_str_mixture(self) -> None: + sio = io.StringIO() + f = capture.TeeCaptureIO(sio) + f.write("\u00f6") + pytest.raises(TypeError, f.write, b"hello") + + +def test_dontreadfrominput() -> None: + from _pytest.capture import DontReadFromInput + + f = DontReadFromInput() + assert f.buffer is f + assert not f.isatty() + pytest.raises(OSError, f.read) + pytest.raises(OSError, f.readlines) + iter_f = iter(f) + pytest.raises(OSError, next, iter_f) + pytest.raises(UnsupportedOperation, f.fileno) + f.close() # just for completeness + + +def test_captureresult() -> None: + cr = CaptureResult("out", "err") + assert len(cr) == 2 + assert cr.out == "out" + assert cr.err == "err" + out, err = cr + assert out == "out" + assert err == "err" + assert cr[0] == "out" + assert cr[1] == "err" + assert cr == cr + assert cr == CaptureResult("out", "err") + assert cr != CaptureResult("wrong", "err") + assert cr == ("out", "err") + assert cr != ("out", "wrong") + assert hash(cr) == hash(CaptureResult("out", "err")) + assert hash(cr) == hash(("out", "err")) + assert hash(cr) != hash(("out", "wrong")) + assert cr < ("z",) + assert cr < ("z", "b") + assert cr < ("z", "b", "c") + assert cr.count("err") == 1 + assert cr.count("wrong") == 0 + assert cr.index("err") == 1 + with pytest.raises(ValueError): + assert cr.index("wrong") == 0 + assert next(iter(cr)) == "out" + assert cr._replace(err="replaced") == ("out", "replaced") + + +@pytest.fixture +def tmpfile(pytester: Pytester) -> Generator[BinaryIO, None, None]: + f = pytester.makepyfile("").open("wb+") + yield f + if not f.closed: + f.close() + + +@contextlib.contextmanager +def lsof_check(): + pid = os.getpid() + try: + out = subprocess.check_output(("lsof", "-p", str(pid))).decode() + except (OSError, subprocess.CalledProcessError, UnicodeDecodeError) as exc: + # about UnicodeDecodeError, see note on pytester + pytest.skip(f"could not run 'lsof' ({exc!r})") + yield + out2 = subprocess.check_output(("lsof", "-p", str(pid))).decode() + len1 = len([x for x in out.split("\n") if "REG" in x]) + len2 = len([x for x in out2.split("\n") if "REG" in x]) + assert len2 < len1 + 3, out2 + + +class TestFDCapture: + def test_simple(self, tmpfile: BinaryIO) -> None: + fd = tmpfile.fileno() + cap = capture.FDCapture(fd) + data = b"hello" + os.write(fd, data) + pytest.raises(AssertionError, cap.snap) + cap.done() + cap = capture.FDCapture(fd) + cap.start() + os.write(fd, data) + s = cap.snap() + cap.done() + assert s == "hello" + + def test_simple_many(self, tmpfile: BinaryIO) -> None: + for i in range(10): + self.test_simple(tmpfile) + + def test_simple_many_check_open_files(self, pytester: Pytester) -> None: + with lsof_check(): + with pytester.makepyfile("").open("wb+") as tmpfile: + self.test_simple_many(tmpfile) + + def test_simple_fail_second_start(self, tmpfile: BinaryIO) -> None: + fd = tmpfile.fileno() + cap = capture.FDCapture(fd) + cap.done() + pytest.raises(AssertionError, cap.start) + + def test_stderr(self) -> None: + cap = capture.FDCapture(2) + cap.start() + print("hello", file=sys.stderr) + s = cap.snap() + cap.done() + assert s == "hello\n" + + def test_stdin(self) -> None: + cap = capture.FDCapture(0) + cap.start() + x = os.read(0, 100).strip() + cap.done() + assert x == b"" + + def test_writeorg(self, tmpfile: BinaryIO) -> None: + data1, data2 = b"foo", b"bar" + cap = capture.FDCapture(tmpfile.fileno()) + cap.start() + tmpfile.write(data1) + tmpfile.flush() + cap.writeorg(data2.decode("ascii")) + scap = cap.snap() + cap.done() + assert scap == data1.decode("ascii") + with open(tmpfile.name, "rb") as stmp_file: + stmp = stmp_file.read() + assert stmp == data2 + + def test_simple_resume_suspend(self) -> None: + with saved_fd(1): + cap = capture.FDCapture(1) + cap.start() + data = b"hello" + os.write(1, data) + sys.stdout.write("whatever") + s = cap.snap() + assert s == "hellowhatever" + cap.suspend() + os.write(1, b"world") + sys.stdout.write("qlwkej") + assert not cap.snap() + cap.resume() + os.write(1, b"but now") + sys.stdout.write(" yes\n") + s = cap.snap() + assert s == "but now yes\n" + cap.suspend() + cap.done() + pytest.raises(AssertionError, cap.suspend) + + assert repr(cap) == ( + "".format( + cap.targetfd_save, cap.tmpfile + ) + ) + # Should not crash with missing "_old". + assert repr(cap.syscapture) == ( + " _state='done' tmpfile={!r}>".format( + cap.syscapture.tmpfile + ) + ) + + def test_capfd_sys_stdout_mode(self, capfd) -> None: + assert "b" not in sys.stdout.mode + + +@contextlib.contextmanager +def saved_fd(fd): + new_fd = os.dup(fd) + try: + yield + finally: + os.dup2(new_fd, fd) + os.close(new_fd) + + +class TestStdCapture: + captureclass = staticmethod(StdCapture) + + @contextlib.contextmanager + def getcapture(self, **kw): + cap = self.__class__.captureclass(**kw) + cap.start_capturing() + try: + yield cap + finally: + cap.stop_capturing() + + def test_capturing_done_simple(self) -> None: + with self.getcapture() as cap: + sys.stdout.write("hello") + sys.stderr.write("world") + out, err = cap.readouterr() + assert out == "hello" + assert err == "world" + + def test_capturing_reset_simple(self) -> None: + with self.getcapture() as cap: + print("hello world") + sys.stderr.write("hello error\n") + out, err = cap.readouterr() + assert out == "hello world\n" + assert err == "hello error\n" + + def test_capturing_readouterr(self) -> None: + with self.getcapture() as cap: + print("hello world") + sys.stderr.write("hello error\n") + out, err = cap.readouterr() + assert out == "hello world\n" + assert err == "hello error\n" + sys.stderr.write("error2") + out, err = cap.readouterr() + assert err == "error2" + + def test_capture_results_accessible_by_attribute(self) -> None: + with self.getcapture() as cap: + sys.stdout.write("hello") + sys.stderr.write("world") + capture_result = cap.readouterr() + assert capture_result.out == "hello" + assert capture_result.err == "world" + + def test_capturing_readouterr_unicode(self) -> None: + with self.getcapture() as cap: + print("hxąć") + out, err = cap.readouterr() + assert out == "hxąć\n" + + def test_reset_twice_error(self) -> None: + with self.getcapture() as cap: + print("hello") + out, err = cap.readouterr() + pytest.raises(ValueError, cap.stop_capturing) + assert out == "hello\n" + assert not err + + def test_capturing_modify_sysouterr_in_between(self) -> None: + oldout = sys.stdout + olderr = sys.stderr + with self.getcapture() as cap: + sys.stdout.write("hello") + sys.stderr.write("world") + sys.stdout = capture.CaptureIO() + sys.stderr = capture.CaptureIO() + print("not seen") + sys.stderr.write("not seen\n") + out, err = cap.readouterr() + assert out == "hello" + assert err == "world" + assert sys.stdout == oldout + assert sys.stderr == olderr + + def test_capturing_error_recursive(self) -> None: + with self.getcapture() as cap1: + print("cap1") + with self.getcapture() as cap2: + print("cap2") + out2, err2 = cap2.readouterr() + out1, err1 = cap1.readouterr() + assert out1 == "cap1\n" + assert out2 == "cap2\n" + + def test_just_out_capture(self) -> None: + with self.getcapture(out=True, err=False) as cap: + sys.stdout.write("hello") + sys.stderr.write("world") + out, err = cap.readouterr() + assert out == "hello" + assert not err + + def test_just_err_capture(self) -> None: + with self.getcapture(out=False, err=True) as cap: + sys.stdout.write("hello") + sys.stderr.write("world") + out, err = cap.readouterr() + assert err == "world" + assert not out + + def test_stdin_restored(self) -> None: + old = sys.stdin + with self.getcapture(in_=True): + newstdin = sys.stdin + assert newstdin != sys.stdin + assert sys.stdin is old + + def test_stdin_nulled_by_default(self) -> None: + print("XXX this test may well hang instead of crashing") + print("XXX which indicates an error in the underlying capturing") + print("XXX mechanisms") + with self.getcapture(): + pytest.raises(OSError, sys.stdin.read) + + +class TestTeeStdCapture(TestStdCapture): + captureclass = staticmethod(TeeStdCapture) + + def test_capturing_error_recursive(self) -> None: + r"""For TeeStdCapture since we passthrough stderr/stdout, cap1 + should get all output, while cap2 should only get "cap2\n".""" + + with self.getcapture() as cap1: + print("cap1") + with self.getcapture() as cap2: + print("cap2") + out2, err2 = cap2.readouterr() + out1, err1 = cap1.readouterr() + assert out1 == "cap1\ncap2\n" + assert out2 == "cap2\n" + + +class TestStdCaptureFD(TestStdCapture): + captureclass = staticmethod(StdCaptureFD) + + def test_simple_only_fd(self, pytester: Pytester) -> None: + pytester.makepyfile( + """\ + import os + def test_x(): + os.write(1, b"hello\\n") + assert 0 + """ + ) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines( + """ + *test_x* + *assert 0* + *Captured stdout* + """ + ) + + def test_intermingling(self): + with self.getcapture() as cap: + os.write(1, b"1") + sys.stdout.write(str(2)) + sys.stdout.flush() + os.write(1, b"3") + os.write(2, b"a") + sys.stderr.write("b") + sys.stderr.flush() + os.write(2, b"c") + out, err = cap.readouterr() + assert out == "123" + assert err == "abc" + + def test_many(self, capfd): + with lsof_check(): + for i in range(10): + cap = StdCaptureFD() + cap.start_capturing() + cap.stop_capturing() + + +class TestStdCaptureFDinvalidFD: + def test_stdcapture_fd_invalid_fd(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import os + from fnmatch import fnmatch + from _pytest import capture + + def StdCaptureFD(out=True, err=True, in_=True): + return capture.MultiCapture( + in_=capture.FDCapture(0) if in_ else None, + out=capture.FDCapture(1) if out else None, + err=capture.FDCapture(2) if err else None, + ) + + def test_stdout(): + os.close(1) + cap = StdCaptureFD(out=True, err=False, in_=False) + assert fnmatch(repr(cap.out), "") + cap.start_capturing() + os.write(1, b"stdout") + assert cap.readouterr() == ("stdout", "") + cap.stop_capturing() + + def test_stderr(): + os.close(2) + cap = StdCaptureFD(out=False, err=True, in_=False) + assert fnmatch(repr(cap.err), "") + cap.start_capturing() + os.write(2, b"stderr") + assert cap.readouterr() == ("", "stderr") + cap.stop_capturing() + + def test_stdin(): + os.close(0) + cap = StdCaptureFD(out=False, err=False, in_=True) + assert fnmatch(repr(cap.in_), "") + cap.stop_capturing() + """ + ) + result = pytester.runpytest_subprocess("--capture=fd") + assert result.ret == 0 + assert result.parseoutcomes()["passed"] == 3 + + def test_fdcapture_invalid_fd_with_fd_reuse(self, pytester: Pytester) -> None: + with saved_fd(1): + os.close(1) + cap = capture.FDCaptureBinary(1) + cap.start() + os.write(1, b"started") + cap.suspend() + os.write(1, b" suspended") + cap.resume() + os.write(1, b" resumed") + assert cap.snap() == b"started resumed" + cap.done() + with pytest.raises(OSError): + os.write(1, b"done") + + def test_fdcapture_invalid_fd_without_fd_reuse(self, pytester: Pytester) -> None: + with saved_fd(1), saved_fd(2): + os.close(1) + os.close(2) + cap = capture.FDCaptureBinary(2) + cap.start() + os.write(2, b"started") + cap.suspend() + os.write(2, b" suspended") + cap.resume() + os.write(2, b" resumed") + assert cap.snap() == b"started resumed" + cap.done() + with pytest.raises(OSError): + os.write(2, b"done") + + +def test_capture_not_started_but_reset() -> None: + capsys = StdCapture() + capsys.stop_capturing() + + +def test_using_capsys_fixture_works_with_sys_stdout_encoding( + capsys: CaptureFixture[str], +) -> None: + test_text = "test text" + + print(test_text.encode(sys.stdout.encoding, "replace")) + (out, err) = capsys.readouterr() + assert out + assert err == "" + + +def test_capsys_results_accessible_by_attribute(capsys: CaptureFixture[str]) -> None: + sys.stdout.write("spam") + sys.stderr.write("eggs") + capture_result = capsys.readouterr() + assert capture_result.out == "spam" + assert capture_result.err == "eggs" + + +def test_fdcapture_tmpfile_remains_the_same() -> None: + cap = StdCaptureFD(out=False, err=True) + try: + cap.start_capturing() + capfile = cap.err.tmpfile + cap.readouterr() + finally: + cap.stop_capturing() + capfile2 = cap.err.tmpfile + assert capfile2 == capfile + + +def test_close_and_capture_again(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import os + def test_close(): + os.close(1) + def test_capture_again(): + os.write(1, b"hello\\n") + assert 0 + """ + ) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines( + """ + *test_capture_again* + *assert 0* + *stdout* + *hello* + """ + ) + + +@pytest.mark.parametrize( + "method", ["SysCapture(2)", "SysCapture(2, tee=True)", "FDCapture(2)"] +) +def test_capturing_and_logging_fundamentals(pytester: Pytester, method: str) -> None: + # here we check a fundamental feature + p = pytester.makepyfile( + """ + import sys, os, logging + from _pytest import capture + cap = capture.MultiCapture( + in_=None, + out=None, + err=capture.%s, + ) + cap.start_capturing() + + logging.warning("hello1") + outerr = cap.readouterr() + print("suspend, captured %%s" %%(outerr,)) + logging.warning("hello2") + + cap.pop_outerr_to_orig() + logging.warning("hello3") + + outerr = cap.readouterr() + print("suspend2, captured %%s" %% (outerr,)) + """ + % (method,) + ) + result = pytester.runpython(p) + result.stdout.fnmatch_lines( + """ + suspend, captured*hello1* + suspend2, captured*WARNING:root:hello3* + """ + ) + result.stderr.fnmatch_lines( + """ + WARNING:root:hello2 + """ + ) + assert "atexit" not in result.stderr.str() + + +def test_error_attribute_issue555(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import sys + def test_capattr(): + assert sys.stdout.errors == "replace" + assert sys.stderr.errors == "replace" + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +@pytest.mark.skipif( + not sys.platform.startswith("win"), + reason="only on windows", +) +def test_py36_windowsconsoleio_workaround_non_standard_streams() -> None: + """ + Ensure _py36_windowsconsoleio_workaround function works with objects that + do not implement the full ``io``-based stream protocol, for example execnet channels (#2666). + """ + from _pytest.capture import _py36_windowsconsoleio_workaround + + class DummyStream: + def write(self, s): + pass + + stream = cast(TextIO, DummyStream()) + _py36_windowsconsoleio_workaround(stream) + + +def test_dontreadfrominput_has_encoding(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import sys + def test_capattr(): + # should not raise AttributeError + assert sys.stdout.encoding + assert sys.stderr.encoding + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +def test_crash_on_closing_tmpfile_py27( + pytester: Pytester, monkeypatch: MonkeyPatch +) -> None: + p = pytester.makepyfile( + """ + import threading + import sys + + printing = threading.Event() + + def spam(): + f = sys.stderr + print('SPAMBEFORE', end='', file=f) + printing.set() + + while True: + try: + f.flush() + except (OSError, ValueError): + break + + def test_spam_in_thread(): + t = threading.Thread(target=spam) + t.daemon = True + t.start() + + printing.wait() + """ + ) + # Do not consider plugins like hypothesis, which might output to stderr. + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + result = pytester.runpytest_subprocess(str(p)) + assert result.ret == 0 + assert result.stderr.str() == "" + result.stdout.no_fnmatch_line("*OSError*") + + +def test_global_capture_with_live_logging(pytester: Pytester) -> None: + # Issue 3819 + # capture should work with live cli logging + + # Teardown report seems to have the capture for the whole process (setup, capture, teardown) + pytester.makeconftest( + """ + def pytest_runtest_logreport(report): + if "test_global" in report.nodeid: + if report.when == "teardown": + with open("caplog", "w") as f: + f.write(report.caplog) + with open("capstdout", "w") as f: + f.write(report.capstdout) + """ + ) + + pytester.makepyfile( + """ + import logging + import sys + import pytest + + logger = logging.getLogger(__name__) + + @pytest.fixture + def fix1(): + print("fix setup") + logging.info("fix setup") + yield + logging.info("fix teardown") + print("fix teardown") + + def test_global(fix1): + print("begin test") + logging.info("something in test") + print("end test") + """ + ) + result = pytester.runpytest_subprocess("--log-cli-level=INFO") + assert result.ret == 0 + + with open("caplog") as f: + caplog = f.read() + + assert "fix setup" in caplog + assert "something in test" in caplog + assert "fix teardown" in caplog + + with open("capstdout") as f: + capstdout = f.read() + + assert "fix setup" in capstdout + assert "begin test" in capstdout + assert "end test" in capstdout + assert "fix teardown" in capstdout + + +@pytest.mark.parametrize("capture_fixture", ["capsys", "capfd"]) +def test_capture_with_live_logging( + pytester: Pytester, capture_fixture: CaptureFixture[str] +) -> None: + # Issue 3819 + # capture should work with live cli logging + + pytester.makepyfile( + """ + import logging + import sys + + logger = logging.getLogger(__name__) + + def test_capture({0}): + print("hello") + sys.stderr.write("world\\n") + captured = {0}.readouterr() + assert captured.out == "hello\\n" + assert captured.err == "world\\n" + + logging.info("something") + print("next") + logging.info("something") + + captured = {0}.readouterr() + assert captured.out == "next\\n" + """.format( + capture_fixture + ) + ) + + result = pytester.runpytest_subprocess("--log-cli-level=INFO") + assert result.ret == 0 + + +def test_typeerror_encodedfile_write(pytester: Pytester) -> None: + """It should behave the same with and without output capturing (#4861).""" + p = pytester.makepyfile( + """ + def test_fails(): + import sys + sys.stdout.write(b"foo") + """ + ) + result_without_capture = pytester.runpytest("-s", str(p)) + result_with_capture = pytester.runpytest(str(p)) + + assert result_with_capture.ret == result_without_capture.ret + out = result_with_capture.stdout.str() + assert ("TypeError: write() argument must be str, not bytes" in out) or ( + "TypeError: unicode argument expected, got 'bytes'" in out + ) + + +def test_stderr_write_returns_len(capsys: CaptureFixture[str]) -> None: + """Write on Encoded files, namely captured stderr, should return number of characters written.""" + assert sys.stderr.write("Foo") == 3 + + +def test_encodedfile_writelines(tmpfile: BinaryIO) -> None: + ef = capture.EncodedFile(tmpfile, encoding="utf-8") + with pytest.raises(TypeError): + ef.writelines([b"line1", b"line2"]) # type: ignore[list-item] + assert ef.writelines(["line3", "line4"]) is None # type: ignore[func-returns-value] + ef.flush() + tmpfile.seek(0) + assert tmpfile.read() == b"line3line4" + tmpfile.close() + with pytest.raises(ValueError): + ef.read() + + +def test__get_multicapture() -> None: + assert isinstance(_get_multicapture("no"), MultiCapture) + pytest.raises(ValueError, _get_multicapture, "unknown").match( + r"^unknown capturing method: 'unknown'" + ) + + +def test_logging_while_collecting(pytester: Pytester) -> None: + """Issue #6240: Calls to logging.xxx() during collection causes all logging calls to be duplicated to stderr""" + p = pytester.makepyfile( + """\ + import logging + + logging.warning("during collection") + + def test_logging(): + logging.warning("during call") + assert False + """ + ) + result = pytester.runpytest_subprocess(p) + assert result.ret == ExitCode.TESTS_FAILED + result.stdout.fnmatch_lines( + [ + "*test_*.py F*", + "====* FAILURES *====", + "____*____", + "*--- Captured log call*", + "WARNING * during call", + "*1 failed*", + ] + ) + result.stdout.no_fnmatch_line("*Captured stderr call*") + result.stdout.no_fnmatch_line("*during collection*") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_collection.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_collection.py new file mode 100644 index 0000000000..6a8a5c1cef --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_collection.py @@ -0,0 +1,1506 @@ +import os +import pprint +import shutil +import sys +import textwrap +from pathlib import Path +from typing import List + +import pytest +from _pytest.config import ExitCode +from _pytest.fixtures import FixtureRequest +from _pytest.main import _in_venv +from _pytest.main import Session +from _pytest.monkeypatch import MonkeyPatch +from _pytest.nodes import Item +from _pytest.pathlib import symlink_or_skip +from _pytest.pytester import HookRecorder +from _pytest.pytester import Pytester + + +def ensure_file(file_path: Path) -> Path: + """Ensure that file exists""" + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.touch(exist_ok=True) + return file_path + + +class TestCollector: + def test_collect_versus_item(self) -> None: + from pytest import Collector + from pytest import Item + + assert not issubclass(Collector, Item) + assert not issubclass(Item, Collector) + + def test_check_equality(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol( + """ + def test_pass(): pass + def test_fail(): assert 0 + """ + ) + fn1 = pytester.collect_by_name(modcol, "test_pass") + assert isinstance(fn1, pytest.Function) + fn2 = pytester.collect_by_name(modcol, "test_pass") + assert isinstance(fn2, pytest.Function) + + assert fn1 == fn2 + assert fn1 != modcol + assert hash(fn1) == hash(fn2) + + fn3 = pytester.collect_by_name(modcol, "test_fail") + assert isinstance(fn3, pytest.Function) + assert not (fn1 == fn3) + assert fn1 != fn3 + + for fn in fn1, fn2, fn3: + assert isinstance(fn, pytest.Function) + assert fn != 3 # type: ignore[comparison-overlap] + assert fn != modcol + assert fn != [1, 2, 3] # type: ignore[comparison-overlap] + assert [1, 2, 3] != fn # type: ignore[comparison-overlap] + assert modcol != fn + + assert pytester.collect_by_name(modcol, "doesnotexist") is None + + def test_getparent_and_accessors(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol( + """ + class TestClass: + def test_foo(self): + pass + """ + ) + cls = pytester.collect_by_name(modcol, "TestClass") + assert isinstance(cls, pytest.Class) + fn = pytester.collect_by_name(cls, "test_foo") + assert isinstance(fn, pytest.Function) + + assert fn.getparent(pytest.Module) is modcol + assert modcol.module is not None + assert modcol.cls is None + assert modcol.instance is None + + assert fn.getparent(pytest.Class) is cls + assert cls.module is not None + assert cls.cls is not None + assert cls.instance is None + + assert fn.getparent(pytest.Function) is fn + assert fn.module is not None + assert fn.cls is not None + assert fn.instance is not None + assert fn.function is not None + + def test_getcustomfile_roundtrip(self, pytester: Pytester) -> None: + hello = pytester.makefile(".xxx", hello="world") + pytester.makepyfile( + conftest=""" + import pytest + class CustomFile(pytest.File): + pass + def pytest_collect_file(file_path, parent): + if file_path.suffix == ".xxx": + return CustomFile.from_parent(path=file_path, parent=parent) + """ + ) + node = pytester.getpathnode(hello) + assert isinstance(node, pytest.File) + assert node.name == "hello.xxx" + nodes = node.session.perform_collect([node.nodeid], genitems=False) + assert len(nodes) == 1 + assert isinstance(nodes[0], pytest.File) + + def test_can_skip_class_with_test_attr(self, pytester: Pytester) -> None: + """Assure test class is skipped when using `__test__=False` (See #2007).""" + pytester.makepyfile( + """ + class TestFoo(object): + __test__ = False + def __init__(self): + pass + def test_foo(): + assert True + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["collected 0 items", "*no tests ran in*"]) + + +class TestCollectFS: + def test_ignored_certain_directories(self, pytester: Pytester) -> None: + tmp_path = pytester.path + ensure_file(tmp_path / "build" / "test_notfound.py") + ensure_file(tmp_path / "dist" / "test_notfound.py") + ensure_file(tmp_path / "_darcs" / "test_notfound.py") + ensure_file(tmp_path / "CVS" / "test_notfound.py") + ensure_file(tmp_path / "{arch}" / "test_notfound.py") + ensure_file(tmp_path / ".whatever" / "test_notfound.py") + ensure_file(tmp_path / ".bzr" / "test_notfound.py") + ensure_file(tmp_path / "normal" / "test_found.py") + for x in tmp_path.rglob("test_*.py"): + x.write_text("def test_hello(): pass", "utf-8") + + result = pytester.runpytest("--collect-only") + s = result.stdout.str() + assert "test_notfound" not in s + assert "test_found" in s + + @pytest.mark.parametrize( + "fname", + ( + "activate", + "activate.csh", + "activate.fish", + "Activate", + "Activate.bat", + "Activate.ps1", + ), + ) + def test_ignored_virtualenvs(self, pytester: Pytester, fname: str) -> None: + bindir = "Scripts" if sys.platform.startswith("win") else "bin" + ensure_file(pytester.path / "virtual" / bindir / fname) + testfile = ensure_file(pytester.path / "virtual" / "test_invenv.py") + testfile.write_text("def test_hello(): pass") + + # by default, ignore tests inside a virtualenv + result = pytester.runpytest() + result.stdout.no_fnmatch_line("*test_invenv*") + # allow test collection if user insists + result = pytester.runpytest("--collect-in-virtualenv") + assert "test_invenv" in result.stdout.str() + # allow test collection if user directly passes in the directory + result = pytester.runpytest("virtual") + assert "test_invenv" in result.stdout.str() + + @pytest.mark.parametrize( + "fname", + ( + "activate", + "activate.csh", + "activate.fish", + "Activate", + "Activate.bat", + "Activate.ps1", + ), + ) + def test_ignored_virtualenvs_norecursedirs_precedence( + self, pytester: Pytester, fname: str + ) -> None: + bindir = "Scripts" if sys.platform.startswith("win") else "bin" + # norecursedirs takes priority + ensure_file(pytester.path / ".virtual" / bindir / fname) + testfile = ensure_file(pytester.path / ".virtual" / "test_invenv.py") + testfile.write_text("def test_hello(): pass") + result = pytester.runpytest("--collect-in-virtualenv") + result.stdout.no_fnmatch_line("*test_invenv*") + # ...unless the virtualenv is explicitly given on the CLI + result = pytester.runpytest("--collect-in-virtualenv", ".virtual") + assert "test_invenv" in result.stdout.str() + + @pytest.mark.parametrize( + "fname", + ( + "activate", + "activate.csh", + "activate.fish", + "Activate", + "Activate.bat", + "Activate.ps1", + ), + ) + def test__in_venv(self, pytester: Pytester, fname: str) -> None: + """Directly test the virtual env detection function""" + bindir = "Scripts" if sys.platform.startswith("win") else "bin" + # no bin/activate, not a virtualenv + base_path = pytester.mkdir("venv") + assert _in_venv(base_path) is False + # with bin/activate, totally a virtualenv + bin_path = base_path.joinpath(bindir) + bin_path.mkdir() + bin_path.joinpath(fname).touch() + assert _in_venv(base_path) is True + + def test_custom_norecursedirs(self, pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + norecursedirs = mydir xyz* + """ + ) + tmp_path = pytester.path + ensure_file(tmp_path / "mydir" / "test_hello.py").write_text( + "def test_1(): pass" + ) + ensure_file(tmp_path / "xyz123" / "test_2.py").write_text("def test_2(): 0/0") + ensure_file(tmp_path / "xy" / "test_ok.py").write_text("def test_3(): pass") + rec = pytester.inline_run() + rec.assertoutcome(passed=1) + rec = pytester.inline_run("xyz123/test_2.py") + rec.assertoutcome(failed=1) + + def test_testpaths_ini(self, pytester: Pytester, monkeypatch: MonkeyPatch) -> None: + pytester.makeini( + """ + [pytest] + testpaths = gui uts + """ + ) + tmp_path = pytester.path + ensure_file(tmp_path / "env" / "test_1.py").write_text("def test_env(): pass") + ensure_file(tmp_path / "gui" / "test_2.py").write_text("def test_gui(): pass") + ensure_file(tmp_path / "uts" / "test_3.py").write_text("def test_uts(): pass") + + # executing from rootdir only tests from `testpaths` directories + # are collected + items, reprec = pytester.inline_genitems("-v") + assert [x.name for x in items] == ["test_gui", "test_uts"] + + # check that explicitly passing directories in the command-line + # collects the tests + for dirname in ("env", "gui", "uts"): + items, reprec = pytester.inline_genitems(tmp_path.joinpath(dirname)) + assert [x.name for x in items] == ["test_%s" % dirname] + + # changing cwd to each subdirectory and running pytest without + # arguments collects the tests in that directory normally + for dirname in ("env", "gui", "uts"): + monkeypatch.chdir(pytester.path.joinpath(dirname)) + items, reprec = pytester.inline_genitems() + assert [x.name for x in items] == ["test_%s" % dirname] + + +class TestCollectPluginHookRelay: + def test_pytest_collect_file(self, pytester: Pytester) -> None: + wascalled = [] + + class Plugin: + def pytest_collect_file(self, file_path: Path) -> None: + if not file_path.name.startswith("."): + # Ignore hidden files, e.g. .testmondata. + wascalled.append(file_path) + + pytester.makefile(".abc", "xyz") + pytest.main(pytester.path, plugins=[Plugin()]) + assert len(wascalled) == 1 + assert wascalled[0].suffix == ".abc" + + +class TestPrunetraceback: + def test_custom_repr_failure(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import not_exists + """ + ) + pytester.makeconftest( + """ + import pytest + def pytest_collect_file(file_path, parent): + return MyFile.from_parent(path=file_path, parent=parent) + class MyError(Exception): + pass + class MyFile(pytest.File): + def collect(self): + raise MyError() + def repr_failure(self, excinfo): + if isinstance(excinfo.value, MyError): + return "hello world" + return pytest.File.repr_failure(self, excinfo) + """ + ) + + result = pytester.runpytest(p) + result.stdout.fnmatch_lines(["*ERROR collecting*", "*hello world*"]) + + @pytest.mark.xfail(reason="other mechanism for adding to reporting needed") + def test_collect_report_postprocessing(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import not_exists + """ + ) + pytester.makeconftest( + """ + import pytest + @pytest.hookimpl(hookwrapper=True) + def pytest_make_collect_report(): + outcome = yield + rep = outcome.get_result() + rep.headerlines += ["header1"] + outcome.force_result(rep) + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines(["*ERROR collecting*", "*header1*"]) + + +class TestCustomConftests: + def test_ignore_collect_path(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_ignore_collect(collection_path, config): + return collection_path.name.startswith("x") or collection_path.name == "test_one.py" + """ + ) + sub = pytester.mkdir("xy123") + ensure_file(sub / "test_hello.py").write_text("syntax error") + sub.joinpath("conftest.py").write_text("syntax error") + pytester.makepyfile("def test_hello(): pass") + pytester.makepyfile(test_one="syntax error") + result = pytester.runpytest("--fulltrace") + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_ignore_collect_not_called_on_argument(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_ignore_collect(collection_path, config): + return True + """ + ) + p = pytester.makepyfile("def test_hello(): pass") + result = pytester.runpytest(p) + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed*"]) + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + result.stdout.fnmatch_lines(["*collected 0 items*"]) + + def test_collectignore_exclude_on_option(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + from pathlib import Path + + class MyPathLike: + def __init__(self, path): + self.path = path + def __fspath__(self): + return "path" + + collect_ignore = [MyPathLike('hello'), 'test_world.py', Path('bye')] + + def pytest_addoption(parser): + parser.addoption("--XX", action="store_true", default=False) + + def pytest_configure(config): + if config.getvalue("XX"): + collect_ignore[:] = [] + """ + ) + pytester.mkdir("hello") + pytester.makepyfile(test_world="def test_hello(): pass") + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + result.stdout.no_fnmatch_line("*passed*") + result = pytester.runpytest("--XX") + assert result.ret == 0 + assert "passed" in result.stdout.str() + + def test_collectignoreglob_exclude_on_option(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + collect_ignore_glob = ['*w*l[dt]*'] + def pytest_addoption(parser): + parser.addoption("--XX", action="store_true", default=False) + def pytest_configure(config): + if config.getvalue("XX"): + collect_ignore_glob[:] = [] + """ + ) + pytester.makepyfile(test_world="def test_hello(): pass") + pytester.makepyfile(test_welt="def test_hallo(): pass") + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + result.stdout.fnmatch_lines(["*collected 0 items*"]) + result = pytester.runpytest("--XX") + assert result.ret == 0 + result.stdout.fnmatch_lines(["*2 passed*"]) + + def test_pytest_fs_collect_hooks_are_seen(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + class MyModule(pytest.Module): + pass + def pytest_collect_file(file_path, parent): + if file_path.suffix == ".py": + return MyModule.from_parent(path=file_path, parent=parent) + """ + ) + pytester.mkdir("sub") + pytester.makepyfile("def test_x(): pass") + result = pytester.runpytest("--co") + result.stdout.fnmatch_lines(["*MyModule*", "*test_x*"]) + + def test_pytest_collect_file_from_sister_dir(self, pytester: Pytester) -> None: + sub1 = pytester.mkpydir("sub1") + sub2 = pytester.mkpydir("sub2") + conf1 = pytester.makeconftest( + """ + import pytest + class MyModule1(pytest.Module): + pass + def pytest_collect_file(file_path, parent): + if file_path.suffix == ".py": + return MyModule1.from_parent(path=file_path, parent=parent) + """ + ) + conf1.replace(sub1.joinpath(conf1.name)) + conf2 = pytester.makeconftest( + """ + import pytest + class MyModule2(pytest.Module): + pass + def pytest_collect_file(file_path, parent): + if file_path.suffix == ".py": + return MyModule2.from_parent(path=file_path, parent=parent) + """ + ) + conf2.replace(sub2.joinpath(conf2.name)) + p = pytester.makepyfile("def test_x(): pass") + shutil.copy(p, sub1.joinpath(p.name)) + shutil.copy(p, sub2.joinpath(p.name)) + result = pytester.runpytest("--co") + result.stdout.fnmatch_lines(["*MyModule1*", "*MyModule2*", "*test_x*"]) + + +class TestSession: + def test_collect_topdir(self, pytester: Pytester) -> None: + p = pytester.makepyfile("def test_func(): pass") + id = "::".join([p.name, "test_func"]) + # XXX migrate to collectonly? (see below) + config = pytester.parseconfig(id) + topdir = pytester.path + rcol = Session.from_config(config) + assert topdir == rcol.path + # rootid = rcol.nodeid + # root2 = rcol.perform_collect([rcol.nodeid], genitems=False)[0] + # assert root2 == rcol, rootid + colitems = rcol.perform_collect([rcol.nodeid], genitems=False) + assert len(colitems) == 1 + assert colitems[0].path == p + + def get_reported_items(self, hookrec: HookRecorder) -> List[Item]: + """Return pytest.Item instances reported by the pytest_collectreport hook""" + calls = hookrec.getcalls("pytest_collectreport") + return [ + x + for call in calls + for x in call.report.result + if isinstance(x, pytest.Item) + ] + + def test_collect_protocol_single_function(self, pytester: Pytester) -> None: + p = pytester.makepyfile("def test_func(): pass") + id = "::".join([p.name, "test_func"]) + items, hookrec = pytester.inline_genitems(id) + (item,) = items + assert item.name == "test_func" + newid = item.nodeid + assert newid == id + pprint.pprint(hookrec.calls) + topdir = pytester.path # noqa + hookrec.assert_contains( + [ + ("pytest_collectstart", "collector.path == topdir"), + ("pytest_make_collect_report", "collector.path == topdir"), + ("pytest_collectstart", "collector.path == p"), + ("pytest_make_collect_report", "collector.path == p"), + ("pytest_pycollect_makeitem", "name == 'test_func'"), + ("pytest_collectreport", "report.result[0].name == 'test_func'"), + ] + ) + # ensure we are reporting the collection of the single test item (#2464) + assert [x.name for x in self.get_reported_items(hookrec)] == ["test_func"] + + def test_collect_protocol_method(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + class TestClass(object): + def test_method(self): + pass + """ + ) + normid = p.name + "::TestClass::test_method" + for id in [p.name, p.name + "::TestClass", normid]: + items, hookrec = pytester.inline_genitems(id) + assert len(items) == 1 + assert items[0].name == "test_method" + newid = items[0].nodeid + assert newid == normid + # ensure we are reporting the collection of the single test item (#2464) + assert [x.name for x in self.get_reported_items(hookrec)] == ["test_method"] + + def test_collect_custom_nodes_multi_id(self, pytester: Pytester) -> None: + p = pytester.makepyfile("def test_func(): pass") + pytester.makeconftest( + """ + import pytest + class SpecialItem(pytest.Item): + def runtest(self): + return # ok + class SpecialFile(pytest.File): + def collect(self): + return [SpecialItem.from_parent(name="check", parent=self)] + def pytest_collect_file(file_path, parent): + if file_path.name == %r: + return SpecialFile.from_parent(path=file_path, parent=parent) + """ + % p.name + ) + id = p.name + + items, hookrec = pytester.inline_genitems(id) + pprint.pprint(hookrec.calls) + assert len(items) == 2 + hookrec.assert_contains( + [ + ("pytest_collectstart", "collector.path == collector.session.path"), + ( + "pytest_collectstart", + "collector.__class__.__name__ == 'SpecialFile'", + ), + ("pytest_collectstart", "collector.__class__.__name__ == 'Module'"), + ("pytest_pycollect_makeitem", "name == 'test_func'"), + ("pytest_collectreport", "report.nodeid.startswith(p.name)"), + ] + ) + assert len(self.get_reported_items(hookrec)) == 2 + + def test_collect_subdir_event_ordering(self, pytester: Pytester) -> None: + p = pytester.makepyfile("def test_func(): pass") + aaa = pytester.mkpydir("aaa") + test_aaa = aaa.joinpath("test_aaa.py") + p.replace(test_aaa) + + items, hookrec = pytester.inline_genitems() + assert len(items) == 1 + pprint.pprint(hookrec.calls) + hookrec.assert_contains( + [ + ("pytest_collectstart", "collector.path == test_aaa"), + ("pytest_pycollect_makeitem", "name == 'test_func'"), + ("pytest_collectreport", "report.nodeid.startswith('aaa/test_aaa.py')"), + ] + ) + + def test_collect_two_commandline_args(self, pytester: Pytester) -> None: + p = pytester.makepyfile("def test_func(): pass") + aaa = pytester.mkpydir("aaa") + bbb = pytester.mkpydir("bbb") + test_aaa = aaa.joinpath("test_aaa.py") + shutil.copy(p, test_aaa) + test_bbb = bbb.joinpath("test_bbb.py") + p.replace(test_bbb) + + id = "." + + items, hookrec = pytester.inline_genitems(id) + assert len(items) == 2 + pprint.pprint(hookrec.calls) + hookrec.assert_contains( + [ + ("pytest_collectstart", "collector.path == test_aaa"), + ("pytest_pycollect_makeitem", "name == 'test_func'"), + ("pytest_collectreport", "report.nodeid == 'aaa/test_aaa.py'"), + ("pytest_collectstart", "collector.path == test_bbb"), + ("pytest_pycollect_makeitem", "name == 'test_func'"), + ("pytest_collectreport", "report.nodeid == 'bbb/test_bbb.py'"), + ] + ) + + def test_serialization_byid(self, pytester: Pytester) -> None: + pytester.makepyfile("def test_func(): pass") + items, hookrec = pytester.inline_genitems() + assert len(items) == 1 + (item,) = items + items2, hookrec = pytester.inline_genitems(item.nodeid) + (item2,) = items2 + assert item2.name == item.name + assert item2.path == item.path + + def test_find_byid_without_instance_parents(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + class TestClass(object): + def test_method(self): + pass + """ + ) + arg = p.name + "::TestClass::test_method" + items, hookrec = pytester.inline_genitems(arg) + assert len(items) == 1 + (item,) = items + assert item.nodeid.endswith("TestClass::test_method") + # ensure we are reporting the collection of the single test item (#2464) + assert [x.name for x in self.get_reported_items(hookrec)] == ["test_method"] + + +class Test_getinitialnodes: + def test_global_file(self, pytester: Pytester) -> None: + tmp_path = pytester.path + x = ensure_file(tmp_path / "x.py") + config = pytester.parseconfigure(x) + col = pytester.getnode(config, x) + assert isinstance(col, pytest.Module) + assert col.name == "x.py" + assert col.parent is not None + assert col.parent.parent is None + for parent in col.listchain(): + assert parent.config is config + + def test_pkgfile(self, pytester: Pytester) -> None: + """Verify nesting when a module is within a package. + The parent chain should match: Module -> Package -> Session. + Session's parent should always be None. + """ + tmp_path = pytester.path + subdir = tmp_path.joinpath("subdir") + x = ensure_file(subdir / "x.py") + ensure_file(subdir / "__init__.py") + with subdir.cwd(): + config = pytester.parseconfigure(x) + col = pytester.getnode(config, x) + assert col is not None + assert col.name == "x.py" + assert isinstance(col, pytest.Module) + assert isinstance(col.parent, pytest.Package) + assert isinstance(col.parent.parent, pytest.Session) + # session is batman (has no parents) + assert col.parent.parent.parent is None + for parent in col.listchain(): + assert parent.config is config + + +class Test_genitems: + def test_check_collect_hashes(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def test_1(): + pass + + def test_2(): + pass + """ + ) + shutil.copy(p, p.parent / (p.stem + "2" + ".py")) + items, reprec = pytester.inline_genitems(p.parent) + assert len(items) == 4 + for numi, i in enumerate(items): + for numj, j in enumerate(items): + if numj != numi: + assert hash(i) != hash(j) + assert i != j + + def test_example_items1(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + def testone(): + pass + + class TestX(object): + def testmethod_one(self): + pass + + class TestY(TestX): + @pytest.mark.parametrize("arg0", [".["]) + def testmethod_two(self, arg0): + pass + """ + ) + items, reprec = pytester.inline_genitems(p) + assert len(items) == 4 + assert items[0].name == "testone" + assert items[1].name == "testmethod_one" + assert items[2].name == "testmethod_one" + assert items[3].name == "testmethod_two[.[]" + + # let's also test getmodpath here + assert items[0].getmodpath() == "testone" # type: ignore[attr-defined] + assert items[1].getmodpath() == "TestX.testmethod_one" # type: ignore[attr-defined] + assert items[2].getmodpath() == "TestY.testmethod_one" # type: ignore[attr-defined] + # PR #6202: Fix incorrect result of getmodpath method. (Resolves issue #6189) + assert items[3].getmodpath() == "TestY.testmethod_two[.[]" # type: ignore[attr-defined] + + s = items[0].getmodpath(stopatmodule=False) # type: ignore[attr-defined] + assert s.endswith("test_example_items1.testone") + print(s) + + def test_class_and_functions_discovery_using_glob(self, pytester: Pytester) -> None: + """Test that Python_classes and Python_functions config options work + as prefixes and glob-like patterns (#600).""" + pytester.makeini( + """ + [pytest] + python_classes = *Suite Test + python_functions = *_test test + """ + ) + p = pytester.makepyfile( + """ + class MyTestSuite(object): + def x_test(self): + pass + + class TestCase(object): + def test_y(self): + pass + """ + ) + items, reprec = pytester.inline_genitems(p) + ids = [x.getmodpath() for x in items] # type: ignore[attr-defined] + assert ids == ["MyTestSuite.x_test", "TestCase.test_y"] + + +def test_matchnodes_two_collections_same_file(pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + def pytest_configure(config): + config.pluginmanager.register(Plugin2()) + + class Plugin2(object): + def pytest_collect_file(self, file_path, parent): + if file_path.suffix == ".abc": + return MyFile2.from_parent(path=file_path, parent=parent) + + def pytest_collect_file(file_path, parent): + if file_path.suffix == ".abc": + return MyFile1.from_parent(path=file_path, parent=parent) + + class MyFile1(pytest.File): + def collect(self): + yield Item1.from_parent(name="item1", parent=self) + + class MyFile2(pytest.File): + def collect(self): + yield Item2.from_parent(name="item2", parent=self) + + class Item1(pytest.Item): + def runtest(self): + pass + + class Item2(pytest.Item): + def runtest(self): + pass + """ + ) + p = pytester.makefile(".abc", "") + result = pytester.runpytest() + assert result.ret == 0 + result.stdout.fnmatch_lines(["*2 passed*"]) + res = pytester.runpytest("%s::item2" % p.name) + res.stdout.fnmatch_lines(["*1 passed*"]) + + +class TestNodeKeywords: + def test_no_under(self, pytester: Pytester) -> None: + modcol = pytester.getmodulecol( + """ + def test_pass(): pass + def test_fail(): assert 0 + """ + ) + values = list(modcol.keywords) + assert modcol.name in values + for x in values: + assert not x.startswith("_") + assert modcol.name in repr(modcol.keywords) + + def test_issue345(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_should_not_be_selected(): + assert False, 'I should not have been selected to run' + + def test___repr__(): + pass + """ + ) + reprec = pytester.inline_run("-k repr") + reprec.assertoutcome(passed=1, failed=0) + + def test_keyword_matching_is_case_insensitive_by_default( + self, pytester: Pytester + ) -> None: + """Check that selection via -k EXPRESSION is case-insensitive. + + Since markers are also added to the node keywords, they too can + be matched without having to think about case sensitivity. + + """ + pytester.makepyfile( + """ + import pytest + + def test_sPeCiFiCToPiC_1(): + assert True + + class TestSpecificTopic_2: + def test(self): + assert True + + @pytest.mark.sPeCiFiCToPic_3 + def test(): + assert True + + @pytest.mark.sPeCiFiCToPic_4 + class Test: + def test(self): + assert True + + def test_failing_5(): + assert False, "This should not match" + + """ + ) + num_matching_tests = 4 + for expression in ("specifictopic", "SPECIFICTOPIC", "SpecificTopic"): + reprec = pytester.inline_run("-k " + expression) + reprec.assertoutcome(passed=num_matching_tests, failed=0) + + def test_duplicates_handled_correctly(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + pytestmark = pytest.mark.kw + class TestClass: + pytestmark = pytest.mark.kw + def test_method(self): pass + test_method.kw = 'method' + """, + "test_method", + ) + assert item.parent is not None and item.parent.parent is not None + item.parent.parent.keywords["kw"] = "class" + + assert item.keywords["kw"] == "method" + assert len(item.keywords) == len(set(item.keywords)) + + +COLLECTION_ERROR_PY_FILES = dict( + test_01_failure=""" + def test_1(): + assert False + """, + test_02_import_error=""" + import asdfasdfasdf + def test_2(): + assert True + """, + test_03_import_error=""" + import asdfasdfasdf + def test_3(): + assert True + """, + test_04_success=""" + def test_4(): + assert True + """, +) + + +def test_exit_on_collection_error(pytester: Pytester) -> None: + """Verify that all collection errors are collected and no tests executed""" + pytester.makepyfile(**COLLECTION_ERROR_PY_FILES) + + res = pytester.runpytest() + assert res.ret == 2 + + res.stdout.fnmatch_lines( + [ + "collected 2 items / 2 errors", + "*ERROR collecting test_02_import_error.py*", + "*No module named *asdfa*", + "*ERROR collecting test_03_import_error.py*", + "*No module named *asdfa*", + ] + ) + + +def test_exit_on_collection_with_maxfail_smaller_than_n_errors( + pytester: Pytester, +) -> None: + """ + Verify collection is aborted once maxfail errors are encountered ignoring + further modules which would cause more collection errors. + """ + pytester.makepyfile(**COLLECTION_ERROR_PY_FILES) + + res = pytester.runpytest("--maxfail=1") + assert res.ret == 1 + res.stdout.fnmatch_lines( + [ + "collected 1 item / 1 error", + "*ERROR collecting test_02_import_error.py*", + "*No module named *asdfa*", + "*! stopping after 1 failures !*", + "*= 1 error in *", + ] + ) + res.stdout.no_fnmatch_line("*test_03*") + + +def test_exit_on_collection_with_maxfail_bigger_than_n_errors( + pytester: Pytester, +) -> None: + """ + Verify the test run aborts due to collection errors even if maxfail count of + errors was not reached. + """ + pytester.makepyfile(**COLLECTION_ERROR_PY_FILES) + + res = pytester.runpytest("--maxfail=4") + assert res.ret == 2 + res.stdout.fnmatch_lines( + [ + "collected 2 items / 2 errors", + "*ERROR collecting test_02_import_error.py*", + "*No module named *asdfa*", + "*ERROR collecting test_03_import_error.py*", + "*No module named *asdfa*", + "*! Interrupted: 2 errors during collection !*", + "*= 2 errors in *", + ] + ) + + +def test_continue_on_collection_errors(pytester: Pytester) -> None: + """ + Verify tests are executed even when collection errors occur when the + --continue-on-collection-errors flag is set + """ + pytester.makepyfile(**COLLECTION_ERROR_PY_FILES) + + res = pytester.runpytest("--continue-on-collection-errors") + assert res.ret == 1 + + res.stdout.fnmatch_lines( + ["collected 2 items / 2 errors", "*1 failed, 1 passed, 2 errors*"] + ) + + +def test_continue_on_collection_errors_maxfail(pytester: Pytester) -> None: + """ + Verify tests are executed even when collection errors occur and that maxfail + is honoured (including the collection error count). + 4 tests: 2 collection errors + 1 failure + 1 success + test_4 is never executed because the test run is with --maxfail=3 which + means it is interrupted after the 2 collection errors + 1 failure. + """ + pytester.makepyfile(**COLLECTION_ERROR_PY_FILES) + + res = pytester.runpytest("--continue-on-collection-errors", "--maxfail=3") + assert res.ret == 1 + + res.stdout.fnmatch_lines(["collected 2 items / 2 errors", "*1 failed, 2 errors*"]) + + +def test_fixture_scope_sibling_conftests(pytester: Pytester) -> None: + """Regression test case for https://github.com/pytest-dev/pytest/issues/2836""" + foo_path = pytester.mkdir("foo") + foo_path.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + @pytest.fixture + def fix(): + return 1 + """ + ) + ) + foo_path.joinpath("test_foo.py").write_text("def test_foo(fix): assert fix == 1") + + # Tests in `food/` should not see the conftest fixture from `foo/` + food_path = pytester.mkpydir("food") + food_path.joinpath("test_food.py").write_text("def test_food(fix): assert fix == 1") + + res = pytester.runpytest() + assert res.ret == 1 + + res.stdout.fnmatch_lines( + [ + "*ERROR at setup of test_food*", + "E*fixture 'fix' not found", + "*1 passed, 1 error*", + ] + ) + + +def test_collect_init_tests(pytester: Pytester) -> None: + """Check that we collect files from __init__.py files when they patch the 'python_files' (#3773)""" + p = pytester.copy_example("collect/collect_init_tests") + result = pytester.runpytest(p, "--collect-only") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "", + " ", + " ", + " ", + " ", + ] + ) + result = pytester.runpytest("./tests", "--collect-only") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "", + " ", + " ", + " ", + " ", + ] + ) + # Ignores duplicates with "." and pkginit (#4310). + result = pytester.runpytest("./tests", ".", "--collect-only") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "", + " ", + " ", + " ", + " ", + ] + ) + # Same as before, but different order. + result = pytester.runpytest(".", "tests", "--collect-only") + result.stdout.fnmatch_lines( + [ + "collected 2 items", + "", + " ", + " ", + " ", + " ", + ] + ) + result = pytester.runpytest("./tests/test_foo.py", "--collect-only") + result.stdout.fnmatch_lines( + ["", " ", " "] + ) + result.stdout.no_fnmatch_line("*test_init*") + result = pytester.runpytest("./tests/__init__.py", "--collect-only") + result.stdout.fnmatch_lines( + ["", " ", " "] + ) + result.stdout.no_fnmatch_line("*test_foo*") + + +def test_collect_invalid_signature_message(pytester: Pytester) -> None: + """Check that we issue a proper message when we can't determine the signature of a test + function (#4026). + """ + pytester.makepyfile( + """ + import pytest + + class TestCase: + @pytest.fixture + def fix(): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + ["Could not determine arguments of *.fix *: invalid method signature"] + ) + + +def test_collect_handles_raising_on_dunder_class(pytester: Pytester) -> None: + """Handle proxy classes like Django's LazySettings that might raise on + ``isinstance`` (#4266). + """ + pytester.makepyfile( + """ + class ImproperlyConfigured(Exception): + pass + + class RaisesOnGetAttr(object): + def raises(self): + raise ImproperlyConfigured + + __class__ = property(raises) + + raises = RaisesOnGetAttr() + + + def test_1(): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed in*"]) + assert result.ret == 0 + + +def test_collect_with_chdir_during_import(pytester: Pytester) -> None: + subdir = pytester.mkdir("sub") + pytester.path.joinpath("conftest.py").write_text( + textwrap.dedent( + """ + import os + os.chdir(%r) + """ + % (str(subdir),) + ) + ) + pytester.makepyfile( + """ + def test_1(): + import os + assert os.getcwd() == %r + """ + % (str(subdir),) + ) + with pytester.path.cwd(): + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed in*"]) + assert result.ret == 0 + + # Handles relative testpaths. + pytester.makeini( + """ + [pytest] + testpaths = . + """ + ) + with pytester.path.cwd(): + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines(["collected 1 item"]) + + +def test_collect_pyargs_with_testpaths( + pytester: Pytester, monkeypatch: MonkeyPatch +) -> None: + testmod = pytester.mkdir("testmod") + # NOTE: __init__.py is not collected since it does not match python_files. + testmod.joinpath("__init__.py").write_text("def test_func(): pass") + testmod.joinpath("test_file.py").write_text("def test_func(): pass") + + root = pytester.mkdir("root") + root.joinpath("pytest.ini").write_text( + textwrap.dedent( + """ + [pytest] + addopts = --pyargs + testpaths = testmod + """ + ) + ) + monkeypatch.setenv("PYTHONPATH", str(pytester.path), prepend=os.pathsep) + with root.cwd(): + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines(["*1 passed in*"]) + + +def test_collect_symlink_file_arg(pytester: Pytester) -> None: + """Collect a direct symlink works even if it does not match python_files (#4325).""" + real = pytester.makepyfile( + real=""" + def test_nodeid(request): + assert request.node.nodeid == "symlink.py::test_nodeid" + """ + ) + symlink = pytester.path.joinpath("symlink.py") + symlink_or_skip(real, symlink) + result = pytester.runpytest("-v", symlink) + result.stdout.fnmatch_lines(["symlink.py::test_nodeid PASSED*", "*1 passed in*"]) + assert result.ret == 0 + + +def test_collect_symlink_out_of_tree(pytester: Pytester) -> None: + """Test collection of symlink via out-of-tree rootdir.""" + sub = pytester.mkdir("sub") + real = sub.joinpath("test_real.py") + real.write_text( + textwrap.dedent( + """ + def test_nodeid(request): + # Should not contain sub/ prefix. + assert request.node.nodeid == "test_real.py::test_nodeid" + """ + ), + ) + + out_of_tree = pytester.mkdir("out_of_tree") + symlink_to_sub = out_of_tree.joinpath("symlink_to_sub") + symlink_or_skip(sub, symlink_to_sub) + os.chdir(sub) + result = pytester.runpytest("-vs", "--rootdir=%s" % sub, symlink_to_sub) + result.stdout.fnmatch_lines( + [ + # Should not contain "sub/"! + "test_real.py::test_nodeid PASSED" + ] + ) + assert result.ret == 0 + + +def test_collect_symlink_dir(pytester: Pytester) -> None: + """A symlinked directory is collected.""" + dir = pytester.mkdir("dir") + dir.joinpath("test_it.py").write_text("def test_it(): pass", "utf-8") + symlink_or_skip(pytester.path.joinpath("symlink_dir"), dir) + result = pytester.runpytest() + result.assert_outcomes(passed=2) + + +def test_collectignore_via_conftest(pytester: Pytester) -> None: + """collect_ignore in parent conftest skips importing child (issue #4592).""" + tests = pytester.mkpydir("tests") + tests.joinpath("conftest.py").write_text("collect_ignore = ['ignore_me']") + + ignore_me = tests.joinpath("ignore_me") + ignore_me.mkdir() + ignore_me.joinpath("__init__.py").touch() + ignore_me.joinpath("conftest.py").write_text("assert 0, 'should_not_be_called'") + + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +def test_collect_pkg_init_and_file_in_args(pytester: Pytester) -> None: + subdir = pytester.mkdir("sub") + init = subdir.joinpath("__init__.py") + init.write_text("def test_init(): pass") + p = subdir.joinpath("test_file.py") + p.write_text("def test_file(): pass") + + # NOTE: without "-o python_files=*.py" this collects test_file.py twice. + # This changed/broke with "Add package scoped fixtures #2283" (2b1410895) + # initially (causing a RecursionError). + result = pytester.runpytest("-v", str(init), str(p)) + result.stdout.fnmatch_lines( + [ + "sub/test_file.py::test_file PASSED*", + "sub/test_file.py::test_file PASSED*", + "*2 passed in*", + ] + ) + + result = pytester.runpytest("-v", "-o", "python_files=*.py", str(init), str(p)) + result.stdout.fnmatch_lines( + [ + "sub/__init__.py::test_init PASSED*", + "sub/test_file.py::test_file PASSED*", + "*2 passed in*", + ] + ) + + +def test_collect_pkg_init_only(pytester: Pytester) -> None: + subdir = pytester.mkdir("sub") + init = subdir.joinpath("__init__.py") + init.write_text("def test_init(): pass") + + result = pytester.runpytest(str(init)) + result.stdout.fnmatch_lines(["*no tests ran in*"]) + + result = pytester.runpytest("-v", "-o", "python_files=*.py", str(init)) + result.stdout.fnmatch_lines(["sub/__init__.py::test_init PASSED*", "*1 passed in*"]) + + +@pytest.mark.parametrize("use_pkg", (True, False)) +def test_collect_sub_with_symlinks(use_pkg: bool, pytester: Pytester) -> None: + """Collection works with symlinked files and broken symlinks""" + sub = pytester.mkdir("sub") + if use_pkg: + sub.joinpath("__init__.py").touch() + sub.joinpath("test_file.py").write_text("def test_file(): pass") + + # Create a broken symlink. + symlink_or_skip("test_doesnotexist.py", sub.joinpath("test_broken.py")) + + # Symlink that gets collected. + symlink_or_skip("test_file.py", sub.joinpath("test_symlink.py")) + + result = pytester.runpytest("-v", str(sub)) + result.stdout.fnmatch_lines( + [ + "sub/test_file.py::test_file PASSED*", + "sub/test_symlink.py::test_file PASSED*", + "*2 passed in*", + ] + ) + + +def test_collector_respects_tbstyle(pytester: Pytester) -> None: + p1 = pytester.makepyfile("assert 0") + result = pytester.runpytest(p1, "--tb=native") + assert result.ret == ExitCode.INTERRUPTED + result.stdout.fnmatch_lines( + [ + "*_ ERROR collecting test_collector_respects_tbstyle.py _*", + "Traceback (most recent call last):", + ' File "*/test_collector_respects_tbstyle.py", line 1, in ', + " assert 0", + "AssertionError: assert 0", + "*! Interrupted: 1 error during collection !*", + "*= 1 error in *", + ] + ) + + +def test_does_not_eagerly_collect_packages(pytester: Pytester) -> None: + pytester.makepyfile("def test(): pass") + pydir = pytester.mkpydir("foopkg") + pydir.joinpath("__init__.py").write_text("assert False") + result = pytester.runpytest() + assert result.ret == ExitCode.OK + + +def test_does_not_put_src_on_path(pytester: Pytester) -> None: + # `src` is not on sys.path so it should not be importable + ensure_file(pytester.path / "src/nope/__init__.py") + pytester.makepyfile( + "import pytest\n" + "def test():\n" + " with pytest.raises(ImportError):\n" + " import nope\n" + ) + result = pytester.runpytest() + assert result.ret == ExitCode.OK + + +def test_fscollector_from_parent(pytester: Pytester, request: FixtureRequest) -> None: + """Ensure File.from_parent can forward custom arguments to the constructor. + + Context: https://github.com/pytest-dev/pytest-cpp/pull/47 + """ + + class MyCollector(pytest.File): + def __init__(self, *k, x, **kw): + super().__init__(*k, **kw) + self.x = x + + collector = MyCollector.from_parent( + parent=request.session, path=pytester.path / "foo", x=10 + ) + assert collector.x == 10 + + +def test_class_from_parent(pytester: Pytester, request: FixtureRequest) -> None: + """Ensure Class.from_parent can forward custom arguments to the constructor.""" + + class MyCollector(pytest.Class): + def __init__(self, name, parent, x): + super().__init__(name, parent) + self.x = x + + @classmethod + def from_parent(cls, parent, *, name, x): + return super().from_parent(parent=parent, name=name, x=x) + + collector = MyCollector.from_parent(parent=request.session, name="foo", x=10) + assert collector.x == 10 + + +class TestImportModeImportlib: + def test_collect_duplicate_names(self, pytester: Pytester) -> None: + """--import-mode=importlib can import modules with same names that are not in packages.""" + pytester.makepyfile( + **{ + "tests_a/test_foo.py": "def test_foo1(): pass", + "tests_b/test_foo.py": "def test_foo2(): pass", + } + ) + result = pytester.runpytest("-v", "--import-mode=importlib") + result.stdout.fnmatch_lines( + [ + "tests_a/test_foo.py::test_foo1 *", + "tests_b/test_foo.py::test_foo2 *", + "* 2 passed in *", + ] + ) + + def test_conftest(self, pytester: Pytester) -> None: + """Directory containing conftest modules are not put in sys.path as a side-effect of + importing them.""" + tests_dir = pytester.path.joinpath("tests") + pytester.makepyfile( + **{ + "tests/conftest.py": "", + "tests/test_foo.py": """ + import sys + def test_check(): + assert r"{tests_dir}" not in sys.path + """.format( + tests_dir=tests_dir + ), + } + ) + result = pytester.runpytest("-v", "--import-mode=importlib") + result.stdout.fnmatch_lines(["* 1 passed in *"]) + + def setup_conftest_and_foo(self, pytester: Pytester) -> None: + """Setup a tests folder to be used to test if modules in that folder can be imported + due to side-effects of --import-mode or not.""" + pytester.makepyfile( + **{ + "tests/conftest.py": "", + "tests/foo.py": """ + def foo(): return 42 + """, + "tests/test_foo.py": """ + def test_check(): + from foo import foo + assert foo() == 42 + """, + } + ) + + def test_modules_importable_as_side_effect(self, pytester: Pytester) -> None: + """In import-modes `prepend` and `append`, we are able to import modules from folders + containing conftest.py files due to the side effect of changing sys.path.""" + self.setup_conftest_and_foo(pytester) + result = pytester.runpytest("-v", "--import-mode=prepend") + result.stdout.fnmatch_lines(["* 1 passed in *"]) + + def test_modules_not_importable_as_side_effect(self, pytester: Pytester) -> None: + """In import-mode `importlib`, modules in folders containing conftest.py are not + importable, as don't change sys.path or sys.modules as side effect of importing + the conftest.py file. + """ + self.setup_conftest_and_foo(pytester) + result = pytester.runpytest("-v", "--import-mode=importlib") + result.stdout.fnmatch_lines( + [ + "*ModuleNotFoundError: No module named 'foo'", + "tests?test_foo.py:2: ModuleNotFoundError", + "* 1 failed in *", + ] + ) + + +def test_does_not_crash_on_error_from_decorated_function(pytester: Pytester) -> None: + """Regression test for an issue around bad exception formatting due to + assertion rewriting mangling lineno's (#4984).""" + pytester.makepyfile( + """ + @pytest.fixture + def a(): return 4 + """ + ) + result = pytester.runpytest() + # Not INTERNAL_ERROR + assert result.ret == ExitCode.INTERRUPTED + + +def test_does_not_crash_on_recursive_symlink(pytester: Pytester) -> None: + """Regression test for an issue around recursive symlinks (#7951).""" + symlink_or_skip("recursive", pytester.path.joinpath("recursive")) + pytester.makepyfile( + """ + def test_foo(): assert True + """ + ) + result = pytester.runpytest() + + assert result.ret == ExitCode.OK + assert result.parseoutcomes() == {"passed": 1} diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_compat.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_compat.py new file mode 100644 index 0000000000..37cf4a077d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_compat.py @@ -0,0 +1,265 @@ +import enum +import sys +from functools import partial +from functools import wraps +from typing import TYPE_CHECKING +from typing import Union + +import pytest +from _pytest.compat import _PytestWrapper +from _pytest.compat import assert_never +from _pytest.compat import cached_property +from _pytest.compat import get_real_func +from _pytest.compat import is_generator +from _pytest.compat import safe_getattr +from _pytest.compat import safe_isclass +from _pytest.outcomes import OutcomeException +from _pytest.pytester import Pytester + +if TYPE_CHECKING: + from typing_extensions import Literal + + +def test_is_generator() -> None: + def zap(): + yield # pragma: no cover + + def foo(): + pass # pragma: no cover + + assert is_generator(zap) + assert not is_generator(foo) + + +def test_real_func_loop_limit() -> None: + class Evil: + def __init__(self): + self.left = 1000 + + def __repr__(self): + return f"" + + def __getattr__(self, attr): + if not self.left: + raise RuntimeError("it's over") # pragma: no cover + self.left -= 1 + return self + + evil = Evil() + + with pytest.raises( + ValueError, + match=( + "could not find real function of \n" + "stopped at " + ), + ): + get_real_func(evil) + + +def test_get_real_func() -> None: + """Check that get_real_func correctly unwraps decorators until reaching the real function""" + + def decorator(f): + @wraps(f) + def inner(): + pass # pragma: no cover + + return inner + + def func(): + pass # pragma: no cover + + wrapped_func = decorator(decorator(func)) + assert get_real_func(wrapped_func) is func + + wrapped_func2 = decorator(decorator(wrapped_func)) + assert get_real_func(wrapped_func2) is func + + # special case for __pytest_wrapped__ attribute: used to obtain the function up until the point + # a function was wrapped by pytest itself + wrapped_func2.__pytest_wrapped__ = _PytestWrapper(wrapped_func) + assert get_real_func(wrapped_func2) is wrapped_func + + +def test_get_real_func_partial() -> None: + """Test get_real_func handles partial instances correctly""" + + def foo(x): + return x + + assert get_real_func(foo) is foo + assert get_real_func(partial(foo)) is foo + + +@pytest.mark.skipif(sys.version_info >= (3, 11), reason="couroutine removed") +def test_is_generator_asyncio(pytester: Pytester) -> None: + pytester.makepyfile( + """ + from _pytest.compat import is_generator + import asyncio + @asyncio.coroutine + def baz(): + yield from [1,2,3] + + def test_is_generator_asyncio(): + assert not is_generator(baz) + """ + ) + # avoid importing asyncio into pytest's own process, + # which in turn imports logging (#8) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_is_generator_async_syntax(pytester: Pytester) -> None: + pytester.makepyfile( + """ + from _pytest.compat import is_generator + def test_is_generator_py35(): + async def foo(): + await foo() + + async def bar(): + pass + + assert not is_generator(foo) + assert not is_generator(bar) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_is_generator_async_gen_syntax(pytester: Pytester) -> None: + pytester.makepyfile( + """ + from _pytest.compat import is_generator + def test_is_generator_py36(): + async def foo(): + yield + await foo() + + async def bar(): + yield + + assert not is_generator(foo) + assert not is_generator(bar) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + + +class ErrorsHelper: + @property + def raise_baseexception(self): + raise BaseException("base exception should be raised") + + @property + def raise_exception(self): + raise Exception("exception should be caught") + + @property + def raise_fail_outcome(self): + pytest.fail("fail should be caught") + + +def test_helper_failures() -> None: + helper = ErrorsHelper() + with pytest.raises(Exception): + helper.raise_exception + with pytest.raises(OutcomeException): + helper.raise_fail_outcome + + +def test_safe_getattr() -> None: + helper = ErrorsHelper() + assert safe_getattr(helper, "raise_exception", "default") == "default" + assert safe_getattr(helper, "raise_fail_outcome", "default") == "default" + with pytest.raises(BaseException): + assert safe_getattr(helper, "raise_baseexception", "default") + + +def test_safe_isclass() -> None: + assert safe_isclass(type) is True + + class CrappyClass(Exception): + # Type ignored because it's bypassed intentionally. + @property # type: ignore + def __class__(self): + assert False, "Should be ignored" + + assert safe_isclass(CrappyClass()) is False + + +def test_cached_property() -> None: + ncalls = 0 + + class Class: + @cached_property + def prop(self) -> int: + nonlocal ncalls + ncalls += 1 + return ncalls + + c1 = Class() + assert ncalls == 0 + assert c1.prop == 1 + assert c1.prop == 1 + c2 = Class() + assert ncalls == 1 + assert c2.prop == 2 + assert c1.prop == 1 + + +def test_assert_never_union() -> None: + x: Union[int, str] = 10 + + if isinstance(x, int): + pass + else: + with pytest.raises(AssertionError): + assert_never(x) # type: ignore[arg-type] + + if isinstance(x, int): + pass + elif isinstance(x, str): + pass + else: + assert_never(x) + + +def test_assert_never_enum() -> None: + E = enum.Enum("E", "a b") + x: E = E.a + + if x is E.a: + pass + else: + with pytest.raises(AssertionError): + assert_never(x) # type: ignore[arg-type] + + if x is E.a: + pass + elif x is E.b: + pass + else: + assert_never(x) + + +def test_assert_never_literal() -> None: + x: Literal["a", "b"] = "a" + + if x == "a": + pass + else: + with pytest.raises(AssertionError): + assert_never(x) # type: ignore[arg-type] + + if x == "a": + pass + elif x == "b": + pass + else: + assert_never(x) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_config.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_config.py new file mode 100644 index 0000000000..8013966f07 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_config.py @@ -0,0 +1,2115 @@ +import os +import re +import sys +import textwrap +from pathlib import Path +from typing import Dict +from typing import List +from typing import Sequence +from typing import Tuple +from typing import Type +from typing import Union + +import attr + +import _pytest._code +import pytest +from _pytest.compat import importlib_metadata +from _pytest.config import _get_plugin_specs_as_list +from _pytest.config import _iter_rewritable_modules +from _pytest.config import _strtobool +from _pytest.config import Config +from _pytest.config import ConftestImportFailure +from _pytest.config import ExitCode +from _pytest.config import parse_warning_filter +from _pytest.config.exceptions import UsageError +from _pytest.config.findpaths import determine_setup +from _pytest.config.findpaths import get_common_ancestor +from _pytest.config.findpaths import locate_config +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pathlib import absolutepath +from _pytest.pytester import Pytester + + +class TestParseIni: + @pytest.mark.parametrize( + "section, filename", [("pytest", "pytest.ini"), ("tool:pytest", "setup.cfg")] + ) + def test_getcfg_and_config( + self, + pytester: Pytester, + tmp_path: Path, + section: str, + filename: str, + monkeypatch: MonkeyPatch, + ) -> None: + sub = tmp_path / "sub" + sub.mkdir() + monkeypatch.chdir(sub) + (tmp_path / filename).write_text( + textwrap.dedent( + """\ + [{section}] + name = value + """.format( + section=section + ) + ), + encoding="utf-8", + ) + _, _, cfg = locate_config([sub]) + assert cfg["name"] == "value" + config = pytester.parseconfigure(str(sub)) + assert config.inicfg["name"] == "value" + + def test_setupcfg_uses_toolpytest_with_pytest(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile("def test(): pass") + pytester.makefile( + ".cfg", + setup=""" + [tool:pytest] + testpaths=%s + [pytest] + testpaths=ignored + """ + % p1.name, + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*, configfile: setup.cfg, *", "* 1 passed in *"]) + assert result.ret == 0 + + def test_append_parse_args( + self, pytester: Pytester, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.setenv("PYTEST_ADDOPTS", '--color no -rs --tb="short"') + tmp_path.joinpath("pytest.ini").write_text( + textwrap.dedent( + """\ + [pytest] + addopts = --verbose + """ + ) + ) + config = pytester.parseconfig(tmp_path) + assert config.option.color == "no" + assert config.option.reportchars == "s" + assert config.option.tbstyle == "short" + assert config.option.verbose + + def test_tox_ini_wrong_version(self, pytester: Pytester) -> None: + pytester.makefile( + ".ini", + tox=""" + [pytest] + minversion=999.0 + """, + ) + result = pytester.runpytest() + assert result.ret != 0 + result.stderr.fnmatch_lines( + ["*tox.ini: 'minversion' requires pytest-999.0, actual pytest-*"] + ) + + @pytest.mark.parametrize( + "section, name", + [("tool:pytest", "setup.cfg"), ("pytest", "tox.ini"), ("pytest", "pytest.ini")], + ) + def test_ini_names(self, pytester: Pytester, name, section) -> None: + pytester.path.joinpath(name).write_text( + textwrap.dedent( + """ + [{section}] + minversion = 1.0 + """.format( + section=section + ) + ) + ) + config = pytester.parseconfig() + assert config.getini("minversion") == "1.0" + + def test_pyproject_toml(self, pytester: Pytester) -> None: + pytester.makepyprojecttoml( + """ + [tool.pytest.ini_options] + minversion = "1.0" + """ + ) + config = pytester.parseconfig() + assert config.getini("minversion") == "1.0" + + def test_toxini_before_lower_pytestini(self, pytester: Pytester) -> None: + sub = pytester.mkdir("sub") + sub.joinpath("tox.ini").write_text( + textwrap.dedent( + """ + [pytest] + minversion = 2.0 + """ + ) + ) + pytester.path.joinpath("pytest.ini").write_text( + textwrap.dedent( + """ + [pytest] + minversion = 1.5 + """ + ) + ) + config = pytester.parseconfigure(sub) + assert config.getini("minversion") == "2.0" + + def test_ini_parse_error(self, pytester: Pytester) -> None: + pytester.path.joinpath("pytest.ini").write_text("addopts = -x") + result = pytester.runpytest() + assert result.ret != 0 + result.stderr.fnmatch_lines(["ERROR: *pytest.ini:1: no section header defined"]) + + @pytest.mark.xfail(reason="probably not needed") + def test_confcutdir(self, pytester: Pytester) -> None: + sub = pytester.mkdir("sub") + os.chdir(sub) + pytester.makeini( + """ + [pytest] + addopts = --qwe + """ + ) + result = pytester.inline_run("--confcutdir=.") + assert result.ret == 0 + + @pytest.mark.parametrize( + "ini_file_text, invalid_keys, warning_output, exception_text", + [ + pytest.param( + """ + [pytest] + unknown_ini = value1 + another_unknown_ini = value2 + """, + ["unknown_ini", "another_unknown_ini"], + [ + "=*= warnings summary =*=", + "*PytestConfigWarning:*Unknown config option: another_unknown_ini", + "*PytestConfigWarning:*Unknown config option: unknown_ini", + ], + "Unknown config option: another_unknown_ini", + id="2-unknowns", + ), + pytest.param( + """ + [pytest] + unknown_ini = value1 + minversion = 5.0.0 + """, + ["unknown_ini"], + [ + "=*= warnings summary =*=", + "*PytestConfigWarning:*Unknown config option: unknown_ini", + ], + "Unknown config option: unknown_ini", + id="1-unknown", + ), + pytest.param( + """ + [some_other_header] + unknown_ini = value1 + [pytest] + minversion = 5.0.0 + """, + [], + [], + "", + id="unknown-in-other-header", + ), + pytest.param( + """ + [pytest] + minversion = 5.0.0 + """, + [], + [], + "", + id="no-unknowns", + ), + pytest.param( + """ + [pytest] + conftest_ini_key = 1 + """, + [], + [], + "", + id="1-known", + ), + ], + ) + @pytest.mark.filterwarnings("default") + def test_invalid_config_options( + self, + pytester: Pytester, + ini_file_text, + invalid_keys, + warning_output, + exception_text, + ) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("conftest_ini_key", "") + """ + ) + pytester.makepyfile("def test(): pass") + pytester.makeini(ini_file_text) + + config = pytester.parseconfig() + assert sorted(config._get_unknown_ini_keys()) == sorted(invalid_keys) + + result = pytester.runpytest() + result.stdout.fnmatch_lines(warning_output) + + result = pytester.runpytest("--strict-config") + if exception_text: + result.stderr.fnmatch_lines("ERROR: " + exception_text) + assert result.ret == pytest.ExitCode.USAGE_ERROR + else: + result.stderr.no_fnmatch_line(exception_text) + assert result.ret == pytest.ExitCode.OK + + @pytest.mark.filterwarnings("default") + def test_silence_unknown_key_warning(self, pytester: Pytester) -> None: + """Unknown config key warnings can be silenced using filterwarnings (#7620)""" + pytester.makeini( + """ + [pytest] + filterwarnings = + ignore:Unknown config option:pytest.PytestConfigWarning + foobar=1 + """ + ) + result = pytester.runpytest() + result.stdout.no_fnmatch_line("*PytestConfigWarning*") + + @pytest.mark.filterwarnings("default::pytest.PytestConfigWarning") + def test_disable_warnings_plugin_disables_config_warnings( + self, pytester: Pytester + ) -> None: + """Disabling 'warnings' plugin also disables config time warnings""" + pytester.makeconftest( + """ + import pytest + def pytest_configure(config): + config.issue_config_time_warning( + pytest.PytestConfigWarning("custom config warning"), + stacklevel=2, + ) + """ + ) + result = pytester.runpytest("-pno:warnings") + result.stdout.no_fnmatch_line("*PytestConfigWarning*") + + @pytest.mark.parametrize( + "ini_file_text, plugin_version, exception_text", + [ + pytest.param( + """ + [pytest] + required_plugins = a z + """, + "1.5", + "Missing required plugins: a, z", + id="2-missing", + ), + pytest.param( + """ + [pytest] + required_plugins = a z myplugin + """, + "1.5", + "Missing required plugins: a, z", + id="2-missing-1-ok", + ), + pytest.param( + """ + [pytest] + required_plugins = myplugin + """, + "1.5", + None, + id="1-ok", + ), + pytest.param( + """ + [pytest] + required_plugins = myplugin==1.5 + """, + "1.5", + None, + id="1-ok-pin-exact", + ), + pytest.param( + """ + [pytest] + required_plugins = myplugin>1.0,<2.0 + """, + "1.5", + None, + id="1-ok-pin-loose", + ), + pytest.param( + """ + [pytest] + required_plugins = myplugin + """, + "1.5a1", + None, + id="1-ok-prerelease", + ), + pytest.param( + """ + [pytest] + required_plugins = myplugin==1.6 + """, + "1.5", + "Missing required plugins: myplugin==1.6", + id="missing-version", + ), + pytest.param( + """ + [pytest] + required_plugins = myplugin==1.6 other==1.0 + """, + "1.5", + "Missing required plugins: myplugin==1.6, other==1.0", + id="missing-versions", + ), + pytest.param( + """ + [some_other_header] + required_plugins = won't be triggered + [pytest] + """, + "1.5", + None, + id="invalid-header", + ), + ], + ) + def test_missing_required_plugins( + self, + pytester: Pytester, + monkeypatch: MonkeyPatch, + ini_file_text: str, + plugin_version: str, + exception_text: str, + ) -> None: + """Check 'required_plugins' option with various settings. + + This test installs a mock "myplugin-1.5" which is used in the parametrized test cases. + """ + + @attr.s + class DummyEntryPoint: + name = attr.ib() + module = attr.ib() + group = "pytest11" + + def load(self): + __import__(self.module) + return sys.modules[self.module] + + entry_points = [ + DummyEntryPoint("myplugin1", "myplugin1_module"), + ] + + @attr.s + class DummyDist: + entry_points = attr.ib() + files = () + version = plugin_version + + @property + def metadata(self): + return {"name": "myplugin"} + + def my_dists(): + return [DummyDist(entry_points)] + + pytester.makepyfile(myplugin1_module="# my plugin module") + pytester.syspathinsert() + + monkeypatch.setattr(importlib_metadata, "distributions", my_dists) + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + + pytester.makeini(ini_file_text) + + if exception_text: + with pytest.raises(pytest.UsageError, match=exception_text): + pytester.parseconfig() + else: + pytester.parseconfig() + + def test_early_config_cmdline( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + """early_config contains options registered by third-party plugins. + + This is a regression involving pytest-cov (and possibly others) introduced in #7700. + """ + pytester.makepyfile( + myplugin=""" + def pytest_addoption(parser): + parser.addoption('--foo', default=None, dest='foo') + + def pytest_load_initial_conftests(early_config, parser, args): + assert early_config.known_args_namespace.foo == "1" + """ + ) + monkeypatch.setenv("PYTEST_PLUGINS", "myplugin") + pytester.syspathinsert() + result = pytester.runpytest("--foo=1") + result.stdout.fnmatch_lines("* no tests ran in *") + + +class TestConfigCmdlineParsing: + def test_parsing_again_fails(self, pytester: Pytester) -> None: + config = pytester.parseconfig() + pytest.raises(AssertionError, lambda: config.parse([])) + + def test_explicitly_specified_config_file_is_loaded( + self, pytester: Pytester + ) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("custom", "") + """ + ) + pytester.makeini( + """ + [pytest] + custom = 0 + """ + ) + pytester.makefile( + ".ini", + custom=""" + [pytest] + custom = 1 + """, + ) + config = pytester.parseconfig("-c", "custom.ini") + assert config.getini("custom") == "1" + + pytester.makefile( + ".cfg", + custom_tool_pytest_section=""" + [tool:pytest] + custom = 1 + """, + ) + config = pytester.parseconfig("-c", "custom_tool_pytest_section.cfg") + assert config.getini("custom") == "1" + + pytester.makefile( + ".toml", + custom=""" + [tool.pytest.ini_options] + custom = 1 + value = [ + ] # this is here on purpose, as it makes this an invalid '.ini' file + """, + ) + config = pytester.parseconfig("-c", "custom.toml") + assert config.getini("custom") == "1" + + def test_absolute_win32_path(self, pytester: Pytester) -> None: + temp_ini_file = pytester.makefile( + ".ini", + custom=""" + [pytest] + addopts = --version + """, + ) + from os.path import normpath + + temp_ini_file_norm = normpath(str(temp_ini_file)) + ret = pytest.main(["-c", temp_ini_file_norm]) + assert ret == ExitCode.OK + + +class TestConfigAPI: + def test_config_trace(self, pytester: Pytester) -> None: + config = pytester.parseconfig() + values: List[str] = [] + config.trace.root.setwriter(values.append) + config.trace("hello") + assert len(values) == 1 + assert values[0] == "hello [config]\n" + + def test_config_getoption(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addoption("--hello", "-X", dest="hello") + """ + ) + config = pytester.parseconfig("--hello=this") + for x in ("hello", "--hello", "-X"): + assert config.getoption(x) == "this" + pytest.raises(ValueError, config.getoption, "qweqwe") + + def test_config_getoption_unicode(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addoption('--hello', type=str) + """ + ) + config = pytester.parseconfig("--hello=this") + assert config.getoption("hello") == "this" + + def test_config_getvalueorskip(self, pytester: Pytester) -> None: + config = pytester.parseconfig() + pytest.raises(pytest.skip.Exception, config.getvalueorskip, "hello") + verbose = config.getvalueorskip("verbose") + assert verbose == config.option.verbose + + def test_config_getvalueorskip_None(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addoption("--hello") + """ + ) + config = pytester.parseconfig() + with pytest.raises(pytest.skip.Exception): + config.getvalueorskip("hello") + + def test_getoption(self, pytester: Pytester) -> None: + config = pytester.parseconfig() + with pytest.raises(ValueError): + config.getvalue("x") + assert config.getoption("x", 1) == 1 + + def test_getconftest_pathlist(self, pytester: Pytester, tmp_path: Path) -> None: + somepath = tmp_path.joinpath("x", "y", "z") + p = tmp_path.joinpath("conftest.py") + p.write_text(f"mylist = {['.', str(somepath)]}") + config = pytester.parseconfigure(p) + assert ( + config._getconftest_pathlist("notexist", path=tmp_path, rootpath=tmp_path) + is None + ) + pl = ( + config._getconftest_pathlist("mylist", path=tmp_path, rootpath=tmp_path) + or [] + ) + print(pl) + assert len(pl) == 2 + assert pl[0] == tmp_path + assert pl[1] == somepath + + @pytest.mark.parametrize("maybe_type", ["not passed", "None", '"string"']) + def test_addini(self, pytester: Pytester, maybe_type: str) -> None: + if maybe_type == "not passed": + type_string = "" + else: + type_string = f", {maybe_type}" + + pytester.makeconftest( + f""" + def pytest_addoption(parser): + parser.addini("myname", "my new ini value"{type_string}) + """ + ) + pytester.makeini( + """ + [pytest] + myname=hello + """ + ) + config = pytester.parseconfig() + val = config.getini("myname") + assert val == "hello" + pytest.raises(ValueError, config.getini, "other") + + @pytest.mark.parametrize("config_type", ["ini", "pyproject"]) + def test_addini_paths(self, pytester: Pytester, config_type: str) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("paths", "my new ini value", type="paths") + parser.addini("abc", "abc value") + """ + ) + if config_type == "ini": + inipath = pytester.makeini( + """ + [pytest] + paths=hello world/sub.py + """ + ) + elif config_type == "pyproject": + inipath = pytester.makepyprojecttoml( + """ + [tool.pytest.ini_options] + paths=["hello", "world/sub.py"] + """ + ) + config = pytester.parseconfig() + values = config.getini("paths") + assert len(values) == 2 + assert values[0] == inipath.parent.joinpath("hello") + assert values[1] == inipath.parent.joinpath("world/sub.py") + pytest.raises(ValueError, config.getini, "other") + + def make_conftest_for_args(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("args", "new args", type="args") + parser.addini("a2", "", "args", default="1 2 3".split()) + """ + ) + + def test_addini_args_ini_files(self, pytester: Pytester) -> None: + self.make_conftest_for_args(pytester) + pytester.makeini( + """ + [pytest] + args=123 "123 hello" "this" + """ + ) + self.check_config_args(pytester) + + def test_addini_args_pyproject_toml(self, pytester: Pytester) -> None: + self.make_conftest_for_args(pytester) + pytester.makepyprojecttoml( + """ + [tool.pytest.ini_options] + args = ["123", "123 hello", "this"] + """ + ) + self.check_config_args(pytester) + + def check_config_args(self, pytester: Pytester) -> None: + config = pytester.parseconfig() + values = config.getini("args") + assert values == ["123", "123 hello", "this"] + values = config.getini("a2") + assert values == list("123") + + def make_conftest_for_linelist(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("xy", "", type="linelist") + parser.addini("a2", "", "linelist") + """ + ) + + def test_addini_linelist_ini_files(self, pytester: Pytester) -> None: + self.make_conftest_for_linelist(pytester) + pytester.makeini( + """ + [pytest] + xy= 123 345 + second line + """ + ) + self.check_config_linelist(pytester) + + def test_addini_linelist_pprojecttoml(self, pytester: Pytester) -> None: + self.make_conftest_for_linelist(pytester) + pytester.makepyprojecttoml( + """ + [tool.pytest.ini_options] + xy = ["123 345", "second line"] + """ + ) + self.check_config_linelist(pytester) + + def check_config_linelist(self, pytester: Pytester) -> None: + config = pytester.parseconfig() + values = config.getini("xy") + assert len(values) == 2 + assert values == ["123 345", "second line"] + values = config.getini("a2") + assert values == [] + + @pytest.mark.parametrize( + "str_val, bool_val", [("True", True), ("no", False), ("no-ini", True)] + ) + def test_addini_bool( + self, pytester: Pytester, str_val: str, bool_val: bool + ) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("strip", "", type="bool", default=True) + """ + ) + if str_val != "no-ini": + pytester.makeini( + """ + [pytest] + strip=%s + """ + % str_val + ) + config = pytester.parseconfig() + assert config.getini("strip") is bool_val + + def test_addinivalue_line_existing(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("xy", "", type="linelist") + """ + ) + pytester.makeini( + """ + [pytest] + xy= 123 + """ + ) + config = pytester.parseconfig() + values = config.getini("xy") + assert len(values) == 1 + assert values == ["123"] + config.addinivalue_line("xy", "456") + values = config.getini("xy") + assert len(values) == 2 + assert values == ["123", "456"] + + def test_addinivalue_line_new(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("xy", "", type="linelist") + """ + ) + config = pytester.parseconfig() + assert not config.getini("xy") + config.addinivalue_line("xy", "456") + values = config.getini("xy") + assert len(values) == 1 + assert values == ["456"] + config.addinivalue_line("xy", "123") + values = config.getini("xy") + assert len(values) == 2 + assert values == ["456", "123"] + + def test_confcutdir_check_isdir(self, pytester: Pytester) -> None: + """Give an error if --confcutdir is not a valid directory (#2078)""" + exp_match = r"^--confcutdir must be a directory, given: " + with pytest.raises(pytest.UsageError, match=exp_match): + pytester.parseconfig("--confcutdir", pytester.path.joinpath("file")) + with pytest.raises(pytest.UsageError, match=exp_match): + pytester.parseconfig("--confcutdir", pytester.path.joinpath("nonexistent")) + + p = pytester.mkdir("dir") + config = pytester.parseconfig("--confcutdir", p) + assert config.getoption("confcutdir") == str(p) + + @pytest.mark.parametrize( + "names, expected", + [ + # dist-info based distributions root are files as will be put in PYTHONPATH + (["bar.py"], ["bar"]), + (["foo/bar.py"], ["bar"]), + (["foo/bar.pyc"], []), + (["foo/__init__.py"], ["foo"]), + (["bar/__init__.py", "xz.py"], ["bar", "xz"]), + (["setup.py"], []), + # egg based distributions root contain the files from the dist root + (["src/bar/__init__.py"], ["bar"]), + (["src/bar/__init__.py", "setup.py"], ["bar"]), + (["source/python/bar/__init__.py", "setup.py"], ["bar"]), + ], + ) + def test_iter_rewritable_modules(self, names, expected) -> None: + assert list(_iter_rewritable_modules(names)) == expected + + +class TestConfigFromdictargs: + def test_basic_behavior(self, _sys_snapshot) -> None: + option_dict = {"verbose": 444, "foo": "bar", "capture": "no"} + args = ["a", "b"] + + config = Config.fromdictargs(option_dict, args) + with pytest.raises(AssertionError): + config.parse(["should refuse to parse again"]) + assert config.option.verbose == 444 + assert config.option.foo == "bar" + assert config.option.capture == "no" + assert config.args == args + + def test_invocation_params_args(self, _sys_snapshot) -> None: + """Show that fromdictargs can handle args in their "orig" format""" + option_dict: Dict[str, object] = {} + args = ["-vvvv", "-s", "a", "b"] + + config = Config.fromdictargs(option_dict, args) + assert config.args == ["a", "b"] + assert config.invocation_params.args == tuple(args) + assert config.option.verbose == 4 + assert config.option.capture == "no" + + def test_inifilename(self, tmp_path: Path) -> None: + d1 = tmp_path.joinpath("foo") + d1.mkdir() + p1 = d1.joinpath("bar.ini") + p1.touch() + p1.write_text( + textwrap.dedent( + """\ + [pytest] + name = value + """ + ) + ) + + inifilename = "../../foo/bar.ini" + option_dict = {"inifilename": inifilename, "capture": "no"} + + cwd = tmp_path.joinpath("a/b") + cwd.mkdir(parents=True) + p2 = cwd.joinpath("pytest.ini") + p2.touch() + p2.write_text( + textwrap.dedent( + """\ + [pytest] + name = wrong-value + should_not_be_set = true + """ + ) + ) + with MonkeyPatch.context() as mp: + mp.chdir(cwd) + config = Config.fromdictargs(option_dict, ()) + inipath = absolutepath(inifilename) + + assert config.args == [str(cwd)] + assert config.option.inifilename == inifilename + assert config.option.capture == "no" + + # this indicates this is the file used for getting configuration values + assert config.inipath == inipath + assert config.inicfg.get("name") == "value" + assert config.inicfg.get("should_not_be_set") is None + + +def test_options_on_small_file_do_not_blow_up(pytester: Pytester) -> None: + def runfiletest(opts: Sequence[str]) -> None: + reprec = pytester.inline_run(*opts) + passed, skipped, failed = reprec.countoutcomes() + assert failed == 2 + assert skipped == passed == 0 + + path = str( + pytester.makepyfile( + """ + def test_f1(): assert 0 + def test_f2(): assert 0 + """ + ) + ) + + runfiletest([path]) + runfiletest(["-l", path]) + runfiletest(["-s", path]) + runfiletest(["--tb=no", path]) + runfiletest(["--tb=short", path]) + runfiletest(["--tb=long", path]) + runfiletest(["--fulltrace", path]) + runfiletest(["--traceconfig", path]) + runfiletest(["-v", path]) + runfiletest(["-v", "-v", path]) + + +def test_preparse_ordering_with_setuptools( + pytester: Pytester, monkeypatch: MonkeyPatch +) -> None: + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + + class EntryPoint: + name = "mytestplugin" + group = "pytest11" + + def load(self): + class PseudoPlugin: + x = 42 + + return PseudoPlugin() + + class Dist: + files = () + metadata = {"name": "foo"} + entry_points = (EntryPoint(),) + + def my_dists(): + return (Dist,) + + monkeypatch.setattr(importlib_metadata, "distributions", my_dists) + pytester.makeconftest( + """ + pytest_plugins = "mytestplugin", + """ + ) + monkeypatch.setenv("PYTEST_PLUGINS", "mytestplugin") + config = pytester.parseconfig() + plugin = config.pluginmanager.getplugin("mytestplugin") + assert plugin.x == 42 + + +def test_setuptools_importerror_issue1479( + pytester: Pytester, monkeypatch: MonkeyPatch +) -> None: + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + + class DummyEntryPoint: + name = "mytestplugin" + group = "pytest11" + + def load(self): + raise ImportError("Don't hide me!") + + class Distribution: + version = "1.0" + files = ("foo.txt",) + metadata = {"name": "foo"} + entry_points = (DummyEntryPoint(),) + + def distributions(): + return (Distribution(),) + + monkeypatch.setattr(importlib_metadata, "distributions", distributions) + with pytest.raises(ImportError): + pytester.parseconfig() + + +def test_importlib_metadata_broken_distribution( + pytester: Pytester, monkeypatch: MonkeyPatch +) -> None: + """Integration test for broken distributions with 'files' metadata being None (#5389)""" + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + + class DummyEntryPoint: + name = "mytestplugin" + group = "pytest11" + + def load(self): + return object() + + class Distribution: + version = "1.0" + files = None + metadata = {"name": "foo"} + entry_points = (DummyEntryPoint(),) + + def distributions(): + return (Distribution(),) + + monkeypatch.setattr(importlib_metadata, "distributions", distributions) + pytester.parseconfig() + + +@pytest.mark.parametrize("block_it", [True, False]) +def test_plugin_preparse_prevents_setuptools_loading( + pytester: Pytester, monkeypatch: MonkeyPatch, block_it: bool +) -> None: + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + + plugin_module_placeholder = object() + + class DummyEntryPoint: + name = "mytestplugin" + group = "pytest11" + + def load(self): + return plugin_module_placeholder + + class Distribution: + version = "1.0" + files = ("foo.txt",) + metadata = {"name": "foo"} + entry_points = (DummyEntryPoint(),) + + def distributions(): + return (Distribution(),) + + monkeypatch.setattr(importlib_metadata, "distributions", distributions) + args = ("-p", "no:mytestplugin") if block_it else () + config = pytester.parseconfig(*args) + config.pluginmanager.import_plugin("mytestplugin") + if block_it: + assert "mytestplugin" not in sys.modules + assert config.pluginmanager.get_plugin("mytestplugin") is None + else: + assert ( + config.pluginmanager.get_plugin("mytestplugin") is plugin_module_placeholder + ) + + +@pytest.mark.parametrize( + "parse_args,should_load", [(("-p", "mytestplugin"), True), ((), False)] +) +def test_disable_plugin_autoload( + pytester: Pytester, + monkeypatch: MonkeyPatch, + parse_args: Union[Tuple[str, str], Tuple[()]], + should_load: bool, +) -> None: + class DummyEntryPoint: + project_name = name = "mytestplugin" + group = "pytest11" + version = "1.0" + + def load(self): + return sys.modules[self.name] + + class Distribution: + metadata = {"name": "foo"} + entry_points = (DummyEntryPoint(),) + files = () + + class PseudoPlugin: + x = 42 + + attrs_used = [] + + def __getattr__(self, name): + assert name == "__loader__" + self.attrs_used.append(name) + return object() + + def distributions(): + return (Distribution(),) + + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + monkeypatch.setattr(importlib_metadata, "distributions", distributions) + monkeypatch.setitem(sys.modules, "mytestplugin", PseudoPlugin()) # type: ignore[misc] + config = pytester.parseconfig(*parse_args) + has_loaded = config.pluginmanager.get_plugin("mytestplugin") is not None + assert has_loaded == should_load + if should_load: + assert PseudoPlugin.attrs_used == ["__loader__"] + else: + assert PseudoPlugin.attrs_used == [] + + +def test_plugin_loading_order(pytester: Pytester) -> None: + """Test order of plugin loading with `-p`.""" + p1 = pytester.makepyfile( + """ + def test_terminal_plugin(request): + import myplugin + assert myplugin.terminal_plugin == [False, True] + """, + **{ + "myplugin": """ + terminal_plugin = [] + + def pytest_configure(config): + terminal_plugin.append(bool(config.pluginmanager.get_plugin("terminalreporter"))) + + def pytest_sessionstart(session): + config = session.config + terminal_plugin.append(bool(config.pluginmanager.get_plugin("terminalreporter"))) + """ + }, + ) + pytester.syspathinsert() + result = pytester.runpytest("-p", "myplugin", str(p1)) + assert result.ret == 0 + + +def test_cmdline_processargs_simple(pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_cmdline_preparse(args): + args.append("-h") + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*pytest*", "*-h*"]) + + +def test_invalid_options_show_extra_information(pytester: Pytester) -> None: + """Display extra information when pytest exits due to unrecognized + options in the command-line.""" + pytester.makeini( + """ + [pytest] + addopts = --invalid-option + """ + ) + result = pytester.runpytest() + result.stderr.fnmatch_lines( + [ + "*error: unrecognized arguments: --invalid-option*", + "* inifile: %s*" % pytester.path.joinpath("tox.ini"), + "* rootdir: %s*" % pytester.path, + ] + ) + + +@pytest.mark.parametrize( + "args", + [ + ["dir1", "dir2", "-v"], + ["dir1", "-v", "dir2"], + ["dir2", "-v", "dir1"], + ["-v", "dir2", "dir1"], + ], +) +def test_consider_args_after_options_for_rootdir( + pytester: Pytester, args: List[str] +) -> None: + """ + Consider all arguments in the command-line for rootdir + discovery, even if they happen to occur after an option. #949 + """ + # replace "dir1" and "dir2" from "args" into their real directory + root = pytester.mkdir("myroot") + d1 = root.joinpath("dir1") + d1.mkdir() + d2 = root.joinpath("dir2") + d2.mkdir() + for i, arg in enumerate(args): + if arg == "dir1": + args[i] = str(d1) + elif arg == "dir2": + args[i] = str(d2) + with MonkeyPatch.context() as mp: + mp.chdir(root) + result = pytester.runpytest(*args) + result.stdout.fnmatch_lines(["*rootdir: *myroot"]) + + +def test_toolongargs_issue224(pytester: Pytester) -> None: + result = pytester.runpytest("-m", "hello" * 500) + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +def test_config_in_subdirectory_colon_command_line_issue2148( + pytester: Pytester, +) -> None: + conftest_source = """ + def pytest_addoption(parser): + parser.addini('foo', 'foo') + """ + + pytester.makefile( + ".ini", + **{"pytest": "[pytest]\nfoo = root", "subdir/pytest": "[pytest]\nfoo = subdir"}, + ) + + pytester.makepyfile( + **{ + "conftest": conftest_source, + "subdir/conftest": conftest_source, + "subdir/test_foo": """\ + def test_foo(pytestconfig): + assert pytestconfig.getini('foo') == 'subdir' + """, + } + ) + + result = pytester.runpytest("subdir/test_foo.py::test_foo") + assert result.ret == 0 + + +def test_notify_exception(pytester: Pytester, capfd) -> None: + config = pytester.parseconfig() + with pytest.raises(ValueError) as excinfo: + raise ValueError(1) + config.notify_exception(excinfo, config.option) + _, err = capfd.readouterr() + assert "ValueError" in err + + class A: + def pytest_internalerror(self): + return True + + config.pluginmanager.register(A()) + config.notify_exception(excinfo, config.option) + _, err = capfd.readouterr() + assert not err + + config = pytester.parseconfig("-p", "no:terminal") + with pytest.raises(ValueError) as excinfo: + raise ValueError(1) + config.notify_exception(excinfo, config.option) + _, err = capfd.readouterr() + assert "ValueError" in err + + +def test_no_terminal_discovery_error(pytester: Pytester) -> None: + pytester.makepyfile("raise TypeError('oops!')") + result = pytester.runpytest("-p", "no:terminal", "--collect-only") + assert result.ret == ExitCode.INTERRUPTED + + +def test_load_initial_conftest_last_ordering(_config_for_test): + pm = _config_for_test.pluginmanager + + class My: + def pytest_load_initial_conftests(self): + pass + + m = My() + pm.register(m) + hc = pm.hook.pytest_load_initial_conftests + hookimpls = [ + ( + hookimpl.function.__module__, + "wrapper" if hookimpl.hookwrapper else "nonwrapper", + ) + for hookimpl in hc.get_hookimpls() + ] + assert hookimpls == [ + ("_pytest.config", "nonwrapper"), + (m.__module__, "nonwrapper"), + ("_pytest.legacypath", "nonwrapper"), + ("_pytest.python_path", "nonwrapper"), + ("_pytest.capture", "wrapper"), + ("_pytest.warnings", "wrapper"), + ] + + +def test_get_plugin_specs_as_list() -> None: + def exp_match(val: object) -> str: + return ( + "Plugins may be specified as a sequence or a ','-separated string of plugin names. Got: %s" + % re.escape(repr(val)) + ) + + with pytest.raises(pytest.UsageError, match=exp_match({"foo"})): + _get_plugin_specs_as_list({"foo"}) # type: ignore[arg-type] + with pytest.raises(pytest.UsageError, match=exp_match({})): + _get_plugin_specs_as_list(dict()) # type: ignore[arg-type] + + assert _get_plugin_specs_as_list(None) == [] + assert _get_plugin_specs_as_list("") == [] + assert _get_plugin_specs_as_list("foo") == ["foo"] + assert _get_plugin_specs_as_list("foo,bar") == ["foo", "bar"] + assert _get_plugin_specs_as_list(["foo", "bar"]) == ["foo", "bar"] + assert _get_plugin_specs_as_list(("foo", "bar")) == ["foo", "bar"] + + +def test_collect_pytest_prefix_bug_integration(pytester: Pytester) -> None: + """Integration test for issue #3775""" + p = pytester.copy_example("config/collect_pytest_prefix") + result = pytester.runpytest(p) + result.stdout.fnmatch_lines(["* 1 passed *"]) + + +def test_collect_pytest_prefix_bug(pytestconfig): + """Ensure we collect only actual functions from conftest files (#3775)""" + + class Dummy: + class pytest_something: + pass + + pm = pytestconfig.pluginmanager + assert pm.parse_hookimpl_opts(Dummy(), "pytest_something") is None + + +class TestRootdir: + def test_simple_noini(self, tmp_path: Path, monkeypatch: MonkeyPatch) -> None: + assert get_common_ancestor([tmp_path]) == tmp_path + a = tmp_path / "a" + a.mkdir() + assert get_common_ancestor([a, tmp_path]) == tmp_path + assert get_common_ancestor([tmp_path, a]) == tmp_path + monkeypatch.chdir(tmp_path) + assert get_common_ancestor([]) == tmp_path + no_path = tmp_path / "does-not-exist" + assert get_common_ancestor([no_path]) == tmp_path + assert get_common_ancestor([no_path / "a"]) == tmp_path + + @pytest.mark.parametrize( + "name, contents", + [ + pytest.param("pytest.ini", "[pytest]\nx=10", id="pytest.ini"), + pytest.param( + "pyproject.toml", "[tool.pytest.ini_options]\nx=10", id="pyproject.toml" + ), + pytest.param("tox.ini", "[pytest]\nx=10", id="tox.ini"), + pytest.param("setup.cfg", "[tool:pytest]\nx=10", id="setup.cfg"), + ], + ) + def test_with_ini(self, tmp_path: Path, name: str, contents: str) -> None: + inipath = tmp_path / name + inipath.write_text(contents, "utf-8") + + a = tmp_path / "a" + a.mkdir() + b = a / "b" + b.mkdir() + for args in ([str(tmp_path)], [str(a)], [str(b)]): + rootpath, parsed_inipath, _ = determine_setup(None, args) + assert rootpath == tmp_path + assert parsed_inipath == inipath + rootpath, parsed_inipath, ini_config = determine_setup(None, [str(b), str(a)]) + assert rootpath == tmp_path + assert parsed_inipath == inipath + assert ini_config == {"x": "10"} + + @pytest.mark.parametrize("name", ["setup.cfg", "tox.ini"]) + def test_pytestini_overrides_empty_other(self, tmp_path: Path, name: str) -> None: + inipath = tmp_path / "pytest.ini" + inipath.touch() + a = tmp_path / "a" + a.mkdir() + (a / name).touch() + rootpath, parsed_inipath, _ = determine_setup(None, [str(a)]) + assert rootpath == tmp_path + assert parsed_inipath == inipath + + def test_setuppy_fallback(self, tmp_path: Path) -> None: + a = tmp_path / "a" + a.mkdir() + (a / "setup.cfg").touch() + (tmp_path / "setup.py").touch() + rootpath, inipath, inicfg = determine_setup(None, [str(a)]) + assert rootpath == tmp_path + assert inipath is None + assert inicfg == {} + + def test_nothing(self, tmp_path: Path, monkeypatch: MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + rootpath, inipath, inicfg = determine_setup(None, [str(tmp_path)]) + assert rootpath == tmp_path + assert inipath is None + assert inicfg == {} + + @pytest.mark.parametrize( + "name, contents", + [ + # pytest.param("pytest.ini", "[pytest]\nx=10", id="pytest.ini"), + pytest.param( + "pyproject.toml", "[tool.pytest.ini_options]\nx=10", id="pyproject.toml" + ), + # pytest.param("tox.ini", "[pytest]\nx=10", id="tox.ini"), + # pytest.param("setup.cfg", "[tool:pytest]\nx=10", id="setup.cfg"), + ], + ) + def test_with_specific_inifile( + self, tmp_path: Path, name: str, contents: str + ) -> None: + p = tmp_path / name + p.touch() + p.write_text(contents, "utf-8") + rootpath, inipath, ini_config = determine_setup(str(p), [str(tmp_path)]) + assert rootpath == tmp_path + assert inipath == p + assert ini_config == {"x": "10"} + + def test_explicit_config_file_sets_rootdir( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + tests_dir = tmp_path / "tests" + tests_dir.mkdir() + + monkeypatch.chdir(tmp_path) + + # No config file is explicitly given: rootdir is determined to be cwd. + rootpath, found_inipath, *_ = determine_setup(None, [str(tests_dir)]) + assert rootpath == tmp_path + assert found_inipath is None + + # Config file is explicitly given: rootdir is determined to be inifile's directory. + inipath = tmp_path / "pytest.ini" + inipath.touch() + rootpath, found_inipath, *_ = determine_setup(str(inipath), [str(tests_dir)]) + assert rootpath == tmp_path + assert found_inipath == inipath + + def test_with_arg_outside_cwd_without_inifile( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + a = tmp_path / "a" + a.mkdir() + b = tmp_path / "b" + b.mkdir() + rootpath, inifile, _ = determine_setup(None, [str(a), str(b)]) + assert rootpath == tmp_path + assert inifile is None + + def test_with_arg_outside_cwd_with_inifile(self, tmp_path: Path) -> None: + a = tmp_path / "a" + a.mkdir() + b = tmp_path / "b" + b.mkdir() + inipath = a / "pytest.ini" + inipath.touch() + rootpath, parsed_inipath, _ = determine_setup(None, [str(a), str(b)]) + assert rootpath == a + assert inipath == parsed_inipath + + @pytest.mark.parametrize("dirs", ([], ["does-not-exist"], ["a/does-not-exist"])) + def test_with_non_dir_arg( + self, dirs: Sequence[str], tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + rootpath, inipath, _ = determine_setup(None, dirs) + assert rootpath == tmp_path + assert inipath is None + + def test_with_existing_file_in_subdir( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: + a = tmp_path / "a" + a.mkdir() + (a / "exists").touch() + monkeypatch.chdir(tmp_path) + rootpath, inipath, _ = determine_setup(None, ["a/exist"]) + assert rootpath == tmp_path + assert inipath is None + + def test_with_config_also_in_parent_directory( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: + """Regression test for #7807.""" + (tmp_path / "setup.cfg").write_text("[tool:pytest]\n", "utf-8") + (tmp_path / "myproject").mkdir() + (tmp_path / "myproject" / "setup.cfg").write_text("[tool:pytest]\n", "utf-8") + (tmp_path / "myproject" / "tests").mkdir() + monkeypatch.chdir(tmp_path / "myproject") + + rootpath, inipath, _ = determine_setup(None, ["tests/"]) + + assert rootpath == tmp_path / "myproject" + assert inipath == tmp_path / "myproject" / "setup.cfg" + + +class TestOverrideIniArgs: + @pytest.mark.parametrize("name", "setup.cfg tox.ini pytest.ini".split()) + def test_override_ini_names(self, pytester: Pytester, name: str) -> None: + section = "[pytest]" if name != "setup.cfg" else "[tool:pytest]" + pytester.path.joinpath(name).write_text( + textwrap.dedent( + """ + {section} + custom = 1.0""".format( + section=section + ) + ) + ) + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("custom", "")""" + ) + pytester.makepyfile( + """ + def test_pass(pytestconfig): + ini_val = pytestconfig.getini("custom") + print('\\ncustom_option:%s\\n' % ini_val)""" + ) + + result = pytester.runpytest("--override-ini", "custom=2.0", "-s") + assert result.ret == 0 + result.stdout.fnmatch_lines(["custom_option:2.0"]) + + result = pytester.runpytest( + "--override-ini", "custom=2.0", "--override-ini=custom=3.0", "-s" + ) + assert result.ret == 0 + result.stdout.fnmatch_lines(["custom_option:3.0"]) + + def test_override_ini_paths(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("paths", "my new ini value", type="paths")""" + ) + pytester.makeini( + """ + [pytest] + paths=blah.py""" + ) + pytester.makepyfile( + r""" + def test_overriden(pytestconfig): + config_paths = pytestconfig.getini("paths") + print(config_paths) + for cpf in config_paths: + print('\nuser_path:%s' % cpf.name) + """ + ) + result = pytester.runpytest( + "--override-ini", "paths=foo/bar1.py foo/bar2.py", "-s" + ) + result.stdout.fnmatch_lines(["user_path:bar1.py", "user_path:bar2.py"]) + + def test_override_multiple_and_default(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + addini = parser.addini + addini("custom_option_1", "", default="o1") + addini("custom_option_2", "", default="o2") + addini("custom_option_3", "", default=False, type="bool") + addini("custom_option_4", "", default=True, type="bool")""" + ) + pytester.makeini( + """ + [pytest] + custom_option_1=custom_option_1 + custom_option_2=custom_option_2 + """ + ) + pytester.makepyfile( + """ + def test_multiple_options(pytestconfig): + prefix = "custom_option" + for x in range(1, 5): + ini_value=pytestconfig.getini("%s_%d" % (prefix, x)) + print('\\nini%d:%s' % (x, ini_value)) + """ + ) + result = pytester.runpytest( + "--override-ini", + "custom_option_1=fulldir=/tmp/user1", + "-o", + "custom_option_2=url=/tmp/user2?a=b&d=e", + "-o", + "custom_option_3=True", + "-o", + "custom_option_4=no", + "-s", + ) + result.stdout.fnmatch_lines( + [ + "ini1:fulldir=/tmp/user1", + "ini2:url=/tmp/user2?a=b&d=e", + "ini3:True", + "ini4:False", + ] + ) + + def test_override_ini_usage_error_bad_style(self, pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + xdist_strict=False + """ + ) + result = pytester.runpytest("--override-ini", "xdist_strict", "True") + result.stderr.fnmatch_lines( + [ + "ERROR: -o/--override-ini expects option=value style (got: 'xdist_strict').", + ] + ) + + @pytest.mark.parametrize("with_ini", [True, False]) + def test_override_ini_handled_asap( + self, pytester: Pytester, with_ini: bool + ) -> None: + """-o should be handled as soon as possible and always override what's in ini files (#2238)""" + if with_ini: + pytester.makeini( + """ + [pytest] + python_files=test_*.py + """ + ) + pytester.makepyfile( + unittest_ini_handle=""" + def test(): + pass + """ + ) + result = pytester.runpytest("--override-ini", "python_files=unittest_*.py") + result.stdout.fnmatch_lines(["*1 passed in*"]) + + def test_addopts_before_initini( + self, monkeypatch: MonkeyPatch, _config_for_test, _sys_snapshot + ) -> None: + cache_dir = ".custom_cache" + monkeypatch.setenv("PYTEST_ADDOPTS", "-o cache_dir=%s" % cache_dir) + config = _config_for_test + config._preparse([], addopts=True) + assert config._override_ini == ["cache_dir=%s" % cache_dir] + + def test_addopts_from_env_not_concatenated( + self, monkeypatch: MonkeyPatch, _config_for_test + ) -> None: + """PYTEST_ADDOPTS should not take values from normal args (#4265).""" + monkeypatch.setenv("PYTEST_ADDOPTS", "-o") + config = _config_for_test + with pytest.raises(UsageError) as excinfo: + config._preparse(["cache_dir=ignored"], addopts=True) + assert ( + "error: argument -o/--override-ini: expected one argument (via PYTEST_ADDOPTS)" + in excinfo.value.args[0] + ) + + def test_addopts_from_ini_not_concatenated(self, pytester: Pytester) -> None: + """`addopts` from ini should not take values from normal args (#4265).""" + pytester.makeini( + """ + [pytest] + addopts=-o + """ + ) + result = pytester.runpytest("cache_dir=ignored") + result.stderr.fnmatch_lines( + [ + "%s: error: argument -o/--override-ini: expected one argument (via addopts config)" + % (pytester._request.config._parser.optparser.prog,) + ] + ) + assert result.ret == _pytest.config.ExitCode.USAGE_ERROR + + def test_override_ini_does_not_contain_paths( + self, _config_for_test, _sys_snapshot + ) -> None: + """Check that -o no longer swallows all options after it (#3103)""" + config = _config_for_test + config._preparse(["-o", "cache_dir=/cache", "/some/test/path"]) + assert config._override_ini == ["cache_dir=/cache"] + + def test_multiple_override_ini_options(self, pytester: Pytester) -> None: + """Ensure a file path following a '-o' option does not generate an error (#3103)""" + pytester.makepyfile( + **{ + "conftest.py": """ + def pytest_addoption(parser): + parser.addini('foo', default=None, help='some option') + parser.addini('bar', default=None, help='some option') + """, + "test_foo.py": """ + def test(pytestconfig): + assert pytestconfig.getini('foo') == '1' + assert pytestconfig.getini('bar') == '0' + """, + "test_bar.py": """ + def test(): + assert False + """, + } + ) + result = pytester.runpytest("-o", "foo=1", "-o", "bar=0", "test_foo.py") + assert "ERROR:" not in result.stderr.str() + result.stdout.fnmatch_lines(["collected 1 item", "*= 1 passed in *="]) + + +def test_help_via_addopts(pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + addopts = --unknown-option-should-allow-for-help --help + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + "usage: *", + "positional arguments:", + # Displays full/default help. + "to see available markers type: pytest --markers", + ] + ) + + +def test_help_and_version_after_argument_error(pytester: Pytester) -> None: + pytester.makeconftest( + """ + def validate(arg): + raise argparse.ArgumentTypeError("argerror") + + def pytest_addoption(parser): + group = parser.getgroup('cov') + group.addoption( + "--invalid-option-should-allow-for-help", + type=validate, + ) + """ + ) + pytester.makeini( + """ + [pytest] + addopts = --invalid-option-should-allow-for-help + """ + ) + result = pytester.runpytest("--help") + result.stdout.fnmatch_lines( + [ + "usage: *", + "positional arguments:", + "NOTE: displaying only minimal help due to UsageError.", + ] + ) + result.stderr.fnmatch_lines( + [ + "ERROR: usage: *", + "%s: error: argument --invalid-option-should-allow-for-help: expected one argument" + % (pytester._request.config._parser.optparser.prog,), + ] + ) + # Does not display full/default help. + assert "to see available markers type: pytest --markers" not in result.stdout.lines + assert result.ret == ExitCode.USAGE_ERROR + + result = pytester.runpytest("--version") + result.stdout.fnmatch_lines([f"pytest {pytest.__version__}"]) + assert result.ret == ExitCode.USAGE_ERROR + + +def test_help_formatter_uses_py_get_terminal_width(monkeypatch: MonkeyPatch) -> None: + from _pytest.config.argparsing import DropShorterLongHelpFormatter + + monkeypatch.setenv("COLUMNS", "90") + formatter = DropShorterLongHelpFormatter("prog") + assert formatter._width == 90 + + monkeypatch.setattr("_pytest._io.get_terminal_width", lambda: 160) + formatter = DropShorterLongHelpFormatter("prog") + assert formatter._width == 160 + + formatter = DropShorterLongHelpFormatter("prog", width=42) + assert formatter._width == 42 + + +def test_config_does_not_load_blocked_plugin_from_args(pytester: Pytester) -> None: + """This tests that pytest's config setup handles "-p no:X".""" + p = pytester.makepyfile("def test(capfd): pass") + result = pytester.runpytest(str(p), "-pno:capture") + result.stdout.fnmatch_lines(["E fixture 'capfd' not found"]) + assert result.ret == ExitCode.TESTS_FAILED + + result = pytester.runpytest(str(p), "-pno:capture", "-s") + result.stderr.fnmatch_lines(["*: error: unrecognized arguments: -s"]) + assert result.ret == ExitCode.USAGE_ERROR + + +def test_invocation_args(pytester: Pytester) -> None: + """Ensure that Config.invocation_* arguments are correctly defined""" + + class DummyPlugin: + pass + + p = pytester.makepyfile("def test(): pass") + plugin = DummyPlugin() + rec = pytester.inline_run(p, "-v", plugins=[plugin]) + calls = rec.getcalls("pytest_runtest_protocol") + assert len(calls) == 1 + call = calls[0] + config = call.item.config + + assert config.invocation_params.args == (str(p), "-v") + assert config.invocation_params.dir == pytester.path + + plugins = config.invocation_params.plugins + assert len(plugins) == 2 + assert plugins[0] is plugin + assert type(plugins[1]).__name__ == "Collect" # installed by pytester.inline_run() + + # args cannot be None + with pytest.raises(TypeError): + Config.InvocationParams(args=None, plugins=None, dir=Path()) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + "plugin", + [ + x + for x in _pytest.config.default_plugins + if x not in _pytest.config.essential_plugins + ], +) +def test_config_blocked_default_plugins(pytester: Pytester, plugin: str) -> None: + if plugin == "debugging": + # Fixed in xdist (after 1.27.0). + # https://github.com/pytest-dev/pytest-xdist/pull/422 + try: + import xdist # noqa: F401 + except ImportError: + pass + else: + pytest.skip("does not work with xdist currently") + + p = pytester.makepyfile("def test(): pass") + result = pytester.runpytest(str(p), "-pno:%s" % plugin) + + if plugin == "python": + assert result.ret == ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines( + [ + "ERROR: not found: */test_config_blocked_default_plugins.py", + "(no name '*/test_config_blocked_default_plugins.py' in any of [])", + ] + ) + return + + assert result.ret == ExitCode.OK + if plugin != "terminal": + result.stdout.fnmatch_lines(["* 1 passed in *"]) + + p = pytester.makepyfile("def test(): assert 0") + result = pytester.runpytest(str(p), "-pno:%s" % plugin) + assert result.ret == ExitCode.TESTS_FAILED + if plugin != "terminal": + result.stdout.fnmatch_lines(["* 1 failed in *"]) + else: + assert result.stdout.lines == [] + + +class TestSetupCfg: + def test_pytest_setup_cfg_unsupported(self, pytester: Pytester) -> None: + pytester.makefile( + ".cfg", + setup=""" + [pytest] + addopts = --verbose + """, + ) + with pytest.raises(pytest.fail.Exception): + pytester.runpytest() + + def test_pytest_custom_cfg_unsupported(self, pytester: Pytester) -> None: + pytester.makefile( + ".cfg", + custom=""" + [pytest] + addopts = --verbose + """, + ) + with pytest.raises(pytest.fail.Exception): + pytester.runpytest("-c", "custom.cfg") + + +class TestPytestPluginsVariable: + def test_pytest_plugins_in_non_top_level_conftest_unsupported( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + **{ + "subdirectory/conftest.py": """ + pytest_plugins=['capture'] + """ + } + ) + pytester.makepyfile( + """ + def test_func(): + pass + """ + ) + res = pytester.runpytest() + assert res.ret == 2 + msg = "Defining 'pytest_plugins' in a non-top-level conftest is no longer supported" + res.stdout.fnmatch_lines([f"*{msg}*", f"*subdirectory{os.sep}conftest.py*"]) + + @pytest.mark.parametrize("use_pyargs", [True, False]) + def test_pytest_plugins_in_non_top_level_conftest_unsupported_pyargs( + self, pytester: Pytester, use_pyargs: bool + ) -> None: + """When using --pyargs, do not emit the warning about non-top-level conftest warnings (#4039, #4044)""" + + files = { + "src/pkg/__init__.py": "", + "src/pkg/conftest.py": "", + "src/pkg/test_root.py": "def test(): pass", + "src/pkg/sub/__init__.py": "", + "src/pkg/sub/conftest.py": "pytest_plugins=['capture']", + "src/pkg/sub/test_bar.py": "def test(): pass", + } + pytester.makepyfile(**files) + pytester.syspathinsert(pytester.path.joinpath("src")) + + args = ("--pyargs", "pkg") if use_pyargs else () + res = pytester.runpytest(*args) + assert res.ret == (0 if use_pyargs else 2) + msg = ( + msg + ) = "Defining 'pytest_plugins' in a non-top-level conftest is no longer supported" + if use_pyargs: + assert msg not in res.stdout.str() + else: + res.stdout.fnmatch_lines([f"*{msg}*"]) + + def test_pytest_plugins_in_non_top_level_conftest_unsupported_no_top_level_conftest( + self, pytester: Pytester + ) -> None: + subdirectory = pytester.path.joinpath("subdirectory") + subdirectory.mkdir() + pytester.makeconftest( + """ + pytest_plugins=['capture'] + """ + ) + pytester.path.joinpath("conftest.py").rename( + subdirectory.joinpath("conftest.py") + ) + + pytester.makepyfile( + """ + def test_func(): + pass + """ + ) + + res = pytester.runpytest_subprocess() + assert res.ret == 2 + msg = "Defining 'pytest_plugins' in a non-top-level conftest is no longer supported" + res.stdout.fnmatch_lines([f"*{msg}*", f"*subdirectory{os.sep}conftest.py*"]) + + def test_pytest_plugins_in_non_top_level_conftest_unsupported_no_false_positives( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + "def test_func(): pass", + **{ + "subdirectory/conftest": "pass", + "conftest": """ + import warnings + warnings.filterwarnings('always', category=DeprecationWarning) + pytest_plugins=['capture'] + """, + }, + ) + res = pytester.runpytest_subprocess() + assert res.ret == 0 + msg = "Defining 'pytest_plugins' in a non-top-level conftest is no longer supported" + assert msg not in res.stdout.str() + + +def test_conftest_import_error_repr(tmp_path: Path) -> None: + """`ConftestImportFailure` should use a short error message and readable + path to the failed conftest.py file.""" + path = tmp_path.joinpath("foo/conftest.py") + with pytest.raises( + ConftestImportFailure, + match=re.escape(f"RuntimeError: some error (from {path})"), + ): + try: + raise RuntimeError("some error") + except Exception as exc: + assert exc.__traceback__ is not None + exc_info = (type(exc), exc, exc.__traceback__) + raise ConftestImportFailure(path, exc_info) from exc + + +def test_strtobool() -> None: + assert _strtobool("YES") + assert not _strtobool("NO") + with pytest.raises(ValueError): + _strtobool("unknown") + + +@pytest.mark.parametrize( + "arg, escape, expected", + [ + ("ignore", False, ("ignore", "", Warning, "", 0)), + ( + "ignore::DeprecationWarning", + False, + ("ignore", "", DeprecationWarning, "", 0), + ), + ( + "ignore:some msg:DeprecationWarning", + False, + ("ignore", "some msg", DeprecationWarning, "", 0), + ), + ( + "ignore::DeprecationWarning:mod", + False, + ("ignore", "", DeprecationWarning, "mod", 0), + ), + ( + "ignore::DeprecationWarning:mod:42", + False, + ("ignore", "", DeprecationWarning, "mod", 42), + ), + ("error:some\\msg:::", True, ("error", "some\\\\msg", Warning, "", 0)), + ("error:::mod\\foo:", True, ("error", "", Warning, "mod\\\\foo\\Z", 0)), + ], +) +def test_parse_warning_filter( + arg: str, escape: bool, expected: Tuple[str, str, Type[Warning], str, int] +) -> None: + assert parse_warning_filter(arg, escape=escape) == expected + + +@pytest.mark.parametrize( + "arg", + [ + # Too much parts. + ":" * 5, + # Invalid action. + "FOO::", + # ImportError when importing the warning class. + "::test_parse_warning_filter_failure.NonExistentClass::", + # Class is not a Warning subclass. + "::list::", + # Negative line number. + "::::-1", + # Not a line number. + "::::not-a-number", + ], +) +def test_parse_warning_filter_failure(arg: str) -> None: + with pytest.raises(pytest.UsageError): + parse_warning_filter(arg, escape=True) + + +class TestDebugOptions: + def test_without_debug_does_not_write_log(self, pytester: Pytester) -> None: + result = pytester.runpytest() + result.stderr.no_fnmatch_line( + "*writing pytest debug information to*pytestdebug.log" + ) + result.stderr.no_fnmatch_line( + "*wrote pytest debug information to*pytestdebug.log" + ) + assert not [f.name for f in pytester.path.glob("**/*.log")] + + def test_with_only_debug_writes_pytestdebug_log(self, pytester: Pytester) -> None: + result = pytester.runpytest("--debug") + result.stderr.fnmatch_lines( + [ + "*writing pytest debug information to*pytestdebug.log", + "*wrote pytest debug information to*pytestdebug.log", + ] + ) + assert "pytestdebug.log" in [f.name for f in pytester.path.glob("**/*.log")] + + def test_multiple_custom_debug_logs(self, pytester: Pytester) -> None: + result = pytester.runpytest("--debug", "bar.log") + result.stderr.fnmatch_lines( + [ + "*writing pytest debug information to*bar.log", + "*wrote pytest debug information to*bar.log", + ] + ) + result = pytester.runpytest("--debug", "foo.log") + result.stderr.fnmatch_lines( + [ + "*writing pytest debug information to*foo.log", + "*wrote pytest debug information to*foo.log", + ] + ) + + assert {"bar.log", "foo.log"} == { + f.name for f in pytester.path.glob("**/*.log") + } + + def test_debug_help(self, pytester: Pytester) -> None: + result = pytester.runpytest("-h") + result.stdout.fnmatch_lines( + [ + "*store internal tracing debug information in this log*", + "*This file is opened with 'w' and truncated as a result*", + "*Defaults to 'pytestdebug.log'.", + ] + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_conftest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_conftest.py new file mode 100644 index 0000000000..64c1014a53 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_conftest.py @@ -0,0 +1,696 @@ +import argparse +import os +import textwrap +from pathlib import Path +from typing import cast +from typing import Dict +from typing import Generator +from typing import List +from typing import Optional + +import pytest +from _pytest.config import ExitCode +from _pytest.config import PytestPluginManager +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pathlib import symlink_or_skip +from _pytest.pytester import Pytester +from _pytest.tmpdir import TempPathFactory + + +def ConftestWithSetinitial(path) -> PytestPluginManager: + conftest = PytestPluginManager() + conftest_setinitial(conftest, [path]) + return conftest + + +def conftest_setinitial( + conftest: PytestPluginManager, args, confcutdir: Optional["os.PathLike[str]"] = None +) -> None: + class Namespace: + def __init__(self) -> None: + self.file_or_dir = args + self.confcutdir = os.fspath(confcutdir) if confcutdir is not None else None + self.noconftest = False + self.pyargs = False + self.importmode = "prepend" + + namespace = cast(argparse.Namespace, Namespace()) + conftest._set_initial_conftests(namespace, rootpath=Path(args[0])) + + +@pytest.mark.usefixtures("_sys_snapshot") +class TestConftestValueAccessGlobal: + @pytest.fixture(scope="module", params=["global", "inpackage"]) + def basedir( + self, request, tmp_path_factory: TempPathFactory + ) -> Generator[Path, None, None]: + tmp_path = tmp_path_factory.mktemp("basedir", numbered=True) + tmp_path.joinpath("adir/b").mkdir(parents=True) + tmp_path.joinpath("adir/conftest.py").write_text("a=1 ; Directory = 3") + tmp_path.joinpath("adir/b/conftest.py").write_text("b=2 ; a = 1.5") + if request.param == "inpackage": + tmp_path.joinpath("adir/__init__.py").touch() + tmp_path.joinpath("adir/b/__init__.py").touch() + + yield tmp_path + + def test_basic_init(self, basedir: Path) -> None: + conftest = PytestPluginManager() + p = basedir / "adir" + assert ( + conftest._rget_with_confmod("a", p, importmode="prepend", rootpath=basedir)[ + 1 + ] + == 1 + ) + + def test_immediate_initialiation_and_incremental_are_the_same( + self, basedir: Path + ) -> None: + conftest = PytestPluginManager() + assert not len(conftest._dirpath2confmods) + conftest._getconftestmodules( + basedir, importmode="prepend", rootpath=Path(basedir) + ) + snap1 = len(conftest._dirpath2confmods) + assert snap1 == 1 + conftest._getconftestmodules( + basedir / "adir", importmode="prepend", rootpath=basedir + ) + assert len(conftest._dirpath2confmods) == snap1 + 1 + conftest._getconftestmodules( + basedir / "b", importmode="prepend", rootpath=basedir + ) + assert len(conftest._dirpath2confmods) == snap1 + 2 + + def test_value_access_not_existing(self, basedir: Path) -> None: + conftest = ConftestWithSetinitial(basedir) + with pytest.raises(KeyError): + conftest._rget_with_confmod( + "a", basedir, importmode="prepend", rootpath=Path(basedir) + ) + + def test_value_access_by_path(self, basedir: Path) -> None: + conftest = ConftestWithSetinitial(basedir) + adir = basedir / "adir" + assert ( + conftest._rget_with_confmod( + "a", adir, importmode="prepend", rootpath=basedir + )[1] + == 1 + ) + assert ( + conftest._rget_with_confmod( + "a", adir / "b", importmode="prepend", rootpath=basedir + )[1] + == 1.5 + ) + + def test_value_access_with_confmod(self, basedir: Path) -> None: + startdir = basedir / "adir" / "b" + startdir.joinpath("xx").mkdir() + conftest = ConftestWithSetinitial(startdir) + mod, value = conftest._rget_with_confmod( + "a", startdir, importmode="prepend", rootpath=Path(basedir) + ) + assert value == 1.5 + path = Path(mod.__file__) + assert path.parent == basedir / "adir" / "b" + assert path.stem == "conftest" + + +def test_conftest_in_nonpkg_with_init(tmp_path: Path, _sys_snapshot) -> None: + tmp_path.joinpath("adir-1.0/b").mkdir(parents=True) + tmp_path.joinpath("adir-1.0/conftest.py").write_text("a=1 ; Directory = 3") + tmp_path.joinpath("adir-1.0/b/conftest.py").write_text("b=2 ; a = 1.5") + tmp_path.joinpath("adir-1.0/b/__init__.py").touch() + tmp_path.joinpath("adir-1.0/__init__.py").touch() + ConftestWithSetinitial(tmp_path.joinpath("adir-1.0", "b")) + + +def test_doubledash_considered(pytester: Pytester) -> None: + conf = pytester.mkdir("--option") + conf.joinpath("conftest.py").touch() + conftest = PytestPluginManager() + conftest_setinitial(conftest, [conf.name, conf.name]) + values = conftest._getconftestmodules( + conf, importmode="prepend", rootpath=pytester.path + ) + assert len(values) == 1 + + +def test_issue151_load_all_conftests(pytester: Pytester) -> None: + names = "code proj src".split() + for name in names: + p = pytester.mkdir(name) + p.joinpath("conftest.py").touch() + + conftest = PytestPluginManager() + conftest_setinitial(conftest, names) + d = list(conftest._conftestpath2mod.values()) + assert len(d) == len(names) + + +def test_conftest_global_import(pytester: Pytester) -> None: + pytester.makeconftest("x=3") + p = pytester.makepyfile( + """ + from pathlib import Path + import pytest + from _pytest.config import PytestPluginManager + conf = PytestPluginManager() + mod = conf._importconftest(Path("conftest.py"), importmode="prepend", rootpath=Path.cwd()) + assert mod.x == 3 + import conftest + assert conftest is mod, (conftest, mod) + sub = Path("sub") + sub.mkdir() + subconf = sub / "conftest.py" + subconf.write_text("y=4") + mod2 = conf._importconftest(subconf, importmode="prepend", rootpath=Path.cwd()) + assert mod != mod2 + assert mod2.y == 4 + import conftest + assert conftest is mod2, (conftest, mod) + """ + ) + res = pytester.runpython(p) + assert res.ret == 0 + + +def test_conftestcutdir(pytester: Pytester) -> None: + conf = pytester.makeconftest("") + p = pytester.mkdir("x") + conftest = PytestPluginManager() + conftest_setinitial(conftest, [pytester.path], confcutdir=p) + values = conftest._getconftestmodules( + p, importmode="prepend", rootpath=pytester.path + ) + assert len(values) == 0 + values = conftest._getconftestmodules( + conf.parent, importmode="prepend", rootpath=pytester.path + ) + assert len(values) == 0 + assert Path(conf) not in conftest._conftestpath2mod + # but we can still import a conftest directly + conftest._importconftest(conf, importmode="prepend", rootpath=pytester.path) + values = conftest._getconftestmodules( + conf.parent, importmode="prepend", rootpath=pytester.path + ) + assert values[0].__file__.startswith(str(conf)) + # and all sub paths get updated properly + values = conftest._getconftestmodules( + p, importmode="prepend", rootpath=pytester.path + ) + assert len(values) == 1 + assert values[0].__file__.startswith(str(conf)) + + +def test_conftestcutdir_inplace_considered(pytester: Pytester) -> None: + conf = pytester.makeconftest("") + conftest = PytestPluginManager() + conftest_setinitial(conftest, [conf.parent], confcutdir=conf.parent) + values = conftest._getconftestmodules( + conf.parent, importmode="prepend", rootpath=pytester.path + ) + assert len(values) == 1 + assert values[0].__file__.startswith(str(conf)) + + +@pytest.mark.parametrize("name", "test tests whatever .dotdir".split()) +def test_setinitial_conftest_subdirs(pytester: Pytester, name: str) -> None: + sub = pytester.mkdir(name) + subconftest = sub.joinpath("conftest.py") + subconftest.touch() + conftest = PytestPluginManager() + conftest_setinitial(conftest, [sub.parent], confcutdir=pytester.path) + key = subconftest.resolve() + if name not in ("whatever", ".dotdir"): + assert key in conftest._conftestpath2mod + assert len(conftest._conftestpath2mod) == 1 + else: + assert key not in conftest._conftestpath2mod + assert len(conftest._conftestpath2mod) == 0 + + +def test_conftest_confcutdir(pytester: Pytester) -> None: + pytester.makeconftest("assert 0") + x = pytester.mkdir("x") + x.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + def pytest_addoption(parser): + parser.addoption("--xyz", action="store_true") + """ + ) + ) + result = pytester.runpytest("-h", "--confcutdir=%s" % x, x) + result.stdout.fnmatch_lines(["*--xyz*"]) + result.stdout.no_fnmatch_line("*warning: could not load initial*") + + +def test_conftest_symlink(pytester: Pytester) -> None: + """`conftest.py` discovery follows normal path resolution and does not resolve symlinks.""" + # Structure: + # /real + # /real/conftest.py + # /real/app + # /real/app/tests + # /real/app/tests/test_foo.py + + # Links: + # /symlinktests -> /real/app/tests (running at symlinktests should fail) + # /symlink -> /real (running at /symlink should work) + + real = pytester.mkdir("real") + realtests = real.joinpath("app/tests") + realtests.mkdir(parents=True) + symlink_or_skip(realtests, pytester.path.joinpath("symlinktests")) + symlink_or_skip(real, pytester.path.joinpath("symlink")) + pytester.makepyfile( + **{ + "real/app/tests/test_foo.py": "def test1(fixture): pass", + "real/conftest.py": textwrap.dedent( + """ + import pytest + + print("conftest_loaded") + + @pytest.fixture + def fixture(): + print("fixture_used") + """ + ), + } + ) + + # Should fail because conftest cannot be found from the link structure. + result = pytester.runpytest("-vs", "symlinktests") + result.stdout.fnmatch_lines(["*fixture 'fixture' not found*"]) + assert result.ret == ExitCode.TESTS_FAILED + + # Should not cause "ValueError: Plugin already registered" (#4174). + result = pytester.runpytest("-vs", "symlink") + assert result.ret == ExitCode.OK + + +def test_conftest_symlink_files(pytester: Pytester) -> None: + """Symlinked conftest.py are found when pytest is executed in a directory with symlinked + files.""" + real = pytester.mkdir("real") + source = { + "app/test_foo.py": "def test1(fixture): pass", + "app/__init__.py": "", + "app/conftest.py": textwrap.dedent( + """ + import pytest + + print("conftest_loaded") + + @pytest.fixture + def fixture(): + print("fixture_used") + """ + ), + } + pytester.makepyfile(**{"real/%s" % k: v for k, v in source.items()}) + + # Create a build directory that contains symlinks to actual files + # but doesn't symlink actual directories. + build = pytester.mkdir("build") + build.joinpath("app").mkdir() + for f in source: + symlink_or_skip(real.joinpath(f), build.joinpath(f)) + os.chdir(build) + result = pytester.runpytest("-vs", "app/test_foo.py") + result.stdout.fnmatch_lines(["*conftest_loaded*", "PASSED"]) + assert result.ret == ExitCode.OK + + +@pytest.mark.skipif( + os.path.normcase("x") != os.path.normcase("X"), + reason="only relevant for case insensitive file systems", +) +def test_conftest_badcase(pytester: Pytester) -> None: + """Check conftest.py loading when directory casing is wrong (#5792).""" + pytester.path.joinpath("JenkinsRoot/test").mkdir(parents=True) + source = {"setup.py": "", "test/__init__.py": "", "test/conftest.py": ""} + pytester.makepyfile(**{"JenkinsRoot/%s" % k: v for k, v in source.items()}) + + os.chdir(pytester.path.joinpath("jenkinsroot/test")) + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +def test_conftest_uppercase(pytester: Pytester) -> None: + """Check conftest.py whose qualified name contains uppercase characters (#5819)""" + source = {"__init__.py": "", "Foo/conftest.py": "", "Foo/__init__.py": ""} + pytester.makepyfile(**source) + + os.chdir(pytester.path) + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +def test_no_conftest(pytester: Pytester) -> None: + pytester.makeconftest("assert 0") + result = pytester.runpytest("--noconftest") + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + result = pytester.runpytest() + assert result.ret == ExitCode.USAGE_ERROR + + +def test_conftest_existing_junitxml(pytester: Pytester) -> None: + x = pytester.mkdir("tests") + x.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + def pytest_addoption(parser): + parser.addoption("--xyz", action="store_true") + """ + ) + ) + pytester.makefile(ext=".xml", junit="") # Writes junit.xml + result = pytester.runpytest("-h", "--junitxml", "junit.xml") + result.stdout.fnmatch_lines(["*--xyz*"]) + + +def test_conftest_import_order(pytester: Pytester, monkeypatch: MonkeyPatch) -> None: + ct1 = pytester.makeconftest("") + sub = pytester.mkdir("sub") + ct2 = sub / "conftest.py" + ct2.write_text("") + + def impct(p, importmode, root): + return p + + conftest = PytestPluginManager() + conftest._confcutdir = pytester.path + monkeypatch.setattr(conftest, "_importconftest", impct) + mods = cast( + List[Path], + conftest._getconftestmodules(sub, importmode="prepend", rootpath=pytester.path), + ) + expected = [ct1, ct2] + assert mods == expected + + +def test_fixture_dependency(pytester: Pytester) -> None: + pytester.makeconftest("") + pytester.path.joinpath("__init__.py").touch() + sub = pytester.mkdir("sub") + sub.joinpath("__init__.py").touch() + sub.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + + @pytest.fixture + def not_needed(): + assert False, "Should not be called!" + + @pytest.fixture + def foo(): + assert False, "Should not be called!" + + @pytest.fixture + def bar(foo): + return 'bar' + """ + ) + ) + subsub = sub.joinpath("subsub") + subsub.mkdir() + subsub.joinpath("__init__.py").touch() + subsub.joinpath("test_bar.py").write_text( + textwrap.dedent( + """\ + import pytest + + @pytest.fixture + def bar(): + return 'sub bar' + + def test_event_fixture(bar): + assert bar == 'sub bar' + """ + ) + ) + result = pytester.runpytest("sub") + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_conftest_found_with_double_dash(pytester: Pytester) -> None: + sub = pytester.mkdir("sub") + sub.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + def pytest_addoption(parser): + parser.addoption("--hello-world", action="store_true") + """ + ) + ) + p = sub.joinpath("test_hello.py") + p.write_text("def test_hello(): pass") + result = pytester.runpytest(str(p) + "::test_hello", "-h") + result.stdout.fnmatch_lines( + """ + *--hello-world* + """ + ) + + +class TestConftestVisibility: + def _setup_tree(self, pytester: Pytester) -> Dict[str, Path]: # for issue616 + # example mostly taken from: + # https://mail.python.org/pipermail/pytest-dev/2014-September/002617.html + runner = pytester.mkdir("empty") + package = pytester.mkdir("package") + + package.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + @pytest.fixture + def fxtr(): + return "from-package" + """ + ) + ) + package.joinpath("test_pkgroot.py").write_text( + textwrap.dedent( + """\ + def test_pkgroot(fxtr): + assert fxtr == "from-package" + """ + ) + ) + + swc = package.joinpath("swc") + swc.mkdir() + swc.joinpath("__init__.py").touch() + swc.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + @pytest.fixture + def fxtr(): + return "from-swc" + """ + ) + ) + swc.joinpath("test_with_conftest.py").write_text( + textwrap.dedent( + """\ + def test_with_conftest(fxtr): + assert fxtr == "from-swc" + """ + ) + ) + + snc = package.joinpath("snc") + snc.mkdir() + snc.joinpath("__init__.py").touch() + snc.joinpath("test_no_conftest.py").write_text( + textwrap.dedent( + """\ + def test_no_conftest(fxtr): + assert fxtr == "from-package" # No local conftest.py, so should + # use value from parent dir's + """ + ) + ) + print("created directory structure:") + for x in pytester.path.rglob(""): + print(" " + str(x.relative_to(pytester.path))) + + return {"runner": runner, "package": package, "swc": swc, "snc": snc} + + # N.B.: "swc" stands for "subdir with conftest.py" + # "snc" stands for "subdir no [i.e. without] conftest.py" + @pytest.mark.parametrize( + "chdir,testarg,expect_ntests_passed", + [ + # Effective target: package/.. + ("runner", "..", 3), + ("package", "..", 3), + ("swc", "../..", 3), + ("snc", "../..", 3), + # Effective target: package + ("runner", "../package", 3), + ("package", ".", 3), + ("swc", "..", 3), + ("snc", "..", 3), + # Effective target: package/swc + ("runner", "../package/swc", 1), + ("package", "./swc", 1), + ("swc", ".", 1), + ("snc", "../swc", 1), + # Effective target: package/snc + ("runner", "../package/snc", 1), + ("package", "./snc", 1), + ("swc", "../snc", 1), + ("snc", ".", 1), + ], + ) + def test_parsefactories_relative_node_ids( + self, pytester: Pytester, chdir: str, testarg: str, expect_ntests_passed: int + ) -> None: + """#616""" + dirs = self._setup_tree(pytester) + print("pytest run in cwd: %s" % (dirs[chdir].relative_to(pytester.path))) + print("pytestarg : %s" % testarg) + print("expected pass : %s" % expect_ntests_passed) + os.chdir(dirs[chdir]) + reprec = pytester.inline_run(testarg, "-q", "--traceconfig") + reprec.assertoutcome(passed=expect_ntests_passed) + + +@pytest.mark.parametrize( + "confcutdir,passed,error", [(".", 2, 0), ("src", 1, 1), (None, 1, 1)] +) +def test_search_conftest_up_to_inifile( + pytester: Pytester, confcutdir: str, passed: int, error: int +) -> None: + """Test that conftest files are detected only up to an ini file, unless + an explicit --confcutdir option is given. + """ + root = pytester.path + src = root.joinpath("src") + src.mkdir() + src.joinpath("pytest.ini").write_text("[pytest]") + src.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + @pytest.fixture + def fix1(): pass + """ + ) + ) + src.joinpath("test_foo.py").write_text( + textwrap.dedent( + """\ + def test_1(fix1): + pass + def test_2(out_of_reach): + pass + """ + ) + ) + root.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + @pytest.fixture + def out_of_reach(): pass + """ + ) + ) + + args = [str(src)] + if confcutdir: + args = ["--confcutdir=%s" % root.joinpath(confcutdir)] + result = pytester.runpytest(*args) + match = "" + if passed: + match += "*%d passed*" % passed + if error: + match += "*%d error*" % error + result.stdout.fnmatch_lines(match) + + +def test_issue1073_conftest_special_objects(pytester: Pytester) -> None: + pytester.makeconftest( + """\ + class DontTouchMe(object): + def __getattr__(self, x): + raise Exception('cant touch me') + + x = DontTouchMe() + """ + ) + pytester.makepyfile( + """\ + def test_some(): + pass + """ + ) + res = pytester.runpytest() + assert res.ret == 0 + + +def test_conftest_exception_handling(pytester: Pytester) -> None: + pytester.makeconftest( + """\ + raise ValueError() + """ + ) + pytester.makepyfile( + """\ + def test_some(): + pass + """ + ) + res = pytester.runpytest() + assert res.ret == 4 + assert "raise ValueError()" in [line.strip() for line in res.errlines] + + +def test_hook_proxy(pytester: Pytester) -> None: + """Session's gethookproxy() would cache conftests incorrectly (#2016). + It was decided to remove the cache altogether. + """ + pytester.makepyfile( + **{ + "root/demo-0/test_foo1.py": "def test1(): pass", + "root/demo-a/test_foo2.py": "def test1(): pass", + "root/demo-a/conftest.py": """\ + def pytest_ignore_collect(collection_path, config): + return True + """, + "root/demo-b/test_foo3.py": "def test1(): pass", + "root/demo-c/test_foo4.py": "def test1(): pass", + } + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + ["*test_foo1.py*", "*test_foo3.py*", "*test_foo4.py*", "*3 passed*"] + ) + + +def test_required_option_help(pytester: Pytester) -> None: + pytester.makeconftest("assert 0") + x = pytester.mkdir("x") + x.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + def pytest_addoption(parser): + parser.addoption("--xyz", action="store_true", required=True) + """ + ) + ) + result = pytester.runpytest("-h", x) + result.stdout.no_fnmatch_line("*argument --xyz is required*") + assert "general:" in result.stdout.str() diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_debugging.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_debugging.py new file mode 100644 index 0000000000..a822bb57f5 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_debugging.py @@ -0,0 +1,1327 @@ +import os +import sys +from typing import List + +import _pytest._code +import pytest +from _pytest.debugging import _validate_usepdb_cls +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester + +try: + # Type ignored for Python <= 3.6. + breakpoint # type: ignore +except NameError: + SUPPORTS_BREAKPOINT_BUILTIN = False +else: + SUPPORTS_BREAKPOINT_BUILTIN = True + + +_ENVIRON_PYTHONBREAKPOINT = os.environ.get("PYTHONBREAKPOINT", "") + + +@pytest.fixture(autouse=True) +def pdb_env(request): + if "pytester" in request.fixturenames: + # Disable pdb++ with inner tests. + pytester = request.getfixturevalue("pytester") + pytester._monkeypatch.setenv("PDBPP_HIJACK_PDB", "0") + + +def runpdb_and_get_report(pytester: Pytester, source: str): + p = pytester.makepyfile(source) + result = pytester.runpytest_inprocess("--pdb", p) + reports = result.reprec.getreports("pytest_runtest_logreport") # type: ignore[attr-defined] + assert len(reports) == 3, reports # setup/call/teardown + return reports[1] + + +@pytest.fixture +def custom_pdb_calls() -> List[str]: + called = [] + + # install dummy debugger class and track which methods were called on it + class _CustomPdb: + quitting = False + + def __init__(self, *args, **kwargs): + called.append("init") + + def reset(self): + called.append("reset") + + def interaction(self, *args): + called.append("interaction") + + _pytest._CustomPdb = _CustomPdb # type: ignore + return called + + +@pytest.fixture +def custom_debugger_hook(): + called = [] + + # install dummy debugger class and track which methods were called on it + class _CustomDebugger: + def __init__(self, *args, **kwargs): + called.append("init") + + def reset(self): + called.append("reset") + + def interaction(self, *args): + called.append("interaction") + + def set_trace(self, frame): + print("**CustomDebugger**") + called.append("set_trace") + + _pytest._CustomDebugger = _CustomDebugger # type: ignore + yield called + del _pytest._CustomDebugger # type: ignore + + +class TestPDB: + @pytest.fixture + def pdblist(self, request): + monkeypatch = request.getfixturevalue("monkeypatch") + pdblist = [] + + def mypdb(*args): + pdblist.append(args) + + plugin = request.config.pluginmanager.getplugin("debugging") + monkeypatch.setattr(plugin, "post_mortem", mypdb) + return pdblist + + def test_pdb_on_fail(self, pytester: Pytester, pdblist) -> None: + rep = runpdb_and_get_report( + pytester, + """ + def test_func(): + assert 0 + """, + ) + assert rep.failed + assert len(pdblist) == 1 + tb = _pytest._code.Traceback(pdblist[0][0]) + assert tb[-1].name == "test_func" + + def test_pdb_on_xfail(self, pytester: Pytester, pdblist) -> None: + rep = runpdb_and_get_report( + pytester, + """ + import pytest + @pytest.mark.xfail + def test_func(): + assert 0 + """, + ) + assert "xfail" in rep.keywords + assert not pdblist + + def test_pdb_on_skip(self, pytester, pdblist) -> None: + rep = runpdb_and_get_report( + pytester, + """ + import pytest + def test_func(): + pytest.skip("hello") + """, + ) + assert rep.skipped + assert len(pdblist) == 0 + + def test_pdb_on_BdbQuit(self, pytester, pdblist) -> None: + rep = runpdb_and_get_report( + pytester, + """ + import bdb + def test_func(): + raise bdb.BdbQuit + """, + ) + assert rep.failed + assert len(pdblist) == 0 + + def test_pdb_on_KeyboardInterrupt(self, pytester, pdblist) -> None: + rep = runpdb_and_get_report( + pytester, + """ + def test_func(): + raise KeyboardInterrupt + """, + ) + assert rep.failed + assert len(pdblist) == 1 + + @staticmethod + def flush(child): + if child.isalive(): + # Read if the test has not (e.g. test_pdb_unittest_skip). + child.read() + child.wait() + assert not child.isalive() + + def test_pdb_unittest_postmortem(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import unittest + class Blub(unittest.TestCase): + def tearDown(self): + self.filename = None + def test_false(self): + self.filename = 'debug' + '.me' + assert 0 + """ + ) + child = pytester.spawn_pytest(f"--pdb {p1}") + child.expect("Pdb") + child.sendline("p self.filename") + child.sendeof() + rest = child.read().decode("utf8") + assert "debug.me" in rest + self.flush(child) + + def test_pdb_unittest_skip(self, pytester: Pytester) -> None: + """Test for issue #2137""" + p1 = pytester.makepyfile( + """ + import unittest + @unittest.skipIf(True, 'Skipping also with pdb active') + class MyTestCase(unittest.TestCase): + def test_one(self): + assert 0 + """ + ) + child = pytester.spawn_pytest(f"-rs --pdb {p1}") + child.expect("Skipping also with pdb active") + child.expect_exact("= 1 skipped in") + child.sendeof() + self.flush(child) + + def test_pdb_print_captured_stdout_and_stderr(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def test_1(): + import sys + sys.stderr.write("get\\x20rekt") + print("get\\x20rekt") + assert False + + def test_not_called_due_to_quit(): + pass + """ + ) + child = pytester.spawn_pytest("--pdb %s" % p1) + child.expect("captured stdout") + child.expect("get rekt") + child.expect("captured stderr") + child.expect("get rekt") + child.expect("traceback") + child.expect("def test_1") + child.expect("Pdb") + child.sendeof() + rest = child.read().decode("utf8") + assert "Exit: Quitting debugger" in rest + assert "= 1 failed in" in rest + assert "def test_1" not in rest + assert "get rekt" not in rest + self.flush(child) + + def test_pdb_dont_print_empty_captured_stdout_and_stderr( + self, pytester: Pytester + ) -> None: + p1 = pytester.makepyfile( + """ + def test_1(): + assert False + """ + ) + child = pytester.spawn_pytest("--pdb %s" % p1) + child.expect("Pdb") + output = child.before.decode("utf8") + child.sendeof() + assert "captured stdout" not in output + assert "captured stderr" not in output + self.flush(child) + + @pytest.mark.parametrize("showcapture", ["all", "no", "log"]) + def test_pdb_print_captured_logs(self, pytester, showcapture: str) -> None: + p1 = pytester.makepyfile( + """ + def test_1(): + import logging + logging.warn("get " + "rekt") + assert False + """ + ) + child = pytester.spawn_pytest(f"--show-capture={showcapture} --pdb {p1}") + if showcapture in ("all", "log"): + child.expect("captured log") + child.expect("get rekt") + child.expect("Pdb") + child.sendeof() + rest = child.read().decode("utf8") + assert "1 failed" in rest + self.flush(child) + + def test_pdb_print_captured_logs_nologging(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def test_1(): + import logging + logging.warn("get " + "rekt") + assert False + """ + ) + child = pytester.spawn_pytest("--show-capture=all --pdb -p no:logging %s" % p1) + child.expect("get rekt") + output = child.before.decode("utf8") + assert "captured log" not in output + child.expect("Pdb") + child.sendeof() + rest = child.read().decode("utf8") + assert "1 failed" in rest + self.flush(child) + + def test_pdb_interaction_exception(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pytest + def globalfunc(): + pass + def test_1(): + pytest.raises(ValueError, globalfunc) + """ + ) + child = pytester.spawn_pytest("--pdb %s" % p1) + child.expect(".*def test_1") + child.expect(".*pytest.raises.*globalfunc") + child.expect("Pdb") + child.sendline("globalfunc") + child.expect(".*function") + child.sendeof() + child.expect("1 failed") + self.flush(child) + + def test_pdb_interaction_on_collection_issue181(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pytest + xxx + """ + ) + child = pytester.spawn_pytest("--pdb %s" % p1) + # child.expect(".*import pytest.*") + child.expect("Pdb") + child.sendline("c") + child.expect("1 error") + self.flush(child) + + def test_pdb_interaction_on_internal_error(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_runtest_protocol(): + 0/0 + """ + ) + p1 = pytester.makepyfile("def test_func(): pass") + child = pytester.spawn_pytest("--pdb %s" % p1) + child.expect("Pdb") + + # INTERNALERROR is only displayed once via terminal reporter. + assert ( + len( + [ + x + for x in child.before.decode().splitlines() + if x.startswith("INTERNALERROR> Traceback") + ] + ) + == 1 + ) + + child.sendeof() + self.flush(child) + + def test_pdb_prevent_ConftestImportFailure_hiding_exception( + self, pytester: Pytester + ) -> None: + pytester.makepyfile("def test_func(): pass") + sub_dir = pytester.path.joinpath("ns") + sub_dir.mkdir() + sub_dir.joinpath("conftest").with_suffix(".py").write_text( + "import unknown", "utf-8" + ) + sub_dir.joinpath("test_file").with_suffix(".py").write_text( + "def test_func(): pass", "utf-8" + ) + + result = pytester.runpytest_subprocess("--pdb", ".") + result.stdout.fnmatch_lines(["-> import unknown"]) + + def test_pdb_interaction_capturing_simple(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pytest + def test_1(): + i = 0 + print("hello17") + pytest.set_trace() + i == 1 + assert 0 + """ + ) + child = pytester.spawn_pytest(str(p1)) + child.expect(r"test_1\(\)") + child.expect("i == 1") + child.expect("Pdb") + child.sendline("c") + rest = child.read().decode("utf-8") + assert "AssertionError" in rest + assert "1 failed" in rest + assert "def test_1" in rest + assert "hello17" in rest # out is captured + self.flush(child) + + def test_pdb_set_trace_kwargs(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pytest + def test_1(): + i = 0 + print("hello17") + pytest.set_trace(header="== my_header ==") + x = 3 + assert 0 + """ + ) + child = pytester.spawn_pytest(str(p1)) + child.expect("== my_header ==") + assert "PDB set_trace" not in child.before.decode() + child.expect("Pdb") + child.sendline("c") + rest = child.read().decode("utf-8") + assert "1 failed" in rest + assert "def test_1" in rest + assert "hello17" in rest # out is captured + self.flush(child) + + def test_pdb_set_trace_interception(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pdb + def test_1(): + pdb.set_trace() + """ + ) + child = pytester.spawn_pytest(str(p1)) + child.expect("test_1") + child.expect("Pdb") + child.sendline("q") + rest = child.read().decode("utf8") + assert "no tests ran" in rest + assert "reading from stdin while output" not in rest + assert "BdbQuit" not in rest + self.flush(child) + + def test_pdb_and_capsys(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pytest + def test_1(capsys): + print("hello1") + pytest.set_trace() + """ + ) + child = pytester.spawn_pytest(str(p1)) + child.expect("test_1") + child.send("capsys.readouterr()\n") + child.expect("hello1") + child.sendeof() + child.read() + self.flush(child) + + def test_pdb_with_caplog_on_pdb_invocation(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def test_1(capsys, caplog): + import logging + logging.getLogger(__name__).warning("some_warning") + assert 0 + """ + ) + child = pytester.spawn_pytest("--pdb %s" % str(p1)) + child.send("caplog.record_tuples\n") + child.expect_exact( + "[('test_pdb_with_caplog_on_pdb_invocation', 30, 'some_warning')]" + ) + child.sendeof() + child.read() + self.flush(child) + + def test_set_trace_capturing_afterwards(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pdb + def test_1(): + pdb.set_trace() + def test_2(): + print("hello") + assert 0 + """ + ) + child = pytester.spawn_pytest(str(p1)) + child.expect("test_1") + child.send("c\n") + child.expect("test_2") + child.expect("Captured") + child.expect("hello") + child.sendeof() + child.read() + self.flush(child) + + def test_pdb_interaction_doctest(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def function_1(): + ''' + >>> i = 0 + >>> assert i == 1 + ''' + """ + ) + child = pytester.spawn_pytest("--doctest-modules --pdb %s" % p1) + child.expect("Pdb") + + assert "UNEXPECTED EXCEPTION: AssertionError()" in child.before.decode("utf8") + + child.sendline("'i=%i.' % i") + child.expect("Pdb") + assert "\r\n'i=0.'\r\n" in child.before.decode("utf8") + + child.sendeof() + rest = child.read().decode("utf8") + assert "! _pytest.outcomes.Exit: Quitting debugger !" in rest + assert "BdbQuit" not in rest + assert "1 failed" in rest + self.flush(child) + + def test_doctest_set_trace_quit(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def function_1(): + ''' + >>> __import__('pdb').set_trace() + ''' + """ + ) + # NOTE: does not use pytest.set_trace, but Python's patched pdb, + # therefore "-s" is required. + child = pytester.spawn_pytest("--doctest-modules --pdb -s %s" % p1) + child.expect("Pdb") + child.sendline("q") + rest = child.read().decode("utf8") + + assert "! _pytest.outcomes.Exit: Quitting debugger !" in rest + assert "= no tests ran in" in rest + assert "BdbQuit" not in rest + assert "UNEXPECTED EXCEPTION" not in rest + + def test_pdb_interaction_capturing_twice(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pytest + def test_1(): + i = 0 + print("hello17") + pytest.set_trace() + x = 3 + print("hello18") + pytest.set_trace() + x = 4 + assert 0 + """ + ) + child = pytester.spawn_pytest(str(p1)) + child.expect(r"PDB set_trace \(IO-capturing turned off\)") + child.expect("test_1") + child.expect("x = 3") + child.expect("Pdb") + child.sendline("c") + child.expect(r"PDB continue \(IO-capturing resumed\)") + child.expect(r"PDB set_trace \(IO-capturing turned off\)") + child.expect("x = 4") + child.expect("Pdb") + child.sendline("c") + child.expect("_ test_1 _") + child.expect("def test_1") + rest = child.read().decode("utf8") + assert "Captured stdout call" in rest + assert "hello17" in rest # out is captured + assert "hello18" in rest # out is captured + assert "1 failed" in rest + self.flush(child) + + def test_pdb_with_injected_do_debug(self, pytester: Pytester) -> None: + """Simulates pdbpp, which injects Pdb into do_debug, and uses + self.__class__ in do_continue. + """ + p1 = pytester.makepyfile( + mytest=""" + import pdb + import pytest + + count_continue = 0 + + class CustomPdb(pdb.Pdb, object): + def do_debug(self, arg): + import sys + import types + + do_debug_func = pdb.Pdb.do_debug + + newglobals = do_debug_func.__globals__.copy() + newglobals['Pdb'] = self.__class__ + orig_do_debug = types.FunctionType( + do_debug_func.__code__, newglobals, + do_debug_func.__name__, do_debug_func.__defaults__, + ) + return orig_do_debug(self, arg) + do_debug.__doc__ = pdb.Pdb.do_debug.__doc__ + + def do_continue(self, *args, **kwargs): + global count_continue + count_continue += 1 + return super(CustomPdb, self).do_continue(*args, **kwargs) + + def foo(): + print("print_from_foo") + + def test_1(): + i = 0 + print("hello17") + pytest.set_trace() + x = 3 + print("hello18") + + assert count_continue == 2, "unexpected_failure: %d != 2" % count_continue + pytest.fail("expected_failure") + """ + ) + child = pytester.spawn_pytest("--pdbcls=mytest:CustomPdb %s" % str(p1)) + child.expect(r"PDB set_trace \(IO-capturing turned off\)") + child.expect(r"\n\(Pdb") + child.sendline("debug foo()") + child.expect("ENTERING RECURSIVE DEBUGGER") + child.expect(r"\n\(\(Pdb") + child.sendline("c") + child.expect("LEAVING RECURSIVE DEBUGGER") + assert b"PDB continue" not in child.before + # No extra newline. + assert child.before.endswith(b"c\r\nprint_from_foo\r\n") + + # set_debug should not raise outcomes. Exit, if used recursively. + child.sendline("debug 42") + child.sendline("q") + child.expect("LEAVING RECURSIVE DEBUGGER") + assert b"ENTERING RECURSIVE DEBUGGER" in child.before + assert b"Quitting debugger" not in child.before + + child.sendline("c") + child.expect(r"PDB continue \(IO-capturing resumed\)") + rest = child.read().decode("utf8") + assert "hello17" in rest # out is captured + assert "hello18" in rest # out is captured + assert "1 failed" in rest + assert "Failed: expected_failure" in rest + assert "AssertionError: unexpected_failure" not in rest + self.flush(child) + + def test_pdb_without_capture(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pytest + def test_1(): + pytest.set_trace() + """ + ) + child = pytester.spawn_pytest("-s %s" % p1) + child.expect(r">>> PDB set_trace >>>") + child.expect("Pdb") + child.sendline("c") + child.expect(r">>> PDB continue >>>") + child.expect("1 passed") + self.flush(child) + + @pytest.mark.parametrize("capture_arg", ("", "-s", "-p no:capture")) + def test_pdb_continue_with_recursive_debug( + self, capture_arg, pytester: Pytester + ) -> None: + """Full coverage for do_debug without capturing. + + This is very similar to test_pdb_interaction_continue_recursive in general, + but mocks out ``pdb.set_trace`` for providing more coverage. + """ + p1 = pytester.makepyfile( + """ + try: + input = raw_input + except NameError: + pass + + def set_trace(): + __import__('pdb').set_trace() + + def test_1(monkeypatch): + import _pytest.debugging + + class pytestPDBTest(_pytest.debugging.pytestPDB): + @classmethod + def set_trace(cls, *args, **kwargs): + # Init PytestPdbWrapper to handle capturing. + _pdb = cls._init_pdb("set_trace", *args, **kwargs) + + # Mock out pdb.Pdb.do_continue. + import pdb + pdb.Pdb.do_continue = lambda self, arg: None + + print("===" + " SET_TRACE ===") + assert input() == "debug set_trace()" + + # Simulate PytestPdbWrapper.do_debug + cls._recursive_debug += 1 + print("ENTERING RECURSIVE DEBUGGER") + print("===" + " SET_TRACE_2 ===") + + assert input() == "c" + _pdb.do_continue("") + print("===" + " SET_TRACE_3 ===") + + # Simulate PytestPdbWrapper.do_debug + print("LEAVING RECURSIVE DEBUGGER") + cls._recursive_debug -= 1 + + print("===" + " SET_TRACE_4 ===") + assert input() == "c" + _pdb.do_continue("") + + def do_continue(self, arg): + print("=== do_continue") + + monkeypatch.setattr(_pytest.debugging, "pytestPDB", pytestPDBTest) + + import pdb + monkeypatch.setattr(pdb, "set_trace", pytestPDBTest.set_trace) + + set_trace() + """ + ) + child = pytester.spawn_pytest(f"--tb=short {p1} {capture_arg}") + child.expect("=== SET_TRACE ===") + before = child.before.decode("utf8") + if not capture_arg: + assert ">>> PDB set_trace (IO-capturing turned off) >>>" in before + else: + assert ">>> PDB set_trace >>>" in before + child.sendline("debug set_trace()") + child.expect("=== SET_TRACE_2 ===") + before = child.before.decode("utf8") + assert "\r\nENTERING RECURSIVE DEBUGGER\r\n" in before + child.sendline("c") + child.expect("=== SET_TRACE_3 ===") + + # No continue message with recursive debugging. + before = child.before.decode("utf8") + assert ">>> PDB continue " not in before + + child.sendline("c") + child.expect("=== SET_TRACE_4 ===") + before = child.before.decode("utf8") + assert "\r\nLEAVING RECURSIVE DEBUGGER\r\n" in before + child.sendline("c") + rest = child.read().decode("utf8") + if not capture_arg: + assert "> PDB continue (IO-capturing resumed) >" in rest + else: + assert "> PDB continue >" in rest + assert "= 1 passed in" in rest + + def test_pdb_used_outside_test(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pytest + pytest.set_trace() + x = 5 + """ + ) + child = pytester.spawn(f"{sys.executable} {p1}") + child.expect("x = 5") + child.expect("Pdb") + child.sendeof() + self.flush(child) + + def test_pdb_used_in_generate_tests(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pytest + def pytest_generate_tests(metafunc): + pytest.set_trace() + x = 5 + def test_foo(a): + pass + """ + ) + child = pytester.spawn_pytest(str(p1)) + child.expect("x = 5") + child.expect("Pdb") + child.sendeof() + self.flush(child) + + def test_pdb_collection_failure_is_shown(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile("xxx") + result = pytester.runpytest_subprocess("--pdb", p1) + result.stdout.fnmatch_lines( + ["E NameError: *xxx*", "*! *Exit: Quitting debugger !*"] # due to EOF + ) + + @pytest.mark.parametrize("post_mortem", (False, True)) + def test_enter_leave_pdb_hooks_are_called( + self, post_mortem, pytester: Pytester + ) -> None: + pytester.makeconftest( + """ + mypdb = None + + def pytest_configure(config): + config.testing_verification = 'configured' + + def pytest_enter_pdb(config, pdb): + assert config.testing_verification == 'configured' + print('enter_pdb_hook') + + global mypdb + mypdb = pdb + mypdb.set_attribute = "bar" + + def pytest_leave_pdb(config, pdb): + assert config.testing_verification == 'configured' + print('leave_pdb_hook') + + global mypdb + assert mypdb is pdb + assert mypdb.set_attribute == "bar" + """ + ) + p1 = pytester.makepyfile( + """ + import pytest + + def test_set_trace(): + pytest.set_trace() + assert 0 + + def test_post_mortem(): + assert 0 + """ + ) + if post_mortem: + child = pytester.spawn_pytest(str(p1) + " --pdb -s -k test_post_mortem") + else: + child = pytester.spawn_pytest(str(p1) + " -k test_set_trace") + child.expect("enter_pdb_hook") + child.sendline("c") + if post_mortem: + child.expect(r"PDB continue") + else: + child.expect(r"PDB continue \(IO-capturing resumed\)") + child.expect("Captured stdout call") + rest = child.read().decode("utf8") + assert "leave_pdb_hook" in rest + assert "1 failed" in rest + self.flush(child) + + def test_pdb_custom_cls( + self, pytester: Pytester, custom_pdb_calls: List[str] + ) -> None: + p1 = pytester.makepyfile("""xxx """) + result = pytester.runpytest_inprocess( + "--pdb", "--pdbcls=_pytest:_CustomPdb", p1 + ) + result.stdout.fnmatch_lines(["*NameError*xxx*", "*1 error*"]) + assert custom_pdb_calls == ["init", "reset", "interaction"] + + def test_pdb_custom_cls_invalid(self, pytester: Pytester) -> None: + result = pytester.runpytest_inprocess("--pdbcls=invalid") + result.stderr.fnmatch_lines( + [ + "*: error: argument --pdbcls: 'invalid' is not in the format 'modname:classname'" + ] + ) + + def test_pdb_validate_usepdb_cls(self): + assert _validate_usepdb_cls("os.path:dirname.__name__") == ( + "os.path", + "dirname.__name__", + ) + + assert _validate_usepdb_cls("pdb:DoesNotExist") == ("pdb", "DoesNotExist") + + def test_pdb_custom_cls_without_pdb( + self, pytester: Pytester, custom_pdb_calls: List[str] + ) -> None: + p1 = pytester.makepyfile("""xxx """) + result = pytester.runpytest_inprocess("--pdbcls=_pytest:_CustomPdb", p1) + result.stdout.fnmatch_lines(["*NameError*xxx*", "*1 error*"]) + assert custom_pdb_calls == [] + + def test_pdb_custom_cls_with_set_trace( + self, + pytester: Pytester, + monkeypatch: MonkeyPatch, + ) -> None: + pytester.makepyfile( + custom_pdb=""" + class CustomPdb(object): + def __init__(self, *args, **kwargs): + skip = kwargs.pop("skip") + assert skip == ["foo.*"] + print("__init__") + super(CustomPdb, self).__init__(*args, **kwargs) + + def set_trace(*args, **kwargs): + print('custom set_trace>') + """ + ) + p1 = pytester.makepyfile( + """ + import pytest + + def test_foo(): + pytest.set_trace(skip=['foo.*']) + """ + ) + monkeypatch.setenv("PYTHONPATH", str(pytester.path)) + child = pytester.spawn_pytest("--pdbcls=custom_pdb:CustomPdb %s" % str(p1)) + + child.expect("__init__") + child.expect("custom set_trace>") + self.flush(child) + + +class TestDebuggingBreakpoints: + def test_supports_breakpoint_module_global(self) -> None: + """Test that supports breakpoint global marks on Python 3.7+.""" + if sys.version_info >= (3, 7): + assert SUPPORTS_BREAKPOINT_BUILTIN is True + + @pytest.mark.skipif( + not SUPPORTS_BREAKPOINT_BUILTIN, reason="Requires breakpoint() builtin" + ) + @pytest.mark.parametrize("arg", ["--pdb", ""]) + def test_sys_breakpointhook_configure_and_unconfigure( + self, pytester: Pytester, arg: str + ) -> None: + """ + Test that sys.breakpointhook is set to the custom Pdb class once configured, test that + hook is reset to system value once pytest has been unconfigured + """ + pytester.makeconftest( + """ + import sys + from pytest import hookimpl + from _pytest.debugging import pytestPDB + + def pytest_configure(config): + config.add_cleanup(check_restored) + + def check_restored(): + assert sys.breakpointhook == sys.__breakpointhook__ + + def test_check(): + assert sys.breakpointhook == pytestPDB.set_trace + """ + ) + pytester.makepyfile( + """ + def test_nothing(): pass + """ + ) + args = (arg,) if arg else () + result = pytester.runpytest_subprocess(*args) + result.stdout.fnmatch_lines(["*1 passed in *"]) + + @pytest.mark.skipif( + not SUPPORTS_BREAKPOINT_BUILTIN, reason="Requires breakpoint() builtin" + ) + def test_pdb_custom_cls(self, pytester: Pytester, custom_debugger_hook) -> None: + p1 = pytester.makepyfile( + """ + def test_nothing(): + breakpoint() + """ + ) + result = pytester.runpytest_inprocess( + "--pdb", "--pdbcls=_pytest:_CustomDebugger", p1 + ) + result.stdout.fnmatch_lines(["*CustomDebugger*", "*1 passed*"]) + assert custom_debugger_hook == ["init", "set_trace"] + + @pytest.mark.parametrize("arg", ["--pdb", ""]) + @pytest.mark.skipif( + not SUPPORTS_BREAKPOINT_BUILTIN, reason="Requires breakpoint() builtin" + ) + def test_environ_custom_class( + self, pytester: Pytester, custom_debugger_hook, arg: str + ) -> None: + pytester.makeconftest( + """ + import os + import sys + + os.environ['PYTHONBREAKPOINT'] = '_pytest._CustomDebugger.set_trace' + + def pytest_configure(config): + config.add_cleanup(check_restored) + + def check_restored(): + assert sys.breakpointhook == sys.__breakpointhook__ + + def test_check(): + import _pytest + assert sys.breakpointhook is _pytest._CustomDebugger.set_trace + """ + ) + pytester.makepyfile( + """ + def test_nothing(): pass + """ + ) + args = (arg,) if arg else () + result = pytester.runpytest_subprocess(*args) + result.stdout.fnmatch_lines(["*1 passed in *"]) + + @pytest.mark.skipif( + not SUPPORTS_BREAKPOINT_BUILTIN, reason="Requires breakpoint() builtin" + ) + @pytest.mark.skipif( + not _ENVIRON_PYTHONBREAKPOINT == "", + reason="Requires breakpoint() default value", + ) + def test_sys_breakpoint_interception(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def test_1(): + breakpoint() + """ + ) + child = pytester.spawn_pytest(str(p1)) + child.expect("test_1") + child.expect("Pdb") + child.sendline("quit") + rest = child.read().decode("utf8") + assert "Quitting debugger" in rest + assert "reading from stdin while output" not in rest + TestPDB.flush(child) + + @pytest.mark.skipif( + not SUPPORTS_BREAKPOINT_BUILTIN, reason="Requires breakpoint() builtin" + ) + def test_pdb_not_altered(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pdb + def test_1(): + pdb.set_trace() + assert 0 + """ + ) + child = pytester.spawn_pytest(str(p1)) + child.expect("test_1") + child.expect("Pdb") + child.sendline("c") + rest = child.read().decode("utf8") + assert "1 failed" in rest + assert "reading from stdin while output" not in rest + TestPDB.flush(child) + + +class TestTraceOption: + def test_trace_sets_breakpoint(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def test_1(): + assert True + + def test_2(): + pass + + def test_3(): + pass + """ + ) + child = pytester.spawn_pytest("--trace " + str(p1)) + child.expect("test_1") + child.expect("Pdb") + child.sendline("c") + child.expect("test_2") + child.expect("Pdb") + child.sendline("c") + child.expect("test_3") + child.expect("Pdb") + child.sendline("q") + child.expect_exact("Exit: Quitting debugger") + rest = child.read().decode("utf8") + assert "= 2 passed in" in rest + assert "reading from stdin while output" not in rest + # Only printed once - not on stderr. + assert "Exit: Quitting debugger" not in child.before.decode("utf8") + TestPDB.flush(child) + + def test_trace_with_parametrize_handles_shared_fixtureinfo( + self, pytester: Pytester + ) -> None: + p1 = pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize('myparam', [1,2]) + def test_1(myparam, request): + assert myparam in (1, 2) + assert request.function.__name__ == "test_1" + @pytest.mark.parametrize('func', [1,2]) + def test_func(func, request): + assert func in (1, 2) + assert request.function.__name__ == "test_func" + @pytest.mark.parametrize('myparam', [1,2]) + def test_func_kw(myparam, request, func="func_kw"): + assert myparam in (1, 2) + assert func == "func_kw" + assert request.function.__name__ == "test_func_kw" + """ + ) + child = pytester.spawn_pytest("--trace " + str(p1)) + for func, argname in [ + ("test_1", "myparam"), + ("test_func", "func"), + ("test_func_kw", "myparam"), + ]: + child.expect_exact("> PDB runcall (IO-capturing turned off) >") + child.expect_exact(func) + child.expect_exact("Pdb") + child.sendline("args") + child.expect_exact(f"{argname} = 1\r\n") + child.expect_exact("Pdb") + child.sendline("c") + child.expect_exact("Pdb") + child.sendline("args") + child.expect_exact(f"{argname} = 2\r\n") + child.expect_exact("Pdb") + child.sendline("c") + child.expect_exact("> PDB continue (IO-capturing resumed) >") + rest = child.read().decode("utf8") + assert "= 6 passed in" in rest + assert "reading from stdin while output" not in rest + # Only printed once - not on stderr. + assert "Exit: Quitting debugger" not in child.before.decode("utf8") + TestPDB.flush(child) + + +def test_trace_after_runpytest(pytester: Pytester) -> None: + """Test that debugging's pytest_configure is re-entrant.""" + p1 = pytester.makepyfile( + """ + from _pytest.debugging import pytestPDB + + def test_outer(pytester) -> None: + assert len(pytestPDB._saved) == 1 + + pytester.makepyfile( + \""" + from _pytest.debugging import pytestPDB + + def test_inner(): + assert len(pytestPDB._saved) == 2 + print() + print("test_inner_" + "end") + \""" + ) + + result = pytester.runpytest("-s", "-k", "test_inner") + assert result.ret == 0 + + assert len(pytestPDB._saved) == 1 + """ + ) + result = pytester.runpytest_subprocess("-s", "-p", "pytester", str(p1)) + result.stdout.fnmatch_lines(["test_inner_end"]) + assert result.ret == 0 + + +def test_quit_with_swallowed_SystemExit(pytester: Pytester) -> None: + """Test that debugging's pytest_configure is re-entrant.""" + p1 = pytester.makepyfile( + """ + def call_pdb_set_trace(): + __import__('pdb').set_trace() + + + def test_1(): + try: + call_pdb_set_trace() + except SystemExit: + pass + + + def test_2(): + pass + """ + ) + child = pytester.spawn_pytest(str(p1)) + child.expect("Pdb") + child.sendline("q") + child.expect_exact("Exit: Quitting debugger") + rest = child.read().decode("utf8") + assert "no tests ran" in rest + TestPDB.flush(child) + + +@pytest.mark.parametrize("fixture", ("capfd", "capsys")) +def test_pdb_suspends_fixture_capturing(pytester: Pytester, fixture: str) -> None: + """Using "-s" with pytest should suspend/resume fixture capturing.""" + p1 = pytester.makepyfile( + """ + def test_inner({fixture}): + import sys + + print("out_inner_before") + sys.stderr.write("err_inner_before\\n") + + __import__("pdb").set_trace() + + print("out_inner_after") + sys.stderr.write("err_inner_after\\n") + + out, err = {fixture}.readouterr() + assert out =="out_inner_before\\nout_inner_after\\n" + assert err =="err_inner_before\\nerr_inner_after\\n" + """.format( + fixture=fixture + ) + ) + + child = pytester.spawn_pytest(str(p1) + " -s") + + child.expect("Pdb") + before = child.before.decode("utf8") + assert ( + "> PDB set_trace (IO-capturing turned off for fixture %s) >" % (fixture) + in before + ) + + # Test that capturing is really suspended. + child.sendline("p 40 + 2") + child.expect("Pdb") + assert "\r\n42\r\n" in child.before.decode("utf8") + + child.sendline("c") + rest = child.read().decode("utf8") + assert "out_inner" not in rest + assert "err_inner" not in rest + + TestPDB.flush(child) + assert child.exitstatus == 0 + assert "= 1 passed in" in rest + assert "> PDB continue (IO-capturing resumed for fixture %s) >" % (fixture) in rest + + +def test_pdbcls_via_local_module(pytester: Pytester) -> None: + """It should be imported in pytest_configure or later only.""" + p1 = pytester.makepyfile( + """ + def test(): + print("before_set_trace") + __import__("pdb").set_trace() + """, + mypdb=""" + class Wrapped: + class MyPdb: + def set_trace(self, *args): + print("set_trace_called", args) + + def runcall(self, *args, **kwds): + print("runcall_called", args, kwds) + """, + ) + result = pytester.runpytest( + str(p1), "--pdbcls=really.invalid:Value", syspathinsert=True + ) + result.stdout.fnmatch_lines( + [ + "*= FAILURES =*", + "E * --pdbcls: could not import 'really.invalid:Value': No module named *really*", + ] + ) + assert result.ret == 1 + + result = pytester.runpytest( + str(p1), "--pdbcls=mypdb:Wrapped.MyPdb", syspathinsert=True + ) + assert result.ret == 0 + result.stdout.fnmatch_lines(["*set_trace_called*", "* 1 passed in *"]) + + # Ensure that it also works with --trace. + result = pytester.runpytest( + str(p1), "--pdbcls=mypdb:Wrapped.MyPdb", "--trace", syspathinsert=True + ) + assert result.ret == 0 + result.stdout.fnmatch_lines(["*runcall_called*", "* 1 passed in *"]) + + +def test_raises_bdbquit_with_eoferror(pytester: Pytester) -> None: + """It is not guaranteed that DontReadFromInput's read is called.""" + + p1 = pytester.makepyfile( + """ + def input_without_read(*args, **kwargs): + raise EOFError() + + def test(monkeypatch): + import builtins + monkeypatch.setattr(builtins, "input", input_without_read) + __import__('pdb').set_trace() + """ + ) + result = pytester.runpytest(str(p1)) + result.stdout.fnmatch_lines(["E *BdbQuit", "*= 1 failed in*"]) + assert result.ret == 1 + + +def test_pdb_wrapper_class_is_reused(pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def test(): + __import__("pdb").set_trace() + __import__("pdb").set_trace() + + import mypdb + instances = mypdb.instances + assert len(instances) == 2 + assert instances[0].__class__ is instances[1].__class__ + """, + mypdb=""" + instances = [] + + class MyPdb: + def __init__(self, *args, **kwargs): + instances.append(self) + + def set_trace(self, *args): + print("set_trace_called", args) + """, + ) + result = pytester.runpytest(str(p1), "--pdbcls=mypdb:MyPdb", syspathinsert=True) + assert result.ret == 0 + result.stdout.fnmatch_lines( + ["*set_trace_called*", "*set_trace_called*", "* 1 passed in *"] + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_doctest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_doctest.py new file mode 100644 index 0000000000..67b8ccdb7e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_doctest.py @@ -0,0 +1,1572 @@ +import inspect +import sys +import textwrap +from pathlib import Path +from typing import Callable +from typing import Optional + +import pytest +from _pytest.doctest import _get_checker +from _pytest.doctest import _is_main_py +from _pytest.doctest import _is_mocked +from _pytest.doctest import _is_setup_py +from _pytest.doctest import _patch_unwrap_mock_aware +from _pytest.doctest import DoctestItem +from _pytest.doctest import DoctestModule +from _pytest.doctest import DoctestTextfile +from _pytest.pytester import Pytester + + +class TestDoctests: + def test_collect_testtextfile(self, pytester: Pytester): + w = pytester.maketxtfile(whatever="") + checkfile = pytester.maketxtfile( + test_something=""" + alskdjalsdk + >>> i = 5 + >>> i-1 + 4 + """ + ) + + for x in (pytester.path, checkfile): + # print "checking that %s returns custom items" % (x,) + items, reprec = pytester.inline_genitems(x) + assert len(items) == 1 + assert isinstance(items[0], DoctestItem) + assert isinstance(items[0].parent, DoctestTextfile) + # Empty file has no items. + items, reprec = pytester.inline_genitems(w) + assert len(items) == 0 + + def test_collect_module_empty(self, pytester: Pytester): + path = pytester.makepyfile(whatever="#") + for p in (path, pytester.path): + items, reprec = pytester.inline_genitems(p, "--doctest-modules") + assert len(items) == 0 + + def test_collect_module_single_modulelevel_doctest(self, pytester: Pytester): + path = pytester.makepyfile(whatever='""">>> pass"""') + for p in (path, pytester.path): + items, reprec = pytester.inline_genitems(p, "--doctest-modules") + assert len(items) == 1 + assert isinstance(items[0], DoctestItem) + assert isinstance(items[0].parent, DoctestModule) + + def test_collect_module_two_doctest_one_modulelevel(self, pytester: Pytester): + path = pytester.makepyfile( + whatever=""" + '>>> x = None' + def my_func(): + ">>> magic = 42 " + """ + ) + for p in (path, pytester.path): + items, reprec = pytester.inline_genitems(p, "--doctest-modules") + assert len(items) == 2 + assert isinstance(items[0], DoctestItem) + assert isinstance(items[1], DoctestItem) + assert isinstance(items[0].parent, DoctestModule) + assert items[0].parent is items[1].parent + + @pytest.mark.parametrize("filename", ["__init__", "whatever"]) + def test_collect_module_two_doctest_no_modulelevel( + self, + pytester: Pytester, + filename: str, + ) -> None: + path = pytester.makepyfile( + **{ + filename: """ + '# Empty' + def my_func(): + ">>> magic = 42 " + def useless(): + ''' + # This is a function + # >>> # it doesn't have any doctest + ''' + def another(): + ''' + # This is another function + >>> import os # this one does have a doctest + ''' + """, + }, + ) + for p in (path, pytester.path): + items, reprec = pytester.inline_genitems(p, "--doctest-modules") + assert len(items) == 2 + assert isinstance(items[0], DoctestItem) + assert isinstance(items[1], DoctestItem) + assert isinstance(items[0].parent, DoctestModule) + assert items[0].parent is items[1].parent + + def test_simple_doctestfile(self, pytester: Pytester): + p = pytester.maketxtfile( + test_doc=""" + >>> x = 1 + >>> x == 1 + False + """ + ) + reprec = pytester.inline_run(p) + reprec.assertoutcome(failed=1) + + def test_new_pattern(self, pytester: Pytester): + p = pytester.maketxtfile( + xdoc=""" + >>> x = 1 + >>> x == 1 + False + """ + ) + reprec = pytester.inline_run(p, "--doctest-glob=x*.txt") + reprec.assertoutcome(failed=1) + + def test_multiple_patterns(self, pytester: Pytester): + """Test support for multiple --doctest-glob arguments (#1255).""" + pytester.maketxtfile( + xdoc=""" + >>> 1 + 1 + """ + ) + pytester.makefile( + ".foo", + test=""" + >>> 1 + 1 + """, + ) + pytester.maketxtfile( + test_normal=""" + >>> 1 + 1 + """ + ) + expected = {"xdoc.txt", "test.foo", "test_normal.txt"} + assert {x.name for x in pytester.path.iterdir()} == expected + args = ["--doctest-glob=xdoc*.txt", "--doctest-glob=*.foo"] + result = pytester.runpytest(*args) + result.stdout.fnmatch_lines(["*test.foo *", "*xdoc.txt *", "*2 passed*"]) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*test_normal.txt *", "*1 passed*"]) + + @pytest.mark.parametrize( + " test_string, encoding", + [("foo", "ascii"), ("öäü", "latin1"), ("öäü", "utf-8")], + ) + def test_encoding(self, pytester, test_string, encoding): + """Test support for doctest_encoding ini option.""" + pytester.makeini( + """ + [pytest] + doctest_encoding={} + """.format( + encoding + ) + ) + doctest = """ + >>> "{}" + {} + """.format( + test_string, repr(test_string) + ) + fn = pytester.path / "test_encoding.txt" + fn.write_text(doctest, encoding=encoding) + + result = pytester.runpytest() + + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_doctest_unexpected_exception(self, pytester: Pytester): + pytester.maketxtfile( + """ + >>> i = 0 + >>> 0 / i + 2 + """ + ) + result = pytester.runpytest("--doctest-modules") + result.stdout.fnmatch_lines( + [ + "test_doctest_unexpected_exception.txt F *", + "", + "*= FAILURES =*", + "*_ [[]doctest[]] test_doctest_unexpected_exception.txt _*", + "001 >>> i = 0", + "002 >>> 0 / i", + "UNEXPECTED EXCEPTION: ZeroDivisionError*", + "Traceback (most recent call last):", + ' File "*/doctest.py", line *, in __run', + " *", + *((" *^^^^*",) if sys.version_info >= (3, 11) else ()), + ' File "", line 1, in ', + "ZeroDivisionError: division by zero", + "*/test_doctest_unexpected_exception.txt:2: UnexpectedException", + ], + consecutive=True, + ) + + def test_doctest_outcomes(self, pytester: Pytester): + pytester.maketxtfile( + test_skip=""" + >>> 1 + 1 + >>> import pytest + >>> pytest.skip("") + >>> 2 + 3 + """, + test_xfail=""" + >>> import pytest + >>> pytest.xfail("xfail_reason") + >>> foo + bar + """, + test_importorskip=""" + >>> import pytest + >>> pytest.importorskip("doesnotexist") + >>> foo + bar + """, + ) + result = pytester.runpytest("--doctest-modules") + result.stdout.fnmatch_lines( + [ + "collected 3 items", + "", + "test_importorskip.txt s *", + "test_skip.txt s *", + "test_xfail.txt x *", + "", + "*= 2 skipped, 1 xfailed in *", + ] + ) + + def test_docstring_partial_context_around_error(self, pytester: Pytester): + """Test that we show some context before the actual line of a failing + doctest. + """ + pytester.makepyfile( + ''' + def foo(): + """ + text-line-1 + text-line-2 + text-line-3 + text-line-4 + text-line-5 + text-line-6 + text-line-7 + text-line-8 + text-line-9 + text-line-10 + text-line-11 + >>> 1 + 1 + 3 + + text-line-after + """ + ''' + ) + result = pytester.runpytest("--doctest-modules") + result.stdout.fnmatch_lines( + [ + "*docstring_partial_context_around_error*", + "005*text-line-3", + "006*text-line-4", + "013*text-line-11", + "014*>>> 1 + 1", + "Expected:", + " 3", + "Got:", + " 2", + ] + ) + # lines below should be trimmed out + result.stdout.no_fnmatch_line("*text-line-2*") + result.stdout.no_fnmatch_line("*text-line-after*") + + def test_docstring_full_context_around_error(self, pytester: Pytester): + """Test that we show the whole context before the actual line of a failing + doctest, provided that the context is up to 10 lines long. + """ + pytester.makepyfile( + ''' + def foo(): + """ + text-line-1 + text-line-2 + + >>> 1 + 1 + 3 + """ + ''' + ) + result = pytester.runpytest("--doctest-modules") + result.stdout.fnmatch_lines( + [ + "*docstring_full_context_around_error*", + "003*text-line-1", + "004*text-line-2", + "006*>>> 1 + 1", + "Expected:", + " 3", + "Got:", + " 2", + ] + ) + + def test_doctest_linedata_missing(self, pytester: Pytester): + pytester.path.joinpath("hello.py").write_text( + textwrap.dedent( + """\ + class Fun(object): + @property + def test(self): + ''' + >>> a = 1 + >>> 1/0 + ''' + """ + ) + ) + result = pytester.runpytest("--doctest-modules") + result.stdout.fnmatch_lines( + ["*hello*", "006*>>> 1/0*", "*UNEXPECTED*ZeroDivision*", "*1 failed*"] + ) + + def test_doctest_linedata_on_property(self, pytester: Pytester): + pytester.makepyfile( + """ + class Sample(object): + @property + def some_property(self): + ''' + >>> Sample().some_property + 'another thing' + ''' + return 'something' + """ + ) + result = pytester.runpytest("--doctest-modules") + result.stdout.fnmatch_lines( + [ + "*= FAILURES =*", + "*_ [[]doctest[]] test_doctest_linedata_on_property.Sample.some_property _*", + "004 ", + "005 >>> Sample().some_property", + "Expected:", + " 'another thing'", + "Got:", + " 'something'", + "", + "*/test_doctest_linedata_on_property.py:5: DocTestFailure", + "*= 1 failed in *", + ] + ) + + def test_doctest_no_linedata_on_overriden_property(self, pytester: Pytester): + pytester.makepyfile( + """ + class Sample(object): + @property + def some_property(self): + ''' + >>> Sample().some_property + 'another thing' + ''' + return 'something' + some_property = property(some_property.__get__, None, None, some_property.__doc__) + """ + ) + result = pytester.runpytest("--doctest-modules") + result.stdout.fnmatch_lines( + [ + "*= FAILURES =*", + "*_ [[]doctest[]] test_doctest_no_linedata_on_overriden_property.Sample.some_property _*", + "EXAMPLE LOCATION UNKNOWN, not showing all tests of that example", + "[?][?][?] >>> Sample().some_property", + "Expected:", + " 'another thing'", + "Got:", + " 'something'", + "", + "*/test_doctest_no_linedata_on_overriden_property.py:None: DocTestFailure", + "*= 1 failed in *", + ] + ) + + def test_doctest_unex_importerror_only_txt(self, pytester: Pytester): + pytester.maketxtfile( + """ + >>> import asdalsdkjaslkdjasd + >>> + """ + ) + result = pytester.runpytest() + # doctest is never executed because of error during hello.py collection + result.stdout.fnmatch_lines( + [ + "*>>> import asdals*", + "*UNEXPECTED*ModuleNotFoundError*", + "ModuleNotFoundError: No module named *asdal*", + ] + ) + + def test_doctest_unex_importerror_with_module(self, pytester: Pytester): + pytester.path.joinpath("hello.py").write_text( + textwrap.dedent( + """\ + import asdalsdkjaslkdjasd + """ + ) + ) + pytester.maketxtfile( + """ + >>> import hello + >>> + """ + ) + result = pytester.runpytest("--doctest-modules") + # doctest is never executed because of error during hello.py collection + result.stdout.fnmatch_lines( + [ + "*ERROR collecting hello.py*", + "*ModuleNotFoundError: No module named *asdals*", + "*Interrupted: 1 error during collection*", + ] + ) + + def test_doctestmodule(self, pytester: Pytester): + p = pytester.makepyfile( + """ + ''' + >>> x = 1 + >>> x == 1 + False + + ''' + """ + ) + reprec = pytester.inline_run(p, "--doctest-modules") + reprec.assertoutcome(failed=1) + + def test_doctestmodule_external_and_issue116(self, pytester: Pytester): + p = pytester.mkpydir("hello") + p.joinpath("__init__.py").write_text( + textwrap.dedent( + """\ + def somefunc(): + ''' + >>> i = 0 + >>> i + 1 + 2 + ''' + """ + ) + ) + result = pytester.runpytest(p, "--doctest-modules") + result.stdout.fnmatch_lines( + [ + "003 *>>> i = 0", + "004 *>>> i + 1", + "*Expected:", + "* 2", + "*Got:", + "* 1", + "*:4: DocTestFailure", + ] + ) + + def test_txtfile_failing(self, pytester: Pytester): + p = pytester.maketxtfile( + """ + >>> i = 0 + >>> i + 1 + 2 + """ + ) + result = pytester.runpytest(p, "-s") + result.stdout.fnmatch_lines( + [ + "001 >>> i = 0", + "002 >>> i + 1", + "Expected:", + " 2", + "Got:", + " 1", + "*test_txtfile_failing.txt:2: DocTestFailure", + ] + ) + + def test_txtfile_with_fixtures(self, pytester: Pytester): + p = pytester.maketxtfile( + """ + >>> p = getfixture('tmp_path') + >>> p.is_dir() + True + """ + ) + reprec = pytester.inline_run(p) + reprec.assertoutcome(passed=1) + + def test_txtfile_with_usefixtures_in_ini(self, pytester: Pytester): + pytester.makeini( + """ + [pytest] + usefixtures = myfixture + """ + ) + pytester.makeconftest( + """ + import pytest + @pytest.fixture + def myfixture(monkeypatch): + monkeypatch.setenv("HELLO", "WORLD") + """ + ) + + p = pytester.maketxtfile( + """ + >>> import os + >>> os.environ["HELLO"] + 'WORLD' + """ + ) + reprec = pytester.inline_run(p) + reprec.assertoutcome(passed=1) + + def test_doctestmodule_with_fixtures(self, pytester: Pytester): + p = pytester.makepyfile( + """ + ''' + >>> p = getfixture('tmp_path') + >>> p.is_dir() + True + ''' + """ + ) + reprec = pytester.inline_run(p, "--doctest-modules") + reprec.assertoutcome(passed=1) + + def test_doctestmodule_three_tests(self, pytester: Pytester): + p = pytester.makepyfile( + """ + ''' + >>> p = getfixture('tmp_path') + >>> p.is_dir() + True + ''' + def my_func(): + ''' + >>> magic = 42 + >>> magic - 42 + 0 + ''' + def useless(): + pass + def another(): + ''' + >>> import os + >>> os is os + True + ''' + """ + ) + reprec = pytester.inline_run(p, "--doctest-modules") + reprec.assertoutcome(passed=3) + + def test_doctestmodule_two_tests_one_fail(self, pytester: Pytester): + p = pytester.makepyfile( + """ + class MyClass(object): + def bad_meth(self): + ''' + >>> magic = 42 + >>> magic + 0 + ''' + def nice_meth(self): + ''' + >>> magic = 42 + >>> magic - 42 + 0 + ''' + """ + ) + reprec = pytester.inline_run(p, "--doctest-modules") + reprec.assertoutcome(failed=1, passed=1) + + def test_ignored_whitespace(self, pytester: Pytester): + pytester.makeini( + """ + [pytest] + doctest_optionflags = ELLIPSIS NORMALIZE_WHITESPACE + """ + ) + p = pytester.makepyfile( + """ + class MyClass(object): + ''' + >>> a = "foo " + >>> print(a) + foo + ''' + pass + """ + ) + reprec = pytester.inline_run(p, "--doctest-modules") + reprec.assertoutcome(passed=1) + + def test_non_ignored_whitespace(self, pytester: Pytester): + pytester.makeini( + """ + [pytest] + doctest_optionflags = ELLIPSIS + """ + ) + p = pytester.makepyfile( + """ + class MyClass(object): + ''' + >>> a = "foo " + >>> print(a) + foo + ''' + pass + """ + ) + reprec = pytester.inline_run(p, "--doctest-modules") + reprec.assertoutcome(failed=1, passed=0) + + def test_ignored_whitespace_glob(self, pytester: Pytester): + pytester.makeini( + """ + [pytest] + doctest_optionflags = ELLIPSIS NORMALIZE_WHITESPACE + """ + ) + p = pytester.maketxtfile( + xdoc=""" + >>> a = "foo " + >>> print(a) + foo + """ + ) + reprec = pytester.inline_run(p, "--doctest-glob=x*.txt") + reprec.assertoutcome(passed=1) + + def test_non_ignored_whitespace_glob(self, pytester: Pytester): + pytester.makeini( + """ + [pytest] + doctest_optionflags = ELLIPSIS + """ + ) + p = pytester.maketxtfile( + xdoc=""" + >>> a = "foo " + >>> print(a) + foo + """ + ) + reprec = pytester.inline_run(p, "--doctest-glob=x*.txt") + reprec.assertoutcome(failed=1, passed=0) + + def test_contains_unicode(self, pytester: Pytester): + """Fix internal error with docstrings containing non-ascii characters.""" + pytester.makepyfile( + '''\ + def foo(): + """ + >>> name = 'с' # not letter 'c' but instead Cyrillic 's'. + 'anything' + """ + ''' + ) + result = pytester.runpytest("--doctest-modules") + result.stdout.fnmatch_lines(["Got nothing", "* 1 failed in*"]) + + def test_ignore_import_errors_on_doctest(self, pytester: Pytester): + p = pytester.makepyfile( + """ + import asdf + + def add_one(x): + ''' + >>> add_one(1) + 2 + ''' + return x + 1 + """ + ) + + reprec = pytester.inline_run( + p, "--doctest-modules", "--doctest-ignore-import-errors" + ) + reprec.assertoutcome(skipped=1, failed=1, passed=0) + + def test_junit_report_for_doctest(self, pytester: Pytester): + """#713: Fix --junit-xml option when used with --doctest-modules.""" + p = pytester.makepyfile( + """ + def foo(): + ''' + >>> 1 + 1 + 3 + ''' + pass + """ + ) + reprec = pytester.inline_run(p, "--doctest-modules", "--junit-xml=junit.xml") + reprec.assertoutcome(failed=1) + + def test_unicode_doctest(self, pytester: Pytester): + """ + Test case for issue 2434: DecodeError on Python 2 when doctest contains non-ascii + characters. + """ + p = pytester.maketxtfile( + test_unicode_doctest=""" + .. doctest:: + + >>> print("Hi\\n\\nByé") + Hi + ... + Byé + >>> 1 / 0 # Byé + 1 + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + ["*UNEXPECTED EXCEPTION: ZeroDivisionError*", "*1 failed*"] + ) + + def test_unicode_doctest_module(self, pytester: Pytester): + """ + Test case for issue 2434: DecodeError on Python 2 when doctest docstring + contains non-ascii characters. + """ + p = pytester.makepyfile( + test_unicode_doctest_module=""" + def fix_bad_unicode(text): + ''' + >>> print(fix_bad_unicode('único')) + único + ''' + return "único" + """ + ) + result = pytester.runpytest(p, "--doctest-modules") + result.stdout.fnmatch_lines(["* 1 passed *"]) + + def test_print_unicode_value(self, pytester: Pytester): + """ + Test case for issue 3583: Printing Unicode in doctest under Python 2.7 + doesn't work + """ + p = pytester.maketxtfile( + test_print_unicode_value=r""" + Here is a doctest:: + + >>> print('\xE5\xE9\xEE\xF8\xFC') + åéîøü + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines(["* 1 passed *"]) + + def test_reportinfo(self, pytester: Pytester): + """Make sure that DoctestItem.reportinfo() returns lineno.""" + p = pytester.makepyfile( + test_reportinfo=""" + def foo(x): + ''' + >>> foo('a') + 'b' + ''' + return 'c' + """ + ) + items, reprec = pytester.inline_genitems(p, "--doctest-modules") + reportinfo = items[0].reportinfo() + assert reportinfo[1] == 1 + + def test_valid_setup_py(self, pytester: Pytester): + """ + Test to make sure that pytest ignores valid setup.py files when ran + with --doctest-modules + """ + p = pytester.makepyfile( + setup=""" + from setuptools import setup, find_packages + if __name__ == '__main__': + setup(name='sample', + version='0.0', + description='description', + packages=find_packages() + ) + """ + ) + result = pytester.runpytest(p, "--doctest-modules") + result.stdout.fnmatch_lines(["*collected 0 items*"]) + + def test_main_py_does_not_cause_import_errors(self, pytester: Pytester): + p = pytester.copy_example("doctest/main_py") + result = pytester.runpytest(p, "--doctest-modules") + result.stdout.fnmatch_lines(["*collected 2 items*", "*1 failed, 1 passed*"]) + + def test_invalid_setup_py(self, pytester: Pytester): + """ + Test to make sure that pytest reads setup.py files that are not used + for python packages when ran with --doctest-modules + """ + p = pytester.makepyfile( + setup=""" + def test_foo(): + return 'bar' + """ + ) + result = pytester.runpytest(p, "--doctest-modules") + result.stdout.fnmatch_lines(["*collected 1 item*"]) + + +class TestLiterals: + @pytest.mark.parametrize("config_mode", ["ini", "comment"]) + def test_allow_unicode(self, pytester, config_mode): + """Test that doctests which output unicode work in all python versions + tested by pytest when the ALLOW_UNICODE option is used (either in + the ini file or by an inline comment). + """ + if config_mode == "ini": + pytester.makeini( + """ + [pytest] + doctest_optionflags = ALLOW_UNICODE + """ + ) + comment = "" + else: + comment = "#doctest: +ALLOW_UNICODE" + + pytester.maketxtfile( + test_doc=""" + >>> b'12'.decode('ascii') {comment} + '12' + """.format( + comment=comment + ) + ) + pytester.makepyfile( + foo=""" + def foo(): + ''' + >>> b'12'.decode('ascii') {comment} + '12' + ''' + """.format( + comment=comment + ) + ) + reprec = pytester.inline_run("--doctest-modules") + reprec.assertoutcome(passed=2) + + @pytest.mark.parametrize("config_mode", ["ini", "comment"]) + def test_allow_bytes(self, pytester, config_mode): + """Test that doctests which output bytes work in all python versions + tested by pytest when the ALLOW_BYTES option is used (either in + the ini file or by an inline comment)(#1287). + """ + if config_mode == "ini": + pytester.makeini( + """ + [pytest] + doctest_optionflags = ALLOW_BYTES + """ + ) + comment = "" + else: + comment = "#doctest: +ALLOW_BYTES" + + pytester.maketxtfile( + test_doc=""" + >>> b'foo' {comment} + 'foo' + """.format( + comment=comment + ) + ) + pytester.makepyfile( + foo=""" + def foo(): + ''' + >>> b'foo' {comment} + 'foo' + ''' + """.format( + comment=comment + ) + ) + reprec = pytester.inline_run("--doctest-modules") + reprec.assertoutcome(passed=2) + + def test_unicode_string(self, pytester: Pytester): + """Test that doctests which output unicode fail in Python 2 when + the ALLOW_UNICODE option is not used. The same test should pass + in Python 3. + """ + pytester.maketxtfile( + test_doc=""" + >>> b'12'.decode('ascii') + '12' + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_bytes_literal(self, pytester: Pytester): + """Test that doctests which output bytes fail in Python 3 when + the ALLOW_BYTES option is not used. (#1287). + """ + pytester.maketxtfile( + test_doc=""" + >>> b'foo' + 'foo' + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(failed=1) + + def test_number_re(self) -> None: + _number_re = _get_checker()._number_re # type: ignore + for s in [ + "1.", + "+1.", + "-1.", + ".1", + "+.1", + "-.1", + "0.1", + "+0.1", + "-0.1", + "1e5", + "+1e5", + "1e+5", + "+1e+5", + "1e-5", + "+1e-5", + "-1e-5", + "1.2e3", + "-1.2e-3", + ]: + print(s) + m = _number_re.match(s) + assert m is not None + assert float(m.group()) == pytest.approx(float(s)) + for s in ["1", "abc"]: + print(s) + assert _number_re.match(s) is None + + @pytest.mark.parametrize("config_mode", ["ini", "comment"]) + def test_number_precision(self, pytester, config_mode): + """Test the NUMBER option.""" + if config_mode == "ini": + pytester.makeini( + """ + [pytest] + doctest_optionflags = NUMBER + """ + ) + comment = "" + else: + comment = "#doctest: +NUMBER" + + pytester.maketxtfile( + test_doc=""" + + Scalars: + + >>> import math + >>> math.pi {comment} + 3.141592653589793 + >>> math.pi {comment} + 3.1416 + >>> math.pi {comment} + 3.14 + >>> -math.pi {comment} + -3.14 + >>> math.pi {comment} + 3. + >>> 3. {comment} + 3.0 + >>> 3. {comment} + 3. + >>> 3. {comment} + 3.01 + >>> 3. {comment} + 2.99 + >>> .299 {comment} + .3 + >>> .301 {comment} + .3 + >>> 951. {comment} + 1e3 + >>> 1049. {comment} + 1e3 + >>> -1049. {comment} + -1e3 + >>> 1e3 {comment} + 1e3 + >>> 1e3 {comment} + 1000. + + Lists: + + >>> [3.1415, 0.097, 13.1, 7, 8.22222e5, 0.598e-2] {comment} + [3.14, 0.1, 13., 7, 8.22e5, 6.0e-3] + >>> [[0.333, 0.667], [0.999, 1.333]] {comment} + [[0.33, 0.667], [0.999, 1.333]] + >>> [[[0.101]]] {comment} + [[[0.1]]] + + Doesn't barf on non-numbers: + + >>> 'abc' {comment} + 'abc' + >>> None {comment} + """.format( + comment=comment + ) + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + @pytest.mark.parametrize( + "expression,output", + [ + # ints shouldn't match floats: + ("3.0", "3"), + ("3e0", "3"), + ("1e3", "1000"), + ("3", "3.0"), + # Rounding: + ("3.1", "3.0"), + ("3.1", "3.2"), + ("3.1", "4.0"), + ("8.22e5", "810000.0"), + # Only the actual output is rounded up, not the expected output: + ("3.0", "2.98"), + ("1e3", "999"), + # The current implementation doesn't understand that numbers inside + # strings shouldn't be treated as numbers: + pytest.param("'3.1416'", "'3.14'", marks=pytest.mark.xfail), + ], + ) + def test_number_non_matches(self, pytester, expression, output): + pytester.maketxtfile( + test_doc=""" + >>> {expression} #doctest: +NUMBER + {output} + """.format( + expression=expression, output=output + ) + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=0, failed=1) + + def test_number_and_allow_unicode(self, pytester: Pytester): + pytester.maketxtfile( + test_doc=""" + >>> from collections import namedtuple + >>> T = namedtuple('T', 'a b c') + >>> T(a=0.2330000001, b=u'str', c=b'bytes') # doctest: +ALLOW_UNICODE, +ALLOW_BYTES, +NUMBER + T(a=0.233, b=u'str', c='bytes') + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +class TestDoctestSkips: + """ + If all examples in a doctest are skipped due to the SKIP option, then + the tests should be SKIPPED rather than PASSED. (#957) + """ + + @pytest.fixture(params=["text", "module"]) + def makedoctest(self, pytester, request): + def makeit(doctest): + mode = request.param + if mode == "text": + pytester.maketxtfile(doctest) + else: + assert mode == "module" + pytester.makepyfile('"""\n%s"""' % doctest) + + return makeit + + def test_one_skipped(self, pytester, makedoctest): + makedoctest( + """ + >>> 1 + 1 # doctest: +SKIP + 2 + >>> 2 + 2 + 4 + """ + ) + reprec = pytester.inline_run("--doctest-modules") + reprec.assertoutcome(passed=1) + + def test_one_skipped_failed(self, pytester, makedoctest): + makedoctest( + """ + >>> 1 + 1 # doctest: +SKIP + 2 + >>> 2 + 2 + 200 + """ + ) + reprec = pytester.inline_run("--doctest-modules") + reprec.assertoutcome(failed=1) + + def test_all_skipped(self, pytester, makedoctest): + makedoctest( + """ + >>> 1 + 1 # doctest: +SKIP + 2 + >>> 2 + 2 # doctest: +SKIP + 200 + """ + ) + reprec = pytester.inline_run("--doctest-modules") + reprec.assertoutcome(skipped=1) + + def test_vacuous_all_skipped(self, pytester, makedoctest): + makedoctest("") + reprec = pytester.inline_run("--doctest-modules") + reprec.assertoutcome(passed=0, skipped=0) + + def test_continue_on_failure(self, pytester: Pytester): + pytester.maketxtfile( + test_something=""" + >>> i = 5 + >>> def foo(): + ... raise ValueError('error1') + >>> foo() + >>> i + >>> i + 2 + 7 + >>> i + 1 + """ + ) + result = pytester.runpytest( + "--doctest-modules", "--doctest-continue-on-failure" + ) + result.assert_outcomes(passed=0, failed=1) + # The lines that contains the failure are 4, 5, and 8. The first one + # is a stack trace and the other two are mismatches. + result.stdout.fnmatch_lines( + ["*4: UnexpectedException*", "*5: DocTestFailure*", "*8: DocTestFailure*"] + ) + + def test_skipping_wrapped_test(self, pytester): + """ + Issue 8796: INTERNALERROR raised when skipping a decorated DocTest + through pytest_collection_modifyitems. + """ + pytester.makeconftest( + """ + import pytest + from _pytest.doctest import DoctestItem + + def pytest_collection_modifyitems(config, items): + skip_marker = pytest.mark.skip() + + for item in items: + if isinstance(item, DoctestItem): + item.add_marker(skip_marker) + """ + ) + + pytester.makepyfile( + """ + from contextlib import contextmanager + + @contextmanager + def my_config_context(): + ''' + >>> import os + ''' + """ + ) + + result = pytester.runpytest("--doctest-modules") + assert "INTERNALERROR" not in result.stdout.str() + result.assert_outcomes(skipped=1) + + +class TestDoctestAutoUseFixtures: + + SCOPES = ["module", "session", "class", "function"] + + def test_doctest_module_session_fixture(self, pytester: Pytester): + """Test that session fixtures are initialized for doctest modules (#768).""" + # session fixture which changes some global data, which will + # be accessed by doctests in a module + pytester.makeconftest( + """ + import pytest + import sys + + @pytest.fixture(autouse=True, scope='session') + def myfixture(): + assert not hasattr(sys, 'pytest_session_data') + sys.pytest_session_data = 1 + yield + del sys.pytest_session_data + """ + ) + pytester.makepyfile( + foo=""" + import sys + + def foo(): + ''' + >>> assert sys.pytest_session_data == 1 + ''' + + def bar(): + ''' + >>> assert sys.pytest_session_data == 1 + ''' + """ + ) + result = pytester.runpytest("--doctest-modules") + result.stdout.fnmatch_lines(["*2 passed*"]) + + @pytest.mark.parametrize("scope", SCOPES) + @pytest.mark.parametrize("enable_doctest", [True, False]) + def test_fixture_scopes(self, pytester, scope, enable_doctest): + """Test that auto-use fixtures work properly with doctest modules. + See #1057 and #1100. + """ + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(autouse=True, scope="{scope}") + def auto(request): + return 99 + """.format( + scope=scope + ) + ) + pytester.makepyfile( + test_1=''' + def test_foo(): + """ + >>> getfixture('auto') + 1 + 100 + """ + def test_bar(): + assert 1 + ''' + ) + params = ("--doctest-modules",) if enable_doctest else () + passes = 3 if enable_doctest else 2 + result = pytester.runpytest(*params) + result.stdout.fnmatch_lines(["*=== %d passed in *" % passes]) + + @pytest.mark.parametrize("scope", SCOPES) + @pytest.mark.parametrize("autouse", [True, False]) + @pytest.mark.parametrize("use_fixture_in_doctest", [True, False]) + def test_fixture_module_doctest_scopes( + self, pytester, scope, autouse, use_fixture_in_doctest + ): + """Test that auto-use fixtures work properly with doctest files. + See #1057 and #1100. + """ + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(autouse={autouse}, scope="{scope}") + def auto(request): + return 99 + """.format( + scope=scope, autouse=autouse + ) + ) + if use_fixture_in_doctest: + pytester.maketxtfile( + test_doc=""" + >>> getfixture('auto') + 99 + """ + ) + else: + pytester.maketxtfile( + test_doc=""" + >>> 1 + 1 + 2 + """ + ) + result = pytester.runpytest("--doctest-modules") + result.stdout.no_fnmatch_line("*FAILURES*") + result.stdout.fnmatch_lines(["*=== 1 passed in *"]) + + @pytest.mark.parametrize("scope", SCOPES) + def test_auto_use_request_attributes(self, pytester, scope): + """Check that all attributes of a request in an autouse fixture + behave as expected when requested for a doctest item. + """ + pytester.makeconftest( + """ + import pytest + + @pytest.fixture(autouse=True, scope="{scope}") + def auto(request): + if "{scope}" == 'module': + assert request.module is None + if "{scope}" == 'class': + assert request.cls is None + if "{scope}" == 'function': + assert request.function is None + return 99 + """.format( + scope=scope + ) + ) + pytester.maketxtfile( + test_doc=""" + >>> 1 + 1 + 2 + """ + ) + result = pytester.runpytest("--doctest-modules") + str(result.stdout.no_fnmatch_line("*FAILURES*")) + result.stdout.fnmatch_lines(["*=== 1 passed in *"]) + + +class TestDoctestNamespaceFixture: + + SCOPES = ["module", "session", "class", "function"] + + @pytest.mark.parametrize("scope", SCOPES) + def test_namespace_doctestfile(self, pytester, scope): + """ + Check that inserting something into the namespace works in a + simple text file doctest + """ + pytester.makeconftest( + """ + import pytest + import contextlib + + @pytest.fixture(autouse=True, scope="{scope}") + def add_contextlib(doctest_namespace): + doctest_namespace['cl'] = contextlib + """.format( + scope=scope + ) + ) + p = pytester.maketxtfile( + """ + >>> print(cl.__name__) + contextlib + """ + ) + reprec = pytester.inline_run(p) + reprec.assertoutcome(passed=1) + + @pytest.mark.parametrize("scope", SCOPES) + def test_namespace_pyfile(self, pytester, scope): + """ + Check that inserting something into the namespace works in a + simple Python file docstring doctest + """ + pytester.makeconftest( + """ + import pytest + import contextlib + + @pytest.fixture(autouse=True, scope="{scope}") + def add_contextlib(doctest_namespace): + doctest_namespace['cl'] = contextlib + """.format( + scope=scope + ) + ) + p = pytester.makepyfile( + """ + def foo(): + ''' + >>> print(cl.__name__) + contextlib + ''' + """ + ) + reprec = pytester.inline_run(p, "--doctest-modules") + reprec.assertoutcome(passed=1) + + +class TestDoctestReportingOption: + def _run_doctest_report(self, pytester, format): + pytester.makepyfile( + """ + def foo(): + ''' + >>> foo() + a b + 0 1 4 + 1 2 4 + 2 3 6 + ''' + print(' a b\\n' + '0 1 4\\n' + '1 2 5\\n' + '2 3 6') + """ + ) + return pytester.runpytest("--doctest-modules", "--doctest-report", format) + + @pytest.mark.parametrize("format", ["udiff", "UDIFF", "uDiFf"]) + def test_doctest_report_udiff(self, pytester, format): + result = self._run_doctest_report(pytester, format) + result.stdout.fnmatch_lines( + [" 0 1 4", " -1 2 4", " +1 2 5", " 2 3 6"] + ) + + def test_doctest_report_cdiff(self, pytester: Pytester): + result = self._run_doctest_report(pytester, "cdiff") + result.stdout.fnmatch_lines( + [ + " a b", + " 0 1 4", + " ! 1 2 4", + " 2 3 6", + " --- 1,4 ----", + " a b", + " 0 1 4", + " ! 1 2 5", + " 2 3 6", + ] + ) + + def test_doctest_report_ndiff(self, pytester: Pytester): + result = self._run_doctest_report(pytester, "ndiff") + result.stdout.fnmatch_lines( + [ + " a b", + " 0 1 4", + " - 1 2 4", + " ? ^", + " + 1 2 5", + " ? ^", + " 2 3 6", + ] + ) + + @pytest.mark.parametrize("format", ["none", "only_first_failure"]) + def test_doctest_report_none_or_only_first_failure(self, pytester, format): + result = self._run_doctest_report(pytester, format) + result.stdout.fnmatch_lines( + [ + "Expected:", + " a b", + " 0 1 4", + " 1 2 4", + " 2 3 6", + "Got:", + " a b", + " 0 1 4", + " 1 2 5", + " 2 3 6", + ] + ) + + def test_doctest_report_invalid(self, pytester: Pytester): + result = self._run_doctest_report(pytester, "obviously_invalid_format") + result.stderr.fnmatch_lines( + [ + "*error: argument --doctest-report: invalid choice: 'obviously_invalid_format' (choose from*" + ] + ) + + +@pytest.mark.parametrize("mock_module", ["mock", "unittest.mock"]) +def test_doctest_mock_objects_dont_recurse_missbehaved(mock_module, pytester: Pytester): + pytest.importorskip(mock_module) + pytester.makepyfile( + """ + from {mock_module} import call + class Example(object): + ''' + >>> 1 + 1 + 2 + ''' + """.format( + mock_module=mock_module + ) + ) + result = pytester.runpytest("--doctest-modules") + result.stdout.fnmatch_lines(["* 1 passed *"]) + + +class Broken: + def __getattr__(self, _): + raise KeyError("This should be an AttributeError") + + +@pytest.mark.parametrize( # pragma: no branch (lambdas are not called) + "stop", [None, _is_mocked, lambda f: None, lambda f: False, lambda f: True] +) +def test_warning_on_unwrap_of_broken_object( + stop: Optional[Callable[[object], object]] +) -> None: + bad_instance = Broken() + assert inspect.unwrap.__module__ == "inspect" + with _patch_unwrap_mock_aware(): + assert inspect.unwrap.__module__ != "inspect" + with pytest.warns( + pytest.PytestWarning, match="^Got KeyError.* when unwrapping" + ): + with pytest.raises(KeyError): + inspect.unwrap(bad_instance, stop=stop) # type: ignore[arg-type] + assert inspect.unwrap.__module__ == "inspect" + + +def test_is_setup_py_not_named_setup_py(tmp_path: Path) -> None: + not_setup_py = tmp_path.joinpath("not_setup.py") + not_setup_py.write_text('from setuptools import setup; setup(name="foo")') + assert not _is_setup_py(not_setup_py) + + +@pytest.mark.parametrize("mod", ("setuptools", "distutils.core")) +def test_is_setup_py_is_a_setup_py(tmp_path: Path, mod: str) -> None: + setup_py = tmp_path.joinpath("setup.py") + setup_py.write_text(f'from {mod} import setup; setup(name="foo")', "utf-8") + assert _is_setup_py(setup_py) + + +@pytest.mark.parametrize("mod", ("setuptools", "distutils.core")) +def test_is_setup_py_different_encoding(tmp_path: Path, mod: str) -> None: + setup_py = tmp_path.joinpath("setup.py") + contents = ( + "# -*- coding: cp1252 -*-\n" + 'from {} import setup; setup(name="foo", description="€")\n'.format(mod) + ) + setup_py.write_bytes(contents.encode("cp1252")) + assert _is_setup_py(setup_py) + + +@pytest.mark.parametrize( + "name, expected", [("__main__.py", True), ("__init__.py", False)] +) +def test_is_main_py(tmp_path: Path, name: str, expected: bool) -> None: + dunder_main = tmp_path.joinpath(name) + assert _is_main_py(dunder_main) == expected diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_entry_points.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_entry_points.py new file mode 100644 index 0000000000..5d00312736 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_entry_points.py @@ -0,0 +1,7 @@ +from _pytest.compat import importlib_metadata + + +def test_pytest_entry_points_are_identical(): + dist = importlib_metadata.distribution("pytest") + entry_map = {ep.name: ep for ep in dist.entry_points} + assert entry_map["pytest"].value == entry_map["py.test"].value diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_error_diffs.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_error_diffs.py new file mode 100644 index 0000000000..1668e929ab --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_error_diffs.py @@ -0,0 +1,283 @@ +""" +Tests and examples for correct "+/-" usage in error diffs. + +See https://github.com/pytest-dev/pytest/issues/3333 for details. + +""" +import sys + +import pytest +from _pytest.pytester import Pytester + + +TESTCASES = [ + pytest.param( + """ + def test_this(): + result = [1, 4, 3] + expected = [1, 2, 3] + assert result == expected + """, + """ + > assert result == expected + E assert [1, 4, 3] == [1, 2, 3] + E At index 1 diff: 4 != 2 + E Full diff: + E - [1, 2, 3] + E ? ^ + E + [1, 4, 3] + E ? ^ + """, + id="Compare lists, one item differs", + ), + pytest.param( + """ + def test_this(): + result = [1, 2, 3] + expected = [1, 2] + assert result == expected + """, + """ + > assert result == expected + E assert [1, 2, 3] == [1, 2] + E Left contains one more item: 3 + E Full diff: + E - [1, 2] + E + [1, 2, 3] + E ? +++ + """, + id="Compare lists, one extra item", + ), + pytest.param( + """ + def test_this(): + result = [1, 3] + expected = [1, 2, 3] + assert result == expected + """, + """ + > assert result == expected + E assert [1, 3] == [1, 2, 3] + E At index 1 diff: 3 != 2 + E Right contains one more item: 3 + E Full diff: + E - [1, 2, 3] + E ? --- + E + [1, 3] + """, + id="Compare lists, one item missing", + ), + pytest.param( + """ + def test_this(): + result = (1, 4, 3) + expected = (1, 2, 3) + assert result == expected + """, + """ + > assert result == expected + E assert (1, 4, 3) == (1, 2, 3) + E At index 1 diff: 4 != 2 + E Full diff: + E - (1, 2, 3) + E ? ^ + E + (1, 4, 3) + E ? ^ + """, + id="Compare tuples", + ), + pytest.param( + """ + def test_this(): + result = {1, 3, 4} + expected = {1, 2, 3} + assert result == expected + """, + """ + > assert result == expected + E assert {1, 3, 4} == {1, 2, 3} + E Extra items in the left set: + E 4 + E Extra items in the right set: + E 2 + E Full diff: + E - {1, 2, 3} + E ? ^ ^ + E + {1, 3, 4} + E ? ^ ^ + """, + id="Compare sets", + ), + pytest.param( + """ + def test_this(): + result = {1: 'spam', 3: 'eggs'} + expected = {1: 'spam', 2: 'eggs'} + assert result == expected + """, + """ + > assert result == expected + E AssertionError: assert {1: 'spam', 3: 'eggs'} == {1: 'spam', 2: 'eggs'} + E Common items: + E {1: 'spam'} + E Left contains 1 more item: + E {3: 'eggs'} + E Right contains 1 more item: + E {2: 'eggs'} + E Full diff: + E - {1: 'spam', 2: 'eggs'} + E ? ^ + E + {1: 'spam', 3: 'eggs'} + E ? ^ + """, + id="Compare dicts with differing keys", + ), + pytest.param( + """ + def test_this(): + result = {1: 'spam', 2: 'eggs'} + expected = {1: 'spam', 2: 'bacon'} + assert result == expected + """, + """ + > assert result == expected + E AssertionError: assert {1: 'spam', 2: 'eggs'} == {1: 'spam', 2: 'bacon'} + E Common items: + E {1: 'spam'} + E Differing items: + E {2: 'eggs'} != {2: 'bacon'} + E Full diff: + E - {1: 'spam', 2: 'bacon'} + E ? ^^^^^ + E + {1: 'spam', 2: 'eggs'} + E ? ^^^^ + """, + id="Compare dicts with differing values", + ), + pytest.param( + """ + def test_this(): + result = {1: 'spam', 2: 'eggs'} + expected = {1: 'spam', 3: 'bacon'} + assert result == expected + """, + """ + > assert result == expected + E AssertionError: assert {1: 'spam', 2: 'eggs'} == {1: 'spam', 3: 'bacon'} + E Common items: + E {1: 'spam'} + E Left contains 1 more item: + E {2: 'eggs'} + E Right contains 1 more item: + E {3: 'bacon'} + E Full diff: + E - {1: 'spam', 3: 'bacon'} + E ? ^ ^^^^^ + E + {1: 'spam', 2: 'eggs'} + E ? ^ ^^^^ + """, + id="Compare dicts with differing items", + ), + pytest.param( + """ + def test_this(): + result = "spmaeggs" + expected = "spameggs" + assert result == expected + """, + """ + > assert result == expected + E AssertionError: assert 'spmaeggs' == 'spameggs' + E - spameggs + E ? - + E + spmaeggs + E ? + + """, + id="Compare strings", + ), + pytest.param( + """ + def test_this(): + result = "spam bacon eggs" + assert "bacon" not in result + """, + """ + > assert "bacon" not in result + E AssertionError: assert 'bacon' not in 'spam bacon eggs' + E 'bacon' is contained here: + E spam bacon eggs + E ? +++++ + """, + id='Test "not in" string', + ), +] +if sys.version_info[:2] >= (3, 7): + TESTCASES.extend( + [ + pytest.param( + """ + from dataclasses import dataclass + + @dataclass + class A: + a: int + b: str + + def test_this(): + result = A(1, 'spam') + expected = A(2, 'spam') + assert result == expected + """, + """ + > assert result == expected + E AssertionError: assert A(a=1, b='spam') == A(a=2, b='spam') + E Matching attributes: + E ['b'] + E Differing attributes: + E ['a'] + E Drill down into differing attribute a: + E a: 1 != 2 + E +1 + E -2 + """, + id="Compare data classes", + ), + pytest.param( + """ + import attr + + @attr.s(auto_attribs=True) + class A: + a: int + b: str + + def test_this(): + result = A(1, 'spam') + expected = A(1, 'eggs') + assert result == expected + """, + """ + > assert result == expected + E AssertionError: assert A(a=1, b='spam') == A(a=1, b='eggs') + E Matching attributes: + E ['a'] + E Differing attributes: + E ['b'] + E Drill down into differing attribute b: + E b: 'spam' != 'eggs' + E - eggs + E + spam + """, + id="Compare attrs classes", + ), + ] + ) + + +@pytest.mark.parametrize("code, expected", TESTCASES) +def test_error_diff(code: str, expected: str, pytester: Pytester) -> None: + expected_lines = [line.lstrip() for line in expected.splitlines()] + p = pytester.makepyfile(code) + result = pytester.runpytest(p, "-vv") + result.stdout.fnmatch_lines(expected_lines) + assert result.ret == 1 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_faulthandler.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_faulthandler.py new file mode 100644 index 0000000000..5b7911f21f --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_faulthandler.py @@ -0,0 +1,172 @@ +import io +import sys + +import pytest +from _pytest.pytester import Pytester + + +def test_enabled(pytester: Pytester) -> None: + """Test single crashing test displays a traceback.""" + pytester.makepyfile( + """ + import faulthandler + def test_crash(): + faulthandler._sigabrt() + """ + ) + result = pytester.runpytest_subprocess() + result.stderr.fnmatch_lines(["*Fatal Python error*"]) + assert result.ret != 0 + + +def setup_crashing_test(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import faulthandler + import atexit + def test_ok(): + atexit.register(faulthandler._sigabrt) + """ + ) + + +def test_crash_during_shutdown_captured(pytester: Pytester) -> None: + """ + Re-enable faulthandler if pytest encountered it enabled during configure. + We should be able to then see crashes during interpreter shutdown. + """ + setup_crashing_test(pytester) + args = (sys.executable, "-Xfaulthandler", "-mpytest") + result = pytester.run(*args) + result.stderr.fnmatch_lines(["*Fatal Python error*"]) + assert result.ret != 0 + + +def test_crash_during_shutdown_not_captured(pytester: Pytester) -> None: + """ + Check that pytest leaves faulthandler disabled if it was not enabled during configure. + This prevents us from seeing crashes during interpreter shutdown (see #8260). + """ + setup_crashing_test(pytester) + args = (sys.executable, "-mpytest") + result = pytester.run(*args) + result.stderr.no_fnmatch_line("*Fatal Python error*") + assert result.ret != 0 + + +def test_disabled(pytester: Pytester) -> None: + """Test option to disable fault handler in the command line.""" + pytester.makepyfile( + """ + import faulthandler + def test_disabled(): + assert not faulthandler.is_enabled() + """ + ) + result = pytester.runpytest_subprocess("-p", "no:faulthandler") + result.stdout.fnmatch_lines(["*1 passed*"]) + assert result.ret == 0 + + +@pytest.mark.parametrize( + "enabled", + [ + pytest.param( + True, marks=pytest.mark.skip(reason="sometimes crashes on CI (#7022)") + ), + False, + ], +) +def test_timeout(pytester: Pytester, enabled: bool) -> None: + """Test option to dump tracebacks after a certain timeout. + + If faulthandler is disabled, no traceback will be dumped. + """ + pytester.makepyfile( + """ + import os, time + def test_timeout(): + time.sleep(1 if "CI" in os.environ else 0.1) + """ + ) + pytester.makeini( + """ + [pytest] + faulthandler_timeout = 0.01 + """ + ) + args = ["-p", "no:faulthandler"] if not enabled else [] + + result = pytester.runpytest_subprocess(*args) + tb_output = "most recent call first" + if enabled: + result.stderr.fnmatch_lines(["*%s*" % tb_output]) + else: + assert tb_output not in result.stderr.str() + result.stdout.fnmatch_lines(["*1 passed*"]) + assert result.ret == 0 + + +@pytest.mark.parametrize("hook_name", ["pytest_enter_pdb", "pytest_exception_interact"]) +def test_cancel_timeout_on_hook(monkeypatch, hook_name) -> None: + """Make sure that we are cancelling any scheduled traceback dumping due + to timeout before entering pdb (pytest-dev/pytest-faulthandler#12) or any + other interactive exception (pytest-dev/pytest-faulthandler#14).""" + import faulthandler + from _pytest import faulthandler as faulthandler_plugin + + called = [] + + monkeypatch.setattr( + faulthandler, "cancel_dump_traceback_later", lambda: called.append(1) + ) + + # call our hook explicitly, we can trust that pytest will call the hook + # for us at the appropriate moment + hook_func = getattr(faulthandler_plugin, hook_name) + hook_func() + assert called == [1] + + +def test_already_initialized_crash(pytester: Pytester) -> None: + """Even if faulthandler is already initialized, we still dump tracebacks on crashes (#8258).""" + pytester.makepyfile( + """ + def test(): + import faulthandler + faulthandler._sigabrt() + """ + ) + result = pytester.run( + sys.executable, + "-X", + "faulthandler", + "-mpytest", + pytester.path, + ) + result.stderr.fnmatch_lines(["*Fatal Python error*"]) + assert result.ret != 0 + + +def test_get_stderr_fileno_invalid_fd() -> None: + """Test for faulthandler being able to handle invalid file descriptors for stderr (#8249).""" + from _pytest.faulthandler import get_stderr_fileno + + class StdErrWrapper(io.StringIO): + """ + Mimic ``twisted.logger.LoggingFile`` to simulate returning an invalid file descriptor. + + https://github.com/twisted/twisted/blob/twisted-20.3.0/src/twisted/logger/_io.py#L132-L139 + """ + + def fileno(self): + return -1 + + wrapper = StdErrWrapper() + + with pytest.MonkeyPatch.context() as mp: + mp.setattr("sys.stderr", wrapper) + + # Even when the stderr wrapper signals an invalid file descriptor, + # ``_get_stderr_fileno()`` should return the real one. + assert get_stderr_fileno() == 2 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_findpaths.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_findpaths.py new file mode 100644 index 0000000000..3a2917261a --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_findpaths.py @@ -0,0 +1,135 @@ +from pathlib import Path +from textwrap import dedent + +import pytest +from _pytest.config import UsageError +from _pytest.config.findpaths import get_common_ancestor +from _pytest.config.findpaths import get_dirs_from_args +from _pytest.config.findpaths import load_config_dict_from_file + + +class TestLoadConfigDictFromFile: + def test_empty_pytest_ini(self, tmp_path: Path) -> None: + """pytest.ini files are always considered for configuration, even if empty""" + fn = tmp_path / "pytest.ini" + fn.write_text("", encoding="utf-8") + assert load_config_dict_from_file(fn) == {} + + def test_pytest_ini(self, tmp_path: Path) -> None: + """[pytest] section in pytest.ini files is read correctly""" + fn = tmp_path / "pytest.ini" + fn.write_text("[pytest]\nx=1", encoding="utf-8") + assert load_config_dict_from_file(fn) == {"x": "1"} + + def test_custom_ini(self, tmp_path: Path) -> None: + """[pytest] section in any .ini file is read correctly""" + fn = tmp_path / "custom.ini" + fn.write_text("[pytest]\nx=1", encoding="utf-8") + assert load_config_dict_from_file(fn) == {"x": "1"} + + def test_custom_ini_without_section(self, tmp_path: Path) -> None: + """Custom .ini files without [pytest] section are not considered for configuration""" + fn = tmp_path / "custom.ini" + fn.write_text("[custom]", encoding="utf-8") + assert load_config_dict_from_file(fn) is None + + def test_custom_cfg_file(self, tmp_path: Path) -> None: + """Custom .cfg files without [tool:pytest] section are not considered for configuration""" + fn = tmp_path / "custom.cfg" + fn.write_text("[custom]", encoding="utf-8") + assert load_config_dict_from_file(fn) is None + + def test_valid_cfg_file(self, tmp_path: Path) -> None: + """Custom .cfg files with [tool:pytest] section are read correctly""" + fn = tmp_path / "custom.cfg" + fn.write_text("[tool:pytest]\nx=1", encoding="utf-8") + assert load_config_dict_from_file(fn) == {"x": "1"} + + def test_unsupported_pytest_section_in_cfg_file(self, tmp_path: Path) -> None: + """.cfg files with [pytest] section are no longer supported and should fail to alert users""" + fn = tmp_path / "custom.cfg" + fn.write_text("[pytest]", encoding="utf-8") + with pytest.raises(pytest.fail.Exception): + load_config_dict_from_file(fn) + + def test_invalid_toml_file(self, tmp_path: Path) -> None: + """Invalid .toml files should raise `UsageError`.""" + fn = tmp_path / "myconfig.toml" + fn.write_text("]invalid toml[", encoding="utf-8") + with pytest.raises(UsageError): + load_config_dict_from_file(fn) + + def test_custom_toml_file(self, tmp_path: Path) -> None: + """.toml files without [tool.pytest.ini_options] are not considered for configuration.""" + fn = tmp_path / "myconfig.toml" + fn.write_text( + dedent( + """ + [build_system] + x = 1 + """ + ), + encoding="utf-8", + ) + assert load_config_dict_from_file(fn) is None + + def test_valid_toml_file(self, tmp_path: Path) -> None: + """.toml files with [tool.pytest.ini_options] are read correctly, including changing + data types to str/list for compatibility with other configuration options.""" + fn = tmp_path / "myconfig.toml" + fn.write_text( + dedent( + """ + [tool.pytest.ini_options] + x = 1 + y = 20.0 + values = ["tests", "integration"] + name = "foo" + heterogeneous_array = [1, "str"] + """ + ), + encoding="utf-8", + ) + assert load_config_dict_from_file(fn) == { + "x": "1", + "y": "20.0", + "values": ["tests", "integration"], + "name": "foo", + "heterogeneous_array": [1, "str"], + } + + +class TestCommonAncestor: + def test_has_ancestor(self, tmp_path: Path) -> None: + fn1 = tmp_path / "foo" / "bar" / "test_1.py" + fn1.parent.mkdir(parents=True) + fn1.touch() + fn2 = tmp_path / "foo" / "zaz" / "test_2.py" + fn2.parent.mkdir(parents=True) + fn2.touch() + assert get_common_ancestor([fn1, fn2]) == tmp_path / "foo" + assert get_common_ancestor([fn1.parent, fn2]) == tmp_path / "foo" + assert get_common_ancestor([fn1.parent, fn2.parent]) == tmp_path / "foo" + assert get_common_ancestor([fn1, fn2.parent]) == tmp_path / "foo" + + def test_single_dir(self, tmp_path: Path) -> None: + assert get_common_ancestor([tmp_path]) == tmp_path + + def test_single_file(self, tmp_path: Path) -> None: + fn = tmp_path / "foo.py" + fn.touch() + assert get_common_ancestor([fn]) == tmp_path + + +def test_get_dirs_from_args(tmp_path): + """get_dirs_from_args() skips over non-existing directories and files""" + fn = tmp_path / "foo.py" + fn.touch() + d = tmp_path / "tests" + d.mkdir() + option = "--foobar=/foo.txt" + # xdist uses options in this format for its rsync feature (#7638) + xdist_rsync_option = "popen=c:/dest" + assert get_dirs_from_args( + [str(fn), str(tmp_path / "does_not_exist"), str(d), option, xdist_rsync_option] + ) == [fn.parent, d] diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_helpconfig.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_helpconfig.py new file mode 100644 index 0000000000..44c2c9295b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_helpconfig.py @@ -0,0 +1,124 @@ +import pytest +from _pytest.config import ExitCode +from _pytest.pytester import Pytester + + +def test_version_verbose(pytester: Pytester, pytestconfig, monkeypatch) -> None: + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + result = pytester.runpytest("--version", "--version") + assert result.ret == 0 + result.stdout.fnmatch_lines([f"*pytest*{pytest.__version__}*imported from*"]) + if pytestconfig.pluginmanager.list_plugin_distinfo(): + result.stdout.fnmatch_lines(["*setuptools registered plugins:", "*at*"]) + + +def test_version_less_verbose(pytester: Pytester, pytestconfig, monkeypatch) -> None: + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + result = pytester.runpytest("--version") + assert result.ret == 0 + result.stdout.fnmatch_lines([f"pytest {pytest.__version__}"]) + + +def test_versions(): + """Regression check for the public version attributes in pytest.""" + assert isinstance(pytest.__version__, str) + assert isinstance(pytest.version_tuple, tuple) + + +def test_help(pytester: Pytester) -> None: + result = pytester.runpytest("--help") + assert result.ret == 0 + result.stdout.fnmatch_lines( + """ + -m MARKEXPR only run tests matching given mark expression. + For example: -m 'mark1 and not mark2'. + reporting: + --durations=N * + -V, --version display pytest version and information about plugins. + When given twice, also display information about + plugins. + *setup.cfg* + *minversion* + *to see*markers*pytest --markers* + *to see*fixtures*pytest --fixtures* + """ + ) + + +def test_none_help_param_raises_exception(pytester: Pytester) -> None: + """Test that a None help param raises a TypeError.""" + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("test_ini", None, default=True, type="bool") + """ + ) + result = pytester.runpytest("--help") + result.stderr.fnmatch_lines( + ["*TypeError: help argument cannot be None for test_ini*"] + ) + + +def test_empty_help_param(pytester: Pytester) -> None: + """Test that an empty help param is displayed correctly.""" + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("test_ini", "", default=True, type="bool") + """ + ) + result = pytester.runpytest("--help") + assert result.ret == 0 + lines = [ + " required_plugins (args):", + " plugins that must be present for pytest to run*", + " test_ini (bool):*", + "environment variables:", + ] + result.stdout.fnmatch_lines(lines, consecutive=True) + + +def test_hookvalidation_unknown(pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_hello(xyz): + pass + """ + ) + result = pytester.runpytest() + assert result.ret != 0 + result.stdout.fnmatch_lines(["*unknown hook*pytest_hello*"]) + + +def test_hookvalidation_optional(pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + @pytest.hookimpl(optionalhook=True) + def pytest_hello(xyz): + pass + """ + ) + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +def test_traceconfig(pytester: Pytester) -> None: + result = pytester.runpytest("--traceconfig") + result.stdout.fnmatch_lines(["*using*pytest*", "*active plugins*"]) + + +def test_debug(pytester: Pytester) -> None: + result = pytester.runpytest_subprocess("--debug") + assert result.ret == ExitCode.NO_TESTS_COLLECTED + p = pytester.path.joinpath("pytestdebug.log") + assert "pytest_sessionstart" in p.read_text("utf-8") + + +def test_PYTEST_DEBUG(pytester: Pytester, monkeypatch) -> None: + monkeypatch.setenv("PYTEST_DEBUG", "1") + result = pytester.runpytest_subprocess() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + result.stderr.fnmatch_lines( + ["*pytest_plugin_registered*", "*manager*PluginManager*"] + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_junitxml.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_junitxml.py new file mode 100644 index 0000000000..02531e8143 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_junitxml.py @@ -0,0 +1,1703 @@ +import os +import platform +from datetime import datetime +from pathlib import Path +from typing import cast +from typing import List +from typing import Optional +from typing import Tuple +from typing import TYPE_CHECKING +from typing import Union +from xml.dom import minidom + +import xmlschema + +import pytest +from _pytest.config import Config +from _pytest.junitxml import bin_xml_escape +from _pytest.junitxml import LogXML +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester +from _pytest.pytester import RunResult +from _pytest.reports import BaseReport +from _pytest.reports import TestReport +from _pytest.stash import Stash + + +@pytest.fixture(scope="session") +def schema() -> xmlschema.XMLSchema: + """Return an xmlschema.XMLSchema object for the junit-10.xsd file.""" + fn = Path(__file__).parent / "example_scripts/junit-10.xsd" + with fn.open() as f: + return xmlschema.XMLSchema(f) + + +class RunAndParse: + def __init__(self, pytester: Pytester, schema: xmlschema.XMLSchema) -> None: + self.pytester = pytester + self.schema = schema + + def __call__( + self, *args: Union[str, "os.PathLike[str]"], family: Optional[str] = "xunit1" + ) -> Tuple[RunResult, "DomNode"]: + if family: + args = ("-o", "junit_family=" + family) + args + xml_path = self.pytester.path.joinpath("junit.xml") + result = self.pytester.runpytest("--junitxml=%s" % xml_path, *args) + if family == "xunit2": + with xml_path.open() as f: + self.schema.validate(f) + xmldoc = minidom.parse(str(xml_path)) + return result, DomNode(xmldoc) + + +@pytest.fixture +def run_and_parse(pytester: Pytester, schema: xmlschema.XMLSchema) -> RunAndParse: + """Fixture that returns a function that can be used to execute pytest and + return the parsed ``DomNode`` of the root xml node. + + The ``family`` parameter is used to configure the ``junit_family`` of the written report. + "xunit2" is also automatically validated against the schema. + """ + return RunAndParse(pytester, schema) + + +def assert_attr(node, **kwargs): + __tracebackhide__ = True + + def nodeval(node, name): + anode = node.getAttributeNode(name) + if anode is not None: + return anode.value + + expected = {name: str(value) for name, value in kwargs.items()} + on_node = {name: nodeval(node, name) for name in expected} + assert on_node == expected + + +class DomNode: + def __init__(self, dom): + self.__node = dom + + def __repr__(self): + return self.__node.toxml() + + def find_first_by_tag(self, tag): + return self.find_nth_by_tag(tag, 0) + + def _by_tag(self, tag): + return self.__node.getElementsByTagName(tag) + + @property + def children(self): + return [type(self)(x) for x in self.__node.childNodes] + + @property + def get_unique_child(self): + children = self.children + assert len(children) == 1 + return children[0] + + def find_nth_by_tag(self, tag, n): + items = self._by_tag(tag) + try: + nth = items[n] + except IndexError: + pass + else: + return type(self)(nth) + + def find_by_tag(self, tag): + t = type(self) + return [t(x) for x in self.__node.getElementsByTagName(tag)] + + def __getitem__(self, key): + node = self.__node.getAttributeNode(key) + if node is not None: + return node.value + + def assert_attr(self, **kwargs): + __tracebackhide__ = True + return assert_attr(self.__node, **kwargs) + + def toxml(self): + return self.__node.toxml() + + @property + def text(self): + return self.__node.childNodes[0].wholeText + + @property + def tag(self): + return self.__node.tagName + + @property + def next_sibling(self): + return type(self)(self.__node.nextSibling) + + +parametrize_families = pytest.mark.parametrize("xunit_family", ["xunit1", "xunit2"]) + + +class TestPython: + @parametrize_families + def test_summing_simple( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + def test_pass(): + pass + def test_fail(): + assert 0 + def test_skip(): + pytest.skip("") + @pytest.mark.xfail + def test_xfail(): + assert 0 + @pytest.mark.xfail + def test_xpass(): + assert 1 + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(name="pytest", errors=0, failures=1, skipped=2, tests=5) + + @parametrize_families + def test_summing_simple_with_errors( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture + def fixture(): + raise Exception() + def test_pass(): + pass + def test_fail(): + assert 0 + def test_error(fixture): + pass + @pytest.mark.xfail + def test_xfail(): + assert False + @pytest.mark.xfail(strict=True) + def test_xpass(): + assert True + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(name="pytest", errors=1, failures=2, skipped=1, tests=5) + + @parametrize_families + def test_hostname_in_xml( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + def test_pass(): + pass + """ + ) + result, dom = run_and_parse(family=xunit_family) + node = dom.find_first_by_tag("testsuite") + node.assert_attr(hostname=platform.node()) + + @parametrize_families + def test_timestamp_in_xml( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + def test_pass(): + pass + """ + ) + start_time = datetime.now() + result, dom = run_and_parse(family=xunit_family) + node = dom.find_first_by_tag("testsuite") + timestamp = datetime.strptime(node["timestamp"], "%Y-%m-%dT%H:%M:%S.%f") + assert start_time <= timestamp < datetime.now() + + def test_timing_function( + self, pytester: Pytester, run_and_parse: RunAndParse, mock_timing + ) -> None: + pytester.makepyfile( + """ + from _pytest import timing + def setup_module(): + timing.sleep(1) + def teardown_module(): + timing.sleep(2) + def test_sleep(): + timing.sleep(4) + """ + ) + result, dom = run_and_parse() + node = dom.find_first_by_tag("testsuite") + tnode = node.find_first_by_tag("testcase") + val = tnode["time"] + assert float(val) == 7.0 + + @pytest.mark.parametrize("duration_report", ["call", "total"]) + def test_junit_duration_report( + self, + pytester: Pytester, + monkeypatch: MonkeyPatch, + duration_report: str, + run_and_parse: RunAndParse, + ) -> None: + + # mock LogXML.node_reporter so it always sets a known duration to each test report object + original_node_reporter = LogXML.node_reporter + + def node_reporter_wrapper(s, report): + report.duration = 1.0 + reporter = original_node_reporter(s, report) + return reporter + + monkeypatch.setattr(LogXML, "node_reporter", node_reporter_wrapper) + + pytester.makepyfile( + """ + def test_foo(): + pass + """ + ) + result, dom = run_and_parse("-o", f"junit_duration_report={duration_report}") + node = dom.find_first_by_tag("testsuite") + tnode = node.find_first_by_tag("testcase") + val = float(tnode["time"]) + if duration_report == "total": + assert val == 3.0 + else: + assert duration_report == "call" + assert val == 1.0 + + @parametrize_families + def test_setup_error( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def arg(request): + raise ValueError("Error reason") + def test_function(arg): + pass + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(errors=1, tests=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(classname="test_setup_error", name="test_function") + fnode = tnode.find_first_by_tag("error") + fnode.assert_attr(message='failed on setup with "ValueError: Error reason"') + assert "ValueError" in fnode.toxml() + + @parametrize_families + def test_teardown_error( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def arg(): + yield + raise ValueError('Error reason') + def test_function(arg): + pass + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret + node = dom.find_first_by_tag("testsuite") + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(classname="test_teardown_error", name="test_function") + fnode = tnode.find_first_by_tag("error") + fnode.assert_attr(message='failed on teardown with "ValueError: Error reason"') + assert "ValueError" in fnode.toxml() + + @parametrize_families + def test_call_failure_teardown_error( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def arg(): + yield + raise Exception("Teardown Exception") + def test_function(arg): + raise Exception("Call Exception") + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(errors=1, failures=1, tests=1) + first, second = dom.find_by_tag("testcase") + assert first + assert second + assert first != second + fnode = first.find_first_by_tag("failure") + fnode.assert_attr(message="Exception: Call Exception") + snode = second.find_first_by_tag("error") + snode.assert_attr( + message='failed on teardown with "Exception: Teardown Exception"' + ) + + @parametrize_families + def test_skip_contains_name_reason( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + def test_skip(): + pytest.skip("hello23") + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret == 0 + node = dom.find_first_by_tag("testsuite") + node.assert_attr(skipped=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(classname="test_skip_contains_name_reason", name="test_skip") + snode = tnode.find_first_by_tag("skipped") + snode.assert_attr(type="pytest.skip", message="hello23") + + @parametrize_families + def test_mark_skip_contains_name_reason( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skip(reason="hello24") + def test_skip(): + assert True + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret == 0 + node = dom.find_first_by_tag("testsuite") + node.assert_attr(skipped=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr( + classname="test_mark_skip_contains_name_reason", name="test_skip" + ) + snode = tnode.find_first_by_tag("skipped") + snode.assert_attr(type="pytest.skip", message="hello24") + + @parametrize_families + def test_mark_skipif_contains_name_reason( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + GLOBAL_CONDITION = True + @pytest.mark.skipif(GLOBAL_CONDITION, reason="hello25") + def test_skip(): + assert True + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret == 0 + node = dom.find_first_by_tag("testsuite") + node.assert_attr(skipped=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr( + classname="test_mark_skipif_contains_name_reason", name="test_skip" + ) + snode = tnode.find_first_by_tag("skipped") + snode.assert_attr(type="pytest.skip", message="hello25") + + @parametrize_families + def test_mark_skip_doesnt_capture_output( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skip(reason="foo") + def test_skip(): + print("bar!") + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret == 0 + node_xml = dom.find_first_by_tag("testsuite").toxml() + assert "bar!" not in node_xml + + @parametrize_families + def test_classname_instance( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + class TestClass(object): + def test_method(self): + assert 0 + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(failures=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr( + classname="test_classname_instance.TestClass", name="test_method" + ) + + @parametrize_families + def test_classname_nested_dir( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + p = pytester.mkdir("sub").joinpath("test_hello.py") + p.write_text("def test_func(): 0/0") + result, dom = run_and_parse(family=xunit_family) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(failures=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(classname="sub.test_hello", name="test_func") + + @parametrize_families + def test_internal_error( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makeconftest("def pytest_runtest_protocol(): 0 / 0") + pytester.makepyfile("def test_function(): pass") + result, dom = run_and_parse(family=xunit_family) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(errors=1, tests=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(classname="pytest", name="internal") + fnode = tnode.find_first_by_tag("error") + fnode.assert_attr(message="internal error") + assert "Division" in fnode.toxml() + + @pytest.mark.parametrize( + "junit_logging", ["no", "log", "system-out", "system-err", "out-err", "all"] + ) + @parametrize_families + def test_failure_function( + self, + pytester: Pytester, + junit_logging, + run_and_parse: RunAndParse, + xunit_family, + ) -> None: + pytester.makepyfile( + """ + import logging + import sys + + def test_fail(): + print("hello-stdout") + sys.stderr.write("hello-stderr\\n") + logging.info('info msg') + logging.warning('warning msg') + raise ValueError(42) + """ + ) + + result, dom = run_and_parse( + "-o", "junit_logging=%s" % junit_logging, family=xunit_family + ) + assert result.ret, "Expected ret > 0" + node = dom.find_first_by_tag("testsuite") + node.assert_attr(failures=1, tests=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(classname="test_failure_function", name="test_fail") + fnode = tnode.find_first_by_tag("failure") + fnode.assert_attr(message="ValueError: 42") + assert "ValueError" in fnode.toxml(), "ValueError not included" + + if junit_logging in ["log", "all"]: + logdata = tnode.find_first_by_tag("system-out") + log_xml = logdata.toxml() + assert logdata.tag == "system-out", "Expected tag: system-out" + assert "info msg" not in log_xml, "Unexpected INFO message" + assert "warning msg" in log_xml, "Missing WARN message" + if junit_logging in ["system-out", "out-err", "all"]: + systemout = tnode.find_first_by_tag("system-out") + systemout_xml = systemout.toxml() + assert systemout.tag == "system-out", "Expected tag: system-out" + assert "info msg" not in systemout_xml, "INFO message found in system-out" + assert ( + "hello-stdout" in systemout_xml + ), "Missing 'hello-stdout' in system-out" + if junit_logging in ["system-err", "out-err", "all"]: + systemerr = tnode.find_first_by_tag("system-err") + systemerr_xml = systemerr.toxml() + assert systemerr.tag == "system-err", "Expected tag: system-err" + assert "info msg" not in systemerr_xml, "INFO message found in system-err" + assert ( + "hello-stderr" in systemerr_xml + ), "Missing 'hello-stderr' in system-err" + assert ( + "warning msg" not in systemerr_xml + ), "WARN message found in system-err" + if junit_logging == "no": + assert not tnode.find_by_tag("log"), "Found unexpected content: log" + assert not tnode.find_by_tag( + "system-out" + ), "Found unexpected content: system-out" + assert not tnode.find_by_tag( + "system-err" + ), "Found unexpected content: system-err" + + @parametrize_families + def test_failure_verbose_message( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import sys + def test_fail(): + assert 0, "An error" + """ + ) + result, dom = run_and_parse(family=xunit_family) + node = dom.find_first_by_tag("testsuite") + tnode = node.find_first_by_tag("testcase") + fnode = tnode.find_first_by_tag("failure") + fnode.assert_attr(message="AssertionError: An error\nassert 0") + + @parametrize_families + def test_failure_escape( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize('arg1', "<&'", ids="<&'") + def test_func(arg1): + print(arg1) + assert 0 + """ + ) + result, dom = run_and_parse( + "-o", "junit_logging=system-out", family=xunit_family + ) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(failures=3, tests=3) + + for index, char in enumerate("<&'"): + + tnode = node.find_nth_by_tag("testcase", index) + tnode.assert_attr( + classname="test_failure_escape", name="test_func[%s]" % char + ) + sysout = tnode.find_first_by_tag("system-out") + text = sysout.text + assert "%s\n" % char in text + + @parametrize_families + def test_junit_prefixing( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + def test_func(): + assert 0 + class TestHello(object): + def test_hello(self): + pass + """ + ) + result, dom = run_and_parse("--junitprefix=xyz", family=xunit_family) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(failures=1, tests=2) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(classname="xyz.test_junit_prefixing", name="test_func") + tnode = node.find_nth_by_tag("testcase", 1) + tnode.assert_attr( + classname="xyz.test_junit_prefixing.TestHello", name="test_hello" + ) + + @parametrize_families + def test_xfailure_function( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + def test_xfail(): + pytest.xfail("42") + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert not result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(skipped=1, tests=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(classname="test_xfailure_function", name="test_xfail") + fnode = tnode.find_first_by_tag("skipped") + fnode.assert_attr(type="pytest.xfail", message="42") + + @parametrize_families + def test_xfailure_marker( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail(reason="42") + def test_xfail(): + assert False + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert not result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(skipped=1, tests=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(classname="test_xfailure_marker", name="test_xfail") + fnode = tnode.find_first_by_tag("skipped") + fnode.assert_attr(type="pytest.xfail", message="42") + + @pytest.mark.parametrize( + "junit_logging", ["no", "log", "system-out", "system-err", "out-err", "all"] + ) + def test_xfail_captures_output_once( + self, pytester: Pytester, junit_logging: str, run_and_parse: RunAndParse + ) -> None: + pytester.makepyfile( + """ + import sys + import pytest + + @pytest.mark.xfail() + def test_fail(): + sys.stdout.write('XFAIL This is stdout') + sys.stderr.write('XFAIL This is stderr') + assert 0 + """ + ) + result, dom = run_and_parse("-o", "junit_logging=%s" % junit_logging) + node = dom.find_first_by_tag("testsuite") + tnode = node.find_first_by_tag("testcase") + if junit_logging in ["system-err", "out-err", "all"]: + assert len(tnode.find_by_tag("system-err")) == 1 + else: + assert len(tnode.find_by_tag("system-err")) == 0 + + if junit_logging in ["log", "system-out", "out-err", "all"]: + assert len(tnode.find_by_tag("system-out")) == 1 + else: + assert len(tnode.find_by_tag("system-out")) == 0 + + @parametrize_families + def test_xfailure_xpass( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail + def test_xpass(): + pass + """ + ) + result, dom = run_and_parse(family=xunit_family) + # assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(skipped=0, tests=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(classname="test_xfailure_xpass", name="test_xpass") + + @parametrize_families + def test_xfailure_xpass_strict( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail(strict=True, reason="This needs to fail!") + def test_xpass(): + pass + """ + ) + result, dom = run_and_parse(family=xunit_family) + # assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(skipped=0, tests=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(classname="test_xfailure_xpass_strict", name="test_xpass") + fnode = tnode.find_first_by_tag("failure") + fnode.assert_attr(message="[XPASS(strict)] This needs to fail!") + + @parametrize_families + def test_collect_error( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makepyfile("syntax error") + result, dom = run_and_parse(family=xunit_family) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(errors=1, tests=1) + tnode = node.find_first_by_tag("testcase") + fnode = tnode.find_first_by_tag("error") + fnode.assert_attr(message="collection failure") + assert "SyntaxError" in fnode.toxml() + + def test_unicode(self, pytester: Pytester, run_and_parse: RunAndParse) -> None: + value = "hx\xc4\x85\xc4\x87\n" + pytester.makepyfile( + """\ + # coding: latin1 + def test_hello(): + print(%r) + assert 0 + """ + % value + ) + result, dom = run_and_parse() + assert result.ret == 1 + tnode = dom.find_first_by_tag("testcase") + fnode = tnode.find_first_by_tag("failure") + assert "hx" in fnode.toxml() + + def test_assertion_binchars( + self, pytester: Pytester, run_and_parse: RunAndParse + ) -> None: + """This test did fail when the escaping wasn't strict.""" + pytester.makepyfile( + """ + + M1 = '\x01\x02\x03\x04' + M2 = '\x01\x02\x03\x05' + + def test_str_compare(): + assert M1 == M2 + """ + ) + result, dom = run_and_parse() + print(dom.toxml()) + + @pytest.mark.parametrize("junit_logging", ["no", "system-out"]) + def test_pass_captures_stdout( + self, pytester: Pytester, run_and_parse: RunAndParse, junit_logging: str + ) -> None: + pytester.makepyfile( + """ + def test_pass(): + print('hello-stdout') + """ + ) + result, dom = run_and_parse("-o", "junit_logging=%s" % junit_logging) + node = dom.find_first_by_tag("testsuite") + pnode = node.find_first_by_tag("testcase") + if junit_logging == "no": + assert not node.find_by_tag( + "system-out" + ), "system-out should not be generated" + if junit_logging == "system-out": + systemout = pnode.find_first_by_tag("system-out") + assert ( + "hello-stdout" in systemout.toxml() + ), "'hello-stdout' should be in system-out" + + @pytest.mark.parametrize("junit_logging", ["no", "system-err"]) + def test_pass_captures_stderr( + self, pytester: Pytester, run_and_parse: RunAndParse, junit_logging: str + ) -> None: + pytester.makepyfile( + """ + import sys + def test_pass(): + sys.stderr.write('hello-stderr') + """ + ) + result, dom = run_and_parse("-o", "junit_logging=%s" % junit_logging) + node = dom.find_first_by_tag("testsuite") + pnode = node.find_first_by_tag("testcase") + if junit_logging == "no": + assert not node.find_by_tag( + "system-err" + ), "system-err should not be generated" + if junit_logging == "system-err": + systemerr = pnode.find_first_by_tag("system-err") + assert ( + "hello-stderr" in systemerr.toxml() + ), "'hello-stderr' should be in system-err" + + @pytest.mark.parametrize("junit_logging", ["no", "system-out"]) + def test_setup_error_captures_stdout( + self, pytester: Pytester, run_and_parse: RunAndParse, junit_logging: str + ) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def arg(request): + print('hello-stdout') + raise ValueError() + def test_function(arg): + pass + """ + ) + result, dom = run_and_parse("-o", "junit_logging=%s" % junit_logging) + node = dom.find_first_by_tag("testsuite") + pnode = node.find_first_by_tag("testcase") + if junit_logging == "no": + assert not node.find_by_tag( + "system-out" + ), "system-out should not be generated" + if junit_logging == "system-out": + systemout = pnode.find_first_by_tag("system-out") + assert ( + "hello-stdout" in systemout.toxml() + ), "'hello-stdout' should be in system-out" + + @pytest.mark.parametrize("junit_logging", ["no", "system-err"]) + def test_setup_error_captures_stderr( + self, pytester: Pytester, run_and_parse: RunAndParse, junit_logging: str + ) -> None: + pytester.makepyfile( + """ + import sys + import pytest + + @pytest.fixture + def arg(request): + sys.stderr.write('hello-stderr') + raise ValueError() + def test_function(arg): + pass + """ + ) + result, dom = run_and_parse("-o", "junit_logging=%s" % junit_logging) + node = dom.find_first_by_tag("testsuite") + pnode = node.find_first_by_tag("testcase") + if junit_logging == "no": + assert not node.find_by_tag( + "system-err" + ), "system-err should not be generated" + if junit_logging == "system-err": + systemerr = pnode.find_first_by_tag("system-err") + assert ( + "hello-stderr" in systemerr.toxml() + ), "'hello-stderr' should be in system-err" + + @pytest.mark.parametrize("junit_logging", ["no", "system-out"]) + def test_avoid_double_stdout( + self, pytester: Pytester, run_and_parse: RunAndParse, junit_logging: str + ) -> None: + pytester.makepyfile( + """ + import sys + import pytest + + @pytest.fixture + def arg(request): + yield + sys.stdout.write('hello-stdout teardown') + raise ValueError() + def test_function(arg): + sys.stdout.write('hello-stdout call') + """ + ) + result, dom = run_and_parse("-o", "junit_logging=%s" % junit_logging) + node = dom.find_first_by_tag("testsuite") + pnode = node.find_first_by_tag("testcase") + if junit_logging == "no": + assert not node.find_by_tag( + "system-out" + ), "system-out should not be generated" + if junit_logging == "system-out": + systemout = pnode.find_first_by_tag("system-out") + assert "hello-stdout call" in systemout.toxml() + assert "hello-stdout teardown" in systemout.toxml() + + +def test_mangle_test_address() -> None: + from _pytest.junitxml import mangle_test_address + + address = "::".join(["a/my.py.thing.py", "Class", "method", "[a-1-::]"]) + newnames = mangle_test_address(address) + assert newnames == ["a.my.py.thing", "Class", "method", "[a-1-::]"] + + +def test_dont_configure_on_workers(tmp_path: Path) -> None: + gotten: List[object] = [] + + class FakeConfig: + if TYPE_CHECKING: + workerinput = None + + def __init__(self): + self.pluginmanager = self + self.option = self + self.stash = Stash() + + def getini(self, name): + return "pytest" + + junitprefix = None + # XXX: shouldn't need tmp_path ? + xmlpath = str(tmp_path.joinpath("junix.xml")) + register = gotten.append + + fake_config = cast(Config, FakeConfig()) + from _pytest import junitxml + + junitxml.pytest_configure(fake_config) + assert len(gotten) == 1 + FakeConfig.workerinput = None + junitxml.pytest_configure(fake_config) + assert len(gotten) == 1 + + +class TestNonPython: + @parametrize_families + def test_summing_simple( + self, pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str + ) -> None: + pytester.makeconftest( + """ + import pytest + def pytest_collect_file(file_path, parent): + if file_path.suffix == ".xyz": + return MyItem.from_parent(name=file_path.name, parent=parent) + class MyItem(pytest.Item): + def runtest(self): + raise ValueError(42) + def repr_failure(self, excinfo): + return "custom item runtest failed" + """ + ) + pytester.path.joinpath("myfile.xyz").write_text("hello") + result, dom = run_and_parse(family=xunit_family) + assert result.ret + node = dom.find_first_by_tag("testsuite") + node.assert_attr(errors=0, failures=1, skipped=0, tests=1) + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(name="myfile.xyz") + fnode = tnode.find_first_by_tag("failure") + fnode.assert_attr(message="custom item runtest failed") + assert "custom item runtest failed" in fnode.toxml() + + +@pytest.mark.parametrize("junit_logging", ["no", "system-out"]) +def test_nullbyte(pytester: Pytester, junit_logging: str) -> None: + # A null byte can not occur in XML (see section 2.2 of the spec) + pytester.makepyfile( + """ + import sys + def test_print_nullbyte(): + sys.stdout.write('Here the null -->' + chr(0) + '<--') + sys.stdout.write('In repr form -->' + repr(chr(0)) + '<--') + assert False + """ + ) + xmlf = pytester.path.joinpath("junit.xml") + pytester.runpytest("--junitxml=%s" % xmlf, "-o", "junit_logging=%s" % junit_logging) + text = xmlf.read_text() + assert "\x00" not in text + if junit_logging == "system-out": + assert "#x00" in text + if junit_logging == "no": + assert "#x00" not in text + + +@pytest.mark.parametrize("junit_logging", ["no", "system-out"]) +def test_nullbyte_replace(pytester: Pytester, junit_logging: str) -> None: + # Check if the null byte gets replaced + pytester.makepyfile( + """ + import sys + def test_print_nullbyte(): + sys.stdout.write('Here the null -->' + chr(0) + '<--') + sys.stdout.write('In repr form -->' + repr(chr(0)) + '<--') + assert False + """ + ) + xmlf = pytester.path.joinpath("junit.xml") + pytester.runpytest("--junitxml=%s" % xmlf, "-o", "junit_logging=%s" % junit_logging) + text = xmlf.read_text() + if junit_logging == "system-out": + assert "#x0" in text + if junit_logging == "no": + assert "#x0" not in text + + +def test_invalid_xml_escape() -> None: + # Test some more invalid xml chars, the full range should be + # tested really but let's just test the edges of the ranges + # instead. + # XXX This only tests low unicode character points for now as + # there are some issues with the testing infrastructure for + # the higher ones. + # XXX Testing 0xD (\r) is tricky as it overwrites the just written + # line in the output, so we skip it too. + invalid = ( + 0x00, + 0x1, + 0xB, + 0xC, + 0xE, + 0x19, + 27, # issue #126 + 0xD800, + 0xDFFF, + 0xFFFE, + 0x0FFFF, + ) # , 0x110000) + valid = (0x9, 0xA, 0x20) + # 0xD, 0xD7FF, 0xE000, 0xFFFD, 0x10000, 0x10FFFF) + + for i in invalid: + got = bin_xml_escape(chr(i)) + if i <= 0xFF: + expected = "#x%02X" % i + else: + expected = "#x%04X" % i + assert got == expected + for i in valid: + assert chr(i) == bin_xml_escape(chr(i)) + + +def test_logxml_path_expansion(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: + home_tilde = Path(os.path.expanduser("~")).joinpath("test.xml") + xml_tilde = LogXML(Path("~", "test.xml"), None) + assert xml_tilde.logfile == str(home_tilde) + + monkeypatch.setenv("HOME", str(tmp_path)) + home_var = os.path.normpath(os.path.expandvars("$HOME/test.xml")) + xml_var = LogXML(Path("$HOME", "test.xml"), None) + assert xml_var.logfile == str(home_var) + + +def test_logxml_changingdir(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_func(): + import os + os.chdir("a") + """ + ) + pytester.mkdir("a") + result = pytester.runpytest("--junitxml=a/x.xml") + assert result.ret == 0 + assert pytester.path.joinpath("a/x.xml").exists() + + +def test_logxml_makedir(pytester: Pytester) -> None: + """--junitxml should automatically create directories for the xml file""" + pytester.makepyfile( + """ + def test_pass(): + pass + """ + ) + result = pytester.runpytest("--junitxml=path/to/results.xml") + assert result.ret == 0 + assert pytester.path.joinpath("path/to/results.xml").exists() + + +def test_logxml_check_isdir(pytester: Pytester) -> None: + """Give an error if --junit-xml is a directory (#2089)""" + result = pytester.runpytest("--junit-xml=.") + result.stderr.fnmatch_lines(["*--junitxml must be a filename*"]) + + +def test_escaped_parametrized_names_xml( + pytester: Pytester, run_and_parse: RunAndParse +) -> None: + pytester.makepyfile( + """\ + import pytest + @pytest.mark.parametrize('char', ["\\x00"]) + def test_func(char): + assert char + """ + ) + result, dom = run_and_parse() + assert result.ret == 0 + node = dom.find_first_by_tag("testcase") + node.assert_attr(name="test_func[\\x00]") + + +def test_double_colon_split_function_issue469( + pytester: Pytester, run_and_parse: RunAndParse +) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize('param', ["double::colon"]) + def test_func(param): + pass + """ + ) + result, dom = run_and_parse() + assert result.ret == 0 + node = dom.find_first_by_tag("testcase") + node.assert_attr(classname="test_double_colon_split_function_issue469") + node.assert_attr(name="test_func[double::colon]") + + +def test_double_colon_split_method_issue469( + pytester: Pytester, run_and_parse: RunAndParse +) -> None: + pytester.makepyfile( + """ + import pytest + class TestClass(object): + @pytest.mark.parametrize('param', ["double::colon"]) + def test_func(self, param): + pass + """ + ) + result, dom = run_and_parse() + assert result.ret == 0 + node = dom.find_first_by_tag("testcase") + node.assert_attr(classname="test_double_colon_split_method_issue469.TestClass") + node.assert_attr(name="test_func[double::colon]") + + +def test_unicode_issue368(pytester: Pytester) -> None: + path = pytester.path.joinpath("test.xml") + log = LogXML(str(path), None) + ustr = "ВНИ!" + + class Report(BaseReport): + longrepr = ustr + sections: List[Tuple[str, str]] = [] + nodeid = "something" + location = "tests/filename.py", 42, "TestClass.method" + when = "teardown" + + test_report = cast(TestReport, Report()) + + # hopefully this is not too brittle ... + log.pytest_sessionstart() + node_reporter = log._opentestcase(test_report) + node_reporter.append_failure(test_report) + node_reporter.append_collect_error(test_report) + node_reporter.append_collect_skipped(test_report) + node_reporter.append_error(test_report) + test_report.longrepr = "filename", 1, ustr + node_reporter.append_skipped(test_report) + test_report.longrepr = "filename", 1, "Skipped: 卡嘣嘣" + node_reporter.append_skipped(test_report) + test_report.wasxfail = ustr # type: ignore[attr-defined] + node_reporter.append_skipped(test_report) + log.pytest_sessionfinish() + + +def test_record_property(pytester: Pytester, run_and_parse: RunAndParse) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def other(record_property): + record_property("bar", 1) + def test_record(record_property, other): + record_property("foo", "<1"); + """ + ) + result, dom = run_and_parse() + node = dom.find_first_by_tag("testsuite") + tnode = node.find_first_by_tag("testcase") + psnode = tnode.find_first_by_tag("properties") + pnodes = psnode.find_by_tag("property") + pnodes[0].assert_attr(name="bar", value="1") + pnodes[1].assert_attr(name="foo", value="<1") + result.stdout.fnmatch_lines(["*= 1 passed in *"]) + + +def test_record_property_same_name( + pytester: Pytester, run_and_parse: RunAndParse +) -> None: + pytester.makepyfile( + """ + def test_record_with_same_name(record_property): + record_property("foo", "bar") + record_property("foo", "baz") + """ + ) + result, dom = run_and_parse() + node = dom.find_first_by_tag("testsuite") + tnode = node.find_first_by_tag("testcase") + psnode = tnode.find_first_by_tag("properties") + pnodes = psnode.find_by_tag("property") + pnodes[0].assert_attr(name="foo", value="bar") + pnodes[1].assert_attr(name="foo", value="baz") + + +@pytest.mark.parametrize("fixture_name", ["record_property", "record_xml_attribute"]) +def test_record_fixtures_without_junitxml( + pytester: Pytester, fixture_name: str +) -> None: + pytester.makepyfile( + """ + def test_record({fixture_name}): + {fixture_name}("foo", "bar") + """.format( + fixture_name=fixture_name + ) + ) + result = pytester.runpytest() + assert result.ret == 0 + + +@pytest.mark.filterwarnings("default") +def test_record_attribute(pytester: Pytester, run_and_parse: RunAndParse) -> None: + pytester.makeini( + """ + [pytest] + junit_family = xunit1 + """ + ) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def other(record_xml_attribute): + record_xml_attribute("bar", 1) + def test_record(record_xml_attribute, other): + record_xml_attribute("foo", "<1"); + """ + ) + result, dom = run_and_parse() + node = dom.find_first_by_tag("testsuite") + tnode = node.find_first_by_tag("testcase") + tnode.assert_attr(bar="1") + tnode.assert_attr(foo="<1") + result.stdout.fnmatch_lines( + ["*test_record_attribute.py:6:*record_xml_attribute is an experimental feature"] + ) + + +@pytest.mark.filterwarnings("default") +@pytest.mark.parametrize("fixture_name", ["record_xml_attribute", "record_property"]) +def test_record_fixtures_xunit2( + pytester: Pytester, fixture_name: str, run_and_parse: RunAndParse +) -> None: + """Ensure record_xml_attribute and record_property drop values when outside of legacy family.""" + pytester.makeini( + """ + [pytest] + junit_family = xunit2 + """ + ) + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def other({fixture_name}): + {fixture_name}("bar", 1) + def test_record({fixture_name}, other): + {fixture_name}("foo", "<1"); + """.format( + fixture_name=fixture_name + ) + ) + + result, dom = run_and_parse(family=None) + expected_lines = [] + if fixture_name == "record_xml_attribute": + expected_lines.append( + "*test_record_fixtures_xunit2.py:6:*record_xml_attribute is an experimental feature" + ) + expected_lines = [ + "*test_record_fixtures_xunit2.py:6:*{fixture_name} is incompatible " + "with junit_family 'xunit2' (use 'legacy' or 'xunit1')".format( + fixture_name=fixture_name + ) + ] + result.stdout.fnmatch_lines(expected_lines) + + +def test_random_report_log_xdist( + pytester: Pytester, monkeypatch: MonkeyPatch, run_and_parse: RunAndParse +) -> None: + """`xdist` calls pytest_runtest_logreport as they are executed by the workers, + with nodes from several nodes overlapping, so junitxml must cope with that + to produce correct reports (#1064).""" + pytest.importorskip("xdist") + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + pytester.makepyfile( + """ + import pytest, time + @pytest.mark.parametrize('i', list(range(30))) + def test_x(i): + assert i != 22 + """ + ) + _, dom = run_and_parse("-n2") + suite_node = dom.find_first_by_tag("testsuite") + failed = [] + for case_node in suite_node.find_by_tag("testcase"): + if case_node.find_first_by_tag("failure"): + failed.append(case_node["name"]) + + assert failed == ["test_x[22]"] + + +@parametrize_families +def test_root_testsuites_tag( + pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str +) -> None: + pytester.makepyfile( + """ + def test_x(): + pass + """ + ) + _, dom = run_and_parse(family=xunit_family) + root = dom.get_unique_child + assert root.tag == "testsuites" + suite_node = root.get_unique_child + assert suite_node.tag == "testsuite" + + +def test_runs_twice(pytester: Pytester, run_and_parse: RunAndParse) -> None: + f = pytester.makepyfile( + """ + def test_pass(): + pass + """ + ) + + result, dom = run_and_parse(f, f) + result.stdout.no_fnmatch_line("*INTERNALERROR*") + first, second = (x["classname"] for x in dom.find_by_tag("testcase")) + assert first == second + + +def test_runs_twice_xdist( + pytester: Pytester, monkeypatch: MonkeyPatch, run_and_parse: RunAndParse +) -> None: + pytest.importorskip("xdist") + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + f = pytester.makepyfile( + """ + def test_pass(): + pass + """ + ) + + result, dom = run_and_parse(f, "--dist", "each", "--tx", "2*popen") + result.stdout.no_fnmatch_line("*INTERNALERROR*") + first, second = (x["classname"] for x in dom.find_by_tag("testcase")) + assert first == second + + +def test_fancy_items_regression(pytester: Pytester, run_and_parse: RunAndParse) -> None: + # issue 1259 + pytester.makeconftest( + """ + import pytest + class FunItem(pytest.Item): + def runtest(self): + pass + class NoFunItem(pytest.Item): + def runtest(self): + pass + + class FunCollector(pytest.File): + def collect(self): + return [ + FunItem.from_parent(name='a', parent=self), + NoFunItem.from_parent(name='a', parent=self), + NoFunItem.from_parent(name='b', parent=self), + ] + + def pytest_collect_file(file_path, parent): + if file_path.suffix == '.py': + return FunCollector.from_parent(path=file_path, parent=parent) + """ + ) + + pytester.makepyfile( + """ + def test_pass(): + pass + """ + ) + + result, dom = run_and_parse() + + result.stdout.no_fnmatch_line("*INTERNALERROR*") + + items = sorted("%(classname)s %(name)s" % x for x in dom.find_by_tag("testcase")) + import pprint + + pprint.pprint(items) + assert items == [ + "conftest a", + "conftest a", + "conftest b", + "test_fancy_items_regression a", + "test_fancy_items_regression a", + "test_fancy_items_regression b", + "test_fancy_items_regression test_pass", + ] + + +@parametrize_families +def test_global_properties(pytester: Pytester, xunit_family: str) -> None: + path = pytester.path.joinpath("test_global_properties.xml") + log = LogXML(str(path), None, family=xunit_family) + + class Report(BaseReport): + sections: List[Tuple[str, str]] = [] + nodeid = "test_node_id" + + log.pytest_sessionstart() + log.add_global_property("foo", "1") + log.add_global_property("bar", "2") + log.pytest_sessionfinish() + + dom = minidom.parse(str(path)) + + properties = dom.getElementsByTagName("properties") + + assert properties.length == 1, "There must be one node" + + property_list = dom.getElementsByTagName("property") + + assert property_list.length == 2, "There most be only 2 property nodes" + + expected = {"foo": "1", "bar": "2"} + actual = {} + + for p in property_list: + k = str(p.getAttribute("name")) + v = str(p.getAttribute("value")) + actual[k] = v + + assert actual == expected + + +def test_url_property(pytester: Pytester) -> None: + test_url = "http://www.github.com/pytest-dev" + path = pytester.path.joinpath("test_url_property.xml") + log = LogXML(str(path), None) + + class Report(BaseReport): + longrepr = "FooBarBaz" + sections: List[Tuple[str, str]] = [] + nodeid = "something" + location = "tests/filename.py", 42, "TestClass.method" + url = test_url + + test_report = cast(TestReport, Report()) + + log.pytest_sessionstart() + node_reporter = log._opentestcase(test_report) + node_reporter.append_failure(test_report) + log.pytest_sessionfinish() + + test_case = minidom.parse(str(path)).getElementsByTagName("testcase")[0] + + assert ( + test_case.getAttribute("url") == test_url + ), "The URL did not get written to the xml" + + +@parametrize_families +def test_record_testsuite_property( + pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str +) -> None: + pytester.makepyfile( + """ + def test_func1(record_testsuite_property): + record_testsuite_property("stats", "all good") + + def test_func2(record_testsuite_property): + record_testsuite_property("stats", 10) + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret == 0 + node = dom.find_first_by_tag("testsuite") + properties_node = node.find_first_by_tag("properties") + p1_node = properties_node.find_nth_by_tag("property", 0) + p2_node = properties_node.find_nth_by_tag("property", 1) + p1_node.assert_attr(name="stats", value="all good") + p2_node.assert_attr(name="stats", value="10") + + +def test_record_testsuite_property_junit_disabled(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_func1(record_testsuite_property): + record_testsuite_property("stats", "all good") + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + + +@pytest.mark.parametrize("junit", [True, False]) +def test_record_testsuite_property_type_checking( + pytester: Pytester, junit: bool +) -> None: + pytester.makepyfile( + """ + def test_func1(record_testsuite_property): + record_testsuite_property(1, 2) + """ + ) + args = ("--junitxml=tests.xml",) if junit else () + result = pytester.runpytest(*args) + assert result.ret == 1 + result.stdout.fnmatch_lines( + ["*TypeError: name parameter needs to be a string, but int given"] + ) + + +@pytest.mark.parametrize("suite_name", ["my_suite", ""]) +@parametrize_families +def test_set_suite_name( + pytester: Pytester, suite_name: str, run_and_parse: RunAndParse, xunit_family: str +) -> None: + if suite_name: + pytester.makeini( + """ + [pytest] + junit_suite_name={suite_name} + junit_family={family} + """.format( + suite_name=suite_name, family=xunit_family + ) + ) + expected = suite_name + else: + expected = "pytest" + pytester.makepyfile( + """ + import pytest + + def test_func(): + pass + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret == 0 + node = dom.find_first_by_tag("testsuite") + node.assert_attr(name=expected) + + +def test_escaped_skipreason_issue3533( + pytester: Pytester, run_and_parse: RunAndParse +) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skip(reason='1 <> 2') + def test_skip(): + pass + """ + ) + _, dom = run_and_parse() + node = dom.find_first_by_tag("testcase") + snode = node.find_first_by_tag("skipped") + assert "1 <> 2" in snode.text + snode.assert_attr(message="1 <> 2") + + +@parametrize_families +def test_logging_passing_tests_disabled_does_not_log_test_output( + pytester: Pytester, run_and_parse: RunAndParse, xunit_family: str +) -> None: + pytester.makeini( + """ + [pytest] + junit_log_passing_tests=False + junit_logging=system-out + junit_family={family} + """.format( + family=xunit_family + ) + ) + pytester.makepyfile( + """ + import pytest + import logging + import sys + + def test_func(): + sys.stdout.write('This is stdout') + sys.stderr.write('This is stderr') + logging.warning('hello') + """ + ) + result, dom = run_and_parse(family=xunit_family) + assert result.ret == 0 + node = dom.find_first_by_tag("testcase") + assert len(node.find_by_tag("system-err")) == 0 + assert len(node.find_by_tag("system-out")) == 0 + + +@parametrize_families +@pytest.mark.parametrize("junit_logging", ["no", "system-out", "system-err"]) +def test_logging_passing_tests_disabled_logs_output_for_failing_test_issue5430( + pytester: Pytester, + junit_logging: str, + run_and_parse: RunAndParse, + xunit_family: str, +) -> None: + pytester.makeini( + """ + [pytest] + junit_log_passing_tests=False + junit_family={family} + """.format( + family=xunit_family + ) + ) + pytester.makepyfile( + """ + import pytest + import logging + import sys + + def test_func(): + logging.warning('hello') + assert 0 + """ + ) + result, dom = run_and_parse( + "-o", "junit_logging=%s" % junit_logging, family=xunit_family + ) + assert result.ret == 1 + node = dom.find_first_by_tag("testcase") + if junit_logging == "system-out": + assert len(node.find_by_tag("system-err")) == 0 + assert len(node.find_by_tag("system-out")) == 1 + elif junit_logging == "system-err": + assert len(node.find_by_tag("system-err")) == 1 + assert len(node.find_by_tag("system-out")) == 0 + else: + assert junit_logging == "no" + assert len(node.find_by_tag("system-err")) == 0 + assert len(node.find_by_tag("system-out")) == 0 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_legacypath.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_legacypath.py new file mode 100644 index 0000000000..8acafe98e7 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_legacypath.py @@ -0,0 +1,180 @@ +from pathlib import Path + +import pytest +from _pytest.compat import LEGACY_PATH +from _pytest.legacypath import TempdirFactory +from _pytest.legacypath import Testdir + + +def test_item_fspath(pytester: pytest.Pytester) -> None: + pytester.makepyfile("def test_func(): pass") + items, hookrec = pytester.inline_genitems() + assert len(items) == 1 + (item,) = items + items2, hookrec = pytester.inline_genitems(item.nodeid) + (item2,) = items2 + assert item2.name == item.name + assert item2.fspath == item.fspath # type: ignore[attr-defined] + assert item2.path == item.path + + +def test_testdir_testtmproot(testdir: Testdir) -> None: + """Check test_tmproot is a py.path attribute for backward compatibility.""" + assert testdir.test_tmproot.check(dir=1) + + +def test_testdir_makefile_dot_prefixes_extension_silently( + testdir: Testdir, +) -> None: + """For backwards compat #8192""" + p1 = testdir.makefile("foo.bar", "") + assert ".foo.bar" in str(p1) + + +def test_testdir_makefile_ext_none_raises_type_error(testdir: Testdir) -> None: + """For backwards compat #8192""" + with pytest.raises(TypeError): + testdir.makefile(None, "") + + +def test_testdir_makefile_ext_empty_string_makes_file(testdir: Testdir) -> None: + """For backwards compat #8192""" + p1 = testdir.makefile("", "") + assert "test_testdir_makefile" in str(p1) + + +def attempt_symlink_to(path: str, to_path: str) -> None: + """Try to make a symlink from "path" to "to_path", skipping in case this platform + does not support it or we don't have sufficient privileges (common on Windows).""" + try: + Path(path).symlink_to(Path(to_path)) + except OSError: + pytest.skip("could not create symbolic link") + + +def test_tmpdir_factory( + tmpdir_factory: TempdirFactory, + tmp_path_factory: pytest.TempPathFactory, +) -> None: + assert str(tmpdir_factory.getbasetemp()) == str(tmp_path_factory.getbasetemp()) + dir = tmpdir_factory.mktemp("foo") + assert dir.exists() + + +def test_tmpdir_equals_tmp_path(tmpdir: LEGACY_PATH, tmp_path: Path) -> None: + assert Path(tmpdir) == tmp_path + + +def test_tmpdir_always_is_realpath(pytester: pytest.Pytester) -> None: + # See test_tmp_path_always_is_realpath. + realtemp = pytester.mkdir("myrealtemp") + linktemp = pytester.path.joinpath("symlinktemp") + attempt_symlink_to(str(linktemp), str(realtemp)) + p = pytester.makepyfile( + """ + def test_1(tmpdir): + import os + assert os.path.realpath(str(tmpdir)) == str(tmpdir) + """ + ) + result = pytester.runpytest("-s", p, "--basetemp=%s/bt" % linktemp) + assert not result.ret + + +def test_cache_makedir(cache: pytest.Cache) -> None: + dir = cache.makedir("foo") # type: ignore[attr-defined] + assert dir.exists() + dir.remove() + + +def test_fixturerequest_getmodulepath(pytester: pytest.Pytester) -> None: + modcol = pytester.getmodulecol("def test_somefunc(): pass") + (item,) = pytester.genitems([modcol]) + req = pytest.FixtureRequest(item, _ispytest=True) + assert req.path == modcol.path + assert req.fspath == modcol.fspath # type: ignore[attr-defined] + + +class TestFixtureRequestSessionScoped: + @pytest.fixture(scope="session") + def session_request(self, request): + return request + + def test_session_scoped_unavailable_attributes(self, session_request): + with pytest.raises( + AttributeError, + match="path not available in session-scoped context", + ): + session_request.fspath + + +@pytest.mark.parametrize("config_type", ["ini", "pyproject"]) +def test_addini_paths(pytester: pytest.Pytester, config_type: str) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("paths", "my new ini value", type="pathlist") + parser.addini("abc", "abc value") + """ + ) + if config_type == "ini": + inipath = pytester.makeini( + """ + [pytest] + paths=hello world/sub.py + """ + ) + elif config_type == "pyproject": + inipath = pytester.makepyprojecttoml( + """ + [tool.pytest.ini_options] + paths=["hello", "world/sub.py"] + """ + ) + config = pytester.parseconfig() + values = config.getini("paths") + assert len(values) == 2 + assert values[0] == inipath.parent.joinpath("hello") + assert values[1] == inipath.parent.joinpath("world/sub.py") + pytest.raises(ValueError, config.getini, "other") + + +def test_override_ini_paths(pytester: pytest.Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("paths", "my new ini value", type="pathlist")""" + ) + pytester.makeini( + """ + [pytest] + paths=blah.py""" + ) + pytester.makepyfile( + r""" + def test_overriden(pytestconfig): + config_paths = pytestconfig.getini("paths") + print(config_paths) + for cpf in config_paths: + print('\nuser_path:%s' % cpf.basename) + """ + ) + result = pytester.runpytest("--override-ini", "paths=foo/bar1.py foo/bar2.py", "-s") + result.stdout.fnmatch_lines(["user_path:bar1.py", "user_path:bar2.py"]) + + +def test_inifile_from_cmdline_main_hook(pytester: pytest.Pytester) -> None: + """Ensure Config.inifile is available during pytest_cmdline_main (#9396).""" + p = pytester.makeini( + """ + [pytest] + """ + ) + pytester.makeconftest( + """ + def pytest_cmdline_main(config): + print("pytest_cmdline_main inifile =", config.inifile) + """ + ) + result = pytester.runpytest_subprocess("-s") + result.stdout.fnmatch_lines(f"*pytest_cmdline_main inifile = {p}") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_link_resolve.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_link_resolve.py new file mode 100644 index 0000000000..60a86ada36 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_link_resolve.py @@ -0,0 +1,80 @@ +import os.path +import subprocess +import sys +import textwrap +from contextlib import contextmanager +from pathlib import Path +from string import ascii_lowercase + +from _pytest.pytester import Pytester + + +@contextmanager +def subst_path_windows(filepath: Path): + for c in ascii_lowercase[7:]: # Create a subst drive from H-Z. + c += ":" + if not os.path.exists(c): + drive = c + break + else: + raise AssertionError("Unable to find suitable drive letter for subst.") + + directory = filepath.parent + basename = filepath.name + + args = ["subst", drive, str(directory)] + subprocess.check_call(args) + assert os.path.exists(drive) + try: + filename = Path(drive, os.sep, basename) + yield filename + finally: + args = ["subst", "/D", drive] + subprocess.check_call(args) + + +@contextmanager +def subst_path_linux(filepath: Path): + directory = filepath.parent + basename = filepath.name + + target = directory / ".." / "sub2" + os.symlink(str(directory), str(target), target_is_directory=True) + try: + filename = target / basename + yield filename + finally: + # We don't need to unlink (it's all in the tempdir). + pass + + +def test_link_resolve(pytester: Pytester) -> None: + """See: https://github.com/pytest-dev/pytest/issues/5965.""" + sub1 = pytester.mkpydir("sub1") + p = sub1.joinpath("test_foo.py") + p.write_text( + textwrap.dedent( + """ + import pytest + def test_foo(): + raise AssertionError() + """ + ) + ) + + subst = subst_path_linux + if sys.platform == "win32": + subst = subst_path_windows + + with subst(p) as subst_p: + result = pytester.runpytest(str(subst_p), "-v") + # i.e.: Make sure that the error is reported as a relative path, not as a + # resolved path. + # See: https://github.com/pytest-dev/pytest/issues/5965 + stdout = result.stdout.str() + assert "sub1/test_foo.py" not in stdout + + # i.e.: Expect drive on windows because we just have drive:filename, whereas + # we expect a relative path on Linux. + expect = f"*{subst_p}*" if sys.platform == "win32" else "*sub2/test_foo.py*" + result.stdout.fnmatch_lines([expect]) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_main.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_main.py new file mode 100644 index 0000000000..2df51bb7bb --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_main.py @@ -0,0 +1,264 @@ +import argparse +import os +import re +import sys +from pathlib import Path +from typing import Optional + +import pytest +from _pytest.config import ExitCode +from _pytest.config import UsageError +from _pytest.main import resolve_collection_argument +from _pytest.main import validate_basetemp +from _pytest.pytester import Pytester + + +@pytest.mark.parametrize( + "ret_exc", + ( + pytest.param((None, ValueError)), + pytest.param((42, SystemExit)), + pytest.param((False, SystemExit)), + ), +) +def test_wrap_session_notify_exception(ret_exc, pytester: Pytester) -> None: + returncode, exc = ret_exc + c1 = pytester.makeconftest( + """ + import pytest + + def pytest_sessionstart(): + raise {exc}("boom") + + def pytest_internalerror(excrepr, excinfo): + returncode = {returncode!r} + if returncode is not False: + pytest.exit("exiting after %s..." % excinfo.typename, returncode={returncode!r}) + """.format( + returncode=returncode, exc=exc.__name__ + ) + ) + result = pytester.runpytest() + if returncode: + assert result.ret == returncode + else: + assert result.ret == ExitCode.INTERNAL_ERROR + assert result.stdout.lines[0] == "INTERNALERROR> Traceback (most recent call last):" + + end_lines = ( + result.stdout.lines[-4:] + if sys.version_info >= (3, 11) + else result.stdout.lines[-3:] + ) + + if exc == SystemExit: + assert end_lines == [ + f'INTERNALERROR> File "{c1}", line 4, in pytest_sessionstart', + 'INTERNALERROR> raise SystemExit("boom")', + *( + ("INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^",) + if sys.version_info >= (3, 11) + else () + ), + "INTERNALERROR> SystemExit: boom", + ] + else: + assert end_lines == [ + f'INTERNALERROR> File "{c1}", line 4, in pytest_sessionstart', + 'INTERNALERROR> raise ValueError("boom")', + *( + ("INTERNALERROR> ^^^^^^^^^^^^^^^^^^^^^^^^",) + if sys.version_info >= (3, 11) + else () + ), + "INTERNALERROR> ValueError: boom", + ] + if returncode is False: + assert result.stderr.lines == ["mainloop: caught unexpected SystemExit!"] + else: + assert result.stderr.lines == [f"Exit: exiting after {exc.__name__}..."] + + +@pytest.mark.parametrize("returncode", (None, 42)) +def test_wrap_session_exit_sessionfinish( + returncode: Optional[int], pytester: Pytester +) -> None: + pytester.makeconftest( + """ + import pytest + def pytest_sessionfinish(): + pytest.exit(reason="exit_pytest_sessionfinish", returncode={returncode}) + """.format( + returncode=returncode + ) + ) + result = pytester.runpytest() + if returncode: + assert result.ret == returncode + else: + assert result.ret == ExitCode.NO_TESTS_COLLECTED + assert result.stdout.lines[-1] == "collected 0 items" + assert result.stderr.lines == ["Exit: exit_pytest_sessionfinish"] + + +@pytest.mark.parametrize("basetemp", ["foo", "foo/bar"]) +def test_validate_basetemp_ok(tmp_path, basetemp, monkeypatch): + monkeypatch.chdir(str(tmp_path)) + validate_basetemp(tmp_path / basetemp) + + +@pytest.mark.parametrize("basetemp", ["", ".", ".."]) +def test_validate_basetemp_fails(tmp_path, basetemp, monkeypatch): + monkeypatch.chdir(str(tmp_path)) + msg = "basetemp must not be empty, the current working directory or any parent directory of it" + with pytest.raises(argparse.ArgumentTypeError, match=msg): + if basetemp: + basetemp = tmp_path / basetemp + validate_basetemp(basetemp) + + +def test_validate_basetemp_integration(pytester: Pytester) -> None: + result = pytester.runpytest("--basetemp=.") + result.stderr.fnmatch_lines("*basetemp must not be*") + + +class TestResolveCollectionArgument: + @pytest.fixture + def invocation_path(self, pytester: Pytester) -> Path: + pytester.syspathinsert(pytester.path / "src") + pytester.chdir() + + pkg = pytester.path.joinpath("src/pkg") + pkg.mkdir(parents=True) + pkg.joinpath("__init__.py").touch() + pkg.joinpath("test.py").touch() + return pytester.path + + def test_file(self, invocation_path: Path) -> None: + """File and parts.""" + assert resolve_collection_argument(invocation_path, "src/pkg/test.py") == ( + invocation_path / "src/pkg/test.py", + [], + ) + assert resolve_collection_argument(invocation_path, "src/pkg/test.py::") == ( + invocation_path / "src/pkg/test.py", + [""], + ) + assert resolve_collection_argument( + invocation_path, "src/pkg/test.py::foo::bar" + ) == (invocation_path / "src/pkg/test.py", ["foo", "bar"]) + assert resolve_collection_argument( + invocation_path, "src/pkg/test.py::foo::bar::" + ) == (invocation_path / "src/pkg/test.py", ["foo", "bar", ""]) + + def test_dir(self, invocation_path: Path) -> None: + """Directory and parts.""" + assert resolve_collection_argument(invocation_path, "src/pkg") == ( + invocation_path / "src/pkg", + [], + ) + + with pytest.raises( + UsageError, match=r"directory argument cannot contain :: selection parts" + ): + resolve_collection_argument(invocation_path, "src/pkg::") + + with pytest.raises( + UsageError, match=r"directory argument cannot contain :: selection parts" + ): + resolve_collection_argument(invocation_path, "src/pkg::foo::bar") + + def test_pypath(self, invocation_path: Path) -> None: + """Dotted name and parts.""" + assert resolve_collection_argument( + invocation_path, "pkg.test", as_pypath=True + ) == (invocation_path / "src/pkg/test.py", []) + assert resolve_collection_argument( + invocation_path, "pkg.test::foo::bar", as_pypath=True + ) == (invocation_path / "src/pkg/test.py", ["foo", "bar"]) + assert resolve_collection_argument(invocation_path, "pkg", as_pypath=True) == ( + invocation_path / "src/pkg", + [], + ) + + with pytest.raises( + UsageError, match=r"package argument cannot contain :: selection parts" + ): + resolve_collection_argument( + invocation_path, "pkg::foo::bar", as_pypath=True + ) + + def test_parametrized_name_with_colons(self, invocation_path: Path) -> None: + ret = resolve_collection_argument( + invocation_path, "src/pkg/test.py::test[a::b]" + ) + assert ret == (invocation_path / "src/pkg/test.py", ["test[a::b]"]) + + def test_does_not_exist(self, invocation_path: Path) -> None: + """Given a file/module that does not exist raises UsageError.""" + with pytest.raises( + UsageError, match=re.escape("file or directory not found: foobar") + ): + resolve_collection_argument(invocation_path, "foobar") + + with pytest.raises( + UsageError, + match=re.escape( + "module or package not found: foobar (missing __init__.py?)" + ), + ): + resolve_collection_argument(invocation_path, "foobar", as_pypath=True) + + def test_absolute_paths_are_resolved_correctly(self, invocation_path: Path) -> None: + """Absolute paths resolve back to absolute paths.""" + full_path = str(invocation_path / "src") + assert resolve_collection_argument(invocation_path, full_path) == ( + Path(os.path.abspath("src")), + [], + ) + + # ensure full paths given in the command-line without the drive letter resolve + # to the full path correctly (#7628) + drive, full_path_without_drive = os.path.splitdrive(full_path) + assert resolve_collection_argument( + invocation_path, full_path_without_drive + ) == (Path(os.path.abspath("src")), []) + + +def test_module_full_path_without_drive(pytester: Pytester) -> None: + """Collect and run test using full path except for the drive letter (#7628). + + Passing a full path without a drive letter would trigger a bug in legacy_path + where it would keep the full path without the drive letter around, instead of resolving + to the full path, resulting in fixtures node ids not matching against test node ids correctly. + """ + pytester.makepyfile( + **{ + "project/conftest.py": """ + import pytest + @pytest.fixture + def fix(): return 1 + """, + } + ) + + pytester.makepyfile( + **{ + "project/tests/dummy_test.py": """ + def test(fix): + assert fix == 1 + """ + } + ) + fn = pytester.path.joinpath("project/tests/dummy_test.py") + assert fn.is_file() + + drive, path = os.path.splitdrive(str(fn)) + + result = pytester.runpytest(path, "-v") + result.stdout.fnmatch_lines( + [ + os.path.join("project", "tests", "dummy_test.py") + "::test PASSED *", + "* 1 passed in *", + ] + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_mark.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_mark.py new file mode 100644 index 0000000000..da67d1ea7b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_mark.py @@ -0,0 +1,1130 @@ +import os +import sys +from typing import List +from typing import Optional +from unittest import mock + +import pytest +from _pytest.config import ExitCode +from _pytest.mark import MarkGenerator +from _pytest.mark.structures import EMPTY_PARAMETERSET_OPTION +from _pytest.nodes import Collector +from _pytest.nodes import Node +from _pytest.pytester import Pytester + + +class TestMark: + @pytest.mark.parametrize("attr", ["mark", "param"]) + def test_pytest_exists_in_namespace_all(self, attr: str) -> None: + module = sys.modules["pytest"] + assert attr in module.__all__ # type: ignore + + def test_pytest_mark_notcallable(self) -> None: + mark = MarkGenerator(_ispytest=True) + with pytest.raises(TypeError): + mark() # type: ignore[operator] + + def test_mark_with_param(self): + def some_function(abc): + pass + + class SomeClass: + pass + + assert pytest.mark.foo(some_function) is some_function + marked_with_args = pytest.mark.foo.with_args(some_function) + assert marked_with_args is not some_function # type: ignore[comparison-overlap] + + assert pytest.mark.foo(SomeClass) is SomeClass + assert pytest.mark.foo.with_args(SomeClass) is not SomeClass # type: ignore[comparison-overlap] + + def test_pytest_mark_name_starts_with_underscore(self) -> None: + mark = MarkGenerator(_ispytest=True) + with pytest.raises(AttributeError): + mark._some_name + + +def test_marked_class_run_twice(pytester: Pytester) -> None: + """Test fails file is run twice that contains marked class. + See issue#683. + """ + py_file = pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize('abc', [1, 2, 3]) + class Test1(object): + def test_1(self, abc): + assert abc in [1, 2, 3] + """ + ) + file_name = os.path.basename(py_file) + rec = pytester.inline_run(file_name, file_name) + rec.assertoutcome(passed=6) + + +def test_ini_markers(pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + markers = + a1: this is a webtest marker + a2: this is a smoke marker + """ + ) + pytester.makepyfile( + """ + def test_markers(pytestconfig): + markers = pytestconfig.getini("markers") + print(markers) + assert len(markers) >= 2 + assert markers[0].startswith("a1:") + assert markers[1].startswith("a2:") + """ + ) + rec = pytester.inline_run() + rec.assertoutcome(passed=1) + + +def test_markers_option(pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + markers = + a1: this is a webtest marker + a1some: another marker + nodescription + """ + ) + result = pytester.runpytest("--markers") + result.stdout.fnmatch_lines( + ["*a1*this is a webtest*", "*a1some*another marker", "*nodescription*"] + ) + + +def test_ini_markers_whitespace(pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + markers = + a1 : this is a whitespace marker + """ + ) + pytester.makepyfile( + """ + import pytest + + @pytest.mark.a1 + def test_markers(): + assert True + """ + ) + rec = pytester.inline_run("--strict-markers", "-m", "a1") + rec.assertoutcome(passed=1) + + +def test_marker_without_description(pytester: Pytester) -> None: + pytester.makefile( + ".cfg", + setup=""" + [tool:pytest] + markers=slow + """, + ) + pytester.makeconftest( + """ + import pytest + pytest.mark.xfail('FAIL') + """ + ) + ftdir = pytester.mkdir("ft1_dummy") + pytester.path.joinpath("conftest.py").replace(ftdir.joinpath("conftest.py")) + rec = pytester.runpytest("--strict-markers") + rec.assert_outcomes() + + +def test_markers_option_with_plugin_in_current_dir(pytester: Pytester) -> None: + pytester.makeconftest('pytest_plugins = "flip_flop"') + pytester.makepyfile( + flip_flop="""\ + def pytest_configure(config): + config.addinivalue_line("markers", "flip:flop") + + def pytest_generate_tests(metafunc): + try: + mark = metafunc.function.flipper + except AttributeError: + return + metafunc.parametrize("x", (10, 20))""" + ) + pytester.makepyfile( + """\ + import pytest + @pytest.mark.flipper + def test_example(x): + assert x""" + ) + + result = pytester.runpytest("--markers") + result.stdout.fnmatch_lines(["*flip*flop*"]) + + +def test_mark_on_pseudo_function(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.mark.r(lambda x: 0/0) + def test_hello(): + pass + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +@pytest.mark.parametrize("option_name", ["--strict-markers", "--strict"]) +def test_strict_prohibits_unregistered_markers( + pytester: Pytester, option_name: str +) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.unregisteredmark + def test_hello(): + pass + """ + ) + result = pytester.runpytest(option_name) + assert result.ret != 0 + result.stdout.fnmatch_lines( + ["'unregisteredmark' not found in `markers` configuration option"] + ) + + +@pytest.mark.parametrize( + ("expr", "expected_passed"), + [ + ("xyz", ["test_one"]), + ("((( xyz)) )", ["test_one"]), + ("not not xyz", ["test_one"]), + ("xyz and xyz2", []), + ("xyz2", ["test_two"]), + ("xyz or xyz2", ["test_one", "test_two"]), + ], +) +def test_mark_option( + expr: str, expected_passed: List[Optional[str]], pytester: Pytester +) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.xyz + def test_one(): + pass + @pytest.mark.xyz2 + def test_two(): + pass + """ + ) + rec = pytester.inline_run("-m", expr) + passed, skipped, fail = rec.listoutcomes() + passed_str = [x.nodeid.split("::")[-1] for x in passed] + assert passed_str == expected_passed + + +@pytest.mark.parametrize( + ("expr", "expected_passed"), + [("interface", ["test_interface"]), ("not interface", ["test_nointer"])], +) +def test_mark_option_custom( + expr: str, expected_passed: List[str], pytester: Pytester +) -> None: + pytester.makeconftest( + """ + import pytest + def pytest_collection_modifyitems(items): + for item in items: + if "interface" in item.nodeid: + item.add_marker(pytest.mark.interface) + """ + ) + pytester.makepyfile( + """ + def test_interface(): + pass + def test_nointer(): + pass + """ + ) + rec = pytester.inline_run("-m", expr) + passed, skipped, fail = rec.listoutcomes() + passed_str = [x.nodeid.split("::")[-1] for x in passed] + assert passed_str == expected_passed + + +@pytest.mark.parametrize( + ("expr", "expected_passed"), + [ + ("interface", ["test_interface"]), + ("not interface", ["test_nointer", "test_pass", "test_1", "test_2"]), + ("pass", ["test_pass"]), + ("not pass", ["test_interface", "test_nointer", "test_1", "test_2"]), + ("not not not (pass)", ["test_interface", "test_nointer", "test_1", "test_2"]), + ("1 or 2", ["test_1", "test_2"]), + ("not (1 or 2)", ["test_interface", "test_nointer", "test_pass"]), + ], +) +def test_keyword_option_custom( + expr: str, expected_passed: List[str], pytester: Pytester +) -> None: + pytester.makepyfile( + """ + def test_interface(): + pass + def test_nointer(): + pass + def test_pass(): + pass + def test_1(): + pass + def test_2(): + pass + """ + ) + rec = pytester.inline_run("-k", expr) + passed, skipped, fail = rec.listoutcomes() + passed_str = [x.nodeid.split("::")[-1] for x in passed] + assert passed_str == expected_passed + + +def test_keyword_option_considers_mark(pytester: Pytester) -> None: + pytester.copy_example("marks/marks_considered_keywords") + rec = pytester.inline_run("-k", "foo") + passed = rec.listoutcomes()[0] + assert len(passed) == 1 + + +@pytest.mark.parametrize( + ("expr", "expected_passed"), + [ + ("None", ["test_func[None]"]), + ("[1.3]", ["test_func[1.3]"]), + ("2-3", ["test_func[2-3]"]), + ], +) +def test_keyword_option_parametrize( + expr: str, expected_passed: List[str], pytester: Pytester +) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize("arg", [None, 1.3, "2-3"]) + def test_func(arg): + pass + """ + ) + rec = pytester.inline_run("-k", expr) + passed, skipped, fail = rec.listoutcomes() + passed_str = [x.nodeid.split("::")[-1] for x in passed] + assert passed_str == expected_passed + + +def test_parametrize_with_module(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize("arg", [pytest,]) + def test_func(arg): + pass + """ + ) + rec = pytester.inline_run() + passed, skipped, fail = rec.listoutcomes() + expected_id = "test_func[" + pytest.__name__ + "]" + assert passed[0].nodeid.split("::")[-1] == expected_id + + +@pytest.mark.parametrize( + ("expr", "expected_error"), + [ + ( + "foo or", + "at column 7: expected not OR left parenthesis OR identifier; got end of input", + ), + ( + "foo or or", + "at column 8: expected not OR left parenthesis OR identifier; got or", + ), + ( + "(foo", + "at column 5: expected right parenthesis; got end of input", + ), + ( + "foo bar", + "at column 5: expected end of input; got identifier", + ), + ( + "or or", + "at column 1: expected not OR left parenthesis OR identifier; got or", + ), + ( + "not or", + "at column 5: expected not OR left parenthesis OR identifier; got or", + ), + ], +) +def test_keyword_option_wrong_arguments( + expr: str, expected_error: str, pytester: Pytester, capsys +) -> None: + pytester.makepyfile( + """ + def test_func(arg): + pass + """ + ) + pytester.inline_run("-k", expr) + err = capsys.readouterr().err + assert expected_error in err + + +def test_parametrized_collected_from_command_line(pytester: Pytester) -> None: + """Parametrized test not collected if test named specified in command + line issue#649.""" + py_file = pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize("arg", [None, 1.3, "2-3"]) + def test_func(arg): + pass + """ + ) + file_name = os.path.basename(py_file) + rec = pytester.inline_run(file_name + "::" + "test_func") + rec.assertoutcome(passed=3) + + +def test_parametrized_collect_with_wrong_args(pytester: Pytester) -> None: + """Test collect parametrized func with wrong number of args.""" + py_file = pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize('foo, bar', [(1, 2, 3)]) + def test_func(foo, bar): + pass + """ + ) + + result = pytester.runpytest(py_file) + result.stdout.fnmatch_lines( + [ + 'test_parametrized_collect_with_wrong_args.py::test_func: in "parametrize" the number of names (2):', + " ['foo', 'bar']", + "must be equal to the number of values (3):", + " (1, 2, 3)", + ] + ) + + +def test_parametrized_with_kwargs(pytester: Pytester) -> None: + """Test collect parametrized func with wrong number of args.""" + py_file = pytester.makepyfile( + """ + import pytest + + @pytest.fixture(params=[1,2]) + def a(request): + return request.param + + @pytest.mark.parametrize(argnames='b', argvalues=[1, 2]) + def test_func(a, b): + pass + """ + ) + + result = pytester.runpytest(py_file) + assert result.ret == 0 + + +def test_parametrize_iterator(pytester: Pytester) -> None: + """`parametrize` should work with generators (#5354).""" + py_file = pytester.makepyfile( + """\ + import pytest + + def gen(): + yield 1 + yield 2 + yield 3 + + @pytest.mark.parametrize('a', gen()) + def test(a): + assert a >= 1 + """ + ) + result = pytester.runpytest(py_file) + assert result.ret == 0 + # should not skip any tests + result.stdout.fnmatch_lines(["*3 passed*"]) + + +class TestFunctional: + def test_merging_markers_deep(self, pytester: Pytester) -> None: + # issue 199 - propagate markers into nested classes + p = pytester.makepyfile( + """ + import pytest + class TestA(object): + pytestmark = pytest.mark.a + def test_b(self): + assert True + class TestC(object): + # this one didn't get marked + def test_d(self): + assert True + """ + ) + items, rec = pytester.inline_genitems(p) + for item in items: + print(item, item.keywords) + assert [x for x in item.iter_markers() if x.name == "a"] + + def test_mark_decorator_subclass_does_not_propagate_to_base( + self, pytester: Pytester + ) -> None: + p = pytester.makepyfile( + """ + import pytest + + @pytest.mark.a + class Base(object): pass + + @pytest.mark.b + class Test1(Base): + def test_foo(self): pass + + class Test2(Base): + def test_bar(self): pass + """ + ) + items, rec = pytester.inline_genitems(p) + self.assert_markers(items, test_foo=("a", "b"), test_bar=("a",)) + + def test_mark_should_not_pass_to_siebling_class(self, pytester: Pytester) -> None: + """#568""" + p = pytester.makepyfile( + """ + import pytest + + class TestBase(object): + def test_foo(self): + pass + + @pytest.mark.b + class TestSub(TestBase): + pass + + + class TestOtherSub(TestBase): + pass + + """ + ) + items, rec = pytester.inline_genitems(p) + base_item, sub_item, sub_item_other = items + print(items, [x.nodeid for x in items]) + # new api segregates + assert not list(base_item.iter_markers(name="b")) + assert not list(sub_item_other.iter_markers(name="b")) + assert list(sub_item.iter_markers(name="b")) + + def test_mark_decorator_baseclasses_merged(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + @pytest.mark.a + class Base(object): pass + + @pytest.mark.b + class Base2(Base): pass + + @pytest.mark.c + class Test1(Base2): + def test_foo(self): pass + + class Test2(Base2): + @pytest.mark.d + def test_bar(self): pass + """ + ) + items, rec = pytester.inline_genitems(p) + self.assert_markers(items, test_foo=("a", "b", "c"), test_bar=("a", "b", "d")) + + def test_mark_closest(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + @pytest.mark.c(location="class") + class Test: + @pytest.mark.c(location="function") + def test_has_own(self): + pass + + def test_has_inherited(self): + pass + + """ + ) + items, rec = pytester.inline_genitems(p) + has_own, has_inherited = items + has_own_marker = has_own.get_closest_marker("c") + has_inherited_marker = has_inherited.get_closest_marker("c") + assert has_own_marker is not None + assert has_inherited_marker is not None + assert has_own_marker.kwargs == {"location": "function"} + assert has_inherited_marker.kwargs == {"location": "class"} + assert has_own.get_closest_marker("missing") is None + + def test_mark_with_wrong_marker(self, pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + import pytest + class pytestmark(object): + pass + def test_func(): + pass + """ + ) + values = reprec.getfailedcollections() + assert len(values) == 1 + assert "TypeError" in str(values[0].longrepr) + + def test_mark_dynamically_in_funcarg(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + @pytest.fixture + def arg(request): + request.applymarker(pytest.mark.hello) + def pytest_terminal_summary(terminalreporter): + values = terminalreporter.stats['passed'] + terminalreporter._tw.line("keyword: %s" % values[0].keywords) + """ + ) + pytester.makepyfile( + """ + def test_func(arg): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["keyword: *hello*"]) + + def test_no_marker_match_on_unmarked_names(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + @pytest.mark.shouldmatch + def test_marked(): + assert 1 + + def test_unmarked(): + assert 1 + """ + ) + reprec = pytester.inline_run("-m", "test_unmarked", p) + passed, skipped, failed = reprec.listoutcomes() + assert len(passed) + len(skipped) + len(failed) == 0 + dlist = reprec.getcalls("pytest_deselected") + deselected_tests = dlist[0].items + assert len(deselected_tests) == 2 + + def test_keywords_at_node_level(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope="session", autouse=True) + def some(request): + request.keywords["hello"] = 42 + assert "world" not in request.keywords + + @pytest.fixture(scope="function", autouse=True) + def funcsetup(request): + assert "world" in request.keywords + assert "hello" in request.keywords + + @pytest.mark.world + def test_function(): + pass + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + def test_keyword_added_for_session(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + def pytest_collection_modifyitems(session): + session.add_marker("mark1") + session.add_marker(pytest.mark.mark2) + session.add_marker(pytest.mark.mark3) + pytest.raises(ValueError, lambda: + session.add_marker(10)) + """ + ) + pytester.makepyfile( + """ + def test_some(request): + assert "mark1" in request.keywords + assert "mark2" in request.keywords + assert "mark3" in request.keywords + assert 10 not in request.keywords + marker = request.node.get_closest_marker("mark1") + assert marker.name == "mark1" + assert marker.args == () + assert marker.kwargs == {} + """ + ) + reprec = pytester.inline_run("-m", "mark1") + reprec.assertoutcome(passed=1) + + def assert_markers(self, items, **expected) -> None: + """Assert that given items have expected marker names applied to them. + expected should be a dict of (item name -> seq of expected marker names). + + Note: this could be moved to ``pytester`` if proven to be useful + to other modules. + """ + items = {x.name: x for x in items} + for name, expected_markers in expected.items(): + markers = {m.name for m in items[name].iter_markers()} + assert markers == set(expected_markers) + + @pytest.mark.filterwarnings("ignore") + def test_mark_from_parameters(self, pytester: Pytester) -> None: + """#1540""" + pytester.makepyfile( + """ + import pytest + + pytestmark = pytest.mark.skipif(True, reason='skip all') + + # skipifs inside fixture params + params = [pytest.mark.skipif(False, reason='dont skip')('parameter')] + + + @pytest.fixture(params=params) + def parameter(request): + return request.param + + + def test_1(parameter): + assert True + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(skipped=1) + + def test_reevaluate_dynamic_expr(self, pytester: Pytester) -> None: + """#7360""" + py_file1 = pytester.makepyfile( + test_reevaluate_dynamic_expr1=""" + import pytest + + skip = True + + @pytest.mark.skipif("skip") + def test_should_skip(): + assert True + """ + ) + py_file2 = pytester.makepyfile( + test_reevaluate_dynamic_expr2=""" + import pytest + + skip = False + + @pytest.mark.skipif("skip") + def test_should_not_skip(): + assert True + """ + ) + + file_name1 = os.path.basename(py_file1) + file_name2 = os.path.basename(py_file2) + reprec = pytester.inline_run(file_name1, file_name2) + reprec.assertoutcome(passed=1, skipped=1) + + +class TestKeywordSelection: + def test_select_simple(self, pytester: Pytester) -> None: + file_test = pytester.makepyfile( + """ + def test_one(): + assert 0 + class TestClass(object): + def test_method_one(self): + assert 42 == 43 + """ + ) + + def check(keyword, name): + reprec = pytester.inline_run("-s", "-k", keyword, file_test) + passed, skipped, failed = reprec.listoutcomes() + assert len(failed) == 1 + assert failed[0].nodeid.split("::")[-1] == name + assert len(reprec.getcalls("pytest_deselected")) == 1 + + for keyword in ["test_one", "est_on"]: + check(keyword, "test_one") + check("TestClass and test", "test_method_one") + + @pytest.mark.parametrize( + "keyword", + [ + "xxx", + "xxx and test_2", + "TestClass", + "xxx and not test_1", + "TestClass and test_2", + "xxx and TestClass and test_2", + ], + ) + def test_select_extra_keywords(self, pytester: Pytester, keyword) -> None: + p = pytester.makepyfile( + test_select=""" + def test_1(): + pass + class TestClass(object): + def test_2(self): + pass + """ + ) + pytester.makepyfile( + conftest=""" + import pytest + @pytest.hookimpl(hookwrapper=True) + def pytest_pycollect_makeitem(name): + outcome = yield + if name == "TestClass": + item = outcome.get_result() + item.extra_keyword_matches.add("xxx") + """ + ) + reprec = pytester.inline_run(p.parent, "-s", "-k", keyword) + print("keyword", repr(keyword)) + passed, skipped, failed = reprec.listoutcomes() + assert len(passed) == 1 + assert passed[0].nodeid.endswith("test_2") + dlist = reprec.getcalls("pytest_deselected") + assert len(dlist) == 1 + assert dlist[0].items[0].name == "test_1" + + def test_select_starton(self, pytester: Pytester) -> None: + threepass = pytester.makepyfile( + test_threepass=""" + def test_one(): assert 1 + def test_two(): assert 1 + def test_three(): assert 1 + """ + ) + reprec = pytester.inline_run( + "-Wignore::pytest.PytestRemovedIn7Warning", "-k", "test_two:", threepass + ) + passed, skipped, failed = reprec.listoutcomes() + assert len(passed) == 2 + assert not failed + dlist = reprec.getcalls("pytest_deselected") + assert len(dlist) == 1 + item = dlist[0].items[0] + assert item.name == "test_one" + + def test_keyword_extra(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def test_one(): + assert 0 + test_one.mykeyword = True + """ + ) + reprec = pytester.inline_run("-k", "mykeyword", p) + passed, skipped, failed = reprec.countoutcomes() + assert failed == 1 + + @pytest.mark.xfail + def test_keyword_extra_dash(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def test_one(): + assert 0 + test_one.mykeyword = True + """ + ) + # with argparse the argument to an option cannot + # start with '-' + reprec = pytester.inline_run("-k", "-mykeyword", p) + passed, skipped, failed = reprec.countoutcomes() + assert passed + skipped + failed == 0 + + @pytest.mark.parametrize( + "keyword", + ["__", "+", ".."], + ) + def test_no_magic_values(self, pytester: Pytester, keyword: str) -> None: + """Make sure the tests do not match on magic values, + no double underscored values, like '__dict__' and '+'. + """ + p = pytester.makepyfile( + """ + def test_one(): assert 1 + """ + ) + + reprec = pytester.inline_run("-k", keyword, p) + passed, skipped, failed = reprec.countoutcomes() + dlist = reprec.getcalls("pytest_deselected") + assert passed + skipped + failed == 0 + deselected_tests = dlist[0].items + assert len(deselected_tests) == 1 + + def test_no_match_directories_outside_the_suite(self, pytester: Pytester) -> None: + """`-k` should not match against directories containing the test suite (#7040).""" + test_contents = """ + def test_aaa(): pass + def test_ddd(): pass + """ + pytester.makepyfile( + **{"ddd/tests/__init__.py": "", "ddd/tests/test_foo.py": test_contents} + ) + + def get_collected_names(*args): + _, rec = pytester.inline_genitems(*args) + calls = rec.getcalls("pytest_collection_finish") + assert len(calls) == 1 + return [x.name for x in calls[0].session.items] + + # sanity check: collect both tests in normal runs + assert get_collected_names() == ["test_aaa", "test_ddd"] + + # do not collect anything based on names outside the collection tree + assert get_collected_names("-k", pytester._name) == [] + + # "-k ddd" should only collect "test_ddd", but not + # 'test_aaa' just because one of its parent directories is named "ddd"; + # this was matched previously because Package.name would contain the full path + # to the package + assert get_collected_names("-k", "ddd") == ["test_ddd"] + + +class TestMarkDecorator: + @pytest.mark.parametrize( + "lhs, rhs, expected", + [ + (pytest.mark.foo(), pytest.mark.foo(), True), + (pytest.mark.foo(), pytest.mark.bar(), False), + (pytest.mark.foo(), "bar", False), + ("foo", pytest.mark.bar(), False), + ], + ) + def test__eq__(self, lhs, rhs, expected) -> None: + assert (lhs == rhs) == expected + + def test_aliases(self) -> None: + md = pytest.mark.foo(1, "2", three=3) + assert md.name == "foo" + assert md.args == (1, "2") + assert md.kwargs == {"three": 3} + + +@pytest.mark.parametrize("mark", [None, "", "skip", "xfail"]) +def test_parameterset_for_parametrize_marks( + pytester: Pytester, mark: Optional[str] +) -> None: + if mark is not None: + pytester.makeini( + """ + [pytest] + {}={} + """.format( + EMPTY_PARAMETERSET_OPTION, mark + ) + ) + + config = pytester.parseconfig() + from _pytest.mark import pytest_configure, get_empty_parameterset_mark + + pytest_configure(config) + result_mark = get_empty_parameterset_mark(config, ["a"], all) + if mark in (None, ""): + # normalize to the requested name + mark = "skip" + assert result_mark.name == mark + assert result_mark.kwargs["reason"].startswith("got empty parameter set ") + if mark == "xfail": + assert result_mark.kwargs.get("run") is False + + +def test_parameterset_for_fail_at_collect(pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + {}=fail_at_collect + """.format( + EMPTY_PARAMETERSET_OPTION + ) + ) + + config = pytester.parseconfig() + from _pytest.mark import pytest_configure, get_empty_parameterset_mark + + pytest_configure(config) + + with pytest.raises( + Collector.CollectError, + match=r"Empty parameter set in 'pytest_configure' at line \d\d+", + ): + get_empty_parameterset_mark(config, ["a"], pytest_configure) + + p1 = pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize("empty", []) + def test(): + pass + """ + ) + result = pytester.runpytest(str(p1)) + result.stdout.fnmatch_lines( + [ + "collected 0 items / 1 error", + "* ERROR collecting test_parameterset_for_fail_at_collect.py *", + "Empty parameter set in 'test' at line 3", + "*= 1 error in *", + ] + ) + assert result.ret == ExitCode.INTERRUPTED + + +def test_parameterset_for_parametrize_bad_markname(pytester: Pytester) -> None: + with pytest.raises(pytest.UsageError): + test_parameterset_for_parametrize_marks(pytester, "bad") + + +def test_mark_expressions_no_smear(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + class BaseTests(object): + def test_something(self): + pass + + @pytest.mark.FOO + class TestFooClass(BaseTests): + pass + + @pytest.mark.BAR + class TestBarClass(BaseTests): + pass + """ + ) + + reprec = pytester.inline_run("-m", "FOO") + passed, skipped, failed = reprec.countoutcomes() + dlist = reprec.getcalls("pytest_deselected") + assert passed == 1 + assert skipped == failed == 0 + deselected_tests = dlist[0].items + assert len(deselected_tests) == 1 + + # todo: fixed + # keywords smear - expected behaviour + # reprec_keywords = pytester.inline_run("-k", "FOO") + # passed_k, skipped_k, failed_k = reprec_keywords.countoutcomes() + # assert passed_k == 2 + # assert skipped_k == failed_k == 0 + + +def test_addmarker_order(pytester) -> None: + session = mock.Mock() + session.own_markers = [] + session.parent = None + session.nodeid = "" + session.path = pytester.path + node = Node.from_parent(session, name="Test") + node.add_marker("foo") + node.add_marker("bar") + node.add_marker("baz", append=False) + extracted = [x.name for x in node.iter_markers()] + assert extracted == ["baz", "foo", "bar"] + + +@pytest.mark.filterwarnings("ignore") +def test_markers_from_parametrize(pytester: Pytester) -> None: + """#3605""" + pytester.makepyfile( + """ + import pytest + + first_custom_mark = pytest.mark.custom_marker + custom_mark = pytest.mark.custom_mark + @pytest.fixture(autouse=True) + def trigger(request): + custom_mark = list(request.node.iter_markers('custom_mark')) + print("Custom mark %s" % custom_mark) + + @custom_mark("custom mark non parametrized") + def test_custom_mark_non_parametrized(): + print("Hey from test") + + @pytest.mark.parametrize( + "obj_type", + [ + first_custom_mark("first custom mark")("template"), + pytest.param( # Think this should be recommended way? + "disk", + marks=custom_mark('custom mark1') + ), + custom_mark("custom mark2")("vm"), # Tried also this + ] + ) + def test_custom_mark_parametrized(obj_type): + print("obj_type is:", obj_type) + """ + ) + + result = pytester.runpytest() + result.assert_outcomes(passed=4) + + +def test_pytest_param_id_requires_string() -> None: + with pytest.raises(TypeError) as excinfo: + pytest.param(id=True) # type: ignore[arg-type] + (msg,) = excinfo.value.args + assert msg == "Expected id to be a string, got : True" + + +@pytest.mark.parametrize("s", (None, "hello world")) +def test_pytest_param_id_allows_none_or_string(s) -> None: + assert pytest.param(id=s) + + +@pytest.mark.parametrize("expr", ("NOT internal_err", "NOT (internal_err)", "bogus=")) +def test_marker_expr_eval_failure_handling(pytester: Pytester, expr) -> None: + foo = pytester.makepyfile( + """ + import pytest + + @pytest.mark.internal_err + def test_foo(): + pass + """ + ) + expected = f"ERROR: Wrong expression passed to '-m': {expr}: *" + result = pytester.runpytest(foo, "-m", expr) + result.stderr.fnmatch_lines([expected]) + assert result.ret == ExitCode.USAGE_ERROR diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_mark_expression.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_mark_expression.py new file mode 100644 index 0000000000..f3643e7b40 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_mark_expression.py @@ -0,0 +1,195 @@ +from typing import Callable + +import pytest +from _pytest.mark.expression import Expression +from _pytest.mark.expression import ParseError + + +def evaluate(input: str, matcher: Callable[[str], bool]) -> bool: + return Expression.compile(input).evaluate(matcher) + + +def test_empty_is_false() -> None: + assert not evaluate("", lambda ident: False) + assert not evaluate("", lambda ident: True) + assert not evaluate(" ", lambda ident: False) + assert not evaluate("\t", lambda ident: False) + + +@pytest.mark.parametrize( + ("expr", "expected"), + ( + ("true", True), + ("true", True), + ("false", False), + ("not true", False), + ("not false", True), + ("not not true", True), + ("not not false", False), + ("true and true", True), + ("true and false", False), + ("false and true", False), + ("true and true and true", True), + ("true and true and false", False), + ("true and true and not true", False), + ("false or false", False), + ("false or true", True), + ("true or true", True), + ("true or true or false", True), + ("true and true or false", True), + ("not true or true", True), + ("(not true) or true", True), + ("not (true or true)", False), + ("true and true or false and false", True), + ("true and (true or false) and false", False), + ("true and (true or (not (not false))) and false", False), + ), +) +def test_basic(expr: str, expected: bool) -> None: + matcher = {"true": True, "false": False}.__getitem__ + assert evaluate(expr, matcher) is expected + + +@pytest.mark.parametrize( + ("expr", "expected"), + ( + (" true ", True), + (" ((((((true)))))) ", True), + (" ( ((\t (((true))))) \t \t)", True), + ("( true and (((false))))", False), + ("not not not not true", True), + ("not not not not not true", False), + ), +) +def test_syntax_oddeties(expr: str, expected: bool) -> None: + matcher = {"true": True, "false": False}.__getitem__ + assert evaluate(expr, matcher) is expected + + +def test_backslash_not_treated_specially() -> None: + r"""When generating nodeids, if the source name contains special characters + like a newline, they are escaped into two characters like \n. Therefore, a + user will never need to insert a literal newline, only \n (two chars). So + mark expressions themselves do not support escaping, instead they treat + backslashes as regular identifier characters.""" + matcher = {r"\nfoo\n"}.__contains__ + + assert evaluate(r"\nfoo\n", matcher) + assert not evaluate(r"foo", matcher) + with pytest.raises(ParseError): + evaluate("\nfoo\n", matcher) + + +@pytest.mark.parametrize( + ("expr", "column", "message"), + ( + ("(", 2, "expected not OR left parenthesis OR identifier; got end of input"), + ( + " (", + 3, + "expected not OR left parenthesis OR identifier; got end of input", + ), + ( + ")", + 1, + "expected not OR left parenthesis OR identifier; got right parenthesis", + ), + ( + ") ", + 1, + "expected not OR left parenthesis OR identifier; got right parenthesis", + ), + ( + "not", + 4, + "expected not OR left parenthesis OR identifier; got end of input", + ), + ( + "not not", + 8, + "expected not OR left parenthesis OR identifier; got end of input", + ), + ( + "(not)", + 5, + "expected not OR left parenthesis OR identifier; got right parenthesis", + ), + ("and", 1, "expected not OR left parenthesis OR identifier; got and"), + ( + "ident and", + 10, + "expected not OR left parenthesis OR identifier; got end of input", + ), + ( + "ident and or", + 11, + "expected not OR left parenthesis OR identifier; got or", + ), + ("ident ident", 7, "expected end of input; got identifier"), + ), +) +def test_syntax_errors(expr: str, column: int, message: str) -> None: + with pytest.raises(ParseError) as excinfo: + evaluate(expr, lambda ident: True) + assert excinfo.value.column == column + assert excinfo.value.message == message + + +@pytest.mark.parametrize( + "ident", + ( + ".", + "...", + ":::", + "a:::c", + "a+-b", + r"\nhe\\l\lo\n\t\rbye", + "a/b", + "אבגד", + "aaאבגדcc", + "a[bcd]", + "1234", + "1234abcd", + "1234and", + "notandor", + "not_and_or", + "not[and]or", + "1234+5678", + "123.232", + "True", + "False", + "None", + "if", + "else", + "while", + ), +) +def test_valid_idents(ident: str) -> None: + assert evaluate(ident, {ident: True}.__getitem__) + + +@pytest.mark.parametrize( + "ident", + ( + "^", + "*", + "=", + "&", + "%", + "$", + "#", + "@", + "!", + "~", + "{", + "}", + '"', + "'", + "|", + ";", + "←", + ), +) +def test_invalid_idents(ident: str) -> None: + with pytest.raises(ParseError): + evaluate(ident, lambda ident: True) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_meta.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_meta.py new file mode 100644 index 0000000000..9201bd2161 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_meta.py @@ -0,0 +1,32 @@ +"""Test importing of all internal packages and modules. + +This ensures all internal packages can be imported without needing the pytest +namespace being set, which is critical for the initialization of xdist. +""" +import pkgutil +import subprocess +import sys +from typing import List + +import _pytest +import pytest + + +def _modules() -> List[str]: + pytest_pkg: str = _pytest.__path__ # type: ignore + return sorted( + n + for _, n, _ in pkgutil.walk_packages(pytest_pkg, prefix=_pytest.__name__ + ".") + ) + + +@pytest.mark.slow +@pytest.mark.parametrize("module", _modules()) +def test_no_warnings(module: str) -> None: + # fmt: off + subprocess.check_call(( + sys.executable, + "-W", "error", + "-c", f"__import__({module!r})", + )) + # fmt: on diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_monkeypatch.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_monkeypatch.py new file mode 100644 index 0000000000..9552181802 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_monkeypatch.py @@ -0,0 +1,455 @@ +import os +import re +import sys +import textwrap +from pathlib import Path +from typing import Dict +from typing import Generator +from typing import Type + +import pytest +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester + + +@pytest.fixture +def mp() -> Generator[MonkeyPatch, None, None]: + cwd = os.getcwd() + sys_path = list(sys.path) + yield MonkeyPatch() + sys.path[:] = sys_path + os.chdir(cwd) + + +def test_setattr() -> None: + class A: + x = 1 + + monkeypatch = MonkeyPatch() + pytest.raises(AttributeError, monkeypatch.setattr, A, "notexists", 2) + monkeypatch.setattr(A, "y", 2, raising=False) + assert A.y == 2 # type: ignore + monkeypatch.undo() + assert not hasattr(A, "y") + + monkeypatch = MonkeyPatch() + monkeypatch.setattr(A, "x", 2) + assert A.x == 2 + monkeypatch.setattr(A, "x", 3) + assert A.x == 3 + monkeypatch.undo() + assert A.x == 1 + + A.x = 5 + monkeypatch.undo() # double-undo makes no modification + assert A.x == 5 + + with pytest.raises(TypeError): + monkeypatch.setattr(A, "y") # type: ignore[call-overload] + + +class TestSetattrWithImportPath: + def test_string_expression(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr("os.path.abspath", lambda x: "hello2") + assert os.path.abspath("123") == "hello2" + + def test_string_expression_class(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr("_pytest.config.Config", 42) + import _pytest + + assert _pytest.config.Config == 42 # type: ignore + + def test_unicode_string(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.setattr("_pytest.config.Config", 42) + import _pytest + + assert _pytest.config.Config == 42 # type: ignore + monkeypatch.delattr("_pytest.config.Config") + + def test_wrong_target(self, monkeypatch: MonkeyPatch) -> None: + with pytest.raises(TypeError): + monkeypatch.setattr(None, None) # type: ignore[call-overload] + + def test_unknown_import(self, monkeypatch: MonkeyPatch) -> None: + with pytest.raises(ImportError): + monkeypatch.setattr("unkn123.classx", None) + + def test_unknown_attr(self, monkeypatch: MonkeyPatch) -> None: + with pytest.raises(AttributeError): + monkeypatch.setattr("os.path.qweqwe", None) + + def test_unknown_attr_non_raising(self, monkeypatch: MonkeyPatch) -> None: + # https://github.com/pytest-dev/pytest/issues/746 + monkeypatch.setattr("os.path.qweqwe", 42, raising=False) + assert os.path.qweqwe == 42 # type: ignore + + def test_delattr(self, monkeypatch: MonkeyPatch) -> None: + monkeypatch.delattr("os.path.abspath") + assert not hasattr(os.path, "abspath") + monkeypatch.undo() + assert os.path.abspath + + +def test_delattr() -> None: + class A: + x = 1 + + monkeypatch = MonkeyPatch() + monkeypatch.delattr(A, "x") + assert not hasattr(A, "x") + monkeypatch.undo() + assert A.x == 1 + + monkeypatch = MonkeyPatch() + monkeypatch.delattr(A, "x") + pytest.raises(AttributeError, monkeypatch.delattr, A, "y") + monkeypatch.delattr(A, "y", raising=False) + monkeypatch.setattr(A, "x", 5, raising=False) + assert A.x == 5 + monkeypatch.undo() + assert A.x == 1 + + +def test_setitem() -> None: + d = {"x": 1} + monkeypatch = MonkeyPatch() + monkeypatch.setitem(d, "x", 2) + monkeypatch.setitem(d, "y", 1700) + monkeypatch.setitem(d, "y", 1700) + assert d["x"] == 2 + assert d["y"] == 1700 + monkeypatch.setitem(d, "x", 3) + assert d["x"] == 3 + monkeypatch.undo() + assert d["x"] == 1 + assert "y" not in d + d["x"] = 5 + monkeypatch.undo() + assert d["x"] == 5 + + +def test_setitem_deleted_meanwhile() -> None: + d: Dict[str, object] = {} + monkeypatch = MonkeyPatch() + monkeypatch.setitem(d, "x", 2) + del d["x"] + monkeypatch.undo() + assert not d + + +@pytest.mark.parametrize("before", [True, False]) +def test_setenv_deleted_meanwhile(before: bool) -> None: + key = "qwpeoip123" + if before: + os.environ[key] = "world" + monkeypatch = MonkeyPatch() + monkeypatch.setenv(key, "hello") + del os.environ[key] + monkeypatch.undo() + if before: + assert os.environ[key] == "world" + del os.environ[key] + else: + assert key not in os.environ + + +def test_delitem() -> None: + d: Dict[str, object] = {"x": 1} + monkeypatch = MonkeyPatch() + monkeypatch.delitem(d, "x") + assert "x" not in d + monkeypatch.delitem(d, "y", raising=False) + pytest.raises(KeyError, monkeypatch.delitem, d, "y") + assert not d + monkeypatch.setitem(d, "y", 1700) + assert d["y"] == 1700 + d["hello"] = "world" + monkeypatch.setitem(d, "x", 1500) + assert d["x"] == 1500 + monkeypatch.undo() + assert d == {"hello": "world", "x": 1} + + +def test_setenv() -> None: + monkeypatch = MonkeyPatch() + with pytest.warns(pytest.PytestWarning): + monkeypatch.setenv("XYZ123", 2) # type: ignore[arg-type] + import os + + assert os.environ["XYZ123"] == "2" + monkeypatch.undo() + assert "XYZ123" not in os.environ + + +def test_delenv() -> None: + name = "xyz1234" + assert name not in os.environ + monkeypatch = MonkeyPatch() + pytest.raises(KeyError, monkeypatch.delenv, name, raising=True) + monkeypatch.delenv(name, raising=False) + monkeypatch.undo() + os.environ[name] = "1" + try: + monkeypatch = MonkeyPatch() + monkeypatch.delenv(name) + assert name not in os.environ + monkeypatch.setenv(name, "3") + assert os.environ[name] == "3" + monkeypatch.undo() + assert os.environ[name] == "1" + finally: + if name in os.environ: + del os.environ[name] + + +class TestEnvironWarnings: + """ + os.environ keys and values should be native strings, otherwise it will cause problems with other modules (notably + subprocess). On Python 2 os.environ accepts anything without complaining, while Python 3 does the right thing + and raises an error. + """ + + VAR_NAME = "PYTEST_INTERNAL_MY_VAR" + + def test_setenv_non_str_warning(self, monkeypatch: MonkeyPatch) -> None: + value = 2 + msg = ( + "Value of environment variable PYTEST_INTERNAL_MY_VAR type should be str, " + "but got 2 (type: int); converted to str implicitly" + ) + with pytest.warns(pytest.PytestWarning, match=re.escape(msg)): + monkeypatch.setenv(str(self.VAR_NAME), value) # type: ignore[arg-type] + + +def test_setenv_prepend() -> None: + import os + + monkeypatch = MonkeyPatch() + monkeypatch.setenv("XYZ123", "2", prepend="-") + monkeypatch.setenv("XYZ123", "3", prepend="-") + assert os.environ["XYZ123"] == "3-2" + monkeypatch.undo() + assert "XYZ123" not in os.environ + + +def test_monkeypatch_plugin(pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + def test_method(monkeypatch): + assert monkeypatch.__class__.__name__ == "MonkeyPatch" + """ + ) + res = reprec.countoutcomes() + assert tuple(res) == (1, 0, 0), res + + +def test_syspath_prepend(mp: MonkeyPatch) -> None: + old = list(sys.path) + mp.syspath_prepend("world") + mp.syspath_prepend("hello") + assert sys.path[0] == "hello" + assert sys.path[1] == "world" + mp.undo() + assert sys.path == old + mp.undo() + assert sys.path == old + + +def test_syspath_prepend_double_undo(mp: MonkeyPatch) -> None: + old_syspath = sys.path[:] + try: + mp.syspath_prepend("hello world") + mp.undo() + sys.path.append("more hello world") + mp.undo() + assert sys.path[-1] == "more hello world" + finally: + sys.path[:] = old_syspath + + +def test_chdir_with_path_local(mp: MonkeyPatch, tmp_path: Path) -> None: + mp.chdir(tmp_path) + assert os.getcwd() == str(tmp_path) + + +def test_chdir_with_str(mp: MonkeyPatch, tmp_path: Path) -> None: + mp.chdir(str(tmp_path)) + assert os.getcwd() == str(tmp_path) + + +def test_chdir_undo(mp: MonkeyPatch, tmp_path: Path) -> None: + cwd = os.getcwd() + mp.chdir(tmp_path) + mp.undo() + assert os.getcwd() == cwd + + +def test_chdir_double_undo(mp: MonkeyPatch, tmp_path: Path) -> None: + mp.chdir(str(tmp_path)) + mp.undo() + os.chdir(tmp_path) + mp.undo() + assert os.getcwd() == str(tmp_path) + + +def test_issue185_time_breaks(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import time + def test_m(monkeypatch): + def f(): + raise Exception + monkeypatch.setattr(time, "time", f) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + """ + *1 passed* + """ + ) + + +def test_importerror(pytester: Pytester) -> None: + p = pytester.mkpydir("package") + p.joinpath("a.py").write_text( + textwrap.dedent( + """\ + import doesnotexist + + x = 1 + """ + ) + ) + pytester.path.joinpath("test_importerror.py").write_text( + textwrap.dedent( + """\ + def test_importerror(monkeypatch): + monkeypatch.setattr('package.a.x', 2) + """ + ) + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + """ + *import error in package.a: No module named 'doesnotexist'* + """ + ) + + +class Sample: + @staticmethod + def hello() -> bool: + return True + + +class SampleInherit(Sample): + pass + + +@pytest.mark.parametrize( + "Sample", + [Sample, SampleInherit], + ids=["new", "new-inherit"], +) +def test_issue156_undo_staticmethod(Sample: Type[Sample]) -> None: + monkeypatch = MonkeyPatch() + + monkeypatch.setattr(Sample, "hello", None) + assert Sample.hello is None + + monkeypatch.undo() # type: ignore[unreachable] + assert Sample.hello() + + +def test_undo_class_descriptors_delattr() -> None: + class SampleParent: + @classmethod + def hello(_cls): + pass + + @staticmethod + def world(): + pass + + class SampleChild(SampleParent): + pass + + monkeypatch = MonkeyPatch() + + original_hello = SampleChild.hello + original_world = SampleChild.world + monkeypatch.delattr(SampleParent, "hello") + monkeypatch.delattr(SampleParent, "world") + assert getattr(SampleParent, "hello", None) is None + assert getattr(SampleParent, "world", None) is None + + monkeypatch.undo() + assert original_hello == SampleChild.hello + assert original_world == SampleChild.world + + +def test_issue1338_name_resolving() -> None: + pytest.importorskip("requests") + monkeypatch = MonkeyPatch() + try: + monkeypatch.delattr("requests.sessions.Session.request") + finally: + monkeypatch.undo() + + +def test_context() -> None: + monkeypatch = MonkeyPatch() + + import functools + import inspect + + with monkeypatch.context() as m: + m.setattr(functools, "partial", 3) + assert not inspect.isclass(functools.partial) + assert inspect.isclass(functools.partial) + + +def test_context_classmethod() -> None: + class A: + x = 1 + + with MonkeyPatch.context() as m: + m.setattr(A, "x", 2) + assert A.x == 2 + assert A.x == 1 + + +def test_syspath_prepend_with_namespace_packages( + pytester: Pytester, monkeypatch: MonkeyPatch +) -> None: + for dirname in "hello", "world": + d = pytester.mkdir(dirname) + ns = d.joinpath("ns_pkg") + ns.mkdir() + ns.joinpath("__init__.py").write_text( + "__import__('pkg_resources').declare_namespace(__name__)" + ) + lib = ns.joinpath(dirname) + lib.mkdir() + lib.joinpath("__init__.py").write_text("def check(): return %r" % dirname) + + monkeypatch.syspath_prepend("hello") + import ns_pkg.hello + + assert ns_pkg.hello.check() == "hello" + + with pytest.raises(ImportError): + import ns_pkg.world + + # Prepending should call fixup_namespace_packages. + monkeypatch.syspath_prepend("world") + import ns_pkg.world + + assert ns_pkg.world.check() == "world" + + # Should invalidate caches via importlib.invalidate_caches. + modules_tmpdir = pytester.mkdir("modules_tmpdir") + monkeypatch.syspath_prepend(str(modules_tmpdir)) + modules_tmpdir.joinpath("main_app.py").write_text("app = True") + from main_app import app # noqa: F401 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_nodes.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_nodes.py new file mode 100644 index 0000000000..df1439e1c4 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_nodes.py @@ -0,0 +1,167 @@ +import re +import warnings +from pathlib import Path +from typing import cast +from typing import List +from typing import Type + +import pytest +from _pytest import nodes +from _pytest.compat import legacy_path +from _pytest.outcomes import OutcomeException +from _pytest.pytester import Pytester +from _pytest.warning_types import PytestWarning + + +@pytest.mark.parametrize( + ("nodeid", "expected"), + ( + ("", [""]), + ("a", ["", "a"]), + ("aa/b", ["", "aa", "aa/b"]), + ("a/b/c", ["", "a", "a/b", "a/b/c"]), + ("a/bbb/c::D", ["", "a", "a/bbb", "a/bbb/c", "a/bbb/c::D"]), + ("a/b/c::D::eee", ["", "a", "a/b", "a/b/c", "a/b/c::D", "a/b/c::D::eee"]), + ("::xx", ["", "::xx"]), + # / only considered until first :: + ("a/b/c::D/d::e", ["", "a", "a/b", "a/b/c", "a/b/c::D/d", "a/b/c::D/d::e"]), + # : alone is not a separator. + ("a/b::D:e:f::g", ["", "a", "a/b", "a/b::D:e:f", "a/b::D:e:f::g"]), + # / not considered if a part of a test name + ("a/b::c/d::e[/test]", ["", "a", "a/b", "a/b::c/d", "a/b::c/d::e[/test]"]), + ), +) +def test_iterparentnodeids(nodeid: str, expected: List[str]) -> None: + result = list(nodes.iterparentnodeids(nodeid)) + assert result == expected + + +def test_node_from_parent_disallowed_arguments() -> None: + with pytest.raises(TypeError, match="session is"): + nodes.Node.from_parent(None, session=None) # type: ignore[arg-type] + with pytest.raises(TypeError, match="config is"): + nodes.Node.from_parent(None, config=None) # type: ignore[arg-type] + + +def test_node_direct_construction_deprecated() -> None: + with pytest.raises( + OutcomeException, + match=( + "Direct construction of _pytest.nodes.Node has been deprecated, please " + "use _pytest.nodes.Node.from_parent.\nSee " + "https://docs.pytest.org/en/stable/deprecations.html#node-construction-changed-to-node-from-parent" + " for more details." + ), + ): + nodes.Node(None, session=None) # type: ignore[arg-type] + + +def test_subclassing_both_item_and_collector_deprecated( + request, tmp_path: Path +) -> None: + """ + Verifies we warn on diamond inheritance as well as correctly managing legacy + inheritance constructors with missing args as found in plugins. + """ + + # We do not expect any warnings messages to issued during class definition. + with warnings.catch_warnings(): + warnings.simplefilter("error") + + class SoWrong(nodes.Item, nodes.File): + def __init__(self, fspath, parent): + """Legacy ctor with legacy call # don't wana see""" + super().__init__(fspath, parent) + + with pytest.warns(PytestWarning) as rec: + SoWrong.from_parent( + request.session, fspath=legacy_path(tmp_path / "broken.txt") + ) + messages = [str(x.message) for x in rec] + assert any( + re.search(".*SoWrong.* not using a cooperative constructor.*", x) + for x in messages + ) + assert any( + re.search("(?m)SoWrong .* should not be a collector", x) for x in messages + ) + + +@pytest.mark.parametrize( + "warn_type, msg", [(DeprecationWarning, "deprecated"), (PytestWarning, "pytest")] +) +def test_node_warn_is_no_longer_only_pytest_warnings( + pytester: Pytester, warn_type: Type[Warning], msg: str +) -> None: + items = pytester.getitems( + """ + def test(): + pass + """ + ) + with pytest.warns(warn_type, match=msg): + items[0].warn(warn_type(msg)) + + +def test_node_warning_enforces_warning_types(pytester: Pytester) -> None: + items = pytester.getitems( + """ + def test(): + pass + """ + ) + with pytest.raises( + ValueError, match="warning must be an instance of Warning or subclass" + ): + items[0].warn(Exception("ok")) # type: ignore[arg-type] + + +def test__check_initialpaths_for_relpath() -> None: + """Ensure that it handles dirs, and does not always use dirname.""" + cwd = Path.cwd() + + class FakeSession1: + _initialpaths = frozenset({cwd}) + + session = cast(pytest.Session, FakeSession1) + + assert nodes._check_initialpaths_for_relpath(session, cwd) == "" + + sub = cwd / "file" + + class FakeSession2: + _initialpaths = frozenset({cwd}) + + session = cast(pytest.Session, FakeSession2) + + assert nodes._check_initialpaths_for_relpath(session, sub) == "file" + + outside = Path("/outside-this-does-not-exist") + assert nodes._check_initialpaths_for_relpath(session, outside) is None + + +def test_failure_with_changed_cwd(pytester: Pytester) -> None: + """ + Test failure lines should use absolute paths if cwd has changed since + invocation, so the path is correct (#6428). + """ + p = pytester.makepyfile( + """ + import os + import pytest + + @pytest.fixture + def private_dir(): + out_dir = 'ddd' + os.mkdir(out_dir) + old_dir = os.getcwd() + os.chdir(out_dir) + yield out_dir + os.chdir(old_dir) + + def test_show_wrong_path(private_dir): + assert False + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines([str(p) + ":*: AssertionError", "*1 failed in *"]) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_nose.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_nose.py new file mode 100644 index 0000000000..1ded8854bb --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_nose.py @@ -0,0 +1,498 @@ +import pytest +from _pytest.pytester import Pytester + + +def setup_module(mod): + mod.nose = pytest.importorskip("nose") + + +def test_nose_setup(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + values = [] + from nose.tools import with_setup + + @with_setup(lambda: values.append(1), lambda: values.append(2)) + def test_hello(): + assert values == [1] + + def test_world(): + assert values == [1,2] + + test_hello.setup = lambda: values.append(1) + test_hello.teardown = lambda: values.append(2) + """ + ) + result = pytester.runpytest(p, "-p", "nose") + result.assert_outcomes(passed=2) + + +def test_setup_func_with_setup_decorator() -> None: + from _pytest.nose import call_optional + + values = [] + + class A: + @pytest.fixture(autouse=True) + def f(self): + values.append(1) + + call_optional(A(), "f") + assert not values + + +def test_setup_func_not_callable() -> None: + from _pytest.nose import call_optional + + class A: + f = 1 + + call_optional(A(), "f") + + +def test_nose_setup_func(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + from nose.tools import with_setup + + values = [] + + def my_setup(): + a = 1 + values.append(a) + + def my_teardown(): + b = 2 + values.append(b) + + @with_setup(my_setup, my_teardown) + def test_hello(): + print(values) + assert values == [1] + + def test_world(): + print(values) + assert values == [1,2] + + """ + ) + result = pytester.runpytest(p, "-p", "nose") + result.assert_outcomes(passed=2) + + +def test_nose_setup_func_failure(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + from nose.tools import with_setup + + values = [] + my_setup = lambda x: 1 + my_teardown = lambda x: 2 + + @with_setup(my_setup, my_teardown) + def test_hello(): + print(values) + assert values == [1] + + def test_world(): + print(values) + assert values == [1,2] + + """ + ) + result = pytester.runpytest(p, "-p", "nose") + result.stdout.fnmatch_lines(["*TypeError: ()*"]) + + +def test_nose_setup_func_failure_2(pytester: Pytester) -> None: + pytester.makepyfile( + """ + values = [] + + my_setup = 1 + my_teardown = 2 + + def test_hello(): + assert values == [] + + test_hello.setup = my_setup + test_hello.teardown = my_teardown + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +def test_nose_setup_partial(pytester: Pytester) -> None: + pytest.importorskip("functools") + p = pytester.makepyfile( + """ + from functools import partial + + values = [] + + def my_setup(x): + a = x + values.append(a) + + def my_teardown(x): + b = x + values.append(b) + + my_setup_partial = partial(my_setup, 1) + my_teardown_partial = partial(my_teardown, 2) + + def test_hello(): + print(values) + assert values == [1] + + def test_world(): + print(values) + assert values == [1,2] + + test_hello.setup = my_setup_partial + test_hello.teardown = my_teardown_partial + """ + ) + result = pytester.runpytest(p, "-p", "nose") + result.stdout.fnmatch_lines(["*2 passed*"]) + + +def test_module_level_setup(pytester: Pytester) -> None: + pytester.makepyfile( + """ + from nose.tools import with_setup + items = {} + + def setup(): + items.setdefault("setup", []).append("up") + + def teardown(): + items.setdefault("setup", []).append("down") + + def setup2(): + items.setdefault("setup2", []).append("up") + + def teardown2(): + items.setdefault("setup2", []).append("down") + + def test_setup_module_setup(): + assert items["setup"] == ["up"] + + def test_setup_module_setup_again(): + assert items["setup"] == ["up"] + + @with_setup(setup2, teardown2) + def test_local_setup(): + assert items["setup"] == ["up"] + assert items["setup2"] == ["up"] + + @with_setup(setup2, teardown2) + def test_local_setup_again(): + assert items["setup"] == ["up"] + assert items["setup2"] == ["up", "down", "up"] + """ + ) + result = pytester.runpytest("-p", "nose") + result.stdout.fnmatch_lines(["*4 passed*"]) + + +def test_nose_style_setup_teardown(pytester: Pytester) -> None: + pytester.makepyfile( + """ + values = [] + + def setup_module(): + values.append(1) + + def teardown_module(): + del values[0] + + def test_hello(): + assert values == [1] + + def test_world(): + assert values == [1] + """ + ) + result = pytester.runpytest("-p", "nose") + result.stdout.fnmatch_lines(["*2 passed*"]) + + +def test_fixtures_nose_setup_issue8394(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def setup_module(): + pass + + def teardown_module(): + pass + + def setup_function(func): + pass + + def teardown_function(func): + pass + + def test_world(): + pass + + class Test(object): + def setup_class(cls): + pass + + def teardown_class(cls): + pass + + def setup_method(self, meth): + pass + + def teardown_method(self, meth): + pass + + def test_method(self): pass + """ + ) + match = "*no docstring available*" + result = pytester.runpytest("--fixtures") + assert result.ret == 0 + result.stdout.no_fnmatch_line(match) + + result = pytester.runpytest("--fixtures", "-v") + assert result.ret == 0 + result.stdout.fnmatch_lines([match, match, match, match]) + + +def test_nose_setup_ordering(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def setup_module(mod): + mod.visited = True + + class TestClass(object): + def setup(self): + assert visited + self.visited_cls = True + def test_first(self): + assert visited + assert self.visited_cls + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_apiwrapper_problem_issue260(pytester: Pytester) -> None: + # this would end up trying a call an optional teardown on the class + # for plain unittests we don't want nose behaviour + pytester.makepyfile( + """ + import unittest + class TestCase(unittest.TestCase): + def setup(self): + #should not be called in unittest testcases + assert 0, 'setup' + def teardown(self): + #should not be called in unittest testcases + assert 0, 'teardown' + def setUp(self): + print('setup') + def tearDown(self): + print('teardown') + def test_fun(self): + pass + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=1) + + +def test_setup_teardown_linking_issue265(pytester: Pytester) -> None: + # we accidentally didn't integrate nose setupstate with normal setupstate + # this test ensures that won't happen again + pytester.makepyfile( + ''' + import pytest + + class TestGeneric(object): + def test_nothing(self): + """Tests the API of the implementation (for generic and specialized).""" + + @pytest.mark.skipif("True", reason= + "Skip tests to check if teardown is skipped as well.") + class TestSkipTeardown(TestGeneric): + + def setup(self): + """Sets up my specialized implementation for $COOL_PLATFORM.""" + raise Exception("should not call setup for skipped tests") + + def teardown(self): + """Undoes the setup.""" + raise Exception("should not call teardown for skipped tests") + ''' + ) + reprec = pytester.runpytest() + reprec.assert_outcomes(passed=1, skipped=1) + + +def test_SkipTest_during_collection(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import nose + raise nose.SkipTest("during collection") + def test_failing(): + assert False + """ + ) + result = pytester.runpytest(p) + result.assert_outcomes(skipped=1, warnings=1) + + +def test_SkipTest_in_test(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import nose + + def test_skipping(): + raise nose.SkipTest("in test") + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(skipped=1) + + +def test_istest_function_decorator(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import nose.tools + @nose.tools.istest + def not_test_prefix(): + pass + """ + ) + result = pytester.runpytest(p) + result.assert_outcomes(passed=1) + + +def test_nottest_function_decorator(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import nose.tools + @nose.tools.nottest + def test_prefix(): + pass + """ + ) + reprec = pytester.inline_run() + assert not reprec.getfailedcollections() + calls = reprec.getreports("pytest_runtest_logreport") + assert not calls + + +def test_istest_class_decorator(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import nose.tools + @nose.tools.istest + class NotTestPrefix(object): + def test_method(self): + pass + """ + ) + result = pytester.runpytest(p) + result.assert_outcomes(passed=1) + + +def test_nottest_class_decorator(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import nose.tools + @nose.tools.nottest + class TestPrefix(object): + def test_method(self): + pass + """ + ) + reprec = pytester.inline_run() + assert not reprec.getfailedcollections() + calls = reprec.getreports("pytest_runtest_logreport") + assert not calls + + +def test_skip_test_with_unicode(pytester: Pytester) -> None: + pytester.makepyfile( + """\ + import unittest + class TestClass(): + def test_io(self): + raise unittest.SkipTest('😊') + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 1 skipped *"]) + + +def test_raises(pytester: Pytester) -> None: + pytester.makepyfile( + """ + from nose.tools import raises + + @raises(RuntimeError) + def test_raises_runtimeerror(): + raise RuntimeError + + @raises(Exception) + def test_raises_baseexception_not_caught(): + raise BaseException + + @raises(BaseException) + def test_raises_baseexception_caught(): + raise BaseException + """ + ) + result = pytester.runpytest("-vv") + result.stdout.fnmatch_lines( + [ + "test_raises.py::test_raises_runtimeerror PASSED*", + "test_raises.py::test_raises_baseexception_not_caught FAILED*", + "test_raises.py::test_raises_baseexception_caught PASSED*", + "*= FAILURES =*", + "*_ test_raises_baseexception_not_caught _*", + "", + "arg = (), kw = {}", + "", + " def newfunc(*arg, **kw):", + " try:", + "> func(*arg, **kw)", + "", + "*/nose/*: ", + "_ _ *", + "", + " @raises(Exception)", + " def test_raises_baseexception_not_caught():", + "> raise BaseException", + "E BaseException", + "", + "test_raises.py:9: BaseException", + "* 1 failed, 2 passed *", + ] + ) + + +def test_nose_setup_skipped_if_non_callable(pytester: Pytester) -> None: + """Regression test for #9391.""" + p = pytester.makepyfile( + __init__="", + setup=""" + """, + teardown=""" + """, + test_it=""" + from . import setup, teardown + + def test_it(): + pass + """, + ) + result = pytester.runpytest(p, "-p", "nose") + assert result.ret == 0 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_parseopt.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_parseopt.py new file mode 100644 index 0000000000..28529d0437 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_parseopt.py @@ -0,0 +1,344 @@ +import argparse +import os +import shlex +import subprocess +import sys +from pathlib import Path + +import pytest +from _pytest.config import argparsing as parseopt +from _pytest.config.exceptions import UsageError +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester + + +@pytest.fixture +def parser() -> parseopt.Parser: + return parseopt.Parser(_ispytest=True) + + +class TestParser: + def test_no_help_by_default(self) -> None: + parser = parseopt.Parser(usage="xyz", _ispytest=True) + pytest.raises(UsageError, lambda: parser.parse(["-h"])) + + def test_custom_prog(self, parser: parseopt.Parser) -> None: + """Custom prog can be set for `argparse.ArgumentParser`.""" + assert parser._getparser().prog == os.path.basename(sys.argv[0]) + parser.prog = "custom-prog" + assert parser._getparser().prog == "custom-prog" + + def test_argument(self) -> None: + with pytest.raises(parseopt.ArgumentError): + # need a short or long option + argument = parseopt.Argument() + argument = parseopt.Argument("-t") + assert argument._short_opts == ["-t"] + assert argument._long_opts == [] + assert argument.dest == "t" + argument = parseopt.Argument("-t", "--test") + assert argument._short_opts == ["-t"] + assert argument._long_opts == ["--test"] + assert argument.dest == "test" + argument = parseopt.Argument("-t", "--test", dest="abc") + assert argument.dest == "abc" + assert str(argument) == ( + "Argument(_short_opts: ['-t'], _long_opts: ['--test'], dest: 'abc')" + ) + + def test_argument_type(self) -> None: + argument = parseopt.Argument("-t", dest="abc", type=int) + assert argument.type is int + argument = parseopt.Argument("-t", dest="abc", type=str) + assert argument.type is str + argument = parseopt.Argument("-t", dest="abc", type=float) + assert argument.type is float + with pytest.warns(DeprecationWarning): + with pytest.raises(KeyError): + argument = parseopt.Argument("-t", dest="abc", type="choice") + argument = parseopt.Argument( + "-t", dest="abc", type=str, choices=["red", "blue"] + ) + assert argument.type is str + + def test_argument_processopt(self) -> None: + argument = parseopt.Argument("-t", type=int) + argument.default = 42 + argument.dest = "abc" + res = argument.attrs() + assert res["default"] == 42 + assert res["dest"] == "abc" + + def test_group_add_and_get(self, parser: parseopt.Parser) -> None: + group = parser.getgroup("hello", description="desc") + assert group.name == "hello" + assert group.description == "desc" + + def test_getgroup_simple(self, parser: parseopt.Parser) -> None: + group = parser.getgroup("hello", description="desc") + assert group.name == "hello" + assert group.description == "desc" + group2 = parser.getgroup("hello") + assert group2 is group + + def test_group_ordering(self, parser: parseopt.Parser) -> None: + parser.getgroup("1") + parser.getgroup("2") + parser.getgroup("3", after="1") + groups = parser._groups + groups_names = [x.name for x in groups] + assert groups_names == list("132") + + def test_group_addoption(self) -> None: + group = parseopt.OptionGroup("hello", _ispytest=True) + group.addoption("--option1", action="store_true") + assert len(group.options) == 1 + assert isinstance(group.options[0], parseopt.Argument) + + def test_group_addoption_conflict(self) -> None: + group = parseopt.OptionGroup("hello again", _ispytest=True) + group.addoption("--option1", "--option-1", action="store_true") + with pytest.raises(ValueError) as err: + group.addoption("--option1", "--option-one", action="store_true") + assert str({"--option1"}) in str(err.value) + + def test_group_shortopt_lowercase(self, parser: parseopt.Parser) -> None: + group = parser.getgroup("hello") + with pytest.raises(ValueError): + group.addoption("-x", action="store_true") + assert len(group.options) == 0 + group._addoption("-x", action="store_true") + assert len(group.options) == 1 + + def test_parser_addoption(self, parser: parseopt.Parser) -> None: + group = parser.getgroup("custom options") + assert len(group.options) == 0 + group.addoption("--option1", action="store_true") + assert len(group.options) == 1 + + def test_parse(self, parser: parseopt.Parser) -> None: + parser.addoption("--hello", dest="hello", action="store") + args = parser.parse(["--hello", "world"]) + assert args.hello == "world" + assert not getattr(args, parseopt.FILE_OR_DIR) + + def test_parse2(self, parser: parseopt.Parser) -> None: + args = parser.parse([Path(".")]) + assert getattr(args, parseopt.FILE_OR_DIR)[0] == "." + + def test_parse_known_args(self, parser: parseopt.Parser) -> None: + parser.parse_known_args([Path(".")]) + parser.addoption("--hello", action="store_true") + ns = parser.parse_known_args(["x", "--y", "--hello", "this"]) + assert ns.hello + assert ns.file_or_dir == ["x"] + + def test_parse_known_and_unknown_args(self, parser: parseopt.Parser) -> None: + parser.addoption("--hello", action="store_true") + ns, unknown = parser.parse_known_and_unknown_args( + ["x", "--y", "--hello", "this"] + ) + assert ns.hello + assert ns.file_or_dir == ["x"] + assert unknown == ["--y", "this"] + + def test_parse_will_set_default(self, parser: parseopt.Parser) -> None: + parser.addoption("--hello", dest="hello", default="x", action="store") + option = parser.parse([]) + assert option.hello == "x" + del option.hello + parser.parse_setoption([], option) + assert option.hello == "x" + + def test_parse_setoption(self, parser: parseopt.Parser) -> None: + parser.addoption("--hello", dest="hello", action="store") + parser.addoption("--world", dest="world", default=42) + + option = argparse.Namespace() + args = parser.parse_setoption(["--hello", "world"], option) + assert option.hello == "world" + assert option.world == 42 + assert not args + + def test_parse_special_destination(self, parser: parseopt.Parser) -> None: + parser.addoption("--ultimate-answer", type=int) + args = parser.parse(["--ultimate-answer", "42"]) + assert args.ultimate_answer == 42 + + def test_parse_split_positional_arguments(self, parser: parseopt.Parser) -> None: + parser.addoption("-R", action="store_true") + parser.addoption("-S", action="store_false") + args = parser.parse(["-R", "4", "2", "-S"]) + assert getattr(args, parseopt.FILE_OR_DIR) == ["4", "2"] + args = parser.parse(["-R", "-S", "4", "2", "-R"]) + assert getattr(args, parseopt.FILE_OR_DIR) == ["4", "2"] + assert args.R is True + assert args.S is False + args = parser.parse(["-R", "4", "-S", "2"]) + assert getattr(args, parseopt.FILE_OR_DIR) == ["4", "2"] + assert args.R is True + assert args.S is False + + def test_parse_defaultgetter(self) -> None: + def defaultget(option): + if not hasattr(option, "type"): + return + if option.type is int: + option.default = 42 + elif option.type is str: + option.default = "world" + + parser = parseopt.Parser(processopt=defaultget, _ispytest=True) + parser.addoption("--this", dest="this", type=int, action="store") + parser.addoption("--hello", dest="hello", type=str, action="store") + parser.addoption("--no", dest="no", action="store_true") + option = parser.parse([]) + assert option.hello == "world" + assert option.this == 42 + assert option.no is False + + def test_drop_short_helper(self) -> None: + parser = argparse.ArgumentParser( + formatter_class=parseopt.DropShorterLongHelpFormatter, allow_abbrev=False + ) + parser.add_argument( + "-t", "--twoword", "--duo", "--two-word", "--two", help="foo" + ) + # throws error on --deux only! + parser.add_argument( + "-d", "--deuxmots", "--deux-mots", action="store_true", help="foo" + ) + parser.add_argument("-s", action="store_true", help="single short") + parser.add_argument("--abc", "-a", action="store_true", help="bar") + parser.add_argument("--klm", "-k", "--kl-m", action="store_true", help="bar") + parser.add_argument( + "-P", "--pq-r", "-p", "--pqr", action="store_true", help="bar" + ) + parser.add_argument( + "--zwei-wort", "--zweiwort", "--zweiwort", action="store_true", help="bar" + ) + parser.add_argument( + "-x", "--exit-on-first", "--exitfirst", action="store_true", help="spam" + ) + parser.add_argument("files_and_dirs", nargs="*") + args = parser.parse_args(["-k", "--duo", "hallo", "--exitfirst"]) + assert args.twoword == "hallo" + assert args.klm is True + assert args.zwei_wort is False + assert args.exit_on_first is True + assert args.s is False + args = parser.parse_args(["--deux-mots"]) + with pytest.raises(AttributeError): + assert args.deux_mots is True + assert args.deuxmots is True + args = parser.parse_args(["file", "dir"]) + assert "|".join(args.files_and_dirs) == "file|dir" + + def test_drop_short_0(self, parser: parseopt.Parser) -> None: + parser.addoption("--funcarg", "--func-arg", action="store_true") + parser.addoption("--abc-def", "--abc-def", action="store_true") + parser.addoption("--klm-hij", action="store_true") + with pytest.raises(UsageError): + parser.parse(["--funcarg", "--k"]) + + def test_drop_short_2(self, parser: parseopt.Parser) -> None: + parser.addoption("--func-arg", "--doit", action="store_true") + args = parser.parse(["--doit"]) + assert args.func_arg is True + + def test_drop_short_3(self, parser: parseopt.Parser) -> None: + parser.addoption("--func-arg", "--funcarg", "--doit", action="store_true") + args = parser.parse(["abcd"]) + assert args.func_arg is False + assert args.file_or_dir == ["abcd"] + + def test_drop_short_help0(self, parser: parseopt.Parser) -> None: + parser.addoption("--func-args", "--doit", help="foo", action="store_true") + parser.parse([]) + help = parser.optparser.format_help() + assert "--func-args, --doit foo" in help + + # testing would be more helpful with all help generated + def test_drop_short_help1(self, parser: parseopt.Parser) -> None: + group = parser.getgroup("general") + group.addoption("--doit", "--func-args", action="store_true", help="foo") + group._addoption( + "-h", + "--help", + action="store_true", + dest="help", + help="show help message and configuration info", + ) + parser.parse(["-h"]) + help = parser.optparser.format_help() + assert "-doit, --func-args foo" in help + + def test_multiple_metavar_help(self, parser: parseopt.Parser) -> None: + """ + Help text for options with a metavar tuple should display help + in the form "--preferences=value1 value2 value3" (#2004). + """ + group = parser.getgroup("general") + group.addoption( + "--preferences", metavar=("value1", "value2", "value3"), nargs=3 + ) + group._addoption("-h", "--help", action="store_true", dest="help") + parser.parse(["-h"]) + help = parser.optparser.format_help() + assert "--preferences=value1 value2 value3" in help + + +def test_argcomplete(pytester: Pytester, monkeypatch: MonkeyPatch) -> None: + try: + bash_version = subprocess.run( + ["bash", "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=True, + universal_newlines=True, + ).stdout + except (OSError, subprocess.CalledProcessError): + pytest.skip("bash is not available") + if "GNU bash" not in bash_version: + # See #7518. + pytest.skip("not a real bash") + + script = str(pytester.path.joinpath("test_argcomplete")) + + with open(str(script), "w") as fp: + # redirect output from argcomplete to stdin and stderr is not trivial + # http://stackoverflow.com/q/12589419/1307905 + # so we use bash + fp.write( + 'COMP_WORDBREAKS="$COMP_WORDBREAKS" {} -m pytest 8>&1 9>&2'.format( + shlex.quote(sys.executable) + ) + ) + # alternative would be extended Pytester.{run(),_run(),popen()} to be able + # to handle a keyword argument env that replaces os.environ in popen or + # extends the copy, advantage: could not forget to restore + monkeypatch.setenv("_ARGCOMPLETE", "1") + monkeypatch.setenv("_ARGCOMPLETE_IFS", "\x0b") + monkeypatch.setenv("COMP_WORDBREAKS", " \\t\\n\"\\'><=;|&(:") + + arg = "--fu" + monkeypatch.setenv("COMP_LINE", "pytest " + arg) + monkeypatch.setenv("COMP_POINT", str(len("pytest " + arg))) + result = pytester.run("bash", str(script), arg) + if result.ret == 255: + # argcomplete not found + pytest.skip("argcomplete not available") + elif not result.stdout.str(): + pytest.skip( + "bash provided no output on stdout, argcomplete not available? (stderr={!r})".format( + result.stderr.str() + ) + ) + else: + result.stdout.fnmatch_lines(["--funcargs", "--fulltrace"]) + os.mkdir("test_argcomplete.d") + arg = "test_argc" + monkeypatch.setenv("COMP_LINE", "pytest " + arg) + monkeypatch.setenv("COMP_POINT", str(len("pytest " + arg))) + result = pytester.run("bash", str(script), arg) + result.stdout.fnmatch_lines(["test_argcomplete", "test_argcomplete.d/"]) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_pastebin.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_pastebin.py new file mode 100644 index 0000000000..b338519ae1 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_pastebin.py @@ -0,0 +1,184 @@ +import io +from typing import List +from typing import Union + +import pytest +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester + + +class TestPasteCapture: + @pytest.fixture + def pastebinlist(self, monkeypatch, request) -> List[Union[str, bytes]]: + pastebinlist: List[Union[str, bytes]] = [] + plugin = request.config.pluginmanager.getplugin("pastebin") + monkeypatch.setattr(plugin, "create_new_paste", pastebinlist.append) + return pastebinlist + + def test_failed(self, pytester: Pytester, pastebinlist) -> None: + testpath = pytester.makepyfile( + """ + import pytest + def test_pass() -> None: + pass + def test_fail(): + assert 0 + def test_skip(): + pytest.skip("") + """ + ) + reprec = pytester.inline_run(testpath, "--pastebin=failed") + assert len(pastebinlist) == 1 + s = pastebinlist[0] + assert s.find("def test_fail") != -1 + assert reprec.countoutcomes() == [1, 1, 1] + + def test_all(self, pytester: Pytester, pastebinlist) -> None: + from _pytest.pytester import LineMatcher + + testpath = pytester.makepyfile( + """ + import pytest + def test_pass(): + pass + def test_fail(): + assert 0 + def test_skip(): + pytest.skip("") + """ + ) + reprec = pytester.inline_run(testpath, "--pastebin=all", "-v") + assert reprec.countoutcomes() == [1, 1, 1] + assert len(pastebinlist) == 1 + contents = pastebinlist[0].decode("utf-8") + matcher = LineMatcher(contents.splitlines()) + matcher.fnmatch_lines( + [ + "*test_pass PASSED*", + "*test_fail FAILED*", + "*test_skip SKIPPED*", + "*== 1 failed, 1 passed, 1 skipped in *", + ] + ) + + def test_non_ascii_paste_text(self, pytester: Pytester, pastebinlist) -> None: + """Make sure that text which contains non-ascii characters is pasted + correctly. See #1219. + """ + pytester.makepyfile( + test_unicode="""\ + def test(): + assert '☺' == 1 + """ + ) + result = pytester.runpytest("--pastebin=all") + expected_msg = "*assert '☺' == 1*" + result.stdout.fnmatch_lines( + [ + expected_msg, + "*== 1 failed in *", + "*Sending information to Paste Service*", + ] + ) + assert len(pastebinlist) == 1 + + +class TestPaste: + @pytest.fixture + def pastebin(self, request): + return request.config.pluginmanager.getplugin("pastebin") + + @pytest.fixture + def mocked_urlopen_fail(self, monkeypatch: MonkeyPatch): + """Monkeypatch the actual urlopen call to emulate a HTTP Error 400.""" + calls = [] + + import urllib.error + import urllib.request + + def mocked(url, data): + calls.append((url, data)) + raise urllib.error.HTTPError(url, 400, "Bad request", {}, io.BytesIO()) + + monkeypatch.setattr(urllib.request, "urlopen", mocked) + return calls + + @pytest.fixture + def mocked_urlopen_invalid(self, monkeypatch: MonkeyPatch): + """Monkeypatch the actual urlopen calls done by the internal plugin + function that connects to bpaste service, but return a url in an + unexpected format.""" + calls = [] + + def mocked(url, data): + calls.append((url, data)) + + class DummyFile: + def read(self): + # part of html of a normal response + return b'View raw.' + + return DummyFile() + + import urllib.request + + monkeypatch.setattr(urllib.request, "urlopen", mocked) + return calls + + @pytest.fixture + def mocked_urlopen(self, monkeypatch: MonkeyPatch): + """Monkeypatch the actual urlopen calls done by the internal plugin + function that connects to bpaste service.""" + calls = [] + + def mocked(url, data): + calls.append((url, data)) + + class DummyFile: + def read(self): + # part of html of a normal response + return b'View raw.' + + return DummyFile() + + import urllib.request + + monkeypatch.setattr(urllib.request, "urlopen", mocked) + return calls + + def test_pastebin_invalid_url(self, pastebin, mocked_urlopen_invalid) -> None: + result = pastebin.create_new_paste(b"full-paste-contents") + assert ( + result + == "bad response: invalid format ('View raw.')" + ) + assert len(mocked_urlopen_invalid) == 1 + + def test_pastebin_http_error(self, pastebin, mocked_urlopen_fail) -> None: + result = pastebin.create_new_paste(b"full-paste-contents") + assert result == "bad response: HTTP Error 400: Bad request" + assert len(mocked_urlopen_fail) == 1 + + def test_create_new_paste(self, pastebin, mocked_urlopen) -> None: + result = pastebin.create_new_paste(b"full-paste-contents") + assert result == "https://bpa.st/show/3c0c6750bd" + assert len(mocked_urlopen) == 1 + url, data = mocked_urlopen[0] + assert type(data) is bytes + lexer = "text" + assert url == "https://bpa.st" + assert "lexer=%s" % lexer in data.decode() + assert "code=full-paste-contents" in data.decode() + assert "expiry=1week" in data.decode() + + def test_create_new_paste_failure(self, pastebin, monkeypatch: MonkeyPatch) -> None: + import io + import urllib.request + + def response(url, data): + stream = io.BytesIO(b"something bad occurred") + return stream + + monkeypatch.setattr(urllib.request, "urlopen", response) + result = pastebin.create_new_paste(b"full-paste-contents") + assert result == "bad response: invalid format ('something bad occurred')" diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_pathlib.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_pathlib.py new file mode 100644 index 0000000000..5eb153e847 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_pathlib.py @@ -0,0 +1,574 @@ +import os.path +import pickle +import sys +import unittest.mock +from pathlib import Path +from textwrap import dedent +from types import ModuleType +from typing import Any +from typing import Generator + +import pytest +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pathlib import bestrelpath +from _pytest.pathlib import commonpath +from _pytest.pathlib import ensure_deletable +from _pytest.pathlib import fnmatch_ex +from _pytest.pathlib import get_extended_length_path_str +from _pytest.pathlib import get_lock_path +from _pytest.pathlib import import_path +from _pytest.pathlib import ImportPathMismatchError +from _pytest.pathlib import insert_missing_modules +from _pytest.pathlib import maybe_delete_a_numbered_dir +from _pytest.pathlib import module_name_from_path +from _pytest.pathlib import resolve_package_path +from _pytest.pathlib import symlink_or_skip +from _pytest.pathlib import visit +from _pytest.tmpdir import TempPathFactory + + +class TestFNMatcherPort: + """Test our port of py.common.FNMatcher (fnmatch_ex).""" + + if sys.platform == "win32": + drv1 = "c:" + drv2 = "d:" + else: + drv1 = "/c" + drv2 = "/d" + + @pytest.mark.parametrize( + "pattern, path", + [ + ("*.py", "foo.py"), + ("*.py", "bar/foo.py"), + ("test_*.py", "foo/test_foo.py"), + ("tests/*.py", "tests/foo.py"), + (f"{drv1}/*.py", f"{drv1}/foo.py"), + (f"{drv1}/foo/*.py", f"{drv1}/foo/foo.py"), + ("tests/**/test*.py", "tests/foo/test_foo.py"), + ("tests/**/doc/test*.py", "tests/foo/bar/doc/test_foo.py"), + ("tests/**/doc/**/test*.py", "tests/foo/doc/bar/test_foo.py"), + ], + ) + def test_matching(self, pattern: str, path: str) -> None: + assert fnmatch_ex(pattern, path) + + def test_matching_abspath(self) -> None: + abspath = os.path.abspath(os.path.join("tests/foo.py")) + assert fnmatch_ex("tests/foo.py", abspath) + + @pytest.mark.parametrize( + "pattern, path", + [ + ("*.py", "foo.pyc"), + ("*.py", "foo/foo.pyc"), + ("tests/*.py", "foo/foo.py"), + (f"{drv1}/*.py", f"{drv2}/foo.py"), + (f"{drv1}/foo/*.py", f"{drv2}/foo/foo.py"), + ("tests/**/test*.py", "tests/foo.py"), + ("tests/**/test*.py", "foo/test_foo.py"), + ("tests/**/doc/test*.py", "tests/foo/bar/doc/foo.py"), + ("tests/**/doc/test*.py", "tests/foo/bar/test_foo.py"), + ], + ) + def test_not_matching(self, pattern: str, path: str) -> None: + assert not fnmatch_ex(pattern, path) + + +class TestImportPath: + """ + + Most of the tests here were copied from py lib's tests for "py.local.path.pyimport". + + Having our own pyimport-like function is inline with removing py.path dependency in the future. + """ + + @pytest.fixture(scope="session") + def path1(self, tmp_path_factory: TempPathFactory) -> Generator[Path, None, None]: + path = tmp_path_factory.mktemp("path") + self.setuptestfs(path) + yield path + assert path.joinpath("samplefile").exists() + + def setuptestfs(self, path: Path) -> None: + # print "setting up test fs for", repr(path) + samplefile = path / "samplefile" + samplefile.write_text("samplefile\n") + + execfile = path / "execfile" + execfile.write_text("x=42") + + execfilepy = path / "execfile.py" + execfilepy.write_text("x=42") + + d = {1: 2, "hello": "world", "answer": 42} + path.joinpath("samplepickle").write_bytes(pickle.dumps(d, 1)) + + sampledir = path / "sampledir" + sampledir.mkdir() + sampledir.joinpath("otherfile").touch() + + otherdir = path / "otherdir" + otherdir.mkdir() + otherdir.joinpath("__init__.py").touch() + + module_a = otherdir / "a.py" + module_a.write_text("from .b import stuff as result\n") + module_b = otherdir / "b.py" + module_b.write_text('stuff="got it"\n') + module_c = otherdir / "c.py" + module_c.write_text( + dedent( + """ + import pluggy; + import otherdir.a + value = otherdir.a.result + """ + ) + ) + module_d = otherdir / "d.py" + module_d.write_text( + dedent( + """ + import pluggy; + from otherdir import a + value2 = a.result + """ + ) + ) + + def test_smoke_test(self, path1: Path) -> None: + obj = import_path(path1 / "execfile.py", root=path1) + assert obj.x == 42 # type: ignore[attr-defined] + assert obj.__name__ == "execfile" + + def test_renamed_dir_creates_mismatch( + self, tmp_path: Path, monkeypatch: MonkeyPatch + ) -> None: + tmp_path.joinpath("a").mkdir() + p = tmp_path.joinpath("a", "test_x123.py") + p.touch() + import_path(p, root=tmp_path) + tmp_path.joinpath("a").rename(tmp_path.joinpath("b")) + with pytest.raises(ImportPathMismatchError): + import_path(tmp_path.joinpath("b", "test_x123.py"), root=tmp_path) + + # Errors can be ignored. + monkeypatch.setenv("PY_IGNORE_IMPORTMISMATCH", "1") + import_path(tmp_path.joinpath("b", "test_x123.py"), root=tmp_path) + + # PY_IGNORE_IMPORTMISMATCH=0 does not ignore error. + monkeypatch.setenv("PY_IGNORE_IMPORTMISMATCH", "0") + with pytest.raises(ImportPathMismatchError): + import_path(tmp_path.joinpath("b", "test_x123.py"), root=tmp_path) + + def test_messy_name(self, tmp_path: Path) -> None: + # https://bitbucket.org/hpk42/py-trunk/issue/129 + path = tmp_path / "foo__init__.py" + path.touch() + module = import_path(path, root=tmp_path) + assert module.__name__ == "foo__init__" + + def test_dir(self, tmp_path: Path) -> None: + p = tmp_path / "hello_123" + p.mkdir() + p_init = p / "__init__.py" + p_init.touch() + m = import_path(p, root=tmp_path) + assert m.__name__ == "hello_123" + m = import_path(p_init, root=tmp_path) + assert m.__name__ == "hello_123" + + def test_a(self, path1: Path) -> None: + otherdir = path1 / "otherdir" + mod = import_path(otherdir / "a.py", root=path1) + assert mod.result == "got it" # type: ignore[attr-defined] + assert mod.__name__ == "otherdir.a" + + def test_b(self, path1: Path) -> None: + otherdir = path1 / "otherdir" + mod = import_path(otherdir / "b.py", root=path1) + assert mod.stuff == "got it" # type: ignore[attr-defined] + assert mod.__name__ == "otherdir.b" + + def test_c(self, path1: Path) -> None: + otherdir = path1 / "otherdir" + mod = import_path(otherdir / "c.py", root=path1) + assert mod.value == "got it" # type: ignore[attr-defined] + + def test_d(self, path1: Path) -> None: + otherdir = path1 / "otherdir" + mod = import_path(otherdir / "d.py", root=path1) + assert mod.value2 == "got it" # type: ignore[attr-defined] + + def test_import_after(self, tmp_path: Path) -> None: + tmp_path.joinpath("xxxpackage").mkdir() + tmp_path.joinpath("xxxpackage", "__init__.py").touch() + mod1path = tmp_path.joinpath("xxxpackage", "module1.py") + mod1path.touch() + mod1 = import_path(mod1path, root=tmp_path) + assert mod1.__name__ == "xxxpackage.module1" + from xxxpackage import module1 + + assert module1 is mod1 + + def test_check_filepath_consistency( + self, monkeypatch: MonkeyPatch, tmp_path: Path + ) -> None: + name = "pointsback123" + p = tmp_path.joinpath(name + ".py") + p.touch() + for ending in (".pyc", ".pyo"): + mod = ModuleType(name) + pseudopath = tmp_path.joinpath(name + ending) + pseudopath.touch() + mod.__file__ = str(pseudopath) + monkeypatch.setitem(sys.modules, name, mod) + newmod = import_path(p, root=tmp_path) + assert mod == newmod + monkeypatch.undo() + mod = ModuleType(name) + pseudopath = tmp_path.joinpath(name + "123.py") + pseudopath.touch() + mod.__file__ = str(pseudopath) + monkeypatch.setitem(sys.modules, name, mod) + with pytest.raises(ImportPathMismatchError) as excinfo: + import_path(p, root=tmp_path) + modname, modfile, orig = excinfo.value.args + assert modname == name + assert modfile == str(pseudopath) + assert orig == p + assert issubclass(ImportPathMismatchError, ImportError) + + def test_issue131_on__init__(self, tmp_path: Path) -> None: + # __init__.py files may be namespace packages, and thus the + # __file__ of an imported module may not be ourselves + # see issue + tmp_path.joinpath("proja").mkdir() + p1 = tmp_path.joinpath("proja", "__init__.py") + p1.touch() + tmp_path.joinpath("sub", "proja").mkdir(parents=True) + p2 = tmp_path.joinpath("sub", "proja", "__init__.py") + p2.touch() + m1 = import_path(p1, root=tmp_path) + m2 = import_path(p2, root=tmp_path) + assert m1 == m2 + + def test_ensuresyspath_append(self, tmp_path: Path) -> None: + root1 = tmp_path / "root1" + root1.mkdir() + file1 = root1 / "x123.py" + file1.touch() + assert str(root1) not in sys.path + import_path(file1, mode="append", root=tmp_path) + assert str(root1) == sys.path[-1] + assert str(root1) not in sys.path[:-1] + + def test_invalid_path(self, tmp_path: Path) -> None: + with pytest.raises(ImportError): + import_path(tmp_path / "invalid.py", root=tmp_path) + + @pytest.fixture + def simple_module(self, tmp_path: Path) -> Path: + fn = tmp_path / "_src/tests/mymod.py" + fn.parent.mkdir(parents=True) + fn.write_text("def foo(x): return 40 + x") + return fn + + def test_importmode_importlib(self, simple_module: Path, tmp_path: Path) -> None: + """`importlib` mode does not change sys.path.""" + module = import_path(simple_module, mode="importlib", root=tmp_path) + assert module.foo(2) == 42 # type: ignore[attr-defined] + assert str(simple_module.parent) not in sys.path + assert module.__name__ in sys.modules + assert module.__name__ == "_src.tests.mymod" + assert "_src" in sys.modules + assert "_src.tests" in sys.modules + + def test_importmode_twice_is_different_module( + self, simple_module: Path, tmp_path: Path + ) -> None: + """`importlib` mode always returns a new module.""" + module1 = import_path(simple_module, mode="importlib", root=tmp_path) + module2 = import_path(simple_module, mode="importlib", root=tmp_path) + assert module1 is not module2 + + def test_no_meta_path_found( + self, simple_module: Path, monkeypatch: MonkeyPatch, tmp_path: Path + ) -> None: + """Even without any meta_path should still import module.""" + monkeypatch.setattr(sys, "meta_path", []) + module = import_path(simple_module, mode="importlib", root=tmp_path) + assert module.foo(2) == 42 # type: ignore[attr-defined] + + # mode='importlib' fails if no spec is found to load the module + import importlib.util + + monkeypatch.setattr( + importlib.util, "spec_from_file_location", lambda *args: None + ) + with pytest.raises(ImportError): + import_path(simple_module, mode="importlib", root=tmp_path) + + +def test_resolve_package_path(tmp_path: Path) -> None: + pkg = tmp_path / "pkg1" + pkg.mkdir() + (pkg / "__init__.py").touch() + (pkg / "subdir").mkdir() + (pkg / "subdir/__init__.py").touch() + assert resolve_package_path(pkg) == pkg + assert resolve_package_path(pkg.joinpath("subdir", "__init__.py")) == pkg + + +def test_package_unimportable(tmp_path: Path) -> None: + pkg = tmp_path / "pkg1-1" + pkg.mkdir() + pkg.joinpath("__init__.py").touch() + subdir = pkg.joinpath("subdir") + subdir.mkdir() + pkg.joinpath("subdir/__init__.py").touch() + assert resolve_package_path(subdir) == subdir + xyz = subdir.joinpath("xyz.py") + xyz.touch() + assert resolve_package_path(xyz) == subdir + assert not resolve_package_path(pkg) + + +def test_access_denied_during_cleanup(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: + """Ensure that deleting a numbered dir does not fail because of OSErrors (#4262).""" + path = tmp_path / "temp-1" + path.mkdir() + + def renamed_failed(*args): + raise OSError("access denied") + + monkeypatch.setattr(Path, "rename", renamed_failed) + + lock_path = get_lock_path(path) + maybe_delete_a_numbered_dir(path) + assert not lock_path.is_file() + + +def test_long_path_during_cleanup(tmp_path: Path) -> None: + """Ensure that deleting long path works (particularly on Windows (#6775)).""" + path = (tmp_path / ("a" * 250)).resolve() + if sys.platform == "win32": + # make sure that the full path is > 260 characters without any + # component being over 260 characters + assert len(str(path)) > 260 + extended_path = "\\\\?\\" + str(path) + else: + extended_path = str(path) + os.mkdir(extended_path) + assert os.path.isdir(extended_path) + maybe_delete_a_numbered_dir(path) + assert not os.path.isdir(extended_path) + + +def test_get_extended_length_path_str() -> None: + assert get_extended_length_path_str(r"c:\foo") == r"\\?\c:\foo" + assert get_extended_length_path_str(r"\\share\foo") == r"\\?\UNC\share\foo" + assert get_extended_length_path_str(r"\\?\UNC\share\foo") == r"\\?\UNC\share\foo" + assert get_extended_length_path_str(r"\\?\c:\foo") == r"\\?\c:\foo" + + +def test_suppress_error_removing_lock(tmp_path: Path) -> None: + """ensure_deletable should be resilient if lock file cannot be removed (#5456, #7491)""" + path = tmp_path / "dir" + path.mkdir() + lock = get_lock_path(path) + lock.touch() + mtime = lock.stat().st_mtime + + with unittest.mock.patch.object(Path, "unlink", side_effect=OSError) as m: + assert not ensure_deletable( + path, consider_lock_dead_if_created_before=mtime + 30 + ) + assert m.call_count == 1 + assert lock.is_file() + + with unittest.mock.patch.object(Path, "is_file", side_effect=OSError) as m: + assert not ensure_deletable( + path, consider_lock_dead_if_created_before=mtime + 30 + ) + assert m.call_count == 1 + assert lock.is_file() + + # check now that we can remove the lock file in normal circumstances + assert ensure_deletable(path, consider_lock_dead_if_created_before=mtime + 30) + assert not lock.is_file() + + +def test_bestrelpath() -> None: + curdir = Path("/foo/bar/baz/path") + assert bestrelpath(curdir, curdir) == "." + assert bestrelpath(curdir, curdir / "hello" / "world") == "hello" + os.sep + "world" + assert bestrelpath(curdir, curdir.parent / "sister") == ".." + os.sep + "sister" + assert bestrelpath(curdir, curdir.parent) == ".." + assert bestrelpath(curdir, Path("hello")) == "hello" + + +def test_commonpath() -> None: + path = Path("/foo/bar/baz/path") + subpath = path / "sampledir" + assert commonpath(path, subpath) == path + assert commonpath(subpath, path) == path + assert commonpath(Path(str(path) + "suffix"), path) == path.parent + assert commonpath(path, path.parent.parent) == path.parent.parent + + +def test_visit_ignores_errors(tmp_path: Path) -> None: + symlink_or_skip("recursive", tmp_path / "recursive") + tmp_path.joinpath("foo").write_bytes(b"") + tmp_path.joinpath("bar").write_bytes(b"") + + assert [ + entry.name for entry in visit(str(tmp_path), recurse=lambda entry: False) + ] == ["bar", "foo"] + + +@pytest.mark.skipif(not sys.platform.startswith("win"), reason="Windows only") +def test_samefile_false_negatives(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: + """ + import_file() should not raise ImportPathMismatchError if the paths are exactly + equal on Windows. It seems directories mounted as UNC paths make os.path.samefile + return False, even when they are clearly equal. + """ + module_path = tmp_path.joinpath("my_module.py") + module_path.write_text("def foo(): return 42") + monkeypatch.syspath_prepend(tmp_path) + + with monkeypatch.context() as mp: + # Forcibly make os.path.samefile() return False here to ensure we are comparing + # the paths too. Using a context to narrow the patch as much as possible given + # this is an important system function. + mp.setattr(os.path, "samefile", lambda x, y: False) + module = import_path(module_path, root=tmp_path) + assert getattr(module, "foo")() == 42 + + +class TestImportLibMode: + @pytest.mark.skipif(sys.version_info < (3, 7), reason="Dataclasses in Python3.7+") + def test_importmode_importlib_with_dataclass(self, tmp_path: Path) -> None: + """Ensure that importlib mode works with a module containing dataclasses (#7856).""" + fn = tmp_path.joinpath("_src/tests/test_dataclass.py") + fn.parent.mkdir(parents=True) + fn.write_text( + dedent( + """ + from dataclasses import dataclass + + @dataclass + class Data: + value: str + """ + ) + ) + + module = import_path(fn, mode="importlib", root=tmp_path) + Data: Any = getattr(module, "Data") + data = Data(value="foo") + assert data.value == "foo" + assert data.__module__ == "_src.tests.test_dataclass" + + def test_importmode_importlib_with_pickle(self, tmp_path: Path) -> None: + """Ensure that importlib mode works with pickle (#7859).""" + fn = tmp_path.joinpath("_src/tests/test_pickle.py") + fn.parent.mkdir(parents=True) + fn.write_text( + dedent( + """ + import pickle + + def _action(): + return 42 + + def round_trip(): + s = pickle.dumps(_action) + return pickle.loads(s) + """ + ) + ) + + module = import_path(fn, mode="importlib", root=tmp_path) + round_trip = getattr(module, "round_trip") + action = round_trip() + assert action() == 42 + + def test_importmode_importlib_with_pickle_separate_modules( + self, tmp_path: Path + ) -> None: + """ + Ensure that importlib mode works can load pickles that look similar but are + defined in separate modules. + """ + fn1 = tmp_path.joinpath("_src/m1/tests/test.py") + fn1.parent.mkdir(parents=True) + fn1.write_text( + dedent( + """ + import attr + import pickle + + @attr.s(auto_attribs=True) + class Data: + x: int = 42 + """ + ) + ) + + fn2 = tmp_path.joinpath("_src/m2/tests/test.py") + fn2.parent.mkdir(parents=True) + fn2.write_text( + dedent( + """ + import attr + import pickle + + @attr.s(auto_attribs=True) + class Data: + x: str = "" + """ + ) + ) + + import pickle + + def round_trip(obj): + s = pickle.dumps(obj) + return pickle.loads(s) + + module = import_path(fn1, mode="importlib", root=tmp_path) + Data1 = getattr(module, "Data") + + module = import_path(fn2, mode="importlib", root=tmp_path) + Data2 = getattr(module, "Data") + + assert round_trip(Data1(20)) == Data1(20) + assert round_trip(Data2("hello")) == Data2("hello") + assert Data1.__module__ == "_src.m1.tests.test" + assert Data2.__module__ == "_src.m2.tests.test" + + def test_module_name_from_path(self, tmp_path: Path) -> None: + result = module_name_from_path(tmp_path / "src/tests/test_foo.py", tmp_path) + assert result == "src.tests.test_foo" + + # Path is not relative to root dir: use the full path to obtain the module name. + result = module_name_from_path(Path("/home/foo/test_foo.py"), Path("/bar")) + assert result == "home.foo.test_foo" + + def test_insert_missing_modules(self) -> None: + modules = {"src.tests.foo": ModuleType("src.tests.foo")} + insert_missing_modules(modules, "src.tests.foo") + assert sorted(modules) == ["src", "src.tests", "src.tests.foo"] + + mod = ModuleType("mod", doc="My Module") + modules = {"src": mod} + insert_missing_modules(modules, "src") + assert modules == {"src": mod} + + modules = {} + insert_missing_modules(modules, "") + assert modules == {} diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_pluginmanager.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_pluginmanager.py new file mode 100644 index 0000000000..9fe23d1779 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_pluginmanager.py @@ -0,0 +1,427 @@ +import os +import shutil +import sys +import types +from typing import List + +import pytest +from _pytest.config import Config +from _pytest.config import ExitCode +from _pytest.config import PytestPluginManager +from _pytest.config.exceptions import UsageError +from _pytest.main import Session +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pathlib import import_path +from _pytest.pytester import Pytester + + +@pytest.fixture +def pytestpm() -> PytestPluginManager: + return PytestPluginManager() + + +class TestPytestPluginInteractions: + def test_addhooks_conftestplugin( + self, pytester: Pytester, _config_for_test: Config + ) -> None: + pytester.makepyfile( + newhooks=""" + def pytest_myhook(xyz): + "new hook" + """ + ) + conf = pytester.makeconftest( + """ + import newhooks + def pytest_addhooks(pluginmanager): + pluginmanager.add_hookspecs(newhooks) + def pytest_myhook(xyz): + return xyz + 1 + """ + ) + config = _config_for_test + pm = config.pluginmanager + pm.hook.pytest_addhooks.call_historic( + kwargs=dict(pluginmanager=config.pluginmanager) + ) + config.pluginmanager._importconftest( + conf, importmode="prepend", rootpath=pytester.path + ) + # print(config.pluginmanager.get_plugins()) + res = config.hook.pytest_myhook(xyz=10) + assert res == [11] + + def test_addhooks_nohooks(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import sys + def pytest_addhooks(pluginmanager): + pluginmanager.add_hookspecs(sys) + """ + ) + res = pytester.runpytest() + assert res.ret != 0 + res.stderr.fnmatch_lines(["*did not find*sys*"]) + + def test_do_option_postinitialize(self, pytester: Pytester) -> None: + config = pytester.parseconfigure() + assert not hasattr(config.option, "test123") + p = pytester.makepyfile( + """ + def pytest_addoption(parser): + parser.addoption('--test123', action="store_true", + default=True) + """ + ) + config.pluginmanager._importconftest( + p, importmode="prepend", rootpath=pytester.path + ) + assert config.option.test123 + + def test_configure(self, pytester: Pytester) -> None: + config = pytester.parseconfig() + values = [] + + class A: + def pytest_configure(self): + values.append(self) + + config.pluginmanager.register(A()) + assert len(values) == 0 + config._do_configure() + assert len(values) == 1 + config.pluginmanager.register(A()) # leads to a configured() plugin + assert len(values) == 2 + assert values[0] != values[1] + + config._ensure_unconfigure() + config.pluginmanager.register(A()) + assert len(values) == 2 + + def test_hook_tracing(self, _config_for_test: Config) -> None: + pytestpm = _config_for_test.pluginmanager # fully initialized with plugins + saveindent = [] + + class api1: + def pytest_plugin_registered(self): + saveindent.append(pytestpm.trace.root.indent) + + class api2: + def pytest_plugin_registered(self): + saveindent.append(pytestpm.trace.root.indent) + raise ValueError() + + values: List[str] = [] + pytestpm.trace.root.setwriter(values.append) + undo = pytestpm.enable_tracing() + try: + indent = pytestpm.trace.root.indent + p = api1() + pytestpm.register(p) + assert pytestpm.trace.root.indent == indent + assert len(values) >= 2 + assert "pytest_plugin_registered" in values[0] + assert "finish" in values[1] + + values[:] = [] + with pytest.raises(ValueError): + pytestpm.register(api2()) + assert pytestpm.trace.root.indent == indent + assert saveindent[0] > indent + finally: + undo() + + def test_hook_proxy(self, pytester: Pytester) -> None: + """Test the gethookproxy function(#2016)""" + config = pytester.parseconfig() + session = Session.from_config(config) + pytester.makepyfile(**{"tests/conftest.py": "", "tests/subdir/conftest.py": ""}) + + conftest1 = pytester.path.joinpath("tests/conftest.py") + conftest2 = pytester.path.joinpath("tests/subdir/conftest.py") + + config.pluginmanager._importconftest( + conftest1, importmode="prepend", rootpath=pytester.path + ) + ihook_a = session.gethookproxy(pytester.path / "tests") + assert ihook_a is not None + config.pluginmanager._importconftest( + conftest2, importmode="prepend", rootpath=pytester.path + ) + ihook_b = session.gethookproxy(pytester.path / "tests") + assert ihook_a is not ihook_b + + def test_hook_with_addoption(self, pytester: Pytester) -> None: + """Test that hooks can be used in a call to pytest_addoption""" + pytester.makepyfile( + newhooks=""" + import pytest + @pytest.hookspec(firstresult=True) + def pytest_default_value(): + pass + """ + ) + pytester.makepyfile( + myplugin=""" + import newhooks + def pytest_addhooks(pluginmanager): + pluginmanager.add_hookspecs(newhooks) + def pytest_addoption(parser, pluginmanager): + default_value = pluginmanager.hook.pytest_default_value() + parser.addoption("--config", help="Config, defaults to %(default)s", default=default_value) + """ + ) + pytester.makeconftest( + """ + pytest_plugins=("myplugin",) + def pytest_default_value(): + return "default_value" + """ + ) + res = pytester.runpytest("--help") + res.stdout.fnmatch_lines(["*--config=CONFIG*default_value*"]) + + +def test_default_markers(pytester: Pytester) -> None: + result = pytester.runpytest("--markers") + result.stdout.fnmatch_lines(["*tryfirst*first*", "*trylast*last*"]) + + +def test_importplugin_error_message( + pytester: Pytester, pytestpm: PytestPluginManager +) -> None: + """Don't hide import errors when importing plugins and provide + an easy to debug message. + + See #375 and #1998. + """ + pytester.syspathinsert(pytester.path) + pytester.makepyfile( + qwe="""\ + def test_traceback(): + raise ImportError('Not possible to import: ☺') + test_traceback() + """ + ) + with pytest.raises(ImportError) as excinfo: + pytestpm.import_plugin("qwe") + + assert str(excinfo.value).endswith( + 'Error importing plugin "qwe": Not possible to import: ☺' + ) + assert "in test_traceback" in str(excinfo.traceback[-1]) + + +class TestPytestPluginManager: + def test_register_imported_modules(self) -> None: + pm = PytestPluginManager() + mod = types.ModuleType("x.y.pytest_hello") + pm.register(mod) + assert pm.is_registered(mod) + values = pm.get_plugins() + assert mod in values + pytest.raises(ValueError, pm.register, mod) + pytest.raises(ValueError, lambda: pm.register(mod)) + # assert not pm.is_registered(mod2) + assert pm.get_plugins() == values + + def test_canonical_import(self, monkeypatch): + mod = types.ModuleType("pytest_xyz") + monkeypatch.setitem(sys.modules, "pytest_xyz", mod) + pm = PytestPluginManager() + pm.import_plugin("pytest_xyz") + assert pm.get_plugin("pytest_xyz") == mod + assert pm.is_registered(mod) + + def test_consider_module( + self, pytester: Pytester, pytestpm: PytestPluginManager + ) -> None: + pytester.syspathinsert() + pytester.makepyfile(pytest_p1="#") + pytester.makepyfile(pytest_p2="#") + mod = types.ModuleType("temp") + mod.__dict__["pytest_plugins"] = ["pytest_p1", "pytest_p2"] + pytestpm.consider_module(mod) + assert pytestpm.get_plugin("pytest_p1").__name__ == "pytest_p1" + assert pytestpm.get_plugin("pytest_p2").__name__ == "pytest_p2" + + def test_consider_module_import_module( + self, pytester: Pytester, _config_for_test: Config + ) -> None: + pytestpm = _config_for_test.pluginmanager + mod = types.ModuleType("x") + mod.__dict__["pytest_plugins"] = "pytest_a" + aplugin = pytester.makepyfile(pytest_a="#") + reprec = pytester.make_hook_recorder(pytestpm) + pytester.syspathinsert(aplugin.parent) + pytestpm.consider_module(mod) + call = reprec.getcall(pytestpm.hook.pytest_plugin_registered.name) + assert call.plugin.__name__ == "pytest_a" + + # check that it is not registered twice + pytestpm.consider_module(mod) + values = reprec.getcalls("pytest_plugin_registered") + assert len(values) == 1 + + def test_consider_env_fails_to_import( + self, monkeypatch: MonkeyPatch, pytestpm: PytestPluginManager + ) -> None: + monkeypatch.setenv("PYTEST_PLUGINS", "nonexisting", prepend=",") + with pytest.raises(ImportError): + pytestpm.consider_env() + + @pytest.mark.filterwarnings("always") + def test_plugin_skip(self, pytester: Pytester, monkeypatch: MonkeyPatch) -> None: + p = pytester.makepyfile( + skipping1=""" + import pytest + pytest.skip("hello", allow_module_level=True) + """ + ) + shutil.copy(p, p.with_name("skipping2.py")) + monkeypatch.setenv("PYTEST_PLUGINS", "skipping2") + result = pytester.runpytest("-p", "skipping1", syspathinsert=True) + assert result.ret == ExitCode.NO_TESTS_COLLECTED + result.stdout.fnmatch_lines( + ["*skipped plugin*skipping1*hello*", "*skipped plugin*skipping2*hello*"] + ) + + def test_consider_env_plugin_instantiation( + self, + pytester: Pytester, + monkeypatch: MonkeyPatch, + pytestpm: PytestPluginManager, + ) -> None: + pytester.syspathinsert() + pytester.makepyfile(xy123="#") + monkeypatch.setitem(os.environ, "PYTEST_PLUGINS", "xy123") + l1 = len(pytestpm.get_plugins()) + pytestpm.consider_env() + l2 = len(pytestpm.get_plugins()) + assert l2 == l1 + 1 + assert pytestpm.get_plugin("xy123") + pytestpm.consider_env() + l3 = len(pytestpm.get_plugins()) + assert l2 == l3 + + def test_pluginmanager_ENV_startup( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + pytester.makepyfile(pytest_x500="#") + p = pytester.makepyfile( + """ + import pytest + def test_hello(pytestconfig): + plugin = pytestconfig.pluginmanager.get_plugin('pytest_x500') + assert plugin is not None + """ + ) + monkeypatch.setenv("PYTEST_PLUGINS", "pytest_x500", prepend=",") + result = pytester.runpytest(p, syspathinsert=True) + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed*"]) + + def test_import_plugin_importname( + self, pytester: Pytester, pytestpm: PytestPluginManager + ) -> None: + pytest.raises(ImportError, pytestpm.import_plugin, "qweqwex.y") + pytest.raises(ImportError, pytestpm.import_plugin, "pytest_qweqwx.y") + + pytester.syspathinsert() + pluginname = "pytest_hello" + pytester.makepyfile(**{pluginname: ""}) + pytestpm.import_plugin("pytest_hello") + len1 = len(pytestpm.get_plugins()) + pytestpm.import_plugin("pytest_hello") + len2 = len(pytestpm.get_plugins()) + assert len1 == len2 + plugin1 = pytestpm.get_plugin("pytest_hello") + assert plugin1.__name__.endswith("pytest_hello") + plugin2 = pytestpm.get_plugin("pytest_hello") + assert plugin2 is plugin1 + + def test_import_plugin_dotted_name( + self, pytester: Pytester, pytestpm: PytestPluginManager + ) -> None: + pytest.raises(ImportError, pytestpm.import_plugin, "qweqwex.y") + pytest.raises(ImportError, pytestpm.import_plugin, "pytest_qweqwex.y") + + pytester.syspathinsert() + pytester.mkpydir("pkg").joinpath("plug.py").write_text("x=3") + pluginname = "pkg.plug" + pytestpm.import_plugin(pluginname) + mod = pytestpm.get_plugin("pkg.plug") + assert mod.x == 3 + + def test_consider_conftest_deps( + self, + pytester: Pytester, + pytestpm: PytestPluginManager, + ) -> None: + mod = import_path( + pytester.makepyfile("pytest_plugins='xyz'"), root=pytester.path + ) + with pytest.raises(ImportError): + pytestpm.consider_conftest(mod) + + +class TestPytestPluginManagerBootstrapming: + def test_preparse_args(self, pytestpm: PytestPluginManager) -> None: + pytest.raises( + ImportError, lambda: pytestpm.consider_preparse(["xyz", "-p", "hello123"]) + ) + + # Handles -p without space (#3532). + with pytest.raises(ImportError) as excinfo: + pytestpm.consider_preparse(["-phello123"]) + assert '"hello123"' in excinfo.value.args[0] + pytestpm.consider_preparse(["-pno:hello123"]) + + # Handles -p without following arg (when used without argparse). + pytestpm.consider_preparse(["-p"]) + + with pytest.raises(UsageError, match="^plugin main cannot be disabled$"): + pytestpm.consider_preparse(["-p", "no:main"]) + + def test_plugin_prevent_register(self, pytestpm: PytestPluginManager) -> None: + pytestpm.consider_preparse(["xyz", "-p", "no:abc"]) + l1 = pytestpm.get_plugins() + pytestpm.register(42, name="abc") + l2 = pytestpm.get_plugins() + assert len(l2) == len(l1) + assert 42 not in l2 + + def test_plugin_prevent_register_unregistered_alredy_registered( + self, pytestpm: PytestPluginManager + ) -> None: + pytestpm.register(42, name="abc") + l1 = pytestpm.get_plugins() + assert 42 in l1 + pytestpm.consider_preparse(["xyz", "-p", "no:abc"]) + l2 = pytestpm.get_plugins() + assert 42 not in l2 + + def test_plugin_prevent_register_stepwise_on_cacheprovider_unregister( + self, pytestpm: PytestPluginManager + ) -> None: + """From PR #4304: The only way to unregister a module is documented at + the end of https://docs.pytest.org/en/stable/how-to/plugins.html. + + When unregister cacheprovider, then unregister stepwise too. + """ + pytestpm.register(42, name="cacheprovider") + pytestpm.register(43, name="stepwise") + l1 = pytestpm.get_plugins() + assert 42 in l1 + assert 43 in l1 + pytestpm.consider_preparse(["xyz", "-p", "no:cacheprovider"]) + l2 = pytestpm.get_plugins() + assert 42 not in l2 + assert 43 not in l2 + + def test_blocked_plugin_can_be_used(self, pytestpm: PytestPluginManager) -> None: + pytestpm.consider_preparse(["xyz", "-p", "no:abc", "-p", "abc"]) + + assert pytestpm.has_plugin("abc") + assert not pytestpm.is_blocked("abc") + assert not pytestpm.is_blocked("pytest_abc") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_pytester.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_pytester.py new file mode 100644 index 0000000000..049f8b22d8 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_pytester.py @@ -0,0 +1,855 @@ +import os +import subprocess +import sys +import time +from pathlib import Path +from types import ModuleType +from typing import List + +import _pytest.pytester as pytester_mod +import pytest +from _pytest.config import ExitCode +from _pytest.config import PytestPluginManager +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import CwdSnapshot +from _pytest.pytester import HookRecorder +from _pytest.pytester import LineMatcher +from _pytest.pytester import Pytester +from _pytest.pytester import SysModulesSnapshot +from _pytest.pytester import SysPathsSnapshot + + +def test_make_hook_recorder(pytester: Pytester) -> None: + item = pytester.getitem("def test_func(): pass") + recorder = pytester.make_hook_recorder(item.config.pluginmanager) + assert not recorder.getfailures() + + # (The silly condition is to fool mypy that the code below this is reachable) + if 1 + 1 == 2: + pytest.xfail("internal reportrecorder tests need refactoring") + + class rep: + excinfo = None + passed = False + failed = True + skipped = False + when = "call" + + recorder.hook.pytest_runtest_logreport(report=rep) # type: ignore[attr-defined] + failures = recorder.getfailures() + assert failures == [rep] # type: ignore[comparison-overlap] + failures = recorder.getfailures() + assert failures == [rep] # type: ignore[comparison-overlap] + + class rep2: + excinfo = None + passed = False + failed = False + skipped = True + when = "call" + + rep2.passed = False + rep2.skipped = True + recorder.hook.pytest_runtest_logreport(report=rep2) # type: ignore[attr-defined] + + modcol = pytester.getmodulecol("") + rep3 = modcol.config.hook.pytest_make_collect_report(collector=modcol) + rep3.passed = False + rep3.failed = True + rep3.skipped = False + recorder.hook.pytest_collectreport(report=rep3) # type: ignore[attr-defined] + + passed, skipped, failed = recorder.listoutcomes() + assert not passed and skipped and failed + + numpassed, numskipped, numfailed = recorder.countoutcomes() + assert numpassed == 0 + assert numskipped == 1 + assert numfailed == 1 + assert len(recorder.getfailedcollections()) == 1 + + recorder.unregister() # type: ignore[attr-defined] + recorder.clear() + recorder.hook.pytest_runtest_logreport(report=rep3) # type: ignore[attr-defined] + pytest.raises(ValueError, recorder.getfailures) + + +def test_parseconfig(pytester: Pytester) -> None: + config1 = pytester.parseconfig() + config2 = pytester.parseconfig() + assert config2 is not config1 + + +def test_pytester_runs_with_plugin(pytester: Pytester) -> None: + pytester.makepyfile( + """ + pytest_plugins = "pytester" + def test_hello(pytester): + assert 1 + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=1) + + +def test_pytester_with_doctest(pytester: Pytester) -> None: + """Check that pytester can be used within doctests. + + It used to use `request.function`, which is `None` with doctests.""" + pytester.makepyfile( + **{ + "sub/t-doctest.py": """ + ''' + >>> import os + >>> pytester = getfixture("pytester") + >>> str(pytester.makepyfile("content")).replace(os.sep, '/') + '.../basetemp/sub.t-doctest0/sub.py' + ''' + """, + "sub/__init__.py": "", + } + ) + result = pytester.runpytest( + "-p", "pytester", "--doctest-modules", "sub/t-doctest.py" + ) + assert result.ret == 0 + + +def test_runresult_assertion_on_xfail(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + pytest_plugins = "pytester" + + @pytest.mark.xfail + def test_potato(): + assert False + """ + ) + result = pytester.runpytest() + result.assert_outcomes(xfailed=1) + assert result.ret == 0 + + +def test_runresult_assertion_on_xpassed(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + pytest_plugins = "pytester" + + @pytest.mark.xfail + def test_potato(): + assert True + """ + ) + result = pytester.runpytest() + result.assert_outcomes(xpassed=1) + assert result.ret == 0 + + +def test_xpassed_with_strict_is_considered_a_failure(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + pytest_plugins = "pytester" + + @pytest.mark.xfail(strict=True) + def test_potato(): + assert True + """ + ) + result = pytester.runpytest() + result.assert_outcomes(failed=1) + assert result.ret != 0 + + +def make_holder(): + class apiclass: + def pytest_xyz(self, arg): + """X""" + + def pytest_xyz_noarg(self): + """X""" + + apimod = type(os)("api") + + def pytest_xyz(arg): + """X""" + + def pytest_xyz_noarg(): + """X""" + + apimod.pytest_xyz = pytest_xyz # type: ignore + apimod.pytest_xyz_noarg = pytest_xyz_noarg # type: ignore + return apiclass, apimod + + +@pytest.mark.parametrize("holder", make_holder()) +def test_hookrecorder_basic(holder) -> None: + pm = PytestPluginManager() + pm.add_hookspecs(holder) + rec = HookRecorder(pm, _ispytest=True) + pm.hook.pytest_xyz(arg=123) + call = rec.popcall("pytest_xyz") + assert call.arg == 123 + assert call._name == "pytest_xyz" + pytest.raises(pytest.fail.Exception, rec.popcall, "abc") + pm.hook.pytest_xyz_noarg() + call = rec.popcall("pytest_xyz_noarg") + assert call._name == "pytest_xyz_noarg" + + +def test_makepyfile_unicode(pytester: Pytester) -> None: + pytester.makepyfile(chr(0xFFFD)) + + +def test_makepyfile_utf8(pytester: Pytester) -> None: + """Ensure makepyfile accepts utf-8 bytes as input (#2738)""" + utf8_contents = """ + def setup_function(function): + mixed_encoding = 'São Paulo' + """.encode() + p = pytester.makepyfile(utf8_contents) + assert "mixed_encoding = 'São Paulo'".encode() in p.read_bytes() + + +class TestInlineRunModulesCleanup: + def test_inline_run_test_module_not_cleaned_up(self, pytester: Pytester) -> None: + test_mod = pytester.makepyfile("def test_foo(): assert True") + result = pytester.inline_run(str(test_mod)) + assert result.ret == ExitCode.OK + # rewrite module, now test should fail if module was re-imported + test_mod.write_text("def test_foo(): assert False") + result2 = pytester.inline_run(str(test_mod)) + assert result2.ret == ExitCode.TESTS_FAILED + + def spy_factory(self): + class SysModulesSnapshotSpy: + instances: List["SysModulesSnapshotSpy"] = [] # noqa: F821 + + def __init__(self, preserve=None) -> None: + SysModulesSnapshotSpy.instances.append(self) + self._spy_restore_count = 0 + self._spy_preserve = preserve + self.__snapshot = SysModulesSnapshot(preserve=preserve) + + def restore(self): + self._spy_restore_count += 1 + return self.__snapshot.restore() + + return SysModulesSnapshotSpy + + def test_inline_run_taking_and_restoring_a_sys_modules_snapshot( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + spy_factory = self.spy_factory() + monkeypatch.setattr(pytester_mod, "SysModulesSnapshot", spy_factory) + pytester.syspathinsert() + original = dict(sys.modules) + pytester.makepyfile(import1="# you son of a silly person") + pytester.makepyfile(import2="# my hovercraft is full of eels") + test_mod = pytester.makepyfile( + """ + import import1 + def test_foo(): import import2""" + ) + pytester.inline_run(str(test_mod)) + assert len(spy_factory.instances) == 1 + spy = spy_factory.instances[0] + assert spy._spy_restore_count == 1 + assert sys.modules == original + assert all(sys.modules[x] is original[x] for x in sys.modules) + + def test_inline_run_sys_modules_snapshot_restore_preserving_modules( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + spy_factory = self.spy_factory() + monkeypatch.setattr(pytester_mod, "SysModulesSnapshot", spy_factory) + test_mod = pytester.makepyfile("def test_foo(): pass") + pytester.inline_run(str(test_mod)) + spy = spy_factory.instances[0] + assert not spy._spy_preserve("black_knight") + assert spy._spy_preserve("zope") + assert spy._spy_preserve("zope.interface") + assert spy._spy_preserve("zopelicious") + + def test_external_test_module_imports_not_cleaned_up( + self, pytester: Pytester + ) -> None: + pytester.syspathinsert() + pytester.makepyfile(imported="data = 'you son of a silly person'") + import imported + + test_mod = pytester.makepyfile( + """ + def test_foo(): + import imported + imported.data = 42""" + ) + pytester.inline_run(str(test_mod)) + assert imported.data == 42 + + +def test_assert_outcomes_after_pytest_error(pytester: Pytester) -> None: + pytester.makepyfile("def test_foo(): assert True") + + result = pytester.runpytest("--unexpected-argument") + with pytest.raises(ValueError, match="Pytest terminal summary report not found"): + result.assert_outcomes(passed=0) + + +def test_cwd_snapshot(pytester: Pytester) -> None: + foo = pytester.mkdir("foo") + bar = pytester.mkdir("bar") + os.chdir(foo) + snapshot = CwdSnapshot() + os.chdir(bar) + assert Path().absolute() == bar + snapshot.restore() + assert Path().absolute() == foo + + +class TestSysModulesSnapshot: + key = "my-test-module" + + def test_remove_added(self) -> None: + original = dict(sys.modules) + assert self.key not in sys.modules + snapshot = SysModulesSnapshot() + sys.modules[self.key] = ModuleType("something") + assert self.key in sys.modules + snapshot.restore() + assert sys.modules == original + + def test_add_removed(self, monkeypatch: MonkeyPatch) -> None: + assert self.key not in sys.modules + monkeypatch.setitem(sys.modules, self.key, ModuleType("something")) + assert self.key in sys.modules + original = dict(sys.modules) + snapshot = SysModulesSnapshot() + del sys.modules[self.key] + assert self.key not in sys.modules + snapshot.restore() + assert sys.modules == original + + def test_restore_reloaded(self, monkeypatch: MonkeyPatch) -> None: + assert self.key not in sys.modules + monkeypatch.setitem(sys.modules, self.key, ModuleType("something")) + assert self.key in sys.modules + original = dict(sys.modules) + snapshot = SysModulesSnapshot() + sys.modules[self.key] = ModuleType("something else") + snapshot.restore() + assert sys.modules == original + + def test_preserve_modules(self, monkeypatch: MonkeyPatch) -> None: + key = [self.key + str(i) for i in range(3)] + assert not any(k in sys.modules for k in key) + for i, k in enumerate(key): + mod = ModuleType("something" + str(i)) + monkeypatch.setitem(sys.modules, k, mod) + original = dict(sys.modules) + + def preserve(name): + return name in (key[0], key[1], "some-other-key") + + snapshot = SysModulesSnapshot(preserve=preserve) + sys.modules[key[0]] = original[key[0]] = ModuleType("something else0") + sys.modules[key[1]] = original[key[1]] = ModuleType("something else1") + sys.modules[key[2]] = ModuleType("something else2") + snapshot.restore() + assert sys.modules == original + + def test_preserve_container(self, monkeypatch: MonkeyPatch) -> None: + original = dict(sys.modules) + assert self.key not in original + replacement = dict(sys.modules) + replacement[self.key] = ModuleType("life of brian") + snapshot = SysModulesSnapshot() + monkeypatch.setattr(sys, "modules", replacement) + snapshot.restore() + assert sys.modules is replacement + assert sys.modules == original + + +@pytest.mark.parametrize("path_type", ("path", "meta_path")) +class TestSysPathsSnapshot: + other_path = {"path": "meta_path", "meta_path": "path"} + + @staticmethod + def path(n: int) -> str: + return "my-dirty-little-secret-" + str(n) + + def test_restore(self, monkeypatch: MonkeyPatch, path_type) -> None: + other_path_type = self.other_path[path_type] + for i in range(10): + assert self.path(i) not in getattr(sys, path_type) + sys_path = [self.path(i) for i in range(6)] + monkeypatch.setattr(sys, path_type, sys_path) + original = list(sys_path) + original_other = list(getattr(sys, other_path_type)) + snapshot = SysPathsSnapshot() + transformation = {"source": (0, 1, 2, 3, 4, 5), "target": (6, 2, 9, 7, 5, 8)} + assert sys_path == [self.path(x) for x in transformation["source"]] + sys_path[1] = self.path(6) + sys_path[3] = self.path(7) + sys_path.append(self.path(8)) + del sys_path[4] + sys_path[3:3] = [self.path(9)] + del sys_path[0] + assert sys_path == [self.path(x) for x in transformation["target"]] + snapshot.restore() + assert getattr(sys, path_type) is sys_path + assert getattr(sys, path_type) == original + assert getattr(sys, other_path_type) == original_other + + def test_preserve_container(self, monkeypatch: MonkeyPatch, path_type) -> None: + other_path_type = self.other_path[path_type] + original_data = list(getattr(sys, path_type)) + original_other = getattr(sys, other_path_type) + original_other_data = list(original_other) + new: List[object] = [] + snapshot = SysPathsSnapshot() + monkeypatch.setattr(sys, path_type, new) + snapshot.restore() + assert getattr(sys, path_type) is new + assert getattr(sys, path_type) == original_data + assert getattr(sys, other_path_type) is original_other + assert getattr(sys, other_path_type) == original_other_data + + +def test_pytester_subprocess(pytester: Pytester) -> None: + testfile = pytester.makepyfile("def test_one(): pass") + assert pytester.runpytest_subprocess(testfile).ret == 0 + + +def test_pytester_subprocess_via_runpytest_arg(pytester: Pytester) -> None: + testfile = pytester.makepyfile( + """ + def test_pytester_subprocess(pytester): + import os + testfile = pytester.makepyfile( + \""" + import os + def test_one(): + assert {} != os.getpid() + \""".format(os.getpid()) + ) + assert pytester.runpytest(testfile).ret == 0 + """ + ) + result = pytester.runpytest_inprocess( + "-p", "pytester", "--runpytest", "subprocess", testfile + ) + assert result.ret == 0 + + +def test_unicode_args(pytester: Pytester) -> None: + result = pytester.runpytest("-k", "אבג") + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +def test_pytester_run_no_timeout(pytester: Pytester) -> None: + testfile = pytester.makepyfile("def test_no_timeout(): pass") + assert pytester.runpytest_subprocess(testfile).ret == ExitCode.OK + + +def test_pytester_run_with_timeout(pytester: Pytester) -> None: + testfile = pytester.makepyfile("def test_no_timeout(): pass") + + timeout = 120 + + start = time.time() + result = pytester.runpytest_subprocess(testfile, timeout=timeout) + end = time.time() + duration = end - start + + assert result.ret == ExitCode.OK + assert duration < timeout + + +def test_pytester_run_timeout_expires(pytester: Pytester) -> None: + testfile = pytester.makepyfile( + """ + import time + + def test_timeout(): + time.sleep(10)""" + ) + with pytest.raises(pytester.TimeoutExpired): + pytester.runpytest_subprocess(testfile, timeout=1) + + +def test_linematcher_with_nonlist() -> None: + """Test LineMatcher with regard to passing in a set (accidentally).""" + from _pytest._code.source import Source + + lm = LineMatcher([]) + with pytest.raises(TypeError, match="invalid type for lines2: set"): + lm.fnmatch_lines(set()) # type: ignore[arg-type] + with pytest.raises(TypeError, match="invalid type for lines2: dict"): + lm.fnmatch_lines({}) # type: ignore[arg-type] + with pytest.raises(TypeError, match="invalid type for lines2: set"): + lm.re_match_lines(set()) # type: ignore[arg-type] + with pytest.raises(TypeError, match="invalid type for lines2: dict"): + lm.re_match_lines({}) # type: ignore[arg-type] + with pytest.raises(TypeError, match="invalid type for lines2: Source"): + lm.fnmatch_lines(Source()) # type: ignore[arg-type] + lm.fnmatch_lines([]) + lm.fnmatch_lines(()) + lm.fnmatch_lines("") + assert lm._getlines({}) == {} # type: ignore[arg-type,comparison-overlap] + assert lm._getlines(set()) == set() # type: ignore[arg-type,comparison-overlap] + assert lm._getlines(Source()) == [] + assert lm._getlines(Source("pass\npass")) == ["pass", "pass"] + + +def test_linematcher_match_failure() -> None: + lm = LineMatcher(["foo", "foo", "bar"]) + with pytest.raises(pytest.fail.Exception) as e: + lm.fnmatch_lines(["foo", "f*", "baz"]) + assert e.value.msg is not None + assert e.value.msg.splitlines() == [ + "exact match: 'foo'", + "fnmatch: 'f*'", + " with: 'foo'", + "nomatch: 'baz'", + " and: 'bar'", + "remains unmatched: 'baz'", + ] + + lm = LineMatcher(["foo", "foo", "bar"]) + with pytest.raises(pytest.fail.Exception) as e: + lm.re_match_lines(["foo", "^f.*", "baz"]) + assert e.value.msg is not None + assert e.value.msg.splitlines() == [ + "exact match: 'foo'", + "re.match: '^f.*'", + " with: 'foo'", + " nomatch: 'baz'", + " and: 'bar'", + "remains unmatched: 'baz'", + ] + + +def test_linematcher_consecutive() -> None: + lm = LineMatcher(["1", "", "2"]) + with pytest.raises(pytest.fail.Exception) as excinfo: + lm.fnmatch_lines(["1", "2"], consecutive=True) + assert str(excinfo.value).splitlines() == [ + "exact match: '1'", + "no consecutive match: '2'", + " with: ''", + ] + + lm.re_match_lines(["1", r"\d?", "2"], consecutive=True) + with pytest.raises(pytest.fail.Exception) as excinfo: + lm.re_match_lines(["1", r"\d", "2"], consecutive=True) + assert str(excinfo.value).splitlines() == [ + "exact match: '1'", + r"no consecutive match: '\\d'", + " with: ''", + ] + + +@pytest.mark.parametrize("function", ["no_fnmatch_line", "no_re_match_line"]) +def test_linematcher_no_matching(function: str) -> None: + if function == "no_fnmatch_line": + good_pattern = "*.py OK*" + bad_pattern = "*X.py OK*" + else: + assert function == "no_re_match_line" + good_pattern = r".*py OK" + bad_pattern = r".*Xpy OK" + + lm = LineMatcher( + [ + "cachedir: .pytest_cache", + "collecting ... collected 1 item", + "", + "show_fixtures_per_test.py OK", + "=== elapsed 1s ===", + ] + ) + + # check the function twice to ensure we don't accumulate the internal buffer + for i in range(2): + with pytest.raises(pytest.fail.Exception) as e: + func = getattr(lm, function) + func(good_pattern) + obtained = str(e.value).splitlines() + if function == "no_fnmatch_line": + assert obtained == [ + f"nomatch: '{good_pattern}'", + " and: 'cachedir: .pytest_cache'", + " and: 'collecting ... collected 1 item'", + " and: ''", + f"fnmatch: '{good_pattern}'", + " with: 'show_fixtures_per_test.py OK'", + ] + else: + assert obtained == [ + f" nomatch: '{good_pattern}'", + " and: 'cachedir: .pytest_cache'", + " and: 'collecting ... collected 1 item'", + " and: ''", + f"re.match: '{good_pattern}'", + " with: 'show_fixtures_per_test.py OK'", + ] + + func = getattr(lm, function) + func(bad_pattern) # bad pattern does not match any line: passes + + +def test_linematcher_no_matching_after_match() -> None: + lm = LineMatcher(["1", "2", "3"]) + lm.fnmatch_lines(["1", "3"]) + with pytest.raises(pytest.fail.Exception) as e: + lm.no_fnmatch_line("*") + assert str(e.value).splitlines() == ["fnmatch: '*'", " with: '1'"] + + +def test_linematcher_string_api() -> None: + lm = LineMatcher(["foo", "bar"]) + assert str(lm) == "foo\nbar" + + +def test_pytest_addopts_before_pytester(request, monkeypatch: MonkeyPatch) -> None: + orig = os.environ.get("PYTEST_ADDOPTS", None) + monkeypatch.setenv("PYTEST_ADDOPTS", "--orig-unused") + pytester: Pytester = request.getfixturevalue("pytester") + assert "PYTEST_ADDOPTS" not in os.environ + pytester._finalize() + assert os.environ.get("PYTEST_ADDOPTS") == "--orig-unused" + monkeypatch.undo() + assert os.environ.get("PYTEST_ADDOPTS") == orig + + +def test_run_stdin(pytester: Pytester) -> None: + with pytest.raises(pytester.TimeoutExpired): + pytester.run( + sys.executable, + "-c", + "import sys, time; time.sleep(1); print(sys.stdin.read())", + stdin=subprocess.PIPE, + timeout=0.1, + ) + + with pytest.raises(pytester.TimeoutExpired): + result = pytester.run( + sys.executable, + "-c", + "import sys, time; time.sleep(1); print(sys.stdin.read())", + stdin=b"input\n2ndline", + timeout=0.1, + ) + + result = pytester.run( + sys.executable, + "-c", + "import sys; print(sys.stdin.read())", + stdin=b"input\n2ndline", + ) + assert result.stdout.lines == ["input", "2ndline"] + assert result.stderr.str() == "" + assert result.ret == 0 + + +def test_popen_stdin_pipe(pytester: Pytester) -> None: + proc = pytester.popen( + [sys.executable, "-c", "import sys; print(sys.stdin.read())"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + ) + stdin = b"input\n2ndline" + stdout, stderr = proc.communicate(input=stdin) + assert stdout.decode("utf8").splitlines() == ["input", "2ndline"] + assert stderr == b"" + assert proc.returncode == 0 + + +def test_popen_stdin_bytes(pytester: Pytester) -> None: + proc = pytester.popen( + [sys.executable, "-c", "import sys; print(sys.stdin.read())"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=b"input\n2ndline", + ) + stdout, stderr = proc.communicate() + assert stdout.decode("utf8").splitlines() == ["input", "2ndline"] + assert stderr == b"" + assert proc.returncode == 0 + + +def test_popen_default_stdin_stderr_and_stdin_None(pytester: Pytester) -> None: + # stdout, stderr default to pipes, + # stdin can be None to not close the pipe, avoiding + # "ValueError: flush of closed file" with `communicate()`. + # + # Wraps the test to make it not hang when run with "-s". + p1 = pytester.makepyfile( + ''' + import sys + + def test_inner(pytester): + p1 = pytester.makepyfile( + """ + import sys + print(sys.stdin.read()) # empty + print('stdout') + sys.stderr.write('stderr') + """ + ) + proc = pytester.popen([sys.executable, str(p1)], stdin=None) + stdout, stderr = proc.communicate(b"ignored") + assert stdout.splitlines() == [b"", b"stdout"] + assert stderr.splitlines() == [b"stderr"] + assert proc.returncode == 0 + ''' + ) + result = pytester.runpytest("-p", "pytester", str(p1)) + assert result.ret == 0 + + +def test_spawn_uses_tmphome(pytester: Pytester) -> None: + tmphome = str(pytester.path) + assert os.environ.get("HOME") == tmphome + + pytester._monkeypatch.setenv("CUSTOMENV", "42") + + p1 = pytester.makepyfile( + """ + import os + + def test(): + assert os.environ["HOME"] == {tmphome!r} + assert os.environ["CUSTOMENV"] == "42" + """.format( + tmphome=tmphome + ) + ) + child = pytester.spawn_pytest(str(p1)) + out = child.read() + assert child.wait() == 0, out.decode("utf8") + + +def test_run_result_repr() -> None: + outlines = ["some", "normal", "output"] + errlines = ["some", "nasty", "errors", "happened"] + + # known exit code + r = pytester_mod.RunResult(1, outlines, errlines, duration=0.5) + assert repr(r) == ( + f"" + ) + + # unknown exit code: just the number + r = pytester_mod.RunResult(99, outlines, errlines, duration=0.5) + assert ( + repr(r) == "" + ) + + +def test_pytester_outcomes_with_multiple_errors(pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def bad_fixture(): + raise Exception("bad") + + def test_error1(bad_fixture): + pass + + def test_error2(bad_fixture): + pass + """ + ) + result = pytester.runpytest(str(p1)) + result.assert_outcomes(errors=2) + + assert result.parseoutcomes() == {"errors": 2} + + +def test_parse_summary_line_always_plural() -> None: + """Parsing summaries always returns plural nouns (#6505)""" + lines = [ + "some output 1", + "some output 2", + "======= 1 failed, 1 passed, 1 warning, 1 error in 0.13s ====", + "done.", + ] + assert pytester_mod.RunResult.parse_summary_nouns(lines) == { + "errors": 1, + "failed": 1, + "passed": 1, + "warnings": 1, + } + + lines = [ + "some output 1", + "some output 2", + "======= 1 failed, 1 passed, 2 warnings, 2 errors in 0.13s ====", + "done.", + ] + assert pytester_mod.RunResult.parse_summary_nouns(lines) == { + "errors": 2, + "failed": 1, + "passed": 1, + "warnings": 2, + } + + +def test_makefile_joins_absolute_path(pytester: Pytester) -> None: + absfile = pytester.path / "absfile" + p1 = pytester.makepyfile(**{str(absfile): ""}) + assert str(p1) == str(pytester.path / "absfile.py") + + +def test_pytester_makefile_dot_prefixes_extension_with_warning( + pytester: Pytester, +) -> None: + with pytest.raises( + ValueError, + match="pytester.makefile expects a file extension, try .foo.bar instead of foo.bar", + ): + pytester.makefile("foo.bar", "") + + +@pytest.mark.filterwarnings("default") +def test_pytester_assert_outcomes_warnings(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import warnings + + def test_with_warning(): + warnings.warn(UserWarning("some custom warning")) + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=1, warnings=1) + # If warnings is not passed, it is not checked at all. + result.assert_outcomes(passed=1) + + +def test_pytester_outcomes_deselected(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_one(): + pass + + def test_two(): + pass + """ + ) + result = pytester.runpytest("-k", "test_one") + result.assert_outcomes(passed=1, deselected=1) + # If deselected is not passed, it is not checked at all. + result.assert_outcomes(passed=1) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_python_path.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_python_path.py new file mode 100644 index 0000000000..5ee0f55e36 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_python_path.py @@ -0,0 +1,110 @@ +import sys +from textwrap import dedent +from typing import Generator +from typing import List +from typing import Optional + +import pytest +from _pytest.pytester import Pytester + + +@pytest.fixture() +def file_structure(pytester: Pytester) -> None: + pytester.makepyfile( + test_foo=""" + from foo import foo + + def test_foo(): + assert foo() == 1 + """ + ) + + pytester.makepyfile( + test_bar=""" + from bar import bar + + def test_bar(): + assert bar() == 2 + """ + ) + + foo_py = pytester.mkdir("sub") / "foo.py" + content = dedent( + """ + def foo(): + return 1 + """ + ) + foo_py.write_text(content, encoding="utf-8") + + bar_py = pytester.mkdir("sub2") / "bar.py" + content = dedent( + """ + def bar(): + return 2 + """ + ) + bar_py.write_text(content, encoding="utf-8") + + +def test_one_dir(pytester: Pytester, file_structure) -> None: + pytester.makefile(".ini", pytest="[pytest]\npythonpath=sub\n") + result = pytester.runpytest("test_foo.py") + assert result.ret == 0 + result.assert_outcomes(passed=1) + + +def test_two_dirs(pytester: Pytester, file_structure) -> None: + pytester.makefile(".ini", pytest="[pytest]\npythonpath=sub sub2\n") + result = pytester.runpytest("test_foo.py", "test_bar.py") + assert result.ret == 0 + result.assert_outcomes(passed=2) + + +def test_module_not_found(pytester: Pytester, file_structure) -> None: + """Without the pythonpath setting, the module should not be found.""" + pytester.makefile(".ini", pytest="[pytest]\n") + result = pytester.runpytest("test_foo.py") + assert result.ret == pytest.ExitCode.INTERRUPTED + result.assert_outcomes(errors=1) + expected_error = "E ModuleNotFoundError: No module named 'foo'" + result.stdout.fnmatch_lines([expected_error]) + + +def test_no_ini(pytester: Pytester, file_structure) -> None: + """If no ini file, test should error.""" + result = pytester.runpytest("test_foo.py") + assert result.ret == pytest.ExitCode.INTERRUPTED + result.assert_outcomes(errors=1) + expected_error = "E ModuleNotFoundError: No module named 'foo'" + result.stdout.fnmatch_lines([expected_error]) + + +def test_clean_up(pytester: Pytester) -> None: + """Test that the plugin cleans up after itself.""" + # This is tough to test behaviorly because the cleanup really runs last. + # So the test make several implementation assumptions: + # - Cleanup is done in pytest_unconfigure(). + # - Not a hookwrapper. + # So we can add a hookwrapper ourselves to test what it does. + pytester.makefile(".ini", pytest="[pytest]\npythonpath=I_SHALL_BE_REMOVED\n") + pytester.makepyfile(test_foo="""def test_foo(): pass""") + + before: Optional[List[str]] = None + after: Optional[List[str]] = None + + class Plugin: + @pytest.hookimpl(hookwrapper=True, tryfirst=True) + def pytest_unconfigure(self) -> Generator[None, None, None]: + nonlocal before, after + before = sys.path.copy() + yield + after = sys.path.copy() + + result = pytester.runpytest_inprocess(plugins=[Plugin()]) + assert result.ret == 0 + + assert before is not None + assert after is not None + assert any("I_SHALL_BE_REMOVED" in entry for entry in before) + assert not any("I_SHALL_BE_REMOVED" in entry for entry in after) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_recwarn.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_recwarn.py new file mode 100644 index 0000000000..d3f218f166 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_recwarn.py @@ -0,0 +1,410 @@ +import re +import warnings +from typing import Optional + +import pytest +from _pytest.pytester import Pytester +from _pytest.recwarn import WarningsRecorder + + +def test_recwarn_stacklevel(recwarn: WarningsRecorder) -> None: + warnings.warn("hello") + warn = recwarn.pop() + assert warn.filename == __file__ + + +def test_recwarn_functional(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import warnings + def test_method(recwarn): + warnings.warn("hello") + warn = recwarn.pop() + assert isinstance(warn.message, UserWarning) + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +@pytest.mark.filterwarnings("") +def test_recwarn_captures_deprecation_warning(recwarn: WarningsRecorder) -> None: + """ + Check that recwarn can capture DeprecationWarning by default + without custom filterwarnings (see #8666). + """ + warnings.warn(DeprecationWarning("some deprecation")) + assert len(recwarn) == 1 + assert recwarn.pop(DeprecationWarning) + + +class TestWarningsRecorderChecker: + def test_recording(self) -> None: + rec = WarningsRecorder(_ispytest=True) + with rec: + assert not rec.list + warnings.warn_explicit("hello", UserWarning, "xyz", 13) + assert len(rec.list) == 1 + warnings.warn(DeprecationWarning("hello")) + assert len(rec.list) == 2 + warn = rec.pop() + assert str(warn.message) == "hello" + values = rec.list + rec.clear() + assert len(rec.list) == 0 + assert values is rec.list + pytest.raises(AssertionError, rec.pop) + + def test_warn_stacklevel(self) -> None: + """#4243""" + rec = WarningsRecorder(_ispytest=True) + with rec: + warnings.warn("test", DeprecationWarning, 2) + + def test_typechecking(self) -> None: + from _pytest.recwarn import WarningsChecker + + with pytest.raises(TypeError): + WarningsChecker(5, _ispytest=True) # type: ignore[arg-type] + with pytest.raises(TypeError): + WarningsChecker(("hi", RuntimeWarning), _ispytest=True) # type: ignore[arg-type] + with pytest.raises(TypeError): + WarningsChecker([DeprecationWarning, RuntimeWarning], _ispytest=True) # type: ignore[arg-type] + + def test_invalid_enter_exit(self) -> None: + # wrap this test in WarningsRecorder to ensure warning state gets reset + with WarningsRecorder(_ispytest=True): + with pytest.raises(RuntimeError): + rec = WarningsRecorder(_ispytest=True) + rec.__exit__(None, None, None) # can't exit before entering + + with pytest.raises(RuntimeError): + rec = WarningsRecorder(_ispytest=True) + with rec: + with rec: + pass # can't enter twice + + +class TestDeprecatedCall: + """test pytest.deprecated_call()""" + + def dep(self, i: int, j: Optional[int] = None) -> int: + if i == 0: + warnings.warn("is deprecated", DeprecationWarning, stacklevel=1) + return 42 + + def dep_explicit(self, i: int) -> None: + if i == 0: + warnings.warn_explicit( + "dep_explicit", category=DeprecationWarning, filename="hello", lineno=3 + ) + + def test_deprecated_call_raises(self) -> None: + with pytest.raises(pytest.fail.Exception, match="No warnings of type"): + pytest.deprecated_call(self.dep, 3, 5) + + def test_deprecated_call(self) -> None: + pytest.deprecated_call(self.dep, 0, 5) + + def test_deprecated_call_ret(self) -> None: + ret = pytest.deprecated_call(self.dep, 0) + assert ret == 42 + + def test_deprecated_call_preserves(self) -> None: + # Type ignored because `onceregistry` and `filters` are not + # documented API. + onceregistry = warnings.onceregistry.copy() # type: ignore + filters = warnings.filters[:] # type: ignore + warn = warnings.warn + warn_explicit = warnings.warn_explicit + self.test_deprecated_call_raises() + self.test_deprecated_call() + assert onceregistry == warnings.onceregistry # type: ignore + assert filters == warnings.filters # type: ignore + assert warn is warnings.warn + assert warn_explicit is warnings.warn_explicit + + def test_deprecated_explicit_call_raises(self) -> None: + with pytest.raises(pytest.fail.Exception): + pytest.deprecated_call(self.dep_explicit, 3) + + def test_deprecated_explicit_call(self) -> None: + pytest.deprecated_call(self.dep_explicit, 0) + pytest.deprecated_call(self.dep_explicit, 0) + + @pytest.mark.parametrize("mode", ["context_manager", "call"]) + def test_deprecated_call_no_warning(self, mode) -> None: + """Ensure deprecated_call() raises the expected failure when its block/function does + not raise a deprecation warning. + """ + + def f(): + pass + + msg = "No warnings of type (.*DeprecationWarning.*, .*PendingDeprecationWarning.*)" + with pytest.raises(pytest.fail.Exception, match=msg): + if mode == "call": + pytest.deprecated_call(f) + else: + with pytest.deprecated_call(): + f() + + @pytest.mark.parametrize( + "warning_type", [PendingDeprecationWarning, DeprecationWarning] + ) + @pytest.mark.parametrize("mode", ["context_manager", "call"]) + @pytest.mark.parametrize("call_f_first", [True, False]) + @pytest.mark.filterwarnings("ignore") + def test_deprecated_call_modes(self, warning_type, mode, call_f_first) -> None: + """Ensure deprecated_call() captures a deprecation warning as expected inside its + block/function. + """ + + def f(): + warnings.warn(warning_type("hi")) + return 10 + + # ensure deprecated_call() can capture the warning even if it has already been triggered + if call_f_first: + assert f() == 10 + if mode == "call": + assert pytest.deprecated_call(f) == 10 + else: + with pytest.deprecated_call(): + assert f() == 10 + + @pytest.mark.parametrize("mode", ["context_manager", "call"]) + def test_deprecated_call_exception_is_raised(self, mode) -> None: + """If the block of the code being tested by deprecated_call() raises an exception, + it must raise the exception undisturbed. + """ + + def f(): + raise ValueError("some exception") + + with pytest.raises(ValueError, match="some exception"): + if mode == "call": + pytest.deprecated_call(f) + else: + with pytest.deprecated_call(): + f() + + def test_deprecated_call_specificity(self) -> None: + other_warnings = [ + Warning, + UserWarning, + SyntaxWarning, + RuntimeWarning, + FutureWarning, + ImportWarning, + UnicodeWarning, + ] + for warning in other_warnings: + + def f(): + warnings.warn(warning("hi")) + + with pytest.raises(pytest.fail.Exception): + pytest.deprecated_call(f) + with pytest.raises(pytest.fail.Exception): + with pytest.deprecated_call(): + f() + + def test_deprecated_call_supports_match(self) -> None: + with pytest.deprecated_call(match=r"must be \d+$"): + warnings.warn("value must be 42", DeprecationWarning) + + with pytest.raises(pytest.fail.Exception): + with pytest.deprecated_call(match=r"must be \d+$"): + warnings.warn("this is not here", DeprecationWarning) + + +class TestWarns: + def test_check_callable(self) -> None: + source = "warnings.warn('w1', RuntimeWarning)" + with pytest.raises(TypeError, match=r".* must be callable"): + pytest.warns(RuntimeWarning, source) # type: ignore + + def test_several_messages(self) -> None: + # different messages, b/c Python suppresses multiple identical warnings + pytest.warns(RuntimeWarning, lambda: warnings.warn("w1", RuntimeWarning)) + with pytest.raises(pytest.fail.Exception): + pytest.warns(UserWarning, lambda: warnings.warn("w2", RuntimeWarning)) + pytest.warns(RuntimeWarning, lambda: warnings.warn("w3", RuntimeWarning)) + + def test_function(self) -> None: + pytest.warns( + SyntaxWarning, lambda msg: warnings.warn(msg, SyntaxWarning), "syntax" + ) + + def test_warning_tuple(self) -> None: + pytest.warns( + (RuntimeWarning, SyntaxWarning), lambda: warnings.warn("w1", RuntimeWarning) + ) + pytest.warns( + (RuntimeWarning, SyntaxWarning), lambda: warnings.warn("w2", SyntaxWarning) + ) + pytest.raises( + pytest.fail.Exception, + lambda: pytest.warns( + (RuntimeWarning, SyntaxWarning), + lambda: warnings.warn("w3", UserWarning), + ), + ) + + def test_as_contextmanager(self) -> None: + with pytest.warns(RuntimeWarning): + warnings.warn("runtime", RuntimeWarning) + + with pytest.warns(UserWarning): + warnings.warn("user", UserWarning) + + with pytest.raises(pytest.fail.Exception) as excinfo: + with pytest.warns(RuntimeWarning): + warnings.warn("user", UserWarning) + excinfo.match( + r"DID NOT WARN. No warnings of type \(.+RuntimeWarning.+,\) were emitted. " + r"The list of emitted warnings is: \[UserWarning\('user',?\)\]." + ) + + with pytest.raises(pytest.fail.Exception) as excinfo: + with pytest.warns(UserWarning): + warnings.warn("runtime", RuntimeWarning) + excinfo.match( + r"DID NOT WARN. No warnings of type \(.+UserWarning.+,\) were emitted. " + r"The list of emitted warnings is: \[RuntimeWarning\('runtime',?\)\]." + ) + + with pytest.raises(pytest.fail.Exception) as excinfo: + with pytest.warns(UserWarning): + pass + excinfo.match( + r"DID NOT WARN. No warnings of type \(.+UserWarning.+,\) were emitted. " + r"The list of emitted warnings is: \[\]." + ) + + warning_classes = (UserWarning, FutureWarning) + with pytest.raises(pytest.fail.Exception) as excinfo: + with pytest.warns(warning_classes) as warninfo: + warnings.warn("runtime", RuntimeWarning) + warnings.warn("import", ImportWarning) + + message_template = ( + "DID NOT WARN. No warnings of type {0} were emitted. " + "The list of emitted warnings is: {1}." + ) + excinfo.match( + re.escape( + message_template.format( + warning_classes, [each.message for each in warninfo] + ) + ) + ) + + def test_record(self) -> None: + with pytest.warns(UserWarning) as record: + warnings.warn("user", UserWarning) + + assert len(record) == 1 + assert str(record[0].message) == "user" + + def test_record_only(self) -> None: + with pytest.warns() as record: + warnings.warn("user", UserWarning) + warnings.warn("runtime", RuntimeWarning) + + assert len(record) == 2 + assert str(record[0].message) == "user" + assert str(record[1].message) == "runtime" + + def test_record_only_none_deprecated_warn(self) -> None: + # This should become an error when WARNS_NONE_ARG is removed in Pytest 8.0 + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with pytest.warns(None) as record: # type: ignore[call-overload] + warnings.warn("user", UserWarning) + warnings.warn("runtime", RuntimeWarning) + + assert len(record) == 2 + assert str(record[0].message) == "user" + assert str(record[1].message) == "runtime" + + def test_record_by_subclass(self) -> None: + with pytest.warns(Warning) as record: + warnings.warn("user", UserWarning) + warnings.warn("runtime", RuntimeWarning) + + assert len(record) == 2 + assert str(record[0].message) == "user" + assert str(record[1].message) == "runtime" + + class MyUserWarning(UserWarning): + pass + + class MyRuntimeWarning(RuntimeWarning): + pass + + with pytest.warns((UserWarning, RuntimeWarning)) as record: + warnings.warn("user", MyUserWarning) + warnings.warn("runtime", MyRuntimeWarning) + + assert len(record) == 2 + assert str(record[0].message) == "user" + assert str(record[1].message) == "runtime" + + def test_double_test(self, pytester: Pytester) -> None: + """If a test is run again, the warning should still be raised""" + pytester.makepyfile( + """ + import pytest + import warnings + + @pytest.mark.parametrize('run', [1, 2]) + def test(run): + with pytest.warns(RuntimeWarning): + warnings.warn("runtime", RuntimeWarning) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*2 passed in*"]) + + def test_match_regex(self) -> None: + with pytest.warns(UserWarning, match=r"must be \d+$"): + warnings.warn("value must be 42", UserWarning) + + with pytest.raises(pytest.fail.Exception): + with pytest.warns(UserWarning, match=r"must be \d+$"): + warnings.warn("this is not here", UserWarning) + + with pytest.raises(pytest.fail.Exception): + with pytest.warns(FutureWarning, match=r"must be \d+$"): + warnings.warn("value must be 42", UserWarning) + + def test_one_from_multiple_warns(self) -> None: + with pytest.warns(UserWarning, match=r"aaa"): + warnings.warn("cccccccccc", UserWarning) + warnings.warn("bbbbbbbbbb", UserWarning) + warnings.warn("aaaaaaaaaa", UserWarning) + + def test_none_of_multiple_warns(self) -> None: + with pytest.raises(pytest.fail.Exception): + with pytest.warns(UserWarning, match=r"aaa"): + warnings.warn("bbbbbbbbbb", UserWarning) + warnings.warn("cccccccccc", UserWarning) + + @pytest.mark.filterwarnings("ignore") + def test_can_capture_previously_warned(self) -> None: + def f() -> int: + warnings.warn(UserWarning("ohai")) + return 10 + + assert f() == 10 + assert pytest.warns(UserWarning, f) == 10 + assert pytest.warns(UserWarning, f) == 10 + assert pytest.warns(UserWarning, f) != "10" # type: ignore[comparison-overlap] + + def test_warns_context_manager_with_kwargs(self) -> None: + with pytest.raises(TypeError) as excinfo: + with pytest.warns(UserWarning, foo="bar"): # type: ignore + pass + assert "Unexpected keyword arguments" in str(excinfo.value) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_reports.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_reports.py new file mode 100644 index 0000000000..31b6cf1afc --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_reports.py @@ -0,0 +1,488 @@ +from typing import Sequence +from typing import Union + +import pytest +from _pytest._code.code import ExceptionChainRepr +from _pytest._code.code import ExceptionRepr +from _pytest.config import Config +from _pytest.pytester import Pytester +from _pytest.reports import CollectReport +from _pytest.reports import TestReport + + +class TestReportSerialization: + def test_xdist_longrepr_to_str_issue_241(self, pytester: Pytester) -> None: + """Regarding issue pytest-xdist#241. + + This test came originally from test_remote.py in xdist (ca03269). + """ + pytester.makepyfile( + """ + def test_a(): assert False + def test_b(): pass + """ + ) + reprec = pytester.inline_run() + reports = reprec.getreports("pytest_runtest_logreport") + assert len(reports) == 6 + test_a_call = reports[1] + assert test_a_call.when == "call" + assert test_a_call.outcome == "failed" + assert test_a_call._to_json()["longrepr"]["reprtraceback"]["style"] == "long" + test_b_call = reports[4] + assert test_b_call.when == "call" + assert test_b_call.outcome == "passed" + assert test_b_call._to_json()["longrepr"] is None + + def test_xdist_report_longrepr_reprcrash_130(self, pytester: Pytester) -> None: + """Regarding issue pytest-xdist#130 + + This test came originally from test_remote.py in xdist (ca03269). + """ + reprec = pytester.inline_runsource( + """ + def test_fail(): + assert False, 'Expected Message' + """ + ) + reports = reprec.getreports("pytest_runtest_logreport") + assert len(reports) == 3 + rep = reports[1] + added_section = ("Failure Metadata", "metadata metadata", "*") + assert isinstance(rep.longrepr, ExceptionRepr) + rep.longrepr.sections.append(added_section) + d = rep._to_json() + a = TestReport._from_json(d) + assert isinstance(a.longrepr, ExceptionRepr) + # Check assembled == rep + assert a.__dict__.keys() == rep.__dict__.keys() + for key in rep.__dict__.keys(): + if key != "longrepr": + assert getattr(a, key) == getattr(rep, key) + assert rep.longrepr.reprcrash is not None + assert a.longrepr.reprcrash is not None + assert rep.longrepr.reprcrash.lineno == a.longrepr.reprcrash.lineno + assert rep.longrepr.reprcrash.message == a.longrepr.reprcrash.message + assert rep.longrepr.reprcrash.path == a.longrepr.reprcrash.path + assert rep.longrepr.reprtraceback.entrysep == a.longrepr.reprtraceback.entrysep + assert ( + rep.longrepr.reprtraceback.extraline == a.longrepr.reprtraceback.extraline + ) + assert rep.longrepr.reprtraceback.style == a.longrepr.reprtraceback.style + assert rep.longrepr.sections == a.longrepr.sections + # Missing section attribute PR171 + assert added_section in a.longrepr.sections + + def test_reprentries_serialization_170(self, pytester: Pytester) -> None: + """Regarding issue pytest-xdist#170 + + This test came originally from test_remote.py in xdist (ca03269). + """ + from _pytest._code.code import ReprEntry + + reprec = pytester.inline_runsource( + """ + def test_repr_entry(): + x = 0 + assert x + """, + "--showlocals", + ) + reports = reprec.getreports("pytest_runtest_logreport") + assert len(reports) == 3 + rep = reports[1] + assert isinstance(rep.longrepr, ExceptionRepr) + d = rep._to_json() + a = TestReport._from_json(d) + assert isinstance(a.longrepr, ExceptionRepr) + + rep_entries = rep.longrepr.reprtraceback.reprentries + a_entries = a.longrepr.reprtraceback.reprentries + for i in range(len(a_entries)): + rep_entry = rep_entries[i] + assert isinstance(rep_entry, ReprEntry) + assert rep_entry.reprfileloc is not None + assert rep_entry.reprfuncargs is not None + assert rep_entry.reprlocals is not None + + a_entry = a_entries[i] + assert isinstance(a_entry, ReprEntry) + assert a_entry.reprfileloc is not None + assert a_entry.reprfuncargs is not None + assert a_entry.reprlocals is not None + + assert rep_entry.lines == a_entry.lines + assert rep_entry.reprfileloc.lineno == a_entry.reprfileloc.lineno + assert rep_entry.reprfileloc.message == a_entry.reprfileloc.message + assert rep_entry.reprfileloc.path == a_entry.reprfileloc.path + assert rep_entry.reprfuncargs.args == a_entry.reprfuncargs.args + assert rep_entry.reprlocals.lines == a_entry.reprlocals.lines + assert rep_entry.style == a_entry.style + + def test_reprentries_serialization_196(self, pytester: Pytester) -> None: + """Regarding issue pytest-xdist#196 + + This test came originally from test_remote.py in xdist (ca03269). + """ + from _pytest._code.code import ReprEntryNative + + reprec = pytester.inline_runsource( + """ + def test_repr_entry_native(): + x = 0 + assert x + """, + "--tb=native", + ) + reports = reprec.getreports("pytest_runtest_logreport") + assert len(reports) == 3 + rep = reports[1] + assert isinstance(rep.longrepr, ExceptionRepr) + d = rep._to_json() + a = TestReport._from_json(d) + assert isinstance(a.longrepr, ExceptionRepr) + + rep_entries = rep.longrepr.reprtraceback.reprentries + a_entries = a.longrepr.reprtraceback.reprentries + for i in range(len(a_entries)): + assert isinstance(rep_entries[i], ReprEntryNative) + assert rep_entries[i].lines == a_entries[i].lines + + def test_itemreport_outcomes(self, pytester: Pytester) -> None: + # This test came originally from test_remote.py in xdist (ca03269). + reprec = pytester.inline_runsource( + """ + import pytest + def test_pass(): pass + def test_fail(): 0/0 + @pytest.mark.skipif("True") + def test_skip(): pass + def test_skip_imperative(): + pytest.skip("hello") + @pytest.mark.xfail("True") + def test_xfail(): 0/0 + def test_xfail_imperative(): + pytest.xfail("hello") + """ + ) + reports = reprec.getreports("pytest_runtest_logreport") + assert len(reports) == 17 # with setup/teardown "passed" reports + for rep in reports: + d = rep._to_json() + newrep = TestReport._from_json(d) + assert newrep.passed == rep.passed + assert newrep.failed == rep.failed + assert newrep.skipped == rep.skipped + if newrep.skipped and not hasattr(newrep, "wasxfail"): + assert isinstance(newrep.longrepr, tuple) + assert len(newrep.longrepr) == 3 + assert newrep.outcome == rep.outcome + assert newrep.when == rep.when + assert newrep.keywords == rep.keywords + if rep.failed: + assert newrep.longreprtext == rep.longreprtext + + def test_collectreport_passed(self, pytester: Pytester) -> None: + """This test came originally from test_remote.py in xdist (ca03269).""" + reprec = pytester.inline_runsource("def test_func(): pass") + reports = reprec.getreports("pytest_collectreport") + for rep in reports: + d = rep._to_json() + newrep = CollectReport._from_json(d) + assert newrep.passed == rep.passed + assert newrep.failed == rep.failed + assert newrep.skipped == rep.skipped + + def test_collectreport_fail(self, pytester: Pytester) -> None: + """This test came originally from test_remote.py in xdist (ca03269).""" + reprec = pytester.inline_runsource("qwe abc") + reports = reprec.getreports("pytest_collectreport") + assert reports + for rep in reports: + d = rep._to_json() + newrep = CollectReport._from_json(d) + assert newrep.passed == rep.passed + assert newrep.failed == rep.failed + assert newrep.skipped == rep.skipped + if rep.failed: + assert newrep.longrepr == str(rep.longrepr) + + def test_extended_report_deserialization(self, pytester: Pytester) -> None: + """This test came originally from test_remote.py in xdist (ca03269).""" + reprec = pytester.inline_runsource("qwe abc") + reports = reprec.getreports("pytest_collectreport") + assert reports + for rep in reports: + rep.extra = True # type: ignore[attr-defined] + d = rep._to_json() + newrep = CollectReport._from_json(d) + assert newrep.extra + assert newrep.passed == rep.passed + assert newrep.failed == rep.failed + assert newrep.skipped == rep.skipped + if rep.failed: + assert newrep.longrepr == str(rep.longrepr) + + def test_paths_support(self, pytester: Pytester) -> None: + """Report attributes which are path-like should become strings.""" + pytester.makepyfile( + """ + def test_a(): + assert False + """ + ) + + class MyPathLike: + def __init__(self, path: str) -> None: + self.path = path + + def __fspath__(self) -> str: + return self.path + + reprec = pytester.inline_run() + reports = reprec.getreports("pytest_runtest_logreport") + assert len(reports) == 3 + test_a_call = reports[1] + test_a_call.path1 = MyPathLike(str(pytester.path)) # type: ignore[attr-defined] + test_a_call.path2 = pytester.path # type: ignore[attr-defined] + data = test_a_call._to_json() + assert data["path1"] == str(pytester.path) + assert data["path2"] == str(pytester.path) + + def test_deserialization_failure(self, pytester: Pytester) -> None: + """Check handling of failure during deserialization of report types.""" + pytester.makepyfile( + """ + def test_a(): + assert False + """ + ) + reprec = pytester.inline_run() + reports = reprec.getreports("pytest_runtest_logreport") + assert len(reports) == 3 + test_a_call = reports[1] + data = test_a_call._to_json() + entry = data["longrepr"]["reprtraceback"]["reprentries"][0] + assert entry["type"] == "ReprEntry" + + entry["type"] = "Unknown" + with pytest.raises( + RuntimeError, match="INTERNALERROR: Unknown entry type returned: Unknown" + ): + TestReport._from_json(data) + + @pytest.mark.parametrize("report_class", [TestReport, CollectReport]) + def test_chained_exceptions( + self, pytester: Pytester, tw_mock, report_class + ) -> None: + """Check serialization/deserialization of report objects containing chained exceptions (#5786)""" + pytester.makepyfile( + """ + def foo(): + raise ValueError('value error') + def test_a(): + try: + foo() + except ValueError as e: + raise RuntimeError('runtime error') from e + if {error_during_import}: + test_a() + """.format( + error_during_import=report_class is CollectReport + ) + ) + + reprec = pytester.inline_run() + if report_class is TestReport: + reports: Union[ + Sequence[TestReport], Sequence[CollectReport] + ] = reprec.getreports("pytest_runtest_logreport") + # we have 3 reports: setup/call/teardown + assert len(reports) == 3 + # get the call report + report = reports[1] + else: + assert report_class is CollectReport + # two collection reports: session and test file + reports = reprec.getreports("pytest_collectreport") + assert len(reports) == 2 + report = reports[1] + + def check_longrepr(longrepr: ExceptionChainRepr) -> None: + """Check the attributes of the given longrepr object according to the test file. + + We can get away with testing both CollectReport and TestReport with this function because + the longrepr objects are very similar. + """ + assert isinstance(longrepr, ExceptionChainRepr) + assert longrepr.sections == [("title", "contents", "=")] + assert len(longrepr.chain) == 2 + entry1, entry2 = longrepr.chain + tb1, fileloc1, desc1 = entry1 + tb2, fileloc2, desc2 = entry2 + + assert "ValueError('value error')" in str(tb1) + assert "RuntimeError('runtime error')" in str(tb2) + + assert ( + desc1 + == "The above exception was the direct cause of the following exception:" + ) + assert desc2 is None + + assert report.failed + assert len(report.sections) == 0 + assert isinstance(report.longrepr, ExceptionChainRepr) + report.longrepr.addsection("title", "contents", "=") + check_longrepr(report.longrepr) + + data = report._to_json() + loaded_report = report_class._from_json(data) + + assert loaded_report.failed + check_longrepr(loaded_report.longrepr) + + # make sure we don't blow up on ``toterminal`` call; we don't test the actual output because it is very + # brittle and hard to maintain, but we can assume it is correct because ``toterminal`` is already tested + # elsewhere and we do check the contents of the longrepr object after loading it. + loaded_report.longrepr.toterminal(tw_mock) + + def test_chained_exceptions_no_reprcrash(self, pytester: Pytester, tw_mock) -> None: + """Regression test for tracebacks without a reprcrash (#5971) + + This happens notably on exceptions raised by multiprocess.pool: the exception transfer + from subprocess to main process creates an artificial exception, which ExceptionInfo + can't obtain the ReprFileLocation from. + """ + pytester.makepyfile( + """ + from concurrent.futures import ProcessPoolExecutor + + def func(): + raise ValueError('value error') + + def test_a(): + with ProcessPoolExecutor() as p: + p.submit(func).result() + """ + ) + + pytester.syspathinsert() + reprec = pytester.inline_run() + + reports = reprec.getreports("pytest_runtest_logreport") + + def check_longrepr(longrepr: object) -> None: + assert isinstance(longrepr, ExceptionChainRepr) + assert len(longrepr.chain) == 2 + entry1, entry2 = longrepr.chain + tb1, fileloc1, desc1 = entry1 + tb2, fileloc2, desc2 = entry2 + + assert "RemoteTraceback" in str(tb1) + assert "ValueError: value error" in str(tb2) + + assert fileloc1 is None + assert fileloc2 is not None + assert fileloc2.message == "ValueError: value error" + + # 3 reports: setup/call/teardown: get the call report + assert len(reports) == 3 + report = reports[1] + + assert report.failed + check_longrepr(report.longrepr) + + data = report._to_json() + loaded_report = TestReport._from_json(data) + + assert loaded_report.failed + check_longrepr(loaded_report.longrepr) + + # for same reasons as previous test, ensure we don't blow up here + assert loaded_report.longrepr is not None + assert isinstance(loaded_report.longrepr, ExceptionChainRepr) + loaded_report.longrepr.toterminal(tw_mock) + + def test_report_prevent_ConftestImportFailure_hiding_exception( + self, pytester: Pytester + ) -> None: + sub_dir = pytester.path.joinpath("ns") + sub_dir.mkdir() + sub_dir.joinpath("conftest.py").write_text("import unknown") + + result = pytester.runpytest_subprocess(".") + result.stdout.fnmatch_lines(["E *Error: No module named 'unknown'"]) + result.stdout.no_fnmatch_line("ERROR - *ConftestImportFailure*") + + +class TestHooks: + """Test that the hooks are working correctly for plugins""" + + def test_test_report(self, pytester: Pytester, pytestconfig: Config) -> None: + pytester.makepyfile( + """ + def test_a(): assert False + def test_b(): pass + """ + ) + reprec = pytester.inline_run() + reports = reprec.getreports("pytest_runtest_logreport") + assert len(reports) == 6 + for rep in reports: + data = pytestconfig.hook.pytest_report_to_serializable( + config=pytestconfig, report=rep + ) + assert data["$report_type"] == "TestReport" + new_rep = pytestconfig.hook.pytest_report_from_serializable( + config=pytestconfig, data=data + ) + assert new_rep.nodeid == rep.nodeid + assert new_rep.when == rep.when + assert new_rep.outcome == rep.outcome + + def test_collect_report(self, pytester: Pytester, pytestconfig: Config) -> None: + pytester.makepyfile( + """ + def test_a(): assert False + def test_b(): pass + """ + ) + reprec = pytester.inline_run() + reports = reprec.getreports("pytest_collectreport") + assert len(reports) == 2 + for rep in reports: + data = pytestconfig.hook.pytest_report_to_serializable( + config=pytestconfig, report=rep + ) + assert data["$report_type"] == "CollectReport" + new_rep = pytestconfig.hook.pytest_report_from_serializable( + config=pytestconfig, data=data + ) + assert new_rep.nodeid == rep.nodeid + assert new_rep.when == "collect" + assert new_rep.outcome == rep.outcome + + @pytest.mark.parametrize( + "hook_name", ["pytest_runtest_logreport", "pytest_collectreport"] + ) + def test_invalid_report_types( + self, pytester: Pytester, pytestconfig: Config, hook_name: str + ) -> None: + pytester.makepyfile( + """ + def test_a(): pass + """ + ) + reprec = pytester.inline_run() + reports = reprec.getreports(hook_name) + assert reports + rep = reports[0] + data = pytestconfig.hook.pytest_report_to_serializable( + config=pytestconfig, report=rep + ) + data["$report_type"] = "Unknown" + with pytest.raises(AssertionError): + _ = pytestconfig.hook.pytest_report_from_serializable( + config=pytestconfig, data=data + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_runner.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_runner.py new file mode 100644 index 0000000000..2e2c462d97 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_runner.py @@ -0,0 +1,1061 @@ +import inspect +import os +import sys +import types +from pathlib import Path +from typing import Dict +from typing import List +from typing import Tuple +from typing import Type + +import pytest +from _pytest import outcomes +from _pytest import reports +from _pytest import runner +from _pytest._code import ExceptionInfo +from _pytest._code.code import ExceptionChainRepr +from _pytest.config import ExitCode +from _pytest.monkeypatch import MonkeyPatch +from _pytest.outcomes import OutcomeException +from _pytest.pytester import Pytester + + +class TestSetupState: + def test_setup(self, pytester: Pytester) -> None: + item = pytester.getitem("def test_func(): pass") + ss = item.session._setupstate + values = [1] + ss.setup(item) + ss.addfinalizer(values.pop, item) + assert values + ss.teardown_exact(None) + assert not values + + def test_teardown_exact_stack_empty(self, pytester: Pytester) -> None: + item = pytester.getitem("def test_func(): pass") + ss = item.session._setupstate + ss.setup(item) + ss.teardown_exact(None) + ss.teardown_exact(None) + ss.teardown_exact(None) + + def test_setup_fails_and_failure_is_cached(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + def setup_module(mod): + raise ValueError(42) + def test_func(): pass + """ + ) + ss = item.session._setupstate + with pytest.raises(ValueError): + ss.setup(item) + with pytest.raises(ValueError): + ss.setup(item) + + def test_teardown_multiple_one_fails(self, pytester: Pytester) -> None: + r = [] + + def fin1(): + r.append("fin1") + + def fin2(): + raise Exception("oops") + + def fin3(): + r.append("fin3") + + item = pytester.getitem("def test_func(): pass") + ss = item.session._setupstate + ss.setup(item) + ss.addfinalizer(fin1, item) + ss.addfinalizer(fin2, item) + ss.addfinalizer(fin3, item) + with pytest.raises(Exception) as err: + ss.teardown_exact(None) + assert err.value.args == ("oops",) + assert r == ["fin3", "fin1"] + + def test_teardown_multiple_fail(self, pytester: Pytester) -> None: + # Ensure the first exception is the one which is re-raised. + # Ideally both would be reported however. + def fin1(): + raise Exception("oops1") + + def fin2(): + raise Exception("oops2") + + item = pytester.getitem("def test_func(): pass") + ss = item.session._setupstate + ss.setup(item) + ss.addfinalizer(fin1, item) + ss.addfinalizer(fin2, item) + with pytest.raises(Exception) as err: + ss.teardown_exact(None) + assert err.value.args == ("oops2",) + + def test_teardown_multiple_scopes_one_fails(self, pytester: Pytester) -> None: + module_teardown = [] + + def fin_func(): + raise Exception("oops1") + + def fin_module(): + module_teardown.append("fin_module") + + item = pytester.getitem("def test_func(): pass") + mod = item.listchain()[-2] + ss = item.session._setupstate + ss.setup(item) + ss.addfinalizer(fin_module, mod) + ss.addfinalizer(fin_func, item) + with pytest.raises(Exception, match="oops1"): + ss.teardown_exact(None) + assert module_teardown == ["fin_module"] + + +class BaseFunctionalTests: + def test_passfunction(self, pytester: Pytester) -> None: + reports = pytester.runitem( + """ + def test_func(): + pass + """ + ) + rep = reports[1] + assert rep.passed + assert not rep.failed + assert rep.outcome == "passed" + assert not rep.longrepr + + def test_failfunction(self, pytester: Pytester) -> None: + reports = pytester.runitem( + """ + def test_func(): + assert 0 + """ + ) + rep = reports[1] + assert not rep.passed + assert not rep.skipped + assert rep.failed + assert rep.when == "call" + assert rep.outcome == "failed" + # assert isinstance(rep.longrepr, ReprExceptionInfo) + + def test_skipfunction(self, pytester: Pytester) -> None: + reports = pytester.runitem( + """ + import pytest + def test_func(): + pytest.skip("hello") + """ + ) + rep = reports[1] + assert not rep.failed + assert not rep.passed + assert rep.skipped + assert rep.outcome == "skipped" + # assert rep.skipped.when == "call" + # assert rep.skipped.when == "call" + # assert rep.skipped == "%sreason == "hello" + # assert rep.skipped.location.lineno == 3 + # assert rep.skipped.location.path + # assert not rep.skipped.failurerepr + + def test_skip_in_setup_function(self, pytester: Pytester) -> None: + reports = pytester.runitem( + """ + import pytest + def setup_function(func): + pytest.skip("hello") + def test_func(): + pass + """ + ) + print(reports) + rep = reports[0] + assert not rep.failed + assert not rep.passed + assert rep.skipped + # assert rep.skipped.reason == "hello" + # assert rep.skipped.location.lineno == 3 + # assert rep.skipped.location.lineno == 3 + assert len(reports) == 2 + assert reports[1].passed # teardown + + def test_failure_in_setup_function(self, pytester: Pytester) -> None: + reports = pytester.runitem( + """ + import pytest + def setup_function(func): + raise ValueError(42) + def test_func(): + pass + """ + ) + rep = reports[0] + assert not rep.skipped + assert not rep.passed + assert rep.failed + assert rep.when == "setup" + assert len(reports) == 2 + + def test_failure_in_teardown_function(self, pytester: Pytester) -> None: + reports = pytester.runitem( + """ + import pytest + def teardown_function(func): + raise ValueError(42) + def test_func(): + pass + """ + ) + print(reports) + assert len(reports) == 3 + rep = reports[2] + assert not rep.skipped + assert not rep.passed + assert rep.failed + assert rep.when == "teardown" + # assert rep.longrepr.reprcrash.lineno == 3 + # assert rep.longrepr.reprtraceback.reprentries + + def test_custom_failure_repr(self, pytester: Pytester) -> None: + pytester.makepyfile( + conftest=""" + import pytest + class Function(pytest.Function): + def repr_failure(self, excinfo): + return "hello" + """ + ) + reports = pytester.runitem( + """ + import pytest + def test_func(): + assert 0 + """ + ) + rep = reports[1] + assert not rep.skipped + assert not rep.passed + assert rep.failed + # assert rep.outcome.when == "call" + # assert rep.failed.where.lineno == 3 + # assert rep.failed.where.path.basename == "test_func.py" + # assert rep.failed.failurerepr == "hello" + + def test_teardown_final_returncode(self, pytester: Pytester) -> None: + rec = pytester.inline_runsource( + """ + def test_func(): + pass + def teardown_function(func): + raise ValueError(42) + """ + ) + assert rec.ret == 1 + + def test_logstart_logfinish_hooks(self, pytester: Pytester) -> None: + rec = pytester.inline_runsource( + """ + import pytest + def test_func(): + pass + """ + ) + reps = rec.getcalls("pytest_runtest_logstart pytest_runtest_logfinish") + assert [x._name for x in reps] == [ + "pytest_runtest_logstart", + "pytest_runtest_logfinish", + ] + for rep in reps: + assert rep.nodeid == "test_logstart_logfinish_hooks.py::test_func" + assert rep.location == ("test_logstart_logfinish_hooks.py", 1, "test_func") + + def test_exact_teardown_issue90(self, pytester: Pytester) -> None: + rec = pytester.inline_runsource( + """ + import pytest + + class TestClass(object): + def test_method(self): + pass + def teardown_class(cls): + raise Exception() + + def test_func(): + import sys + # on python2 exc_info is kept till a function exits + # so we would end up calling test functions while + # sys.exc_info would return the indexerror + # from guessing the lastitem + excinfo = sys.exc_info() + import traceback + assert excinfo[0] is None, \ + traceback.format_exception(*excinfo) + def teardown_function(func): + raise ValueError(42) + """ + ) + reps = rec.getreports("pytest_runtest_logreport") + print(reps) + for i in range(2): + assert reps[i].nodeid.endswith("test_method") + assert reps[i].passed + assert reps[2].when == "teardown" + assert reps[2].failed + assert len(reps) == 6 + for i in range(3, 5): + assert reps[i].nodeid.endswith("test_func") + assert reps[i].passed + assert reps[5].when == "teardown" + assert reps[5].nodeid.endswith("test_func") + assert reps[5].failed + + def test_exact_teardown_issue1206(self, pytester: Pytester) -> None: + """Issue shadowing error with wrong number of arguments on teardown_method.""" + rec = pytester.inline_runsource( + """ + import pytest + + class TestClass(object): + def teardown_method(self, x, y, z): + pass + + def test_method(self): + assert True + """ + ) + reps = rec.getreports("pytest_runtest_logreport") + print(reps) + assert len(reps) == 3 + # + assert reps[0].nodeid.endswith("test_method") + assert reps[0].passed + assert reps[0].when == "setup" + # + assert reps[1].nodeid.endswith("test_method") + assert reps[1].passed + assert reps[1].when == "call" + # + assert reps[2].nodeid.endswith("test_method") + assert reps[2].failed + assert reps[2].when == "teardown" + longrepr = reps[2].longrepr + assert isinstance(longrepr, ExceptionChainRepr) + assert longrepr.reprcrash + assert longrepr.reprcrash.message in ( + "TypeError: teardown_method() missing 2 required positional arguments: 'y' and 'z'", + # Python >= 3.10 + "TypeError: TestClass.teardown_method() missing 2 required positional arguments: 'y' and 'z'", + ) + + def test_failure_in_setup_function_ignores_custom_repr( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + conftest=""" + import pytest + class Function(pytest.Function): + def repr_failure(self, excinfo): + assert 0 + """ + ) + reports = pytester.runitem( + """ + def setup_function(func): + raise ValueError(42) + def test_func(): + pass + """ + ) + assert len(reports) == 2 + rep = reports[0] + print(rep) + assert not rep.skipped + assert not rep.passed + assert rep.failed + # assert rep.outcome.when == "setup" + # assert rep.outcome.where.lineno == 3 + # assert rep.outcome.where.path.basename == "test_func.py" + # assert instanace(rep.failed.failurerepr, PythonFailureRepr) + + def test_systemexit_does_not_bail_out(self, pytester: Pytester) -> None: + try: + reports = pytester.runitem( + """ + def test_func(): + raise SystemExit(42) + """ + ) + except SystemExit: + assert False, "runner did not catch SystemExit" + rep = reports[1] + assert rep.failed + assert rep.when == "call" + + def test_exit_propagates(self, pytester: Pytester) -> None: + try: + pytester.runitem( + """ + import pytest + def test_func(): + raise pytest.exit.Exception() + """ + ) + except pytest.exit.Exception: + pass + else: + assert False, "did not raise" + + +class TestExecutionNonForked(BaseFunctionalTests): + def getrunner(self): + def f(item): + return runner.runtestprotocol(item, log=False) + + return f + + def test_keyboardinterrupt_propagates(self, pytester: Pytester) -> None: + try: + pytester.runitem( + """ + def test_func(): + raise KeyboardInterrupt("fake") + """ + ) + except KeyboardInterrupt: + pass + else: + assert False, "did not raise" + + +class TestSessionReports: + def test_collect_result(self, pytester: Pytester) -> None: + col = pytester.getmodulecol( + """ + def test_func1(): + pass + class TestClass(object): + pass + """ + ) + rep = runner.collect_one_node(col) + assert not rep.failed + assert not rep.skipped + assert rep.passed + locinfo = rep.location + assert locinfo[0] == col.path.name + assert not locinfo[1] + assert locinfo[2] == col.path.name + res = rep.result + assert len(res) == 2 + assert res[0].name == "test_func1" + assert res[1].name == "TestClass" + + +reporttypes: List[Type[reports.BaseReport]] = [ + reports.BaseReport, + reports.TestReport, + reports.CollectReport, +] + + +@pytest.mark.parametrize( + "reporttype", reporttypes, ids=[x.__name__ for x in reporttypes] +) +def test_report_extra_parameters(reporttype: Type[reports.BaseReport]) -> None: + args = list(inspect.signature(reporttype.__init__).parameters.keys())[1:] + basekw: Dict[str, List[object]] = dict.fromkeys(args, []) + report = reporttype(newthing=1, **basekw) + assert report.newthing == 1 + + +def test_callinfo() -> None: + ci = runner.CallInfo.from_call(lambda: 0, "collect") + assert ci.when == "collect" + assert ci.result == 0 + assert "result" in repr(ci) + assert repr(ci) == "" + assert str(ci) == "" + + ci2 = runner.CallInfo.from_call(lambda: 0 / 0, "collect") + assert ci2.when == "collect" + assert not hasattr(ci2, "result") + assert repr(ci2) == f"" + assert str(ci2) == repr(ci2) + assert ci2.excinfo + + # Newlines are escaped. + def raise_assertion(): + assert 0, "assert_msg" + + ci3 = runner.CallInfo.from_call(raise_assertion, "call") + assert repr(ci3) == f"" + assert "\n" not in repr(ci3) + + +# design question: do we want general hooks in python files? +# then something like the following functional tests makes sense + + +@pytest.mark.xfail +def test_runtest_in_module_ordering(pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + import pytest + def pytest_runtest_setup(item): # runs after class-level! + item.function.mylist.append("module") + class TestClass(object): + def pytest_runtest_setup(self, item): + assert not hasattr(item.function, 'mylist') + item.function.mylist = ['class'] + @pytest.fixture + def mylist(self, request): + return request.function.mylist + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_call(self, item): + try: + (yield).get_result() + except ValueError: + pass + def test_hello1(self, mylist): + assert mylist == ['class', 'module'], mylist + raise ValueError() + def test_hello2(self, mylist): + assert mylist == ['class', 'module'], mylist + def pytest_runtest_teardown(item): + del item.function.mylist + """ + ) + result = pytester.runpytest(p1) + result.stdout.fnmatch_lines(["*2 passed*"]) + + +def test_outcomeexception_exceptionattributes() -> None: + outcome = outcomes.OutcomeException("test") + assert outcome.args[0] == outcome.msg + + +def test_outcomeexception_passes_except_Exception() -> None: + with pytest.raises(outcomes.OutcomeException): + try: + raise outcomes.OutcomeException("test") + except Exception as e: + raise NotImplementedError from e + + +def test_pytest_exit() -> None: + with pytest.raises(pytest.exit.Exception) as excinfo: + pytest.exit("hello") + assert excinfo.errisinstance(pytest.exit.Exception) + + +def test_pytest_fail() -> None: + with pytest.raises(pytest.fail.Exception) as excinfo: + pytest.fail("hello") + s = excinfo.exconly(tryshort=True) + assert s.startswith("Failed") + + +def test_pytest_exit_msg(pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + + def pytest_configure(config): + pytest.exit('oh noes') + """ + ) + result = pytester.runpytest() + result.stderr.fnmatch_lines(["Exit: oh noes"]) + + +def _strip_resource_warnings(lines): + # Assert no output on stderr, except for unreliable ResourceWarnings. + # (https://github.com/pytest-dev/pytest/issues/5088) + return [ + x + for x in lines + if not x.startswith(("Exception ignored in:", "ResourceWarning")) + ] + + +def test_pytest_exit_returncode(pytester: Pytester) -> None: + pytester.makepyfile( + """\ + import pytest + def test_foo(): + pytest.exit("some exit msg", 99) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*! *Exit: some exit msg !*"]) + + assert _strip_resource_warnings(result.stderr.lines) == [] + assert result.ret == 99 + + # It prints to stderr also in case of exit during pytest_sessionstart. + pytester.makeconftest( + """\ + import pytest + + def pytest_sessionstart(): + pytest.exit("during_sessionstart", 98) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*! *Exit: during_sessionstart !*"]) + assert _strip_resource_warnings(result.stderr.lines) == [ + "Exit: during_sessionstart" + ] + assert result.ret == 98 + + +def test_pytest_fail_notrace_runtest(pytester: Pytester) -> None: + """Test pytest.fail(..., pytrace=False) does not show tracebacks during test run.""" + pytester.makepyfile( + """ + import pytest + def test_hello(): + pytest.fail("hello", pytrace=False) + def teardown_function(function): + pytest.fail("world", pytrace=False) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["world", "hello"]) + result.stdout.no_fnmatch_line("*def teardown_function*") + + +def test_pytest_fail_notrace_collection(pytester: Pytester) -> None: + """Test pytest.fail(..., pytrace=False) does not show tracebacks during collection.""" + pytester.makepyfile( + """ + import pytest + def some_internal_function(): + pytest.fail("hello", pytrace=False) + some_internal_function() + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["hello"]) + result.stdout.no_fnmatch_line("*def some_internal_function()*") + + +def test_pytest_fail_notrace_non_ascii(pytester: Pytester) -> None: + """Fix pytest.fail with pytrace=False with non-ascii characters (#1178). + + This tests with native and unicode strings containing non-ascii chars. + """ + pytester.makepyfile( + """\ + import pytest + + def test_hello(): + pytest.fail('oh oh: ☺', pytrace=False) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*test_hello*", "oh oh: ☺"]) + result.stdout.no_fnmatch_line("*def test_hello*") + + +def test_pytest_no_tests_collected_exit_status(pytester: Pytester) -> None: + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*collected 0 items*"]) + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + pytester.makepyfile( + test_foo=""" + def test_foo(): + assert 1 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*collected 1 item*"]) + result.stdout.fnmatch_lines(["*1 passed*"]) + assert result.ret == ExitCode.OK + + result = pytester.runpytest("-k nonmatch") + result.stdout.fnmatch_lines(["*collected 1 item*"]) + result.stdout.fnmatch_lines(["*1 deselected*"]) + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +def test_exception_printing_skip() -> None: + assert pytest.skip.Exception == pytest.skip.Exception + try: + pytest.skip("hello") + except pytest.skip.Exception: + excinfo = ExceptionInfo.from_current() + s = excinfo.exconly(tryshort=True) + assert s.startswith("Skipped") + + +def test_importorskip(monkeypatch) -> None: + importorskip = pytest.importorskip + + def f(): + importorskip("asdlkj") + + try: + sysmod = importorskip("sys") + assert sysmod is sys + # path = pytest.importorskip("os.path") + # assert path == os.path + excinfo = pytest.raises(pytest.skip.Exception, f) + assert excinfo is not None + excrepr = excinfo.getrepr() + assert excrepr is not None + assert excrepr.reprcrash is not None + path = Path(excrepr.reprcrash.path) + # check that importorskip reports the actual call + # in this test the test_runner.py file + assert path.stem == "test_runner" + pytest.raises(SyntaxError, pytest.importorskip, "x y z") + pytest.raises(SyntaxError, pytest.importorskip, "x=y") + mod = types.ModuleType("hello123") + mod.__version__ = "1.3" # type: ignore + monkeypatch.setitem(sys.modules, "hello123", mod) + with pytest.raises(pytest.skip.Exception): + pytest.importorskip("hello123", minversion="1.3.1") + mod2 = pytest.importorskip("hello123", minversion="1.3") + assert mod2 == mod + except pytest.skip.Exception: # pragma: no cover + assert False, f"spurious skip: {ExceptionInfo.from_current()}" + + +def test_importorskip_imports_last_module_part() -> None: + ospath = pytest.importorskip("os.path") + assert os.path == ospath + + +def test_importorskip_dev_module(monkeypatch) -> None: + try: + mod = types.ModuleType("mockmodule") + mod.__version__ = "0.13.0.dev-43290" # type: ignore + monkeypatch.setitem(sys.modules, "mockmodule", mod) + mod2 = pytest.importorskip("mockmodule", minversion="0.12.0") + assert mod2 == mod + with pytest.raises(pytest.skip.Exception): + pytest.importorskip("mockmodule1", minversion="0.14.0") + except pytest.skip.Exception: # pragma: no cover + assert False, f"spurious skip: {ExceptionInfo.from_current()}" + + +def test_importorskip_module_level(pytester: Pytester) -> None: + """`importorskip` must be able to skip entire modules when used at module level.""" + pytester.makepyfile( + """ + import pytest + foobarbaz = pytest.importorskip("foobarbaz") + + def test_foo(): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*collected 0 items / 1 skipped*"]) + + +def test_importorskip_custom_reason(pytester: Pytester) -> None: + """Make sure custom reasons are used.""" + pytester.makepyfile( + """ + import pytest + foobarbaz = pytest.importorskip("foobarbaz2", reason="just because") + + def test_foo(): + pass + """ + ) + result = pytester.runpytest("-ra") + result.stdout.fnmatch_lines(["*just because*"]) + result.stdout.fnmatch_lines(["*collected 0 items / 1 skipped*"]) + + +def test_pytest_cmdline_main(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + def test_hello(): + assert 1 + if __name__ == '__main__': + pytest.cmdline.main([__file__]) + """ + ) + import subprocess + + popen = subprocess.Popen([sys.executable, str(p)], stdout=subprocess.PIPE) + popen.communicate() + ret = popen.wait() + assert ret == 0 + + +def test_unicode_in_longrepr(pytester: Pytester) -> None: + pytester.makeconftest( + """\ + import pytest + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_makereport(): + outcome = yield + rep = outcome.get_result() + if rep.when == "call": + rep.longrepr = 'ä' + """ + ) + pytester.makepyfile( + """ + def test_out(): + assert 0 + """ + ) + result = pytester.runpytest() + assert result.ret == 1 + assert "UnicodeEncodeError" not in result.stderr.str() + + +def test_failure_in_setup(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def setup_module(): + 0/0 + def test_func(): + pass + """ + ) + result = pytester.runpytest("--tb=line") + result.stdout.no_fnmatch_line("*def setup_module*") + + +def test_makereport_getsource(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_foo(): + if False: pass + else: assert False + """ + ) + result = pytester.runpytest() + result.stdout.no_fnmatch_line("*INTERNALERROR*") + result.stdout.fnmatch_lines(["*else: assert False*"]) + + +def test_makereport_getsource_dynamic_code( + pytester: Pytester, monkeypatch: MonkeyPatch +) -> None: + """Test that exception in dynamically generated code doesn't break getting the source line.""" + import inspect + + original_findsource = inspect.findsource + + def findsource(obj): + # Can be triggered by dynamically created functions + if obj.__name__ == "foo": + raise IndexError() + return original_findsource(obj) + + monkeypatch.setattr(inspect, "findsource", findsource) + + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def foo(missing): + pass + + def test_fix(foo): + assert False + """ + ) + result = pytester.runpytest("-vv") + result.stdout.no_fnmatch_line("*INTERNALERROR*") + result.stdout.fnmatch_lines(["*test_fix*", "*fixture*'missing'*not found*"]) + + +def test_store_except_info_on_error() -> None: + """Test that upon test failure, the exception info is stored on + sys.last_traceback and friends.""" + # Simulate item that might raise a specific exception, depending on `raise_error` class var + class ItemMightRaise: + nodeid = "item_that_raises" + raise_error = True + + def runtest(self): + if self.raise_error: + raise IndexError("TEST") + + try: + runner.pytest_runtest_call(ItemMightRaise()) # type: ignore[arg-type] + except IndexError: + pass + # Check that exception info is stored on sys + assert sys.last_type is IndexError + assert isinstance(sys.last_value, IndexError) + assert sys.last_value.args[0] == "TEST" + assert sys.last_traceback + + # The next run should clear the exception info stored by the previous run + ItemMightRaise.raise_error = False + runner.pytest_runtest_call(ItemMightRaise()) # type: ignore[arg-type] + assert not hasattr(sys, "last_type") + assert not hasattr(sys, "last_value") + assert not hasattr(sys, "last_traceback") + + +def test_current_test_env_var(pytester: Pytester, monkeypatch: MonkeyPatch) -> None: + pytest_current_test_vars: List[Tuple[str, str]] = [] + monkeypatch.setattr( + sys, "pytest_current_test_vars", pytest_current_test_vars, raising=False + ) + pytester.makepyfile( + """ + import pytest + import sys + import os + + @pytest.fixture + def fix(): + sys.pytest_current_test_vars.append(('setup', os.environ['PYTEST_CURRENT_TEST'])) + yield + sys.pytest_current_test_vars.append(('teardown', os.environ['PYTEST_CURRENT_TEST'])) + + def test(fix): + sys.pytest_current_test_vars.append(('call', os.environ['PYTEST_CURRENT_TEST'])) + """ + ) + result = pytester.runpytest_inprocess() + assert result.ret == 0 + test_id = "test_current_test_env_var.py::test" + assert pytest_current_test_vars == [ + ("setup", test_id + " (setup)"), + ("call", test_id + " (call)"), + ("teardown", test_id + " (teardown)"), + ] + assert "PYTEST_CURRENT_TEST" not in os.environ + + +class TestReportContents: + """Test user-level API of ``TestReport`` objects.""" + + def getrunner(self): + return lambda item: runner.runtestprotocol(item, log=False) + + def test_longreprtext_pass(self, pytester: Pytester) -> None: + reports = pytester.runitem( + """ + def test_func(): + pass + """ + ) + rep = reports[1] + assert rep.longreprtext == "" + + def test_longreprtext_skip(self, pytester: Pytester) -> None: + """TestReport.longreprtext can handle non-str ``longrepr`` attributes (#7559)""" + reports = pytester.runitem( + """ + import pytest + def test_func(): + pytest.skip() + """ + ) + _, call_rep, _ = reports + assert isinstance(call_rep.longrepr, tuple) + assert "Skipped" in call_rep.longreprtext + + def test_longreprtext_collect_skip(self, pytester: Pytester) -> None: + """CollectReport.longreprtext can handle non-str ``longrepr`` attributes (#7559)""" + pytester.makepyfile( + """ + import pytest + pytest.skip(allow_module_level=True) + """ + ) + rec = pytester.inline_run() + calls = rec.getcalls("pytest_collectreport") + _, call = calls + assert isinstance(call.report.longrepr, tuple) + assert "Skipped" in call.report.longreprtext + + def test_longreprtext_failure(self, pytester: Pytester) -> None: + reports = pytester.runitem( + """ + def test_func(): + x = 1 + assert x == 4 + """ + ) + rep = reports[1] + assert "assert 1 == 4" in rep.longreprtext + + def test_captured_text(self, pytester: Pytester) -> None: + reports = pytester.runitem( + """ + import pytest + import sys + + @pytest.fixture + def fix(): + sys.stdout.write('setup: stdout\\n') + sys.stderr.write('setup: stderr\\n') + yield + sys.stdout.write('teardown: stdout\\n') + sys.stderr.write('teardown: stderr\\n') + assert 0 + + def test_func(fix): + sys.stdout.write('call: stdout\\n') + sys.stderr.write('call: stderr\\n') + assert 0 + """ + ) + setup, call, teardown = reports + assert setup.capstdout == "setup: stdout\n" + assert call.capstdout == "setup: stdout\ncall: stdout\n" + assert teardown.capstdout == "setup: stdout\ncall: stdout\nteardown: stdout\n" + + assert setup.capstderr == "setup: stderr\n" + assert call.capstderr == "setup: stderr\ncall: stderr\n" + assert teardown.capstderr == "setup: stderr\ncall: stderr\nteardown: stderr\n" + + def test_no_captured_text(self, pytester: Pytester) -> None: + reports = pytester.runitem( + """ + def test_func(): + pass + """ + ) + rep = reports[1] + assert rep.capstdout == "" + assert rep.capstderr == "" + + def test_longrepr_type(self, pytester: Pytester) -> None: + reports = pytester.runitem( + """ + import pytest + def test_func(): + pytest.fail(pytrace=False) + """ + ) + rep = reports[1] + assert isinstance(rep.longrepr, ExceptionChainRepr) + + +def test_outcome_exception_bad_msg() -> None: + """Check that OutcomeExceptions validate their input to prevent confusing errors (#5578)""" + + def func() -> None: + raise NotImplementedError() + + expected = ( + "OutcomeException expected string as 'msg' parameter, got 'function' instead.\n" + "Perhaps you meant to use a mark?" + ) + with pytest.raises(TypeError) as excinfo: + OutcomeException(func) # type: ignore + assert str(excinfo.value) == expected diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_runner_xunit.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_runner_xunit.py new file mode 100644 index 0000000000..e077ac41e2 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_runner_xunit.py @@ -0,0 +1,297 @@ +"""Test correct setup/teardowns at module, class, and instance level.""" +from typing import List + +import pytest +from _pytest.pytester import Pytester + + +def test_module_and_function_setup(pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + modlevel = [] + def setup_module(module): + assert not modlevel + module.modlevel.append(42) + + def teardown_module(module): + modlevel.pop() + + def setup_function(function): + function.answer = 17 + + def teardown_function(function): + del function.answer + + def test_modlevel(): + assert modlevel[0] == 42 + assert test_modlevel.answer == 17 + + class TestFromClass(object): + def test_module(self): + assert modlevel[0] == 42 + assert not hasattr(test_modlevel, 'answer') + """ + ) + rep = reprec.matchreport("test_modlevel") + assert rep.passed + rep = reprec.matchreport("test_module") + assert rep.passed + + +def test_module_setup_failure_no_teardown(pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + values = [] + def setup_module(module): + values.append(1) + 0/0 + + def test_nothing(): + pass + + def teardown_module(module): + values.append(2) + """ + ) + reprec.assertoutcome(failed=1) + calls = reprec.getcalls("pytest_runtest_setup") + assert calls[0].item.module.values == [1] + + +def test_setup_function_failure_no_teardown(pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + modlevel = [] + def setup_function(function): + modlevel.append(1) + 0/0 + + def teardown_function(module): + modlevel.append(2) + + def test_func(): + pass + """ + ) + calls = reprec.getcalls("pytest_runtest_setup") + assert calls[0].item.module.modlevel == [1] + + +def test_class_setup(pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + class TestSimpleClassSetup(object): + clslevel = [] + def setup_class(cls): + cls.clslevel.append(23) + + def teardown_class(cls): + cls.clslevel.pop() + + def test_classlevel(self): + assert self.clslevel[0] == 23 + + class TestInheritedClassSetupStillWorks(TestSimpleClassSetup): + def test_classlevel_anothertime(self): + assert self.clslevel == [23] + + def test_cleanup(): + assert not TestSimpleClassSetup.clslevel + assert not TestInheritedClassSetupStillWorks.clslevel + """ + ) + reprec.assertoutcome(passed=1 + 2 + 1) + + +def test_class_setup_failure_no_teardown(pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + class TestSimpleClassSetup(object): + clslevel = [] + def setup_class(cls): + 0/0 + + def teardown_class(cls): + cls.clslevel.append(1) + + def test_classlevel(self): + pass + + def test_cleanup(): + assert not TestSimpleClassSetup.clslevel + """ + ) + reprec.assertoutcome(failed=1, passed=1) + + +def test_method_setup(pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + class TestSetupMethod(object): + def setup_method(self, meth): + self.methsetup = meth + def teardown_method(self, meth): + del self.methsetup + + def test_some(self): + assert self.methsetup == self.test_some + + def test_other(self): + assert self.methsetup == self.test_other + """ + ) + reprec.assertoutcome(passed=2) + + +def test_method_setup_failure_no_teardown(pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + class TestMethodSetup(object): + clslevel = [] + def setup_method(self, method): + self.clslevel.append(1) + 0/0 + + def teardown_method(self, method): + self.clslevel.append(2) + + def test_method(self): + pass + + def test_cleanup(): + assert TestMethodSetup.clslevel == [1] + """ + ) + reprec.assertoutcome(failed=1, passed=1) + + +def test_method_setup_uses_fresh_instances(pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + class TestSelfState1(object): + memory = [] + def test_hello(self): + self.memory.append(self) + + def test_afterhello(self): + assert self != self.memory[0] + """ + ) + reprec.assertoutcome(passed=2, failed=0) + + +def test_setup_that_skips_calledagain(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + def setup_module(mod): + pytest.skip("x") + def test_function1(): + pass + def test_function2(): + pass + """ + ) + reprec = pytester.inline_run(p) + reprec.assertoutcome(skipped=2) + + +def test_setup_fails_again_on_all_tests(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + def setup_module(mod): + raise ValueError(42) + def test_function1(): + pass + def test_function2(): + pass + """ + ) + reprec = pytester.inline_run(p) + reprec.assertoutcome(failed=2) + + +def test_setup_funcarg_setup_when_outer_scope_fails(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + def setup_module(mod): + raise ValueError(42) + @pytest.fixture + def hello(request): + raise ValueError("xyz43") + def test_function1(hello): + pass + def test_function2(hello): + pass + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + [ + "*function1*", + "*ValueError*42*", + "*function2*", + "*ValueError*42*", + "*2 errors*", + ] + ) + result.stdout.no_fnmatch_line("*xyz43*") + + +@pytest.mark.parametrize("arg", ["", "arg"]) +def test_setup_teardown_function_level_with_optional_argument( + pytester: Pytester, + monkeypatch, + arg: str, +) -> None: + """Parameter to setup/teardown xunit-style functions parameter is now optional (#1728).""" + import sys + + trace_setups_teardowns: List[str] = [] + monkeypatch.setattr( + sys, "trace_setups_teardowns", trace_setups_teardowns, raising=False + ) + p = pytester.makepyfile( + """ + import pytest + import sys + + trace = sys.trace_setups_teardowns.append + + def setup_module({arg}): trace('setup_module') + def teardown_module({arg}): trace('teardown_module') + + def setup_function({arg}): trace('setup_function') + def teardown_function({arg}): trace('teardown_function') + + def test_function_1(): pass + def test_function_2(): pass + + class Test(object): + def setup_method(self, {arg}): trace('setup_method') + def teardown_method(self, {arg}): trace('teardown_method') + + def test_method_1(self): pass + def test_method_2(self): pass + """.format( + arg=arg + ) + ) + result = pytester.inline_run(p) + result.assertoutcome(passed=4) + + expected = [ + "setup_module", + "setup_function", + "teardown_function", + "setup_function", + "teardown_function", + "setup_method", + "teardown_method", + "setup_method", + "teardown_method", + "teardown_module", + ] + assert trace_setups_teardowns == expected diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_scope.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_scope.py new file mode 100644 index 0000000000..09ee1343a8 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_scope.py @@ -0,0 +1,39 @@ +import re + +import pytest +from _pytest.scope import Scope + + +def test_ordering() -> None: + assert Scope.Session > Scope.Package + assert Scope.Package > Scope.Module + assert Scope.Module > Scope.Class + assert Scope.Class > Scope.Function + + +def test_next_lower() -> None: + assert Scope.Session.next_lower() is Scope.Package + assert Scope.Package.next_lower() is Scope.Module + assert Scope.Module.next_lower() is Scope.Class + assert Scope.Class.next_lower() is Scope.Function + + with pytest.raises(ValueError, match="Function is the lower-most scope"): + Scope.Function.next_lower() + + +def test_next_higher() -> None: + assert Scope.Function.next_higher() is Scope.Class + assert Scope.Class.next_higher() is Scope.Module + assert Scope.Module.next_higher() is Scope.Package + assert Scope.Package.next_higher() is Scope.Session + + with pytest.raises(ValueError, match="Session is the upper-most scope"): + Scope.Session.next_higher() + + +def test_from_user() -> None: + assert Scope.from_user("module", "for parametrize", "some::id") is Scope.Module + + expected_msg = "for parametrize from some::id got an unexpected scope value 'foo'" + with pytest.raises(pytest.fail.Exception, match=re.escape(expected_msg)): + Scope.from_user("foo", "for parametrize", "some::id") # type:ignore[arg-type] diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_session.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_session.py new file mode 100644 index 0000000000..3ca6d39038 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_session.py @@ -0,0 +1,369 @@ +import pytest +from _pytest.config import ExitCode +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester + + +class SessionTests: + def test_basic_testitem_events(self, pytester: Pytester) -> None: + tfile = pytester.makepyfile( + """ + def test_one(): + pass + def test_one_one(): + assert 0 + def test_other(): + raise ValueError(23) + class TestClass(object): + def test_two(self, someargs): + pass + """ + ) + reprec = pytester.inline_run(tfile) + passed, skipped, failed = reprec.listoutcomes() + assert len(skipped) == 0 + assert len(passed) == 1 + assert len(failed) == 3 + + def end(x): + return x.nodeid.split("::")[-1] + + assert end(failed[0]) == "test_one_one" + assert end(failed[1]) == "test_other" + itemstarted = reprec.getcalls("pytest_itemcollected") + assert len(itemstarted) == 4 + # XXX check for failing funcarg setup + # colreports = reprec.getcalls("pytest_collectreport") + # assert len(colreports) == 4 + # assert colreports[1].report.failed + + def test_nested_import_error(self, pytester: Pytester) -> None: + tfile = pytester.makepyfile( + """ + import import_fails + def test_this(): + assert import_fails.a == 1 + """, + import_fails=""" + import does_not_work + a = 1 + """, + ) + reprec = pytester.inline_run(tfile) + values = reprec.getfailedcollections() + assert len(values) == 1 + out = str(values[0].longrepr) + assert out.find("does_not_work") != -1 + + def test_raises_output(self, pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + import pytest + def test_raises_doesnt(): + pytest.raises(ValueError, int, "3") + """ + ) + passed, skipped, failed = reprec.listoutcomes() + assert len(failed) == 1 + out = failed[0].longrepr.reprcrash.message # type: ignore[union-attr] + assert "DID NOT RAISE" in out + + def test_syntax_error_module(self, pytester: Pytester) -> None: + reprec = pytester.inline_runsource("this is really not python") + values = reprec.getfailedcollections() + assert len(values) == 1 + out = str(values[0].longrepr) + assert out.find("not python") != -1 + + def test_exit_first_problem(self, pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + def test_one(): assert 0 + def test_two(): assert 0 + """, + "--exitfirst", + ) + passed, skipped, failed = reprec.countoutcomes() + assert failed == 1 + assert passed == skipped == 0 + + def test_maxfail(self, pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + def test_one(): assert 0 + def test_two(): assert 0 + def test_three(): assert 0 + """, + "--maxfail=2", + ) + passed, skipped, failed = reprec.countoutcomes() + assert failed == 2 + assert passed == skipped == 0 + + def test_broken_repr(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + class reprexc(BaseException): + def __str__(self): + return "Ha Ha fooled you, I'm a broken repr()." + + class BrokenRepr1(object): + foo=0 + def __repr__(self): + raise reprexc + + class TestBrokenClass(object): + def test_explicit_bad_repr(self): + t = BrokenRepr1() + with pytest.raises(BaseException, match="broken repr"): + repr(t) + + def test_implicit_bad_repr1(self): + t = BrokenRepr1() + assert t.foo == 1 + + """ + ) + reprec = pytester.inline_run(p) + passed, skipped, failed = reprec.listoutcomes() + assert (len(passed), len(skipped), len(failed)) == (1, 0, 1) + out = failed[0].longrepr.reprcrash.message # type: ignore[union-attr] + assert out.find("<[reprexc() raised in repr()] BrokenRepr1") != -1 + + def test_broken_repr_with_showlocals_verbose(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + class ObjWithErrorInRepr: + def __repr__(self): + raise NotImplementedError + + def test_repr_error(): + x = ObjWithErrorInRepr() + assert x == "value" + """ + ) + reprec = pytester.inline_run("--showlocals", "-vv", p) + passed, skipped, failed = reprec.listoutcomes() + assert (len(passed), len(skipped), len(failed)) == (0, 0, 1) + entries = failed[0].longrepr.reprtraceback.reprentries # type: ignore[union-attr] + assert len(entries) == 1 + repr_locals = entries[0].reprlocals + assert repr_locals.lines + assert len(repr_locals.lines) == 1 + assert repr_locals.lines[0].startswith( + "x = <[NotImplementedError() raised in repr()] ObjWithErrorInRepr" + ) + + def test_skip_file_by_conftest(self, pytester: Pytester) -> None: + pytester.makepyfile( + conftest=""" + import pytest + def pytest_collect_file(): + pytest.skip("intentional") + """, + test_file=""" + def test_one(): pass + """, + ) + try: + reprec = pytester.inline_run(pytester.path) + except pytest.skip.Exception: # pragma: no cover + pytest.fail("wrong skipped caught") + reports = reprec.getreports("pytest_collectreport") + assert len(reports) == 1 + assert reports[0].skipped + + +class TestNewSession(SessionTests): + def test_order_of_execution(self, pytester: Pytester) -> None: + reprec = pytester.inline_runsource( + """ + values = [] + def test_1(): + values.append(1) + def test_2(): + values.append(2) + def test_3(): + assert values == [1,2] + class Testmygroup(object): + reslist = values + def test_1(self): + self.reslist.append(1) + def test_2(self): + self.reslist.append(2) + def test_3(self): + self.reslist.append(3) + def test_4(self): + assert self.reslist == [1,2,1,2,3] + """ + ) + passed, skipped, failed = reprec.countoutcomes() + assert failed == skipped == 0 + assert passed == 7 + + def test_collect_only_with_various_situations(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + test_one=""" + def test_one(): + raise ValueError() + + class TestX(object): + def test_method_one(self): + pass + + class TestY(TestX): + pass + """, + test_three="xxxdsadsadsadsa", + __init__="", + ) + reprec = pytester.inline_run("--collect-only", p.parent) + + itemstarted = reprec.getcalls("pytest_itemcollected") + assert len(itemstarted) == 3 + assert not reprec.getreports("pytest_runtest_logreport") + started = reprec.getcalls("pytest_collectstart") + finished = reprec.getreports("pytest_collectreport") + assert len(started) == len(finished) + assert len(started) == 6 + colfail = [x for x in finished if x.failed] + assert len(colfail) == 1 + + def test_minus_x_import_error(self, pytester: Pytester) -> None: + pytester.makepyfile(__init__="") + pytester.makepyfile(test_one="xxxx", test_two="yyyy") + reprec = pytester.inline_run("-x", pytester.path) + finished = reprec.getreports("pytest_collectreport") + colfail = [x for x in finished if x.failed] + assert len(colfail) == 1 + + def test_minus_x_overridden_by_maxfail(self, pytester: Pytester) -> None: + pytester.makepyfile(__init__="") + pytester.makepyfile(test_one="xxxx", test_two="yyyy", test_third="zzz") + reprec = pytester.inline_run("-x", "--maxfail=2", pytester.path) + finished = reprec.getreports("pytest_collectreport") + colfail = [x for x in finished if x.failed] + assert len(colfail) == 2 + + +def test_plugin_specify(pytester: Pytester) -> None: + with pytest.raises(ImportError): + pytester.parseconfig("-p", "nqweotexistent") + # pytest.raises(ImportError, + # "config.do_configure(config)" + # ) + + +def test_plugin_already_exists(pytester: Pytester) -> None: + config = pytester.parseconfig("-p", "terminal") + assert config.option.plugins == ["terminal"] + config._do_configure() + config._ensure_unconfigure() + + +def test_exclude(pytester: Pytester) -> None: + hellodir = pytester.mkdir("hello") + hellodir.joinpath("test_hello.py").write_text("x y syntaxerror") + hello2dir = pytester.mkdir("hello2") + hello2dir.joinpath("test_hello2.py").write_text("x y syntaxerror") + pytester.makepyfile(test_ok="def test_pass(): pass") + result = pytester.runpytest("--ignore=hello", "--ignore=hello2") + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_exclude_glob(pytester: Pytester) -> None: + hellodir = pytester.mkdir("hello") + hellodir.joinpath("test_hello.py").write_text("x y syntaxerror") + hello2dir = pytester.mkdir("hello2") + hello2dir.joinpath("test_hello2.py").write_text("x y syntaxerror") + hello3dir = pytester.mkdir("hallo3") + hello3dir.joinpath("test_hello3.py").write_text("x y syntaxerror") + subdir = pytester.mkdir("sub") + subdir.joinpath("test_hello4.py").write_text("x y syntaxerror") + pytester.makepyfile(test_ok="def test_pass(): pass") + result = pytester.runpytest("--ignore-glob=*h[ea]llo*") + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_deselect(pytester: Pytester) -> None: + pytester.makepyfile( + test_a=""" + import pytest + + def test_a1(): pass + + @pytest.mark.parametrize('b', range(3)) + def test_a2(b): pass + + class TestClass: + def test_c1(self): pass + + def test_c2(self): pass + """ + ) + result = pytester.runpytest( + "-v", + "--deselect=test_a.py::test_a2[1]", + "--deselect=test_a.py::test_a2[2]", + "--deselect=test_a.py::TestClass::test_c1", + ) + assert result.ret == 0 + result.stdout.fnmatch_lines(["*3 passed, 3 deselected*"]) + for line in result.stdout.lines: + assert not line.startswith(("test_a.py::test_a2[1]", "test_a.py::test_a2[2]")) + + +def test_sessionfinish_with_start(pytester: Pytester) -> None: + pytester.makeconftest( + """ + import os + values = [] + def pytest_sessionstart(): + values.append(os.getcwd()) + os.chdir("..") + + def pytest_sessionfinish(): + assert values[0] == os.getcwd() + + """ + ) + res = pytester.runpytest("--collect-only") + assert res.ret == ExitCode.NO_TESTS_COLLECTED + + +@pytest.mark.parametrize("path", ["root", "{relative}/root", "{environment}/root"]) +def test_rootdir_option_arg( + pytester: Pytester, monkeypatch: MonkeyPatch, path: str +) -> None: + monkeypatch.setenv("PY_ROOTDIR_PATH", str(pytester.path)) + path = path.format(relative=str(pytester.path), environment="$PY_ROOTDIR_PATH") + + rootdir = pytester.path / "root" / "tests" + rootdir.mkdir(parents=True) + pytester.makepyfile( + """ + import os + def test_one(): + assert 1 + """ + ) + + result = pytester.runpytest(f"--rootdir={path}") + result.stdout.fnmatch_lines( + [ + f"*rootdir: {pytester.path}/root", + "root/test_rootdir_option_arg.py *", + "*1 passed*", + ] + ) + + +def test_rootdir_wrong_option_arg(pytester: Pytester) -> None: + result = pytester.runpytest("--rootdir=wrong_dir") + result.stderr.fnmatch_lines( + ["*Directory *wrong_dir* not found. Check your '--rootdir' option.*"] + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_setuponly.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_setuponly.py new file mode 100644 index 0000000000..fe4bdc514e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_setuponly.py @@ -0,0 +1,318 @@ +import sys + +import pytest +from _pytest.config import ExitCode +from _pytest.pytester import Pytester + + +@pytest.fixture(params=["--setup-only", "--setup-plan", "--setup-show"], scope="module") +def mode(request): + return request.param + + +def test_show_only_active_fixtures( + pytester: Pytester, mode, dummy_yaml_custom_test +) -> None: + pytester.makepyfile( + ''' + import pytest + @pytest.fixture + def _arg0(): + """hidden arg0 fixture""" + @pytest.fixture + def arg1(): + """arg1 docstring""" + def test_arg1(arg1): + pass + ''' + ) + + result = pytester.runpytest(mode) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + ["*SETUP F arg1*", "*test_arg1 (fixtures used: arg1)*", "*TEARDOWN F arg1*"] + ) + result.stdout.no_fnmatch_line("*_arg0*") + + +def test_show_different_scopes(pytester: Pytester, mode) -> None: + p = pytester.makepyfile( + ''' + import pytest + @pytest.fixture + def arg_function(): + """function scoped fixture""" + @pytest.fixture(scope='session') + def arg_session(): + """session scoped fixture""" + def test_arg1(arg_session, arg_function): + pass + ''' + ) + + result = pytester.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + [ + "SETUP S arg_session*", + "*SETUP F arg_function*", + "*test_arg1 (fixtures used: arg_function, arg_session)*", + "*TEARDOWN F arg_function*", + "TEARDOWN S arg_session*", + ] + ) + + +def test_show_nested_fixtures(pytester: Pytester, mode) -> None: + pytester.makeconftest( + ''' + import pytest + @pytest.fixture(scope='session') + def arg_same(): + """session scoped fixture""" + ''' + ) + p = pytester.makepyfile( + ''' + import pytest + @pytest.fixture(scope='function') + def arg_same(arg_same): + """function scoped fixture""" + def test_arg1(arg_same): + pass + ''' + ) + + result = pytester.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + [ + "SETUP S arg_same*", + "*SETUP F arg_same (fixtures used: arg_same)*", + "*test_arg1 (fixtures used: arg_same)*", + "*TEARDOWN F arg_same*", + "TEARDOWN S arg_same*", + ] + ) + + +def test_show_fixtures_with_autouse(pytester: Pytester, mode) -> None: + p = pytester.makepyfile( + ''' + import pytest + @pytest.fixture + def arg_function(): + """function scoped fixture""" + @pytest.fixture(scope='session', autouse=True) + def arg_session(): + """session scoped fixture""" + def test_arg1(arg_function): + pass + ''' + ) + + result = pytester.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + [ + "SETUP S arg_session*", + "*SETUP F arg_function*", + "*test_arg1 (fixtures used: arg_function, arg_session)*", + ] + ) + + +def test_show_fixtures_with_parameters(pytester: Pytester, mode) -> None: + pytester.makeconftest( + ''' + import pytest + @pytest.fixture(scope='session', params=['foo', 'bar']) + def arg_same(): + """session scoped fixture""" + ''' + ) + p = pytester.makepyfile( + ''' + import pytest + @pytest.fixture(scope='function') + def arg_other(arg_same): + """function scoped fixture""" + def test_arg1(arg_other): + pass + ''' + ) + + result = pytester.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + [ + "SETUP S arg_same?'foo'?", + "TEARDOWN S arg_same?'foo'?", + "SETUP S arg_same?'bar'?", + "TEARDOWN S arg_same?'bar'?", + ] + ) + + +def test_show_fixtures_with_parameter_ids(pytester: Pytester, mode) -> None: + pytester.makeconftest( + ''' + import pytest + @pytest.fixture( + scope='session', params=['foo', 'bar'], ids=['spam', 'ham']) + def arg_same(): + """session scoped fixture""" + ''' + ) + p = pytester.makepyfile( + ''' + import pytest + @pytest.fixture(scope='function') + def arg_other(arg_same): + """function scoped fixture""" + def test_arg1(arg_other): + pass + ''' + ) + + result = pytester.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + ["SETUP S arg_same?'spam'?", "SETUP S arg_same?'ham'?"] + ) + + +def test_show_fixtures_with_parameter_ids_function(pytester: Pytester, mode) -> None: + p = pytester.makepyfile( + """ + import pytest + @pytest.fixture(params=['foo', 'bar'], ids=lambda p: p.upper()) + def foobar(): + pass + def test_foobar(foobar): + pass + """ + ) + + result = pytester.runpytest(mode, p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + ["*SETUP F foobar?'FOO'?", "*SETUP F foobar?'BAR'?"] + ) + + +def test_dynamic_fixture_request(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + @pytest.fixture() + def dynamically_requested_fixture(): + pass + @pytest.fixture() + def dependent_fixture(request): + request.getfixturevalue('dynamically_requested_fixture') + def test_dyn(dependent_fixture): + pass + """ + ) + + result = pytester.runpytest("--setup-only", p) + assert result.ret == 0 + + result.stdout.fnmatch_lines( + [ + "*SETUP F dynamically_requested_fixture", + "*TEARDOWN F dynamically_requested_fixture", + ] + ) + + +def test_capturing(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest, sys + @pytest.fixture() + def one(): + sys.stdout.write('this should be captured') + sys.stderr.write('this should also be captured') + @pytest.fixture() + def two(one): + assert 0 + def test_capturing(two): + pass + """ + ) + + result = pytester.runpytest("--setup-only", p) + result.stdout.fnmatch_lines( + ["this should be captured", "this should also be captured"] + ) + + +def test_show_fixtures_and_execute_test(pytester: Pytester) -> None: + """Verify that setups are shown and tests are executed.""" + p = pytester.makepyfile( + """ + import pytest + @pytest.fixture + def arg(): + assert True + def test_arg(arg): + assert False + """ + ) + + result = pytester.runpytest("--setup-show", p) + assert result.ret == 1 + + result.stdout.fnmatch_lines( + ["*SETUP F arg*", "*test_arg (fixtures used: arg)F*", "*TEARDOWN F arg*"] + ) + + +def test_setup_show_with_KeyboardInterrupt_in_test(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + @pytest.fixture + def arg(): + pass + def test_arg(arg): + raise KeyboardInterrupt() + """ + ) + result = pytester.runpytest("--setup-show", p, no_reraise_ctrlc=True) + result.stdout.fnmatch_lines( + [ + "*SETUP F arg*", + "*test_arg (fixtures used: arg)*", + "*TEARDOWN F arg*", + "*! KeyboardInterrupt !*", + "*= no tests ran in *", + ] + ) + assert result.ret == ExitCode.INTERRUPTED + + +def test_show_fixture_action_with_bytes(pytester: Pytester) -> None: + # Issue 7126, BytesWarning when using --setup-show with bytes parameter + test_file = pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize('data', [b'Hello World']) + def test_data(data): + pass + """ + ) + result = pytester.run( + sys.executable, "-bb", "-m", "pytest", "--setup-show", str(test_file) + ) + assert result.ret == 0 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_setupplan.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_setupplan.py new file mode 100644 index 0000000000..d51a187395 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_setupplan.py @@ -0,0 +1,120 @@ +from _pytest.pytester import Pytester + + +def test_show_fixtures_and_test( + pytester: Pytester, dummy_yaml_custom_test: None +) -> None: + """Verify that fixtures are not executed.""" + pytester.makepyfile( + """ + import pytest + @pytest.fixture + def arg(): + assert False + def test_arg(arg): + assert False + """ + ) + + result = pytester.runpytest("--setup-plan") + assert result.ret == 0 + + result.stdout.fnmatch_lines( + ["*SETUP F arg*", "*test_arg (fixtures used: arg)", "*TEARDOWN F arg*"] + ) + + +def test_show_multi_test_fixture_setup_and_teardown_correctly_simple( + pytester: Pytester, +) -> None: + """Verify that when a fixture lives for longer than a single test, --setup-plan + correctly displays the SETUP/TEARDOWN indicators the right number of times. + + As reported in https://github.com/pytest-dev/pytest/issues/2049 + --setup-plan was showing SETUP/TEARDOWN on every test, even when the fixture + should persist through multiple tests. + + (Note that this bug never affected actual test execution, which used the + correct fixture lifetimes. It was purely a display bug for --setup-plan, and + did not affect the related --setup-show or --setup-only.) + """ + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope = 'class') + def fix(): + return object() + class TestClass: + def test_one(self, fix): + assert False + def test_two(self, fix): + assert False + """ + ) + + result = pytester.runpytest("--setup-plan") + assert result.ret == 0 + + setup_fragment = "SETUP C fix" + setup_count = 0 + + teardown_fragment = "TEARDOWN C fix" + teardown_count = 0 + + for line in result.stdout.lines: + if setup_fragment in line: + setup_count += 1 + if teardown_fragment in line: + teardown_count += 1 + + # before the fix this tests, there would have been a setup/teardown + # message for each test, so the counts would each have been 2 + assert setup_count == 1 + assert teardown_count == 1 + + +def test_show_multi_test_fixture_setup_and_teardown_same_as_setup_show( + pytester: Pytester, +) -> None: + """Verify that SETUP/TEARDOWN messages match what comes out of --setup-show.""" + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope = 'session') + def sess(): + return True + @pytest.fixture(scope = 'module') + def mod(): + return True + @pytest.fixture(scope = 'class') + def cls(): + return True + @pytest.fixture(scope = 'function') + def func(): + return True + def test_outside(sess, mod, cls, func): + assert True + class TestCls: + def test_one(self, sess, mod, cls, func): + assert True + def test_two(self, sess, mod, cls, func): + assert True + """ + ) + + plan_result = pytester.runpytest("--setup-plan") + show_result = pytester.runpytest("--setup-show") + + # the number and text of these lines should be identical + plan_lines = [ + line + for line in plan_result.stdout.lines + if "SETUP" in line or "TEARDOWN" in line + ] + show_lines = [ + line + for line in show_result.stdout.lines + if "SETUP" in line or "TEARDOWN" in line + ] + + assert plan_lines == show_lines diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_skipping.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_skipping.py new file mode 100644 index 0000000000..3010943607 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_skipping.py @@ -0,0 +1,1533 @@ +import sys +import textwrap + +import pytest +from _pytest.pytester import Pytester +from _pytest.runner import runtestprotocol +from _pytest.skipping import evaluate_skip_marks +from _pytest.skipping import evaluate_xfail_marks +from _pytest.skipping import pytest_runtest_setup + + +class TestEvaluation: + def test_no_marker(self, pytester: Pytester) -> None: + item = pytester.getitem("def test_func(): pass") + skipped = evaluate_skip_marks(item) + assert not skipped + + def test_marked_xfail_no_args(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + @pytest.mark.xfail + def test_func(): + pass + """ + ) + xfailed = evaluate_xfail_marks(item) + assert xfailed + assert xfailed.reason == "" + assert xfailed.run + + def test_marked_skipif_no_args(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + @pytest.mark.skipif + def test_func(): + pass + """ + ) + skipped = evaluate_skip_marks(item) + assert skipped + assert skipped.reason == "" + + def test_marked_one_arg(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + @pytest.mark.skipif("hasattr(os, 'sep')") + def test_func(): + pass + """ + ) + skipped = evaluate_skip_marks(item) + assert skipped + assert skipped.reason == "condition: hasattr(os, 'sep')" + + def test_marked_one_arg_with_reason(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + @pytest.mark.skipif("hasattr(os, 'sep')", attr=2, reason="hello world") + def test_func(): + pass + """ + ) + skipped = evaluate_skip_marks(item) + assert skipped + assert skipped.reason == "hello world" + + def test_marked_one_arg_twice(self, pytester: Pytester) -> None: + lines = [ + """@pytest.mark.skipif("not hasattr(os, 'murks')")""", + """@pytest.mark.skipif(condition="hasattr(os, 'murks')")""", + ] + for i in range(0, 2): + item = pytester.getitem( + """ + import pytest + %s + %s + def test_func(): + pass + """ + % (lines[i], lines[(i + 1) % 2]) + ) + skipped = evaluate_skip_marks(item) + assert skipped + assert skipped.reason == "condition: not hasattr(os, 'murks')" + + def test_marked_one_arg_twice2(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + @pytest.mark.skipif("hasattr(os, 'murks')") + @pytest.mark.skipif("not hasattr(os, 'murks')") + def test_func(): + pass + """ + ) + skipped = evaluate_skip_marks(item) + assert skipped + assert skipped.reason == "condition: not hasattr(os, 'murks')" + + def test_marked_skipif_with_boolean_without_reason( + self, pytester: Pytester + ) -> None: + item = pytester.getitem( + """ + import pytest + @pytest.mark.skipif(False) + def test_func(): + pass + """ + ) + with pytest.raises(pytest.fail.Exception) as excinfo: + evaluate_skip_marks(item) + assert excinfo.value.msg is not None + assert ( + """Error evaluating 'skipif': you need to specify reason=STRING when using booleans as conditions.""" + in excinfo.value.msg + ) + + def test_marked_skipif_with_invalid_boolean(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + + class InvalidBool: + def __bool__(self): + raise TypeError("INVALID") + + @pytest.mark.skipif(InvalidBool(), reason="xxx") + def test_func(): + pass + """ + ) + with pytest.raises(pytest.fail.Exception) as excinfo: + evaluate_skip_marks(item) + assert excinfo.value.msg is not None + assert "Error evaluating 'skipif' condition as a boolean" in excinfo.value.msg + assert "INVALID" in excinfo.value.msg + + def test_skipif_class(self, pytester: Pytester) -> None: + (item,) = pytester.getitems( + """ + import pytest + class TestClass(object): + pytestmark = pytest.mark.skipif("config._hackxyz") + def test_func(self): + pass + """ + ) + item.config._hackxyz = 3 # type: ignore[attr-defined] + skipped = evaluate_skip_marks(item) + assert skipped + assert skipped.reason == "condition: config._hackxyz" + + def test_skipif_markeval_namespace(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + + def pytest_markeval_namespace(): + return {"color": "green"} + """ + ) + p = pytester.makepyfile( + """ + import pytest + + @pytest.mark.skipif("color == 'green'") + def test_1(): + assert True + + @pytest.mark.skipif("color == 'red'") + def test_2(): + assert True + """ + ) + res = pytester.runpytest(p) + assert res.ret == 0 + res.stdout.fnmatch_lines(["*1 skipped*"]) + res.stdout.fnmatch_lines(["*1 passed*"]) + + def test_skipif_markeval_namespace_multiple(self, pytester: Pytester) -> None: + """Keys defined by ``pytest_markeval_namespace()`` in nested plugins override top-level ones.""" + root = pytester.mkdir("root") + root.joinpath("__init__.py").touch() + root.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + + def pytest_markeval_namespace(): + return {"arg": "root"} + """ + ) + ) + root.joinpath("test_root.py").write_text( + textwrap.dedent( + """\ + import pytest + + @pytest.mark.skipif("arg == 'root'") + def test_root(): + assert False + """ + ) + ) + foo = root.joinpath("foo") + foo.mkdir() + foo.joinpath("__init__.py").touch() + foo.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + + def pytest_markeval_namespace(): + return {"arg": "foo"} + """ + ) + ) + foo.joinpath("test_foo.py").write_text( + textwrap.dedent( + """\ + import pytest + + @pytest.mark.skipif("arg == 'foo'") + def test_foo(): + assert False + """ + ) + ) + bar = root.joinpath("bar") + bar.mkdir() + bar.joinpath("__init__.py").touch() + bar.joinpath("conftest.py").write_text( + textwrap.dedent( + """\ + import pytest + + def pytest_markeval_namespace(): + return {"arg": "bar"} + """ + ) + ) + bar.joinpath("test_bar.py").write_text( + textwrap.dedent( + """\ + import pytest + + @pytest.mark.skipif("arg == 'bar'") + def test_bar(): + assert False + """ + ) + ) + + reprec = pytester.inline_run("-vs", "--capture=no") + reprec.assertoutcome(skipped=3) + + def test_skipif_markeval_namespace_ValueError(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + + def pytest_markeval_namespace(): + return True + """ + ) + p = pytester.makepyfile( + """ + import pytest + + @pytest.mark.skipif("color == 'green'") + def test_1(): + assert True + """ + ) + res = pytester.runpytest(p) + assert res.ret == 1 + res.stdout.fnmatch_lines( + [ + "*ValueError: pytest_markeval_namespace() needs to return a dict, got True*" + ] + ) + + +class TestXFail: + @pytest.mark.parametrize("strict", [True, False]) + def test_xfail_simple(self, pytester: Pytester, strict: bool) -> None: + item = pytester.getitem( + """ + import pytest + @pytest.mark.xfail(strict=%s) + def test_func(): + assert 0 + """ + % strict + ) + reports = runtestprotocol(item, log=False) + assert len(reports) == 3 + callreport = reports[1] + assert callreport.skipped + assert callreport.wasxfail == "" + + def test_xfail_xpassed(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + @pytest.mark.xfail(reason="this is an xfail") + def test_func(): + assert 1 + """ + ) + reports = runtestprotocol(item, log=False) + assert len(reports) == 3 + callreport = reports[1] + assert callreport.passed + assert callreport.wasxfail == "this is an xfail" + + def test_xfail_using_platform(self, pytester: Pytester) -> None: + """Verify that platform can be used with xfail statements.""" + item = pytester.getitem( + """ + import pytest + @pytest.mark.xfail("platform.platform() == platform.platform()") + def test_func(): + assert 0 + """ + ) + reports = runtestprotocol(item, log=False) + assert len(reports) == 3 + callreport = reports[1] + assert callreport.wasxfail + + def test_xfail_xpassed_strict(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + @pytest.mark.xfail(strict=True, reason="nope") + def test_func(): + assert 1 + """ + ) + reports = runtestprotocol(item, log=False) + assert len(reports) == 3 + callreport = reports[1] + assert callreport.failed + assert str(callreport.longrepr) == "[XPASS(strict)] nope" + assert not hasattr(callreport, "wasxfail") + + def test_xfail_run_anyway(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail + def test_func(): + assert 0 + def test_func2(): + pytest.xfail("hello") + """ + ) + result = pytester.runpytest("--runxfail") + result.stdout.fnmatch_lines( + ["*def test_func():*", "*assert 0*", "*1 failed*1 pass*"] + ) + + @pytest.mark.parametrize( + "test_input,expected", + [ + ( + ["-rs"], + ["SKIPPED [1] test_sample.py:2: unconditional skip", "*1 skipped*"], + ), + ( + ["-rs", "--runxfail"], + ["SKIPPED [1] test_sample.py:2: unconditional skip", "*1 skipped*"], + ), + ], + ) + def test_xfail_run_with_skip_mark( + self, pytester: Pytester, test_input, expected + ) -> None: + pytester.makepyfile( + test_sample=""" + import pytest + @pytest.mark.skip + def test_skip_location() -> None: + assert 0 + """ + ) + result = pytester.runpytest(*test_input) + result.stdout.fnmatch_lines(expected) + + def test_xfail_evalfalse_but_fails(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + @pytest.mark.xfail('False') + def test_func(): + assert 0 + """ + ) + reports = runtestprotocol(item, log=False) + callreport = reports[1] + assert callreport.failed + assert not hasattr(callreport, "wasxfail") + assert "xfail" in callreport.keywords + + def test_xfail_not_report_default(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + test_one=""" + import pytest + @pytest.mark.xfail + def test_this(): + assert 0 + """ + ) + pytester.runpytest(p, "-v") + # result.stdout.fnmatch_lines([ + # "*HINT*use*-r*" + # ]) + + def test_xfail_not_run_xfail_reporting(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + test_one=""" + import pytest + @pytest.mark.xfail(run=False, reason="noway") + def test_this(): + assert 0 + @pytest.mark.xfail("True", run=False) + def test_this_true(): + assert 0 + @pytest.mark.xfail("False", run=False, reason="huh") + def test_this_false(): + assert 1 + """ + ) + result = pytester.runpytest(p, "-rx") + result.stdout.fnmatch_lines( + [ + "*test_one*test_this*", + "*NOTRUN*noway", + "*test_one*test_this_true*", + "*NOTRUN*condition:*True*", + "*1 passed*", + ] + ) + + def test_xfail_not_run_no_setup_run(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + test_one=""" + import pytest + @pytest.mark.xfail(run=False, reason="hello") + def test_this(): + assert 0 + def setup_module(mod): + raise ValueError(42) + """ + ) + result = pytester.runpytest(p, "-rx") + result.stdout.fnmatch_lines( + ["*test_one*test_this*", "*NOTRUN*hello", "*1 xfailed*"] + ) + + def test_xfail_xpass(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + test_one=""" + import pytest + @pytest.mark.xfail + def test_that(): + assert 1 + """ + ) + result = pytester.runpytest(p, "-rX") + result.stdout.fnmatch_lines(["*XPASS*test_that*", "*1 xpassed*"]) + assert result.ret == 0 + + def test_xfail_imperative(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + def test_this(): + pytest.xfail("hello") + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines(["*1 xfailed*"]) + result = pytester.runpytest(p, "-rx") + result.stdout.fnmatch_lines(["*XFAIL*test_this*", "*reason:*hello*"]) + result = pytester.runpytest(p, "--runxfail") + result.stdout.fnmatch_lines(["*1 pass*"]) + + def test_xfail_imperative_in_setup_function(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + def setup_function(function): + pytest.xfail("hello") + + def test_this(): + assert 0 + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines(["*1 xfailed*"]) + result = pytester.runpytest(p, "-rx") + result.stdout.fnmatch_lines(["*XFAIL*test_this*", "*reason:*hello*"]) + result = pytester.runpytest(p, "--runxfail") + result.stdout.fnmatch_lines( + """ + *def test_this* + *1 fail* + """ + ) + + def xtest_dynamic_xfail_set_during_setup(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + def setup_function(function): + pytest.mark.xfail(function) + def test_this(): + assert 0 + def test_that(): + assert 1 + """ + ) + result = pytester.runpytest(p, "-rxX") + result.stdout.fnmatch_lines(["*XFAIL*test_this*", "*XPASS*test_that*"]) + + def test_dynamic_xfail_no_run(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + @pytest.fixture + def arg(request): + request.applymarker(pytest.mark.xfail(run=False)) + def test_this(arg): + assert 0 + """ + ) + result = pytester.runpytest(p, "-rxX") + result.stdout.fnmatch_lines(["*XFAIL*test_this*", "*NOTRUN*"]) + + def test_dynamic_xfail_set_during_funcarg_setup(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + @pytest.fixture + def arg(request): + request.applymarker(pytest.mark.xfail) + def test_this2(arg): + assert 0 + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines(["*1 xfailed*"]) + + def test_dynamic_xfail_set_during_runtest_failed(self, pytester: Pytester) -> None: + # Issue #7486. + p = pytester.makepyfile( + """ + import pytest + def test_this(request): + request.node.add_marker(pytest.mark.xfail(reason="xfail")) + assert 0 + """ + ) + result = pytester.runpytest(p) + result.assert_outcomes(xfailed=1) + + def test_dynamic_xfail_set_during_runtest_passed_strict( + self, pytester: Pytester + ) -> None: + # Issue #7486. + p = pytester.makepyfile( + """ + import pytest + def test_this(request): + request.node.add_marker(pytest.mark.xfail(reason="xfail", strict=True)) + """ + ) + result = pytester.runpytest(p) + result.assert_outcomes(failed=1) + + @pytest.mark.parametrize( + "expected, actual, matchline", + [ + ("TypeError", "TypeError", "*1 xfailed*"), + ("(AttributeError, TypeError)", "TypeError", "*1 xfailed*"), + ("TypeError", "IndexError", "*1 failed*"), + ("(AttributeError, TypeError)", "IndexError", "*1 failed*"), + ], + ) + def test_xfail_raises( + self, expected, actual, matchline, pytester: Pytester + ) -> None: + p = pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail(raises=%s) + def test_raises(): + raise %s() + """ + % (expected, actual) + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines([matchline]) + + def test_strict_sanity(self, pytester: Pytester) -> None: + """Sanity check for xfail(strict=True): a failing test should behave + exactly like a normal xfail.""" + p = pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail(reason='unsupported feature', strict=True) + def test_foo(): + assert 0 + """ + ) + result = pytester.runpytest(p, "-rxX") + result.stdout.fnmatch_lines(["*XFAIL*", "*unsupported feature*"]) + assert result.ret == 0 + + @pytest.mark.parametrize("strict", [True, False]) + def test_strict_xfail(self, pytester: Pytester, strict: bool) -> None: + p = pytester.makepyfile( + """ + import pytest + + @pytest.mark.xfail(reason='unsupported feature', strict=%s) + def test_foo(): + with open('foo_executed', 'w'): pass # make sure test executes + """ + % strict + ) + result = pytester.runpytest(p, "-rxX") + if strict: + result.stdout.fnmatch_lines( + ["*test_foo*", "*XPASS(strict)*unsupported feature*"] + ) + else: + result.stdout.fnmatch_lines( + [ + "*test_strict_xfail*", + "XPASS test_strict_xfail.py::test_foo unsupported feature", + ] + ) + assert result.ret == (1 if strict else 0) + assert pytester.path.joinpath("foo_executed").exists() + + @pytest.mark.parametrize("strict", [True, False]) + def test_strict_xfail_condition(self, pytester: Pytester, strict: bool) -> None: + p = pytester.makepyfile( + """ + import pytest + + @pytest.mark.xfail(False, reason='unsupported feature', strict=%s) + def test_foo(): + pass + """ + % strict + ) + result = pytester.runpytest(p, "-rxX") + result.stdout.fnmatch_lines(["*1 passed*"]) + assert result.ret == 0 + + @pytest.mark.parametrize("strict", [True, False]) + def test_xfail_condition_keyword(self, pytester: Pytester, strict: bool) -> None: + p = pytester.makepyfile( + """ + import pytest + + @pytest.mark.xfail(condition=False, reason='unsupported feature', strict=%s) + def test_foo(): + pass + """ + % strict + ) + result = pytester.runpytest(p, "-rxX") + result.stdout.fnmatch_lines(["*1 passed*"]) + assert result.ret == 0 + + @pytest.mark.parametrize("strict_val", ["true", "false"]) + def test_strict_xfail_default_from_file( + self, pytester: Pytester, strict_val + ) -> None: + pytester.makeini( + """ + [pytest] + xfail_strict = %s + """ + % strict_val + ) + p = pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail(reason='unsupported feature') + def test_foo(): + pass + """ + ) + result = pytester.runpytest(p, "-rxX") + strict = strict_val == "true" + result.stdout.fnmatch_lines(["*1 failed*" if strict else "*1 xpassed*"]) + assert result.ret == (1 if strict else 0) + + def test_xfail_markeval_namespace(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + + def pytest_markeval_namespace(): + return {"color": "green"} + """ + ) + p = pytester.makepyfile( + """ + import pytest + + @pytest.mark.xfail("color == 'green'") + def test_1(): + assert False + + @pytest.mark.xfail("color == 'red'") + def test_2(): + assert False + """ + ) + res = pytester.runpytest(p) + assert res.ret == 1 + res.stdout.fnmatch_lines(["*1 failed*"]) + res.stdout.fnmatch_lines(["*1 xfailed*"]) + + +class TestXFailwithSetupTeardown: + def test_failing_setup_issue9(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + def setup_function(func): + assert 0 + + @pytest.mark.xfail + def test_func(): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 xfail*"]) + + def test_failing_teardown_issue9(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + def teardown_function(func): + assert 0 + + @pytest.mark.xfail + def test_func(): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 xfail*"]) + + +class TestSkip: + def test_skip_class(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skip + class TestSomething(object): + def test_foo(self): + pass + def test_bar(self): + pass + + def test_baz(): + pass + """ + ) + rec = pytester.inline_run() + rec.assertoutcome(skipped=2, passed=1) + + def test_skips_on_false_string(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skip('False') + def test_foo(): + pass + """ + ) + rec = pytester.inline_run() + rec.assertoutcome(skipped=1) + + def test_arg_as_reason(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skip('testing stuff') + def test_bar(): + pass + """ + ) + result = pytester.runpytest("-rs") + result.stdout.fnmatch_lines(["*testing stuff*", "*1 skipped*"]) + + def test_skip_no_reason(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skip + def test_foo(): + pass + """ + ) + result = pytester.runpytest("-rs") + result.stdout.fnmatch_lines(["*unconditional skip*", "*1 skipped*"]) + + def test_skip_with_reason(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skip(reason="for lolz") + def test_bar(): + pass + """ + ) + result = pytester.runpytest("-rs") + result.stdout.fnmatch_lines(["*for lolz*", "*1 skipped*"]) + + def test_only_skips_marked_test(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skip + def test_foo(): + pass + @pytest.mark.skip(reason="nothing in particular") + def test_bar(): + pass + def test_baz(): + assert True + """ + ) + result = pytester.runpytest("-rs") + result.stdout.fnmatch_lines(["*nothing in particular*", "*1 passed*2 skipped*"]) + + def test_strict_and_skip(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skip + def test_hello(): + pass + """ + ) + result = pytester.runpytest("-rs", "--strict-markers") + result.stdout.fnmatch_lines(["*unconditional skip*", "*1 skipped*"]) + + def test_wrong_skip_usage(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skip(False, reason="I thought this was skipif") + def test_hello(): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*TypeError: *__init__() got multiple values for argument 'reason'" + " - maybe you meant pytest.mark.skipif?" + ] + ) + + +class TestSkipif: + def test_skipif_conditional(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + @pytest.mark.skipif("hasattr(os, 'sep')") + def test_func(): + pass + """ + ) + x = pytest.raises(pytest.skip.Exception, lambda: pytest_runtest_setup(item)) + assert x.value.msg == "condition: hasattr(os, 'sep')" + + @pytest.mark.parametrize( + "params", ["\"hasattr(sys, 'platform')\"", 'True, reason="invalid platform"'] + ) + def test_skipif_reporting(self, pytester: Pytester, params) -> None: + p = pytester.makepyfile( + test_foo=""" + import pytest + @pytest.mark.skipif(%(params)s) + def test_that(): + assert 0 + """ + % dict(params=params) + ) + result = pytester.runpytest(p, "-s", "-rs") + result.stdout.fnmatch_lines(["*SKIP*1*test_foo.py*platform*", "*1 skipped*"]) + assert result.ret == 0 + + def test_skipif_using_platform(self, pytester: Pytester) -> None: + item = pytester.getitem( + """ + import pytest + @pytest.mark.skipif("platform.platform() == platform.platform()") + def test_func(): + pass + """ + ) + pytest.raises(pytest.skip.Exception, lambda: pytest_runtest_setup(item)) + + @pytest.mark.parametrize( + "marker, msg1, msg2", + [("skipif", "SKIP", "skipped"), ("xfail", "XPASS", "xpassed")], + ) + def test_skipif_reporting_multiple( + self, pytester: Pytester, marker, msg1, msg2 + ) -> None: + pytester.makepyfile( + test_foo=""" + import pytest + @pytest.mark.{marker}(False, reason='first_condition') + @pytest.mark.{marker}(True, reason='second_condition') + def test_foobar(): + assert 1 + """.format( + marker=marker + ) + ) + result = pytester.runpytest("-s", "-rsxX") + result.stdout.fnmatch_lines( + [f"*{msg1}*test_foo.py*second_condition*", f"*1 {msg2}*"] + ) + assert result.ret == 0 + + +def test_skip_not_report_default(pytester: Pytester) -> None: + p = pytester.makepyfile( + test_one=""" + import pytest + def test_this(): + pytest.skip("hello") + """ + ) + result = pytester.runpytest(p, "-v") + result.stdout.fnmatch_lines( + [ + # "*HINT*use*-r*", + "*1 skipped*" + ] + ) + + +def test_skipif_class(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + class TestClass(object): + pytestmark = pytest.mark.skipif("True") + def test_that(self): + assert 0 + def test_though(self): + assert 0 + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines(["*2 skipped*"]) + + +def test_skipped_reasons_functional(pytester: Pytester) -> None: + pytester.makepyfile( + test_one=""" + import pytest + from conftest import doskip + + def setup_function(func): + doskip() + + def test_func(): + pass + + class TestClass(object): + def test_method(self): + doskip() + + @pytest.mark.skip("via_decorator") + def test_deco(self): + assert 0 + """, + conftest=""" + import pytest, sys + def doskip(): + assert sys._getframe().f_lineno == 3 + pytest.skip('test') + """, + ) + result = pytester.runpytest("-rs") + result.stdout.fnmatch_lines_random( + [ + "SKIPPED [[]2[]] conftest.py:4: test", + "SKIPPED [[]1[]] test_one.py:14: via_decorator", + ] + ) + assert result.ret == 0 + + +def test_skipped_folding(pytester: Pytester) -> None: + pytester.makepyfile( + test_one=""" + import pytest + pytestmark = pytest.mark.skip("Folding") + def setup_function(func): + pass + def test_func(): + pass + class TestClass(object): + def test_method(self): + pass + """ + ) + result = pytester.runpytest("-rs") + result.stdout.fnmatch_lines(["*SKIP*2*test_one.py: Folding"]) + assert result.ret == 0 + + +def test_reportchars(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + def test_1(): + assert 0 + @pytest.mark.xfail + def test_2(): + assert 0 + @pytest.mark.xfail + def test_3(): + pass + def test_4(): + pytest.skip("four") + """ + ) + result = pytester.runpytest("-rfxXs") + result.stdout.fnmatch_lines( + ["FAIL*test_1*", "XFAIL*test_2*", "XPASS*test_3*", "SKIP*four*"] + ) + + +def test_reportchars_error(pytester: Pytester) -> None: + pytester.makepyfile( + conftest=""" + def pytest_runtest_teardown(): + assert 0 + """, + test_simple=""" + def test_foo(): + pass + """, + ) + result = pytester.runpytest("-rE") + result.stdout.fnmatch_lines(["ERROR*test_foo*"]) + + +def test_reportchars_all(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + def test_1(): + assert 0 + @pytest.mark.xfail + def test_2(): + assert 0 + @pytest.mark.xfail + def test_3(): + pass + def test_4(): + pytest.skip("four") + @pytest.fixture + def fail(): + assert 0 + def test_5(fail): + pass + """ + ) + result = pytester.runpytest("-ra") + result.stdout.fnmatch_lines( + [ + "SKIP*four*", + "XFAIL*test_2*", + "XPASS*test_3*", + "ERROR*test_5*", + "FAIL*test_1*", + ] + ) + + +def test_reportchars_all_error(pytester: Pytester) -> None: + pytester.makepyfile( + conftest=""" + def pytest_runtest_teardown(): + assert 0 + """, + test_simple=""" + def test_foo(): + pass + """, + ) + result = pytester.runpytest("-ra") + result.stdout.fnmatch_lines(["ERROR*test_foo*"]) + + +def test_errors_in_xfail_skip_expressions(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skipif("asd") + def test_nameerror(): + pass + @pytest.mark.xfail("syntax error") + def test_syntax(): + pass + + def test_func(): + pass + """ + ) + result = pytester.runpytest() + markline = " ^" + pypy_version_info = getattr(sys, "pypy_version_info", None) + if pypy_version_info is not None and pypy_version_info < (6,): + markline = markline[5:] + elif sys.version_info >= (3, 8) or hasattr(sys, "pypy_version_info"): + markline = markline[4:] + + if sys.version_info[:2] >= (3, 10): + expected = [ + "*ERROR*test_nameerror*", + "*asd*", + "", + "During handling of the above exception, another exception occurred:", + ] + else: + expected = [ + "*ERROR*test_nameerror*", + ] + + expected += [ + "*evaluating*skipif*condition*", + "*asd*", + "*ERROR*test_syntax*", + "*evaluating*xfail*condition*", + " syntax error", + markline, + "SyntaxError: invalid syntax", + "*1 pass*2 errors*", + ] + result.stdout.fnmatch_lines(expected) + + +def test_xfail_skipif_with_globals(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + x = 3 + @pytest.mark.skipif("x == 3") + def test_skip1(): + pass + @pytest.mark.xfail("x == 3") + def test_boolean(): + assert 0 + """ + ) + result = pytester.runpytest("-rsx") + result.stdout.fnmatch_lines(["*SKIP*x == 3*", "*XFAIL*test_boolean*", "*x == 3*"]) + + +def test_default_markers(pytester: Pytester) -> None: + result = pytester.runpytest("--markers") + result.stdout.fnmatch_lines( + [ + "*skipif(condition, ..., [*], reason=...)*skip*", + "*xfail(condition, ..., [*], reason=..., run=True, raises=None, strict=xfail_strict)*expected failure*", + ] + ) + + +def test_xfail_test_setup_exception(pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_runtest_setup(): + 0 / 0 + """ + ) + p = pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail + def test_func(): + assert 0 + """ + ) + result = pytester.runpytest(p) + assert result.ret == 0 + assert "xfailed" in result.stdout.str() + result.stdout.no_fnmatch_line("*xpassed*") + + +def test_imperativeskip_on_xfail_test(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail + def test_that_fails(): + assert 0 + + @pytest.mark.skipif("True") + def test_hello(): + pass + """ + ) + pytester.makeconftest( + """ + import pytest + def pytest_runtest_setup(item): + pytest.skip("abc") + """ + ) + result = pytester.runpytest("-rsxX") + result.stdout.fnmatch_lines_random( + """ + *SKIP*abc* + *SKIP*condition: True* + *2 skipped* + """ + ) + + +class TestBooleanCondition: + def test_skipif(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skipif(True, reason="True123") + def test_func1(): + pass + @pytest.mark.skipif(False, reason="True123") + def test_func2(): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + """ + *1 passed*1 skipped* + """ + ) + + def test_skipif_noreason(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.skipif(True) + def test_func(): + pass + """ + ) + result = pytester.runpytest("-rs") + result.stdout.fnmatch_lines( + """ + *1 error* + """ + ) + + def test_xfail(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail(True, reason="True123") + def test_func(): + assert 0 + """ + ) + result = pytester.runpytest("-rxs") + result.stdout.fnmatch_lines( + """ + *XFAIL* + *True123* + *1 xfail* + """ + ) + + +def test_xfail_item(pytester: Pytester) -> None: + # Ensure pytest.xfail works with non-Python Item + pytester.makeconftest( + """ + import pytest + + class MyItem(pytest.Item): + nodeid = 'foo' + def runtest(self): + pytest.xfail("Expected Failure") + + def pytest_collect_file(file_path, parent): + return MyItem.from_parent(name="foo", parent=parent) + """ + ) + result = pytester.inline_run() + passed, skipped, failed = result.listoutcomes() + assert not failed + xfailed = [r for r in skipped if hasattr(r, "wasxfail")] + assert xfailed + + +def test_module_level_skip_error(pytester: Pytester) -> None: + """Verify that using pytest.skip at module level causes a collection error.""" + pytester.makepyfile( + """ + import pytest + pytest.skip("skip_module_level") + + def test_func(): + assert True + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + ["*Using pytest.skip outside of a test will skip the entire module*"] + ) + + +def test_module_level_skip_with_allow_module_level(pytester: Pytester) -> None: + """Verify that using pytest.skip(allow_module_level=True) is allowed.""" + pytester.makepyfile( + """ + import pytest + pytest.skip("skip_module_level", allow_module_level=True) + + def test_func(): + assert 0 + """ + ) + result = pytester.runpytest("-rxs") + result.stdout.fnmatch_lines(["*SKIP*skip_module_level"]) + + +def test_invalid_skip_keyword_parameter(pytester: Pytester) -> None: + """Verify that using pytest.skip() with unknown parameter raises an error.""" + pytester.makepyfile( + """ + import pytest + pytest.skip("skip_module_level", unknown=1) + + def test_func(): + assert 0 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*TypeError:*['unknown']*"]) + + +def test_mark_xfail_item(pytester: Pytester) -> None: + # Ensure pytest.mark.xfail works with non-Python Item + pytester.makeconftest( + """ + import pytest + + class MyItem(pytest.Item): + nodeid = 'foo' + def setup(self): + marker = pytest.mark.xfail("1 == 2", reason="Expected failure - false") + self.add_marker(marker) + marker = pytest.mark.xfail(True, reason="Expected failure - true") + self.add_marker(marker) + def runtest(self): + assert False + + def pytest_collect_file(file_path, parent): + return MyItem.from_parent(name="foo", parent=parent) + """ + ) + result = pytester.inline_run() + passed, skipped, failed = result.listoutcomes() + assert not failed + xfailed = [r for r in skipped if hasattr(r, "wasxfail")] + assert xfailed + + +def test_summary_list_after_errors(pytester: Pytester) -> None: + """Ensure the list of errors/fails/xfails/skips appears after tracebacks in terminal reporting.""" + pytester.makepyfile( + """ + import pytest + def test_fail(): + assert 0 + """ + ) + result = pytester.runpytest("-ra") + result.stdout.fnmatch_lines( + [ + "=* FAILURES *=", + "*= short test summary info =*", + "FAILED test_summary_list_after_errors.py::test_fail - assert 0", + ] + ) + + +def test_importorskip() -> None: + with pytest.raises( + pytest.skip.Exception, + match="^could not import 'doesnotexist': No module named .*", + ): + pytest.importorskip("doesnotexist") + + +def test_relpath_rootdir(pytester: Pytester) -> None: + pytester.makepyfile( + **{ + "tests/test_1.py": """ + import pytest + @pytest.mark.skip() + def test_pass(): + pass + """, + } + ) + result = pytester.runpytest("-rs", "tests/test_1.py", "--rootdir=tests") + result.stdout.fnmatch_lines( + ["SKIPPED [[]1[]] tests/test_1.py:2: unconditional skip"] + ) + + +def test_skip_using_reason_works_ok(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + def test_skipping_reason(): + pytest.skip(reason="skippedreason") + """ + ) + result = pytester.runpytest(p) + result.stdout.no_fnmatch_line("*PytestDeprecationWarning*") + result.assert_outcomes(skipped=1) + + +def test_fail_using_reason_works_ok(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + def test_failing_reason(): + pytest.fail(reason="failedreason") + """ + ) + result = pytester.runpytest(p) + result.stdout.no_fnmatch_line("*PytestDeprecationWarning*") + result.assert_outcomes(failed=1) + + +def test_fail_fails_with_msg_and_reason(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + def test_fail_both_arguments(): + pytest.fail(reason="foo", msg="bar") + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + "*UsageError: Passing both ``reason`` and ``msg`` to pytest.fail(...) is not permitted.*" + ) + result.assert_outcomes(failed=1) + + +def test_skip_fails_with_msg_and_reason(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + def test_skip_both_arguments(): + pytest.skip(reason="foo", msg="bar") + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + "*UsageError: Passing both ``reason`` and ``msg`` to pytest.skip(...) is not permitted.*" + ) + result.assert_outcomes(failed=1) + + +def test_exit_with_msg_and_reason_fails(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + def test_exit_both_arguments(): + pytest.exit(reason="foo", msg="bar") + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines( + "*UsageError: cannot pass reason and msg to exit(), `msg` is deprecated, use `reason`.*" + ) + result.assert_outcomes(failed=1) + + +def test_exit_with_reason_works_ok(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + def test_exit_reason_only(): + pytest.exit(reason="foo") + """ + ) + result = pytester.runpytest(p) + result.stdout.fnmatch_lines("*_pytest.outcomes.Exit: foo*") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_stash.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_stash.py new file mode 100644 index 0000000000..2c9df4832e --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_stash.py @@ -0,0 +1,67 @@ +import pytest +from _pytest.stash import Stash +from _pytest.stash import StashKey + + +def test_stash() -> None: + stash = Stash() + + assert len(stash) == 0 + assert not stash + + key1 = StashKey[str]() + key2 = StashKey[int]() + + # Basic functionality - single key. + assert key1 not in stash + stash[key1] = "hello" + assert key1 in stash + assert stash[key1] == "hello" + assert stash.get(key1, None) == "hello" + stash[key1] = "world" + assert stash[key1] == "world" + # Has correct type (no mypy error). + stash[key1] + "string" + assert len(stash) == 1 + assert stash + + # No interaction with another key. + assert key2 not in stash + assert stash.get(key2, None) is None + with pytest.raises(KeyError): + stash[key2] + with pytest.raises(KeyError): + del stash[key2] + stash[key2] = 1 + assert stash[key2] == 1 + # Has correct type (no mypy error). + stash[key2] + 20 + del stash[key1] + with pytest.raises(KeyError): + del stash[key1] + with pytest.raises(KeyError): + stash[key1] + + # setdefault + stash[key1] = "existing" + assert stash.setdefault(key1, "default") == "existing" + assert stash[key1] == "existing" + key_setdefault = StashKey[bytes]() + assert stash.setdefault(key_setdefault, b"default") == b"default" + assert stash[key_setdefault] == b"default" + assert len(stash) == 3 + assert stash + + # Can't accidentally add attributes to stash object itself. + with pytest.raises(AttributeError): + stash.foo = "nope" # type: ignore[attr-defined] + + # No interaction with another stash. + stash2 = Stash() + key3 = StashKey[int]() + assert key2 not in stash2 + stash2[key2] = 100 + stash2[key3] = 200 + assert stash2[key2] + stash2[key3] == 300 + assert stash[key2] == 1 + assert key3 not in stash diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_stepwise.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_stepwise.py new file mode 100644 index 0000000000..63d29d6241 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_stepwise.py @@ -0,0 +1,280 @@ +import pytest +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester + + +@pytest.fixture +def stepwise_pytester(pytester: Pytester) -> Pytester: + # Rather than having to modify our testfile between tests, we introduce + # a flag for whether or not the second test should fail. + pytester.makeconftest( + """ +def pytest_addoption(parser): + group = parser.getgroup('general') + group.addoption('--fail', action='store_true', dest='fail') + group.addoption('--fail-last', action='store_true', dest='fail_last') +""" + ) + + # Create a simple test suite. + pytester.makepyfile( + test_a=""" +def test_success_before_fail(): + assert 1 + +def test_fail_on_flag(request): + assert not request.config.getvalue('fail') + +def test_success_after_fail(): + assert 1 + +def test_fail_last_on_flag(request): + assert not request.config.getvalue('fail_last') + +def test_success_after_last_fail(): + assert 1 +""" + ) + + pytester.makepyfile( + test_b=""" +def test_success(): + assert 1 +""" + ) + + # customize cache directory so we don't use the tox's cache directory, which makes tests in this module flaky + pytester.makeini( + """ + [pytest] + cache_dir = .cache + """ + ) + + return pytester + + +@pytest.fixture +def error_pytester(pytester: Pytester) -> Pytester: + pytester.makepyfile( + test_a=""" +def test_error(nonexisting_fixture): + assert 1 + +def test_success_after_fail(): + assert 1 +""" + ) + + return pytester + + +@pytest.fixture +def broken_pytester(pytester: Pytester) -> Pytester: + pytester.makepyfile( + working_testfile="def test_proper(): assert 1", broken_testfile="foobar" + ) + return pytester + + +def _strip_resource_warnings(lines): + # Strip unreliable ResourceWarnings, so no-output assertions on stderr can work. + # (https://github.com/pytest-dev/pytest/issues/5088) + return [ + x + for x in lines + if not x.startswith(("Exception ignored in:", "ResourceWarning")) + ] + + +def test_run_without_stepwise(stepwise_pytester: Pytester) -> None: + result = stepwise_pytester.runpytest("-v", "--strict-markers", "--fail") + result.stdout.fnmatch_lines(["*test_success_before_fail PASSED*"]) + result.stdout.fnmatch_lines(["*test_fail_on_flag FAILED*"]) + result.stdout.fnmatch_lines(["*test_success_after_fail PASSED*"]) + + +def test_stepwise_output_summary(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize("expected", [True, True, True, True, False]) + def test_data(expected): + assert expected + """ + ) + result = pytester.runpytest("-v", "--stepwise") + result.stdout.fnmatch_lines(["stepwise: no previously failed tests, not skipping."]) + result = pytester.runpytest("-v", "--stepwise") + result.stdout.fnmatch_lines( + ["stepwise: skipping 4 already passed items.", "*1 failed, 4 deselected*"] + ) + + +def test_fail_and_continue_with_stepwise(stepwise_pytester: Pytester) -> None: + # Run the tests with a failing second test. + result = stepwise_pytester.runpytest( + "-v", "--strict-markers", "--stepwise", "--fail" + ) + assert _strip_resource_warnings(result.stderr.lines) == [] + + stdout = result.stdout.str() + # Make sure we stop after first failing test. + assert "test_success_before_fail PASSED" in stdout + assert "test_fail_on_flag FAILED" in stdout + assert "test_success_after_fail" not in stdout + + # "Fix" the test that failed in the last run and run it again. + result = stepwise_pytester.runpytest("-v", "--strict-markers", "--stepwise") + assert _strip_resource_warnings(result.stderr.lines) == [] + + stdout = result.stdout.str() + # Make sure the latest failing test runs and then continues. + assert "test_success_before_fail" not in stdout + assert "test_fail_on_flag PASSED" in stdout + assert "test_success_after_fail PASSED" in stdout + + +@pytest.mark.parametrize("stepwise_skip", ["--stepwise-skip", "--sw-skip"]) +def test_run_with_skip_option(stepwise_pytester: Pytester, stepwise_skip: str) -> None: + result = stepwise_pytester.runpytest( + "-v", + "--strict-markers", + "--stepwise", + stepwise_skip, + "--fail", + "--fail-last", + ) + assert _strip_resource_warnings(result.stderr.lines) == [] + + stdout = result.stdout.str() + # Make sure first fail is ignore and second fail stops the test run. + assert "test_fail_on_flag FAILED" in stdout + assert "test_success_after_fail PASSED" in stdout + assert "test_fail_last_on_flag FAILED" in stdout + assert "test_success_after_last_fail" not in stdout + + +def test_fail_on_errors(error_pytester: Pytester) -> None: + result = error_pytester.runpytest("-v", "--strict-markers", "--stepwise") + + assert _strip_resource_warnings(result.stderr.lines) == [] + stdout = result.stdout.str() + + assert "test_error ERROR" in stdout + assert "test_success_after_fail" not in stdout + + +def test_change_testfile(stepwise_pytester: Pytester) -> None: + result = stepwise_pytester.runpytest( + "-v", "--strict-markers", "--stepwise", "--fail", "test_a.py" + ) + assert _strip_resource_warnings(result.stderr.lines) == [] + + stdout = result.stdout.str() + assert "test_fail_on_flag FAILED" in stdout + + # Make sure the second test run starts from the beginning, since the + # test to continue from does not exist in testfile_b. + result = stepwise_pytester.runpytest( + "-v", "--strict-markers", "--stepwise", "test_b.py" + ) + assert _strip_resource_warnings(result.stderr.lines) == [] + + stdout = result.stdout.str() + assert "test_success PASSED" in stdout + + +@pytest.mark.parametrize("broken_first", [True, False]) +def test_stop_on_collection_errors( + broken_pytester: Pytester, broken_first: bool +) -> None: + """Stop during collection errors. Broken test first or broken test last + actually surfaced a bug (#5444), so we test both situations.""" + files = ["working_testfile.py", "broken_testfile.py"] + if broken_first: + files.reverse() + result = broken_pytester.runpytest("-v", "--strict-markers", "--stepwise", *files) + result.stdout.fnmatch_lines("*error during collection*") + + +def test_xfail_handling(pytester: Pytester, monkeypatch: MonkeyPatch) -> None: + """Ensure normal xfail is ignored, and strict xfail interrupts the session in sw mode + + (#5547) + """ + monkeypatch.setattr("sys.dont_write_bytecode", True) + + contents = """ + import pytest + def test_a(): pass + + @pytest.mark.xfail(strict={strict}) + def test_b(): assert {assert_value} + + def test_c(): pass + def test_d(): pass + """ + pytester.makepyfile(contents.format(assert_value="0", strict="False")) + result = pytester.runpytest("--sw", "-v") + result.stdout.fnmatch_lines( + [ + "*::test_a PASSED *", + "*::test_b XFAIL *", + "*::test_c PASSED *", + "*::test_d PASSED *", + "* 3 passed, 1 xfailed in *", + ] + ) + + pytester.makepyfile(contents.format(assert_value="1", strict="True")) + result = pytester.runpytest("--sw", "-v") + result.stdout.fnmatch_lines( + [ + "*::test_a PASSED *", + "*::test_b FAILED *", + "* Interrupted*", + "* 1 failed, 1 passed in *", + ] + ) + + pytester.makepyfile(contents.format(assert_value="0", strict="True")) + result = pytester.runpytest("--sw", "-v") + result.stdout.fnmatch_lines( + [ + "*::test_b XFAIL *", + "*::test_c PASSED *", + "*::test_d PASSED *", + "* 2 passed, 1 deselected, 1 xfailed in *", + ] + ) + + +def test_stepwise_skip_is_independent(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_one(): + assert False + + def test_two(): + assert False + + def test_three(): + assert False + + """ + ) + result = pytester.runpytest("--tb", "no", "--stepwise-skip") + result.assert_outcomes(failed=2) + result.stdout.fnmatch_lines( + [ + "FAILED test_stepwise_skip_is_independent.py::test_one - assert False", + "FAILED test_stepwise_skip_is_independent.py::test_two - assert False", + "*Interrupted: Test failed, continuing from this test next run.*", + ] + ) + + +def test_sw_skip_help(pytester: Pytester) -> None: + result = pytester.runpytest("-h") + result.stdout.fnmatch_lines("*implicitly enables --stepwise.") diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_terminal.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_terminal.py new file mode 100644 index 0000000000..23f597e332 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_terminal.py @@ -0,0 +1,2486 @@ +"""Terminal reporting of the full testing process.""" +import collections +import os +import sys +import textwrap +from io import StringIO +from pathlib import Path +from types import SimpleNamespace +from typing import cast +from typing import Dict +from typing import List +from typing import Tuple + +import pluggy + +import _pytest.config +import _pytest.terminal +import pytest +from _pytest._io.wcwidth import wcswidth +from _pytest.config import Config +from _pytest.config import ExitCode +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester +from _pytest.reports import BaseReport +from _pytest.reports import CollectReport +from _pytest.reports import TestReport +from _pytest.terminal import _folded_skips +from _pytest.terminal import _format_trimmed +from _pytest.terminal import _get_line_with_reprcrash_message +from _pytest.terminal import _get_raw_skip_reason +from _pytest.terminal import _plugin_nameversions +from _pytest.terminal import getreportopt +from _pytest.terminal import TerminalReporter + +DistInfo = collections.namedtuple("DistInfo", ["project_name", "version"]) + + +TRANS_FNMATCH = str.maketrans({"[": "[[]", "]": "[]]"}) + + +class Option: + def __init__(self, verbosity=0): + self.verbosity = verbosity + + @property + def args(self): + values = [] + values.append("--verbosity=%d" % self.verbosity) + return values + + +@pytest.fixture( + params=[Option(verbosity=0), Option(verbosity=1), Option(verbosity=-1)], + ids=["default", "verbose", "quiet"], +) +def option(request): + return request.param + + +@pytest.mark.parametrize( + "input,expected", + [ + ([DistInfo(project_name="test", version=1)], ["test-1"]), + ([DistInfo(project_name="pytest-test", version=1)], ["test-1"]), + ( + [ + DistInfo(project_name="test", version=1), + DistInfo(project_name="test", version=1), + ], + ["test-1"], + ), + ], + ids=["normal", "prefix-strip", "deduplicate"], +) +def test_plugin_nameversion(input, expected): + pluginlist = [(None, x) for x in input] + result = _plugin_nameversions(pluginlist) + assert result == expected + + +class TestTerminal: + def test_pass_skip_fail(self, pytester: Pytester, option) -> None: + pytester.makepyfile( + """ + import pytest + def test_ok(): + pass + def test_skip(): + pytest.skip("xx") + def test_func(): + assert 0 + """ + ) + result = pytester.runpytest(*option.args) + if option.verbosity > 0: + result.stdout.fnmatch_lines( + [ + "*test_pass_skip_fail.py::test_ok PASS*", + "*test_pass_skip_fail.py::test_skip SKIP*", + "*test_pass_skip_fail.py::test_func FAIL*", + ] + ) + elif option.verbosity == 0: + result.stdout.fnmatch_lines(["*test_pass_skip_fail.py .sF*"]) + else: + result.stdout.fnmatch_lines([".sF*"]) + result.stdout.fnmatch_lines( + [" def test_func():", "> assert 0", "E assert 0"] + ) + + def test_internalerror(self, pytester: Pytester, linecomp) -> None: + modcol = pytester.getmodulecol("def test_one(): pass") + rep = TerminalReporter(modcol.config, file=linecomp.stringio) + with pytest.raises(ValueError) as excinfo: + raise ValueError("hello") + rep.pytest_internalerror(excinfo.getrepr()) + linecomp.assert_contains_lines(["INTERNALERROR> *ValueError*hello*"]) + + def test_writeline(self, pytester: Pytester, linecomp) -> None: + modcol = pytester.getmodulecol("def test_one(): pass") + rep = TerminalReporter(modcol.config, file=linecomp.stringio) + rep.write_fspath_result(modcol.nodeid, ".") + rep.write_line("hello world") + lines = linecomp.stringio.getvalue().split("\n") + assert not lines[0] + assert lines[1].endswith(modcol.name + " .") + assert lines[2] == "hello world" + + def test_show_runtest_logstart(self, pytester: Pytester, linecomp) -> None: + item = pytester.getitem("def test_func(): pass") + tr = TerminalReporter(item.config, file=linecomp.stringio) + item.config.pluginmanager.register(tr) + location = item.reportinfo() + tr.config.hook.pytest_runtest_logstart( + nodeid=item.nodeid, location=location, fspath=str(item.path) + ) + linecomp.assert_contains_lines(["*test_show_runtest_logstart.py*"]) + + def test_runtest_location_shown_before_test_starts( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """ + def test_1(): + import time + time.sleep(20) + """ + ) + child = pytester.spawn_pytest("") + child.expect(".*test_runtest_location.*py") + child.sendeof() + child.kill(15) + + def test_report_collect_after_half_a_second( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + """Test for "collecting" being updated after 0.5s""" + + pytester.makepyfile( + **{ + "test1.py": """ + import _pytest.terminal + + _pytest.terminal.REPORT_COLLECTING_RESOLUTION = 0 + + def test_1(): + pass + """, + "test2.py": "def test_2(): pass", + } + ) + # Explicitly test colored output. + monkeypatch.setenv("PY_COLORS", "1") + + child = pytester.spawn_pytest("-v test1.py test2.py") + child.expect(r"collecting \.\.\.") + child.expect(r"collecting 1 item") + child.expect(r"collecting 2 items") + child.expect(r"collected 2 items") + rest = child.read().decode("utf8") + assert "= \x1b[32m\x1b[1m2 passed\x1b[0m\x1b[32m in" in rest + + def test_itemreport_subclasses_show_subclassed_file( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + **{ + "tests/test_p1": """ + class BaseTests(object): + fail = False + + def test_p1(self): + if self.fail: assert 0 + """, + "tests/test_p2": """ + from test_p1 import BaseTests + + class TestMore(BaseTests): pass + """, + "tests/test_p3.py": """ + from test_p1 import BaseTests + + BaseTests.fail = True + + class TestMore(BaseTests): pass + """, + } + ) + result = pytester.runpytest("tests/test_p2.py", "--rootdir=tests") + result.stdout.fnmatch_lines(["tests/test_p2.py .*", "=* 1 passed in *"]) + + result = pytester.runpytest("-vv", "-rA", "tests/test_p2.py", "--rootdir=tests") + result.stdout.fnmatch_lines( + [ + "tests/test_p2.py::TestMore::test_p1 <- test_p1.py PASSED *", + "*= short test summary info =*", + "PASSED tests/test_p2.py::TestMore::test_p1", + ] + ) + result = pytester.runpytest("-vv", "-rA", "tests/test_p3.py", "--rootdir=tests") + result.stdout.fnmatch_lines( + [ + "tests/test_p3.py::TestMore::test_p1 <- test_p1.py FAILED *", + "*_ TestMore.test_p1 _*", + " def test_p1(self):", + "> if self.fail: assert 0", + "E assert 0", + "", + "tests/test_p1.py:5: AssertionError", + "*= short test summary info =*", + "FAILED tests/test_p3.py::TestMore::test_p1 - assert 0", + "*= 1 failed in *", + ] + ) + + def test_itemreport_directclasses_not_shown_as_subclasses( + self, pytester: Pytester + ) -> None: + a = pytester.mkpydir("a123") + a.joinpath("test_hello123.py").write_text( + textwrap.dedent( + """\ + class TestClass(object): + def test_method(self): + pass + """ + ) + ) + result = pytester.runpytest("-vv") + assert result.ret == 0 + result.stdout.fnmatch_lines(["*a123/test_hello123.py*PASS*"]) + result.stdout.no_fnmatch_line("* <- *") + + @pytest.mark.parametrize("fulltrace", ("", "--fulltrace")) + def test_keyboard_interrupt(self, pytester: Pytester, fulltrace) -> None: + pytester.makepyfile( + """ + def test_foobar(): + assert 0 + def test_spamegg(): + import py; pytest.skip('skip me please!') + def test_interrupt_me(): + raise KeyboardInterrupt # simulating the user + """ + ) + + result = pytester.runpytest(fulltrace, no_reraise_ctrlc=True) + result.stdout.fnmatch_lines( + [ + " def test_foobar():", + "> assert 0", + "E assert 0", + "*_keyboard_interrupt.py:6: KeyboardInterrupt*", + ] + ) + if fulltrace: + result.stdout.fnmatch_lines( + ["*raise KeyboardInterrupt # simulating the user*"] + ) + else: + result.stdout.fnmatch_lines( + ["(to show a full traceback on KeyboardInterrupt use --full-trace)"] + ) + result.stdout.fnmatch_lines(["*KeyboardInterrupt*"]) + + def test_keyboard_in_sessionstart(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_sessionstart(): + raise KeyboardInterrupt + """ + ) + pytester.makepyfile( + """ + def test_foobar(): + pass + """ + ) + + result = pytester.runpytest(no_reraise_ctrlc=True) + assert result.ret == 2 + result.stdout.fnmatch_lines(["*KeyboardInterrupt*"]) + + def test_collect_single_item(self, pytester: Pytester) -> None: + """Use singular 'item' when reporting a single test item""" + pytester.makepyfile( + """ + def test_foobar(): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["collected 1 item"]) + + def test_rewrite(self, pytester: Pytester, monkeypatch) -> None: + config = pytester.parseconfig() + f = StringIO() + monkeypatch.setattr(f, "isatty", lambda *args: True) + tr = TerminalReporter(config, f) + tr._tw.fullwidth = 10 + tr.write("hello") + tr.rewrite("hey", erase=True) + assert f.getvalue() == "hello" + "\r" + "hey" + (6 * " ") + + def test_report_teststatus_explicit_markup( + self, monkeypatch: MonkeyPatch, pytester: Pytester, color_mapping + ) -> None: + """Test that TerminalReporter handles markup explicitly provided by + a pytest_report_teststatus hook.""" + monkeypatch.setenv("PY_COLORS", "1") + pytester.makeconftest( + """ + def pytest_report_teststatus(report): + return 'foo', 'F', ('FOO', {'red': True}) + """ + ) + pytester.makepyfile( + """ + def test_foobar(): + pass + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + color_mapping.format_for_fnmatch(["*{red}FOO{reset}*"]) + ) + + def test_verbose_skip_reason(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.mark.skip(reason="123") + def test_1(): + pass + + @pytest.mark.xfail(reason="456") + def test_2(): + pass + + @pytest.mark.xfail(reason="789") + def test_3(): + assert False + + @pytest.mark.xfail(reason="") + def test_4(): + assert False + + @pytest.mark.skip + def test_5(): + pass + + @pytest.mark.xfail + def test_6(): + pass + + def test_7(): + pytest.skip() + + def test_8(): + pytest.skip("888 is great") + + def test_9(): + pytest.xfail() + + def test_10(): + pytest.xfail("It's 🕙 o'clock") + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "test_verbose_skip_reason.py::test_1 SKIPPED (123) *", + "test_verbose_skip_reason.py::test_2 XPASS (456) *", + "test_verbose_skip_reason.py::test_3 XFAIL (789) *", + "test_verbose_skip_reason.py::test_4 XFAIL *", + "test_verbose_skip_reason.py::test_5 SKIPPED (unconditional skip) *", + "test_verbose_skip_reason.py::test_6 XPASS *", + "test_verbose_skip_reason.py::test_7 SKIPPED *", + "test_verbose_skip_reason.py::test_8 SKIPPED (888 is great) *", + "test_verbose_skip_reason.py::test_9 XFAIL *", + "test_verbose_skip_reason.py::test_10 XFAIL (It's 🕙 o'clock) *", + ] + ) + + +class TestCollectonly: + def test_collectonly_basic(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_func(): + pass + """ + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines( + ["", " "] + ) + + def test_collectonly_skipped_module(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + pytest.skip("hello") + """ + ) + result = pytester.runpytest("--collect-only", "-rs") + result.stdout.fnmatch_lines(["*ERROR collecting*"]) + + def test_collectonly_displays_test_description( + self, pytester: Pytester, dummy_yaml_custom_test + ) -> None: + """Used dummy_yaml_custom_test for an Item without ``obj``.""" + pytester.makepyfile( + """ + def test_with_description(): + ''' This test has a description. + + more1. + more2.''' + """ + ) + result = pytester.runpytest("--collect-only", "--verbose") + result.stdout.fnmatch_lines( + [ + "", + " ", + "", + " ", + " This test has a description.", + " ", + " more1.", + " more2.", + ], + consecutive=True, + ) + + def test_collectonly_failed_module(self, pytester: Pytester) -> None: + pytester.makepyfile("""raise ValueError(0)""") + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines(["*raise ValueError*", "*1 error*"]) + + def test_collectonly_fatal(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_collectstart(collector): + assert 0, "urgs" + """ + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines(["*INTERNAL*args*"]) + assert result.ret == 3 + + def test_collectonly_simple(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + def test_func1(): + pass + class TestClass(object): + def test_method(self): + pass + """ + ) + result = pytester.runpytest("--collect-only", p) + # assert stderr.startswith("inserting into sys.path") + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + "*", + "* ", + "* ", + "* ", + ] + ) + + def test_collectonly_error(self, pytester: Pytester) -> None: + p = pytester.makepyfile("import Errlkjqweqwe") + result = pytester.runpytest("--collect-only", p) + assert result.ret == 2 + result.stdout.fnmatch_lines( + textwrap.dedent( + """\ + *ERROR* + *ImportError* + *No module named *Errlk* + *1 error* + """ + ).strip() + ) + + def test_collectonly_missing_path(self, pytester: Pytester) -> None: + """Issue 115: failure in parseargs will cause session not to + have the items attribute.""" + result = pytester.runpytest("--collect-only", "uhm_missing_path") + assert result.ret == 4 + result.stderr.fnmatch_lines( + ["*ERROR: file or directory not found: uhm_missing_path"] + ) + + def test_collectonly_quiet(self, pytester: Pytester) -> None: + pytester.makepyfile("def test_foo(): pass") + result = pytester.runpytest("--collect-only", "-q") + result.stdout.fnmatch_lines(["*test_foo*"]) + + def test_collectonly_more_quiet(self, pytester: Pytester) -> None: + pytester.makepyfile(test_fun="def test_foo(): pass") + result = pytester.runpytest("--collect-only", "-qq") + result.stdout.fnmatch_lines(["*test_fun.py: 1*"]) + + def test_collect_only_summary_status(self, pytester: Pytester) -> None: + """Custom status depending on test selection using -k or -m. #7701.""" + pytester.makepyfile( + test_collect_foo=""" + def test_foo(): pass + """, + test_collect_bar=""" + def test_foobar(): pass + def test_bar(): pass + """, + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines("*== 3 tests collected in * ==*") + + result = pytester.runpytest("--collect-only", "test_collect_foo.py") + result.stdout.fnmatch_lines("*== 1 test collected in * ==*") + + result = pytester.runpytest("--collect-only", "-k", "foo") + result.stdout.fnmatch_lines("*== 2/3 tests collected (1 deselected) in * ==*") + + result = pytester.runpytest("--collect-only", "-k", "test_bar") + result.stdout.fnmatch_lines("*== 1/3 tests collected (2 deselected) in * ==*") + + result = pytester.runpytest("--collect-only", "-k", "invalid") + result.stdout.fnmatch_lines("*== no tests collected (3 deselected) in * ==*") + + pytester.mkdir("no_tests_here") + result = pytester.runpytest("--collect-only", "no_tests_here") + result.stdout.fnmatch_lines("*== no tests collected in * ==*") + + pytester.makepyfile( + test_contains_error=""" + raise RuntimeError + """, + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines("*== 3 tests collected, 1 error in * ==*") + result = pytester.runpytest("--collect-only", "-k", "foo") + result.stdout.fnmatch_lines( + "*== 2/3 tests collected (1 deselected), 1 error in * ==*" + ) + + +class TestFixtureReporting: + def test_setup_fixture_error(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def setup_function(function): + print("setup func") + assert 0 + def test_nada(): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*ERROR at setup of test_nada*", + "*setup_function(function):*", + "*setup func*", + "*assert 0*", + "*1 error*", + ] + ) + assert result.ret != 0 + + def test_teardown_fixture_error(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_nada(): + pass + def teardown_function(function): + print("teardown func") + assert 0 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*ERROR at teardown*", + "*teardown_function(function):*", + "*assert 0*", + "*Captured stdout*", + "*teardown func*", + "*1 passed*1 error*", + ] + ) + + def test_teardown_fixture_error_and_test_failure(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_fail(): + assert 0, "failingfunc" + + def teardown_function(function): + print("teardown func") + assert False + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*ERROR at teardown of test_fail*", + "*teardown_function(function):*", + "*assert False*", + "*Captured stdout*", + "*teardown func*", + "*test_fail*", + "*def test_fail():", + "*failingfunc*", + "*1 failed*1 error*", + ] + ) + + def test_setup_teardown_output_and_test_failure(self, pytester: Pytester) -> None: + """Test for issue #442.""" + pytester.makepyfile( + """ + def setup_function(function): + print("setup func") + + def test_fail(): + assert 0, "failingfunc" + + def teardown_function(function): + print("teardown func") + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*test_fail*", + "*def test_fail():", + "*failingfunc*", + "*Captured stdout setup*", + "*setup func*", + "*Captured stdout teardown*", + "*teardown func*", + "*1 failed*", + ] + ) + + +class TestTerminalFunctional: + def test_deselected(self, pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + def test_one(): + pass + def test_two(): + pass + def test_three(): + pass + """ + ) + result = pytester.runpytest( + "-Wignore::pytest.PytestRemovedIn7Warning", "-k", "test_two:", testpath + ) + result.stdout.fnmatch_lines( + ["collected 3 items / 1 deselected / 2 selected", "*test_deselected.py ..*"] + ) + assert result.ret == 0 + + def test_deselected_with_hookwrapper(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + + @pytest.hookimpl(hookwrapper=True) + def pytest_collection_modifyitems(config, items): + yield + deselected = items.pop() + config.hook.pytest_deselected(items=[deselected]) + """ + ) + testpath = pytester.makepyfile( + """ + def test_one(): + pass + def test_two(): + pass + def test_three(): + pass + """ + ) + result = pytester.runpytest(testpath) + result.stdout.fnmatch_lines( + [ + "collected 3 items / 1 deselected / 2 selected", + "*= 2 passed, 1 deselected in*", + ] + ) + assert result.ret == 0 + + def test_show_deselected_items_using_markexpr_before_test_execution( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + test_show_deselected=""" + import pytest + + @pytest.mark.foo + def test_foobar(): + pass + + @pytest.mark.bar + def test_bar(): + pass + + def test_pass(): + pass + """ + ) + result = pytester.runpytest("-m", "not foo") + result.stdout.fnmatch_lines( + [ + "collected 3 items / 1 deselected / 2 selected", + "*test_show_deselected.py ..*", + "*= 2 passed, 1 deselected in * =*", + ] + ) + result.stdout.no_fnmatch_line("*= 1 deselected =*") + assert result.ret == 0 + + def test_no_skip_summary_if_failure(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + def test_ok(): + pass + def test_fail(): + assert 0 + def test_skip(): + pytest.skip("dontshow") + """ + ) + result = pytester.runpytest() + assert result.stdout.str().find("skip test summary") == -1 + assert result.ret == 1 + + def test_passes(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def test_passes(): + pass + class TestClass(object): + def test_method(self): + pass + """ + ) + old = p1.parent + pytester.chdir() + try: + result = pytester.runpytest() + finally: + os.chdir(old) + result.stdout.fnmatch_lines(["test_passes.py ..*", "* 2 pass*"]) + assert result.ret == 0 + + def test_header_trailer_info( + self, monkeypatch: MonkeyPatch, pytester: Pytester, request + ) -> None: + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + pytester.makepyfile( + """ + def test_passes(): + pass + """ + ) + result = pytester.runpytest() + verinfo = ".".join(map(str, sys.version_info[:3])) + result.stdout.fnmatch_lines( + [ + "*===== test session starts ====*", + "platform %s -- Python %s*pytest-%s**pluggy-%s" + % ( + sys.platform, + verinfo, + pytest.__version__, + pluggy.__version__, + ), + "*test_header_trailer_info.py .*", + "=* 1 passed*in *.[0-9][0-9]s *=", + ] + ) + if request.config.pluginmanager.list_plugin_distinfo(): + result.stdout.fnmatch_lines(["plugins: *"]) + + def test_no_header_trailer_info( + self, monkeypatch: MonkeyPatch, pytester: Pytester, request + ) -> None: + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + pytester.makepyfile( + """ + def test_passes(): + pass + """ + ) + result = pytester.runpytest("--no-header") + verinfo = ".".join(map(str, sys.version_info[:3])) + result.stdout.no_fnmatch_line( + "platform %s -- Python %s*pytest-%s**pluggy-%s" + % ( + sys.platform, + verinfo, + pytest.__version__, + pluggy.__version__, + ) + ) + if request.config.pluginmanager.list_plugin_distinfo(): + result.stdout.no_fnmatch_line("plugins: *") + + def test_header(self, pytester: Pytester) -> None: + pytester.path.joinpath("tests").mkdir() + pytester.path.joinpath("gui").mkdir() + + # no ini file + result = pytester.runpytest() + result.stdout.fnmatch_lines(["rootdir: *test_header0"]) + + # with configfile + pytester.makeini("""[pytest]""") + result = pytester.runpytest() + result.stdout.fnmatch_lines(["rootdir: *test_header0, configfile: tox.ini"]) + + # with testpaths option, and not passing anything in the command-line + pytester.makeini( + """ + [pytest] + testpaths = tests gui + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + ["rootdir: *test_header0, configfile: tox.ini, testpaths: tests, gui"] + ) + + # with testpaths option, passing directory in command-line: do not show testpaths then + result = pytester.runpytest("tests") + result.stdout.fnmatch_lines(["rootdir: *test_header0, configfile: tox.ini"]) + + def test_header_absolute_testpath( + self, pytester: Pytester, monkeypatch: MonkeyPatch + ) -> None: + """Regresstion test for #7814.""" + tests = pytester.path.joinpath("tests") + tests.mkdir() + pytester.makepyprojecttoml( + """ + [tool.pytest.ini_options] + testpaths = ['{}'] + """.format( + tests + ) + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "rootdir: *absolute_testpath0, configfile: pyproject.toml, testpaths: {}".format( + tests + ) + ] + ) + + def test_no_header(self, pytester: Pytester) -> None: + pytester.path.joinpath("tests").mkdir() + pytester.path.joinpath("gui").mkdir() + + # with testpaths option, and not passing anything in the command-line + pytester.makeini( + """ + [pytest] + testpaths = tests gui + """ + ) + result = pytester.runpytest("--no-header") + result.stdout.no_fnmatch_line( + "rootdir: *test_header0, inifile: tox.ini, testpaths: tests, gui" + ) + + # with testpaths option, passing directory in command-line: do not show testpaths then + result = pytester.runpytest("tests", "--no-header") + result.stdout.no_fnmatch_line("rootdir: *test_header0, inifile: tox.ini") + + def test_no_summary(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def test_no_summary(): + assert false + """ + ) + result = pytester.runpytest(p1, "--no-summary") + result.stdout.no_fnmatch_line("*= FAILURES =*") + + def test_showlocals(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def test_showlocals(): + x = 3 + y = "x" * 5000 + assert 0 + """ + ) + result = pytester.runpytest(p1, "-l") + result.stdout.fnmatch_lines( + [ + # "_ _ * Locals *", + "x* = 3", + "y* = 'xxxxxx*", + ] + ) + + def test_showlocals_short(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def test_showlocals_short(): + x = 3 + y = "xxxx" + assert 0 + """ + ) + result = pytester.runpytest(p1, "-l", "--tb=short") + result.stdout.fnmatch_lines( + [ + "test_showlocals_short.py:*", + " assert 0", + "E assert 0", + " x = 3", + " y = 'xxxx'", + ] + ) + + @pytest.fixture + def verbose_testfile(self, pytester: Pytester) -> Path: + return pytester.makepyfile( + """ + import pytest + def test_fail(): + raise ValueError() + def test_pass(): + pass + class TestClass(object): + def test_skip(self): + pytest.skip("hello") + def test_gen(): + def check(x): + assert x == 1 + yield check, 0 + """ + ) + + def test_verbose_reporting(self, verbose_testfile, pytester: Pytester) -> None: + result = pytester.runpytest( + verbose_testfile, "-v", "-Walways::pytest.PytestWarning" + ) + result.stdout.fnmatch_lines( + [ + "*test_verbose_reporting.py::test_fail *FAIL*", + "*test_verbose_reporting.py::test_pass *PASS*", + "*test_verbose_reporting.py::TestClass::test_skip *SKIP*", + "*test_verbose_reporting.py::test_gen *XFAIL*", + ] + ) + assert result.ret == 1 + + def test_verbose_reporting_xdist( + self, + verbose_testfile, + monkeypatch: MonkeyPatch, + pytester: Pytester, + pytestconfig, + ) -> None: + if not pytestconfig.pluginmanager.get_plugin("xdist"): + pytest.skip("xdist plugin not installed") + + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + result = pytester.runpytest( + verbose_testfile, "-v", "-n 1", "-Walways::pytest.PytestWarning" + ) + result.stdout.fnmatch_lines( + ["*FAIL*test_verbose_reporting_xdist.py::test_fail*"] + ) + assert result.ret == 1 + + def test_quiet_reporting(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile("def test_pass(): pass") + result = pytester.runpytest(p1, "-q") + s = result.stdout.str() + assert "test session starts" not in s + assert p1.name not in s + assert "===" not in s + assert "passed" in s + + def test_more_quiet_reporting(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile("def test_pass(): pass") + result = pytester.runpytest(p1, "-qq") + s = result.stdout.str() + assert "test session starts" not in s + assert p1.name not in s + assert "===" not in s + assert "passed" not in s + + @pytest.mark.parametrize( + "params", [(), ("--collect-only",)], ids=["no-params", "collect-only"] + ) + def test_report_collectionfinish_hook(self, pytester: Pytester, params) -> None: + pytester.makeconftest( + """ + def pytest_report_collectionfinish(config, start_path, items): + return [f'hello from hook: {len(items)} items'] + """ + ) + pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize('i', range(3)) + def test(i): + pass + """ + ) + result = pytester.runpytest(*params) + result.stdout.fnmatch_lines(["collected 3 items", "hello from hook: 3 items"]) + + def test_summary_f_alias(self, pytester: Pytester) -> None: + """Test that 'f' and 'F' report chars are aliases and don't show up twice in the summary (#6334)""" + pytester.makepyfile( + """ + def test(): + assert False + """ + ) + result = pytester.runpytest("-rfF") + expected = "FAILED test_summary_f_alias.py::test - assert False" + result.stdout.fnmatch_lines([expected]) + assert result.stdout.lines.count(expected) == 1 + + def test_summary_s_alias(self, pytester: Pytester) -> None: + """Test that 's' and 'S' report chars are aliases and don't show up twice in the summary""" + pytester.makepyfile( + """ + import pytest + + @pytest.mark.skip + def test(): + pass + """ + ) + result = pytester.runpytest("-rsS") + expected = "SKIPPED [1] test_summary_s_alias.py:3: unconditional skip" + result.stdout.fnmatch_lines([expected]) + assert result.stdout.lines.count(expected) == 1 + + +def test_fail_extra_reporting(pytester: Pytester, monkeypatch) -> None: + monkeypatch.setenv("COLUMNS", "80") + pytester.makepyfile("def test_this(): assert 0, 'this_failed' * 100") + result = pytester.runpytest("-rN") + result.stdout.no_fnmatch_line("*short test summary*") + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*test summary*", + "FAILED test_fail_extra_reporting.py::test_this - AssertionError: this_failedt...", + ] + ) + + +def test_fail_reporting_on_pass(pytester: Pytester) -> None: + pytester.makepyfile("def test_this(): assert 1") + result = pytester.runpytest("-rf") + result.stdout.no_fnmatch_line("*short test summary*") + + +def test_pass_extra_reporting(pytester: Pytester) -> None: + pytester.makepyfile("def test_this(): assert 1") + result = pytester.runpytest() + result.stdout.no_fnmatch_line("*short test summary*") + result = pytester.runpytest("-rp") + result.stdout.fnmatch_lines(["*test summary*", "PASS*test_pass_extra_reporting*"]) + + +def test_pass_reporting_on_fail(pytester: Pytester) -> None: + pytester.makepyfile("def test_this(): assert 0") + result = pytester.runpytest("-rp") + result.stdout.no_fnmatch_line("*short test summary*") + + +def test_pass_output_reporting(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def setup_module(): + print("setup_module") + + def teardown_module(): + print("teardown_module") + + def test_pass_has_output(): + print("Four score and seven years ago...") + + def test_pass_no_output(): + pass + """ + ) + result = pytester.runpytest() + s = result.stdout.str() + assert "test_pass_has_output" not in s + assert "Four score and seven years ago..." not in s + assert "test_pass_no_output" not in s + result = pytester.runpytest("-rPp") + result.stdout.fnmatch_lines( + [ + "*= PASSES =*", + "*_ test_pass_has_output _*", + "*- Captured stdout setup -*", + "setup_module", + "*- Captured stdout call -*", + "Four score and seven years ago...", + "*- Captured stdout teardown -*", + "teardown_module", + "*= short test summary info =*", + "PASSED test_pass_output_reporting.py::test_pass_has_output", + "PASSED test_pass_output_reporting.py::test_pass_no_output", + "*= 2 passed in *", + ] + ) + + +def test_color_yes(pytester: Pytester, color_mapping) -> None: + p1 = pytester.makepyfile( + """ + def fail(): + assert 0 + + def test_this(): + fail() + """ + ) + result = pytester.runpytest("--color=yes", str(p1)) + result.stdout.fnmatch_lines( + color_mapping.format_for_fnmatch( + [ + "{bold}=*= test session starts =*={reset}", + "collected 1 item", + "", + "test_color_yes.py {red}F{reset}{red} * [100%]{reset}", + "", + "=*= FAILURES =*=", + "{red}{bold}_*_ test_this _*_{reset}", + "", + " {kw}def{hl-reset} {function}test_this{hl-reset}():", + "> fail()", + "", + "{bold}{red}test_color_yes.py{reset}:5: ", + "_ _ * _ _*", + "", + " {kw}def{hl-reset} {function}fail{hl-reset}():", + "> {kw}assert{hl-reset} {number}0{hl-reset}", + "{bold}{red}E assert 0{reset}", + "", + "{bold}{red}test_color_yes.py{reset}:2: AssertionError", + "{red}=*= {red}{bold}1 failed{reset}{red} in *s{reset}{red} =*={reset}", + ] + ) + ) + result = pytester.runpytest("--color=yes", "--tb=short", str(p1)) + result.stdout.fnmatch_lines( + color_mapping.format_for_fnmatch( + [ + "{bold}=*= test session starts =*={reset}", + "collected 1 item", + "", + "test_color_yes.py {red}F{reset}{red} * [100%]{reset}", + "", + "=*= FAILURES =*=", + "{red}{bold}_*_ test_this _*_{reset}", + "{bold}{red}test_color_yes.py{reset}:5: in test_this", + " fail()", + "{bold}{red}test_color_yes.py{reset}:2: in fail", + " {kw}assert{hl-reset} {number}0{hl-reset}", + "{bold}{red}E assert 0{reset}", + "{red}=*= {red}{bold}1 failed{reset}{red} in *s{reset}{red} =*={reset}", + ] + ) + ) + + +def test_color_no(pytester: Pytester) -> None: + pytester.makepyfile("def test_this(): assert 1") + result = pytester.runpytest("--color=no") + assert "test session starts" in result.stdout.str() + result.stdout.no_fnmatch_line("*\x1b[1m*") + + +@pytest.mark.parametrize("verbose", [True, False]) +def test_color_yes_collection_on_non_atty(pytester: Pytester, verbose) -> None: + """#1397: Skip collect progress report when working on non-terminals.""" + pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize('i', range(10)) + def test_this(i): + assert 1 + """ + ) + args = ["--color=yes"] + if verbose: + args.append("-vv") + result = pytester.runpytest(*args) + assert "test session starts" in result.stdout.str() + assert "\x1b[1m" in result.stdout.str() + result.stdout.no_fnmatch_line("*collecting 10 items*") + if verbose: + assert "collecting ..." in result.stdout.str() + assert "collected 10 items" in result.stdout.str() + + +def test_getreportopt() -> None: + from _pytest.terminal import _REPORTCHARS_DEFAULT + + class FakeConfig: + class Option: + reportchars = _REPORTCHARS_DEFAULT + disable_warnings = False + + option = Option() + + config = cast(Config, FakeConfig()) + + assert _REPORTCHARS_DEFAULT == "fE" + + # Default. + assert getreportopt(config) == "wfE" + + config.option.reportchars = "sf" + assert getreportopt(config) == "wsf" + + config.option.reportchars = "sfxw" + assert getreportopt(config) == "sfxw" + + config.option.reportchars = "a" + assert getreportopt(config) == "wsxXEf" + + config.option.reportchars = "N" + assert getreportopt(config) == "w" + + config.option.reportchars = "NwfE" + assert getreportopt(config) == "wfE" + + config.option.reportchars = "NfENx" + assert getreportopt(config) == "wx" + + # Now with --disable-warnings. + config.option.disable_warnings = True + config.option.reportchars = "a" + assert getreportopt(config) == "sxXEf" + + config.option.reportchars = "sfx" + assert getreportopt(config) == "sfx" + + config.option.reportchars = "sfxw" + assert getreportopt(config) == "sfx" + + config.option.reportchars = "a" + assert getreportopt(config) == "sxXEf" + + config.option.reportchars = "A" + assert getreportopt(config) == "PpsxXEf" + + config.option.reportchars = "AN" + assert getreportopt(config) == "" + + config.option.reportchars = "NwfE" + assert getreportopt(config) == "fE" + + +def test_terminalreporter_reportopt_addopts(pytester: Pytester) -> None: + pytester.makeini("[pytest]\naddopts=-rs") + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def tr(request): + tr = request.config.pluginmanager.getplugin("terminalreporter") + return tr + def test_opt(tr): + assert tr.hasopt('skipped') + assert not tr.hasopt('qwe') + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_tbstyle_short(pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def arg(request): + return 42 + def test_opt(arg): + x = 0 + assert x + """ + ) + result = pytester.runpytest("--tb=short") + s = result.stdout.str() + assert "arg = 42" not in s + assert "x = 0" not in s + result.stdout.fnmatch_lines(["*%s:8*" % p.name, " assert x", "E assert*"]) + result = pytester.runpytest() + s = result.stdout.str() + assert "x = 0" in s + assert "assert x" in s + + +def test_traceconfig(pytester: Pytester) -> None: + result = pytester.runpytest("--traceconfig") + result.stdout.fnmatch_lines(["*active plugins*"]) + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +class TestGenericReporting: + """Test class which can be subclassed with a different option provider to + run e.g. distributed tests.""" + + def test_collect_fail(self, pytester: Pytester, option) -> None: + pytester.makepyfile("import xyz\n") + result = pytester.runpytest(*option.args) + result.stdout.fnmatch_lines( + ["ImportError while importing*", "*No module named *xyz*", "*1 error*"] + ) + + def test_maxfailures(self, pytester: Pytester, option) -> None: + pytester.makepyfile( + """ + def test_1(): + assert 0 + def test_2(): + assert 0 + def test_3(): + assert 0 + """ + ) + result = pytester.runpytest("--maxfail=2", *option.args) + result.stdout.fnmatch_lines( + [ + "*def test_1():*", + "*def test_2():*", + "*! stopping after 2 failures !*", + "*2 failed*", + ] + ) + + def test_maxfailures_with_interrupted(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test(request): + request.session.shouldstop = "session_interrupted" + assert 0 + """ + ) + result = pytester.runpytest("--maxfail=1", "-ra") + result.stdout.fnmatch_lines( + [ + "*= short test summary info =*", + "FAILED *", + "*! stopping after 1 failures !*", + "*! session_interrupted !*", + "*= 1 failed in*", + ] + ) + + def test_tb_option(self, pytester: Pytester, option) -> None: + pytester.makepyfile( + """ + import pytest + def g(): + raise IndexError + def test_func(): + print(6*7) + g() # --calling-- + """ + ) + for tbopt in ["long", "short", "no"]: + print("testing --tb=%s..." % tbopt) + result = pytester.runpytest("-rN", "--tb=%s" % tbopt) + s = result.stdout.str() + if tbopt == "long": + assert "print(6*7)" in s + else: + assert "print(6*7)" not in s + if tbopt != "no": + assert "--calling--" in s + assert "IndexError" in s + else: + assert "FAILURES" not in s + assert "--calling--" not in s + assert "IndexError" not in s + + def test_tb_crashline(self, pytester: Pytester, option) -> None: + p = pytester.makepyfile( + """ + import pytest + def g(): + raise IndexError + def test_func1(): + print(6*7) + g() # --calling-- + def test_func2(): + assert 0, "hello" + """ + ) + result = pytester.runpytest("--tb=line") + bn = p.name + result.stdout.fnmatch_lines( + ["*%s:3: IndexError*" % bn, "*%s:8: AssertionError: hello*" % bn] + ) + s = result.stdout.str() + assert "def test_func2" not in s + + def test_pytest_report_header(self, pytester: Pytester, option) -> None: + pytester.makeconftest( + """ + def pytest_sessionstart(session): + session.config._somevalue = 42 + def pytest_report_header(config): + return "hello: %s" % config._somevalue + """ + ) + pytester.mkdir("a").joinpath("conftest.py").write_text( + """ +def pytest_report_header(config, start_path): + return ["line1", str(start_path)] +""" + ) + result = pytester.runpytest("a") + result.stdout.fnmatch_lines(["*hello: 42*", "line1", str(pytester.path)]) + + def test_show_capture(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import sys + import logging + def test_one(): + sys.stdout.write('!This is stdout!') + sys.stderr.write('!This is stderr!') + logging.warning('!This is a warning log msg!') + assert False, 'Something failed' + """ + ) + + result = pytester.runpytest("--tb=short") + result.stdout.fnmatch_lines( + [ + "!This is stdout!", + "!This is stderr!", + "*WARNING*!This is a warning log msg!", + ] + ) + + result = pytester.runpytest("--show-capture=all", "--tb=short") + result.stdout.fnmatch_lines( + [ + "!This is stdout!", + "!This is stderr!", + "*WARNING*!This is a warning log msg!", + ] + ) + + stdout = pytester.runpytest("--show-capture=stdout", "--tb=short").stdout.str() + assert "!This is stderr!" not in stdout + assert "!This is stdout!" in stdout + assert "!This is a warning log msg!" not in stdout + + stdout = pytester.runpytest("--show-capture=stderr", "--tb=short").stdout.str() + assert "!This is stdout!" not in stdout + assert "!This is stderr!" in stdout + assert "!This is a warning log msg!" not in stdout + + stdout = pytester.runpytest("--show-capture=log", "--tb=short").stdout.str() + assert "!This is stdout!" not in stdout + assert "!This is stderr!" not in stdout + assert "!This is a warning log msg!" in stdout + + stdout = pytester.runpytest("--show-capture=no", "--tb=short").stdout.str() + assert "!This is stdout!" not in stdout + assert "!This is stderr!" not in stdout + assert "!This is a warning log msg!" not in stdout + + def test_show_capture_with_teardown_logs(self, pytester: Pytester) -> None: + """Ensure that the capturing of teardown logs honor --show-capture setting""" + pytester.makepyfile( + """ + import logging + import sys + import pytest + + @pytest.fixture(scope="function", autouse="True") + def hook_each_test(request): + yield + sys.stdout.write("!stdout!") + sys.stderr.write("!stderr!") + logging.warning("!log!") + + def test_func(): + assert False + """ + ) + + result = pytester.runpytest("--show-capture=stdout", "--tb=short").stdout.str() + assert "!stdout!" in result + assert "!stderr!" not in result + assert "!log!" not in result + + result = pytester.runpytest("--show-capture=stderr", "--tb=short").stdout.str() + assert "!stdout!" not in result + assert "!stderr!" in result + assert "!log!" not in result + + result = pytester.runpytest("--show-capture=log", "--tb=short").stdout.str() + assert "!stdout!" not in result + assert "!stderr!" not in result + assert "!log!" in result + + result = pytester.runpytest("--show-capture=no", "--tb=short").stdout.str() + assert "!stdout!" not in result + assert "!stderr!" not in result + assert "!log!" not in result + + +@pytest.mark.xfail("not hasattr(os, 'dup')") +def test_fdopen_kept_alive_issue124(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import os, sys + k = [] + def test_open_file_and_keep_alive(capfd): + stdout = os.fdopen(1, 'w', 1) + k.append(stdout) + + def test_close_kept_alive_file(): + stdout = k.pop() + stdout.close() + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*2 passed*"]) + + +def test_tbstyle_native_setup_error(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture + def setup_error_fixture(): + raise Exception("error in exception") + + def test_error_fixture(setup_error_fixture): + pass + """ + ) + result = pytester.runpytest("--tb=native") + result.stdout.fnmatch_lines( + ['*File *test_tbstyle_native_setup_error.py", line *, in setup_error_fixture*'] + ) + + +def test_terminal_summary(pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_terminal_summary(terminalreporter, exitstatus): + w = terminalreporter + w.section("hello") + w.line("world") + w.line("exitstatus: {0}".format(exitstatus)) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + """ + *==== hello ====* + world + exitstatus: 5 + """ + ) + + +@pytest.mark.filterwarnings("default::UserWarning") +def test_terminal_summary_warnings_are_displayed(pytester: Pytester) -> None: + """Test that warnings emitted during pytest_terminal_summary are displayed. + (#1305). + """ + pytester.makeconftest( + """ + import warnings + def pytest_terminal_summary(terminalreporter): + warnings.warn(UserWarning('internal warning')) + """ + ) + pytester.makepyfile( + """ + def test_failure(): + import warnings + warnings.warn("warning_from_" + "test") + assert 0 + """ + ) + result = pytester.runpytest("-ra") + result.stdout.fnmatch_lines( + [ + "*= warnings summary =*", + "*warning_from_test*", + "*= short test summary info =*", + "*= warnings summary (final) =*", + "*conftest.py:3:*internal warning", + "*== 1 failed, 2 warnings in *", + ] + ) + result.stdout.no_fnmatch_line("*None*") + stdout = result.stdout.str() + assert stdout.count("warning_from_test") == 1 + assert stdout.count("=== warnings summary ") == 2 + + +@pytest.mark.filterwarnings("default::UserWarning") +def test_terminal_summary_warnings_header_once(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_failure(): + import warnings + warnings.warn("warning_from_" + "test") + assert 0 + """ + ) + result = pytester.runpytest("-ra") + result.stdout.fnmatch_lines( + [ + "*= warnings summary =*", + "*warning_from_test*", + "*= short test summary info =*", + "*== 1 failed, 1 warning in *", + ] + ) + result.stdout.no_fnmatch_line("*None*") + stdout = result.stdout.str() + assert stdout.count("warning_from_test") == 1 + assert stdout.count("=== warnings summary ") == 1 + + +@pytest.mark.filterwarnings("default") +def test_terminal_no_summary_warnings_header_once(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_failure(): + import warnings + warnings.warn("warning_from_" + "test") + assert 0 + """ + ) + result = pytester.runpytest("--no-summary") + result.stdout.no_fnmatch_line("*= warnings summary =*") + result.stdout.no_fnmatch_line("*= short test summary info =*") + + +@pytest.fixture(scope="session") +def tr() -> TerminalReporter: + config = _pytest.config._prepareconfig() + return TerminalReporter(config) + + +@pytest.mark.parametrize( + "exp_color, exp_line, stats_arg", + [ + # The method under test only cares about the length of each + # dict value, not the actual contents, so tuples of anything + # suffice + # Important statuses -- the highest priority of these always wins + ("red", [("1 failed", {"bold": True, "red": True})], {"failed": [1]}), + ( + "red", + [ + ("1 failed", {"bold": True, "red": True}), + ("1 passed", {"bold": False, "green": True}), + ], + {"failed": [1], "passed": [1]}, + ), + ("red", [("1 error", {"bold": True, "red": True})], {"error": [1]}), + ("red", [("2 errors", {"bold": True, "red": True})], {"error": [1, 2]}), + ( + "red", + [ + ("1 passed", {"bold": False, "green": True}), + ("1 error", {"bold": True, "red": True}), + ], + {"error": [1], "passed": [1]}, + ), + # (a status that's not known to the code) + ("yellow", [("1 weird", {"bold": True, "yellow": True})], {"weird": [1]}), + ( + "yellow", + [ + ("1 passed", {"bold": False, "green": True}), + ("1 weird", {"bold": True, "yellow": True}), + ], + {"weird": [1], "passed": [1]}, + ), + ("yellow", [("1 warning", {"bold": True, "yellow": True})], {"warnings": [1]}), + ( + "yellow", + [ + ("1 passed", {"bold": False, "green": True}), + ("1 warning", {"bold": True, "yellow": True}), + ], + {"warnings": [1], "passed": [1]}, + ), + ( + "green", + [("5 passed", {"bold": True, "green": True})], + {"passed": [1, 2, 3, 4, 5]}, + ), + # "Boring" statuses. These have no effect on the color of the summary + # line. Thus, if *every* test has a boring status, the summary line stays + # at its default color, i.e. yellow, to warn the user that the test run + # produced no useful information + ("yellow", [("1 skipped", {"bold": True, "yellow": True})], {"skipped": [1]}), + ( + "green", + [ + ("1 passed", {"bold": True, "green": True}), + ("1 skipped", {"bold": False, "yellow": True}), + ], + {"skipped": [1], "passed": [1]}, + ), + ( + "yellow", + [("1 deselected", {"bold": True, "yellow": True})], + {"deselected": [1]}, + ), + ( + "green", + [ + ("1 passed", {"bold": True, "green": True}), + ("1 deselected", {"bold": False, "yellow": True}), + ], + {"deselected": [1], "passed": [1]}, + ), + ("yellow", [("1 xfailed", {"bold": True, "yellow": True})], {"xfailed": [1]}), + ( + "green", + [ + ("1 passed", {"bold": True, "green": True}), + ("1 xfailed", {"bold": False, "yellow": True}), + ], + {"xfailed": [1], "passed": [1]}, + ), + ("yellow", [("1 xpassed", {"bold": True, "yellow": True})], {"xpassed": [1]}), + ( + "yellow", + [ + ("1 passed", {"bold": False, "green": True}), + ("1 xpassed", {"bold": True, "yellow": True}), + ], + {"xpassed": [1], "passed": [1]}, + ), + # Likewise if no tests were found at all + ("yellow", [("no tests ran", {"yellow": True})], {}), + # Test the empty-key special case + ("yellow", [("no tests ran", {"yellow": True})], {"": [1]}), + ( + "green", + [("1 passed", {"bold": True, "green": True})], + {"": [1], "passed": [1]}, + ), + # A couple more complex combinations + ( + "red", + [ + ("1 failed", {"bold": True, "red": True}), + ("2 passed", {"bold": False, "green": True}), + ("3 xfailed", {"bold": False, "yellow": True}), + ], + {"passed": [1, 2], "failed": [1], "xfailed": [1, 2, 3]}, + ), + ( + "green", + [ + ("1 passed", {"bold": True, "green": True}), + ("2 skipped", {"bold": False, "yellow": True}), + ("3 deselected", {"bold": False, "yellow": True}), + ("2 xfailed", {"bold": False, "yellow": True}), + ], + { + "passed": [1], + "skipped": [1, 2], + "deselected": [1, 2, 3], + "xfailed": [1, 2], + }, + ), + ], +) +def test_summary_stats( + tr: TerminalReporter, + exp_line: List[Tuple[str, Dict[str, bool]]], + exp_color: str, + stats_arg: Dict[str, List[object]], +) -> None: + tr.stats = stats_arg + + # Fake "_is_last_item" to be True. + class fake_session: + testscollected = 0 + + tr._session = fake_session # type: ignore[assignment] + assert tr._is_last_item + + # Reset cache. + tr._main_color = None + + print("Based on stats: %s" % stats_arg) + print(f'Expect summary: "{exp_line}"; with color "{exp_color}"') + (line, color) = tr.build_summary_stats_line() + print(f'Actually got: "{line}"; with color "{color}"') + assert line == exp_line + assert color == exp_color + + +def test_skip_counting_towards_summary(tr): + class DummyReport(BaseReport): + count_towards_summary = True + + r1 = DummyReport() + r2 = DummyReport() + tr.stats = {"failed": (r1, r2)} + tr._main_color = None + res = tr.build_summary_stats_line() + assert res == ([("2 failed", {"bold": True, "red": True})], "red") + + r1.count_towards_summary = False + tr.stats = {"failed": (r1, r2)} + tr._main_color = None + res = tr.build_summary_stats_line() + assert res == ([("1 failed", {"bold": True, "red": True})], "red") + + +class TestClassicOutputStyle: + """Ensure classic output style works as expected (#3883)""" + + @pytest.fixture + def test_files(self, pytester: Pytester) -> None: + pytester.makepyfile( + **{ + "test_one.py": "def test_one(): pass", + "test_two.py": "def test_two(): assert 0", + "sub/test_three.py": """ + def test_three_1(): pass + def test_three_2(): assert 0 + def test_three_3(): pass + """, + } + ) + + def test_normal_verbosity(self, pytester: Pytester, test_files) -> None: + result = pytester.runpytest("-o", "console_output_style=classic") + result.stdout.fnmatch_lines( + [ + "test_one.py .", + "test_two.py F", + f"sub{os.sep}test_three.py .F.", + "*2 failed, 3 passed in*", + ] + ) + + def test_verbose(self, pytester: Pytester, test_files) -> None: + result = pytester.runpytest("-o", "console_output_style=classic", "-v") + result.stdout.fnmatch_lines( + [ + "test_one.py::test_one PASSED", + "test_two.py::test_two FAILED", + f"sub{os.sep}test_three.py::test_three_1 PASSED", + f"sub{os.sep}test_three.py::test_three_2 FAILED", + f"sub{os.sep}test_three.py::test_three_3 PASSED", + "*2 failed, 3 passed in*", + ] + ) + + def test_quiet(self, pytester: Pytester, test_files) -> None: + result = pytester.runpytest("-o", "console_output_style=classic", "-q") + result.stdout.fnmatch_lines([".F.F.", "*2 failed, 3 passed in*"]) + + +class TestProgressOutputStyle: + @pytest.fixture + def many_tests_files(self, pytester: Pytester) -> None: + pytester.makepyfile( + test_bar=""" + import pytest + @pytest.mark.parametrize('i', range(10)) + def test_bar(i): pass + """, + test_foo=""" + import pytest + @pytest.mark.parametrize('i', range(5)) + def test_foo(i): pass + """, + test_foobar=""" + import pytest + @pytest.mark.parametrize('i', range(5)) + def test_foobar(i): pass + """, + ) + + def test_zero_tests_collected(self, pytester: Pytester) -> None: + """Some plugins (testmon for example) might issue pytest_runtest_logreport without any tests being + actually collected (#2971).""" + pytester.makeconftest( + """ + def pytest_collection_modifyitems(items, config): + from _pytest.runner import CollectReport + for node_id in ('nodeid1', 'nodeid2'): + rep = CollectReport(node_id, 'passed', None, None) + rep.when = 'passed' + rep.duration = 0.1 + config.hook.pytest_runtest_logreport(report=rep) + """ + ) + output = pytester.runpytest() + output.stdout.no_fnmatch_line("*ZeroDivisionError*") + output.stdout.fnmatch_lines(["=* 2 passed in *="]) + + def test_normal(self, many_tests_files, pytester: Pytester) -> None: + output = pytester.runpytest() + output.stdout.re_match_lines( + [ + r"test_bar.py \.{10} \s+ \[ 50%\]", + r"test_foo.py \.{5} \s+ \[ 75%\]", + r"test_foobar.py \.{5} \s+ \[100%\]", + ] + ) + + def test_colored_progress( + self, pytester: Pytester, monkeypatch, color_mapping + ) -> None: + monkeypatch.setenv("PY_COLORS", "1") + pytester.makepyfile( + test_axfail=""" + import pytest + @pytest.mark.xfail + def test_axfail(): assert 0 + """, + test_bar=""" + import pytest + @pytest.mark.parametrize('i', range(10)) + def test_bar(i): pass + """, + test_foo=""" + import pytest + import warnings + @pytest.mark.parametrize('i', range(5)) + def test_foo(i): + warnings.warn(DeprecationWarning("collection")) + pass + """, + test_foobar=""" + import pytest + @pytest.mark.parametrize('i', range(5)) + def test_foobar(i): raise ValueError() + """, + ) + result = pytester.runpytest() + result.stdout.re_match_lines( + color_mapping.format_for_rematch( + [ + r"test_axfail.py {yellow}x{reset}{green} \s+ \[ 4%\]{reset}", + r"test_bar.py ({green}\.{reset}){{10}}{green} \s+ \[ 52%\]{reset}", + r"test_foo.py ({green}\.{reset}){{5}}{yellow} \s+ \[ 76%\]{reset}", + r"test_foobar.py ({red}F{reset}){{5}}{red} \s+ \[100%\]{reset}", + ] + ) + ) + + # Only xfail should have yellow progress indicator. + result = pytester.runpytest("test_axfail.py") + result.stdout.re_match_lines( + color_mapping.format_for_rematch( + [ + r"test_axfail.py {yellow}x{reset}{yellow} \s+ \[100%\]{reset}", + r"^{yellow}=+ ({yellow}{bold}|{bold}{yellow})1 xfailed{reset}{yellow} in ", + ] + ) + ) + + def test_count(self, many_tests_files, pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + console_output_style = count + """ + ) + output = pytester.runpytest() + output.stdout.re_match_lines( + [ + r"test_bar.py \.{10} \s+ \[10/20\]", + r"test_foo.py \.{5} \s+ \[15/20\]", + r"test_foobar.py \.{5} \s+ \[20/20\]", + ] + ) + + def test_verbose(self, many_tests_files, pytester: Pytester) -> None: + output = pytester.runpytest("-v") + output.stdout.re_match_lines( + [ + r"test_bar.py::test_bar\[0\] PASSED \s+ \[ 5%\]", + r"test_foo.py::test_foo\[4\] PASSED \s+ \[ 75%\]", + r"test_foobar.py::test_foobar\[4\] PASSED \s+ \[100%\]", + ] + ) + + def test_verbose_count(self, many_tests_files, pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + console_output_style = count + """ + ) + output = pytester.runpytest("-v") + output.stdout.re_match_lines( + [ + r"test_bar.py::test_bar\[0\] PASSED \s+ \[ 1/20\]", + r"test_foo.py::test_foo\[4\] PASSED \s+ \[15/20\]", + r"test_foobar.py::test_foobar\[4\] PASSED \s+ \[20/20\]", + ] + ) + + def test_xdist_normal( + self, many_tests_files, pytester: Pytester, monkeypatch + ) -> None: + pytest.importorskip("xdist") + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + output = pytester.runpytest("-n2") + output.stdout.re_match_lines([r"\.{20} \s+ \[100%\]"]) + + def test_xdist_normal_count( + self, many_tests_files, pytester: Pytester, monkeypatch + ) -> None: + pytest.importorskip("xdist") + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + pytester.makeini( + """ + [pytest] + console_output_style = count + """ + ) + output = pytester.runpytest("-n2") + output.stdout.re_match_lines([r"\.{20} \s+ \[20/20\]"]) + + def test_xdist_verbose( + self, many_tests_files, pytester: Pytester, monkeypatch + ) -> None: + pytest.importorskip("xdist") + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + output = pytester.runpytest("-n2", "-v") + output.stdout.re_match_lines_random( + [ + r"\[gw\d\] \[\s*\d+%\] PASSED test_bar.py::test_bar\[1\]", + r"\[gw\d\] \[\s*\d+%\] PASSED test_foo.py::test_foo\[1\]", + r"\[gw\d\] \[\s*\d+%\] PASSED test_foobar.py::test_foobar\[1\]", + ] + ) + output.stdout.fnmatch_lines_random( + [ + line.translate(TRANS_FNMATCH) + for line in [ + "test_bar.py::test_bar[0] ", + "test_foo.py::test_foo[0] ", + "test_foobar.py::test_foobar[0] ", + "[gw?] [ 5%] PASSED test_*[?] ", + "[gw?] [ 10%] PASSED test_*[?] ", + "[gw?] [ 55%] PASSED test_*[?] ", + "[gw?] [ 60%] PASSED test_*[?] ", + "[gw?] [ 95%] PASSED test_*[?] ", + "[gw?] [100%] PASSED test_*[?] ", + ] + ] + ) + + def test_capture_no(self, many_tests_files, pytester: Pytester) -> None: + output = pytester.runpytest("-s") + output.stdout.re_match_lines( + [r"test_bar.py \.{10}", r"test_foo.py \.{5}", r"test_foobar.py \.{5}"] + ) + + output = pytester.runpytest("--capture=no") + output.stdout.no_fnmatch_line("*%]*") + + +class TestProgressWithTeardown: + """Ensure we show the correct percentages for tests that fail during teardown (#3088)""" + + @pytest.fixture + def contest_with_teardown_fixture(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + + @pytest.fixture + def fail_teardown(): + yield + assert False + """ + ) + + @pytest.fixture + def many_files(self, pytester: Pytester, contest_with_teardown_fixture) -> None: + pytester.makepyfile( + test_bar=""" + import pytest + @pytest.mark.parametrize('i', range(5)) + def test_bar(fail_teardown, i): + pass + """, + test_foo=""" + import pytest + @pytest.mark.parametrize('i', range(15)) + def test_foo(fail_teardown, i): + pass + """, + ) + + def test_teardown_simple( + self, pytester: Pytester, contest_with_teardown_fixture + ) -> None: + pytester.makepyfile( + """ + def test_foo(fail_teardown): + pass + """ + ) + output = pytester.runpytest() + output.stdout.re_match_lines([r"test_teardown_simple.py \.E\s+\[100%\]"]) + + def test_teardown_with_test_also_failing( + self, pytester: Pytester, contest_with_teardown_fixture + ) -> None: + pytester.makepyfile( + """ + def test_foo(fail_teardown): + assert 0 + """ + ) + output = pytester.runpytest("-rfE") + output.stdout.re_match_lines( + [ + r"test_teardown_with_test_also_failing.py FE\s+\[100%\]", + "FAILED test_teardown_with_test_also_failing.py::test_foo - assert 0", + "ERROR test_teardown_with_test_also_failing.py::test_foo - assert False", + ] + ) + + def test_teardown_many(self, pytester: Pytester, many_files) -> None: + output = pytester.runpytest() + output.stdout.re_match_lines( + [r"test_bar.py (\.E){5}\s+\[ 25%\]", r"test_foo.py (\.E){15}\s+\[100%\]"] + ) + + def test_teardown_many_verbose( + self, pytester: Pytester, many_files, color_mapping + ) -> None: + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + color_mapping.format_for_fnmatch( + [ + "test_bar.py::test_bar[0] PASSED * [ 5%]", + "test_bar.py::test_bar[0] ERROR * [ 5%]", + "test_bar.py::test_bar[4] PASSED * [ 25%]", + "test_foo.py::test_foo[14] PASSED * [100%]", + "test_foo.py::test_foo[14] ERROR * [100%]", + "=* 20 passed, 20 errors in *", + ] + ) + ) + + def test_xdist_normal(self, many_files, pytester: Pytester, monkeypatch) -> None: + pytest.importorskip("xdist") + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + output = pytester.runpytest("-n2") + output.stdout.re_match_lines([r"[\.E]{40} \s+ \[100%\]"]) + + +def test_skip_reasons_folding() -> None: + path = "xyz" + lineno = 3 + message = "justso" + longrepr = (path, lineno, message) + + class X: + pass + + ev1 = cast(CollectReport, X()) + ev1.when = "execute" + ev1.skipped = True # type: ignore[misc] + ev1.longrepr = longrepr + + ev2 = cast(CollectReport, X()) + ev2.when = "execute" + ev2.longrepr = longrepr + ev2.skipped = True # type: ignore[misc] + + # ev3 might be a collection report + ev3 = cast(CollectReport, X()) + ev3.when = "collect" + ev3.longrepr = longrepr + ev3.skipped = True # type: ignore[misc] + + values = _folded_skips(Path.cwd(), [ev1, ev2, ev3]) + assert len(values) == 1 + num, fspath, lineno_, reason = values[0] + assert num == 3 + assert fspath == path + assert lineno_ == lineno + assert reason == message + + +def test_line_with_reprcrash(monkeypatch: MonkeyPatch) -> None: + mocked_verbose_word = "FAILED" + + mocked_pos = "some::nodeid" + + def mock_get_pos(*args): + return mocked_pos + + monkeypatch.setattr(_pytest.terminal, "_get_pos", mock_get_pos) + + class config: + pass + + class rep: + def _get_verbose_word(self, *args): + return mocked_verbose_word + + class longrepr: + class reprcrash: + pass + + def check(msg, width, expected): + __tracebackhide__ = True + if msg: + rep.longrepr.reprcrash.message = msg # type: ignore + actual = _get_line_with_reprcrash_message(config, rep(), width) # type: ignore + + assert actual == expected + if actual != f"{mocked_verbose_word} {mocked_pos}": + assert len(actual) <= width + assert wcswidth(actual) <= width + + # AttributeError with message + check(None, 80, "FAILED some::nodeid") + + check("msg", 80, "FAILED some::nodeid - msg") + check("msg", 3, "FAILED some::nodeid") + + check("msg", 24, "FAILED some::nodeid") + check("msg", 25, "FAILED some::nodeid - msg") + + check("some longer msg", 24, "FAILED some::nodeid") + check("some longer msg", 25, "FAILED some::nodeid - ...") + check("some longer msg", 26, "FAILED some::nodeid - s...") + + check("some\nmessage", 25, "FAILED some::nodeid - ...") + check("some\nmessage", 26, "FAILED some::nodeid - some") + check("some\nmessage", 80, "FAILED some::nodeid - some") + + # Test unicode safety. + check("🉐🉐🉐🉐🉐\n2nd line", 25, "FAILED some::nodeid - ...") + check("🉐🉐🉐🉐🉐\n2nd line", 26, "FAILED some::nodeid - ...") + check("🉐🉐🉐🉐🉐\n2nd line", 27, "FAILED some::nodeid - 🉐...") + check("🉐🉐🉐🉐🉐\n2nd line", 28, "FAILED some::nodeid - 🉐...") + check("🉐🉐🉐🉐🉐\n2nd line", 29, "FAILED some::nodeid - 🉐🉐...") + + # NOTE: constructed, not sure if this is supported. + mocked_pos = "nodeid::🉐::withunicode" + check("🉐🉐🉐🉐🉐\n2nd line", 29, "FAILED nodeid::🉐::withunicode") + check("🉐🉐🉐🉐🉐\n2nd line", 40, "FAILED nodeid::🉐::withunicode - 🉐🉐...") + check("🉐🉐🉐🉐🉐\n2nd line", 41, "FAILED nodeid::🉐::withunicode - 🉐🉐...") + check("🉐🉐🉐🉐🉐\n2nd line", 42, "FAILED nodeid::🉐::withunicode - 🉐🉐🉐...") + check("🉐🉐🉐🉐🉐\n2nd line", 80, "FAILED nodeid::🉐::withunicode - 🉐🉐🉐🉐🉐") + + +@pytest.mark.parametrize( + "seconds, expected", + [ + (10.0, "10.00s"), + (10.34, "10.34s"), + (59.99, "59.99s"), + (60.55, "60.55s (0:01:00)"), + (123.55, "123.55s (0:02:03)"), + (60 * 60 + 0.5, "3600.50s (1:00:00)"), + ], +) +def test_format_session_duration(seconds, expected): + from _pytest.terminal import format_session_duration + + assert format_session_duration(seconds) == expected + + +def test_collecterror(pytester: Pytester) -> None: + p1 = pytester.makepyfile("raise SyntaxError()") + result = pytester.runpytest("-ra", str(p1)) + result.stdout.fnmatch_lines( + [ + "collected 0 items / 1 error", + "*= ERRORS =*", + "*_ ERROR collecting test_collecterror.py _*", + "E SyntaxError: *", + "*= short test summary info =*", + "ERROR test_collecterror.py", + "*! Interrupted: 1 error during collection !*", + "*= 1 error in *", + ] + ) + + +def test_no_summary_collecterror(pytester: Pytester) -> None: + p1 = pytester.makepyfile("raise SyntaxError()") + result = pytester.runpytest("-ra", "--no-summary", str(p1)) + result.stdout.no_fnmatch_line("*= ERRORS =*") + + +def test_via_exec(pytester: Pytester) -> None: + p1 = pytester.makepyfile("exec('def test_via_exec(): pass')") + result = pytester.runpytest(str(p1), "-vv") + result.stdout.fnmatch_lines( + ["test_via_exec.py::test_via_exec <- PASSED*", "*= 1 passed in *"] + ) + + +class TestCodeHighlight: + def test_code_highlight_simple(self, pytester: Pytester, color_mapping) -> None: + pytester.makepyfile( + """ + def test_foo(): + assert 1 == 10 + """ + ) + result = pytester.runpytest("--color=yes") + result.stdout.fnmatch_lines( + color_mapping.format_for_fnmatch( + [ + " {kw}def{hl-reset} {function}test_foo{hl-reset}():", + "> {kw}assert{hl-reset} {number}1{hl-reset} == {number}10{hl-reset}", + "{bold}{red}E assert 1 == 10{reset}", + ] + ) + ) + + def test_code_highlight_continuation( + self, pytester: Pytester, color_mapping + ) -> None: + pytester.makepyfile( + """ + def test_foo(): + print(''' + '''); assert 0 + """ + ) + result = pytester.runpytest("--color=yes") + + result.stdout.fnmatch_lines( + color_mapping.format_for_fnmatch( + [ + " {kw}def{hl-reset} {function}test_foo{hl-reset}():", + " {print}print{hl-reset}({str}'''{hl-reset}{str}{hl-reset}", + "> {str} {hl-reset}{str}'''{hl-reset}); {kw}assert{hl-reset} {number}0{hl-reset}", + "{bold}{red}E assert 0{reset}", + ] + ) + ) + + def test_code_highlight_custom_theme( + self, pytester: Pytester, color_mapping, monkeypatch: MonkeyPatch + ) -> None: + pytester.makepyfile( + """ + def test_foo(): + assert 1 == 10 + """ + ) + monkeypatch.setenv("PYTEST_THEME", "solarized-dark") + monkeypatch.setenv("PYTEST_THEME_MODE", "dark") + result = pytester.runpytest("--color=yes") + result.stdout.fnmatch_lines( + color_mapping.format_for_fnmatch( + [ + " {kw}def{hl-reset} {function}test_foo{hl-reset}():", + "> {kw}assert{hl-reset} {number}1{hl-reset} == {number}10{hl-reset}", + "{bold}{red}E assert 1 == 10{reset}", + ] + ) + ) + + def test_code_highlight_invalid_theme( + self, pytester: Pytester, color_mapping, monkeypatch: MonkeyPatch + ) -> None: + pytester.makepyfile( + """ + def test_foo(): + assert 1 == 10 + """ + ) + monkeypatch.setenv("PYTEST_THEME", "invalid") + result = pytester.runpytest_subprocess("--color=yes") + result.stderr.fnmatch_lines( + "ERROR: PYTEST_THEME environment variable had an invalid value: 'invalid'. " + "Only valid pygment styles are allowed." + ) + + def test_code_highlight_invalid_theme_mode( + self, pytester: Pytester, color_mapping, monkeypatch: MonkeyPatch + ) -> None: + pytester.makepyfile( + """ + def test_foo(): + assert 1 == 10 + """ + ) + monkeypatch.setenv("PYTEST_THEME_MODE", "invalid") + result = pytester.runpytest_subprocess("--color=yes") + result.stderr.fnmatch_lines( + "ERROR: PYTEST_THEME_MODE environment variable had an invalid value: 'invalid'. " + "The only allowed values are 'dark' and 'light'." + ) + + +def test_raw_skip_reason_skipped() -> None: + report = SimpleNamespace() + report.skipped = True + report.longrepr = ("xyz", 3, "Skipped: Just so") + + reason = _get_raw_skip_reason(cast(TestReport, report)) + assert reason == "Just so" + + +def test_raw_skip_reason_xfail() -> None: + report = SimpleNamespace() + report.wasxfail = "reason: To everything there is a season" + + reason = _get_raw_skip_reason(cast(TestReport, report)) + assert reason == "To everything there is a season" + + +def test_format_trimmed() -> None: + msg = "unconditional skip" + + assert _format_trimmed(" ({}) ", msg, len(msg) + 4) == " (unconditional skip) " + assert _format_trimmed(" ({}) ", msg, len(msg) + 3) == " (unconditional ...) " diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_threadexception.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_threadexception.py new file mode 100644 index 0000000000..5b7519f27d --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_threadexception.py @@ -0,0 +1,137 @@ +import sys + +import pytest +from _pytest.pytester import Pytester + + +if sys.version_info < (3, 8): + pytest.skip("threadexception plugin needs Python>=3.8", allow_module_level=True) + + +@pytest.mark.filterwarnings("default::pytest.PytestUnhandledThreadExceptionWarning") +def test_unhandled_thread_exception(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + import threading + + def test_it(): + def oops(): + raise ValueError("Oops") + + t = threading.Thread(target=oops, name="MyThread") + t.start() + t.join() + + def test_2(): pass + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + assert result.parseoutcomes() == {"passed": 2, "warnings": 1} + result.stdout.fnmatch_lines( + [ + "*= warnings summary =*", + "test_it.py::test_it", + " * PytestUnhandledThreadExceptionWarning: Exception in thread MyThread", + " ", + " Traceback (most recent call last):", + " ValueError: Oops", + " ", + " warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg))", + ] + ) + + +@pytest.mark.filterwarnings("default::pytest.PytestUnhandledThreadExceptionWarning") +def test_unhandled_thread_exception_in_setup(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + import threading + import pytest + + @pytest.fixture + def threadexc(): + def oops(): + raise ValueError("Oops") + t = threading.Thread(target=oops, name="MyThread") + t.start() + t.join() + + def test_it(threadexc): pass + def test_2(): pass + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + assert result.parseoutcomes() == {"passed": 2, "warnings": 1} + result.stdout.fnmatch_lines( + [ + "*= warnings summary =*", + "test_it.py::test_it", + " * PytestUnhandledThreadExceptionWarning: Exception in thread MyThread", + " ", + " Traceback (most recent call last):", + " ValueError: Oops", + " ", + " warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg))", + ] + ) + + +@pytest.mark.filterwarnings("default::pytest.PytestUnhandledThreadExceptionWarning") +def test_unhandled_thread_exception_in_teardown(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + import threading + import pytest + + @pytest.fixture + def threadexc(): + def oops(): + raise ValueError("Oops") + yield + t = threading.Thread(target=oops, name="MyThread") + t.start() + t.join() + + def test_it(threadexc): pass + def test_2(): pass + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + assert result.parseoutcomes() == {"passed": 2, "warnings": 1} + result.stdout.fnmatch_lines( + [ + "*= warnings summary =*", + "test_it.py::test_it", + " * PytestUnhandledThreadExceptionWarning: Exception in thread MyThread", + " ", + " Traceback (most recent call last):", + " ValueError: Oops", + " ", + " warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg))", + ] + ) + + +@pytest.mark.filterwarnings("error::pytest.PytestUnhandledThreadExceptionWarning") +def test_unhandled_thread_exception_warning_error(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + import threading + import pytest + + def test_it(): + def oops(): + raise ValueError("Oops") + t = threading.Thread(target=oops, name="MyThread") + t.start() + t.join() + + def test_2(): pass + """ + ) + result = pytester.runpytest() + assert result.ret == pytest.ExitCode.TESTS_FAILED + assert result.parseoutcomes() == {"passed": 1, "failed": 1} diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_tmpdir.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_tmpdir.py new file mode 100644 index 0000000000..4f7c538470 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_tmpdir.py @@ -0,0 +1,480 @@ +import os +import stat +import sys +import warnings +from pathlib import Path +from typing import Callable +from typing import cast +from typing import List + +import attr + +import pytest +from _pytest import pathlib +from _pytest.config import Config +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pathlib import cleanup_numbered_dir +from _pytest.pathlib import create_cleanup_lock +from _pytest.pathlib import make_numbered_dir +from _pytest.pathlib import maybe_delete_a_numbered_dir +from _pytest.pathlib import on_rm_rf_error +from _pytest.pathlib import register_cleanup_lock_removal +from _pytest.pathlib import rm_rf +from _pytest.pytester import Pytester +from _pytest.tmpdir import get_user +from _pytest.tmpdir import TempPathFactory + + +def test_tmp_path_fixture(pytester: Pytester) -> None: + p = pytester.copy_example("tmpdir/tmp_path_fixture.py") + results = pytester.runpytest(p) + results.stdout.fnmatch_lines(["*1 passed*"]) + + +@attr.s +class FakeConfig: + basetemp = attr.ib() + + @property + def trace(self): + return self + + def get(self, key): + return lambda *k: None + + @property + def option(self): + return self + + +class TestTmpPathHandler: + def test_mktemp(self, tmp_path): + config = cast(Config, FakeConfig(tmp_path)) + t = TempPathFactory.from_config(config, _ispytest=True) + tmp = t.mktemp("world") + assert str(tmp.relative_to(t.getbasetemp())) == "world0" + tmp = t.mktemp("this") + assert str(tmp.relative_to(t.getbasetemp())).startswith("this") + tmp2 = t.mktemp("this") + assert str(tmp2.relative_to(t.getbasetemp())).startswith("this") + assert tmp2 != tmp + + def test_tmppath_relative_basetemp_absolute(self, tmp_path, monkeypatch): + """#4425""" + monkeypatch.chdir(tmp_path) + config = cast(Config, FakeConfig("hello")) + t = TempPathFactory.from_config(config, _ispytest=True) + assert t.getbasetemp().resolve() == (tmp_path / "hello").resolve() + + +class TestConfigTmpPath: + def test_getbasetemp_custom_removes_old(self, pytester: Pytester) -> None: + mytemp = pytester.path.joinpath("xyz") + p = pytester.makepyfile( + """ + def test_1(tmp_path): + pass + """ + ) + pytester.runpytest(p, "--basetemp=%s" % mytemp) + assert mytemp.exists() + mytemp.joinpath("hello").touch() + + pytester.runpytest(p, "--basetemp=%s" % mytemp) + assert mytemp.exists() + assert not mytemp.joinpath("hello").exists() + + +testdata = [ + ("mypath", True), + ("/mypath1", False), + ("./mypath1", True), + ("../mypath3", False), + ("../../mypath4", False), + ("mypath5/..", False), + ("mypath6/../mypath6", True), + ("mypath7/../mypath7/..", False), +] + + +@pytest.mark.parametrize("basename, is_ok", testdata) +def test_mktemp(pytester: Pytester, basename: str, is_ok: bool) -> None: + mytemp = pytester.mkdir("mytemp") + p = pytester.makepyfile( + """ + def test_abs_path(tmp_path_factory): + tmp_path_factory.mktemp('{}', numbered=False) + """.format( + basename + ) + ) + + result = pytester.runpytest(p, "--basetemp=%s" % mytemp) + if is_ok: + assert result.ret == 0 + assert mytemp.joinpath(basename).exists() + else: + assert result.ret == 1 + result.stdout.fnmatch_lines("*ValueError*") + + +def test_tmp_path_always_is_realpath(pytester: Pytester, monkeypatch) -> None: + # the reason why tmp_path should be a realpath is that + # when you cd to it and do "os.getcwd()" you will anyway + # get the realpath. Using the symlinked path can thus + # easily result in path-inequality + # XXX if that proves to be a problem, consider using + # os.environ["PWD"] + realtemp = pytester.mkdir("myrealtemp") + linktemp = pytester.path.joinpath("symlinktemp") + attempt_symlink_to(linktemp, str(realtemp)) + monkeypatch.setenv("PYTEST_DEBUG_TEMPROOT", str(linktemp)) + pytester.makepyfile( + """ + def test_1(tmp_path): + assert tmp_path.resolve() == tmp_path + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +def test_tmp_path_too_long_on_parametrization(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.parametrize("arg", ["1"*1000]) + def test_some(arg, tmp_path): + tmp_path.joinpath("hello").touch() + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +def test_tmp_path_factory(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.fixture(scope='session') + def session_dir(tmp_path_factory): + return tmp_path_factory.mktemp('data', numbered=False) + def test_some(session_dir): + assert session_dir.is_dir() + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +def test_tmp_path_fallback_tox_env(pytester: Pytester, monkeypatch) -> None: + """Test that tmp_path works even if environment variables required by getpass + module are missing (#1010). + """ + monkeypatch.delenv("USER", raising=False) + monkeypatch.delenv("USERNAME", raising=False) + pytester.makepyfile( + """ + def test_some(tmp_path): + assert tmp_path.is_dir() + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +@pytest.fixture +def break_getuser(monkeypatch): + monkeypatch.setattr("os.getuid", lambda: -1) + # taken from python 2.7/3.4 + for envvar in ("LOGNAME", "USER", "LNAME", "USERNAME"): + monkeypatch.delenv(envvar, raising=False) + + +@pytest.mark.usefixtures("break_getuser") +@pytest.mark.skipif(sys.platform.startswith("win"), reason="no os.getuid on windows") +def test_tmp_path_fallback_uid_not_found(pytester: Pytester) -> None: + """Test that tmp_path works even if the current process's user id does not + correspond to a valid user. + """ + + pytester.makepyfile( + """ + def test_some(tmp_path): + assert tmp_path.is_dir() + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +@pytest.mark.usefixtures("break_getuser") +@pytest.mark.skipif(sys.platform.startswith("win"), reason="no os.getuid on windows") +def test_get_user_uid_not_found(): + """Test that get_user() function works even if the current process's + user id does not correspond to a valid user (e.g. running pytest in a + Docker container with 'docker run -u'. + """ + assert get_user() is None + + +@pytest.mark.skipif(not sys.platform.startswith("win"), reason="win only") +def test_get_user(monkeypatch): + """Test that get_user() function works even if environment variables + required by getpass module are missing from the environment on Windows + (#1010). + """ + monkeypatch.delenv("USER", raising=False) + monkeypatch.delenv("USERNAME", raising=False) + assert get_user() is None + + +class TestNumberedDir: + PREFIX = "fun-" + + def test_make(self, tmp_path): + for i in range(10): + d = make_numbered_dir(root=tmp_path, prefix=self.PREFIX) + assert d.name.startswith(self.PREFIX) + assert d.name.endswith(str(i)) + + symlink = tmp_path.joinpath(self.PREFIX + "current") + if symlink.exists(): + # unix + assert symlink.is_symlink() + assert symlink.resolve() == d.resolve() + + def test_cleanup_lock_create(self, tmp_path): + d = tmp_path.joinpath("test") + d.mkdir() + lockfile = create_cleanup_lock(d) + with pytest.raises(OSError, match="cannot create lockfile in .*"): + create_cleanup_lock(d) + + lockfile.unlink() + + def test_lock_register_cleanup_removal(self, tmp_path: Path) -> None: + lock = create_cleanup_lock(tmp_path) + + registry: List[Callable[..., None]] = [] + register_cleanup_lock_removal(lock, register=registry.append) + + (cleanup_func,) = registry + + assert lock.is_file() + + cleanup_func(original_pid="intentionally_different") + + assert lock.is_file() + + cleanup_func() + + assert not lock.exists() + + cleanup_func() + + assert not lock.exists() + + def _do_cleanup(self, tmp_path: Path) -> None: + self.test_make(tmp_path) + cleanup_numbered_dir( + root=tmp_path, + prefix=self.PREFIX, + keep=2, + consider_lock_dead_if_created_before=0, + ) + + def test_cleanup_keep(self, tmp_path): + self._do_cleanup(tmp_path) + a, b = (x for x in tmp_path.iterdir() if not x.is_symlink()) + print(a, b) + + def test_cleanup_locked(self, tmp_path): + p = make_numbered_dir(root=tmp_path, prefix=self.PREFIX) + + create_cleanup_lock(p) + + assert not pathlib.ensure_deletable( + p, consider_lock_dead_if_created_before=p.stat().st_mtime - 1 + ) + assert pathlib.ensure_deletable( + p, consider_lock_dead_if_created_before=p.stat().st_mtime + 1 + ) + + def test_cleanup_ignores_symlink(self, tmp_path): + the_symlink = tmp_path / (self.PREFIX + "current") + attempt_symlink_to(the_symlink, tmp_path / (self.PREFIX + "5")) + self._do_cleanup(tmp_path) + + def test_removal_accepts_lock(self, tmp_path): + folder = make_numbered_dir(root=tmp_path, prefix=self.PREFIX) + create_cleanup_lock(folder) + maybe_delete_a_numbered_dir(folder) + assert folder.is_dir() + + +class TestRmRf: + def test_rm_rf(self, tmp_path): + adir = tmp_path / "adir" + adir.mkdir() + rm_rf(adir) + + assert not adir.exists() + + adir.mkdir() + afile = adir / "afile" + afile.write_bytes(b"aa") + + rm_rf(adir) + assert not adir.exists() + + def test_rm_rf_with_read_only_file(self, tmp_path): + """Ensure rm_rf can remove directories with read-only files in them (#5524)""" + fn = tmp_path / "dir/foo.txt" + fn.parent.mkdir() + + fn.touch() + + self.chmod_r(fn) + + rm_rf(fn.parent) + + assert not fn.parent.is_dir() + + def chmod_r(self, path): + mode = os.stat(str(path)).st_mode + os.chmod(str(path), mode & ~stat.S_IWRITE) + + def test_rm_rf_with_read_only_directory(self, tmp_path): + """Ensure rm_rf can remove read-only directories (#5524)""" + adir = tmp_path / "dir" + adir.mkdir() + + (adir / "foo.txt").touch() + self.chmod_r(adir) + + rm_rf(adir) + + assert not adir.is_dir() + + def test_on_rm_rf_error(self, tmp_path: Path) -> None: + adir = tmp_path / "dir" + adir.mkdir() + + fn = adir / "foo.txt" + fn.touch() + self.chmod_r(fn) + + # unknown exception + with pytest.warns(pytest.PytestWarning): + exc_info1 = (None, RuntimeError(), None) + on_rm_rf_error(os.unlink, str(fn), exc_info1, start_path=tmp_path) + assert fn.is_file() + + # we ignore FileNotFoundError + exc_info2 = (None, FileNotFoundError(), None) + assert not on_rm_rf_error(None, str(fn), exc_info2, start_path=tmp_path) + + # unknown function + with pytest.warns( + pytest.PytestWarning, + match=r"^\(rm_rf\) unknown function None when removing .*foo.txt:\nNone: ", + ): + exc_info3 = (None, PermissionError(), None) + on_rm_rf_error(None, str(fn), exc_info3, start_path=tmp_path) + assert fn.is_file() + + # ignored function + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + with pytest.warns(None) as warninfo: # type: ignore[call-overload] + exc_info4 = (None, PermissionError(), None) + on_rm_rf_error(os.open, str(fn), exc_info4, start_path=tmp_path) + assert fn.is_file() + assert not [x.message for x in warninfo] + + exc_info5 = (None, PermissionError(), None) + on_rm_rf_error(os.unlink, str(fn), exc_info5, start_path=tmp_path) + assert not fn.is_file() + + +def attempt_symlink_to(path, to_path): + """Try to make a symlink from "path" to "to_path", skipping in case this platform + does not support it or we don't have sufficient privileges (common on Windows).""" + try: + Path(path).symlink_to(Path(to_path)) + except OSError: + pytest.skip("could not create symbolic link") + + +def test_basetemp_with_read_only_files(pytester: Pytester) -> None: + """Integration test for #5524""" + pytester.makepyfile( + """ + import os + import stat + + def test(tmp_path): + fn = tmp_path / 'foo.txt' + fn.write_text('hello') + mode = os.stat(str(fn)).st_mode + os.chmod(str(fn), mode & ~stat.S_IREAD) + """ + ) + result = pytester.runpytest("--basetemp=tmp") + assert result.ret == 0 + # running a second time and ensure we don't crash + result = pytester.runpytest("--basetemp=tmp") + assert result.ret == 0 + + +def test_tmp_path_factory_handles_invalid_dir_characters( + tmp_path_factory: TempPathFactory, monkeypatch: MonkeyPatch +) -> None: + monkeypatch.setattr("getpass.getuser", lambda: "os/<:*?;>agnostic") + # _basetemp / _given_basetemp are cached / set in parallel runs, patch them + monkeypatch.setattr(tmp_path_factory, "_basetemp", None) + monkeypatch.setattr(tmp_path_factory, "_given_basetemp", None) + p = tmp_path_factory.getbasetemp() + assert "pytest-of-unknown" in str(p) + + +@pytest.mark.skipif(not hasattr(os, "getuid"), reason="checks unix permissions") +def test_tmp_path_factory_create_directory_with_safe_permissions( + tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + """Verify that pytest creates directories under /tmp with private permissions.""" + # Use the test's tmp_path as the system temproot (/tmp). + monkeypatch.setenv("PYTEST_DEBUG_TEMPROOT", str(tmp_path)) + tmp_factory = TempPathFactory(None, lambda *args: None, _ispytest=True) + basetemp = tmp_factory.getbasetemp() + + # No world-readable permissions. + assert (basetemp.stat().st_mode & 0o077) == 0 + # Parent too (pytest-of-foo). + assert (basetemp.parent.stat().st_mode & 0o077) == 0 + + +@pytest.mark.skipif(not hasattr(os, "getuid"), reason="checks unix permissions") +def test_tmp_path_factory_fixes_up_world_readable_permissions( + tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + """Verify that if a /tmp/pytest-of-foo directory already exists with + world-readable permissions, it is fixed. + + pytest used to mkdir with such permissions, that's why we fix it up. + """ + # Use the test's tmp_path as the system temproot (/tmp). + monkeypatch.setenv("PYTEST_DEBUG_TEMPROOT", str(tmp_path)) + tmp_factory = TempPathFactory(None, lambda *args: None, _ispytest=True) + basetemp = tmp_factory.getbasetemp() + + # Before - simulate bad perms. + os.chmod(basetemp.parent, 0o777) + assert (basetemp.parent.stat().st_mode & 0o077) != 0 + + tmp_factory = TempPathFactory(None, lambda *args: None, _ispytest=True) + basetemp = tmp_factory.getbasetemp() + + # After - fixed. + assert (basetemp.parent.stat().st_mode & 0o077) == 0 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_unittest.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_unittest.py new file mode 100644 index 0000000000..1601086d5b --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_unittest.py @@ -0,0 +1,1500 @@ +import gc +import sys +from typing import List + +import pytest +from _pytest.config import ExitCode +from _pytest.monkeypatch import MonkeyPatch +from _pytest.pytester import Pytester + + +def test_simple_unittest(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + def testpassing(self): + self.assertEqual('foo', 'foo') + def test_failing(self): + self.assertEqual('foo', 'bar') + """ + ) + reprec = pytester.inline_run(testpath) + assert reprec.matchreport("testpassing").passed + assert reprec.matchreport("test_failing").failed + + +def test_runTest_method(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import unittest + class MyTestCaseWithRunTest(unittest.TestCase): + def runTest(self): + self.assertEqual('foo', 'foo') + class MyTestCaseWithoutRunTest(unittest.TestCase): + def runTest(self): + self.assertEqual('foo', 'foo') + def test_something(self): + pass + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + """ + *MyTestCaseWithRunTest::runTest* + *MyTestCaseWithoutRunTest::test_something* + *2 passed* + """ + ) + + +def test_isclasscheck_issue53(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + class _E(object): + def __getattr__(self, tag): + pass + E = _E() + """ + ) + result = pytester.runpytest(testpath) + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +def test_setup(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + def setUp(self): + self.foo = 1 + def setup_method(self, method): + self.foo2 = 1 + def test_both(self): + self.assertEqual(1, self.foo) + assert self.foo2 == 1 + def teardown_method(self, method): + assert 0, "42" + + """ + ) + reprec = pytester.inline_run("-s", testpath) + assert reprec.matchreport("test_both", when="call").passed + rep = reprec.matchreport("test_both", when="teardown") + assert rep.failed and "42" in str(rep.longrepr) + + +def test_setUpModule(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + values = [] + + def setUpModule(): + values.append(1) + + def tearDownModule(): + del values[0] + + def test_hello(): + assert values == [1] + + def test_world(): + assert values == [1] + """ + ) + result = pytester.runpytest(testpath) + result.stdout.fnmatch_lines(["*2 passed*"]) + + +def test_setUpModule_failing_no_teardown(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + values = [] + + def setUpModule(): + 0/0 + + def tearDownModule(): + values.append(1) + + def test_hello(): + pass + """ + ) + reprec = pytester.inline_run(testpath) + reprec.assertoutcome(passed=0, failed=1) + call = reprec.getcalls("pytest_runtest_setup")[0] + assert not call.item.module.values + + +def test_new_instances(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + def test_func1(self): + self.x = 2 + def test_func2(self): + assert not hasattr(self, 'x') + """ + ) + reprec = pytester.inline_run(testpath) + reprec.assertoutcome(passed=2) + + +def test_function_item_obj_is_instance(pytester: Pytester) -> None: + """item.obj should be a bound method on unittest.TestCase function items (#5390).""" + pytester.makeconftest( + """ + def pytest_runtest_makereport(item, call): + if call.when == 'call': + class_ = item.parent.obj + assert isinstance(item.obj.__self__, class_) + """ + ) + pytester.makepyfile( + """ + import unittest + + class Test(unittest.TestCase): + def test_foo(self): + pass + """ + ) + result = pytester.runpytest_inprocess() + result.stdout.fnmatch_lines(["* 1 passed in*"]) + + +def test_teardown(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + values = [] + def test_one(self): + pass + def tearDown(self): + self.values.append(None) + class Second(unittest.TestCase): + def test_check(self): + self.assertEqual(MyTestCase.values, [None]) + """ + ) + reprec = pytester.inline_run(testpath) + passed, skipped, failed = reprec.countoutcomes() + assert failed == 0, failed + assert passed == 2 + assert passed + skipped + failed == 2 + + +def test_teardown_issue1649(pytester: Pytester) -> None: + """ + Are TestCase objects cleaned up? Often unittest TestCase objects set + attributes that are large and expensive during setUp. + + The TestCase will not be cleaned up if the test fails, because it + would then exist in the stackframe. + """ + testpath = pytester.makepyfile( + """ + import unittest + class TestCaseObjectsShouldBeCleanedUp(unittest.TestCase): + def setUp(self): + self.an_expensive_object = 1 + def test_demo(self): + pass + + """ + ) + pytester.inline_run("-s", testpath) + gc.collect() + for obj in gc.get_objects(): + assert type(obj).__name__ != "TestCaseObjectsShouldBeCleanedUp" + + +def test_unittest_skip_issue148(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + + @unittest.skip("hello") + class MyTestCase(unittest.TestCase): + @classmethod + def setUpClass(self): + xxx + def test_one(self): + pass + @classmethod + def tearDownClass(self): + xxx + """ + ) + reprec = pytester.inline_run(testpath) + reprec.assertoutcome(skipped=1) + + +def test_method_and_teardown_failing_reporting(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import unittest + class TC(unittest.TestCase): + def tearDown(self): + assert 0, "down1" + def test_method(self): + assert False, "down2" + """ + ) + result = pytester.runpytest("-s") + assert result.ret == 1 + result.stdout.fnmatch_lines( + [ + "*tearDown*", + "*assert 0*", + "*test_method*", + "*assert False*", + "*1 failed*1 error*", + ] + ) + + +def test_setup_failure_is_shown(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import unittest + import pytest + class TC(unittest.TestCase): + def setUp(self): + assert 0, "down1" + def test_method(self): + print("never42") + xyz + """ + ) + result = pytester.runpytest("-s") + assert result.ret == 1 + result.stdout.fnmatch_lines(["*setUp*", "*assert 0*down1*", "*1 failed*"]) + result.stdout.no_fnmatch_line("*never42*") + + +def test_setup_setUpClass(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + import pytest + class MyTestCase(unittest.TestCase): + x = 0 + @classmethod + def setUpClass(cls): + cls.x += 1 + def test_func1(self): + assert self.x == 1 + def test_func2(self): + assert self.x == 1 + @classmethod + def tearDownClass(cls): + cls.x -= 1 + def test_teareddown(): + assert MyTestCase.x == 0 + """ + ) + reprec = pytester.inline_run(testpath) + reprec.assertoutcome(passed=3) + + +def test_fixtures_setup_setUpClass_issue8394(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + @classmethod + def setUpClass(cls): + pass + def test_func1(self): + pass + @classmethod + def tearDownClass(cls): + pass + """ + ) + result = pytester.runpytest("--fixtures") + assert result.ret == 0 + result.stdout.no_fnmatch_line("*no docstring available*") + + result = pytester.runpytest("--fixtures", "-v") + assert result.ret == 0 + result.stdout.fnmatch_lines(["*no docstring available*"]) + + +def test_setup_class(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + import pytest + class MyTestCase(unittest.TestCase): + x = 0 + def setup_class(cls): + cls.x += 1 + def test_func1(self): + assert self.x == 1 + def test_func2(self): + assert self.x == 1 + def teardown_class(cls): + cls.x -= 1 + def test_teareddown(): + assert MyTestCase.x == 0 + """ + ) + reprec = pytester.inline_run(testpath) + reprec.assertoutcome(passed=3) + + +@pytest.mark.parametrize("type", ["Error", "Failure"]) +def test_testcase_adderrorandfailure_defers(pytester: Pytester, type: str) -> None: + pytester.makepyfile( + """ + from unittest import TestCase + import pytest + class MyTestCase(TestCase): + def run(self, result): + excinfo = pytest.raises(ZeroDivisionError, lambda: 0/0) + try: + result.add%s(self, excinfo._excinfo) + except KeyboardInterrupt: + raise + except: + pytest.fail("add%s should not raise") + def test_hello(self): + pass + """ + % (type, type) + ) + result = pytester.runpytest() + result.stdout.no_fnmatch_line("*should not raise*") + + +@pytest.mark.parametrize("type", ["Error", "Failure"]) +def test_testcase_custom_exception_info(pytester: Pytester, type: str) -> None: + pytester.makepyfile( + """ + from typing import Generic, TypeVar + from unittest import TestCase + import pytest, _pytest._code + + class MyTestCase(TestCase): + def run(self, result): + excinfo = pytest.raises(ZeroDivisionError, lambda: 0/0) + # We fake an incompatible exception info. + class FakeExceptionInfo(Generic[TypeVar("E")]): + def __init__(self, *args, **kwargs): + mp.undo() + raise TypeError() + @classmethod + def from_current(cls): + return cls() + @classmethod + def from_exc_info(cls, *args, **kwargs): + return cls() + mp = pytest.MonkeyPatch() + mp.setattr(_pytest._code, 'ExceptionInfo', FakeExceptionInfo) + try: + excinfo = excinfo._excinfo + result.add%(type)s(self, excinfo) + finally: + mp.undo() + + def test_hello(self): + pass + """ + % locals() + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "NOTE: Incompatible Exception Representation*", + "*ZeroDivisionError*", + "*1 failed*", + ] + ) + + +def test_testcase_totally_incompatible_exception_info(pytester: Pytester) -> None: + import _pytest.unittest + + (item,) = pytester.getitems( + """ + from unittest import TestCase + class MyTestCase(TestCase): + def test_hello(self): + pass + """ + ) + assert isinstance(item, _pytest.unittest.TestCaseFunction) + item.addError(None, 42) # type: ignore[arg-type] + excinfo = item._excinfo + assert excinfo is not None + assert "ERROR: Unknown Incompatible" in str(excinfo.pop(0).getrepr()) + + +def test_module_level_pytestmark(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + import pytest + pytestmark = pytest.mark.xfail + class MyTestCase(unittest.TestCase): + def test_func1(self): + assert 0 + """ + ) + reprec = pytester.inline_run(testpath, "-s") + reprec.assertoutcome(skipped=1) + + +class TestTrialUnittest: + def setup_class(cls): + cls.ut = pytest.importorskip("twisted.trial.unittest") + # on windows trial uses a socket for a reactor and apparently doesn't close it properly + # https://twistedmatrix.com/trac/ticket/9227 + cls.ignore_unclosed_socket_warning = ("-W", "always") + + def test_trial_testcase_runtest_not_collected(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + from twisted.trial.unittest import TestCase + + class TC(TestCase): + def test_hello(self): + pass + """ + ) + reprec = pytester.inline_run(*self.ignore_unclosed_socket_warning) + reprec.assertoutcome(passed=1) + pytester.makepyfile( + """ + from twisted.trial.unittest import TestCase + + class TC(TestCase): + def runTest(self): + pass + """ + ) + reprec = pytester.inline_run(*self.ignore_unclosed_socket_warning) + reprec.assertoutcome(passed=1) + + def test_trial_exceptions_with_skips(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + from twisted.trial import unittest + import pytest + class TC(unittest.TestCase): + def test_hello(self): + pytest.skip("skip_in_method") + @pytest.mark.skipif("sys.version_info != 1") + def test_hello2(self): + pass + @pytest.mark.xfail(reason="iwanto") + def test_hello3(self): + assert 0 + def test_hello4(self): + pytest.xfail("i2wanto") + def test_trial_skip(self): + pass + test_trial_skip.skip = "trialselfskip" + + def test_trial_todo(self): + assert 0 + test_trial_todo.todo = "mytodo" + + def test_trial_todo_success(self): + pass + test_trial_todo_success.todo = "mytodo" + + class TC2(unittest.TestCase): + def setup_class(cls): + pytest.skip("skip_in_setup_class") + def test_method(self): + pass + """ + ) + result = pytester.runpytest("-rxs", *self.ignore_unclosed_socket_warning) + result.stdout.fnmatch_lines_random( + [ + "*XFAIL*test_trial_todo*", + "*trialselfskip*", + "*skip_in_setup_class*", + "*iwanto*", + "*i2wanto*", + "*sys.version_info*", + "*skip_in_method*", + "*1 failed*4 skipped*3 xfailed*", + ] + ) + assert result.ret == 1 + + def test_trial_error(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + from twisted.trial.unittest import TestCase + from twisted.internet.defer import Deferred + from twisted.internet import reactor + + class TC(TestCase): + def test_one(self): + crash + + def test_two(self): + def f(_): + crash + + d = Deferred() + d.addCallback(f) + reactor.callLater(0.3, d.callback, None) + return d + + def test_three(self): + def f(): + pass # will never get called + reactor.callLater(0.3, f) + # will crash at teardown + + def test_four(self): + def f(_): + reactor.callLater(0.3, f) + crash + + d = Deferred() + d.addCallback(f) + reactor.callLater(0.3, d.callback, None) + return d + # will crash both at test time and at teardown + """ + ) + result = pytester.runpytest( + "-vv", "-oconsole_output_style=classic", "-W", "ignore::DeprecationWarning" + ) + result.stdout.fnmatch_lines( + [ + "test_trial_error.py::TC::test_four FAILED", + "test_trial_error.py::TC::test_four ERROR", + "test_trial_error.py::TC::test_one FAILED", + "test_trial_error.py::TC::test_three FAILED", + "test_trial_error.py::TC::test_two FAILED", + "*ERRORS*", + "*_ ERROR at teardown of TC.test_four _*", + "*DelayedCalls*", + "*= FAILURES =*", + "*_ TC.test_four _*", + "*NameError*crash*", + "*_ TC.test_one _*", + "*NameError*crash*", + "*_ TC.test_three _*", + "*DelayedCalls*", + "*_ TC.test_two _*", + "*NameError*crash*", + "*= 4 failed, 1 error in *", + ] + ) + + def test_trial_pdb(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """ + from twisted.trial import unittest + import pytest + class TC(unittest.TestCase): + def test_hello(self): + assert 0, "hellopdb" + """ + ) + child = pytester.spawn_pytest(str(p)) + child.expect("hellopdb") + child.sendeof() + + def test_trial_testcase_skip_property(self, pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + from twisted.trial import unittest + class MyTestCase(unittest.TestCase): + skip = 'dont run' + def test_func(self): + pass + """ + ) + reprec = pytester.inline_run(testpath, "-s") + reprec.assertoutcome(skipped=1) + + def test_trial_testfunction_skip_property(self, pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + from twisted.trial import unittest + class MyTestCase(unittest.TestCase): + def test_func(self): + pass + test_func.skip = 'dont run' + """ + ) + reprec = pytester.inline_run(testpath, "-s") + reprec.assertoutcome(skipped=1) + + def test_trial_testcase_todo_property(self, pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + from twisted.trial import unittest + class MyTestCase(unittest.TestCase): + todo = 'dont run' + def test_func(self): + assert 0 + """ + ) + reprec = pytester.inline_run(testpath, "-s") + reprec.assertoutcome(skipped=1) + + def test_trial_testfunction_todo_property(self, pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + from twisted.trial import unittest + class MyTestCase(unittest.TestCase): + def test_func(self): + assert 0 + test_func.todo = 'dont run' + """ + ) + reprec = pytester.inline_run( + testpath, "-s", *self.ignore_unclosed_socket_warning + ) + reprec.assertoutcome(skipped=1) + + +def test_djangolike_testcase(pytester: Pytester) -> None: + # contributed from Morten Breekevold + pytester.makepyfile( + """ + from unittest import TestCase, main + + class DjangoLikeTestCase(TestCase): + + def setUp(self): + print("setUp()") + + def test_presetup_has_been_run(self): + print("test_thing()") + self.assertTrue(hasattr(self, 'was_presetup')) + + def tearDown(self): + print("tearDown()") + + def __call__(self, result=None): + try: + self._pre_setup() + except (KeyboardInterrupt, SystemExit): + raise + except Exception: + import sys + result.addError(self, sys.exc_info()) + return + super(DjangoLikeTestCase, self).__call__(result) + try: + self._post_teardown() + except (KeyboardInterrupt, SystemExit): + raise + except Exception: + import sys + result.addError(self, sys.exc_info()) + return + + def _pre_setup(self): + print("_pre_setup()") + self.was_presetup = True + + def _post_teardown(self): + print("_post_teardown()") + """ + ) + result = pytester.runpytest("-s") + assert result.ret == 0 + result.stdout.fnmatch_lines( + [ + "*_pre_setup()*", + "*setUp()*", + "*test_thing()*", + "*tearDown()*", + "*_post_teardown()*", + ] + ) + + +def test_unittest_not_shown_in_traceback(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import unittest + class t(unittest.TestCase): + def test_hello(self): + x = 3 + self.assertEqual(x, 4) + """ + ) + res = pytester.runpytest() + res.stdout.no_fnmatch_line("*failUnlessEqual*") + + +def test_unorderable_types(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import unittest + class TestJoinEmpty(unittest.TestCase): + pass + + def make_test(): + class Test(unittest.TestCase): + pass + Test.__name__ = "TestFoo" + return Test + TestFoo = make_test() + """ + ) + result = pytester.runpytest() + result.stdout.no_fnmatch_line("*TypeError*") + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +def test_unittest_typerror_traceback(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import unittest + class TestJoinEmpty(unittest.TestCase): + def test_hello(self, arg1): + pass + """ + ) + result = pytester.runpytest() + assert "TypeError" in result.stdout.str() + assert result.ret == 1 + + +@pytest.mark.parametrize("runner", ["pytest", "unittest"]) +def test_unittest_expected_failure_for_failing_test_is_xfail( + pytester: Pytester, runner +) -> None: + script = pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + @unittest.expectedFailure + def test_failing_test_is_xfail(self): + assert False + if __name__ == '__main__': + unittest.main() + """ + ) + if runner == "pytest": + result = pytester.runpytest("-rxX") + result.stdout.fnmatch_lines( + ["*XFAIL*MyTestCase*test_failing_test_is_xfail*", "*1 xfailed*"] + ) + else: + result = pytester.runpython(script) + result.stderr.fnmatch_lines(["*1 test in*", "*OK*(expected failures=1)*"]) + assert result.ret == 0 + + +@pytest.mark.parametrize("runner", ["pytest", "unittest"]) +def test_unittest_expected_failure_for_passing_test_is_fail( + pytester: Pytester, + runner: str, +) -> None: + script = pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + @unittest.expectedFailure + def test_passing_test_is_fail(self): + assert True + if __name__ == '__main__': + unittest.main() + """ + ) + + if runner == "pytest": + result = pytester.runpytest("-rxX") + result.stdout.fnmatch_lines( + [ + "*MyTestCase*test_passing_test_is_fail*", + "Unexpected success", + "*1 failed*", + ] + ) + else: + result = pytester.runpython(script) + result.stderr.fnmatch_lines(["*1 test in*", "*(unexpected successes=1)*"]) + + assert result.ret == 1 + + +@pytest.mark.parametrize("stmt", ["return", "yield"]) +def test_unittest_setup_interaction(pytester: Pytester, stmt: str) -> None: + pytester.makepyfile( + """ + import unittest + import pytest + class MyTestCase(unittest.TestCase): + @pytest.fixture(scope="class", autouse=True) + def perclass(self, request): + request.cls.hello = "world" + {stmt} + @pytest.fixture(scope="function", autouse=True) + def perfunction(self, request): + request.instance.funcname = request.function.__name__ + {stmt} + + def test_method1(self): + assert self.funcname == "test_method1" + assert self.hello == "world" + + def test_method2(self): + assert self.funcname == "test_method2" + + def test_classattr(self): + assert self.__class__.hello == "world" + """.format( + stmt=stmt + ) + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*3 passed*"]) + + +def test_non_unittest_no_setupclass_support(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + class TestFoo(object): + x = 0 + + @classmethod + def setUpClass(cls): + cls.x = 1 + + def test_method1(self): + assert self.x == 0 + + @classmethod + def tearDownClass(cls): + cls.x = 1 + + def test_not_teareddown(): + assert TestFoo.x == 0 + + """ + ) + reprec = pytester.inline_run(testpath) + reprec.assertoutcome(passed=2) + + +def test_no_teardown_if_setupclass_failed(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + + class MyTestCase(unittest.TestCase): + x = 0 + + @classmethod + def setUpClass(cls): + cls.x = 1 + assert False + + def test_func1(self): + cls.x = 10 + + @classmethod + def tearDownClass(cls): + cls.x = 100 + + def test_notTornDown(): + assert MyTestCase.x == 1 + """ + ) + reprec = pytester.inline_run(testpath) + reprec.assertoutcome(passed=1, failed=1) + + +def test_cleanup_functions(pytester: Pytester) -> None: + """Ensure functions added with addCleanup are always called after each test ends (#6947)""" + pytester.makepyfile( + """ + import unittest + + cleanups = [] + + class Test(unittest.TestCase): + + def test_func_1(self): + self.addCleanup(cleanups.append, "test_func_1") + + def test_func_2(self): + self.addCleanup(cleanups.append, "test_func_2") + assert 0 + + def test_func_3_check_cleanups(self): + assert cleanups == ["test_func_1", "test_func_2"] + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "*::test_func_1 PASSED *", + "*::test_func_2 FAILED *", + "*::test_func_3_check_cleanups PASSED *", + ] + ) + + +def test_issue333_result_clearing(pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + @pytest.hookimpl(hookwrapper=True) + def pytest_runtest_call(item): + yield + assert 0 + """ + ) + pytester.makepyfile( + """ + import unittest + class TestIt(unittest.TestCase): + def test_func(self): + 0/0 + """ + ) + + reprec = pytester.inline_run() + reprec.assertoutcome(failed=1) + + +def test_unittest_raise_skip_issue748(pytester: Pytester) -> None: + pytester.makepyfile( + test_foo=""" + import unittest + + class MyTestCase(unittest.TestCase): + def test_one(self): + raise unittest.SkipTest('skipping due to reasons') + """ + ) + result = pytester.runpytest("-v", "-rs") + result.stdout.fnmatch_lines( + """ + *SKIP*[1]*test_foo.py*skipping due to reasons* + *1 skipped* + """ + ) + + +def test_unittest_skip_issue1169(pytester: Pytester) -> None: + pytester.makepyfile( + test_foo=""" + import unittest + + class MyTestCase(unittest.TestCase): + @unittest.skip("skipping due to reasons") + def test_skip(self): + self.fail() + """ + ) + result = pytester.runpytest("-v", "-rs") + result.stdout.fnmatch_lines( + """ + *SKIP*[1]*skipping due to reasons* + *1 skipped* + """ + ) + + +def test_class_method_containing_test_issue1558(pytester: Pytester) -> None: + pytester.makepyfile( + test_foo=""" + import unittest + + class MyTestCase(unittest.TestCase): + def test_should_run(self): + pass + def test_should_not_run(self): + pass + test_should_not_run.__test__ = False + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(passed=1) + + +@pytest.mark.parametrize("base", ["builtins.object", "unittest.TestCase"]) +def test_usefixtures_marker_on_unittest(base, pytester: Pytester) -> None: + """#3498""" + module = base.rsplit(".", 1)[0] + pytest.importorskip(module) + pytester.makepyfile( + conftest=""" + import pytest + + @pytest.fixture(scope='function') + def fixture1(request, monkeypatch): + monkeypatch.setattr(request.instance, 'fixture1', True ) + + + @pytest.fixture(scope='function') + def fixture2(request, monkeypatch): + monkeypatch.setattr(request.instance, 'fixture2', True ) + + def node_and_marks(item): + print(item.nodeid) + for mark in item.iter_markers(): + print(" ", mark) + + @pytest.fixture(autouse=True) + def my_marks(request): + node_and_marks(request.node) + + def pytest_collection_modifyitems(items): + for item in items: + node_and_marks(item) + + """ + ) + + pytester.makepyfile( + """ + import pytest + import {module} + + class Tests({base}): + fixture1 = False + fixture2 = False + + @pytest.mark.usefixtures("fixture1") + def test_one(self): + assert self.fixture1 + assert not self.fixture2 + + @pytest.mark.usefixtures("fixture1", "fixture2") + def test_two(self): + assert self.fixture1 + assert self.fixture2 + + + """.format( + module=module, base=base + ) + ) + + result = pytester.runpytest("-s") + result.assert_outcomes(passed=2) + + +def test_testcase_handles_init_exceptions(pytester: Pytester) -> None: + """ + Regression test to make sure exceptions in the __init__ method are bubbled up correctly. + See https://github.com/pytest-dev/pytest/issues/3788 + """ + pytester.makepyfile( + """ + from unittest import TestCase + import pytest + class MyTestCase(TestCase): + def __init__(self, *args, **kwargs): + raise Exception("should raise this exception") + def test_hello(self): + pass + """ + ) + result = pytester.runpytest() + assert "should raise this exception" in result.stdout.str() + result.stdout.no_fnmatch_line("*ERROR at teardown of MyTestCase.test_hello*") + + +def test_error_message_with_parametrized_fixtures(pytester: Pytester) -> None: + pytester.copy_example("unittest/test_parametrized_fixture_error_message.py") + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*test_two does not support fixtures*", + "*TestSomethingElse::test_two", + "*Function type: TestCaseFunction", + ] + ) + + +@pytest.mark.parametrize( + "test_name, expected_outcome", + [ + ("test_setup_skip.py", "1 skipped"), + ("test_setup_skip_class.py", "1 skipped"), + ("test_setup_skip_module.py", "1 error"), + ], +) +def test_setup_inheritance_skipping( + pytester: Pytester, test_name, expected_outcome +) -> None: + """Issue #4700""" + pytester.copy_example(f"unittest/{test_name}") + result = pytester.runpytest() + result.stdout.fnmatch_lines([f"* {expected_outcome} in *"]) + + +def test_BdbQuit(pytester: Pytester) -> None: + pytester.makepyfile( + test_foo=""" + import unittest + + class MyTestCase(unittest.TestCase): + def test_bdbquit(self): + import bdb + raise bdb.BdbQuit() + + def test_should_not_run(self): + pass + """ + ) + reprec = pytester.inline_run() + reprec.assertoutcome(failed=1, passed=1) + + +def test_exit_outcome(pytester: Pytester) -> None: + pytester.makepyfile( + test_foo=""" + import pytest + import unittest + + class MyTestCase(unittest.TestCase): + def test_exit_outcome(self): + pytest.exit("pytest_exit called") + + def test_should_not_run(self): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*Exit: pytest_exit called*", "*= no tests ran in *"]) + + +def test_trace(pytester: Pytester, monkeypatch: MonkeyPatch) -> None: + calls = [] + + def check_call(*args, **kwargs): + calls.append((args, kwargs)) + assert args == ("runcall",) + + class _pdb: + def runcall(*args, **kwargs): + calls.append((args, kwargs)) + + return _pdb + + monkeypatch.setattr("_pytest.debugging.pytestPDB._init_pdb", check_call) + + p1 = pytester.makepyfile( + """ + import unittest + + class MyTestCase(unittest.TestCase): + def test(self): + self.assertEqual('foo', 'foo') + """ + ) + result = pytester.runpytest("--trace", str(p1)) + assert len(calls) == 2 + assert result.ret == 0 + + +def test_pdb_teardown_called(pytester: Pytester, monkeypatch: MonkeyPatch) -> None: + """Ensure tearDown() is always called when --pdb is given in the command-line. + + We delay the normal tearDown() calls when --pdb is given, so this ensures we are calling + tearDown() eventually to avoid memory leaks when using --pdb. + """ + teardowns: List[str] = [] + monkeypatch.setattr( + pytest, "test_pdb_teardown_called_teardowns", teardowns, raising=False + ) + + pytester.makepyfile( + """ + import unittest + import pytest + + class MyTestCase(unittest.TestCase): + + def tearDown(self): + pytest.test_pdb_teardown_called_teardowns.append(self.id()) + + def test_1(self): + pass + def test_2(self): + pass + """ + ) + result = pytester.runpytest_inprocess("--pdb") + result.stdout.fnmatch_lines("* 2 passed in *") + assert teardowns == [ + "test_pdb_teardown_called.MyTestCase.test_1", + "test_pdb_teardown_called.MyTestCase.test_2", + ] + + +@pytest.mark.parametrize("mark", ["@unittest.skip", "@pytest.mark.skip"]) +def test_pdb_teardown_skipped( + pytester: Pytester, monkeypatch: MonkeyPatch, mark: str +) -> None: + """With --pdb, setUp and tearDown should not be called for skipped tests.""" + tracked: List[str] = [] + monkeypatch.setattr(pytest, "test_pdb_teardown_skipped", tracked, raising=False) + + pytester.makepyfile( + """ + import unittest + import pytest + + class MyTestCase(unittest.TestCase): + + def setUp(self): + pytest.test_pdb_teardown_skipped.append("setUp:" + self.id()) + + def tearDown(self): + pytest.test_pdb_teardown_skipped.append("tearDown:" + self.id()) + + {mark}("skipped for reasons") + def test_1(self): + pass + + """.format( + mark=mark + ) + ) + result = pytester.runpytest_inprocess("--pdb") + result.stdout.fnmatch_lines("* 1 skipped in *") + assert tracked == [] + + +def test_async_support(pytester: Pytester) -> None: + pytest.importorskip("unittest.async_case") + + pytester.copy_example("unittest/test_unittest_asyncio.py") + reprec = pytester.inline_run() + reprec.assertoutcome(failed=1, passed=2) + + +def test_asynctest_support(pytester: Pytester) -> None: + """Check asynctest support (#7110)""" + pytest.importorskip("asynctest") + + pytester.copy_example("unittest/test_unittest_asynctest.py") + reprec = pytester.inline_run() + reprec.assertoutcome(failed=1, passed=2) + + +def test_plain_unittest_does_not_support_async(pytester: Pytester) -> None: + """Async functions in plain unittest.TestCase subclasses are not supported without plugins. + + This test exists here to avoid introducing this support by accident, leading users + to expect that it works, rather than doing so intentionally as a feature. + + See https://github.com/pytest-dev/pytest-asyncio/issues/180 for more context. + """ + pytester.copy_example("unittest/test_unittest_plain_async.py") + result = pytester.runpytest_subprocess() + if hasattr(sys, "pypy_version_info"): + # in PyPy we can't reliable get the warning about the coroutine not being awaited, + # because it depends on the coroutine being garbage collected; given that + # we are running in a subprocess, that's difficult to enforce + expected_lines = ["*1 passed*"] + else: + expected_lines = [ + "*RuntimeWarning: coroutine * was never awaited", + "*1 passed*", + ] + result.stdout.fnmatch_lines(expected_lines) + + +@pytest.mark.skipif( + sys.version_info < (3, 8), reason="Feature introduced in Python 3.8" +) +def test_do_class_cleanups_on_success(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + values = [] + @classmethod + def setUpClass(cls): + def cleanup(): + cls.values.append(1) + cls.addClassCleanup(cleanup) + def test_one(self): + pass + def test_two(self): + pass + def test_cleanup_called_exactly_once(): + assert MyTestCase.values == [1] + """ + ) + reprec = pytester.inline_run(testpath) + passed, skipped, failed = reprec.countoutcomes() + assert failed == 0 + assert passed == 3 + + +@pytest.mark.skipif( + sys.version_info < (3, 8), reason="Feature introduced in Python 3.8" +) +def test_do_class_cleanups_on_setupclass_failure(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + values = [] + @classmethod + def setUpClass(cls): + def cleanup(): + cls.values.append(1) + cls.addClassCleanup(cleanup) + assert False + def test_one(self): + pass + def test_cleanup_called_exactly_once(): + assert MyTestCase.values == [1] + """ + ) + reprec = pytester.inline_run(testpath) + passed, skipped, failed = reprec.countoutcomes() + assert failed == 1 + assert passed == 1 + + +@pytest.mark.skipif( + sys.version_info < (3, 8), reason="Feature introduced in Python 3.8" +) +def test_do_class_cleanups_on_teardownclass_failure(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + values = [] + @classmethod + def setUpClass(cls): + def cleanup(): + cls.values.append(1) + cls.addClassCleanup(cleanup) + @classmethod + def tearDownClass(cls): + assert False + def test_one(self): + pass + def test_two(self): + pass + def test_cleanup_called_exactly_once(): + assert MyTestCase.values == [1] + """ + ) + reprec = pytester.inline_run(testpath) + passed, skipped, failed = reprec.countoutcomes() + assert passed == 3 + + +def test_do_cleanups_on_success(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + values = [] + def setUp(self): + def cleanup(): + self.values.append(1) + self.addCleanup(cleanup) + def test_one(self): + pass + def test_two(self): + pass + def test_cleanup_called_the_right_number_of_times(): + assert MyTestCase.values == [1, 1] + """ + ) + reprec = pytester.inline_run(testpath) + passed, skipped, failed = reprec.countoutcomes() + assert failed == 0 + assert passed == 3 + + +def test_do_cleanups_on_setup_failure(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + values = [] + def setUp(self): + def cleanup(): + self.values.append(1) + self.addCleanup(cleanup) + assert False + def test_one(self): + pass + def test_two(self): + pass + def test_cleanup_called_the_right_number_of_times(): + assert MyTestCase.values == [1, 1] + """ + ) + reprec = pytester.inline_run(testpath) + passed, skipped, failed = reprec.countoutcomes() + assert failed == 2 + assert passed == 1 + + +def test_do_cleanups_on_teardown_failure(pytester: Pytester) -> None: + testpath = pytester.makepyfile( + """ + import unittest + class MyTestCase(unittest.TestCase): + values = [] + def setUp(self): + def cleanup(): + self.values.append(1) + self.addCleanup(cleanup) + def tearDown(self): + assert False + def test_one(self): + pass + def test_two(self): + pass + def test_cleanup_called_the_right_number_of_times(): + assert MyTestCase.values == [1, 1] + """ + ) + reprec = pytester.inline_run(testpath) + passed, skipped, failed = reprec.countoutcomes() + assert failed == 2 + assert passed == 1 + + +def test_traceback_pruning(pytester: Pytester) -> None: + """Regression test for #9610 - doesn't crash during traceback pruning.""" + pytester.makepyfile( + """ + import unittest + + class MyTestCase(unittest.TestCase): + def __init__(self, test_method): + unittest.TestCase.__init__(self, test_method) + + class TestIt(MyTestCase): + @classmethod + def tearDownClass(cls) -> None: + assert False + + def test_it(self): + pass + """ + ) + reprec = pytester.inline_run() + passed, skipped, failed = reprec.countoutcomes() + assert passed == 1 + assert failed == 1 + assert reprec.ret == 1 diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_unraisableexception.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_unraisableexception.py new file mode 100644 index 0000000000..f625833dce --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_unraisableexception.py @@ -0,0 +1,133 @@ +import sys + +import pytest +from _pytest.pytester import Pytester + + +if sys.version_info < (3, 8): + pytest.skip("unraisableexception plugin needs Python>=3.8", allow_module_level=True) + + +@pytest.mark.filterwarnings("default::pytest.PytestUnraisableExceptionWarning") +def test_unraisable(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + class BrokenDel: + def __del__(self): + raise ValueError("del is broken") + + def test_it(): + obj = BrokenDel() + del obj + + def test_2(): pass + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + assert result.parseoutcomes() == {"passed": 2, "warnings": 1} + result.stdout.fnmatch_lines( + [ + "*= warnings summary =*", + "test_it.py::test_it", + " * PytestUnraisableExceptionWarning: Exception ignored in: ", + " ", + " Traceback (most recent call last):", + " ValueError: del is broken", + " ", + " warnings.warn(pytest.PytestUnraisableExceptionWarning(msg))", + ] + ) + + +@pytest.mark.filterwarnings("default::pytest.PytestUnraisableExceptionWarning") +def test_unraisable_in_setup(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + import pytest + + class BrokenDel: + def __del__(self): + raise ValueError("del is broken") + + @pytest.fixture + def broken_del(): + obj = BrokenDel() + del obj + + def test_it(broken_del): pass + def test_2(): pass + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + assert result.parseoutcomes() == {"passed": 2, "warnings": 1} + result.stdout.fnmatch_lines( + [ + "*= warnings summary =*", + "test_it.py::test_it", + " * PytestUnraisableExceptionWarning: Exception ignored in: ", + " ", + " Traceback (most recent call last):", + " ValueError: del is broken", + " ", + " warnings.warn(pytest.PytestUnraisableExceptionWarning(msg))", + ] + ) + + +@pytest.mark.filterwarnings("default::pytest.PytestUnraisableExceptionWarning") +def test_unraisable_in_teardown(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + import pytest + + class BrokenDel: + def __del__(self): + raise ValueError("del is broken") + + @pytest.fixture + def broken_del(): + yield + obj = BrokenDel() + del obj + + def test_it(broken_del): pass + def test_2(): pass + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + assert result.parseoutcomes() == {"passed": 2, "warnings": 1} + result.stdout.fnmatch_lines( + [ + "*= warnings summary =*", + "test_it.py::test_it", + " * PytestUnraisableExceptionWarning: Exception ignored in: ", + " ", + " Traceback (most recent call last):", + " ValueError: del is broken", + " ", + " warnings.warn(pytest.PytestUnraisableExceptionWarning(msg))", + ] + ) + + +@pytest.mark.filterwarnings("error::pytest.PytestUnraisableExceptionWarning") +def test_unraisable_warning_error(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + class BrokenDel: + def __del__(self) -> None: + raise ValueError("del is broken") + + def test_it() -> None: + obj = BrokenDel() + del obj + + def test_2(): pass + """ + ) + result = pytester.runpytest() + assert result.ret == pytest.ExitCode.TESTS_FAILED + assert result.parseoutcomes() == {"passed": 1, "failed": 1} diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_warning_types.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_warning_types.py new file mode 100644 index 0000000000..b49cc68f9c --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_warning_types.py @@ -0,0 +1,38 @@ +import inspect + +import pytest +from _pytest import warning_types +from _pytest.pytester import Pytester + + +@pytest.mark.parametrize( + "warning_class", + [ + w + for n, w in vars(warning_types).items() + if inspect.isclass(w) and issubclass(w, Warning) + ], +) +def test_warning_types(warning_class: UserWarning) -> None: + """Make sure all warnings declared in _pytest.warning_types are displayed as coming + from 'pytest' instead of the internal module (#5452). + """ + assert warning_class.__module__ == "pytest" + + +@pytest.mark.filterwarnings("error::pytest.PytestWarning") +def test_pytest_warnings_repr_integration_test(pytester: Pytester) -> None: + """Small integration test to ensure our small hack of setting the __module__ attribute + of our warnings actually works (#5452). + """ + pytester.makepyfile( + """ + import pytest + import warnings + + def test(): + warnings.warn(pytest.PytestWarning("some warning")) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["E pytest.PytestWarning: some warning"]) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/test_warnings.py b/testing/web-platform/tests/tools/third_party/pytest/testing/test_warnings.py new file mode 100644 index 0000000000..5663c46cea --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/test_warnings.py @@ -0,0 +1,775 @@ +import os +import warnings +from typing import List +from typing import Optional +from typing import Tuple + +import pytest +from _pytest.fixtures import FixtureRequest +from _pytest.pytester import Pytester + +WARNINGS_SUMMARY_HEADER = "warnings summary" + + +@pytest.fixture +def pyfile_with_warnings(pytester: Pytester, request: FixtureRequest) -> str: + """Create a test file which calls a function in a module which generates warnings.""" + pytester.syspathinsert() + test_name = request.function.__name__ + module_name = test_name.lstrip("test_") + "_module" + test_file = pytester.makepyfile( + """ + import {module_name} + def test_func(): + assert {module_name}.foo() == 1 + """.format( + module_name=module_name + ), + **{ + module_name: """ + import warnings + def foo(): + warnings.warn(UserWarning("user warning")) + warnings.warn(RuntimeWarning("runtime warning")) + return 1 + """, + }, + ) + return str(test_file) + + +@pytest.mark.filterwarnings("default::UserWarning", "default::RuntimeWarning") +def test_normal_flow(pytester: Pytester, pyfile_with_warnings) -> None: + """Check that the warnings section is displayed.""" + result = pytester.runpytest(pyfile_with_warnings) + result.stdout.fnmatch_lines( + [ + "*== %s ==*" % WARNINGS_SUMMARY_HEADER, + "test_normal_flow.py::test_func", + "*normal_flow_module.py:3: UserWarning: user warning", + '* warnings.warn(UserWarning("user warning"))', + "*normal_flow_module.py:4: RuntimeWarning: runtime warning", + '* warnings.warn(RuntimeWarning("runtime warning"))', + "* 1 passed, 2 warnings*", + ] + ) + + +@pytest.mark.filterwarnings("always::UserWarning") +def test_setup_teardown_warnings(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import warnings + import pytest + + @pytest.fixture + def fix(): + warnings.warn(UserWarning("warning during setup")) + yield + warnings.warn(UserWarning("warning during teardown")) + + def test_func(fix): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*== %s ==*" % WARNINGS_SUMMARY_HEADER, + "*test_setup_teardown_warnings.py:6: UserWarning: warning during setup", + '*warnings.warn(UserWarning("warning during setup"))', + "*test_setup_teardown_warnings.py:8: UserWarning: warning during teardown", + '*warnings.warn(UserWarning("warning during teardown"))', + "* 1 passed, 2 warnings*", + ] + ) + + +@pytest.mark.parametrize("method", ["cmdline", "ini"]) +def test_as_errors(pytester: Pytester, pyfile_with_warnings, method) -> None: + args = ("-W", "error") if method == "cmdline" else () + if method == "ini": + pytester.makeini( + """ + [pytest] + filterwarnings=error + """ + ) + # Use a subprocess, since changing logging level affects other threads + # (xdist). + result = pytester.runpytest_subprocess(*args, pyfile_with_warnings) + result.stdout.fnmatch_lines( + [ + "E UserWarning: user warning", + "as_errors_module.py:3: UserWarning", + "* 1 failed in *", + ] + ) + + +@pytest.mark.parametrize("method", ["cmdline", "ini"]) +def test_ignore(pytester: Pytester, pyfile_with_warnings, method) -> None: + args = ("-W", "ignore") if method == "cmdline" else () + if method == "ini": + pytester.makeini( + """ + [pytest] + filterwarnings= ignore + """ + ) + + result = pytester.runpytest(*args, pyfile_with_warnings) + result.stdout.fnmatch_lines(["* 1 passed in *"]) + assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() + + +@pytest.mark.filterwarnings("always::UserWarning") +def test_unicode(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import warnings + import pytest + + + @pytest.fixture + def fix(): + warnings.warn("测试") + yield + + def test_func(fix): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*== %s ==*" % WARNINGS_SUMMARY_HEADER, + "*test_unicode.py:7: UserWarning: \u6d4b\u8bd5*", + "* 1 passed, 1 warning*", + ] + ) + + +def test_works_with_filterwarnings(pytester: Pytester) -> None: + """Ensure our warnings capture does not mess with pre-installed filters (#2430).""" + pytester.makepyfile( + """ + import warnings + + class MyWarning(Warning): + pass + + warnings.filterwarnings("error", category=MyWarning) + + class TestWarnings(object): + def test_my_warning(self): + try: + warnings.warn(MyWarning("warn!")) + assert False + except MyWarning: + assert True + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*== 1 passed in *"]) + + +@pytest.mark.parametrize("default_config", ["ini", "cmdline"]) +def test_filterwarnings_mark(pytester: Pytester, default_config) -> None: + """Test ``filterwarnings`` mark works and takes precedence over command + line and ini options.""" + if default_config == "ini": + pytester.makeini( + """ + [pytest] + filterwarnings = always::RuntimeWarning + """ + ) + pytester.makepyfile( + """ + import warnings + import pytest + + @pytest.mark.filterwarnings('ignore::RuntimeWarning') + def test_ignore_runtime_warning(): + warnings.warn(RuntimeWarning()) + + @pytest.mark.filterwarnings('error') + def test_warning_error(): + warnings.warn(RuntimeWarning()) + + def test_show_warning(): + warnings.warn(RuntimeWarning()) + """ + ) + result = pytester.runpytest( + "-W always::RuntimeWarning" if default_config == "cmdline" else "" + ) + result.stdout.fnmatch_lines(["*= 1 failed, 2 passed, 1 warning in *"]) + + +def test_non_string_warning_argument(pytester: Pytester) -> None: + """Non-str argument passed to warning breaks pytest (#2956)""" + pytester.makepyfile( + """\ + import warnings + import pytest + + def test(): + warnings.warn(UserWarning(1, 'foo')) + """ + ) + result = pytester.runpytest("-W", "always::UserWarning") + result.stdout.fnmatch_lines(["*= 1 passed, 1 warning in *"]) + + +def test_filterwarnings_mark_registration(pytester: Pytester) -> None: + """Ensure filterwarnings mark is registered""" + pytester.makepyfile( + """ + import pytest + + @pytest.mark.filterwarnings('error') + def test_func(): + pass + """ + ) + result = pytester.runpytest("--strict-markers") + assert result.ret == 0 + + +@pytest.mark.filterwarnings("always::UserWarning") +def test_warning_captured_hook(pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_configure(config): + config.issue_config_time_warning(UserWarning("config warning"), stacklevel=2) + """ + ) + pytester.makepyfile( + """ + import pytest, warnings + + warnings.warn(UserWarning("collect warning")) + + @pytest.fixture + def fix(): + warnings.warn(UserWarning("setup warning")) + yield 1 + warnings.warn(UserWarning("teardown warning")) + + def test_func(fix): + warnings.warn(UserWarning("call warning")) + assert fix == 1 + """ + ) + + collected = [] + + class WarningCollector: + def pytest_warning_recorded(self, warning_message, when, nodeid, location): + collected.append((str(warning_message.message), when, nodeid, location)) + + result = pytester.runpytest(plugins=[WarningCollector()]) + result.stdout.fnmatch_lines(["*1 passed*"]) + + expected = [ + ("config warning", "config", ""), + ("collect warning", "collect", ""), + ("setup warning", "runtest", "test_warning_captured_hook.py::test_func"), + ("call warning", "runtest", "test_warning_captured_hook.py::test_func"), + ("teardown warning", "runtest", "test_warning_captured_hook.py::test_func"), + ] + for index in range(len(expected)): + collected_result = collected[index] + expected_result = expected[index] + + assert collected_result[0] == expected_result[0], str(collected) + assert collected_result[1] == expected_result[1], str(collected) + assert collected_result[2] == expected_result[2], str(collected) + + # NOTE: collected_result[3] is location, which differs based on the platform you are on + # thus, the best we can do here is assert the types of the parameters match what we expect + # and not try and preload it in the expected array + if collected_result[3] is not None: + assert type(collected_result[3][0]) is str, str(collected) + assert type(collected_result[3][1]) is int, str(collected) + assert type(collected_result[3][2]) is str, str(collected) + else: + assert collected_result[3] is None, str(collected) + + +@pytest.mark.filterwarnings("always::UserWarning") +def test_collection_warnings(pytester: Pytester) -> None: + """Check that we also capture warnings issued during test collection (#3251).""" + pytester.makepyfile( + """ + import warnings + + warnings.warn(UserWarning("collection warning")) + + def test_foo(): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*== %s ==*" % WARNINGS_SUMMARY_HEADER, + " *collection_warnings.py:3: UserWarning: collection warning", + ' warnings.warn(UserWarning("collection warning"))', + "* 1 passed, 1 warning*", + ] + ) + + +@pytest.mark.filterwarnings("always::UserWarning") +def test_mark_regex_escape(pytester: Pytester) -> None: + """@pytest.mark.filterwarnings should not try to escape regex characters (#3936)""" + pytester.makepyfile( + r""" + import pytest, warnings + + @pytest.mark.filterwarnings(r"ignore:some \(warning\)") + def test_foo(): + warnings.warn(UserWarning("some (warning)")) + """ + ) + result = pytester.runpytest() + assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() + + +@pytest.mark.filterwarnings("default::pytest.PytestWarning") +@pytest.mark.parametrize("ignore_pytest_warnings", ["no", "ini", "cmdline"]) +def test_hide_pytest_internal_warnings( + pytester: Pytester, ignore_pytest_warnings +) -> None: + """Make sure we can ignore internal pytest warnings using a warnings filter.""" + pytester.makepyfile( + """ + import pytest + import warnings + + warnings.warn(pytest.PytestWarning("some internal warning")) + + def test_bar(): + pass + """ + ) + if ignore_pytest_warnings == "ini": + pytester.makeini( + """ + [pytest] + filterwarnings = ignore::pytest.PytestWarning + """ + ) + args = ( + ["-W", "ignore::pytest.PytestWarning"] + if ignore_pytest_warnings == "cmdline" + else [] + ) + result = pytester.runpytest(*args) + if ignore_pytest_warnings != "no": + assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() + else: + result.stdout.fnmatch_lines( + [ + "*== %s ==*" % WARNINGS_SUMMARY_HEADER, + "*test_hide_pytest_internal_warnings.py:4: PytestWarning: some internal warning", + "* 1 passed, 1 warning *", + ] + ) + + +@pytest.mark.parametrize("ignore_on_cmdline", [True, False]) +def test_option_precedence_cmdline_over_ini( + pytester: Pytester, ignore_on_cmdline +) -> None: + """Filters defined in the command-line should take precedence over filters in ini files (#3946).""" + pytester.makeini( + """ + [pytest] + filterwarnings = error::UserWarning + """ + ) + pytester.makepyfile( + """ + import warnings + def test(): + warnings.warn(UserWarning('hello')) + """ + ) + args = ["-W", "ignore"] if ignore_on_cmdline else [] + result = pytester.runpytest(*args) + if ignore_on_cmdline: + result.stdout.fnmatch_lines(["* 1 passed in*"]) + else: + result.stdout.fnmatch_lines(["* 1 failed in*"]) + + +def test_option_precedence_mark(pytester: Pytester) -> None: + """Filters defined by marks should always take precedence (#3946).""" + pytester.makeini( + """ + [pytest] + filterwarnings = ignore + """ + ) + pytester.makepyfile( + """ + import pytest, warnings + @pytest.mark.filterwarnings('error') + def test(): + warnings.warn(UserWarning('hello')) + """ + ) + result = pytester.runpytest("-W", "ignore") + result.stdout.fnmatch_lines(["* 1 failed in*"]) + + +class TestDeprecationWarningsByDefault: + """ + Note: all pytest runs are executed in a subprocess so we don't inherit warning filters + from pytest's own test suite + """ + + def create_file(self, pytester: Pytester, mark="") -> None: + pytester.makepyfile( + """ + import pytest, warnings + + warnings.warn(DeprecationWarning("collection")) + + {mark} + def test_foo(): + warnings.warn(PendingDeprecationWarning("test run")) + """.format( + mark=mark + ) + ) + + @pytest.mark.parametrize("customize_filters", [True, False]) + def test_shown_by_default(self, pytester: Pytester, customize_filters) -> None: + """Show deprecation warnings by default, even if user has customized the warnings filters (#4013).""" + self.create_file(pytester) + if customize_filters: + pytester.makeini( + """ + [pytest] + filterwarnings = + once::UserWarning + """ + ) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines( + [ + "*== %s ==*" % WARNINGS_SUMMARY_HEADER, + "*test_shown_by_default.py:3: DeprecationWarning: collection", + "*test_shown_by_default.py:7: PendingDeprecationWarning: test run", + "* 1 passed, 2 warnings*", + ] + ) + + def test_hidden_by_ini(self, pytester: Pytester) -> None: + self.create_file(pytester) + pytester.makeini( + """ + [pytest] + filterwarnings = + ignore::DeprecationWarning + ignore::PendingDeprecationWarning + """ + ) + result = pytester.runpytest_subprocess() + assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() + + def test_hidden_by_mark(self, pytester: Pytester) -> None: + """Should hide the deprecation warning from the function, but the warning during collection should + be displayed normally. + """ + self.create_file( + pytester, + mark='@pytest.mark.filterwarnings("ignore::PendingDeprecationWarning")', + ) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines( + [ + "*== %s ==*" % WARNINGS_SUMMARY_HEADER, + "*test_hidden_by_mark.py:3: DeprecationWarning: collection", + "* 1 passed, 1 warning*", + ] + ) + + def test_hidden_by_cmdline(self, pytester: Pytester) -> None: + self.create_file(pytester) + result = pytester.runpytest_subprocess( + "-W", + "ignore::DeprecationWarning", + "-W", + "ignore::PendingDeprecationWarning", + ) + assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() + + def test_hidden_by_system(self, pytester: Pytester, monkeypatch) -> None: + self.create_file(pytester) + monkeypatch.setenv("PYTHONWARNINGS", "once::UserWarning") + result = pytester.runpytest_subprocess() + assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() + + +@pytest.mark.parametrize("change_default", [None, "ini", "cmdline"]) +def test_removed_in_x_warning_as_error(pytester: Pytester, change_default) -> None: + """This ensures that PytestRemovedInXWarnings raised by pytest are turned into errors. + + This test should be enabled as part of each major release, and skipped again afterwards + to ensure our deprecations are turning into warnings as expected. + """ + pytester.makepyfile( + """ + import warnings, pytest + def test(): + warnings.warn(pytest.PytestRemovedIn7Warning("some warning")) + """ + ) + if change_default == "ini": + pytester.makeini( + """ + [pytest] + filterwarnings = + ignore::pytest.PytestRemovedIn7Warning + """ + ) + + args = ( + ("-Wignore::pytest.PytestRemovedIn7Warning",) + if change_default == "cmdline" + else () + ) + result = pytester.runpytest(*args) + if change_default is None: + result.stdout.fnmatch_lines(["* 1 failed in *"]) + else: + assert change_default in ("ini", "cmdline") + result.stdout.fnmatch_lines(["* 1 passed in *"]) + + +class TestAssertionWarnings: + @staticmethod + def assert_result_warns(result, msg) -> None: + result.stdout.fnmatch_lines(["*PytestAssertRewriteWarning: %s*" % msg]) + + def test_tuple_warning(self, pytester: Pytester) -> None: + pytester.makepyfile( + """\ + def test_foo(): + assert (1,2) + """ + ) + result = pytester.runpytest() + self.assert_result_warns( + result, "assertion is always true, perhaps remove parentheses?" + ) + + +def test_warnings_checker_twice() -> None: + """Issue #4617""" + expectation = pytest.warns(UserWarning) + with expectation: + warnings.warn("Message A", UserWarning) + with expectation: + warnings.warn("Message B", UserWarning) + + +@pytest.mark.filterwarnings("always::UserWarning") +def test_group_warnings_by_message(pytester: Pytester) -> None: + pytester.copy_example("warnings/test_group_warnings_by_message.py") + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*== %s ==*" % WARNINGS_SUMMARY_HEADER, + "test_group_warnings_by_message.py::test_foo[[]0[]]", + "test_group_warnings_by_message.py::test_foo[[]1[]]", + "test_group_warnings_by_message.py::test_foo[[]2[]]", + "test_group_warnings_by_message.py::test_foo[[]3[]]", + "test_group_warnings_by_message.py::test_foo[[]4[]]", + "test_group_warnings_by_message.py::test_foo_1", + " */test_group_warnings_by_message.py:*: UserWarning: foo", + " warnings.warn(UserWarning(msg))", + "", + "test_group_warnings_by_message.py::test_bar[[]0[]]", + "test_group_warnings_by_message.py::test_bar[[]1[]]", + "test_group_warnings_by_message.py::test_bar[[]2[]]", + "test_group_warnings_by_message.py::test_bar[[]3[]]", + "test_group_warnings_by_message.py::test_bar[[]4[]]", + " */test_group_warnings_by_message.py:*: UserWarning: bar", + " warnings.warn(UserWarning(msg))", + "", + "-- Docs: *", + "*= 11 passed, 11 warnings *", + ], + consecutive=True, + ) + + +@pytest.mark.filterwarnings("always::UserWarning") +def test_group_warnings_by_message_summary(pytester: Pytester) -> None: + pytester.copy_example("warnings/test_group_warnings_by_message_summary") + pytester.syspathinsert() + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*== %s ==*" % WARNINGS_SUMMARY_HEADER, + "test_1.py: 21 warnings", + "test_2.py: 1 warning", + " */test_1.py:7: UserWarning: foo", + " warnings.warn(UserWarning(msg))", + "", + "test_1.py: 20 warnings", + " */test_1.py:7: UserWarning: bar", + " warnings.warn(UserWarning(msg))", + "", + "-- Docs: *", + "*= 42 passed, 42 warnings *", + ], + consecutive=True, + ) + + +def test_pytest_configure_warning(pytester: Pytester, recwarn) -> None: + """Issue 5115.""" + pytester.makeconftest( + """ + def pytest_configure(): + import warnings + + warnings.warn("from pytest_configure") + """ + ) + + result = pytester.runpytest() + assert result.ret == 5 + assert "INTERNALERROR" not in result.stderr.str() + warning = recwarn.pop() + assert str(warning.message) == "from pytest_configure" + + +class TestStackLevel: + @pytest.fixture + def capwarn(self, pytester: Pytester): + class CapturedWarnings: + captured: List[ + Tuple[warnings.WarningMessage, Optional[Tuple[str, int, str]]] + ] = [] + + @classmethod + def pytest_warning_recorded(cls, warning_message, when, nodeid, location): + cls.captured.append((warning_message, location)) + + pytester.plugins = [CapturedWarnings()] + + return CapturedWarnings + + def test_issue4445_rewrite(self, pytester: Pytester, capwarn) -> None: + """#4445: Make sure the warning points to a reasonable location + See origin of _issue_warning_captured at: _pytest.assertion.rewrite.py:241 + """ + pytester.makepyfile(some_mod="") + conftest = pytester.makeconftest( + """ + import some_mod + import pytest + + pytest.register_assert_rewrite("some_mod") + """ + ) + pytester.parseconfig() + + # with stacklevel=5 the warning originates from register_assert_rewrite + # function in the created conftest.py + assert len(capwarn.captured) == 1 + warning, location = capwarn.captured.pop() + file, lineno, func = location + + assert "Module already imported" in str(warning.message) + assert file == str(conftest) + assert func == "" # the above conftest.py + assert lineno == 4 + + def test_issue4445_preparse(self, pytester: Pytester, capwarn) -> None: + """#4445: Make sure the warning points to a reasonable location + See origin of _issue_warning_captured at: _pytest.config.__init__.py:910 + """ + pytester.makeconftest( + """ + import nothing + """ + ) + pytester.parseconfig("--help") + + # with stacklevel=2 the warning should originate from config._preparse and is + # thrown by an erroneous conftest.py + assert len(capwarn.captured) == 1 + warning, location = capwarn.captured.pop() + file, _, func = location + + assert "could not load initial conftests" in str(warning.message) + assert f"config{os.sep}__init__.py" in file + assert func == "_preparse" + + @pytest.mark.filterwarnings("default") + def test_conftest_warning_captured(self, pytester: Pytester) -> None: + """Warnings raised during importing of conftest.py files is captured (#2891).""" + pytester.makeconftest( + """ + import warnings + warnings.warn(UserWarning("my custom warning")) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + ["conftest.py:2", "*UserWarning: my custom warning*"] + ) + + def test_issue4445_import_plugin(self, pytester: Pytester, capwarn) -> None: + """#4445: Make sure the warning points to a reasonable location""" + pytester.makepyfile( + some_plugin=""" + import pytest + pytest.skip("thing", allow_module_level=True) + """ + ) + pytester.syspathinsert() + pytester.parseconfig("-p", "some_plugin") + + # with stacklevel=2 the warning should originate from + # config.PytestPluginManager.import_plugin is thrown by a skipped plugin + + assert len(capwarn.captured) == 1 + warning, location = capwarn.captured.pop() + file, _, func = location + + assert "skipped plugin 'some_plugin': thing" in str(warning.message) + assert f"config{os.sep}__init__.py" in file + assert func == "_warn_about_skipped_plugins" + + def test_issue4445_issue5928_mark_generator(self, pytester: Pytester) -> None: + """#4445 and #5928: Make sure the warning from an unknown mark points to + the test file where this mark is used. + """ + testfile = pytester.makepyfile( + """ + import pytest + + @pytest.mark.unknown + def test_it(): + pass + """ + ) + result = pytester.runpytest_subprocess() + # with stacklevel=2 the warning should originate from the above created test file + result.stdout.fnmatch_lines_random( + [ + f"*{testfile}:3*", + "*Unknown pytest.mark.unknown*", + ] + ) diff --git a/testing/web-platform/tests/tools/third_party/pytest/testing/typing_checks.py b/testing/web-platform/tests/tools/third_party/pytest/testing/typing_checks.py new file mode 100644 index 0000000000..0a6b5ad284 --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/testing/typing_checks.py @@ -0,0 +1,24 @@ +"""File for checking typing issues. + +This file is not executed, it is only checked by mypy to ensure that +none of the code triggers any mypy errors. +""" +import pytest + + +# Issue #7488. +@pytest.mark.xfail(raises=RuntimeError) +def check_mark_xfail_raises() -> None: + pass + + +# Issue #7494. +@pytest.fixture(params=[(0, 0), (1, 1)], ids=lambda x: str(x[0])) +def check_fixture_ids_callable() -> None: + pass + + +# Issue #7494. +@pytest.mark.parametrize("func", [str, int], ids=lambda x: str(x.__name__)) +def check_parametrize_ids_callable(func) -> None: + pass diff --git a/testing/web-platform/tests/tools/third_party/pytest/tox.ini b/testing/web-platform/tests/tools/third_party/pytest/tox.ini new file mode 100644 index 0000000000..b2f90008ce --- /dev/null +++ b/testing/web-platform/tests/tools/third_party/pytest/tox.ini @@ -0,0 +1,184 @@ +[tox] +isolated_build = True +minversion = 3.20.0 +distshare = {homedir}/.tox/distshare +envlist = + linting + py36 + py37 + py38 + py39 + py310 + py311 + pypy3 + py37-{pexpect,xdist,unittestextras,numpy,pluggymain} + doctesting + plugins + py37-freeze + docs + docs-checklinks + + + +[testenv] +commands = + {env:_PYTEST_TOX_COVERAGE_RUN:} pytest {posargs:{env:_PYTEST_TOX_DEFAULT_POSARGS:}} + doctesting: {env:_PYTEST_TOX_COVERAGE_RUN:} pytest --doctest-modules --pyargs _pytest + coverage: coverage combine + coverage: coverage report -m +passenv = USER USERNAME COVERAGE_* PYTEST_ADDOPTS TERM SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST +setenv = + _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} + + # Configuration to run with coverage similar to CI, e.g. + # "tox -e py37-coverage". + coverage: _PYTEST_TOX_COVERAGE_RUN=coverage run -m + coverage: _PYTEST_TOX_EXTRA_DEP=coverage-enable-subprocess + coverage: COVERAGE_FILE={toxinidir}/.coverage + coverage: COVERAGE_PROCESS_START={toxinidir}/.coveragerc + + doctesting: _PYTEST_TOX_POSARGS_DOCTESTING=doc/en + + nobyte: PYTHONDONTWRITEBYTECODE=1 + + lsof: _PYTEST_TOX_POSARGS_LSOF=--lsof + + xdist: _PYTEST_TOX_POSARGS_XDIST=-n auto +extras = testing +deps = + doctesting: PyYAML + numpy: numpy>=1.19.4 + pexpect: pexpect>=4.8.0 + pluggymain: pluggy @ git+https://github.com/pytest-dev/pluggy.git + unittestextras: twisted + unittestextras: asynctest + xdist: pytest-xdist>=2.1.0 + xdist: -e . + {env:_PYTEST_TOX_EXTRA_DEP:} + +[testenv:linting] +skip_install = True +basepython = python3 +deps = pre-commit>=2.9.3 +commands = pre-commit run --all-files --show-diff-on-failure {posargs:} + +[testenv:docs] +basepython = python3 +usedevelop = True +deps = + -r{toxinidir}/doc/en/requirements.txt + # https://github.com/twisted/towncrier/issues/340 + towncrier<21.3.0 +commands = + python scripts/towncrier-draft-to-file.py + # the '-t changelog_towncrier_draft' tags makes sphinx include the draft + # changelog in the docs; this does not happen on ReadTheDocs because it uses + # the standard sphinx command so the 'changelog_towncrier_draft' is never set there + sphinx-build -W --keep-going -b html doc/en doc/en/_build/html -t changelog_towncrier_draft {posargs:} + +[testenv:docs-checklinks] +basepython = python3 +usedevelop = True +changedir = doc/en +deps = -r{toxinidir}/doc/en/requirements.txt +commands = + sphinx-build -W -q --keep-going -b linkcheck . _build + +[testenv:regen] +changedir = doc/en +basepython = python3 +passenv = SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST +deps = + dataclasses + PyYAML + regendoc>=0.8.1 + sphinx +whitelist_externals = + make +commands = + make regen + +[testenv:plugins] +# use latest versions of all plugins, including pre-releases +pip_pre=true +# use latest pip and new dependency resolver (#7783) +download=true +install_command=python -m pip --use-feature=2020-resolver install {opts} {packages} +changedir = testing/plugins_integration +deps = -rtesting/plugins_integration/requirements.txt +setenv = + PYTHONPATH=. +commands = + pip check + pytest bdd_wallet.py + pytest --cov=. simple_integration.py + pytest --ds=django_settings simple_integration.py + pytest --html=simple.html simple_integration.py + pytest --reruns 5 simple_integration.py + pytest pytest_anyio_integration.py + pytest pytest_asyncio_integration.py + pytest pytest_mock_integration.py + pytest pytest_trio_integration.py + pytest pytest_twisted_integration.py + pytest simple_integration.py --force-sugar --flakes + +[testenv:py37-freeze] +changedir = testing/freeze +deps = + pyinstaller +commands = + {envpython} create_executable.py + {envpython} tox_run.py + +[testenv:release] +description = do a release, required posarg of the version number +basepython = python3 +usedevelop = True +passenv = * +deps = + colorama + github3.py + pre-commit>=2.9.3 + wheel + # https://github.com/twisted/towncrier/issues/340 + towncrier<21.3.0 +commands = python scripts/release.py {posargs} + +[testenv:prepare-release-pr] +description = prepare a release PR from a manual trigger in GitHub actions +usedevelop = {[testenv:release]usedevelop} +passenv = {[testenv:release]passenv} +deps = {[testenv:release]deps} +commands = python scripts/prepare-release-pr.py {posargs} + +[testenv:publish-gh-release-notes] +description = create GitHub release after deployment +basepython = python3 +usedevelop = True +passenv = GH_RELEASE_NOTES_TOKEN GITHUB_REF GITHUB_REPOSITORY +deps = + github3.py + pypandoc +commands = python scripts/publish-gh-release-notes.py {posargs} + +[flake8] +max-line-length = 120 +extend-ignore = + ; whitespace before ':' + E203 + ; Missing Docstrings + D100,D101,D102,D103,D104,D105,D106,D107 + ; Whitespace Issues + D202,D203,D204,D205,D209,D213 + ; Quotes Issues + D302 + ; Docstring Content Issues + D400,D401,D401,D402,D405,D406,D407,D408,D409,D410,D411,D412,D413,D414,D415,D416,D417 + + +[isort] +; This config mimics what reorder-python-imports does. +force_single_line = 1 +known_localfolder = pytest,_pytest +known_third_party = test_source,test_excinfo +force_alphabetical_sort_within_sections = 1 -- cgit v1.2.3