diff options
Diffstat (limited to 'src/debputy/lsp/lsp_debian_control_reference_data.py')
| -rw-r--r-- | src/debputy/lsp/lsp_debian_control_reference_data.py | 578 |
1 files changed, 565 insertions, 13 deletions
diff --git a/src/debputy/lsp/lsp_debian_control_reference_data.py b/src/debputy/lsp/lsp_debian_control_reference_data.py index 689866f..3e16f3c 100644 --- a/src/debputy/lsp/lsp_debian_control_reference_data.py +++ b/src/debputy/lsp/lsp_debian_control_reference_data.py @@ -2,6 +2,7 @@ import dataclasses import functools import itertools import re +import sys import textwrap from abc import ABC from enum import Enum, auto @@ -17,6 +18,7 @@ from typing import ( Union, Callable, Tuple, + Any, ) from debian.debian_support import DpkgArchTable @@ -37,8 +39,22 @@ from debputy.lsp.vendoring._deb822_repro.parsing import ( LIST_SPACE_SEPARATED_INTERPRETATION, Deb822ParagraphElement, Deb822FileElement, + Interpretation, + LIST_COMMA_SEPARATED_INTERPRETATION, + ListInterpretation, + _parsed_value_render_factory, + Deb822ParsedValueElement, + LIST_UPLOADERS_INTERPRETATION, + _parse_whitespace_list_value, +) +from debputy.lsp.vendoring._deb822_repro.tokens import ( + Deb822FieldNameToken, + _value_line_tokenizer, + Deb822ValueToken, + Deb822Token, + _RE_WHITESPACE_SEPARATED_WORD_LIST, + Deb822SpaceSeparatorToken, ) -from debputy.lsp.vendoring._deb822_repro.tokens import Deb822FieldNameToken from debputy.util import PKGNAME_REGEX try: @@ -55,6 +71,43 @@ F = TypeVar("F", bound="Deb822KnownField") S = TypeVar("S", bound="StanzaMetadata") +# FIXME: should go into python3-debian +_RE_COMMA = re.compile("([^,]*),([^,]*)") + + +@_value_line_tokenizer +def comma_or_space_split_tokenizer(v): + # type: (str) -> Iterable[Deb822Token] + assert "\n" not in v + for match in _RE_WHITESPACE_SEPARATED_WORD_LIST.finditer(v): + space_before, word, space_after = match.groups() + if space_before: + yield Deb822SpaceSeparatorToken(sys.intern(space_before)) + if "," in word: + for m in _RE_COMMA.finditer(word): + word_before, word_after = m.groups() + if word_before: + yield Deb822ValueToken(word_before) + # ... not quite a whitespace, but it is too much pain to make it a non-whitespace token. + yield Deb822SpaceSeparatorToken(",") + if word_after: + yield Deb822ValueToken(word_after) + else: + yield Deb822ValueToken(word) + if space_after: + yield Deb822SpaceSeparatorToken(sys.intern(space_after)) + + +# FIXME: should go into python3-debian +LIST_COMMA_OR_SPACE_SEPARATED_INTERPRETATION = ListInterpretation( + comma_or_space_split_tokenizer, + _parse_whitespace_list_value, + Deb822ParsedValueElement, + Deb822SpaceSeparatorToken, + Deb822SpaceSeparatorToken, + _parsed_value_render_factory, +) + CustomFieldCheck = Callable[ [ "F", @@ -387,13 +440,17 @@ def _combined_custom_field_check(*checks: CustomFieldCheck) -> CustomFieldCheck: class FieldValueClass(Enum): - SINGLE_VALUE = auto() - SPACE_SEPARATED_LIST = auto() - BUILD_PROFILES_LIST = auto() - COMMA_SEPARATED_LIST = auto() - COMMA_SEPARATED_EMAIL_LIST = auto() - FREE_TEXT_FIELD = auto() - DEP5_FILE_LIST = auto() + SINGLE_VALUE = auto(), LIST_SPACE_SEPARATED_INTERPRETATION + SPACE_SEPARATED_LIST = auto(), LIST_SPACE_SEPARATED_INTERPRETATION + BUILD_PROFILES_LIST = auto(), None # TODO + COMMA_SEPARATED_LIST = auto(), LIST_COMMA_SEPARATED_INTERPRETATION + COMMA_SEPARATED_EMAIL_LIST = auto(), LIST_UPLOADERS_INTERPRETATION + COMMA_OR_SPACE_SEPARATED_LIST = auto(), LIST_COMMA_OR_SPACE_SEPARATED_INTERPRETATION + FREE_TEXT_FIELD = auto(), None + DEP5_FILE_LIST = auto(), None # TODO + + def interpreter(self) -> Optional[Interpretation[Any]]: + return self.value[1] @dataclasses.dataclass(slots=True, frozen=True) @@ -505,10 +562,11 @@ class Deb822KnownField: ) -> Iterable[Diagnostic]: unknown_value_severity = self.unknown_value_diagnostic_severity allowed_values = self.known_values - if not allowed_values: + interpreter = self.field_value_class.interpreter() + if not allowed_values or interpreter is None: return hint_text = None - values = kvpair.interpret_as(LIST_SPACE_SEPARATED_INTERPRETATION) + values = kvpair.interpret_as(interpreter) value_off = kvpair.value_element.position_in_parent().relative_to( field_position_te ) @@ -1053,9 +1111,9 @@ SOURCE_FIELDS = _fields( If it breaks and you cannot figure out how to fix it, then reset the field to `binary-targets` and move on until you have time to fix it. - The default value for this field depends on the ``dpkg-build-api`` version. If the package - `` Build-Depends`` on ``dpkg-build-api (>= 1)`` or later, the default is ``no``. Otherwise, - the default is ``binary-target`` + The default value for this field depends on the `dpkg-build-api` version. If the package + ` Build-Depends` on `dpkg-build-api (>= 1)` or later, the default is `no`. Otherwise, + the default is `binary-target` Note it is **not** possible to require running the package as "true root". """ @@ -2150,6 +2208,472 @@ _DEP5_LICENSE_FIELDS = _fields( ), ) +_DTESTSCTRL_FIELDS = _fields( + Deb822KnownField( + "Architecture", + FieldValueClass.SPACE_SEPARATED_LIST, + unknown_value_diagnostic_severity=None, + known_values=_allowed_values(*dpkg_arch_and_wildcards()), + hover_text=textwrap.dedent( + """\ + When package tests are only supported on a limited set of + architectures, or are known to not work on a particular (set of) + architecture(s), this field can be used to define the supported + architectures. The autopkgtest will be skipped when the + architecture of the testbed doesn't match the content of this + field. The format is the same as in (Build-)Depends, with the + understanding that `all` is not allowed, and `any` means that + the test will be run on every architecture, which is the default + when not specifying this field at all. + """ + ), + ), + Deb822KnownField( + "Classes", + FieldValueClass.FREE_TEXT_FIELD, + hover_text=textwrap.dedent( + """\ + Most package tests should work in a minimal environment and are + usually not hardware specific. However, some packages like the + kernel, X.org, or graphics drivers should be tested on particular + hardware, and also run on a set of different platforms rather than + just a single virtual testbeds. + + This field can specify a list of abstract class names such as + "desktop" or "graphics-driver". Consumers of autopkgtest can then + map these class names to particular machines/platforms/policies. + Unknown class names should be ignored. + + This is purely an informational field for autopkgtest itself and + will be ignored. + """ + ), + ), + Deb822KnownField( + "Depends", + FieldValueClass.COMMA_SEPARATED_LIST, + default_value="@", + hover_text="""\ + Declares that the specified packages must be installed for the test + to go ahead. This supports all features of dpkg dependencies, including + the architecture qualifiers (see + https://www.debian.org/doc/debian-policy/ch-relationships.html), + plus the following extensions: + + `@` stands for the package(s) generated by the source package + containing the tests; each dependency (strictly, or-clause, which + may contain `|`s but not commas) containing `@` is replicated + once for each such binary package, with the binary package name + substituted for each `@` (but normally `@` should occur only + once and without a version restriction). + + `@builddeps@` will be replaced by the package's + `Build-Depends:`, `Build-Depends-Indep:`, `Build-Depends-Arch:`, and + `build-essential`. This is useful if you have many build + dependencies which are only necessary for running the test suite and + you don't want to replicate them in the test `Depends:`. However, + please use this sparingly, as this can easily lead to missing binary + package dependencies being overlooked if they get pulled in via + build dependencies. + + `@recommends@` stands for all the packages listed in the + `Recommends:` fields of all the binary packages mentioned in the + `debian/control` file. Please note that variables are stripped, + so if some required test dependencies aren't explicitly mentioned, + they may not be installed. + + If no Depends field is present, `Depends: @` is assumed. Note that + the source tree's Build-Dependencies are *not* necessarily + installed, and if you specify any Depends, no binary packages from + the source are installed unless explicitly requested. + """, + ), + Deb822KnownField( + "Features", + FieldValueClass.COMMA_OR_SPACE_SEPARATED_LIST, + hover_text=textwrap.dedent( + """\ + Declares some additional capabilities or good properties of the + tests defined in this stanza. Any unknown features declared will be + completely ignored. See below for the defined features. + + Features are separated by commas and/or whitespace. + """ + ), + ), + Deb822KnownField( + "Restrictions", + FieldValueClass.COMMA_OR_SPACE_SEPARATED_LIST, + unknown_value_diagnostic_severity=DiagnosticSeverity.Warning, + known_values=_allowed_values( + Keyword( + "allow-stderr", + hover_text=textwrap.dedent( + """\ + Output to stderr is not considered a failure. This is useful for + tests which write e. g. lots of logging to stderr. + """ + ), + ), + Keyword( + "breaks-testbed", + hover_text=textwrap.dedent( + """\ + The test, when run, is liable to break the testbed system. This + includes causing data loss, causing services that the machine is + running to malfunction, or permanently disabling services; it does + not include causing services on the machine to temporarily fail. + + When this restriction is present the test will usually be skipped + unless the testbed's virtualisation arrangements are sufficiently + powerful, or alternatively if the user explicitly requests. + """ + ), + ), + Keyword( + "build-needed", + hover_text=textwrap.dedent( + """\ + The tests need to be run from a built source tree. The test runner + will build the source tree (honouring the source package's build + dependencies), before running the tests. However, the tests are + *not* entitled to assume that the source package's build + dependencies will be installed when the test is run. + + Please use this considerately, as for large builds it unnecessarily + builds the entire project when you only need a tiny subset (like the + tests/ subdirectory). It is often possible to run `make -C tests` + instead, or copy the test code to `$AUTOPKGTEST_TMP` and build it + there with some custom commands. This cuts down the load on the + Continuous Integration servers and also makes tests more robust as + it prevents accidentally running them against the built source tree + instead of the installed packages. + """ + ), + ), + Keyword( + "flaky", + hover_text=textwrap.dedent( + """\ + The test is expected to fail intermittently, and is not suitable for + gating continuous integration. This indicates a bug in either the + package under test, a dependency or the test itself, but such bugs + can be difficult to fix, and it is often difficult to know when the + bug has been fixed without running the test for a while. If a + `flaky` test succeeds, it will be treated like any other + successful test, but if it fails it will be treated as though it + had been skipped. + """ + ), + ), + Keyword( + "hint-testsuite-triggers", + hover_text=textwrap.dedent( + """\ + This test exists purely as a hint to suggest when rerunning the + tests is likely to be useful. Specifically, it exists to + influence the way dpkg-source generates the Testsuite-Triggers + .dsc header from test metadata: the Depends for this test are + to be added to Testsuite-Triggers. (Just as they are for any other + test.) + + The test with the hint-testsuite-triggers restriction should not + actually be run. + + The packages listed as Depends for this test are usually indirect + dependencies, updates to which are considered to pose a risk of + regressions in other tests defined in this package. + + There is currently no way to specify this hint on a per-test + basis; but in any case the debian.org machinery is not able to + think about triggering individual tests. + """ + ), + ), + Keyword( + "isolation-container", + hover_text=textwrap.dedent( + """\ + The test wants to start services or open network TCP ports. This + commonly fails in a simple chroot/schroot, so tests need to be run + in their own container (e. g. autopkgtest-virt-lxc) or their own + machine/VM (e. g. autopkgtest-virt-qemu or autopkgtest-virt-null). + When running the test in a virtualization server which does not + provide this (like autopkgtest-schroot) it will be skipped. + + Tests may assume that this restriction implies that process 1 in the + container's process namespace is a system service manager (init system) + such as systemd or sysvinit + sysv-rc, and therefore system services + are available via the `service(8)`, `invoke-rc.d(8)` and + `update-rc.d(8))` interfaces. + + Tests must not assume that a specific init system is in use: a + dependency such as `systemd-sysv` or `sysvinit-core` does not work + in practice, because switching the init system often cannot be done + automatically. Tests that require a specific init system should use the + `skippable` restriction, and skip the test if the required init system + was not detected. + + Many implementations of the `isolation-container` restriction will + also provide `systemd-logind(8)` or a compatible interface, but this + is not guaranteed. Tests requiring a login session registered with + logind should declare a dependency on `default-logind | logind` + or on a more specific implementation of `logind`, and should use the + `skippable` restriction to exit gracefully if its functionality is + not available at runtime. + + """ + ), + ), + Keyword( + "isolation-machine", + hover_text=textwrap.dedent( + """\ + The test wants to interact with the kernel, reboot the machine, or + other things which fail in a simple schroot and even a container. + Those tests need to be run in their own machine/VM (e. g. + autopkgtest-virt-qemu or autopkgtest-virt-null). When running the + test in a virtualization server which does not provide this it will + be skipped. + + This restriction also provides the same facilities as + `isolation-container`. + """ + ), + ), + Keyword( + "needs-internet", + hover_text=textwrap.dedent( + """\ + The test needs unrestricted internet access, e.g. to download test data + that's not shipped as a package, or to test a protocol implementation + against a test server. Please also see the note about Network access later + in this document. + """ + ), + ), + Keyword( + "needs-reboot", + hover_text=textwrap.dedent( + """\ + The test wants to reboot the machine using + `/tmp/autopkgtest-reboot`. + """ + ), + ), + Keyword( + "needs-recommends", + is_obsolete=True, + hover_text=textwrap.dedent( + """\ + Please use `@recommends@` in your test `Depends:` instead. + """ + ), + ), + Keyword( + "needs-root", + hover_text=textwrap.dedent( + """\ + The test script must be run as root. + + While running tests with this restriction, some test runners will + set the `AUTOPKGTEST_NORMAL_USER` environment variable to the name + of an ordinary user account. If so, the test script may drop + privileges from root to that user, for example via the `runuser` + command. Test scripts must not assume that this environment variable + will always be set. + + For tests that declare both the `needs-root` and `isolation-machine` + restrictions, the test may assume that it has "global root" with full + control over the kernel that is running the test, and not just root + in a container (more formally, it has uid 0 and full capabilities in + the initial user namespace as defined in `user_namespaces(7)`). + For example, it can expect that mounting block devices will succeed. + + For tests that declare the `needs-root` restriction but not the + `isolation-machine` restriction, the test will be run as uid 0 in + a user namespace with a reasonable range of system and user uids + available, but will not necessarily have full control over the kernel, + and in particular it is not guaranteed to have elevated capabilities + in the initial user namespace as defined by `user_namespaces(7)`. + For example, it might be run in a namespace where uid 0 is mapped to + an ordinary uid in the initial user namespace, or it might run in a + Docker-style container where global uid 0 is used but its ability to + carry out operations that affect the whole system is restricted by + capabilities and system call filtering. Tests requiring particular + privileges should use the `skippable` restriction to check for + required functionality at runtime. + """ + ), + ), + Keyword( + "needs-sudo", + hover_text=textwrap.dedent( + """\ + The test script needs to be run as a non-root user who is a member of + the `sudo` group, and has the ability to elevate privileges to root + on-demand. + + This is useful for testing user components which should not normally + be run as root, in test scenarios that require configuring a system + service to support the test. For example, gvfs has a test-case which + uses sudo for privileged configuration of a Samba server, so that + the unprivileged gvfs service under test can communicate with that server. + + While running a test with this restriction, `sudo(8)` will be + installed and configured to allow members of the `sudo` group to run + any command without password authentication. + + Because the test user is a member of the `sudo` group, they will + also gain the ability to take any other privileged actions that are + controlled by membership in that group. In particular, several packages + install `polkit(8)` policies allowing members of group `sudo` to + take administrative actions with or without authentication. + + If the test requires access to additional privileged actions, it may + use its access to `sudo(8)` to install additional configuration + files, for example configuring `polkit(8)` or `doas.conf(5)` + to allow running `pkexec(1)` or `doas(1)` without authentication. + + Commands run via `sudo(8)` or another privilege-elevation tool could + be run with either "global root" or root in a container, depending + on the presence or absence of the `isolation-machine` restriction, + in the same way described for `needs-root`. + """ + ), + ), + Keyword( + "rw-build-tree", + hover_text=textwrap.dedent( + """\ + The test(s) needs write access to the built source tree (so it may + need to be copied first). Even with this restriction, the test is + not allowed to make any change to the built source tree which (i) + isn't cleaned up by debian/rules clean, (ii) affects the future + results of any test, or (iii) affects binary packages produced by + the build tree in the future. + """ + ), + ), + Keyword( + "skip-not-installable", + hover_text=textwrap.dedent( + """\ + This restrictions may cause a test to miss a regression due to + installability issues, so use with caution. If one only wants to + skip certain architectures, use the `Architecture` field for + that. + + This test might have test dependencies that can't be fulfilled in + all suites or in derivatives. Therefore, when apt-get installs the + test dependencies, it will fail. Don't treat this as a test + failure, but instead treat it as if the test was skipped. + """ + ), + ), + Keyword( + "skippable", + hover_text=textwrap.dedent( + """\ + The test might need to be skipped for reasons that cannot be + described by an existing restriction such as isolation-machine or + breaks-testbed, but must instead be detected at runtime. If the + test exits with status 77 (a convention borrowed from Automake), it + will be treated as though it had been skipped. If it exits with any + other status, its success or failure will be derived from the exit + status and stderr as usual. Test authors must be careful to ensure + that `skippable` tests never exit with status 77 for reasons that + should be treated as a failure. + """ + ), + ), + Keyword( + "superficial", + hover_text=textwrap.dedent( + """\ + The test does not provide significant test coverage, so if it + passes, that does not necessarily mean that the package under test + is actually functional. If a `superficial` test fails, it will be + treated like any other failing test, but if it succeeds, this is + only a weak indication of success. Continuous integration systems + should treat a package where all non-superficial tests are skipped as + equivalent to a package where all tests are skipped. + + For example, a C library might have a superficial test that simply + compiles, links and executes a "hello world" program against the + library under test but does not attempt to make use of the library's + functionality, while a Python or Perl library might have a + superficial test that runs `import foo` or `require Foo;` but + does not attempt to use the library beyond that. + """ + ), + ), + ), + hover_text=textwrap.dedent( + """\ + Declares some restrictions or problems with the tests defined in + this stanza. Depending on the test environment capabilities, user + requests, and so on, restrictions can cause tests to be skipped or + can cause the test to be run in a different manner. Tests which + declare unknown restrictions will be skipped. See below for the + defined restrictions. + + Restrictions are separated by commas and/or whitespace. + """ + ), + ), + Deb822KnownField( + "Tests", + FieldValueClass.COMMA_OR_SPACE_SEPARATED_LIST, + hover_text=textwrap.dedent( + """\ + This field names the tests which are defined by this stanza, and map + to executables/scripts in the test directory. All of the other + fields in the same stanza apply to all of the named tests. Either + this field or `Test-Command:` must be present. + + Test names are separated by comma and/or whitespace and should + contain only characters which are legal in package names. It is + permitted, but not encouraged, to use upper-case characters as well. + """ + ), + ), + Deb822KnownField( + "Test-Command", + FieldValueClass.FREE_TEXT_FIELD, + hover_text=textwrap.dedent( + """\ + If your test only contains a shell command or two, or you want to + re-use an existing upstream test executable and just need to wrap it + with some command like `dbus-launch` or `env`, you can use this + field to specify the shell command directly. It will be run under + `bash -e`. This is mutually exclusive with the `Tests:` field. + + This is also useful for running the same script under different + interpreters and/or with different dependencies, such as + `Test-Command: python debian/tests/mytest.py` and + `Test-Command: python3 debian/tests/mytest.py`. + """ + ), + ), + Deb822KnownField( + "Test-Directory", + FieldValueClass.FREE_TEXT_FIELD, # TODO: Single path + hover_text=textwrap.dedent( + """\ + Replaces the path segment `debian/tests` in the filenames of the + test programs with `path`. I. e., the tests are run by executing + `built/source/tree/path/testname`. `path` must be a relative + path and is interpreted starting from the root of the built source + tree. + + This allows tests to live outside the debian/ metadata area, so that + they can more palatably be shared with non-Debian distributions. + """ + ), + ), +) + @dataclasses.dataclass(slots=True, frozen=True) class StanzaMetadata(Mapping[str, F], Generic[F], ABC): @@ -2196,6 +2720,17 @@ class DctrlStanzaMetadata(StanzaMetadata[DctrlKnownField]): pass +@dataclasses.dataclass(slots=True, frozen=True) +class DTestsCtrlStanzaMetadata(StanzaMetadata[Deb822KnownField]): + + def stanza_diagnostics( + self, + stanza: Deb822ParagraphElement, + stanza_position_in_file: "TEPosition", + ) -> Iterable[Diagnostic]: + pass + + class Deb822FileMetadata(Generic[S]): def classify_stanza(self, stanza: Deb822ParagraphElement, stanza_idx: int) -> S: return self.guess_stanza_classification_by_idx(stanza_idx) @@ -2241,6 +2776,8 @@ _DEP5_LICENSE_STANZA = Dep5StanzaMetadata( _DEP5_LICENSE_FIELDS, ) +_DTESTSCTRL_STANZA = DTestsCtrlStanzaMetadata("Tests", _DTESTSCTRL_FIELDS) + class Dep5FileMetadata(Deb822FileMetadata[Dep5StanzaMetadata]): def classify_stanza(self, stanza: Deb822ParagraphElement, stanza_idx: int) -> S: @@ -2292,3 +2829,18 @@ class DctrlFileMetadata(Deb822FileMetadata[DctrlStanzaMetadata]): if item == "Package": return _DCTRL_PACKAGE_STANZA raise KeyError(item) + + +class DTestsCtrlFileMetadata(Deb822FileMetadata[DctrlStanzaMetadata]): + def guess_stanza_classification_by_idx(self, stanza_idx: int) -> S: + if stanza_idx >= 0: + return _DTESTSCTRL_STANZA + raise ValueError("The stanza_idx must be 0 or greater") + + def stanza_types(self) -> Iterable[S]: + yield _DTESTSCTRL_STANZA + + def __getitem__(self, item: str) -> S: + if item == "Tests": + return _DTESTSCTRL_STANZA + raise KeyError(item) |