diff options
| -rw-r--r-- | GETTING-STARTED-WITH-dh-debputy.md | 53 | ||||
| -rw-r--r-- | MANIFEST-FORMAT.md | 102 | ||||
| -rw-r--r-- | src/debputy/commands/debputy_cmd/plugin_cmds.py | 37 | ||||
| -rw-r--r-- | src/debputy/deb_packaging_support.py | 202 | ||||
| -rw-r--r-- | src/debputy/highlevel_manifest.py | 4 | ||||
| -rw-r--r-- | src/debputy/highlevel_manifest_parser.py | 7 | ||||
| -rw-r--r-- | src/debputy/lsp/logins-and-people.dic | 8 | ||||
| -rw-r--r-- | src/debputy/manifest_parser/declarative_parser.py | 174 | ||||
| -rw-r--r-- | src/debputy/plugin/api/impl.py | 33 | ||||
| -rw-r--r-- | src/debputy/plugin/api/impl_types.py | 2 | ||||
| -rw-r--r-- | src/debputy/plugin/api/spec.py | 6 | ||||
| -rw-r--r-- | src/debputy/plugin/debputy/binary_package_rules.py | 235 | ||||
| -rw-r--r-- | src/debputy/plugin/debputy/package_processors.py | 5 | ||||
| -rw-r--r-- | src/debputy/plugin/debputy/service_management.py | 14 |
14 files changed, 776 insertions, 106 deletions
diff --git a/GETTING-STARTED-WITH-dh-debputy.md b/GETTING-STARTED-WITH-dh-debputy.md index 227f6d7..ac0b034 100644 --- a/GETTING-STARTED-WITH-dh-debputy.md +++ b/GETTING-STARTED-WITH-dh-debputy.md @@ -169,13 +169,9 @@ commands. Other tools will have some form of support (often at least a commonly * `dh_installemacsen` **(!)** * `dh_installinfo` * `dh_installinit` - - Only default service mode (enable, standard restart) are supported. Any use of `--no-start`, etc. - is unsupported * `dh_installsysusers` * `dh_installtmpfiles` * `dh_installsystemd` - - Only default service mode (enable, standard restart) are supported. Any use of `--no-start`, etc. - is unsupported * `dh_installsystemduser` **(!)** * `dh_installmenu` **(!)** * `dh_installmime` @@ -316,6 +312,55 @@ _Remember to merge your manifest with previous steps rather than replacing it!_ `debputy migrate-from-dh` will merge its changes into existing manifests and can safely be re-run after adding/writing this base manifest. +### Covert your overrides for `dh_installsystemd`, `dh_installinit` (if any) + +If your package overrides any of the service related helpers, the following use-cases have a trivial +known solution: + + * Use of `--name` + * Use of `name.service` (etc.) with `dh_installsystemd` + * Use of `--no-start`, `--no-enable`, or similar options + * Any combination of the above. + +Dealing with `--name` is generally depends on "why" it is used. If it is about having the helper +pick up `debian/pkg.NAME.service` (etc.), then the `--name` can be dropped. This is because `debputy` +automatically resolves the `NAME` without this option. + +For uses that involve `--no-start`, `--no-enable`, etc., you will have to add a `services` section +to the package manifest. As an example: + + override_dh_installinit: + dh_installinit --name foo --no-start + + override_dh_installsystemd: + dh_installsystemd foo.service --no-start + +Would become: + + manifest-version: "0.1" + packages: + foo: + services: + - service: foo + enable-on-install: false + +If `sysvinit` and `systemd` should use different options, then you could do something like: + + + manifest-version: "0.1" + packages: + foo: + services: + # In systemd, the service is reloaded, but for sysvinit we use the "stop in preinst, upgrade than start" + # approach. + - service: foo + on-upgrade: reload + service-manager: systemd + - service: foo + on-upgrade: stop-then-start + service-manager: sysvinit + + ### Convert your overrides for `dh_gencontrol` (if any) If the package uses an override to choose a custom version for a binary package, then it is possible in `debputy` diff --git a/MANIFEST-FORMAT.md b/MANIFEST-FORMAT.md index 1b369cf..37773c2 100644 --- a/MANIFEST-FORMAT.md +++ b/MANIFEST-FORMAT.md @@ -1304,6 +1304,108 @@ done (except all symlinks will be ignored) +## Service management (`services`) + +If you have non-standard requirements for certain services in the package, you can define those via +the `services` attribute. + + packages: + foo: + services: + - service: "foo" + enable-on-install: false + - service: "bar" + on-upgrade: stop-then-start + + +The `services` attribute must contain a non-empty list, where each element in that list is a mapping. +Each mapping has the following key/value pairs: + + * `service` (required): Name of the service to match. The name is usually the basename of the service file. + However, aliases can also be used for relevant system managers. When aliases **and** multiple service + managers are involved, then the rule will apply to all matches. See alias handling below. + + - Note: For systemd, the `.service` suffix can be omitted from name, but other suffixes such as `.timer` + cannot. + + * `type-of-service` (optional, defaults to `service`): The type of service this rule applies to. To act on a + `systemd` timer, you would set this to `timer` (etc.). Each service manager defines its own set of types + of services. + + * `service-scope` (optional, defaults to `system`): The scope of the service. It must be either `system` and + `user`. + - Note: The keyword is defined to support `user`, but `debputy` does not support `user` services at the moment + (the detection logic is missing). + + * `service-manager` or `service-managers` (optional): Which service managers this rule is for. When omitted, all + service managers with this service will be affected. This can be used to specify separate rules for the same + service under different service managers. + - When this attribute is explicitly given, then all the listed service managers must provide at least one + service matching the definition. In contract, when it is omitted, then all service manager integrations + are consulted but as long as at least one service is match from any service manager, the rule is accepted. + + * `enable-on-install` (optional): Whether to automatically enable the service on installation. Note: This does + **not** affect whether the service will be started nor how restarts during upgrades will happen. + - If omitted, the plugin detecting the service decides the default. + + * `start-on-install` (optional): Whether to automatically start the service on installation. Whether it is + enabled or how upgrades are handled have separate attributes. + - If omitted, the plugin detecting the service decides the default. + + * `on-upgrade` (optional): How `debputy` should handle the service during upgrades. The default depends on the + plugin detecting the service. Valid values are: + + - `do-nothing`: During an upgrade, the package should not attempt to stop, reload or restart the service. + - `reload`: During an upgrade, prefer reloading the service rather than restarting if possible. Note that + the result may become `restart` instead if the service manager integration determines that `reload` is + not supported. + - `restart`: During an upgrade, `restart` the service post upgrade. The service will be left running during + the upgrade process. + - `stop-then-start`: Stop the service before the upgrade, preform the upgrade and then start the service. + +### Service managers and aliases + +When defining a service rule, you can use any name that any of the relevant service managers would call the +service. As an example, consider a package that has the following services: + + * A `sysvinit` service called `foo` + * A `systemd` service called `bar.service` with `Alias=foo.service` in its definition. + +Here, depending on which service managers are relevant to the rule, you can use different names to match. +When the rule applies to the `systemd` service manager, then either of the following names can be used: + + * `bar.service` (the "canonical" name in the systemd world) + * `foo.service` (the defined alias) + * `bar` + `foo` (automatic aliases based on the above) + +Now, if rule *also* applies to the `sysvinit` service manager, then any of those 4 names would cause the +rule to apply to both the `systemd` and the `sysvinit` services. + +To show concrete examples: + + ...: + services: + # Only applies to systemd. Either of the 4 names would have work. + - service: "foo.service" + on-upgrade: stop-then-start + service-manager: systemd + + ...: + services: + # Only applies to sysvinit. Must use `foo` since the 3 other names only applies when systemd + # is involved. + - service: "foo" + on-upgrade: stop-then-start + service-manager: sysvinit + + ...: + services: + # Applies to both systemd and sysvinit; this works because the `systemd` service provides an + # alias for `foo`. If the systemd service did not have that alias, only the `systemd` service + # would have been matched. + - service: bar + enable-on-install: false + ## Custom binary version (`binary-version`) In the *rare* case that you need a binary package to have a custom version, you can use the `binary-version:` diff --git a/src/debputy/commands/debputy_cmd/plugin_cmds.py b/src/debputy/commands/debputy_cmd/plugin_cmds.py index 1343b2e..2456902 100644 --- a/src/debputy/commands/debputy_cmd/plugin_cmds.py +++ b/src/debputy/commands/debputy_cmd/plugin_cmds.py @@ -1,4 +1,5 @@ import argparse +import itertools import operator import os import sys @@ -560,12 +561,17 @@ def _render_rule( required = declarative_parser.input_time_required_parameters conditionally_required = declarative_parser.at_least_one_of mutually_exclusive = declarative_parser.mutually_exclusive_attributes + is_list_wrapped = declarative_parser.is_list_wrapped else: attributes = {} required = frozenset() conditionally_required = frozenset() mutually_exclusive = frozenset() - print("Attributes:") + is_list_wrapped = False + if is_list_wrapped: + print("List where each element has the following attributes:") + else: + print("Attributes:") attribute_docs = ( parser_doc.attribute_doc if parser_doc.attribute_doc is not None else [] ) @@ -617,17 +623,30 @@ def _render_rule( or any(pd.conflicting_attributes for pd in attributes.values()) ): print() - print("This rule enforces the following restrictions:") + if is_list_wrapped: + print( + "This rule enforces the following restrictions on each element in the list:" + ) + else: + print("This rule enforces the following restrictions:") - if conditionally_required: - for cr in conditionally_required: - anames = "`, `".join( - attributes[a].source_attribute_name for a in cr - ) - if cr in mutually_exclusive: + if conditionally_required or mutually_exclusive: + all_groups = set( + itertools.chain(conditionally_required, mutually_exclusive) + ) + for g in all_groups: + anames = "`, `".join(g) + is_mx = g in mutually_exclusive + is_cr = g in conditionally_required + if is_mx and is_cr: print(f" - The rule must use exactly one of: `{anames}`") - else: + elif is_cr: print(f" - The rule must use at least one of: `{anames}`") + else: + assert is_mx + print( + f" - The following attributes are mutually exclusive: `{anames}`" + ) if mutually_exclusive or any( pd.conflicting_attributes for pd in attributes.values() diff --git a/src/debputy/deb_packaging_support.py b/src/debputy/deb_packaging_support.py index bb5979a..b38cbc2 100644 --- a/src/debputy/deb_packaging_support.py +++ b/src/debputy/deb_packaging_support.py @@ -26,6 +26,9 @@ from typing import ( TypeVar, FrozenSet, cast, + Any, + Union, + Mapping, ) import debian.deb822 @@ -64,12 +67,15 @@ from debputy.plugin.api.impl import ServiceRegistryImpl from debputy.plugin.api.impl_types import ( MetadataOrMaintscriptDetector, PackageDataTable, + ServiceManagerDetails, ) from debputy.plugin.api.spec import ( FlushableSubstvars, VirtualPath, PackageProcessingContext, + ServiceDefinition, ) +from debputy.plugin.debputy.binary_package_rules import ServiceRule from debputy.util import ( _error, ensure_dir, @@ -908,6 +914,164 @@ def cross_package_control_files( ) +def _relevant_service_definitions( + service_rule: ServiceRule, + service_managers: Union[List[str], FrozenSet[str]], + by_service_manager_key: Mapping[ + Tuple[str, str, str, str], Tuple[ServiceManagerDetails, ServiceDefinition[Any]] + ], + aliases: Mapping[str, Sequence[Tuple[str, str, str, str]]], +) -> Iterable[Tuple[Tuple[str, str, str, str], ServiceDefinition[Any]]]: + as_keys = (key for key in aliases[service_rule.service]) + + pending_queue = { + key + for key in as_keys + if key in by_service_manager_key + and service_rule.applies_to_service_manager(key[-1]) + } + relevant_names = {} + seen_keys = set() + + if not pending_queue: + service_manager_names = ", ".join(sorted(service_managers)) + _error( + f"No none of the service managers ({service_manager_names}) detected a service named" + f" {service_rule.service} (type: {service_rule.type_of_service}, scope: {service_rule.service_scope})," + f" but the manifest definition at {service_rule.definition_source} requested that." + ) + + while pending_queue: + next_key = pending_queue.pop() + seen_keys.add(next_key) + _, definition = by_service_manager_key[next_key] + yield next_key, definition + for name in definition.names: + for target_key in aliases[name]: + if ( + target_key not in seen_keys + and service_rule.applies_to_service_manager(target_key[-1]) + ): + pending_queue.add(target_key) + + return relevant_names + + +def handle_service_management( + binary_package_data: BinaryPackageData, + manifest: HighLevelManifest, + package_metadata_context: PackageProcessingContext, + fs_root: VirtualPath, + feature_set: PluginProvidedFeatureSet, +) -> None: + + by_service_manager_key = {} + aliases_by_name = collections.defaultdict(list) + + state = manifest.package_state_for(binary_package_data.binary_package.name) + all_service_managers = list(feature_set.service_managers) + requested_service_rules = state.requested_service_rules + for requested_service_rule in requested_service_rules: + if not requested_service_rule.service_managers: + continue + for manager in requested_service_rule.service_managers: + if manager not in feature_set.service_managers: + # FIXME: Missing definition source; move to parsing. + _error( + f"Unknown service manager {manager} used at {requested_service_rule.definition_source}" + ) + + for service_manager_details in feature_set.service_managers.values(): + service_registry = ServiceRegistryImpl(service_manager_details) + service_manager_details.service_detector( + fs_root, + service_registry, + package_metadata_context, + ) + + service_definitions = service_registry.detected_services + if not service_definitions: + continue + + for plugin_provided_definition in service_definitions: + key = ( + plugin_provided_definition.name, + plugin_provided_definition.type_of_service, + plugin_provided_definition.service_scope, + service_manager_details.service_manager, + ) + by_service_manager_key[key] = ( + service_manager_details, + plugin_provided_definition, + ) + + for name in plugin_provided_definition.names: + aliases_by_name[name].append(key) + + for requested_service_rule in requested_service_rules: + explicit_service_managers = requested_service_rule.service_managers is not None + related_service_managers = ( + requested_service_rule.service_managers or all_service_managers + ) + seen_service_managers = set() + for service_key, service_definition in _relevant_service_definitions( + requested_service_rule, + related_service_managers, + by_service_manager_key, + aliases_by_name, + ): + sm = service_key[-1] + seen_service_managers.add(sm) + by_service_manager_key[service_key] = ( + by_service_manager_key[service_key][0], + requested_service_rule.apply_to_service_definition(service_definition), + ) + if ( + explicit_service_managers + and seen_service_managers != related_service_managers + ): + missing_sms = ", ".join( + sorted(related_service_managers - seen_service_managers) + ) + _error( + f"The rule {requested_service_rule.definition_source} explicitly requested which service managers" + f" it should apply to. However, the following service managers did not provide a service of that" + f" name, type and scope: {missing_sms}. Please check the rule is correct and either provide the" + f" missing service or update the definition match the relevant services." + ) + + per_service_manager = {} + + for ( + service_manager_details, + plugin_provided_definition, + ) in by_service_manager_key.values(): + service_manager = service_manager_details.service_manager + if service_manager not in per_service_manager: + per_service_manager[service_manager] = ( + service_manager_details, + [plugin_provided_definition], + ) + else: + per_service_manager[service_manager][1].append(plugin_provided_definition) + + for ( + service_manager_details, + final_service_definitions, + ) in per_service_manager.values(): + ctrl = binary_package_data.ctrl_creator.for_plugin( + service_manager_details.plugin_metadata, + service_manager_details.service_manager, + default_snippet_order="service", + ) + _info(f"Applying {final_service_definitions}") + service_manager_details.service_integrator( + final_service_definitions, + ctrl, + package_metadata_context, + ) + + def setup_control_files( binary_package_data: BinaryPackageData, manifest: HighLevelManifest, @@ -948,26 +1112,13 @@ def setup_control_files( control_output_dir, ) - for service_manager_details in feature_set.service_managers.values(): - service_registry = ServiceRegistryImpl(service_manager_details) - service_manager_details.service_detector( - fs_root, - service_registry, - package_metadata_context, - ) - - ctrl = binary_package_data.ctrl_creator.for_plugin( - service_manager_details.plugin_metadata, - service_manager_details.service_manager, - ) - service_definitions = service_registry.detected_services - if not service_definitions: - continue - service_manager_details.service_integrator( - service_definitions, - ctrl, - package_metadata_context, - ) + handle_service_management( + binary_package_data, + manifest, + package_metadata_context, + fs_root, + feature_set, + ) plugin_detector_definition: MetadataOrMaintscriptDetector for plugin_detector_definition in itertools.chain.from_iterable( @@ -991,6 +1142,12 @@ def setup_control_files( ) else: + state = manifest.package_state_for(binary_package_data.binary_package.name) + if state.requested_service_rules: + service_source = state.requested_service_rules[0].definition_source + _error( + f"Use of service definitions (such as {service_source}) is not supported in this integration mode" + ) for script, snippet_container in package_state.maintscript_snippets.items(): for snippet in snippet_container.all_snippets(): source = snippet.definition_source @@ -1185,14 +1342,15 @@ def _generate_dbgsym_control_file_if_relevant( extra_params.append(f"-VInstalled-Size={total_size}") extra_params.extend(extra_common_params) - package = ( + package = binary_package.name + package_selector = ( binary_package.name if dctrl == "debian/control" else f"{binary_package.name}-dbgsym" ) dpkg_cmd = [ "dpkg-gencontrol", - f"-p{package}", + f"-p{package_selector}", # FIXME: Support d/<pkg>.changelog at some point. "-ldebian/changelog", "-T/dev/null", diff --git a/src/debputy/highlevel_manifest.py b/src/debputy/highlevel_manifest.py index bae5cdb..1e92210 100644 --- a/src/debputy/highlevel_manifest.py +++ b/src/debputy/highlevel_manifest.py @@ -56,13 +56,14 @@ from .manifest_parser.base_types import FileSystemMatchRule, FileSystemExactMatc from .manifest_parser.util import AttributePath from .packager_provided_files import PackagerProvidedFile from .packages import BinaryPackage, SourcePackage +from .plugin.api.feature_set import PluginProvidedFeatureSet from .plugin.api.impl import BinaryCtrlAccessorProviderCreator from .plugin.api.impl_types import ( PackageProcessingContextProvider, PackageDataTable, ) -from .plugin.api.feature_set import PluginProvidedFeatureSet from .plugin.api.spec import FlushableSubstvars, VirtualPath +from .plugin.debputy.binary_package_rules import ServiceRule from .substitution import Substitution from .transformation_rules import ( TransformationRule, @@ -117,6 +118,7 @@ class PackageTransformationDefinition: default_factory=dict ) install_rules: List[InstallRule] = field(default_factory=list) + requested_service_rules: List[ServiceRule] = field(default_factory=list) def _path_to_tar_member( diff --git a/src/debputy/highlevel_manifest_parser.py b/src/debputy/highlevel_manifest_parser.py index 6181603..24d05c7 100644 --- a/src/debputy/highlevel_manifest_parser.py +++ b/src/debputy/highlevel_manifest_parser.py @@ -468,6 +468,7 @@ class YAMLManifestParser(HighLevelManifestParser): definition_source.path, ) + package_state: PackageTransformationDefinition with self.binary_package_context(package_name) as package_state: if package_state.is_auto_generated_package: # Maybe lift (part) of this restriction. @@ -481,7 +482,7 @@ class YAMLManifestParser(HighLevelManifestParser): ] ) parsed = cast( - "BinaryPackageRule", + "Mapping[str, Any]", package_rule_parser.parse( v, definition_source, parser_context=self ), @@ -499,12 +500,16 @@ class YAMLManifestParser(HighLevelManifestParser): package_state.search_dirs = search_dirs transformations = parsed.get("transformations") conffile_management = parsed.get("conffile_management") + service_rules = parsed.get("services") if transformations: package_state.transformations.extend(transformations) if conffile_management: package_state.dpkg_maintscript_helper_snippets.extend( conffile_management ) + if service_rules: + package_state.requested_service_rules.extend(service_rules) + return self.build_manifest() def _parse_manifest(self, fd: Union[IO[bytes], str]) -> HighLevelManifest: diff --git a/src/debputy/lsp/logins-and-people.dic b/src/debputy/lsp/logins-and-people.dic index a7c468b..8c231b2 100644 --- a/src/debputy/lsp/logins-and-people.dic +++ b/src/debputy/lsp/logins-and-people.dic @@ -8,6 +8,7 @@ Alteholz Américo Andreas Andrej +Andrey Andrius Ansgar Aoki @@ -162,6 +163,7 @@ Masato Matej Mattia Maximiliano +McVittie Mennucc Merkys Metzler @@ -203,8 +205,9 @@ Praveen Prévot Raboud Ragwitz -Raphaël Reiner +Rakhmatullin +Raphaël Reyer Rivero Rizzolo @@ -232,7 +235,7 @@ Sérgio Seyeong Shachnev Shadura -smcv McVittie +smcv Smedegaard Sprickerhof Stapelberg @@ -271,6 +274,7 @@ wferi Whitton Wilk Wouter +wRAR Yamane Yann zeha diff --git a/src/debputy/manifest_parser/declarative_parser.py b/src/debputy/manifest_parser/declarative_parser.py index 84e0230..bb901fc 100644 --- a/src/debputy/manifest_parser/declarative_parser.py +++ b/src/debputy/manifest_parser/declarative_parser.py @@ -1,6 +1,5 @@ import collections import dataclasses -import itertools from typing import ( Any, Callable, @@ -30,14 +29,8 @@ from typing import ( from debputy.manifest_parser.base_types import ( DebputyParsedContent, - StaticFileSystemOwner, - StaticFileSystemGroup, - FileSystemMode, - OctalMode, - SymlinkTarget, FileSystemMatchRule, FileSystemExactMatchRule, - FileSystemExactNonDirMatchRule, DebputyDispatchableType, TypeMapping, ) @@ -45,14 +38,12 @@ from debputy.manifest_parser.exceptions import ( ManifestParseException, ) from debputy.manifest_parser.mapper_code import ( - type_mapper_str2package, normalize_into_list, wrap_into_list, map_each_element, ) from debputy.manifest_parser.parser_data import ParserContextData from debputy.manifest_parser.util import AttributePath, unpack_type, find_annotation -from debputy.packages import BinaryPackage from debputy.plugin.api.impl_types import ( DeclarativeInputParser, TD, @@ -288,50 +279,43 @@ class DeclarativeMappingInputParser(DeclarativeInputParser[TD], Generic[TD, SF]) _per_attribute_conflicts_cache: Optional[Mapping[str, FrozenSet[str]]] = None inline_reference_documentation: Optional[ParserDocumentation] = None path_hint_source_attributes: Sequence[str] = tuple() + # TODO: List-wrapping should probably be its own parser that delegetes to subparsers + is_list_wrapped: bool = False - def parse_input( + def _parse_alt_form( self, value: object, path: AttributePath, *, parser_context: Optional["ParserContextData"] = None, ) -> TD: - if self.reference_documentation_url is not None: - doc_ref = f" (Documentation: {self.reference_documentation_url})" - else: - doc_ref = "" - if value is None: - form_note = " The attribute must be a mapping." - if self.alt_form_parser is not None: - form_note = ( - " The attribute can be a mapping or a non-mapping format" - ' (usually, "non-mapping format" means a string or a list of strings).' - ) - if self.reference_documentation_url is not None: - doc_ref = f" Please see {self.reference_documentation_url} for the documentation." + alt_form_parser = self.alt_form_parser + if alt_form_parser is None: raise ManifestParseException( - f"The attribute {path.path} was missing a value. {form_note}{doc_ref}" + f"The attribute {path.path} must be a mapping.{self._doc_url_error_suffix()}" ) - if not isinstance(value, dict): - alt_form_parser = self.alt_form_parser - if alt_form_parser is None: - raise ManifestParseException( - f"The attribute {path.path} must be a mapping.{doc_ref}" - ) - _extract_path_hint(value, path) - alt_form_parser.type_validator.ensure_type(value, path) - assert ( - value is not None - ), "The alternative form was None, but the parser should have rejected None earlier." - attribute = alt_form_parser.target_attribute - alias_mapping = { - attribute: ("", None), - } - v = alt_form_parser.type_validator.map_type(value, path, parser_context) - path.alias_mapping = alias_mapping - return cast("TD", {attribute: v}) + _extract_path_hint(value, path) + alt_form_parser.type_validator.ensure_type(value, path) + assert ( + value is not None + ), "The alternative form was None, but the parser should have rejected None earlier." + attribute = alt_form_parser.target_attribute + alias_mapping = { + attribute: ("", None), + } + v = alt_form_parser.type_validator.map_type(value, path, parser_context) + path.alias_mapping = alias_mapping + return cast("TD", {attribute: v}) + def _validate_expected_keys( + self, + value: Dict[Any, Any], + path: AttributePath, + *, + parser_context: Optional["ParserContextData"] = None, + ) -> None: unknown_keys = value.keys() - self.all_parameters + doc_ref = self._doc_url_error_suffix() if unknown_keys: for k in unknown_keys: if isinstance(k, str): @@ -380,6 +364,15 @@ class DeclarativeMappingInputParser(DeclarativeInputParser[TD], Generic[TD, SF]) f"Could not parse {path.path}: The following attributes are" f" mutually exclusive: {ck}{doc_ref}" ) + + def _parse_typed_dict_form( + self, + value: Dict[Any, Any], + path: AttributePath, + *, + parser_context: Optional["ParserContextData"] = None, + ) -> TD: + self._validate_expected_keys(value, path, parser_context=parser_context) result = {} per_attribute_conflicts = self._per_attribute_conflicts() alias_mapping = {} @@ -394,7 +387,7 @@ class DeclarativeMappingInputParser(DeclarativeInputParser[TD], Generic[TD, SF]) ck = ", ".join(repr(k) for k in sorted(matched)) raise ManifestParseException( f'The attribute "{k}" at {path.path} cannot be used with the following' - f" attributes: {ck}{doc_ref}" + f" attributes: {ck}{self._doc_url_error_suffix()}" ) nk = attr.target_attribute key_path = path[k] @@ -409,6 +402,72 @@ class DeclarativeMappingInputParser(DeclarativeInputParser[TD], Generic[TD, SF]) path.alias_mapping = alias_mapping return cast("TD", result) + def _doc_url_error_suffix(self, *, see_url_version: bool = False) -> str: + doc_url = self.reference_documentation_url + if doc_url is not None: + if see_url_version: + return f" Please see {doc_url} for the documentation." + return f" (Documentation: {doc_url})" + return "" + + def _parse_input( + self, + value: object, + path: AttributePath, + *, + parser_context: Optional["ParserContextData"] = None, + is_list_wrapped: bool, + ) -> TD: + if value is None: + if is_list_wrapped: + form_note = " The attribute must be a list of mappings" + else: + form_note = " The attribute must be a mapping." + if self.alt_form_parser is not None: + form_note = ( + " The attribute can be a mapping or a non-mapping format" + ' (usually, "non-mapping format" means a string or a list of strings).' + ) + doc_ref = self._doc_url_error_suffix(see_url_version=True) + raise ManifestParseException( + f"The attribute {path.path} was missing a value. {form_note}{doc_ref}" + ) + if is_list_wrapped: + if not isinstance(value, list) or not value: + doc_ref = self._doc_url_error_suffix(see_url_version=True) + raise ManifestParseException( + f"The attribute {path.path} must be a non-empty list.{doc_ref}" + ) + result = [] + for idx, element in enumerate(value): + element_path = path[idx] + result.append( + self._parse_input( + element, + element_path, + parser_context=parser_context, + is_list_wrapped=False, + ) + ) + return result + if not isinstance(value, dict): + return self._parse_alt_form(value, path, parser_context=parser_context) + return self._parse_typed_dict_form(value, path, parser_context=parser_context) + + def parse_input( + self, + value: object, + path: AttributePath, + *, + parser_context: Optional["ParserContextData"] = None, + ) -> TD: + return self._parse_input( + value, + path, + parser_context=parser_context, + is_list_wrapped=self.is_list_wrapped, + ) + def _per_attribute_conflicts(self) -> Mapping[str, FrozenSet[str]]: conflicts = self._per_attribute_conflicts_cache if conflicts is not None: @@ -813,7 +872,9 @@ class ParserGenerator: for concrete features that may be useful to you. :param parsed_content: A DebputyParsedContent / TypedDict describing the desired model of the input once parsed. - (DebputyParsedContent is a TypedDict subclass that work around some inadequate type checkers) + (DebputyParsedContent is a TypedDict subclass that work around some inadequate type checkers). + It can also be a `List[DebputyParsedContent]`. In that case, `source_content` must be a + `List[TypedDict[...]]`. :param source_content: Optionally, a TypedDict describing the input allowed by the user. This can be useful to describe more variations than in `parsed_content` that the parser will normalize for you. If omitted, the parsed_content is also considered the source_content (which affects what annotations are allowed in it). @@ -824,15 +885,26 @@ class ParserGenerator: :param inline_reference_documentation: Optionally, programmatic documentation :return: An input parser capable of reading input matching the TypedDict(s) used as reference. """ + orig_parsed_content = parsed_content + if source_content is parsed_content: + raise ValueError( + "Do not provide source_content if it is the same as parsed_content" + ) + is_list_wrapped = False + if get_origin(orig_parsed_content) == list: + parsed_content = get_args(orig_parsed_content)[0] + is_list_wrapped = True if not is_typeddict(parsed_content): raise ValueError( f"Unsupported parsed_content descriptor: {parsed_content.__qualname__}." ' Only "TypedDict"-based types supported.' ) - if source_content is parsed_content: - raise ValueError( - "Do not provide source_content if it is the same as parsed_content" - ) + if is_list_wrapped: + if get_origin(source_content) != list: + raise ValueError( + "If the parsed_content is a List type, then source_format must be a List type as well." + ) + source_content = get_args(source_content)[0] target_attributes = self._parse_types( parsed_content, @@ -988,6 +1060,11 @@ class ParserGenerator: parsed_alt_form is not None, ) if non_mapping_source_only: + if is_list_wrapped: + raise ValueError( + f"Unsupported case: {non_mapping_source_only=} + {is_list_wrapped=}" + " (TODO: Look whether it is feasible)" + ) return DeclarativeNonMappingInputParser( assume_not_none(parsed_alt_form), inline_reference_documentation=inline_reference_documentation, @@ -1003,6 +1080,7 @@ class ParserGenerator: at_least_one_of=at_least_one_of, inline_reference_documentation=inline_reference_documentation, path_hint_source_attributes=tuple(path_hint_source_attributes), + is_list_wrapped=is_list_wrapped, ) def _as_type_validator( diff --git a/src/debputy/plugin/api/impl.py b/src/debputy/plugin/api/impl.py index 8b75322..3c9da60 100644 --- a/src/debputy/plugin/api/impl.py +++ b/src/debputy/plugin/api/impl.py @@ -29,6 +29,7 @@ from typing import ( cast, FrozenSet, Any, + Literal, ) from debputy import DEBPUTY_DOC_ROOT_DIR @@ -1152,6 +1153,7 @@ class MaintscriptAccessorProvider(MaintscriptAccessorProviderBase): "_maintscript_snippets", "_plugin_source_id", "_package_substitution", + "_default_snippet_order", ) def __init__( @@ -1160,11 +1162,14 @@ class MaintscriptAccessorProvider(MaintscriptAccessorProviderBase): plugin_source_id: str, maintscript_snippets: Dict[str, MaintscriptSnippetContainer], package_substitution: Substitution, + *, + default_snippet_order: Optional[Literal["service"]] = None, ): self._plugin_metadata = plugin_metadata self._plugin_source_id = plugin_source_id self._maintscript_snippets = maintscript_snippets self._package_substitution = package_substitution + self._default_snippet_order = default_snippet_order def _append_script( self, @@ -1178,7 +1183,11 @@ class MaintscriptAccessorProvider(MaintscriptAccessorProviderBase): if perform_substitution: full_script = self._package_substitution.substitute(full_script, def_source) - snippet = MaintscriptSnippet(snippet=full_script, definition_source=def_source) + snippet = MaintscriptSnippet( + snippet=full_script, + definition_source=def_source, + snippet_order=self._default_snippet_order, + ) self._maintscript_snippets[maintscript].append(snippet) @@ -1283,6 +1292,8 @@ class BinaryCtrlAccessorProvider(BinaryCtrlAccessorProviderBase): maintscript_snippets: Dict[str, MaintscriptSnippetContainer], package_substitution: Substitution, shlibs_details: Tuple[Optional[str], Optional[List[str]]], + *, + default_snippet_order: Optional[Literal["service"]] = None, ) -> None: super().__init__( plugin_metadata, @@ -1299,6 +1310,7 @@ class BinaryCtrlAccessorProvider(BinaryCtrlAccessorProviderBase): plugin_source_id, maintscript_snippets, package_substitution, + default_snippet_order=default_snippet_order, ) def _create_maintscript_accessor(self) -> MaintscriptAccessor: @@ -1329,6 +1341,8 @@ class BinaryCtrlAccessorProviderCreator: self, plugin_metadata: DebputyPluginMetadata, plugin_source_id: str, + *, + default_snippet_order: Optional[Literal["service"]] = None, ) -> BinaryCtrlAccessor: return BinaryCtrlAccessorProvider( plugin_metadata, @@ -1339,6 +1353,7 @@ class BinaryCtrlAccessorProviderCreator: self._maintscript_snippets, self._substitution, self.shlibs_details, + default_snippet_order=default_snippet_order, ) def generated_triggers(self) -> Iterable[PluginProvidedTrigger]: @@ -1873,19 +1888,23 @@ class ServiceDefinitionImpl(ServiceDefinition[DSD]): type_of_service: str service_scope: str auto_enable_on_install: bool - auto_start_in_install: bool + auto_start_on_install: bool on_upgrade: ServiceUpgradeRule definition_source: str is_plugin_provided_definition: bool service_context: Optional[DSD] + def replace(self, **changes: Any) -> "ServiceDefinitionImpl[DSD]": + return dataclasses.replace(self, **changes) + class ServiceRegistryImpl(ServiceRegistry[DSD]): - __slots__ = ("_service_manager_details", "_service_definitions") + __slots__ = ("_service_manager_details", "_service_definitions", "_seen_services") def __init__(self, service_manager_details: ServiceManagerDetails) -> None: self._service_manager_details = service_manager_details self._service_definitions: List[ServiceDefinition[DSD]] = [] + self._seen_services = set() @property def detected_services(self) -> Sequence[ServiceDefinition[DSD]]: @@ -1908,6 +1927,14 @@ class ServiceRegistryImpl(ServiceRegistry[DSD]): raise ValueError( f"The service must have at least one name - {path.absolute} did not have any" ) + for n in names: + key = (n, type_of_service, service_scope) + if key in self._seen_services: + raise PluginAPIViolationError( + f"The service manager (from {self._service_manager_details.plugin_metadata.plugin_name}) used" + f" the service name {n} (type: {type_of_service}, scope: {service_scope}) twice. This is not" + " allowed by the debputy plugin API." + ) # TODO: We cannot create a service definition immediate once the manifest is involved self._service_definitions.append( ServiceDefinitionImpl( diff --git a/src/debputy/plugin/api/impl_types.py b/src/debputy/plugin/api/impl_types.py index f32b008..76579fb 100644 --- a/src/debputy/plugin/api/impl_types.py +++ b/src/debputy/plugin/api/impl_types.py @@ -82,7 +82,7 @@ _PACKAGE_TYPE_DEB_ONLY = frozenset(["deb"]) _ALL_PACKAGE_TYPES = frozenset(["deb", "udeb"]) -TD = TypeVar("TD", bound="DebputyParsedContent") +TD = TypeVar("TD", bound="Union[DebputyParsedContent, List[DebputyParsedContent]]") PF = TypeVar("PF") SF = TypeVar("SF") TP = TypeVar("TP") diff --git a/src/debputy/plugin/api/spec.py b/src/debputy/plugin/api/spec.py index d034a28..5d7a261 100644 --- a/src/debputy/plugin/api/spec.py +++ b/src/debputy/plugin/api/spec.py @@ -1679,7 +1679,7 @@ class ServiceDefinition(Generic[DSD]): raise NotImplementedError @property - def auto_start_in_install(self) -> bool: + def auto_start_on_install(self) -> bool: """Whether the service should be auto-started on install :return: True if the service should be started automatically, false if not. @@ -1727,8 +1727,8 @@ class ServiceDefinition(Generic[DSD]): def is_plugin_provided_definition(self) -> bool: """Whether the definition source points to the plugin or a package provided definition - :return: True if definition is from the plugin. False if the definition is defined - in another place (usually, the manifest) + :return: True if definition is 100% from the plugin. False if the definition is partially + or fully from another source (usually, the packager via the manifest). """ raise NotImplementedError diff --git a/src/debputy/plugin/debputy/binary_package_rules.py b/src/debputy/plugin/debputy/binary_package_rules.py index 686d71a..4753c79 100644 --- a/src/debputy/plugin/debputy/binary_package_rules.py +++ b/src/debputy/plugin/debputy/binary_package_rules.py @@ -1,3 +1,4 @@ +import dataclasses import os import textwrap from typing import ( @@ -9,6 +10,9 @@ from typing import ( TypedDict, Annotated, Optional, + FrozenSet, + Self, + cast, ) from debputy import DEBPUTY_DOC_ROOT_DIR @@ -26,10 +30,19 @@ from debputy.manifest_parser.parser_data import ParserContextData from debputy.manifest_parser.util import AttributePath from debputy.path_matcher import MatchRule, MATCH_ANYTHING, ExactFileSystemPath from debputy.plugin.api import reference_documentation -from debputy.plugin.api.impl import DebputyPluginInitializerProvider +from debputy.plugin.api.impl import ( + DebputyPluginInitializerProvider, + ServiceDefinitionImpl, +) from debputy.plugin.api.impl_types import OPARSER_PACKAGES +from debputy.plugin.api.spec import ( + ServiceUpgradeRule, + ServiceDefinition, + DSD, + documented_attr, +) from debputy.transformation_rules import TransformationRule - +from debputy.util import _error ACCEPTABLE_CLEAN_ON_REMOVAL_FOR_GLOBS_AND_EXACT_MATCHES = frozenset( [ @@ -141,6 +154,123 @@ def register_binary_package_rules(api: DebputyPluginInitializerProvider) -> None api.pluggable_manifest_rule( OPARSER_PACKAGES, + "services", + List[ServiceRuleParsedFormat], + _process_service_rules, + source_format=List[ServiceRuleSourceFormat], + inline_reference_documentation=reference_documentation( + title="Define how services in the package will be handled (`services`)", + description=textwrap.dedent( + """\ + If you have non-standard requirements for certain services in the package, you can define those via + the `services` attribute. The `services` attribute is a list of service rules. Example: + + packages: + foo: + services: + - service: "foo" + enable-on-install: false + - service: "bar" + on-upgrade: stop-then-start + """ + ), + attributes=[ + documented_attr( + "service", + textwrap.dedent( + f"""\ + Name of the service to match. The name is usually the basename of the service file. + However, aliases can also be used for relevant system managers. When aliases **and** + multiple service managers are involved, then the rule will apply to all matches. + For details on aliases, please see + {DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#service-managers-and-aliases. + + - Note: For systemd, the `.service` suffix can be omitted from name, but other + suffixes such as `.timer` cannot. + """ + ), + ), + documented_attr( + "type_of_service", + textwrap.dedent( + """\ + The type of service this rule applies to. To act on a `systemd` timer, you would + set this to `timer` (etc.). Each service manager defines its own set of types + of services. + """ + ), + ), + documented_attr( + "service_scope", + textwrap.dedent( + """\ + The scope of the service. It must be either `system` and `user`. + - Note: The keyword is defined to support `user`, but `debputy` does not support `user` + services at the moment (the detection logic is missing). + """ + ), + ), + documented_attr( + ["service_manager", "service_managers"], + textwrap.dedent( + """\ + Which service managers this rule is for. When omitted, all service managers with this + service will be affected. This can be used to specify separate rules for the same + service under different service managers. + - When this attribute is explicitly given, then all the listed service managers must + provide at least one service matching the definition. In contract, when it is omitted, + then all service manager integrations are consulted but as long as at least one + service is match from any service manager, the rule is accepted. + """ + ), + ), + documented_attr( + "enable_on_install", + textwrap.dedent( + """\ + Whether to automatically enable the service on installation. Note: This does + **not** affect whether the service will be started nor how restarts during + upgrades will happen. + - If omitted, the plugin detecting the service decides the default. + """ + ), + ), + documented_attr( + "start_on_install", + textwrap.dedent( + """\ + Whether to automatically start the service on installation. Whether it is + enabled or how upgrades are handled have separate attributes. + - If omitted, the plugin detecting the service decides the default. + """ + ), + ), + documented_attr( + "on_upgrade", + textwrap.dedent( + """\ + How `debputy` should handle the service during upgrades. The default depends on the + plugin detecting the service. Valid values are: + + - `do-nothing`: During an upgrade, the package should not attempt to stop, reload or + restart the service. + - `reload`: During an upgrade, prefer reloading the service rather than restarting + if possible. Note that the result may become `restart` instead if the service + manager integration determines that `reload` is not supported. + - `restart`: During an upgrade, `restart` the service post upgrade. The service + will be left running during the upgrade process. + - `stop-then-start`: Stop the service before the upgrade, preform the upgrade and + then start the service. + """ + ), + ), + ], + reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#service-management-services", + ), + ) + + api.pluggable_manifest_rule( + OPARSER_PACKAGES, "clean-after-removal", ListParsedFormat, _parse_clean_after_removal, @@ -251,6 +381,93 @@ def register_binary_package_rules(api: DebputyPluginInitializerProvider) -> None ) +class ServiceRuleSourceFormat(TypedDict): + service: str + type_of_service: NotRequired[str] + service_scope: NotRequired[Literal["system", "user"]] + enable_on_install: NotRequired[bool] + start_on_install: NotRequired[bool] + on_upgrade: NotRequired[ServiceUpgradeRule] + service_manager: NotRequired[ + Annotated[str, DebputyParseHint.target_attribute("service_managers")] + ] + service_managers: NotRequired[List[str]] + + +class ServiceRuleParsedFormat(DebputyParsedContent): + service: str + type_of_service: NotRequired[str] + service_scope: NotRequired[Literal["system", "user"]] + enable_on_install: NotRequired[bool] + start_on_install: NotRequired[bool] + on_upgrade: NotRequired[ServiceUpgradeRule] + service_managers: NotRequired[List[str]] + + +@dataclasses.dataclass(slots=True, frozen=True) +class ServiceRule: + definition_source: str + service: str + type_of_service: str + service_scope: Literal["system", "user"] + enable_on_install: Optional[bool] + start_on_install: Optional[bool] + on_upgrade: Optional[ServiceUpgradeRule] + service_managers: Optional[FrozenSet[str]] + + @classmethod + def from_service_rule_parsed_format( + cls, + data: ServiceRuleParsedFormat, + attribute_path: AttributePath, + ) -> "Self": + service_managers = data.get("service_managers") + return cls( + attribute_path.path, + data["service"], + data.get("type_of_service", "service"), + cast("Literal['system', 'user']", data.get("service_scope", "system")), + data.get("enable_on_install"), + data.get("start_on_install"), + data.get("on_upgrade"), + frozenset(service_managers) if service_managers else service_managers, + ) + + def applies_to_service_manager(self, service_manager: str) -> bool: + return self.service_managers is None or service_manager in self.service_managers + + def apply_to_service_definition( + self, + service_definition: ServiceDefinition[DSD], + ) -> ServiceDefinition[DSD]: + assert isinstance(service_definition, ServiceDefinitionImpl) + if not service_definition.is_plugin_provided_definition: + _error( + f"Conflicting definitions related to {self.service} (type: {self.type_of_service}," + f" scope: {self.service_scope}). First definition at {service_definition.definition_source}," + f" the second at {self.definition_source}). If they are for different service managers," + " you can often avoid this problem by explicitly defining which service managers are applicable" + ' to each rule via the "service-managers" keyword.' + ) + changes = { + "definition_source": self.definition_source, + "is_plugin_provided_definition": False, + } + if ( + self.service != service_definition.name + and self.service in service_definition.names + ): + changes["name"] = self.service + if self.enable_on_install is not None: + changes["auto_start_on_install"] = self.enable_on_install + if self.start_on_install is not None: + changes["auto_start_on_install"] = self.start_on_install + if self.on_upgrade is not None: + changes["on_upgrade"] = self.on_upgrade + + return service_definition.replace(**changes) + + class BinaryVersionParsedFormat(DebputyParsedContent): binary_version: str @@ -289,6 +506,18 @@ def _parse_installation_search_dirs( return parsed_data["installation_search_dirs"] +def _process_service_rules( + _name: str, + parsed_data: List[ServiceRuleParsedFormat], + attribute_path: AttributePath, + _parser_context: ParserContextData, +) -> List[ServiceRule]: + return [ + ServiceRule.from_service_rule_parsed_format(x, attribute_path[i]) + for i, x in enumerate(parsed_data) + ] + + def _unpack_list( _name: str, parsed_data: ListParsedFormat, @@ -314,7 +543,7 @@ class CleanAfterRemovalRule(DebputyParsedContent): # FIXME: Not optimal that we are doing an initialization of ParserGenerator here. But the rule is not depending on any -# complex types that is regiersted by plugins, so it will work for now. +# complex types that is registered by plugins, so it will work for now. _CLEAN_AFTER_REMOVAL_RULE_PARSER = ParserGenerator().parser_from_typed_dict( CleanAfterRemovalRule, source_content=Union[CleanAfterRemovalRuleSourceFormat, str, List[str]], diff --git a/src/debputy/plugin/debputy/package_processors.py b/src/debputy/plugin/debputy/package_processors.py index 3747755..1d19b66 100644 --- a/src/debputy/plugin/debputy/package_processors.py +++ b/src/debputy/plugin/debputy/package_processors.py @@ -117,7 +117,10 @@ def process_manpages(fs_root: VirtualPath, _unused1: Any, _unused2: Any) -> None " what went wrong." ) for manpage in manpages: - os.rename(f"{manpage}.encoded", manpage) + dest_name = manpage + if dest_name.endswith(".gz"): + dest_name = dest_name[:-3] + os.rename(f"{dest_name}.encoded", manpage) def _filter_compress_paths() -> Callable[[VirtualPath], Iterator[VirtualPath]]: diff --git a/src/debputy/plugin/debputy/service_management.py b/src/debputy/plugin/debputy/service_management.py index 1ec8c1b..64f2733 100644 --- a/src/debputy/plugin/debputy/service_management.py +++ b/src/debputy/plugin/debputy/service_management.py @@ -62,7 +62,7 @@ def generate_snippets_for_systemd_units( ctrl: BinaryCtrlAccessor, _context: PackageProcessingContext, ) -> None: - stop_in_prerm: List[str] = [] + stop_before_upgrade: List[str] = [] stop_then_start_scripts = [] on_purge = [] start_on_install = [] @@ -118,14 +118,12 @@ def generate_snippets_for_systemd_units( f' so that it does not enable the service or does not apply to "systemd"' ) - if service_def.auto_start_in_install: + if service_def.auto_start_on_install: start_on_install.append(service_name) if service_def.on_upgrade == "stop-then-start": stop_then_start_scripts.append(service_name) elif service_def.on_upgrade in ("restart", "reload"): action: str = service_def.on_upgrade - if not service_def.auto_start_in_install and action != "reload": - action = f"try-{action}" action_on_upgrade[action].append(service_name) elif service_def.on_upgrade != "do-nothing": raise AssertionError( @@ -187,7 +185,7 @@ def generate_snippets_for_systemd_units( ), ) - if stop_in_prerm: + if stop_before_upgrade: ctrl.maintscript.on_before_removal( """\ if {EMPTY_DPKG_ROOT_CONDITION} && {SERVICE_MANAGER_IS_SYSTEMD_CONDITION} ; then @@ -196,7 +194,7 @@ def generate_snippets_for_systemd_units( """.format( EMPTY_DPKG_ROOT_CONDITION=EMPTY_DPKG_ROOT_CONDITION, SERVICE_MANAGER_IS_SYSTEMD_CONDITION=SERVICE_MANAGER_IS_SYSTEMD_CONDITION, - UNIT_FILES=ctrl.maintscript.escape_shell_words(*stop_in_prerm), + UNIT_FILES=ctrl.maintscript.escape_shell_words(*stop_before_upgrade), ) ) if on_purge: @@ -206,7 +204,7 @@ def generate_snippets_for_systemd_units( deb-systemd-helper purge {UNITFILES} >/dev/null || true fi """.format( - UNITFILES=ctrl.maintscript.escape_shell_words(*stop_in_prerm), + UNITFILES=ctrl.maintscript.escape_shell_words(*stop_before_upgrade), ) ) ctrl.maintscript.on_removed( @@ -347,7 +345,7 @@ def generate_snippets_for_init_scripts( ] if ( - service_def.auto_start_in_install + service_def.auto_start_on_install and service_def.on_upgrade != "stop-then-start" ): lines.append(' if [ -z "$2" ]; then') |