diff options
69 files changed, 6869 insertions, 879 deletions
diff --git a/MANIFEST-FORMAT.md b/MANIFEST-FORMAT.md index d1474bf..b7d8262 100644 --- a/MANIFEST-FORMAT.md +++ b/MANIFEST-FORMAT.md @@ -542,6 +542,60 @@ Limitations: `{{PACKAGE}}` cannot be used when defining a variable. This restriction may be lifted in the future. +# Build environment (`build-environment`) + +Define the environment variables used in all build commands. + +The environment definition can be used to tweak the environment variables used by the +build commands. An example: + + environment: + set: + ENV_VAR: foo + ANOTHER_ENV_VAR: bar + +The environment definition has multiple attributes for setting environment variables +which determines when the definition is applied. The resulting environment is the +result of the following order of operations. + + 1. The environment `debputy` received from its parent process. + 2. Apply all the variable definitions from `set` (if the attribute is present) + 3. Apply all computed variables (such as variables from `dpkg-buildflags`). + 4. Apply all the variable definitions from `override` (if the attribute is present) + 5. Remove all variables listed in `unset` (if the attribute is present). + +Accordingly, both `override` and `unset` will overrule any computed variables while +`set` will be overruled by any computed variables. + +Note that these variables are not available via manifest substitution (they are only +visible to build commands). They are only available to build commands. + +The `build-environment` attribute is a mapping and has the following attributes: + + - `set` (optional): Mapping of string + + A mapping of environment variables to be set. + + Note these environment variables are set before computed variables (such + as `dpkg-buildflags`) are provided. They can affect the content of the + computed variables, but they cannot overrule them. If you need to overrule + a computed variable, please use `override` instead. + + - `override` (optional): Mapping of string + + A mapping of environment variables to set. + + Similar to `set`, but it can overrule computed variables like those from + `dpkg-buildflags`. + + - `unset` (optional): List of string + + A list of environment variables to unset. + + Any environment variable named here will be unset. No warnings or errors + will be raised if a given variable was not set. + + # Installations For source packages building a single binary, the `dh_auto_install` from debhelper will default to @@ -625,7 +679,7 @@ you could do: installations: - install: sources: - # By-pass automatic discard of `libfoo.la` - no globs *cannot* be used! + # By-pass automatic discard of `libfoo.la` - globs *cannot* be used! - "usr/lib/libfoo.la" - "usr/lib/libfoo*.so*" into: libfoo1 @@ -755,8 +809,8 @@ alternative name. ## Install examples (`install-examples`) -This install rule resemble that of `dh_installexamples`. It is a shorthand over the generic ` -install` rule with the following key features: +This install rule resemble that of `dh_installexamples`. It is a shorthand over the generic +`install` rule with the following key features: 1) It pre-defines the `dest-dir` that respects the "main documentation package" recommendation from Debian Policy. The `install-examples` will use the `examples` subdir for the package documentation @@ -15,10 +15,12 @@ install: $(DESTDIR)/$(DEBPUTY_INSTALLED_ROOT_DIR) \ $(DESTDIR)/$(DEBPUTY_INSTALLED_PLUGIN_ROOT_DIR)/debputy \ $(DESTDIR)/usr/share/perl5/Debian/Debhelper/Sequence \ + $(DESTDIR)/usr/share/perl5/Dpkg/BuildDriver \ $(DESTDIR)/usr/share/man/man1 install -m0755 -t $(DESTDIR)/usr/bin dh_debputy dh_installdebputy assets/debputy install -m0755 -t $(DESTDIR)/$(DEBPUTY_INSTALLED_ROOT_DIR) deb_packer.py deb_materialization.py install -m0644 -t $(DESTDIR)/usr/share/perl5/Debian/Debhelper/Sequence lib/Debian/Debhelper/Sequence/*.pm + install -m0644 -t $(DESTDIR)/usr/share/perl5/Dpkg/BuildDriver lib/Dpkg/BuildDriver/*.pm cp -a --reflink=auto src/debputy $(DESTDIR)/$(DEBPUTY_INSTALLED_ROOT_DIR) cp -a --reflink=auto debputy $(DESTDIR)/$(DEBPUTY_INSTALLED_PLUGIN_ROOT_DIR) sed -i "s/^__version__ =.*/__version__ = '$$(dpkg-parsechangelog -SVersion)'/; s/^__release_commit__ =.*/__release_commit__ = 'N\\/A'/;" \ diff --git a/docs/IMPLEMENTATION-DECISIONS.md b/docs/IMPLEMENTATION-DECISIONS.md index d3faef6..b1d3b57 100644 --- a/docs/IMPLEMENTATION-DECISIONS.md +++ b/docs/IMPLEMENTATION-DECISIONS.md @@ -204,3 +204,27 @@ integration up and running. When designing it, the following things were import At the time of writing, the plugin integration is still in development. What is important can change as we get actual users. + +# Plugin module import vs. initialization design + +When loading the python code for plugins, there were multiple ways to initialize the plugin: + + 1. The plugin could have an initialization function that is called by `debputy` + + 2. The plugin could use `@some_feature` annotations on types to be registered. This is similar + in spirit to the `add-on` system where module load is considered the initialization event. + +The 1. option was chosen because it is more reliable at directing the control flow and enables +the plugin module to be importable by non-`debputy` code. The latter might not seem directly +useful, but code like `py.test` attempts to import everything it can (even in `debian/` by +default) which could cause unwanted breakage for plugin providers by simply adding a +Python-based plugin. + +With the module autoloads mechanism, `debputy` would have to assume everything imported is +part of the plugin. This can cause issues (misattribution of errors) when a plugin loads +code or definitions from another plugin in addition to the `py.test` problem above. Safeguards +could be added, but the solutions found would either break the "py.test can import the module"- +property (see above) or be "safety is opt-in rather than always on/opt-out" by having an explicit +"This is plugin X code coming now" - the absence of a marker then causing misattribution). + +If these problems could be solved, the annotation based loading could be considered. diff --git a/lib/Dpkg/BuildDriver/Debputy.pm b/lib/Dpkg/BuildDriver/Debputy.pm new file mode 100644 index 0000000..957f608 --- /dev/null +++ b/lib/Dpkg/BuildDriver/Debputy.pm @@ -0,0 +1,44 @@ +package Dpkg::BuildDriver::Debputy; +use strict; +use warnings FATAL => 'all'; +use Dpkg::ErrorHandling; + +sub _run_cmd { + my @cmd = @_; + printcmd(@cmd); + system @cmd and subprocerr("@cmd"); +} + +sub new { + my ($this, %opts) = @_; + my $class = ref($this) || $this; + my $self = bless({ + 'ctrl' => $opts{ctrl}, + 'debputy_cmd' => 'debputy', + }, $class); + return $self; +} + + +sub pre_check { + my ($self) = @_; + my $ctrl_src = $self->{'ctrl'}->get_source(); + my $debputy_self_hosting_cmd = './debputy.sh'; + if ($ctrl_src->{"Source"} eq 'debputy' and -f -x $debputy_self_hosting_cmd) { + $self->{'debputy_cmd'} = $debputy_self_hosting_cmd; + notice("Detected this is a self-hosting build of debputy. Using \"${debputy_self_hosting_cmd}\" to self-host."); + } + return; +} + +sub need_build_task { + return 0; +} + +sub run_task { + my ($self, $task) = @_; + _run_cmd($self->{'debputy_cmd'}, 'internal-command', 'dpkg-build-driver-run-task', $task); + return; +} + +1; diff --git a/src/debputy/_manifest_constants.py b/src/debputy/_manifest_constants.py index 3ed992b..974ef7b 100644 --- a/src/debputy/_manifest_constants.py +++ b/src/debputy/_manifest_constants.py @@ -8,6 +8,8 @@ assert DEFAULT_MANIFEST_VERSION in SUPPORTED_MANIFEST_VERSIONS MK_MANIFEST_VERSION = "manifest-version" MK_PACKAGES = "packages" +MK_BUILDS = "builds" + MK_INSTALLATIONS = "installations" MK_INSTALLATIONS_INSTALL = "install" MK_INSTALLATIONS_MULTI_DEST_INSTALL = "multi-dest-install" diff --git a/src/debputy/build_support/__init__.py b/src/debputy/build_support/__init__.py new file mode 100644 index 0000000..8123659 --- /dev/null +++ b/src/debputy/build_support/__init__.py @@ -0,0 +1,7 @@ +from debputy.build_support.build_logic import perform_builds +from debputy.build_support.clean_logic import perform_clean + +__all__ = [ + "perform_clean", + "perform_builds", +] diff --git a/src/debputy/build_support/build_context.py b/src/debputy/build_support/build_context.py new file mode 100644 index 0000000..2eeef66 --- /dev/null +++ b/src/debputy/build_support/build_context.py @@ -0,0 +1,100 @@ +from typing import Mapping, Optional + +from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable +from debputy.commands.debputy_cmd.context import CommandContext +from debputy.highlevel_manifest import HighLevelManifest +from debputy.manifest_conditions import _run_build_time_tests + + +class BuildContext: + @staticmethod + def from_command_context( + cmd_context: CommandContext, + ) -> "BuildContext": + return BuildContextImpl(cmd_context) + + @property + def deb_build_options(self) -> Mapping[str, Optional[str]]: + raise NotImplementedError + + def parallelization_limit(self, *, support_zero_as_unlimited: bool = False) -> int: + """Parallelization limit of the build + + This is accessor that reads the `parallel` option from `DEB_BUILD_OPTIONS` with relevant + fallback behavior. + + :param support_zero_as_unlimited: The debhelper framework allowed `0` to mean unlimited + in some build systems. If the build system supports this, it should set this option + to True, which will allow `0` as a possible return value. WHen this option is False + (which is the default), `0` will be remapped to a high number to preserve the effect + in spirit (said fallback number is also from `debhelper`). + """ + limit = self.deb_build_options.get("parallel") + if limit is None: + return 1 + try: + v = int(limit) + except ValueError: + return 1 + if v == 0 and not support_zero_as_unlimited: + # debhelper allowed "0" to be used as unlimited in some cases. Preserve that feature + # for callers that are prepared for it. For everyone else, remap 0 to an obscene number + # that de facto has the same behaviour + # + # The number is taken out of `cmake.pm` from `debhelper` to be "Bug compatible" with + # debhelper on the fallback as well. + return 999 + return v + + @property + def is_terse_build(self) -> bool: + """Whether the build is terse + + This is a shorthand for testing for `terse` in DEB_BUILD_OPTIONS + """ + return "terse" in self.deb_build_options + + @property + def is_cross_compiling(self) -> bool: + """Whether the build is considered a cross build + + Note: Do **not** use this as indicator for whether tests should run. Use `should_run_tests` instead. + To the naive eye, they seem like they overlap in functionality, but they do not. There are cross + builds where tests can be run. Additionally, there are non-cross-builds where tests should be + skipped. + """ + return self.dpkg_architecture_variables.is_cross_compiling + + def cross_tool(self, command: str) -> str: + if not self.is_cross_compiling: + return command + cross_prefix = self.dpkg_architecture_variables["DEB_HOST_GNU_TYPE"] + return f"{cross_prefix}-{command}" + + @property + def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable: + raise NotImplementedError + + @property + def should_run_tests(self) -> bool: + return _run_build_time_tests(self.deb_build_options) + + +class BuildContextImpl(BuildContext): + def __init__( + self, + cmd_context: CommandContext, + ) -> None: + self._cmd_context = cmd_context + + @property + def deb_build_options(self) -> Mapping[str, Optional[str]]: + return self._cmd_context.deb_build_options + + @property + def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable: + return self._cmd_context.dpkg_architecture_variables() + + @property + def manifest(self) -> HighLevelManifest: + return self._manifest diff --git a/src/debputy/build_support/build_logic.py b/src/debputy/build_support/build_logic.py new file mode 100644 index 0000000..ee247e7 --- /dev/null +++ b/src/debputy/build_support/build_logic.py @@ -0,0 +1,193 @@ +import collections +import contextlib +import os +from typing import ( + Iterator, + Mapping, + List, + Dict, + Optional, +) + +from debputy.build_support.build_context import BuildContext +from debputy.build_support.buildsystem_detection import ( + auto_detect_buildsystem, +) +from debputy.commands.debputy_cmd.context import CommandContext +from debputy.highlevel_manifest import HighLevelManifest +from debputy.manifest_parser.base_types import BuildEnvironmentDefinition +from debputy.plugin.debputy.to_be_api_types import BuildRule +from debputy.util import ( + _error, + _info, + _non_verbose_info, +) + + +@contextlib.contextmanager +def in_build_env(build_env: BuildEnvironmentDefinition): + remove_unnecessary_env() + # Should possibly be per build + with _setup_build_env(build_env): + yield + + +def _set_stem_if_absent(stems: List[Optional[str]], idx: int, stem: str) -> None: + if stems[idx] is None: + stems[idx] = stem + + +def assign_stems( + build_rules: List[BuildRule], + manifest: HighLevelManifest, +) -> None: + if not build_rules: + return + if len(build_rules) == 1: + build_rules[0].auto_generated_stem = "" + return + + debs = {p.name for p in manifest.all_packages if p.package_type == "deb"} + udebs = {p.name for p in manifest.all_packages if p.package_type == "udeb"} + deb_only_builds: List[int] = [] + udeb_only_builds: List[int] = [] + by_name_only_builds: Dict[str, List[int]] = collections.defaultdict(list) + stems = [rule.name for rule in build_rules] + reserved_stems = set(n for n in stems if n is not None) + + for idx, rule in enumerate(build_rules): + stem = stems[idx] + if stem is not None: + continue + pkg_names = {p.name for p in rule.for_packages} + if pkg_names == debs: + deb_only_builds.append(idx) + elif pkg_names == udebs: + udeb_only_builds.append(idx) + + if len(pkg_names) == 1: + pkg_name = next(iter(pkg_names)) + by_name_only_builds[pkg_name].append(idx) + + if "deb" not in reserved_stems and len(deb_only_builds) == 1: + _set_stem_if_absent(stems, deb_only_builds[0], "deb") + + if "udeb" not in reserved_stems and len(udeb_only_builds) == 1: + _set_stem_if_absent(stems, udeb_only_builds[0], "udeb") + + for pkg, idxs in by_name_only_builds.items(): + if len(idxs) != 1 or pkg in reserved_stems: + continue + _set_stem_if_absent(stems, idxs[0], pkg) + + for idx, rule in enumerate(build_rules): + stem = stems[idx] + if stem is None: + stem = f"bno_{idx}" + rule.auto_generated_stem = stem + _info(f"Assigned {rule.auto_generated_stem} [{stem}] to step {idx}") + + +def perform_builds( + context: CommandContext, + manifest: HighLevelManifest, +) -> None: + build_rules = manifest.build_rules + if build_rules is not None: + if not build_rules: + # Defined but empty disables the auto-detected build system + return + active_packages = frozenset(manifest.active_packages) + condition_context = manifest.source_condition_context + build_context = BuildContext.from_command_context(context) + assign_stems(build_rules, manifest) + for step_no, build_rule in enumerate(build_rules): + step_ref = ( + f"step {step_no} [{build_rule.auto_generated_stem}]" + if build_rule.name is None + else f"step {step_no} [{build_rule.name}]" + ) + if build_rule.for_packages.isdisjoint(active_packages): + _info( + f"Skipping build for {step_ref}: None of the relevant packages are being built" + ) + continue + manifest_condition = build_rule.manifest_condition + if manifest_condition is not None and not manifest_condition.evaluate( + condition_context + ): + _info( + f"Skipping build for {step_ref}: The condition clause evaluated to false" + ) + continue + _info(f"Starting build for {step_ref}.") + with in_build_env(build_rule.environment): + try: + build_rule.run_build(build_context, manifest) + except (RuntimeError, AttributeError) as e: + if context.parsed_args.debug_mode: + raise e + _error( + f"An error occurred during build/install at {step_ref} (defined at {build_rule.attribute_path.path}): {str(e)}" + ) + _info(f"Completed build for {step_ref}.") + + else: + build_system = auto_detect_buildsystem(manifest) + if build_system: + _info(f"Auto-detected build system: {build_system.__class__.__name__}") + build_context = BuildContext.from_command_context(context) + with in_build_env(build_system.environment): + with in_build_env(build_system.environment): + build_system.run_build( + build_context, + manifest, + ) + + _non_verbose_info("Upstream builds completed successfully") + else: + _info("No build system was detected from the current plugin set.") + + +def remove_unnecessary_env() -> None: + vs = [ + "XDG_CACHE_HOME", + "XDG_CONFIG_DIRS", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_DATA_DIRS", + "XDG_RUNTIME_DIR", + ] + for v in vs: + if v in os.environ: + del os.environ[v] + + # FIXME: Add custom HOME + XDG_RUNTIME_DIR + + +@contextlib.contextmanager +def _setup_build_env(build_env: BuildEnvironmentDefinition) -> Iterator[None]: + env_backup = dict(os.environ) + env = dict(env_backup) + had_delta = False + build_env.update_env(env) + if env != env_backup: + _set_env(env) + had_delta = True + _info("Updated environment to match build") + yield + if had_delta or env != env_backup: + _set_env(env_backup) + + +def _set_env(desired_env: Mapping[str, str]) -> None: + os_env = os.environ + for key in os_env.keys() | desired_env.keys(): + desired_value = desired_env.get(key) + if desired_value is None: + try: + del os_env[key] + except KeyError: + pass + else: + os_env[key] = desired_value diff --git a/src/debputy/build_support/buildsystem_detection.py b/src/debputy/build_support/buildsystem_detection.py new file mode 100644 index 0000000..47415fd --- /dev/null +++ b/src/debputy/build_support/buildsystem_detection.py @@ -0,0 +1,112 @@ +from typing import ( + Optional, +) + +from debputy.exceptions import ( + DebputyPluginRuntimeError, + PluginBaseError, +) +from debputy.filesystem_scan import FSRootDir, FSROOverlay +from debputy.highlevel_manifest import HighLevelManifest +from debputy.manifest_parser.base_types import BuildEnvironmentDefinition +from debputy.manifest_parser.util import AttributePath +from debputy.plugin.debputy.to_be_api_types import ( + BuildSystemRule, +) +from debputy.plugin.plugin_state import run_in_context_of_plugin_wrap_errors +from debputy.util import ( + _error, + _debug_log, +) + + +def default_build_environment_only( + manifest: HighLevelManifest, +) -> BuildEnvironmentDefinition: + build_envs = manifest.build_environments + if build_envs.environments: + _error( + 'When automatic build system detection is used, the manifest cannot use "build-environments"' + ) + build_env = build_envs.default_environment + assert build_env is not None + return build_env + + +def auto_detect_buildsystem( + manifest: HighLevelManifest, +) -> Optional[BuildSystemRule]: + auto_detectable_build_systems = ( + manifest.plugin_provided_feature_set.auto_detectable_build_systems + ) + excludes = set() + options = [] + _debug_log("Auto-detecting build systems.") + source_root = FSROOverlay.create_root_dir("", ".") + for ppadbs in auto_detectable_build_systems.values(): + detected = ppadbs.detector(source_root) + if not isinstance(detected, bool): + _error( + f'The auto-detector for the build system {ppadbs.manifest_keyword} returned a "non-bool"' + f" ({detected!r}), which could be a bug in the plugin or the plugin relying on a newer" + " version of `debputy` that changed the auto-detection protocol." + ) + if not detected: + _debug_log( + f"Skipping build system {ppadbs.manifest_keyword}: Detector returned False!" + ) + continue + _debug_log( + f"Considering build system {ppadbs.manifest_keyword} as its Detector returned True!" + ) + if ppadbs.auto_detection_shadow_build_systems: + names = ", ".join( + sorted(x for x in ppadbs.auto_detection_shadow_build_systems) + ) + _debug_log(f"Build system {ppadbs.manifest_keyword} excludes: {names}!") + excludes.update(ppadbs.auto_detection_shadow_build_systems) + options.append(ppadbs) + + if not options: + _debug_log("Zero candidates; continuing without a build system") + return None + + if excludes: + names = ", ".join(sorted(x for x in excludes)) + _debug_log(f"The following build systems have been excluded: {names}!") + remaining_options = [o for o in options if o.manifest_keyword not in excludes] + else: + remaining_options = options + + if len(remaining_options) > 1: + names = ", ".join(o.manifest_keyword for o in remaining_options) + # TODO: This means adding an auto-detectable build system to an existing plugin causes FTBFS + # We need a better way of handling this. Probably the build systems should include + # a grace timer based on d/changelog. Anything before the changelog date is in + # "grace mode" and will not conflict with a build system that is. If all choices + # are in "grace mode", "oldest one" wins. + _error( + f"Multiple build systems match, please pick one explicitly (under `builds:`): {names}" + ) + + if not remaining_options: + names = ", ".join(o.build_system_rule_type.__name__ for o in options) + # TODO: Detect at registration time + _error( + f"Multiple build systems matched but they all shadowed each other: {names}." + f" There is a bug in at least one of them!" + ) + + chosen_build_system = remaining_options[0] + environment = default_build_environment_only(manifest) + bs = run_in_context_of_plugin_wrap_errors( + chosen_build_system.plugin_metadata.plugin_name, + chosen_build_system.constructor, + { + "environment": environment, + }, + AttributePath.builtin_path(), + manifest, + ) + bs.auto_generated_stem = "" + return bs diff --git a/src/debputy/build_support/clean_logic.py b/src/debputy/build_support/clean_logic.py new file mode 100644 index 0000000..13347b0 --- /dev/null +++ b/src/debputy/build_support/clean_logic.py @@ -0,0 +1,233 @@ +import os.path +from typing import ( + Set, + cast, + List, +) + +from debputy.build_support.build_context import BuildContext +from debputy.build_support.build_logic import ( + in_build_env, + assign_stems, +) +from debputy.build_support.buildsystem_detection import auto_detect_buildsystem +from debputy.commands.debputy_cmd.context import CommandContext +from debputy.highlevel_manifest import HighLevelManifest +from debputy.plugin.debputy.to_be_api_types import BuildSystemRule, CleanHelper +from debputy.util import _info, print_command, _error, _debug_log, _warn +from debputy.util import ( + run_build_system_command, +) + +_REMOVE_DIRS = frozenset( + [ + "__pycache__", + "autom4te.cache", + ] +) +_IGNORE_DIRS = frozenset( + [ + ".git", + ".svn", + ".bzr", + ".hg", + "CVS", + ".pc", + "_darcs", + ] +) +DELETE_FILE_EXT = ( + "~", + ".orig", + ".rej", + ".bak", +) +DELETE_FILE_BASENAMES = { + "DEADJOE", + ".SUMS", + "TAGS", +} + + +def _debhelper_left_overs() -> bool: + if os.path.lexists("debian/.debhelper") or os.path.lexists( + "debian/debhelper-build-stamp" + ): + return True + with os.scandir(".") as root_dir: + for child in root_dir: + if child.is_file(follow_symlinks=False) and ( + child.name.endswith(".debhelper.log") + or child.name.endswith(".debhelper") + ): + return True + return False + + +class CleanHelperImpl(CleanHelper): + + def __init__(self) -> None: + self.files_to_remove: Set[str] = set() + self.dirs_to_remove: Set[str] = set() + + def schedule_removal_of_files(self, *args: str) -> None: + self.files_to_remove.update(args) + + def schedule_removal_of_directories(self, *args: str) -> None: + if any(p == "/" for p in args): + raise ValueError("Refusing to delete '/'") + self.dirs_to_remove.update(args) + + +def _scan_for_standard_removals(clean_helper: CleanHelperImpl) -> None: + remove_files = clean_helper.files_to_remove + remove_dirs = clean_helper.dirs_to_remove + with os.scandir(".") as root_dir: + for child in root_dir: + if child.is_file(follow_symlinks=False) and child.name.endswith("-stamp"): + remove_files.add(child.path) + for current_dir, subdirs, files in os.walk("."): + for remove_dir in [d for d in subdirs if d in _REMOVE_DIRS]: + path = os.path.join(current_dir, remove_dir) + remove_dirs.add(path) + subdirs.remove(remove_dir) + for skip_dir in [d for d in subdirs if d in _IGNORE_DIRS]: + subdirs.remove(skip_dir) + + for basename in files: + if ( + basename.endswith(DELETE_FILE_EXT) + or basename in DELETE_FILE_BASENAMES + or (basename.startswith("#") and basename.endswith("#")) + ): + path = os.path.join(current_dir, basename) + remove_files.add(path) + + +def perform_clean( + context: CommandContext, + manifest: HighLevelManifest, +) -> None: + clean_helper = CleanHelperImpl() + + build_rules = manifest.build_rules + if build_rules is not None: + if not build_rules: + # Defined but empty disables the auto-detected build system + return + active_packages = frozenset(manifest.active_packages) + condition_context = manifest.source_condition_context + build_context = BuildContext.from_command_context(context) + assign_stems(build_rules, manifest) + for step_no, build_rule in enumerate(build_rules): + step_ref = ( + f"step {step_no} [{build_rule.auto_generated_stem}]" + if build_rule.name is None + else f"step {step_no} [{build_rule.name}]" + ) + if not build_rule.is_buildsystem: + _debug_log(f"Skipping clean for {step_ref}: Not a build system") + continue + build_system_rule: BuildSystemRule = cast("BuildSystemRule", build_rule) + if build_system_rule.for_packages.isdisjoint(active_packages): + _info( + f"Skipping build for {step_ref}: None of the relevant packages are being built" + ) + continue + manifest_condition = build_system_rule.manifest_condition + if manifest_condition is not None and not manifest_condition.evaluate( + condition_context + ): + _info( + f"Skipping clean for {step_ref}: The condition clause evaluated to false" + ) + continue + _info(f"Starting clean for {step_ref}.") + with in_build_env(build_rule.environment): + try: + build_system_rule.run_clean( + build_context, + manifest, + clean_helper, + ) + except (RuntimeError, AttributeError) as e: + if context.parsed_args.debug_mode: + raise e + _error( + f"An error occurred during clean at {step_ref} (defined at {build_rule.attribute_path.path}): {str(e)}" + ) + _info(f"Completed clean for {step_ref}.") + else: + build_system = auto_detect_buildsystem(manifest) + if build_system: + _info(f"Auto-detected build system: {build_system.__class__.__name__}") + build_context = BuildContext.from_command_context(context) + with in_build_env(build_system.environment): + build_system.run_clean( + build_context, + manifest, + clean_helper, + ) + else: + _info("No build system was detected from the current plugin set.") + + dh_autoreconf_used = os.path.lexists("debian/autoreconf.before") + debhelper_used = False + + if dh_autoreconf_used or _debhelper_left_overs(): + debhelper_used = True + + _scan_for_standard_removals(clean_helper) + + for package in manifest.all_packages: + package_staging_dir = os.path.join("debian", package.name) + if os.path.lexists(package_staging_dir): + clean_helper.schedule_removal_of_directories(package_staging_dir) + + remove_files = clean_helper.files_to_remove + remove_dirs = clean_helper.dirs_to_remove + if remove_files: + print_command("rm", "-f", *remove_files) + _remove_files_if_exists(*remove_files) + if remove_dirs: + run_build_system_command("rm", "-fr", *remove_dirs) + + if debhelper_used: + _info( + "Noted traces of debhelper commands being used; invoking dh_clean to clean up after them" + ) + if dh_autoreconf_used: + run_build_system_command("dh_autoreconf_clean") + run_build_system_command("dh_clean") + + try: + run_build_system_command("dpkg-buildtree", "clean") + except FileNotFoundError: + _warn("The dpkg-buildtree command is not present. Emulating it") + # This is from the manpage of dpkg-buildtree for 1.22.11. + _remove_files_if_exists( + "debian/files", + "debian/files.new", + "debian/substvars", + "debian/substvars.new", + ) + run_build_system_command("rm", "-fr", "debian/tmp") + # Remove debian/.debputy as a separate step. While `rm -fr` should process things in order, + # it will continue on error, which could cause our manifests of things to delete to be deleted + # while leaving things half-removed unless we do this extra step. + run_build_system_command("rm", "-fr", "debian/.debputy") + + +def _remove_files_if_exists(*args: str) -> None: + for path in args: + try: + os.unlink(path) + except FileNotFoundError: + continue + except OSError as e: + if os.path.isdir(path): + _error( + f"Failed to remove {path}: It is a directory, but it should have been a non-directory." + " Please verify everything is as expected and, if it is, remove it manually." + ) + _error(f"Failed to remove {path}: {str(e)}") diff --git a/src/debputy/builtin_manifest_rules.py b/src/debputy/builtin_manifest_rules.py index e420cda..e31a50e 100644 --- a/src/debputy/builtin_manifest_rules.py +++ b/src/debputy/builtin_manifest_rules.py @@ -17,7 +17,7 @@ from debputy.path_matcher import ( ) from debputy.substitution import Substitution from debputy.types import VP -from debputy.util import _normalize_path, perl_module_dirs +from debputy.util import _normalize_path, resolve_perl_config # Imported from dh_fixperms _PERMISSION_NORMALIZATION_SOURCE_DEFINITION = "permission normalization" @@ -218,17 +218,19 @@ def builtin_mode_normalization_rules( OctalMode(0o0644), ) + perl_config_data = resolve_perl_config(dpkg_architecture_variables, dctrl_bin) + yield from ( ( BasenameGlobMatch( "*.pm", - only_when_in_directory=perl_dir, + only_when_in_directory=_normalize_path(perl_dir), path_type=PathType.FILE, recursive_match=True, ), _STD_FILE_MODE, ) - for perl_dir in perl_module_dirs(dpkg_architecture_variables, dctrl_bin) + for perl_dir in (perl_config_data.vendorlib, perl_config_data.vendorarch) ) yield ( diff --git a/src/debputy/commands/debputy_cmd/__main__.py b/src/debputy/commands/debputy_cmd/__main__.py index 3270737..05bd135 100644 --- a/src/debputy/commands/debputy_cmd/__main__.py +++ b/src/debputy/commands/debputy_cmd/__main__.py @@ -24,6 +24,7 @@ from typing import ( from debputy import DEBPUTY_ROOT_DIR, DEBPUTY_PLUGIN_ROOT_DIR from debputy.analysis import REFERENCE_DATA_TABLE from debputy.analysis.debian_dir import scan_debian_dir +from debputy.build_support import perform_clean, perform_builds from debputy.commands.debputy_cmd.context import ( CommandContext, add_arg, @@ -40,10 +41,15 @@ from debputy.exceptions import ( UnhandledOrUnexpectedErrorFromPluginError, SymlinkLoopError, ) +from debputy.highlevel_manifest import HighLevelManifest from debputy.package_build.assemble_deb import ( assemble_debs, ) -from debputy.plugin.api.spec import INTEGRATION_MODE_DH_DEBPUTY_RRR +from debputy.plugin.api.spec import ( + INTEGRATION_MODE_DH_DEBPUTY_RRR, + DebputyIntegrationMode, + INTEGRATION_MODE_FULL, +) try: from argcomplete import autocomplete @@ -92,8 +98,9 @@ from debputy.util import ( escape_shell, program_name, integrated_with_debhelper, - change_log_level, + PRINT_BUILD_SYSTEM_COMMAND, PRINT_COMMAND, + change_log_level, ) @@ -255,6 +262,18 @@ def _add_packages_args(parser: argparse.ArgumentParser) -> None: ) +def _build_subcommand_log_level(context: CommandContext) -> int: + parsed_args = context.parsed_args + log_level: Optional[int] = None + if os.environ.get("DH_VERBOSE", "") != "": + log_level = PRINT_COMMAND + if parsed_args.debug_mode: + log_level = logging.INFO + if log_level is not None: + change_log_level(log_level) + return PRINT_BUILD_SYSTEM_COMMAND + + internal_commands = ROOT_COMMAND.add_dispatching_subcommand( "internal-command", dest="internal_command", @@ -630,10 +649,71 @@ def _run_tests_for_plugin(context: CommandContext) -> None: @internal_commands.register_subcommand( + "dpkg-build-driver-run-task", + help_description="[Internal command] Perform a given Dpkg::BuildDriver task (Not stable API)", + requested_plugins_only=True, + default_log_level=_build_subcommand_log_level, + argparser=[ + add_arg( + "task_name", + metavar="task-name", + choices=[ + "clean", + "build", + "build-arch", + "build-indep", + "binary", + "binary-arch", + "binary-indep", + ], + help="The task to run", + ), + add_arg( + "output", + nargs="?", + default="..", + metavar="output", + help="Where to place the resulting packages. Should be a directory", + ), + ], +) +def _dpkg_build_driver_integration(context: CommandContext) -> None: + parsed_args = context.parsed_args + log_level = context.set_log_level_for_build_subcommand() + task_name = parsed_args.task_name + + if task_name.endswith("-indep"): + context.package_set = "indep" + elif task_name.endswith("arch"): + context.package_set = "arch" + + manifest = context.parse_manifest() + + plugins = context.load_plugins().plugin_data + for plugin in plugins.values(): + if not plugin.is_bundled: + _info(f"Loaded plugin {plugin.plugin_name}") + if task_name == "clean": + perform_clean(context, manifest) + elif task_name in ("build", "build-indep", "build-arch"): + perform_builds(context, manifest) + elif task_name in ("binary", "binary-indep", "binary-arch"): + perform_builds(context, manifest) + assemble( + context, + manifest, + INTEGRATION_MODE_FULL, + debug_materialization=log_level is not None, + ) + else: + _error(f"Unsupported Dpkg::BuildDriver task: {task_name}.") + + +@internal_commands.register_subcommand( "dh-integration-generate-debs", help_description="[Internal command] Generate .deb/.udebs packages from debian/<pkg> (Not stable API)", requested_plugins_only=True, - default_log_level=logging.WARN, + default_log_level=_build_subcommand_log_level, argparser=[ _add_packages_args, add_arg( @@ -653,14 +733,7 @@ def _run_tests_for_plugin(context: CommandContext) -> None: ) def _dh_integration_generate_debs(context: CommandContext) -> None: integrated_with_debhelper() - parsed_args = context.parsed_args - log_level: Optional[int] = None - if os.environ.get("DH_VERBOSE", "") != "": - log_level = PRINT_COMMAND - if parsed_args.debug_mode: - log_level = logging.INFO - if log_level is not None: - change_log_level(log_level) + log_level = context.set_log_level_for_build_subcommand() integration_mode = context.resolve_integration_mode() is_dh_rrr_only_mode = integration_mode == INTEGRATION_MODE_DH_DEBPUTY_RRR if is_dh_rrr_only_mode: @@ -678,13 +751,28 @@ def _dh_integration_generate_debs(context: CommandContext) -> None: _info(f"Loaded plugin {plugin.plugin_name}") manifest = context.parse_manifest() - package_data_table = manifest.perform_installations( - enable_manifest_installation_feature=not is_dh_rrr_only_mode + assemble( + context, + manifest, + integration_mode, + debug_materialization=log_level is not None, ) + + +def assemble( + context: CommandContext, + manifest: HighLevelManifest, + integration_mode: DebputyIntegrationMode, + *, + debug_materialization: bool = False, +) -> None: source_fs = FSROOverlay.create_root_dir("..", ".") source_version = manifest.source_version() is_native = "-" not in source_version - + is_dh_rrr_only_mode = integration_mode == INTEGRATION_MODE_DH_DEBPUTY_RRR + package_data_table = manifest.perform_installations( + enable_manifest_installation_feature=not is_dh_rrr_only_mode + ) if not is_dh_rrr_only_mode: for dctrl_bin in manifest.active_packages: package = dctrl_bin.name @@ -702,7 +790,7 @@ def _dh_integration_generate_debs(context: CommandContext) -> None: fs_root, dctrl_data.substvars, ) - if "nostrip" not in manifest.build_env.deb_build_options: + if "nostrip" not in manifest.deb_options_and_profiles.deb_build_options: dbgsym_ids = relocate_dwarves_into_dbgsym_packages( dctrl_bin, fs_root, @@ -714,7 +802,7 @@ def _dh_integration_generate_debs(context: CommandContext) -> None: dctrl_bin, fs_root, is_native, - manifest.build_env, + manifest.deb_options_and_profiles, ) if not is_native: install_upstream_changelog( @@ -739,7 +827,7 @@ def _dh_integration_generate_debs(context: CommandContext) -> None: manifest, package_data_table, is_dh_rrr_only_mode, - debug_materialization=log_level is not None, + debug_materialization=debug_materialization, ) @@ -901,10 +989,10 @@ def _json_output(data: Any) -> None: ], ) def _migrate_from_dh(context: CommandContext) -> None: + context.must_be_called_in_source_root() parsed_args = context.parsed_args - resolved_migration_target = _check_migration_target( - context.debian_dir, + context, parsed_args.migration_target, ) context.debputy_integration_mode = resolved_migration_target diff --git a/src/debputy/commands/debputy_cmd/context.py b/src/debputy/commands/debputy_cmd/context.py index 0c184c7..a9c0a13 100644 --- a/src/debputy/commands/debputy_cmd/context.py +++ b/src/debputy/commands/debputy_cmd/context.py @@ -15,6 +15,7 @@ from typing import ( Callable, Dict, TYPE_CHECKING, + Literal, ) from debian.debian_support import DpkgArchTable @@ -45,7 +46,14 @@ from debputy.substitution import ( SubstitutionImpl, NULL_SUBSTITUTION, ) -from debputy.util import _error, PKGNAME_REGEX, resolve_source_date_epoch, setup_logging +from debputy.util import ( + _error, + PKGNAME_REGEX, + resolve_source_date_epoch, + setup_logging, + PRINT_COMMAND, + change_log_level, +) if TYPE_CHECKING: from argparse import _SubParsersAction @@ -110,6 +118,19 @@ class CommandContext: Mapping[str, "BinaryPackage"], ] ] = None + self._package_set: Literal["both", "arch", "indep"] = "both" + + @property + def package_set(self) -> Literal["both", "arch", "indep"]: + return self._package_set + + @package_set.setter + def package_set(self, new_value: Literal["both", "arch", "indep"]) -> None: + if self._dctrl_parser is not None: + raise TypeError( + "package_set cannot be redefined once the debian/control parser has been initialized" + ) + self._package_set = new_value @property def debian_dir(self) -> VirtualPath: @@ -135,9 +156,10 @@ class CommandContext: parser = DctrlParser( packages, # -p/--package set(), # -N/--no-package - False, # -i - False, # -a - build_env=DebBuildOptionsAndProfiles.instance(), + # binary-indep and binary-indep (dpkg BuildDriver integration only) + self._package_set == "indep", + self._package_set == "arch", + deb_options_and_profiles=DebBuildOptionsAndProfiles.instance(), dpkg_architecture_variables=dpkg_architecture_table(), dpkg_arch_query_table=DpkgArchTable.load_arch_table(), ) @@ -152,6 +174,9 @@ class CommandContext: _, binary_package_table = self._parse_dctrl() return binary_package_table + def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable: + return self.dctrl_parser.dpkg_architecture_variables + def requested_plugins(self) -> Sequence[str]: if self._requested_plugins is None: self._requested_plugins = self._resolve_requested_plugins() @@ -162,7 +187,7 @@ class CommandContext: @property def deb_build_options_and_profiles(self) -> "DebBuildOptionsAndProfiles": - return self.dctrl_parser.build_env + return self.dctrl_parser.deb_options_and_profiles @property def deb_build_options(self) -> Mapping[str, Optional[str]]: @@ -292,20 +317,37 @@ class CommandContext: debian_control = self.debian_dir.get("control") return debian_control is not None - def resolve_integration_mode(self) -> DebputyIntegrationMode: + def resolve_integration_mode( + self, + require_integration: bool = True, + ) -> DebputyIntegrationMode: integration_mode = self.debputy_integration_mode if integration_mode is None: r = read_dh_addon_sequences(self.debian_dir) bd_sequences, dr_sequences, _ = r all_sequences = bd_sequences | dr_sequences - integration_mode = determine_debputy_integration_mode(all_sequences) - if integration_mode is None: + integration_mode = determine_debputy_integration_mode( + self.source_package().fields, + all_sequences, + ) + if integration_mode is None and not require_integration: _error( "Cannot resolve the integration mode expected for this package. Is this package using `debputy`?" ) self.debputy_integration_mode = integration_mode return integration_mode + def set_log_level_for_build_subcommand(self) -> Optional[int]: + parsed_args = self.parsed_args + log_level: Optional[int] = None + if os.environ.get("DH_VERBOSE", "") != "": + log_level = PRINT_COMMAND + if parsed_args.debug_mode or os.environ.get("DEBPUTY_DEBUG", "") != "": + log_level = logging.DEBUG + if log_level is not None: + change_log_level(log_level) + return log_level + def manifest_parser( self, *, @@ -320,7 +362,6 @@ class CommandContext: manifest_path = self.parsed_args.debputy_manifest if manifest_path is None: manifest_path = os.path.join(self.debian_dir.fs_path, "debputy.manifest") - debian_dir = self.debian_dir return YAMLManifestParser( manifest_path, source_package, @@ -328,7 +369,7 @@ class CommandContext: substitution, dctrl_parser.dpkg_architecture_variables, dctrl_parser.dpkg_arch_query_table, - dctrl_parser.build_env, + dctrl_parser.deb_options_and_profiles, self.load_plugins(), self.resolve_integration_mode(), debian_dir=self.debian_dir, @@ -420,7 +461,7 @@ class GenericSubCommand(SubcommandBase): require_substitution: bool = True, requested_plugins_only: bool = False, log_only_to_stderr: bool = False, - default_log_level: int = logging.INFO, + default_log_level: Union[int, Callable[[CommandContext], int]] = logging.INFO, ) -> None: super().__init__(name, aliases=aliases, help_description=help_description) self._handler = handler @@ -452,7 +493,18 @@ class GenericSubCommand(SubcommandBase): ) if self._log_only_to_stderr: setup_logging(reconfigure_logging=True, log_only_to_stderr=True) - logging.getLogger().setLevel(self._default_log_level) + + default_log_level = self._default_log_level + if isinstance(default_log_level, int): + level = default_log_level + else: + assert callable(default_log_level) + level = default_log_level(context) + change_log_level(level) + if level > logging.DEBUG and ( + context.parsed_args.debug_mode or os.environ.get("DEBPUTY_DEBUG", "") != "" + ): + change_log_level(logging.DEBUG) return self._handler(context) @@ -494,7 +546,7 @@ class DispatchingCommandMixin(CommandBase): require_substitution: bool = True, requested_plugins_only: bool = False, log_only_to_stderr: bool = False, - default_log_level: int = logging.INFO, + default_log_level: Union[int, Callable[[CommandContext], int]] = logging.INFO, ) -> Callable[[CommandHandler], GenericSubCommand]: if isinstance(name, str): cmd_name = name diff --git a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py index eaab750..a03126b 100644 --- a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py +++ b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py @@ -267,7 +267,6 @@ def lsp_describe_features(context: CommandContext) -> None: "--spellcheck", dest="spellcheck", action="store_true", - shared=True, help="Enable spellchecking", ), add_arg( diff --git a/src/debputy/commands/debputy_cmd/plugin_cmds.py b/src/debputy/commands/debputy_cmd/plugin_cmds.py index 83bb88f..9721702 100644 --- a/src/debputy/commands/debputy_cmd/plugin_cmds.py +++ b/src/debputy/commands/debputy_cmd/plugin_cmds.py @@ -2,6 +2,7 @@ import argparse import operator import os import sys +import textwrap from itertools import chain from typing import ( Sequence, @@ -28,7 +29,7 @@ from debputy.commands.debputy_cmd.output import ( ) from debputy.exceptions import DebputySubstitutionError from debputy.filesystem_scan import build_virtual_fs -from debputy.manifest_parser.base_types import TypeMapping +from debputy.manifest_parser.tagging_types import TypeMapping from debputy.manifest_parser.declarative_parser import ( BASIC_SIMPLE_TYPES, ) @@ -45,13 +46,15 @@ from debputy.plugin.api.impl_types import ( PackagerProvidedFileClassSpec, PluginProvidedManifestVariable, DispatchingParserBase, - SUPPORTED_DISPATCHABLE_TABLE_PARSERS, - OPARSER_MANIFEST_ROOT, PluginProvidedDiscardRule, AutomaticDiscardRuleExample, MetadataOrMaintscriptDetector, PluginProvidedTypeMapping, ) +from debputy.plugin.api.parser_tables import ( + SUPPORTED_DISPATCHABLE_TABLE_PARSERS, + OPARSER_MANIFEST_ROOT, +) from debputy.plugin.api.spec import ( TypeMappingExample, ) @@ -538,7 +541,16 @@ def _plugin_cmd_show_manifest_variables(context: CommandContext) -> None: variable_value=None, is_context_specific_variable=False, is_documentation_placeholder=True, - variable_reference_documentation=f'Environment variable "{env_var}"', + variable_reference_documentation=textwrap.dedent( + f"""\ + Environment variable "{env_var}" + + Note that uses beneath `builds:` may use the environment variable defined by + `build-environment:` (depends on whether the rule uses eager or lazy + substitution) while uses outside `builds:` will generally not use a definition + from `build-environment:`. + """ + ), ) else: variable = variables.get(variable_name) diff --git a/src/debputy/deb_packaging_support.py b/src/debputy/deb_packaging_support.py index 875b3b1..92f57a2 100644 --- a/src/debputy/deb_packaging_support.py +++ b/src/debputy/deb_packaging_support.py @@ -81,7 +81,7 @@ from debputy.util import ( _error, ensure_dir, assume_not_none, - perl_module_dirs, + resolve_perl_config, perlxs_api_dependency, detect_fakeroot, grouper, @@ -186,11 +186,11 @@ def handle_perl_code( fs_root: FSPath, substvars: FlushableSubstvars, ) -> None: - known_perl_inc_dirs = perl_module_dirs(dpkg_architecture_variables, dctrl_bin) + perl_config_data = resolve_perl_config(dpkg_architecture_variables, dctrl_bin) detected_dep_requirements = 0 # MakeMaker always makes lib and share dirs, but typically only one directory is actually used. - for perl_inc_dir in known_perl_inc_dirs: + for perl_inc_dir in (perl_config_data.vendorarch, perl_config_data.vendorlib): p = fs_root.lookup(perl_inc_dir) if p and p.is_dir: p.prune_if_empty_dir() @@ -198,8 +198,8 @@ def handle_perl_code( # FIXME: 80% of this belongs in a metadata detector, but that requires us to expose .walk() in the public API, # which will not be today. for d, pm_mode in [ - (known_perl_inc_dirs.vendorlib, PERL_DEP_INDEP_PM_MODULE), - (known_perl_inc_dirs.vendorarch, PERL_DEP_ARCH_PM_MODULE), + (perl_config_data.vendorlib, PERL_DEP_INDEP_PM_MODULE), + (perl_config_data.vendorarch, PERL_DEP_ARCH_PM_MODULE), ]: inc_dir = fs_root.lookup(d) if not inc_dir: diff --git a/src/debputy/dh_migration/migration.py b/src/debputy/dh_migration/migration.py index f7b7d9e..62f739e 100644 --- a/src/debputy/dh_migration/migration.py +++ b/src/debputy/dh_migration/migration.py @@ -3,10 +3,11 @@ import os import re import subprocess from itertools import chain -from typing import Optional, List, Callable, Set, Container +from typing import Optional, List, Callable, Set, Container, Mapping, FrozenSet from debian.deb822 import Deb822 +from debputy.commands.debputy_cmd.context import CommandContext from debputy.dh.debhelper_emulation import CannotEmulateExecutableDHConfigFile from debputy.dh_migration.migrators import MIGRATORS from debputy.dh_migration.migrators_impl import ( @@ -24,9 +25,25 @@ from debputy.highlevel_manifest import HighLevelManifest from debputy.integration_detection import determine_debputy_integration_mode from debputy.manifest_parser.exceptions import ManifestParseException from debputy.plugin.api import VirtualPath -from debputy.plugin.api.spec import DebputyIntegrationMode +from debputy.plugin.api.spec import DebputyIntegrationMode, INTEGRATION_MODE_FULL from debputy.util import _error, _warn, _info, escape_shell, assume_not_none +SUPPORTED_MIGRATIONS: Mapping[ + DebputyIntegrationMode, FrozenSet[DebputyIntegrationMode] +] = { + INTEGRATION_MODE_FULL: frozenset([INTEGRATION_MODE_FULL]), + INTEGRATION_MODE_DH_DEBPUTY: frozenset( + [INTEGRATION_MODE_DH_DEBPUTY, INTEGRATION_MODE_FULL] + ), + INTEGRATION_MODE_DH_DEBPUTY_RRR: frozenset( + [ + INTEGRATION_MODE_DH_DEBPUTY_RRR, + INTEGRATION_MODE_DH_DEBPUTY, + INTEGRATION_MODE_FULL, + ] + ), +} + def _print_migration_summary( migrations: List[FeatureMigration], @@ -143,22 +160,33 @@ def _requested_debputy_plugins(debian_dir: VirtualPath) -> Optional[Set[str]]: def _check_migration_target( - debian_dir: VirtualPath, + context: CommandContext, migration_target: Optional[DebputyIntegrationMode], ) -> DebputyIntegrationMode: - r = read_dh_addon_sequences(debian_dir) - if r is None and migration_target is None: - _error("debian/control is missing and no migration target was provided") - bd_sequences, dr_sequences, _ = r - all_sequences = bd_sequences | dr_sequences - - detected_migration_target = determine_debputy_integration_mode(all_sequences) - - if ( - migration_target == INTEGRATION_MODE_DH_DEBPUTY_RRR - and detected_migration_target == INTEGRATION_MODE_DH_DEBPUTY - ): - _error("Cannot migrate from (zz-)debputy to zz-debputy-rrr") + r = read_dh_addon_sequences(context.debian_dir) + if r is not None: + bd_sequences, dr_sequences, _ = r + all_sequences = bd_sequences | dr_sequences + detected_migration_target = determine_debputy_integration_mode( + context.source_package().fields, + all_sequences, + ) + else: + detected_migration_target = None + + if migration_target is not None and detected_migration_target is not None: + supported_migrations = SUPPORTED_MIGRATIONS.get( + detected_migration_target, + frozenset([detected_migration_target]), + ) + + if ( + migration_target != detected_migration_target + and migration_target not in supported_migrations + ): + _error( + f"Cannot migrate from {detected_migration_target} to {migration_target}" + ) if migration_target is not None: resolved_migration_target = migration_target diff --git a/src/debputy/dh_migration/migrators.py b/src/debputy/dh_migration/migrators.py index 8eff679..8a057b7 100644 --- a/src/debputy/dh_migration/migrators.py +++ b/src/debputy/dh_migration/migrators.py @@ -13,7 +13,7 @@ from debputy.dh_migration.migrators_impl import ( migrate_lintian_overrides_files, detect_unsupported_zz_debputy_features, detect_pam_files, - detect_dh_addons, + detect_dh_addons_with_zz_integration, migrate_not_installed_file, migrate_installman_file, migrate_bash_completion, @@ -21,6 +21,7 @@ from debputy.dh_migration.migrators_impl import ( migrate_dh_installsystemd_files, detect_obsolete_substvars, detect_dh_addons_zz_debputy_rrr, + detect_dh_addons_with_full_integration, ) from debputy.dh_migration.models import AcceptableMigrationIssues, FeatureMigration from debputy.highlevel_manifest import HighLevelManifest @@ -29,13 +30,42 @@ from debputy.plugin.api.spec import ( DebputyIntegrationMode, INTEGRATION_MODE_DH_DEBPUTY_RRR, INTEGRATION_MODE_DH_DEBPUTY, + INTEGRATION_MODE_FULL, ) Migrator = Callable[ - [VirtualPath, HighLevelManifest, AcceptableMigrationIssues, FeatureMigration, str], + [ + VirtualPath, + HighLevelManifest, + AcceptableMigrationIssues, + FeatureMigration, + DebputyIntegrationMode, + ], None, ] +_DH_DEBPUTY_MIGRATORS = [ + detect_unsupported_zz_debputy_features, + detect_pam_files, + migrate_dh_hook_targets, + migrate_dh_installsystemd_files, + migrate_install_file, + migrate_installdocs_file, + migrate_installexamples_file, + migrate_installman_file, + migrate_installinfo_file, + migrate_misspelled_readme_debian_files, + migrate_doc_base_files, + migrate_links_files, + migrate_maintscript, + migrate_tmpfile, + migrate_lintian_overrides_files, + migrate_bash_completion, + detect_obsolete_substvars, + # not-installed should go last, so its rules appear after other installations + # It is not perfect, but it is a start. + migrate_not_installed_file, +] MIGRATORS: Mapping[DebputyIntegrationMode, List[Migrator]] = { INTEGRATION_MODE_DH_DEBPUTY_RRR: [ @@ -45,26 +75,12 @@ MIGRATORS: Mapping[DebputyIntegrationMode, List[Migrator]] = { detect_obsolete_substvars, ], INTEGRATION_MODE_DH_DEBPUTY: [ - detect_unsupported_zz_debputy_features, - detect_pam_files, - migrate_dh_hook_targets, - migrate_dh_installsystemd_files, - migrate_install_file, - migrate_installdocs_file, - migrate_installexamples_file, - migrate_installman_file, - migrate_installinfo_file, - migrate_misspelled_readme_debian_files, - migrate_doc_base_files, - migrate_links_files, - migrate_maintscript, - migrate_tmpfile, - migrate_lintian_overrides_files, - migrate_bash_completion, - detect_dh_addons, - detect_obsolete_substvars, - # not-installed should go last, so its rules appear after other installations - # It is not perfect, but it is a start. - migrate_not_installed_file, + *_DH_DEBPUTY_MIGRATORS, + detect_dh_addons_with_zz_integration, + ], + INTEGRATION_MODE_FULL: [ + *_DH_DEBPUTY_MIGRATORS, + detect_dh_addons_with_full_integration, ], } +del _DH_DEBPUTY_MIGRATORS diff --git a/src/debputy/dh_migration/migrators_impl.py b/src/debputy/dh_migration/migrators_impl.py index 91ea8cd..97b0fd2 100644 --- a/src/debputy/dh_migration/migrators_impl.py +++ b/src/debputy/dh_migration/migrators_impl.py @@ -17,6 +17,7 @@ from typing import ( Callable, TypeVar, Dict, + Container, ) from debian.deb822 import Deb822 @@ -51,6 +52,8 @@ from debputy.plugin.api import VirtualPath from debputy.plugin.api.spec import ( INTEGRATION_MODE_DH_DEBPUTY_RRR, INTEGRATION_MODE_DH_DEBPUTY, + DebputyIntegrationMode, + INTEGRATION_MODE_FULL, ) from debputy.util import ( _error, @@ -61,8 +64,15 @@ from debputy.util import ( has_glob_magic, ) + +class ContainsEverything: + + def __contains__(self, item: str) -> bool: + return True + + # Align with debputy.py -DH_COMMANDS_REPLACED = { +DH_COMMANDS_REPLACED: Mapping[DebputyIntegrationMode, Container[str]] = { INTEGRATION_MODE_DH_DEBPUTY_RRR: frozenset( { "dh_fixperms", @@ -124,6 +134,7 @@ DH_COMMANDS_REPLACED = { "dh_builddeb", } ), + INTEGRATION_MODE_FULL: ContainsEverything(), } _GS_DOC = f"{DEBPUTY_DOC_ROOT_DIR}/GETTING-STARTED-WITH-dh-debputy.md" @@ -375,7 +386,7 @@ def migrate_bash_completion( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_bash-completion files" is_single_binary = sum(1 for _ in manifest.all_packages) == 1 @@ -466,7 +477,7 @@ def migrate_dh_installsystemd_files( manifest: HighLevelManifest, _acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installsystemd files" for dctrl_bin in manifest.all_packages: @@ -499,7 +510,7 @@ def migrate_maintscript( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installdeb files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -608,7 +619,7 @@ def migrate_install_file( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_install config files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -799,7 +810,7 @@ def migrate_installdocs_file( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installdocs config files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -846,7 +857,7 @@ def migrate_installexamples_file( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installexamples config files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -900,7 +911,7 @@ def migrate_installinfo_file( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installinfo config files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -975,7 +986,7 @@ def migrate_installman_file( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installman config files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -1095,7 +1106,7 @@ def migrate_not_installed_file( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_missing's not-installed config file" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -1135,7 +1146,7 @@ def detect_pam_files( manifest: HighLevelManifest, _acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "detect dh_installpam files (min dh compat)" for dctrl_bin in manifest.all_packages: @@ -1150,7 +1161,7 @@ def migrate_tmpfile( manifest: HighLevelManifest, _acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installtmpfiles config files" for dctrl_bin in manifest.all_packages: @@ -1174,7 +1185,7 @@ def migrate_lintian_overrides_files( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_lintian config files" for dctrl_bin in manifest.all_packages: @@ -1198,7 +1209,7 @@ def migrate_links_files( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_link files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -1272,7 +1283,7 @@ def migrate_misspelled_readme_debian_files( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "misspelled README.Debian files" for dctrl_bin in manifest.all_packages: @@ -1304,7 +1315,7 @@ def migrate_doc_base_files( manifest: HighLevelManifest, _: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "doc-base files" # ignore the dh_make ".EX" file if one should still be present. The dh_installdocs tool ignores it too. @@ -1355,7 +1366,7 @@ def migrate_dh_hook_targets( _: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - migration_target: str, + migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh hook targets" source_root = os.path.dirname(debian_dir.fs_path) @@ -1407,7 +1418,7 @@ def detect_unsupported_zz_debputy_features( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "Known unsupported features" @@ -1426,7 +1437,7 @@ def detect_obsolete_substvars( _manifest: HighLevelManifest, _acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = ( "Check for obsolete ${foo:var} variables in debian/control" @@ -1507,7 +1518,7 @@ def detect_dh_addons_zz_debputy_rrr( _manifest: HighLevelManifest, _acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "Check for dh-sequence-addons" r = read_dh_addon_sequences(debian_dir) @@ -1527,12 +1538,28 @@ def detect_dh_addons_zz_debputy_rrr( feature_migration.warn("Missing Build-Depends on dh-sequence-zz-debputy-rrr") -def detect_dh_addons( +def detect_dh_addons_with_full_integration( + _debian_dir: VirtualPath, + _manifest: HighLevelManifest, + _acceptable_migration_issues: AcceptableMigrationIssues, + feature_migration: FeatureMigration, + _migration_target: DebputyIntegrationMode, +) -> None: + feature_migration.tagline = "Check for dh-sequence-addons and Build-Depends" + feature_migration.warn( + "TODO: Not implemented: Please remove any dh-sequence Build-Dependency" + ) + feature_migration.warn( + "TODO: Not implemented: Please ensure there is a Build-Dependency on `debputy (>= 0.1.45~)" + ) + + +def detect_dh_addons_with_zz_integration( debian_dir: VirtualPath, _manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "Check for dh-sequence-addons" r = read_dh_addon_sequences(debian_dir) @@ -1544,6 +1571,8 @@ def detect_dh_addons( ) return + assert _migration_target != INTEGRATION_MODE_FULL + bd_sequences, dr_sequences, _ = r remaining_sequences = bd_sequences | dr_sequences diff --git a/src/debputy/exceptions.py b/src/debputy/exceptions.py index a445997..b3ff7d5 100644 --- a/src/debputy/exceptions.py +++ b/src/debputy/exceptions.py @@ -10,6 +10,10 @@ class DebputyRuntimeError(RuntimeError): return cast("str", self.args[0]) +class DebputyBuildStepError(DebputyRuntimeError): + pass + + class DebputySubstitutionError(DebputyRuntimeError): pass @@ -64,6 +68,10 @@ class PluginInitializationError(PluginBaseError): pass +class PluginIncorrectRegistrationError(PluginInitializationError): + pass + + class PluginMetadataError(PluginBaseError): pass diff --git a/src/debputy/highlevel_manifest.py b/src/debputy/highlevel_manifest.py index 9bdc225..6c910ab 100644 --- a/src/debputy/highlevel_manifest.py +++ b/src/debputy/highlevel_manifest.py @@ -49,7 +49,11 @@ from .maintscript_snippet import ( MaintscriptSnippetContainer, ) from .manifest_conditions import ConditionContext -from .manifest_parser.base_types import FileSystemMatchRule, FileSystemExactMatchRule +from .manifest_parser.base_types import ( + FileSystemMatchRule, + FileSystemExactMatchRule, + BuildEnvironments, +) from .manifest_parser.util import AttributePath from .packager_provided_files import PackagerProvidedFile from .packages import BinaryPackage, SourcePackage @@ -61,6 +65,8 @@ from .plugin.api.impl_types import ( ) from .plugin.api.spec import FlushableSubstvars, VirtualPath from .plugin.debputy.binary_package_rules import ServiceRule +from .plugin.debputy.to_be_api_types import BuildRule +from .plugin.plugin_state import run_in_context_of_plugin from .substitution import Substitution from .transformation_rules import ( TransformationRule, @@ -1036,7 +1042,9 @@ def _install_everything_from_source_dir_if_present( ) -> None: attribute_path = AttributePath.builtin_path()[f"installing {source_dir.fs_path}"] pkg_set = frozenset([dctrl_bin]) - install_rule = InstallRule.install_dest( + install_rule = run_in_context_of_plugin( + "debputy", + InstallRule.install_dest, [FileSystemMatchRule.from_path_match("*", attribute_path, substitution)], None, pkg_set, @@ -1086,6 +1094,8 @@ class HighLevelManifest: dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable, dpkg_arch_query_table: DpkgArchTable, build_env: DebBuildOptionsAndProfiles, + build_environments: BuildEnvironments, + build_rules: Optional[List[BuildRule]], plugin_provided_feature_set: PluginProvidedFeatureSet, debian_dir: VirtualPath, ) -> None: @@ -1100,8 +1110,17 @@ class HighLevelManifest: self._dpkg_arch_query_table = dpkg_arch_query_table self._build_env = build_env self._used_for: Set[str] = set() + self.build_environments = build_environments + self.build_rules = build_rules self._plugin_provided_feature_set = plugin_provided_feature_set self._debian_dir = debian_dir + self._source_condition_context = ConditionContext( + binary_package=None, + substitution=self.substitution, + deb_options_and_profiles=self._build_env, + dpkg_architecture_variables=self._dpkg_architecture_variables, + dpkg_arch_query_table=self._dpkg_arch_query_table, + ) def source_version(self, include_binnmu_version: bool = True) -> str: # TODO: There should an easier way to determine the source version; really. @@ -1116,6 +1135,10 @@ class HighLevelManifest: raise AssertionError(f"Could not resolve {version_var}") from e @property + def source_condition_context(self) -> ConditionContext: + return self._source_condition_context + + @property def debian_dir(self) -> VirtualPath: return self._debian_dir @@ -1124,7 +1147,7 @@ class HighLevelManifest: return self._dpkg_architecture_variables @property - def build_env(self) -> DebBuildOptionsAndProfiles: + def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles: return self._build_env @property @@ -1270,13 +1293,7 @@ class HighLevelManifest: ] path_matcher = SourcePathMatcher(discard_rules) - source_condition_context = ConditionContext( - binary_package=None, - substitution=self.substitution, - build_env=self._build_env, - dpkg_architecture_variables=self._dpkg_architecture_variables, - dpkg_arch_query_table=self._dpkg_arch_query_table, - ) + source_condition_context = self._source_condition_context for dctrl_bin in self.active_packages: package = dctrl_bin.name @@ -1416,23 +1433,14 @@ class HighLevelManifest: self, binary_package: Optional[Union[BinaryPackage, str]] ) -> ConditionContext: if binary_package is None: - return ConditionContext( - binary_package=None, - substitution=self.substitution, - build_env=self._build_env, - dpkg_architecture_variables=self._dpkg_architecture_variables, - dpkg_arch_query_table=self._dpkg_arch_query_table, - ) + return self._source_condition_context if not isinstance(binary_package, str): binary_package = binary_package.name package_transformation = self.package_transformations[binary_package] - return ConditionContext( + return self._source_condition_context.replace( binary_package=package_transformation.binary_package, substitution=package_transformation.substitution, - build_env=self._build_env, - dpkg_architecture_variables=self._dpkg_architecture_variables, - dpkg_arch_query_table=self._dpkg_arch_query_table, ) def apply_fs_transformations( @@ -1452,7 +1460,7 @@ class HighLevelManifest: condition_context = ConditionContext( binary_package=package_transformation.binary_package, substitution=package_transformation.substitution, - build_env=self._build_env, + deb_options_and_profiles=self._build_env, dpkg_architecture_variables=self._dpkg_architecture_variables, dpkg_arch_query_table=self._dpkg_arch_query_table, ) @@ -1466,7 +1474,7 @@ class HighLevelManifest: norm_mode_transformation_rule = ModeNormalizationTransformationRule(norm_rules) norm_mode_transformation_rule.transform_file_system(fs_root, condition_context) for transformation in package_transformation.transformations: - transformation.transform_file_system(fs_root, condition_context) + transformation.run_transform_file_system(fs_root, condition_context) interpreter_normalization = NormalizeShebangLineTransformation() interpreter_normalization.transform_file_system(fs_root, condition_context) diff --git a/src/debputy/highlevel_manifest_parser.py b/src/debputy/highlevel_manifest_parser.py index b7a4600..18d9fa7 100644 --- a/src/debputy/highlevel_manifest_parser.py +++ b/src/debputy/highlevel_manifest_parser.py @@ -43,20 +43,22 @@ from ._deb_options_profiles import DebBuildOptionsAndProfiles from .architecture_support import DpkgArchitectureBuildProcessValuesTable from .filesystem_scan import FSROOverlay from .installations import InstallRule, PPFInstallRule +from .manifest_parser.base_types import BuildEnvironments, BuildEnvironmentDefinition from .manifest_parser.exceptions import ManifestParseException from .manifest_parser.parser_data import ParserContextData from .manifest_parser.util import AttributePath from .packager_provided_files import detect_all_packager_provided_files from .plugin.api import VirtualPath +from .plugin.api.feature_set import PluginProvidedFeatureSet from .plugin.api.impl_types import ( TP, TTP, DispatchingTableParser, - OPARSER_MANIFEST_ROOT, PackageContextData, ) -from .plugin.api.feature_set import PluginProvidedFeatureSet +from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT from .plugin.api.spec import DebputyIntegrationMode +from .plugin.debputy.to_be_api_types import BuildRule from .yaml import YAMLError, MANIFEST_YAML try: @@ -131,11 +133,19 @@ class HighLevelManifestParser(ParserContextData): self._substitution = substitution self._dpkg_architecture_variables = dpkg_architecture_variables self._dpkg_arch_query_table = dpkg_arch_query_table - self._build_env = build_env + self._deb_options_and_profiles = build_env self._package_state_stack: List[PackageTransformationDefinition] = [] self._plugin_provided_feature_set = plugin_provided_feature_set self._debputy_integration_mode = debputy_integration_mode self._declared_variables = {} + self._used_named_envs = set() + self._build_environments: Optional[BuildEnvironments] = BuildEnvironments( + {}, + None, + ) + self._has_set_default_build_environment = False + self._read_build_environment = False + self._build_rules: Optional[List[BuildRule]] = None if isinstance(debian_dir, str): debian_dir = FSROOverlay.create_root_dir("debian", debian_dir) @@ -202,10 +212,21 @@ class HighLevelManifestParser(ParserContextData): return self._dpkg_arch_query_table @property - def build_env(self) -> DebBuildOptionsAndProfiles: - return self._build_env + def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles: + return self._deb_options_and_profiles + + def _self_check(self) -> None: + unused_envs = ( + self._build_environments.environments.keys() - self._used_named_envs + ) + if unused_envs: + unused_env_names = ", ".join(unused_envs) + raise ManifestParseException( + f"The following named environments were never referenced: {unused_env_names}" + ) def build_manifest(self) -> HighLevelManifest: + self._self_check() if self._used: raise TypeError("build_manifest can only be called once!") self._used = True @@ -240,6 +261,8 @@ class HighLevelManifestParser(ParserContextData): ppf_result.reserved_only ) self._transform_dpkg_maintscript_helpers_to_snippets() + build_environments = self.build_environments() + assert build_environments is not None return HighLevelManifest( self.manifest_path, @@ -251,7 +274,9 @@ class HighLevelManifestParser(ParserContextData): self._package_states, self._dpkg_architecture_variables, self._dpkg_arch_query_table, - self._build_env, + self._deb_options_and_profiles, + build_environments, + self._build_rules, self._plugin_provided_feature_set, self._debian_dir, ) @@ -325,6 +350,69 @@ class HighLevelManifestParser(ParserContextData): def debputy_integration_mode(self, new_value: DebputyIntegrationMode) -> None: self._debputy_integration_mode = new_value + def _register_build_environment( + self, + name: Optional[str], + build_environment: BuildEnvironmentDefinition, + attribute_path: AttributePath, + is_default: bool = False, + ) -> None: + assert not self._read_build_environment + + # TODO: Reference the paths of the original environments for the error messages where that is relevant. + if is_default: + if self._has_set_default_build_environment: + raise ManifestParseException( + f"There cannot be multiple default environments and" + f" therefore {attribute_path.path} cannot be a default environment" + ) + self._has_set_default_build_environment = True + self._build_environments.default_environment = build_environment + if name is None: + return + elif name is None: + raise ManifestParseException( + f"Useless environment defined at {attribute_path.path}. It is neither the" + " default environment nor does it have a name (so no rules can reference it" + " explicitly)" + ) + + if name in self._build_environments.environments: + raise ManifestParseException( + f'The environment defined at {attribute_path.path} reuse the name "{name}".' + " The environment name must be unique." + ) + self._build_environments.environments[name] = build_environment + + def resolve_build_environment( + self, + name: Optional[str], + attribute_path: AttributePath, + ) -> BuildEnvironmentDefinition: + if name is None: + return self.build_environments().default_environment + try: + env = self.build_environments().environments[name] + except KeyError: + raise ManifestParseException( + f'The environment "{name}" requested at {attribute_path.path} was not' + f" defined in the `build-environments`" + ) + else: + self._used_named_envs.add(name) + return env + + def build_environments(self) -> BuildEnvironments: + v = self._build_environments + if ( + not self._read_build_environment + and not self._build_environments.environments + and self._build_environments.default_environment is None + ): + self._build_environments.default_environment = BuildEnvironmentDefinition() + self._read_build_environment = True + return v + def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None: package_state = self.current_binary_package_state for dmh in package_state.dpkg_maintscript_helper_snippets: @@ -504,6 +592,7 @@ class YAMLManifestParser(HighLevelManifestParser): ) if service_rules: package_state.requested_service_rules.extend(service_rules) + self._build_rules = parsed_data.get("builds") return self.build_manifest() diff --git a/src/debputy/installations.py b/src/debputy/installations.py index b781757..806a964 100644 --- a/src/debputy/installations.py +++ b/src/debputy/installations.py @@ -31,10 +31,11 @@ from debputy.manifest_conditions import ( from debputy.manifest_parser.base_types import ( FileSystemMatchRule, FileSystemExactMatchRule, - DebputyDispatchableType, ) +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.packages import BinaryPackage from debputy.path_matcher import MatchRule, ExactFileSystemPath, MATCH_ANYTHING +from debputy.plugin.plugin_state import run_in_context_of_plugin from debputy.substitution import Substitution from debputy.util import _error, _warn @@ -585,6 +586,7 @@ class InstallRule(DebputyDispatchableType): *, match_filter: Optional[Callable[["VirtualPath"], bool]] = None, ) -> None: + super().__init__() self._condition = condition self._definition_source = definition_source self._match_filter = match_filter @@ -1025,7 +1027,9 @@ class PPFInstallRule(InstallRule): substitution: Substitution, ppfs: Sequence["PackagerProvidedFile"], ) -> None: - super().__init__( + run_in_context_of_plugin( + "debputy", + super().__init__, None, "<built-in; PPF install rule>", ) diff --git a/src/debputy/integration_detection.py b/src/debputy/integration_detection.py index f412268..cc19057 100644 --- a/src/debputy/integration_detection.py +++ b/src/debputy/integration_detection.py @@ -1,16 +1,21 @@ -from typing import Container, Optional +from typing import Container, Optional, Mapping from debputy.plugin.api.spec import ( DebputyIntegrationMode, INTEGRATION_MODE_DH_DEBPUTY_RRR, INTEGRATION_MODE_DH_DEBPUTY, + INTEGRATION_MODE_FULL, ) def determine_debputy_integration_mode( + source_fields: Mapping[str, str], all_sequences: Container[str], ) -> Optional[DebputyIntegrationMode]: + if source_fields.get("Build-Driver", "").lower() == "debputy": + return INTEGRATION_MODE_FULL + has_zz_debputy = "zz-debputy" in all_sequences or "debputy" in all_sequences has_zz_debputy_rrr = "zz-debputy-rrr" in all_sequences has_any_existing = has_zz_debputy or has_zz_debputy_rrr @@ -18,4 +23,7 @@ def determine_debputy_integration_mode( return INTEGRATION_MODE_DH_DEBPUTY_RRR if has_any_existing: return INTEGRATION_MODE_DH_DEBPUTY + if source_fields.get("Source", "") == "debputy": + # Self-hosting. We cannot set the Build-Driver field since that creates a self-circular dependency loop + return INTEGRATION_MODE_FULL return None diff --git a/src/debputy/linting/lint_impl.py b/src/debputy/linting/lint_impl.py index ddc7e93..0f37dce 100644 --- a/src/debputy/linting/lint_impl.py +++ b/src/debputy/linting/lint_impl.py @@ -35,7 +35,7 @@ from debputy.lsp.lsp_debian_tests_control import ( ) from debputy.lsp.maint_prefs import ( MaintainerPreferenceTable, - EffectivePreference, + EffectiveFormattingPreference, determine_effective_preference, ) from debputy.lsp.quickfixes import provide_standard_quickfixes_from_diagnostics @@ -92,7 +92,7 @@ class LintContext: parsed_deb822_file_content: Optional[Deb822FileElement] = None source_package: Optional[SourcePackage] = None binary_packages: Optional[Mapping[str, BinaryPackage]] = None - effective_preference: Optional[EffectivePreference] = None + effective_preference: Optional[EffectiveFormattingPreference] = None style_tool: Optional[str] = None unsupported_preference_reason: Optional[str] = None salsa_ci: Optional[CommentedMap] = None @@ -233,8 +233,6 @@ def perform_reformat( named_style: Optional[str] = None, ) -> None: parsed_args = context.parsed_args - if not parsed_args.spellcheck: - disable_spellchecking() fo = _output_styling(context.parsed_args, sys.stdout) lint_context = gather_lint_info(context) if named_style is not None: diff --git a/src/debputy/linting/lint_util.py b/src/debputy/linting/lint_util.py index 6346508..017a1dc 100644 --- a/src/debputy/linting/lint_util.py +++ b/src/debputy/linting/lint_util.py @@ -43,7 +43,7 @@ if TYPE_CHECKING: from debputy.lsp.text_util import LintCapablePositionCodec from debputy.lsp.maint_prefs import ( MaintainerPreferenceTable, - EffectivePreference, + EffectiveFormattingPreference, ) @@ -56,9 +56,14 @@ class DebputyMetadata: debputy_integration_mode: Optional[DebputyIntegrationMode] @classmethod - def from_data(cls, dh_sequencer_data: DhSequencerData) -> typing.Self: + def from_data( + cls, + source_fields: Mapping[str, str], + dh_sequencer_data: DhSequencerData, + ) -> typing.Self: integration_mode = determine_debputy_integration_mode( - dh_sequencer_data.sequences + source_fields, + dh_sequencer_data.sequences, ) return cls(integration_mode) @@ -114,12 +119,17 @@ class LintState: raise NotImplementedError @property - def effective_preference(self) -> Optional["EffectivePreference"]: + def effective_preference(self) -> Optional["EffectiveFormattingPreference"]: raise NotImplementedError @property def debputy_metadata(self) -> DebputyMetadata: - return DebputyMetadata.from_data(self.dh_sequencer_data) + src_pkg = self.source_package + src_fields = src_pkg.fields if src_pkg else {} + return DebputyMetadata.from_data( + src_fields, + self.dh_sequencer_data, + ) @property def dh_sequencer_data(self) -> DhSequencerData: @@ -137,7 +147,7 @@ class LintStateImpl(LintState): lines: List[str] source_package: Optional[SourcePackage] = None binary_packages: Optional[Mapping[str, BinaryPackage]] = None - effective_preference: Optional["EffectivePreference"] = None + effective_preference: Optional["EffectiveFormattingPreference"] = None _parsed_cache: Optional[Deb822FileElement] = None _dh_sequencer_cache: Optional[DhSequencerData] = None diff --git a/src/debputy/lsp/lsp_debian_control.py b/src/debputy/lsp/lsp_debian_control.py index 2b8f9b0..ac08266 100644 --- a/src/debputy/lsp/lsp_debian_control.py +++ b/src/debputy/lsp/lsp_debian_control.py @@ -677,7 +677,7 @@ def _doc_inlay_hint( stanza_range = stanza.range_in_parent() if stanza_no < 1: continue - pkg_kvpair = stanza.get_kvpair_element("Package", use_get=True) + pkg_kvpair = stanza.get_kvpair_element(("Package", 0), use_get=True) if pkg_kvpair is None: continue @@ -778,7 +778,7 @@ def _binary_package_checks( ) -> None: package_name = stanza.get("Package", "") source_section = source_stanza.get("Section") - section_kvpair = stanza.get_kvpair_element("Section", use_get=True) + section_kvpair = stanza.get_kvpair_element(("Section", 0), use_get=True) section: Optional[str] = None if section_kvpair is not None: section, section_range = extract_first_value_and_position( @@ -1227,7 +1227,7 @@ def _package_range_of_stanza( binary_stanzas: List[Tuple[Deb822ParagraphElement, TEPosition]], ) -> Iterable[Tuple[str, Optional[str], Range]]: for stanza, stanza_position in binary_stanzas: - kvpair = stanza.get_kvpair_element("Package", use_get=True) + kvpair = stanza.get_kvpair_element(("Package", 0), use_get=True) if kvpair is None: continue representation_field_range = kvpair.range_in_parent().relative_to( diff --git a/src/debputy/lsp/lsp_debian_control_reference_data.py b/src/debputy/lsp/lsp_debian_control_reference_data.py index 2ec885b..1d24045 100644 --- a/src/debputy/lsp/lsp_debian_control_reference_data.py +++ b/src/debputy/lsp/lsp_debian_control_reference_data.py @@ -104,7 +104,7 @@ except ImportError: if TYPE_CHECKING: - from debputy.lsp.maint_prefs import EffectivePreference + from debputy.lsp.maint_prefs import EffectiveFormattingPreference F = TypeVar("F", bound="Deb822KnownField") @@ -519,7 +519,7 @@ def _dctrl_ma_field_validation( stanza_position: "TEPosition", lint_state: LintState, ) -> Iterable[Diagnostic]: - ma_kvpair = stanza.get_kvpair_element("Multi-Arch", use_get=True) + ma_kvpair = stanza.get_kvpair_element(("Multi-Arch", 0), use_get=True) arch = stanza.get("Architecture", "any") if arch == "all" and ma_kvpair is not None: ma_value, ma_value_range = extract_first_value_and_position( @@ -1035,6 +1035,36 @@ def _dctrl_validate_dep( ) +def _rrr_build_driver_mismatch( + _known_field: "F", + _deb822_file: Deb822FileElement, + _kvpair: Deb822KeyValuePairElement, + kvpair_range_te: "TERange", + _field_name_range: "TERange", + stanza: Deb822ParagraphElement, + _stanza_position: "TEPosition", + lint_state: LintState, +) -> Iterable[Diagnostic]: + dr = stanza.get("Build-Driver", "debian-rules") + if dr != "debian-rules": + yield Diagnostic( + lint_state.position_codec.range_to_client_units( + lint_state.lines, + te_range_to_lsp(kvpair_range_te), + ), + f"The Rules-Requires-Root field is irrelevant for the `Build-Driver` `{dr}`.", + DiagnosticSeverity.Warning, + source="debputy", + data=DiagnosticData( + quickfixes=[ + propose_remove_range_quick_fix( + proposed_title="Remove Rules-Requires-Root" + ) + ] + ), + ) + + class Dep5Matcher(BasenameGlobMatch): def __init__(self, basename_glob: str) -> None: super().__init__( @@ -1979,7 +2009,7 @@ class Deb822KnownField: def reformat_field( self, - effective_preference: "EffectivePreference", + effective_preference: "EffectiveFormattingPreference", stanza_range: TERange, kvpair: Deb822KeyValuePairElement, formatter: FormatterCallback, @@ -2002,7 +2032,7 @@ class DctrlLikeKnownField(Deb822KnownField): def reformat_field( self, - effective_preference: "EffectivePreference", + effective_preference: "EffectiveFormattingPreference", stanza_range: TERange, kvpair: Deb822KeyValuePairElement, formatter: FormatterCallback, @@ -2011,7 +2041,7 @@ class DctrlLikeKnownField(Deb822KnownField): ) -> Iterable[TextEdit]: interpretation = self.field_value_class.interpreter() if ( - not effective_preference.formatting_deb822_normalize_field_content + not effective_preference.deb822_normalize_field_content or interpretation is None ): yield from super(DctrlLikeKnownField, self).reformat_field( @@ -2133,7 +2163,7 @@ class DctrlKnownField(DctrlLikeKnownField): def reformat_field( self, - effective_preference: "EffectivePreference", + effective_preference: "EffectiveFormattingPreference", stanza_range: TERange, kvpair: Deb822KeyValuePairElement, formatter: FormatterCallback, @@ -2142,7 +2172,7 @@ class DctrlKnownField(DctrlLikeKnownField): ) -> Iterable[TextEdit]: if ( self.name == "Architecture" - and effective_preference.formatting_deb822_normalize_field_content + and effective_preference.deb822_normalize_field_content ): interpretation = self.field_value_class.interpreter() assert interpretation is not None @@ -2317,6 +2347,47 @@ SOURCE_FIELDS = _fields( ), ), DctrlKnownField( + "Build-Driver", + FieldValueClass.SINGLE_VALUE, + default_value="debian-rules", + known_values=allowed_values( + Keyword( + "debian-rules", + synopsis_doc="Build via `debian/rules`", + hover_text=textwrap.dedent( + """\ + Use the `debian/rules` interface for building packages. + + This is the historical default and the interface that Debian Packages have used for + decades to build debs. + """ + ), + ), + Keyword( + "debputy", + synopsis_doc="Build with `debputy`", + hover_text=textwrap.dedent( + """\ + Use the `debputy` interface for building the package. + + This is provides the "full" integration mode with `debputy` where all parts of the + package build is handled by `debputy`. + + This *may* make any `debhelper` build-dependency redundant depending on which build + system is used. Some build systems (such as `autoconf` still use `debhelper` based tools). + """ + ), + ), + ), + synopsis_doc="Which implementation dpkg should use for the build", + hover_text=textwrap.dedent( + """\ + The name of the build driver that dpkg (`dpkg-buildpackage`) will use for assembling the + package. + """ + ), + ), + DctrlKnownField( "Vcs-Browser", FieldValueClass.SINGLE_VALUE, synopsis_doc="URL for browsers to interact with packaging VCS", @@ -2580,6 +2651,7 @@ SOURCE_FIELDS = _fields( "Rules-Requires-Root", FieldValueClass.SPACE_SEPARATED_LIST, unknown_value_diagnostic_severity=None, + custom_field_check=_rrr_build_driver_mismatch, known_values=allowed_values( Keyword( "no", @@ -2642,6 +2714,9 @@ SOURCE_FIELDS = _fields( ` Build-Depends` on `dpkg-build-api (>= 1)` or later, the default is `no`. Otherwise, the default is `binary-target` + This field is only relevant when when the `Build-Driver` is `debian-rules` (which it is by + default). + Note it is **not** possible to require running the package as "true root". """ ), @@ -3905,6 +3980,57 @@ _DEP5_HEADER_FIELDS = _fields( """ ), ), + Deb822KnownField( + "Files-Excluded", + FieldValueClass.FREE_TEXT_FIELD, + hover_text=textwrap.dedent( + """\ + Remove the listed files from the tarball when repacking (commonly via uscan). This can be useful when the + listed files are non-free but not necessary for the Debian package. In this case, the upstream version of + the package should generally end with `~dfsg` or `+dfsg` (to mark the content changed due to the + Debian Free Software Guidelines). The exclusion can also be useful to remove large files or directories + that are not used by Debian or pre-built binaries. In this case, `~ds` or `+ds` should be added to the + version instead of `~dfsg` or `+dfsg` for "Debian Source" to mark it as altered by Debian. If both reasons + are used, the `~dfsg` or `+dfsg` version is used as that is the more important reason for the repacking. + + Example: + ``` + Files-Excluded: exclude-this + exclude-dir + */exclude-dir + .* + */js/jquery.js + ``` + + The `Files-Included` field can be used to "re-include" files matched by `Files-Excluded`. + + It is also possible to exclude files in specific "upstream components" for source packages with multiple + upstream tarballs. This is done by adding a field called `Files-Excluded-<component>`. The `<component>` + part should then match the component name exactly (case sensitive). + + Defined by: mk-origtargz (usually used via uscan) + """ + ), + ), + Deb822KnownField( + "Files-Included", + FieldValueClass.FREE_TEXT_FIELD, + hover_text=textwrap.dedent( + """\ + Re-include files that were marked for exclusion by `Files-Excluded`. This can be useful for "exclude + everything except X" style semantics where `Files-Excluded` has a very broad pattern and + `Files-Included` then marks a few exceptions. + + It is also possible to re-include files in specific "upstream components" for source packages with multiple + upstream tarballs. This is done by adding a field called `Files-Include-<component>` which is then used + in tandem with `Files-Exclude-<component>`. The `<component>` part should then match the component name + exactly (case sensitive). + + + Defined by: mk-origtargz (usually used via uscan) + """ + ), + ), ) _DEP5_FILES_FIELDS = _fields( Deb822KnownField( @@ -4664,7 +4790,7 @@ class StanzaMetadata(Mapping[str, F], Generic[F], ABC): def reformat_stanza( self, - effective_preference: "EffectivePreference", + effective_preference: "EffectiveFormattingPreference", stanza: Deb822ParagraphElement, stanza_range: TERange, formatter: FormatterCallback, @@ -4672,7 +4798,7 @@ class StanzaMetadata(Mapping[str, F], Generic[F], ABC): lines: List[str], ) -> Iterable[TextEdit]: for known_field in self.stanza_fields.values(): - kvpair = stanza.get_kvpair_element(known_field.name, use_get=True) + kvpair = stanza.get_kvpair_element((known_field.name, 0), use_get=True) if kvpair is None: continue yield from known_field.reformat_field( @@ -4744,7 +4870,7 @@ class Deb822FileMetadata(Generic[S]): def reformat( self, - effective_preference: "EffectivePreference", + effective_preference: "EffectiveFormattingPreference", deb822_file: Deb822FileElement, formatter: FormatterCallback, _content: str, @@ -4856,7 +4982,7 @@ class DctrlFileMetadata(Deb822FileMetadata[DctrlStanzaMetadata]): def reformat( self, - effective_preference: "EffectivePreference", + effective_preference: "EffectiveFormattingPreference", deb822_file: Deb822FileElement, formatter: FormatterCallback, content: str, @@ -4875,7 +5001,7 @@ class DctrlFileMetadata(Deb822FileMetadata[DctrlStanzaMetadata]): ) if ( - not effective_preference.formatting_deb822_normalize_stanza_order + not effective_preference.deb822_normalize_stanza_order or deb822_file.find_first_error_element() is not None ): return edits diff --git a/src/debputy/lsp/lsp_debian_debputy_manifest.py b/src/debputy/lsp/lsp_debian_debputy_manifest.py index a8a2fdf..8c7aeac 100644 --- a/src/debputy/lsp/lsp_debian_debputy_manifest.py +++ b/src/debputy/lsp/lsp_debian_debputy_manifest.py @@ -47,7 +47,7 @@ from debputy.lsprotocol.types import ( DiagnosticRelatedInformation, Location, ) -from debputy.manifest_parser.base_types import DebputyDispatchableType +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.manifest_parser.declarative_parser import ( AttributeDescription, ParserGenerator, @@ -57,7 +57,6 @@ from debputy.manifest_parser.declarative_parser import DeclarativeMappingInputPa from debputy.manifest_parser.util import AttributePath from debputy.plugin.api.impl import plugin_metadata_for_debputys_own_plugin from debputy.plugin.api.impl_types import ( - OPARSER_MANIFEST_ROOT, DeclarativeInputParser, DispatchingParserBase, DebputyPluginMetadata, @@ -65,6 +64,7 @@ from debputy.plugin.api.impl_types import ( InPackageContextParser, DeclarativeValuelessKeywordInputParser, ) +from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT from debputy.plugin.api.spec import DebputyIntegrationMode from debputy.plugin.debputy.private_api import Capability, load_libcap from debputy.util import _info, detect_possible_typo @@ -742,6 +742,22 @@ def _insert_snippet(lines: List[str], server_position: Position) -> bool: return True +def _maybe_quote(v: str) -> str: + if v and v[0].isdigit(): + try: + float(v) + return f"'{v}'" + except ValueError: + pass + return v + + +def _complete_value(v: Any) -> str: + if isinstance(v, str): + return _maybe_quote(v) + return str(v) + + @lsp_completer(_LANGUAGE_IDS) def debputy_manifest_completer( ls: "DebputyLanguageServer", @@ -820,7 +836,9 @@ def debputy_manifest_completer( if isinstance(parser, DispatchingParserBase): if matched_key: items = [ - CompletionItem(k if has_colon else f"{k}:") + CompletionItem( + _maybe_quote(k) if has_colon else f"{_maybe_quote(k)}:" + ) for k in parser.registered_keywords() if k not in parent and not isinstance( @@ -830,7 +848,7 @@ def debputy_manifest_completer( ] else: items = [ - CompletionItem(k) + CompletionItem(_maybe_quote(k)) for k in parser.registered_keywords() if k not in parent and isinstance( @@ -842,7 +860,9 @@ def debputy_manifest_completer( binary_packages = ls.lint_state(doc).binary_packages if binary_packages is not None: items = [ - CompletionItem(p if has_colon else f"{p}:") + CompletionItem( + _maybe_quote(p) if has_colon else f"{_maybe_quote(p)}:" + ) for p in binary_packages if p not in parent ] @@ -858,7 +878,9 @@ def debputy_manifest_completer( locked.add(attr_name) break items = [ - CompletionItem(k if has_colon else f"{k}:") + CompletionItem( + _maybe_quote(k) if has_colon else f"{_maybe_quote(k)}:" + ) for k in parser.manifest_attributes if k not in locked ] @@ -913,7 +935,7 @@ def _completion_from_attr( _info(f"Already filled: {matched} is one of {valid_values}") return None if valid_values: - return [CompletionItem(x) for x in valid_values] + return [CompletionItem(_complete_value(x)) for x in valid_values] return None diff --git a/src/debputy/lsp/lsp_generic_yaml.py b/src/debputy/lsp/lsp_generic_yaml.py index 94267f7..5e67428 100644 --- a/src/debputy/lsp/lsp_generic_yaml.py +++ b/src/debputy/lsp/lsp_generic_yaml.py @@ -1,6 +1,6 @@ from typing import Union, Any, Optional, List, Tuple -from debputy.manifest_parser.base_types import DebputyDispatchableType +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.manifest_parser.declarative_parser import DeclarativeMappingInputParser from debputy.manifest_parser.parser_doc import ( render_rule, diff --git a/src/debputy/lsp/maint_prefs.py b/src/debputy/lsp/maint_prefs.py index fa6315b..4cc70d5 100644 --- a/src/debputy/lsp/maint_prefs.py +++ b/src/debputy/lsp/maint_prefs.py @@ -32,37 +32,37 @@ PT = TypeVar("PT", bool, str, int) BUILTIN_STYLES = os.path.join(os.path.dirname(__file__), "maint-preferences.yaml") -_NORMALISE_FIELD_CONTENT_KEY = ["formatting", "deb822", "normalize-field-content"] +_NORMALISE_FIELD_CONTENT_KEY = ["deb822", "normalize-field-content"] _UPLOADER_SPLIT_RE = re.compile(r"(?<=>)\s*,") _WAS_OPTIONS = { - "-a": ("formatting_deb822_always_wrap", True), - "--always-wrap": ("formatting_deb822_always_wrap", True), - "-s": ("formatting_deb822_short_indent", True), - "--short-indent": ("formatting_deb822_short_indent", True), - "-t": ("formatting_deb822_trailing_separator", True), - "--trailing-separator": ("formatting_deb822_trailing_separator", True), + "-a": ("deb822_always_wrap", True), + "--always-wrap": ("deb822_always_wrap", True), + "-s": ("deb822_short_indent", True), + "--short-indent": ("deb822_short_indent", True), + "-t": ("deb822_trailing_separator", True), + "--trailing-separator": ("deb822_trailing_separator", True), # Noise option for us; we do not accept `--no-keep-first` though "-k": (None, True), "--keep-first": (None, True), "--no-keep-first": ("DISABLE_NORMALIZE_STANZA_ORDER", True), - "-b": ("formatting_deb822_normalize_stanza_order", True), - "--sort-binary-packages": ("formatting_deb822_normalize_stanza_order", True), + "-b": ("deb822_normalize_stanza_order", True), + "--sort-binary-packages": ("deb822_normalize_stanza_order", True), } _WAS_DEFAULTS = { - "formatting_deb822_always_wrap": False, - "formatting_deb822_short_indent": False, - "formatting_deb822_trailing_separator": False, - "formatting_deb822_normalize_stanza_order": False, - "formatting_deb822_normalize_field_content": True, + "deb822_always_wrap": False, + "deb822_short_indent": False, + "deb822_trailing_separator": False, + "deb822_normalize_stanza_order": False, + "deb822_normalize_field_content": True, } @dataclasses.dataclass(slots=True, frozen=True, kw_only=True) class PreferenceOption(Generic[PT]): key: Union[str, List[str]] - expected_type: Type[PT] + expected_type: Union[Type[PT], Callable[[Any], Optional[str]]] description: str default_value: Optional[Union[PT, Callable[[CommentedMap], Optional[PT]]]] = None @@ -88,11 +88,17 @@ class PreferenceOption(Generic[PT]): if callable(default_value): return default_value(data) return default_value - if isinstance(v, self.expected_type): + val_issue: Optional[str] = None + expected_type = self.expected_type + if not isinstance(expected_type, type) and callable(self.expected_type): + val_issue = self.expected_type(v) + elif not isinstance(v, self.expected_type): + val_issue = f"It should have been a {self.expected_type} but it was not" + + if val_issue is None: return v raise ValueError( - f'The value "{self.name}" for key {key} in file "{filename}" should have been a' - f" {self.expected_type} but it was not" + f'The value "{self.name}" for key {key} in file "{filename}" was incorrect: {val_issue}' ) @@ -108,7 +114,7 @@ def _false_when_formatting_content(m: CommentedMap) -> Optional[bool]: return m.mlget(_NORMALISE_FIELD_CONTENT_KEY, list_ok=True, default=False) is True -OPTIONS: List[PreferenceOption] = [ +MAINT_OPTIONS: List[PreferenceOption] = [ PreferenceOption( key="canonical-name", expected_type=str, @@ -139,7 +145,25 @@ OPTIONS: List[PreferenceOption] = [ ), ), PreferenceOption( - key=["formatting", "deb822", "short-indent"], + key="formatting", + expected_type=lambda x: ( + None + if isinstance(x, EffectiveFormattingPreference) + else "It should have been a EffectiveFormattingPreference but it was not" + ), + default_value=None, + description=textwrap.dedent( + """\ + The formatting preference of the maintainer. Can either be a string for a named style or an inline + style. + """ + ), + ), +] + +FORMATTING_OPTIONS = [ + PreferenceOption( + key=["deb822", "short-indent"], expected_type=bool, description=textwrap.dedent( """\ @@ -175,7 +199,7 @@ OPTIONS: List[PreferenceOption] = [ ), ), PreferenceOption( - key=["formatting", "deb822", "always-wrap"], + key=["deb822", "always-wrap"], expected_type=bool, description=textwrap.dedent( """\ @@ -210,7 +234,7 @@ OPTIONS: List[PreferenceOption] = [ ), ), PreferenceOption( - key=["formatting", "deb822", "trailing-separator"], + key=["deb822", "trailing-separator"], expected_type=bool, default_value=False, description=textwrap.dedent( @@ -241,7 +265,7 @@ OPTIONS: List[PreferenceOption] = [ ), ), PreferenceOption( - key=["formatting", "deb822", "max-line-length"], + key=["deb822", "max-line-length"], expected_type=int, default_value=79, description=textwrap.dedent( @@ -297,7 +321,7 @@ OPTIONS: List[PreferenceOption] = [ ), ), PreferenceOption( - key=["formatting", "deb822", "normalize-field-order"], + key=["deb822", "normalize-field-order"], expected_type=bool, default_value=False, description=textwrap.dedent( @@ -332,7 +356,7 @@ OPTIONS: List[PreferenceOption] = [ ), ), PreferenceOption( - key=["formatting", "deb822", "normalize-stanza-order"], + key=["deb822", "normalize-stanza-order"], expected_type=bool, default_value=False, description=textwrap.dedent( @@ -385,43 +409,43 @@ OPTIONS: List[PreferenceOption] = [ @dataclasses.dataclass(slots=True, frozen=True) -class EffectivePreference: - formatting_deb822_short_indent: Optional[bool] = None - formatting_deb822_always_wrap: Optional[bool] = None - formatting_deb822_trailing_separator: bool = False - formatting_deb822_normalize_field_content: bool = False - formatting_deb822_normalize_field_order: bool = False - formatting_deb822_normalize_stanza_order: bool = False - formatting_deb822_max_line_length: int = 79 +class EffectiveFormattingPreference: + deb822_short_indent: Optional[bool] = None + deb822_always_wrap: Optional[bool] = None + deb822_trailing_separator: bool = False + deb822_normalize_field_content: bool = False + deb822_normalize_field_order: bool = False + deb822_normalize_stanza_order: bool = False + deb822_max_line_length: int = 79 @classmethod def from_file( cls, filename: str, key: str, - stylees: CommentedMap, + styles: CommentedMap, ) -> Self: attr = {} - for option in OPTIONS: + for option in FORMATTING_OPTIONS: if not hasattr(cls, option.attribute_name): continue - value = option.extract_value(filename, key, stylees) + value = option.extract_value(filename, key, styles) attr[option.attribute_name] = value return cls(**attr) # type: ignore @classmethod def aligned_preference( cls, - a: Optional["EffectivePreference"], - b: Optional["EffectivePreference"], - ) -> Optional["EffectivePreference"]: + a: Optional["EffectiveFormattingPreference"], + b: Optional["EffectiveFormattingPreference"], + ) -> Optional["EffectiveFormattingPreference"]: if a is None or b is None: return None - for option in OPTIONS: + for option in MAINT_OPTIONS: attr_name = option.attribute_name - if not hasattr(EffectivePreference, attr_name): + if not hasattr(EffectiveFormattingPreference, attr_name): continue a_value = getattr(a, attr_name) b_value = getattr(b, attr_name) @@ -430,14 +454,12 @@ class EffectivePreference: return a def deb822_formatter(self) -> FormatterCallback: - line_length = self.formatting_deb822_max_line_length + line_length = self.deb822_max_line_length return wrap_and_sort_formatter( - 1 if self.formatting_deb822_short_indent else "FIELD_NAME_LENGTH", - trailing_separator=self.formatting_deb822_trailing_separator, - immediate_empty_line=self.formatting_deb822_short_indent or False, - max_line_length_one_liner=( - 0 if self.formatting_deb822_always_wrap else line_length - ), + 1 if self.deb822_short_indent else "FIELD_NAME_LENGTH", + trailing_separator=self.deb822_trailing_separator, + immediate_empty_line=self.deb822_short_indent or False, + max_line_length_one_liner=(0 if self.deb822_always_wrap else line_length), ) def replace(self, /, **changes: Any) -> Self: @@ -445,24 +467,33 @@ class EffectivePreference: @dataclasses.dataclass(slots=True, frozen=True) -class MaintainerPreference(EffectivePreference): +class MaintainerPreference: canonical_name: Optional[str] = None is_packaging_team: bool = False + formatting: Optional[EffectiveFormattingPreference] = None - def as_effective_pref(self) -> EffectivePreference: - fields = { - k: v - for k, v in dataclasses.asdict(self).items() - if hasattr(EffectivePreference, k) - } - return EffectivePreference(**fields) + @classmethod + def from_file( + cls, + filename: str, + key: str, + styles: CommentedMap, + ) -> Self: + attr = {} + + for option in MAINT_OPTIONS: + if not hasattr(cls, option.attribute_name): + continue + value = option.extract_value(filename, key, styles) + attr[option.attribute_name] = value + return cls(**attr) # type: ignore class MaintainerPreferenceTable: def __init__( self, - named_styles: Mapping[str, EffectivePreference], + named_styles: Mapping[str, EffectiveFormattingPreference], maintainer_preferences: Mapping[str, MaintainerPreference], ) -> None: self._named_styles = named_styles @@ -470,7 +501,7 @@ class MaintainerPreferenceTable: @classmethod def load_preferences(cls) -> Self: - named_styles: Dict[str, EffectivePreference] = {} + named_styles: Dict[str, EffectiveFormattingPreference] = {} maintainer_preferences: Dict[str, MaintainerPreference] = {} with open(BUILTIN_STYLES) as fd: parse_file(named_styles, maintainer_preferences, BUILTIN_STYLES, fd) @@ -488,7 +519,7 @@ class MaintainerPreferenceTable: return cls(named_styles, maintainer_preferences) @property - def named_styles(self) -> Mapping[str, EffectivePreference]: + def named_styles(self) -> Mapping[str, EffectiveFormattingPreference]: return self._named_styles @property @@ -497,7 +528,7 @@ class MaintainerPreferenceTable: def parse_file( - named_styles: Dict[str, EffectivePreference], + named_styles: Dict[str, EffectiveFormattingPreference], maintainer_preferences: Dict[str, MaintainerPreference], filename: str, fd, @@ -520,38 +551,45 @@ def parse_file( named_styles_raw = {} for style_name, content in named_styles_raw.items(): - wrapped_style = CommentedMap({"formatting": content}) - style = EffectivePreference.from_file( + style = EffectiveFormattingPreference.from_file( filename, style_name, - wrapped_style, + content, ) named_styles[style_name] = style - for maintainer_email, maintainer_styles in maintainer_rules.items(): - if not isinstance(maintainer_styles, CommentedMap): + for maintainer_email, maintainer_pref in maintainer_rules.items(): + if not isinstance(maintainer_pref, CommentedMap): line_no = maintainer_rules.lc.key(maintainer_email).line raise ValueError( f'The value for maintainer "{maintainer_email}" should have been a mapping,' f' but it is not. The problem entry is at line {line_no} in "{filename}"' ) - formatting = maintainer_styles.get("formatting") + formatting = maintainer_pref.get("formatting") if isinstance(formatting, str): try: - style = named_styles_raw[formatting] + style = named_styles[formatting] except KeyError: line_no = maintainer_rules.lc.key(maintainer_email).line raise ValueError( f'The maintainer "{maintainer_email}" requested the named style "{formatting}",' f' but said style was not defined {filename}. The problem entry is at line {line_no} in "{filename}"' ) from None - maintainer_styles["formatting"] = style - maintainer_preferences[maintainer_email] = MaintainerPreference.from_file( + maintainer_pref["formatting"] = style + elif formatting is not None: + maintainer_pref["formatting"] = EffectiveFormattingPreference.from_file( + filename, + "formatting", + formatting, + ) + mp = MaintainerPreference.from_file( filename, maintainer_email, - maintainer_styles, + maintainer_pref, ) + maintainer_preferences[maintainer_email] = mp + @functools.lru_cache(64) def extract_maint_email(maint: str) -> str: @@ -569,7 +607,7 @@ def determine_effective_preference( maint_preference_table: MaintainerPreferenceTable, source_package: Optional[SourcePackage], salsa_ci: Optional[CommentedMap], -) -> Tuple[Optional[EffectivePreference], Optional[str], Optional[str]]: +) -> Tuple[Optional[EffectiveFormattingPreference], Optional[str], Optional[str]]: style = source_package.fields.get("X-Style") if source_package is not None else None if style is not None: if style not in ALL_PUBLIC_NAMED_STYLES: @@ -612,7 +650,6 @@ def determine_effective_preference( else: msg = None return detected_style, tool_w_args, msg - if source_package is None: return None, None, None @@ -620,47 +657,44 @@ def determine_effective_preference( if maint is None: return None, None, None maint_email = extract_maint_email(maint) - maint_style = maint_preference_table.maintainer_preferences.get(maint_email) + maint_pref = maint_preference_table.maintainer_preferences.get(maint_email) # Special-case "@packages.debian.org" when missing, since they are likely to be "ad-hoc" # teams that will not be registered. In that case, we fall back to looking at the uploader # preferences as-if the maintainer had not been listed at all. - if maint_style is None and not maint_email.endswith("@packages.debian.org"): + if maint_pref is None and not maint_email.endswith("@packages.debian.org"): return None, None, None - if maint_style is not None and maint_style.is_packaging_team: + if maint_pref is not None and maint_pref.is_packaging_team: # When the maintainer is registered as a packaging team, then we assume the packaging # team's style applies unconditionally. - effective = maint_style.as_effective_pref() + effective = maint_pref.formatting tool_w_args = _guess_tool_from_style(maint_preference_table, effective) return effective, tool_w_args, None uploaders = source_package.fields.get("Uploaders") if uploaders is None: - detected_style = ( - maint_style.as_effective_pref() if maint_style is not None else None - ) + detected_style = maint_pref.formatting if maint_pref is not None else None tool_w_args = _guess_tool_from_style(maint_preference_table, detected_style) return detected_style, tool_w_args, None - all_styles: List[Optional[EffectivePreference]] = [] - if maint_style is not None: - all_styles.append(maint_style) + all_styles: List[Optional[EffectiveFormattingPreference]] = [] + if maint_pref is not None: + all_styles.append(maint_pref.formatting) for uploader in _UPLOADER_SPLIT_RE.split(uploaders): uploader_email = extract_maint_email(uploader) - uploader_style = maint_preference_table.maintainer_preferences.get( + uploader_pref = maint_preference_table.maintainer_preferences.get( uploader_email ) - all_styles.append(uploader_style) + all_styles.append(uploader_pref.formatting if uploader_pref else None) if not all_styles: return None, None, None - r = functools.reduce(EffectivePreference.aligned_preference, all_styles) - if isinstance(r, MaintainerPreference): - r = r.as_effective_pref() + r = functools.reduce(EffectiveFormattingPreference.aligned_preference, all_styles) + assert not isinstance(r, MaintainerPreference) tool_w_args = _guess_tool_from_style(maint_preference_table, r) return r, tool_w_args, None def _guess_tool_from_style( maint_preference_table: MaintainerPreferenceTable, - pref: Optional[EffectivePreference], + pref: Optional[EffectiveFormattingPreference], ) -> Optional[str]: if pref is None: return None @@ -682,7 +716,9 @@ def _split_options(args: Iterable[str]) -> Iterable[str]: @functools.lru_cache -def parse_salsa_ci_wrap_and_sort_args(args: str) -> Optional[EffectivePreference]: +def parse_salsa_ci_wrap_and_sort_args( + args: str, +) -> Optional[EffectiveFormattingPreference]: options = dict(_WAS_DEFAULTS) for arg in _split_options(args.split()): v = _WAS_OPTIONS.get(arg) @@ -694,6 +730,6 @@ def parse_salsa_ci_wrap_and_sort_args(args: str) -> Optional[EffectivePreference options[varname] = value if "DISABLE_NORMALIZE_STANZA_ORDER" in options: del options["DISABLE_NORMALIZE_STANZA_ORDER"] - options["formatting_deb822_normalize_stanza_order"] = False + options["deb822_normalize_stanza_order"] = False - return EffectivePreference(**options) # type: ignore + return EffectiveFormattingPreference(**options) # type: ignore diff --git a/src/debputy/maintscript_snippet.py b/src/debputy/maintscript_snippet.py index ca81ca5..58a6bba 100644 --- a/src/debputy/maintscript_snippet.py +++ b/src/debputy/maintscript_snippet.py @@ -1,7 +1,7 @@ import dataclasses from typing import Sequence, Optional, List, Literal, Iterable, Dict, Self -from debputy.manifest_parser.base_types import DebputyDispatchableType +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.manifest_parser.util import AttributePath STD_CONTROL_SCRIPTS = frozenset( diff --git a/src/debputy/manifest_conditions.py b/src/debputy/manifest_conditions.py index 0f5c298..3e97b00 100644 --- a/src/debputy/manifest_conditions.py +++ b/src/debputy/manifest_conditions.py @@ -1,12 +1,12 @@ import dataclasses from enum import Enum -from typing import List, Callable, Optional, Sequence +from typing import List, Callable, Optional, Sequence, Any, Self, Mapping from debian.debian_support import DpkgArchTable from debputy._deb_options_profiles import DebBuildOptionsAndProfiles from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable -from debputy.manifest_parser.base_types import DebputyDispatchableType +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.packages import BinaryPackage from debputy.substitution import Substitution from debputy.util import active_profiles_match @@ -15,11 +15,14 @@ from debputy.util import active_profiles_match @dataclasses.dataclass(slots=True, frozen=True) class ConditionContext: binary_package: Optional[BinaryPackage] - build_env: DebBuildOptionsAndProfiles + deb_options_and_profiles: DebBuildOptionsAndProfiles substitution: Substitution dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable dpkg_arch_query_table: DpkgArchTable + def replace(self, /, **changes: Any) -> "Self": + return dataclasses.replace(self, **changes) + class ManifestCondition(DebputyDispatchableType): __slots__ = () @@ -72,6 +75,7 @@ class NegatedManifestCondition(ManifestCondition): __slots__ = ("_condition",) def __init__(self, condition: ManifestCondition) -> None: + super().__init__() self._condition = condition def negated(self) -> "ManifestCondition": @@ -107,6 +111,7 @@ class ManifestConditionGroup(ManifestCondition): match_type: _ConditionGroupMatchType, conditions: Sequence[ManifestCondition], ) -> None: + super().__init__() self.match_type = match_type self._conditions = conditions @@ -132,6 +137,7 @@ class ArchMatchManifestConditionBase(ManifestCondition): __slots__ = ("_arch_spec", "_is_negated") def __init__(self, arch_spec: List[str], *, is_negated: bool = False) -> None: + super().__init__() self._arch_spec = arch_spec self._is_negated = is_negated @@ -177,6 +183,7 @@ class BuildProfileMatch(ManifestCondition): __slots__ = ("_profile_spec", "_is_negated") def __init__(self, profile_spec: str, *, is_negated: bool = False) -> None: + super().__init__() self._profile_spec = profile_spec self._is_negated = is_negated @@ -190,7 +197,7 @@ class BuildProfileMatch(ManifestCondition): def evaluate(self, context: ConditionContext) -> bool: match = active_profiles_match( - self._profile_spec, context.build_env.deb_build_profiles + self._profile_spec, context.deb_options_and_profiles.deb_build_profiles ) return not match if self._is_negated else match @@ -211,7 +218,14 @@ def _can_run_built_binaries(context: ConditionContext) -> bool: if not context.dpkg_architecture_variables.is_cross_compiling: return True # User / Builder asserted that we could even though we are cross-compiling, so we have to assume it is true - return "crossbuildcanrunhostbinaries" in context.build_env.deb_build_options + return ( + "crossbuildcanrunhostbinaries" + in context.deb_options_and_profiles.deb_build_options + ) + + +def _run_build_time_tests(deb_build_options: Mapping[str, Optional[str]]) -> bool: + return "nocheck" not in deb_build_options _IS_CROSS_BUILDING = _SingletonCondition( @@ -226,12 +240,12 @@ _CAN_EXECUTE_COMPILED_BINARIES = _SingletonCondition( _RUN_BUILD_TIME_TESTS = _SingletonCondition( "Run build time tests", - lambda c: "nocheck" not in c.build_env.deb_build_options, + lambda c: _run_build_time_tests(c.deb_options_and_profiles.deb_build_options), ) _BUILD_DOCS_BDO = _SingletonCondition( "Build docs (nodocs not in DEB_BUILD_OPTIONS)", - lambda c: "nodocs" not in c.build_env.deb_build_options, + lambda c: "nodocs" not in c.deb_options_and_profiles.deb_build_options, ) diff --git a/src/debputy/manifest_parser/base_types.py b/src/debputy/manifest_parser/base_types.py index 865e320..106c30e 100644 --- a/src/debputy/manifest_parser/base_types.py +++ b/src/debputy/manifest_parser/base_types.py @@ -1,9 +1,8 @@ import dataclasses import os +import subprocess from functools import lru_cache from typing import ( - TypedDict, - NotRequired, Sequence, Optional, Union, @@ -12,12 +11,14 @@ from typing import ( Mapping, Iterable, TYPE_CHECKING, - Callable, - Type, - Generic, + Dict, + MutableMapping, + NotRequired, ) +from debputy.manifest_conditions import ManifestCondition from debputy.manifest_parser.exceptions import ManifestParseException +from debputy.manifest_parser.tagging_types import DebputyParsedContent from debputy.manifest_parser.util import ( AttributePath, _SymbolicModeSegment, @@ -25,37 +26,20 @@ from debputy.manifest_parser.util import ( ) from debputy.path_matcher import MatchRule, ExactFileSystemPath from debputy.substitution import Substitution -from debputy.types import S -from debputy.util import _normalize_path, T +from debputy.util import _normalize_path, _error, _warn, _debug_log if TYPE_CHECKING: - from debputy.manifest_conditions import ManifestCondition from debputy.manifest_parser.parser_data import ParserContextData -class DebputyParsedContent(TypedDict): - pass - - -class DebputyDispatchableType: - __slots__ = () - - -class DebputyParsedContentStandardConditional(DebputyParsedContent): - when: NotRequired["ManifestCondition"] - - @dataclasses.dataclass(slots=True, frozen=True) class OwnershipDefinition: entity_name: str entity_id: int -@dataclasses.dataclass -class TypeMapping(Generic[S, T]): - target_type: Type[T] - source_type: Type[S] - mapper: Callable[[S, AttributePath, Optional["ParserContextData"]], T] +class DebputyParsedContentStandardConditional(DebputyParsedContent): + when: NotRequired[ManifestCondition] ROOT_DEFINITION = OwnershipDefinition("root", 0) @@ -438,3 +422,76 @@ class FileSystemExactMatchRule(FileSystemMatchRule): class FileSystemExactNonDirMatchRule(FileSystemExactMatchRule): pass + + +class BuildEnvironmentDefinition: + + def dpkg_buildflags_env( + self, + env: Mapping[str, str], + definition_source: Optional[str], + ) -> Dict[str, str]: + dpkg_env = {} + try: + bf_output = subprocess.check_output(["dpkg-buildflags"], env=env) + except FileNotFoundError: + if definition_source is None: + _error( + "The dpkg-buildflags command was not available and is necessary to set the relevant" + "env variables by default." + ) + _error( + "The dpkg-buildflags command was not available and is necessary to set the relevant" + f"env variables for the environment defined at {definition_source}." + ) + except subprocess.CalledProcessError as e: + if definition_source is None: + _error( + f"The dpkg-buildflags command failed with exit code {e.returncode}. Please review the output from" + f" dpkg-buildflags above to resolve the issue." + ) + _error( + f"The dpkg-buildflags command failed with exit code {e.returncode}. Please review the output from" + f" dpkg-buildflags above to resolve the issue. The environment definition that triggered this call" + f" was {definition_source}" + ) + else: + warned = False + for line in bf_output.decode("utf-8").splitlines(keepends=False): + if "=" not in line or line.startswith("="): + if not warned: + _warn( + f"Unexpected output from dpkg-buildflags (not a K=V line): {line}" + ) + continue + k, v = line.split("=", 1) + if k.strip() != k: + if not warned: + _warn( + f'Unexpected output from dpkg-buildflags (Key had spaces): "{line}"' + ) + continue + dpkg_env[k] = v + + return dpkg_env + + def log_computed_env(self, source: str, computed_env: Mapping[str, str]) -> None: + _debug_log(f"Computed environment variables from {source}") + for k, v in computed_env.items(): + _debug_log(f" {k}={v}") + + def update_env(self, env: MutableMapping[str, str]) -> None: + dpkg_env = self.dpkg_buildflags_env(env, None) + self.log_computed_env("dpkg-buildflags", dpkg_env) + env.update(dpkg_env) + + +class BuildEnvironments: + + def __init__( + self, + environments: Dict[str, BuildEnvironmentDefinition], + default_environment: Optional[BuildEnvironmentDefinition], + ) -> None: + self.environments = environments + self.default_environment = default_environment diff --git a/src/debputy/manifest_parser/declarative_parser.py b/src/debputy/manifest_parser/declarative_parser.py index 6cbbce3..2c350a0 100644 --- a/src/debputy/manifest_parser/declarative_parser.py +++ b/src/debputy/manifest_parser/declarative_parser.py @@ -1,5 +1,6 @@ import collections import dataclasses +import typing from typing import ( Any, Callable, @@ -16,7 +17,6 @@ from typing import ( Mapping, Optional, cast, - is_typeddict, Type, Union, List, @@ -28,13 +28,7 @@ from typing import ( Container, ) -from debputy.manifest_parser.base_types import ( - DebputyParsedContent, - FileSystemMatchRule, - FileSystemExactMatchRule, - DebputyDispatchableType, - TypeMapping, -) +from debputy.manifest_parser.base_types import FileSystemMatchRule from debputy.manifest_parser.exceptions import ( ManifestParseException, ) @@ -43,7 +37,20 @@ from debputy.manifest_parser.mapper_code import ( wrap_into_list, map_each_element, ) +from debputy.manifest_parser.parse_hints import ( + ConditionalRequired, + DebputyParseHint, + TargetAttribute, + ManifestAttribute, + ConflictWithSourceAttribute, + NotPathHint, +) from debputy.manifest_parser.parser_data import ParserContextData +from debputy.manifest_parser.tagging_types import ( + DebputyParsedContent, + DebputyDispatchableType, + TypeMapping, +) from debputy.manifest_parser.util import ( AttributePath, unpack_type, @@ -53,8 +60,6 @@ from debputy.manifest_parser.util import ( from debputy.plugin.api.impl_types import ( DeclarativeInputParser, TD, - _ALL_PACKAGE_TYPES, - resolve_package_type_selectors, ListWrappedDeclarativeInputParser, DispatchingObjectParser, DispatchingTableParser, @@ -64,8 +69,11 @@ from debputy.plugin.api.impl_types import ( ) from debputy.plugin.api.spec import ( ParserDocumentation, - PackageTypeSelector, DebputyIntegrationMode, + StandardParserAttributeDocumentation, + undocumented_attr, + ParserAttributeDocumentation, + reference_documentation, ) from debputy.util import _info, _warn, assume_not_none @@ -478,242 +486,6 @@ class DeclarativeMappingInputParser(DeclarativeInputParser[TD], Generic[TD, SF]) return self._per_attribute_conflicts_cache -class DebputyParseHint: - @classmethod - def target_attribute(cls, target_attribute: str) -> "DebputyParseHint": - """Define this source attribute to have a different target attribute name - - As an example: - - >>> class SourceType(TypedDict): - ... source: Annotated[NotRequired[str], DebputyParseHint.target_attribute("sources")] - ... sources: NotRequired[List[str]] - >>> class TargetType(TypedDict): - ... sources: List[str] - >>> pg = ParserGenerator() - >>> parser = pg.generate_parser(TargetType, source_content=SourceType) - - In this example, the user can provide either `source` or `sources` and the parser will - map them to the `sources` attribute in the `TargetType`. Note this example relies on - the builtin mapping of `str` to `List[str]` to align the types between `source` (from - SourceType) and `sources` (from TargetType). - - The following rules apply: - - * All source attributes that map to the same target attribute will be mutually exclusive - (that is, the user cannot give `source` *and* `sources` as input). - * When the target attribute is required, the source attributes are conditionally - mandatory requiring the user to provide exactly one of them. - * When multiple source attributes point to a single target attribute, none of the source - attributes can be Required. - * The annotation can only be used for the source type specification and the source type - specification must be different from the target type specification. - - The `target_attribute` annotation can be used without having multiple source attributes. This - can be useful if the source attribute name is not valid as a python variable identifier to - rename it to a valid python identifier. - - :param target_attribute: The attribute name in the target content - :return: The annotation. - """ - return TargetAttribute(target_attribute) - - @classmethod - def conflicts_with_source_attributes( - cls, - *conflicting_source_attributes: str, - ) -> "DebputyParseHint": - """Declare a conflict with one or more source attributes - - Example: - - >>> class SourceType(TypedDict): - ... source: Annotated[NotRequired[str], DebputyParseHint.target_attribute("sources")] - ... sources: NotRequired[List[str]] - ... into_dir: NotRequired[str] - ... renamed_to: Annotated[ - ... NotRequired[str], - ... DebputyParseHint.conflicts_with_source_attributes("sources", "into_dir") - ... ] - >>> class TargetType(TypedDict): - ... sources: List[str] - ... into_dir: NotRequired[str] - ... renamed_to: NotRequired[str] - >>> pg = ParserGenerator() - >>> parser = pg.generate_parser(TargetType, source_content=SourceType) - - In this example, if the user was to provide `renamed_to` with `sources` or `into_dir` the parser would report - an error. However, the parser will allow `renamed_to` with `source` as the conflict is considered only for - the input source. That is, it is irrelevant that `sources` and `source´ happens to "map" to the same target - attribute. - - The following rules apply: - * It is not possible for a target attribute to declare conflicts unless the target type spec is reused as - source type spec. - * All attributes involved in a conflict must be NotRequired. If any of the attributes are Required, then - the parser generator will reject the input. - * All attributes listed in the conflict must be valid attributes in the source type spec. - - Note you do not have to specify conflicts between two attributes with the same target attribute name. The - `target_attribute` annotation will handle that for you. - - :param conflicting_source_attributes: All source attributes that cannot be used with this attribute. - :return: The annotation. - """ - if len(conflicting_source_attributes) < 1: - raise ValueError( - "DebputyParseHint.conflicts_with_source_attributes requires at least one attribute as input" - ) - return ConflictWithSourceAttribute(frozenset(conflicting_source_attributes)) - - @classmethod - def required_when_single_binary( - cls, - *, - package_type: PackageTypeSelector = _ALL_PACKAGE_TYPES, - ) -> "DebputyParseHint": - """Declare a source attribute as required when the source package produces exactly one binary package - - The attribute in question must always be declared as `NotRequired` in the TypedDict and this condition - can only be used for source attributes. - """ - resolved_package_types = resolve_package_type_selectors(package_type) - reason = "The field is required for source packages producing exactly one binary package" - if resolved_package_types != _ALL_PACKAGE_TYPES: - types = ", ".join(sorted(resolved_package_types)) - reason += f" of type {types}" - return ConditionalRequired( - reason, - lambda c: len( - [ - p - for p in c.binary_packages.values() - if p.package_type in package_type - ] - ) - == 1, - ) - return ConditionalRequired( - reason, - lambda c: c.is_single_binary_package, - ) - - @classmethod - def required_when_multi_binary( - cls, - *, - package_type: PackageTypeSelector = _ALL_PACKAGE_TYPES, - ) -> "DebputyParseHint": - """Declare a source attribute as required when the source package produces two or more binary package - - The attribute in question must always be declared as `NotRequired` in the TypedDict and this condition - can only be used for source attributes. - """ - resolved_package_types = resolve_package_type_selectors(package_type) - reason = "The field is required for source packages producing two or more binary packages" - if resolved_package_types != _ALL_PACKAGE_TYPES: - types = ", ".join(sorted(resolved_package_types)) - reason = ( - "The field is required for source packages producing not producing exactly one binary packages" - f" of type {types}" - ) - return ConditionalRequired( - reason, - lambda c: len( - [ - p - for p in c.binary_packages.values() - if p.package_type in package_type - ] - ) - != 1, - ) - return ConditionalRequired( - reason, - lambda c: not c.is_single_binary_package, - ) - - @classmethod - def manifest_attribute(cls, attribute: str) -> "DebputyParseHint": - """Declare what the attribute name (as written in the manifest) should be - - By default, debputy will do an attribute normalizing that will take valid python identifiers such - as `dest_dir` and remap it to the manifest variant (such as `dest-dir`) automatically. If you have - a special case, where this built-in normalization is insufficient or the python name is considerably - different from what the user would write in the manifest, you can use this parse hint to set the - name that the user would have to write in the manifest for this attribute. - - >>> class SourceType(TypedDict): - ... source: List[FileSystemMatchRule] - ... # Use "as" in the manifest because "as_" was not pretty enough - ... install_as: Annotated[NotRequired[FileSystemExactMatchRule], DebputyParseHint.manifest_attribute("as")] - - In this example, we use the parse hint to use "as" as the name in the manifest, because we cannot - use "as" a valid python identifier (it is a keyword). While debputy would map `as_` to `as` for us, - we have chosen to use `install_as` as a python identifier. - """ - return ManifestAttribute(attribute) - - @classmethod - def not_path_error_hint(cls) -> "DebputyParseHint": - """Mark this attribute as not a "path hint" when it comes to reporting errors - - By default, `debputy` will pick up attributes that uses path names (FileSystemMatchRule) as - candidates for parse error hints (the little "<Search for: VALUE>" in error messages). - - Most rules only have one active path-based attribute and paths tends to be unique enough - that it helps people spot the issue faster. However, in rare cases, you can have multiple - attributes that fit the bill. In this case, this hint can be used to "hide" the suboptimal - choice. As an example: - - >>> class SourceType(TypedDict): - ... source: List[FileSystemMatchRule] - ... install_as: Annotated[NotRequired[FileSystemExactMatchRule], DebputyParseHint.not_path_error_hint()] - - In this case, without the hint, `debputy` might pick up `install_as` as the attribute to - use as hint for error reporting. However, here we have decided that we never want `install_as` - leaving `source` as the only option. - - Generally, this type hint must be placed on the **source** format. Any source attribute matching - the parsed format will be ignored. - - Mind the asymmetry: The annotation is placed in the **source** format while `debputy` looks at - the type of the target attribute to determine if it counts as path. - """ - return NOT_PATH_HINT - - -@dataclasses.dataclass(frozen=True, slots=True) -class TargetAttribute(DebputyParseHint): - attribute: str - - -@dataclasses.dataclass(frozen=True, slots=True) -class ConflictWithSourceAttribute(DebputyParseHint): - conflicting_attributes: FrozenSet[str] - - -@dataclasses.dataclass(frozen=True, slots=True) -class ConditionalRequired(DebputyParseHint): - reason: str - condition: Callable[["ParserContextData"], bool] - - def condition_applies(self, context: "ParserContextData") -> bool: - return self.condition(context) - - -@dataclasses.dataclass(frozen=True, slots=True) -class ManifestAttribute(DebputyParseHint): - attribute: str - - -class NotPathHint(DebputyParseHint): - pass - - -NOT_PATH_HINT = NotPathHint() - - def _is_path_attribute_candidate( source_attribute: AttributeDescription, target_attribute: AttributeDescription ) -> bool: @@ -730,6 +502,16 @@ def _is_path_attribute_candidate( return isinstance(match_type, type) and issubclass(match_type, FileSystemMatchRule) +if typing.is_typeddict(DebputyParsedContent): + is_typeddict = typing.is_typeddict +else: + + def is_typeddict(t: Any) -> bool: + if typing.is_typeddict(t): + return True + return isinstance(t, type) and issubclass(t, DebputyParsedContent) + + class ParserGenerator: def __init__(self) -> None: self._registered_types: Dict[Any, TypeMapping[Any, Any]] = {} @@ -811,6 +593,9 @@ class ParserGenerator: expected_debputy_integration_mode: Optional[ Container[DebputyIntegrationMode] ] = None, + automatic_docs: Optional[ + Mapping[Type[Any], Sequence[StandardParserAttributeDocumentation]] + ] = None, ) -> DeclarativeInputParser[TD]: """Derive a parser from a TypedDict @@ -978,7 +763,7 @@ class ParserGenerator: f"Unsupported parsed_content descriptor: {parsed_content.__qualname__}." ' Only "TypedDict"-based types and a subset of "DebputyDispatchableType" are supported.' ) - if is_list_wrapped: + if is_list_wrapped and source_content is not None: if get_origin(source_content) != list: raise ValueError( "If the parsed_content is a List type, then source_format must be a List type as well." @@ -1133,10 +918,15 @@ class ParserGenerator: parsed_alt_form.type_validator.combine_mapper(bridge_mapper) ) - _verify_inline_reference_documentation( - source_content_attributes, - inline_reference_documentation, - parsed_alt_form is not None, + inline_reference_documentation = ( + _verify_and_auto_correct_inline_reference_documentation( + parsed_content, + source_typed_dict, + source_content_attributes, + inline_reference_documentation, + parsed_alt_form is not None, + automatic_docs, + ) ) if non_mapping_source_only: parser = DeclarativeNonMappingInputParser( @@ -1700,45 +1490,133 @@ class ParserGenerator: return orig_td -def _verify_inline_reference_documentation( +def _sort_key(attr: StandardParserAttributeDocumentation) -> Any: + key = next(iter(attr.attributes)) + return attr.sort_category, key + + +def _apply_std_docs( + std_doc_table: Optional[ + Mapping[Type[Any], Sequence[StandardParserAttributeDocumentation]] + ], + source_format_typed_dict: Type[Any], + attribute_docs: Optional[Sequence[ParserAttributeDocumentation]], +) -> Optional[Sequence[ParserAttributeDocumentation]]: + if std_doc_table is None or not std_doc_table: + return attribute_docs + + has_docs_for = set() + if attribute_docs: + for attribute_doc in attribute_docs: + has_docs_for.update(attribute_doc.attributes) + + base_seen = set() + std_docs_used = [] + + remaining_bases = set(getattr(source_format_typed_dict, "__orig_bases__", [])) + base_seen.update(remaining_bases) + while remaining_bases: + base = remaining_bases.pop() + new_bases_to_check = { + x for x in getattr(base, "__orig_bases__", []) if x not in base_seen + } + remaining_bases.update(new_bases_to_check) + base_seen.update(new_bases_to_check) + std_docs = std_doc_table.get(base) + if std_docs: + for std_doc in std_docs: + if any(a in has_docs_for for a in std_doc.attributes): + # If there is any overlap, do not add the docs + continue + has_docs_for.update(std_doc.attributes) + std_docs_used.append(std_doc) + + if not std_docs_used: + return attribute_docs + docs = sorted(std_docs_used, key=_sort_key) + if attribute_docs: + # Plugin provided attributes first + c = list(attribute_docs) + c.extend(docs) + docs = c + return tuple(docs) + + +def _verify_and_auto_correct_inline_reference_documentation( + parsed_content: Type[TD], + source_typed_dict: Type[Any], source_content_attributes: Mapping[str, AttributeDescription], inline_reference_documentation: Optional[ParserDocumentation], has_alt_form: bool, -) -> None: - if inline_reference_documentation is None: - return - attribute_doc = inline_reference_documentation.attribute_doc - if attribute_doc: + automatic_docs: Optional[ + Mapping[Type[Any], Sequence[StandardParserAttributeDocumentation]] + ] = None, +) -> Optional[ParserDocumentation]: + orig_attribute_docs = ( + inline_reference_documentation.attribute_doc + if inline_reference_documentation + else None + ) + attribute_docs = _apply_std_docs( + automatic_docs, + source_typed_dict, + orig_attribute_docs, + ) + if inline_reference_documentation is None and attribute_docs is None: + return None + changes = {} + if attribute_docs: seen = set() - for attr_doc in attribute_doc: + had_any_custom_docs = False + for attr_doc in attribute_docs: + if not isinstance(attr_doc, StandardParserAttributeDocumentation): + had_any_custom_docs = True for attr_name in attr_doc.attributes: attr = source_content_attributes.get(attr_name) if attr is None: raise ValueError( - f'The inline_reference_documentation references an attribute "{attr_name}", which does not' - f" exist in the source format." + f"The inline_reference_documentation for the source format of {parsed_content.__qualname__}" + f' references an attribute "{attr_name}", which does not exist in the source format.' ) if attr_name in seen: raise ValueError( - f'The inline_reference_documentation has documentation for "{attr_name}" twice,' - f" which is not supported. Please document it at most once" + f"The inline_reference_documentation for the source format of {parsed_content.__qualname__}" + f' has documentation for "{attr_name}" twice, which is not supported.' + f" Please document it at most once" ) seen.add(attr_name) - undocumented = source_content_attributes.keys() - seen if undocumented: - undocumented_attrs = ", ".join(undocumented) - raise ValueError( - "The following attributes were not documented. If this is deliberate, then please" - ' declare each them as undocumented (via undocumented_attr("foo")):' - f" {undocumented_attrs}" - ) + if had_any_custom_docs: + undocumented_attrs = ", ".join(undocumented) + raise ValueError( + f"The following attributes were not documented for the source format of" + f" {parsed_content.__qualname__}. If this is deliberate, then please" + ' declare each them as undocumented (via undocumented_attr("foo")):' + f" {undocumented_attrs}" + ) + combined_docs = list(attribute_docs) + combined_docs.extend(undocumented_attr(a) for a in sorted(undocumented)) + attribute_docs = combined_docs + + if attribute_docs and orig_attribute_docs != attribute_docs: + assert attribute_docs is not None + changes["attribute_doc"] = tuple(attribute_docs) - if inline_reference_documentation.alt_parser_description and not has_alt_form: + if ( + inline_reference_documentation is not None + and inline_reference_documentation.alt_parser_description + and not has_alt_form + ): raise ValueError( "The inline_reference_documentation had documentation for an non-mapping format," " but the source format does not have a non-mapping format." ) + if changes: + if inline_reference_documentation is None: + inline_reference_documentation = reference_documentation() + return inline_reference_documentation.replace(**changes) + return inline_reference_documentation def _check_conflicts( diff --git a/src/debputy/manifest_parser/exceptions.py b/src/debputy/manifest_parser/exceptions.py index 671ec1b..f058458 100644 --- a/src/debputy/manifest_parser/exceptions.py +++ b/src/debputy/manifest_parser/exceptions.py @@ -1,9 +1,17 @@ from debputy.exceptions import DebputyRuntimeError -class ManifestParseException(DebputyRuntimeError): +class ManifestException(DebputyRuntimeError): + pass + + +class ManifestParseException(ManifestException): pass class ManifestTypeException(ManifestParseException): pass + + +class ManifestInvalidUserDataException(ManifestException): + pass diff --git a/src/debputy/manifest_parser/mapper_code.py b/src/debputy/manifest_parser/mapper_code.py index d7a08c3..f206af9 100644 --- a/src/debputy/manifest_parser/mapper_code.py +++ b/src/debputy/manifest_parser/mapper_code.py @@ -4,22 +4,25 @@ from typing import ( Union, List, Callable, + TYPE_CHECKING, ) from debputy.manifest_parser.exceptions import ManifestTypeException -from debputy.manifest_parser.parser_data import ParserContextData -from debputy.manifest_parser.util import AttributePath from debputy.packages import BinaryPackage from debputy.util import assume_not_none +if TYPE_CHECKING: + from debputy.manifest_parser.util import AttributePath + from debputy.manifest_parser.parser_data import ParserContextData + S = TypeVar("S") T = TypeVar("T") def type_mapper_str2package( raw_package_name: str, - ap: AttributePath, - opc: Optional[ParserContextData], + ap: "AttributePath", + opc: Optional["ParserContextData"], ) -> BinaryPackage: pc = assume_not_none(opc) if "{{" in raw_package_name: @@ -50,7 +53,7 @@ def type_mapper_str2package( def wrap_into_list( x: T, - _ap: AttributePath, + _ap: "AttributePath", _pc: Optional["ParserContextData"], ) -> List[T]: return [x] @@ -58,18 +61,18 @@ def wrap_into_list( def normalize_into_list( x: Union[T, List[T]], - _ap: AttributePath, + _ap: "AttributePath", _pc: Optional["ParserContextData"], ) -> List[T]: return x if isinstance(x, list) else [x] def map_each_element( - mapper: Callable[[S, AttributePath, Optional["ParserContextData"]], T], -) -> Callable[[List[S], AttributePath, Optional["ParserContextData"]], List[T]]: + mapper: Callable[[S, "AttributePath", Optional["ParserContextData"]], T], +) -> Callable[[List[S], "AttributePath", Optional["ParserContextData"]], List[T]]: def _generated_mapper( xs: List[S], - ap: AttributePath, + ap: "AttributePath", pc: Optional["ParserContextData"], ) -> List[T]: return [mapper(s, ap[i], pc) for i, s in enumerate(xs)] diff --git a/src/debputy/manifest_parser/parse_hints.py b/src/debputy/manifest_parser/parse_hints.py new file mode 100644 index 0000000..30b8aca --- /dev/null +++ b/src/debputy/manifest_parser/parse_hints.py @@ -0,0 +1,259 @@ +import dataclasses +from typing import ( + NotRequired, + TypedDict, + TYPE_CHECKING, + Callable, + FrozenSet, + Annotated, + List, +) + +from debputy.manifest_parser.util import ( + resolve_package_type_selectors, + _ALL_PACKAGE_TYPES, +) +from debputy.plugin.api.spec import PackageTypeSelector + +if TYPE_CHECKING: + from debputy.manifest_parser.parser_data import ParserContextData + + +class DebputyParseHint: + @classmethod + def target_attribute(cls, target_attribute: str) -> "DebputyParseHint": + """Define this source attribute to have a different target attribute name + + As an example: + + >>> from debputy.manifest_parser.declarative_parser import ParserGenerator + >>> class SourceType(TypedDict): + ... source: Annotated[NotRequired[str], DebputyParseHint.target_attribute("sources")] + ... sources: NotRequired[List[str]] + >>> class TargetType(TypedDict): + ... sources: List[str] + >>> pg = ParserGenerator() + >>> parser = pg.generate_parser(TargetType, source_content=SourceType) + + In this example, the user can provide either `source` or `sources` and the parser will + map them to the `sources` attribute in the `TargetType`. Note this example relies on + the builtin mapping of `str` to `List[str]` to align the types between `source` (from + SourceType) and `sources` (from TargetType). + + The following rules apply: + + * All source attributes that map to the same target attribute will be mutually exclusive + (that is, the user cannot give `source` *and* `sources` as input). + * When the target attribute is required, the source attributes are conditionally + mandatory requiring the user to provide exactly one of them. + * When multiple source attributes point to a single target attribute, none of the source + attributes can be Required. + * The annotation can only be used for the source type specification and the source type + specification must be different from the target type specification. + + The `target_attribute` annotation can be used without having multiple source attributes. This + can be useful if the source attribute name is not valid as a python variable identifier to + rename it to a valid python identifier. + + :param target_attribute: The attribute name in the target content + :return: The annotation. + """ + return TargetAttribute(target_attribute) + + @classmethod + def conflicts_with_source_attributes( + cls, + *conflicting_source_attributes: str, + ) -> "DebputyParseHint": + """Declare a conflict with one or more source attributes + + Example: + + >>> from debputy.manifest_parser.declarative_parser import ParserGenerator + >>> class SourceType(TypedDict): + ... source: Annotated[NotRequired[str], DebputyParseHint.target_attribute("sources")] + ... sources: NotRequired[List[str]] + ... into_dir: NotRequired[str] + ... renamed_to: Annotated[ + ... NotRequired[str], + ... DebputyParseHint.conflicts_with_source_attributes("sources", "into_dir") + ... ] + >>> class TargetType(TypedDict): + ... sources: List[str] + ... into_dir: NotRequired[str] + ... renamed_to: NotRequired[str] + >>> pg = ParserGenerator() + >>> parser = pg.generate_parser(TargetType, source_content=SourceType) + + In this example, if the user was to provide `renamed_to` with `sources` or `into_dir` the parser would report + an error. However, the parser will allow `renamed_to` with `source` as the conflict is considered only for + the input source. That is, it is irrelevant that `sources` and `source´ happens to "map" to the same target + attribute. + + The following rules apply: + * It is not possible for a target attribute to declare conflicts unless the target type spec is reused as + source type spec. + * All attributes involved in a conflict must be NotRequired. If any of the attributes are Required, then + the parser generator will reject the input. + * All attributes listed in the conflict must be valid attributes in the source type spec. + + Note you do not have to specify conflicts between two attributes with the same target attribute name. The + `target_attribute` annotation will handle that for you. + + :param conflicting_source_attributes: All source attributes that cannot be used with this attribute. + :return: The annotation. + """ + if len(conflicting_source_attributes) < 1: + raise ValueError( + "DebputyParseHint.conflicts_with_source_attributes requires at least one attribute as input" + ) + return ConflictWithSourceAttribute(frozenset(conflicting_source_attributes)) + + @classmethod + def required_when_single_binary( + cls, + *, + package_type: PackageTypeSelector = _ALL_PACKAGE_TYPES, + ) -> "DebputyParseHint": + """Declare a source attribute as required when the source package produces exactly one binary package + + The attribute in question must always be declared as `NotRequired` in the TypedDict and this condition + can only be used for source attributes. + """ + resolved_package_types = resolve_package_type_selectors(package_type) + reason = "The field is required for source packages producing exactly one binary package" + if resolved_package_types != _ALL_PACKAGE_TYPES: + types = ", ".join(sorted(resolved_package_types)) + reason += f" of type {types}" + return ConditionalRequired( + reason, + lambda c: len( + [ + p + for p in c.binary_packages.values() + if p.package_type in package_type + ] + ) + == 1, + ) + return ConditionalRequired( + reason, + lambda c: c.is_single_binary_package, + ) + + @classmethod + def required_when_multi_binary( + cls, + *, + package_type: PackageTypeSelector = _ALL_PACKAGE_TYPES, + ) -> "DebputyParseHint": + """Declare a source attribute as required when the source package produces two or more binary package + + The attribute in question must always be declared as `NotRequired` in the TypedDict and this condition + can only be used for source attributes. + """ + resolved_package_types = resolve_package_type_selectors(package_type) + reason = "The field is required for source packages producing two or more binary packages" + if resolved_package_types != _ALL_PACKAGE_TYPES: + types = ", ".join(sorted(resolved_package_types)) + reason = ( + "The field is required for source packages producing not producing exactly one binary packages" + f" of type {types}" + ) + return ConditionalRequired( + reason, + lambda c: len( + [ + p + for p in c.binary_packages.values() + if p.package_type in package_type + ] + ) + != 1, + ) + return ConditionalRequired( + reason, + lambda c: not c.is_single_binary_package, + ) + + @classmethod + def manifest_attribute(cls, attribute: str) -> "DebputyParseHint": + """Declare what the attribute name (as written in the manifest) should be + + By default, debputy will do an attribute normalizing that will take valid python identifiers such + as `dest_dir` and remap it to the manifest variant (such as `dest-dir`) automatically. If you have + a special case, where this built-in normalization is insufficient or the python name is considerably + different from what the user would write in the manifest, you can use this parse hint to set the + name that the user would have to write in the manifest for this attribute. + + >>> from debputy.manifest_parser.base_types import FileSystemMatchRule, FileSystemExactMatchRule + >>> class SourceType(TypedDict): + ... source: List[FileSystemMatchRule] + ... # Use "as" in the manifest because "as_" was not pretty enough + ... install_as: Annotated[NotRequired[FileSystemExactMatchRule], DebputyParseHint.manifest_attribute("as")] + + In this example, we use the parse hint to use "as" as the name in the manifest, because we cannot + use "as" a valid python identifier (it is a keyword). While debputy would map `as_` to `as` for us, + we have chosen to use `install_as` as a python identifier. + """ + return ManifestAttribute(attribute) + + @classmethod + def not_path_error_hint(cls) -> "DebputyParseHint": + """Mark this attribute as not a "path hint" when it comes to reporting errors + + By default, `debputy` will pick up attributes that uses path names (FileSystemMatchRule) as + candidates for parse error hints (the little "<Search for: VALUE>" in error messages). + + Most rules only have one active path-based attribute and paths tends to be unique enough + that it helps people spot the issue faster. However, in rare cases, you can have multiple + attributes that fit the bill. In this case, this hint can be used to "hide" the suboptimal + choice. As an example: + + >>> from debputy.manifest_parser.base_types import FileSystemMatchRule, FileSystemExactMatchRule + >>> class SourceType(TypedDict): + ... source: List[FileSystemMatchRule] + ... install_as: Annotated[NotRequired[FileSystemExactMatchRule], DebputyParseHint.not_path_error_hint()] + + In this case, without the hint, `debputy` might pick up `install_as` as the attribute to + use as hint for error reporting. However, here we have decided that we never want `install_as` + leaving `source` as the only option. + + Generally, this type hint must be placed on the **source** format. Any source attribute matching + the parsed format will be ignored. + + Mind the asymmetry: The annotation is placed in the **source** format while `debputy` looks at + the type of the target attribute to determine if it counts as path. + """ + return NOT_PATH_HINT + + +@dataclasses.dataclass(frozen=True, slots=True) +class TargetAttribute(DebputyParseHint): + attribute: str + + +@dataclasses.dataclass(frozen=True, slots=True) +class ConflictWithSourceAttribute(DebputyParseHint): + conflicting_attributes: FrozenSet[str] + + +@dataclasses.dataclass(frozen=True, slots=True) +class ConditionalRequired(DebputyParseHint): + reason: str + condition: Callable[["ParserContextData"], bool] + + def condition_applies(self, context: "ParserContextData") -> bool: + return self.condition(context) + + +@dataclasses.dataclass(frozen=True, slots=True) +class ManifestAttribute(DebputyParseHint): + attribute: str + + +class NotPathHint(DebputyParseHint): + pass + + +NOT_PATH_HINT = NotPathHint() diff --git a/src/debputy/manifest_parser/parser_data.py b/src/debputy/manifest_parser/parser_data.py index 30d9ce0..acc5c67 100644 --- a/src/debputy/manifest_parser/parser_data.py +++ b/src/debputy/manifest_parser/parser_data.py @@ -11,12 +11,15 @@ from debian.debian_support import DpkgArchTable from debputy._deb_options_profiles import DebBuildOptionsAndProfiles from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable +from debputy.manifest_parser.base_types import BuildEnvironmentDefinition from debputy.manifest_parser.exceptions import ManifestParseException from debputy.manifest_parser.util import AttributePath -from debputy.packages import BinaryPackage -from debputy.plugin.api.impl_types import ( +from debputy.manifest_parser.util import ( _ALL_PACKAGE_TYPES, resolve_package_type_selectors, +) +from debputy.packages import BinaryPackage +from debputy.plugin.api.impl_types import ( TP, DispatchingTableParser, TTP, @@ -101,7 +104,7 @@ class ParserContextData: raise NotImplementedError @property - def build_env(self) -> DebBuildOptionsAndProfiles: + def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles: raise NotImplementedError @contextlib.contextmanager @@ -129,3 +132,8 @@ class ParserContextData: @property def debputy_integration_mode(self) -> DebputyIntegrationMode: raise NotImplementedError + + def resolve_build_environment( + self, name: Optional[str], attribute_path: AttributePath + ) -> BuildEnvironmentDefinition: + raise NotImplementedError diff --git a/src/debputy/manifest_parser/tagging_types.py b/src/debputy/manifest_parser/tagging_types.py new file mode 100644 index 0000000..83030f0 --- /dev/null +++ b/src/debputy/manifest_parser/tagging_types.py @@ -0,0 +1,36 @@ +import dataclasses +from typing import ( + TypedDict, + TYPE_CHECKING, + Generic, + Type, + Callable, + Optional, +) + +from debputy.plugin.plugin_state import current_debputy_plugin_required +from debputy.types import S +from debputy.util import T + +if TYPE_CHECKING: + from debputy.manifest_parser.parser_data import ParserContextData + + from debputy.manifest_parser.util import AttributePath + + +class DebputyParsedContent(TypedDict): + pass + + +class DebputyDispatchableType: + __slots__ = ("_debputy_plugin",) + + def __init__(self) -> None: + self._debputy_plugin = current_debputy_plugin_required() + + +@dataclasses.dataclass +class TypeMapping(Generic[S, T]): + target_type: Type[T] + source_type: Type[S] + mapper: Callable[[S, "AttributePath", Optional["ParserContextData"]], T] diff --git a/src/debputy/manifest_parser/util.py b/src/debputy/manifest_parser/util.py index bcaa617..4e8fd7c 100644 --- a/src/debputy/manifest_parser/util.py +++ b/src/debputy/manifest_parser/util.py @@ -15,6 +15,8 @@ from typing import ( Iterable, Container, Literal, + FrozenSet, + cast, ) from debputy.yaml.compat import CommentedBase @@ -23,7 +25,7 @@ from debputy.manifest_parser.exceptions import ManifestParseException if TYPE_CHECKING: from debputy.manifest_parser.parser_data import ParserContextData - from debputy.plugin.api.spec import DebputyIntegrationMode + from debputy.plugin.api.spec import DebputyIntegrationMode, PackageTypeSelector MP = TypeVar("MP", bound="DebputyParseHint") @@ -34,6 +36,25 @@ AttributePathAliasMapping = Mapping[ LineReportKind = Literal["key", "value", "container"] +_PACKAGE_TYPE_DEB_ONLY = frozenset(["deb"]) +_ALL_PACKAGE_TYPES = frozenset(["deb", "udeb"]) + + +def resolve_package_type_selectors( + package_type: "PackageTypeSelector", +) -> FrozenSet[str]: + if package_type is _ALL_PACKAGE_TYPES or package_type is _PACKAGE_TYPE_DEB_ONLY: + return cast("FrozenSet[str]", package_type) + if isinstance(package_type, str): + return ( + _PACKAGE_TYPE_DEB_ONLY + if package_type == "deb" + else frozenset([package_type]) + ) + else: + return frozenset(package_type) + + class AttributePath: __slots__ = ("parent", "container", "name", "alias_mapping", "path_hint") @@ -94,7 +115,7 @@ class AttributePath: lc_data = lc.value(key) else: lc_data = lc.item(key) - except (AttributeError, RuntimeError, LookupError): + except (AttributeError, RuntimeError, LookupError, TypeError): lc_data = None else: lc_data = None @@ -161,7 +182,7 @@ class AttributePath: if container is not None: try: child_container = self.container[item] - except (AttributeError, RuntimeError, LookupError): + except (AttributeError, RuntimeError, LookupError, TypeError): child_container = None else: child_container = None diff --git a/src/debputy/package_build/assemble_deb.py b/src/debputy/package_build/assemble_deb.py index fd92f37..21fe0d6 100644 --- a/src/debputy/package_build/assemble_deb.py +++ b/src/debputy/package_build/assemble_deb.py @@ -95,8 +95,8 @@ def assemble_debs( package_metadata_context = dctrl_data.package_metadata_context if ( dbgsym_package_name in package_data_table - or "noautodbgsym" in manifest.build_env.deb_build_options - or "noddebs" in manifest.build_env.deb_build_options + or "noautodbgsym" in manifest.deb_options_and_profiles.deb_build_options + or "noddebs" in manifest.deb_options_and_profiles.deb_build_options ): # Discard the dbgsym part if it conflicts with a real package, or # we were asked not to build it. diff --git a/src/debputy/packager_provided_files.py b/src/debputy/packager_provided_files.py index a35beec..5657ad2 100644 --- a/src/debputy/packager_provided_files.py +++ b/src/debputy/packager_provided_files.py @@ -35,6 +35,8 @@ _KNOWN_NON_TYPO_EXTENSIONS = frozenset( "bash", "pl", "py", + # Fairly common image format in older packages + "xpm", } ) diff --git a/src/debputy/packages.py b/src/debputy/packages.py index 3a6ee16..0a3876a 100644 --- a/src/debputy/packages.py +++ b/src/debputy/packages.py @@ -44,15 +44,15 @@ class DctrlParser: DpkgArchitectureBuildProcessValuesTable ] = None, dpkg_arch_query_table: Optional[DpkgArchTable] = None, - build_env: Optional[DebBuildOptionsAndProfiles] = None, + deb_options_and_profiles: Optional[DebBuildOptionsAndProfiles] = None, ignore_errors: bool = False, ) -> None: if dpkg_architecture_variables is None: dpkg_architecture_variables = dpkg_architecture_table() if dpkg_arch_query_table is None: dpkg_arch_query_table = DpkgArchTable.load_arch_table() - if build_env is None: - build_env = DebBuildOptionsAndProfiles.instance() + if deb_options_and_profiles is None: + deb_options_and_profiles = DebBuildOptionsAndProfiles.instance() # If no selection option is set, then all packages are acted on (except the # excluded ones) @@ -66,7 +66,7 @@ class DctrlParser: self.select_arch_any = select_arch_any self.dpkg_architecture_variables = dpkg_architecture_variables self.dpkg_arch_query_table = dpkg_arch_query_table - self.build_env = build_env + self.deb_options_and_profiles = deb_options_and_profiles self.ignore_errors = ignore_errors @overload @@ -138,7 +138,7 @@ class DctrlParser: self.select_arch_any, self.dpkg_architecture_variables, self.dpkg_arch_query_table, - self.build_env, + self.deb_options_and_profiles, i, ) ) diff --git a/src/debputy/path_matcher.py b/src/debputy/path_matcher.py index 2917b14..a7b8356 100644 --- a/src/debputy/path_matcher.py +++ b/src/debputy/path_matcher.py @@ -161,7 +161,7 @@ class MatchRule: _error( f'The pattern "{path_or_glob}" (defined in {definition_source}) looks like it contains a' f' brace expansion (such as "{{a,b}}" or "{{a..b}}"). Brace expansions are not supported.' - " If you wanted to match the literal path a brace in it, please use a substitution to insert" + " If you wanted to match the literal path with a brace in it, please use a substitution to insert" f' the opening brace. As an example: "{replacement}"' ) diff --git a/src/debputy/plugin/api/feature_set.py b/src/debputy/plugin/api/feature_set.py index a56f37b..30d79be 100644 --- a/src/debputy/plugin/api/feature_set.py +++ b/src/debputy/plugin/api/feature_set.py @@ -1,29 +1,24 @@ import dataclasses -import textwrap -from typing import Dict, List, Tuple, Sequence, Any +from typing import Dict, List, Tuple, Sequence, Any, Optional, Type -from debputy import DEBPUTY_DOC_ROOT_DIR from debputy.manifest_parser.declarative_parser import ParserGenerator -from debputy.plugin.api import reference_documentation from debputy.plugin.api.impl_types import ( DebputyPluginMetadata, PackagerProvidedFileClassSpec, MetadataOrMaintscriptDetector, - TTP, - DispatchingTableParser, - TP, - SUPPORTED_DISPATCHABLE_TABLE_PARSERS, - DispatchingObjectParser, - SUPPORTED_DISPATCHABLE_OBJECT_PARSERS, PluginProvidedManifestVariable, PluginProvidedPackageProcessor, PluginProvidedDiscardRule, ServiceManagerDetails, PluginProvidedKnownPackagingFile, PluginProvidedTypeMapping, - OPARSER_PACKAGES, - OPARSER_PACKAGES_ROOT, + PluginProvidedBuildSystemAutoDetection, +) +from debputy.plugin.api.parser_tables import ( + SUPPORTED_DISPATCHABLE_OBJECT_PARSERS, + SUPPORTED_DISPATCHABLE_TABLE_PARSERS, ) +from debputy.plugin.debputy.to_be_api_types import BuildSystemRule def _initialize_parser_generator() -> ParserGenerator: @@ -70,6 +65,9 @@ class PluginProvidedFeatureSet: manifest_parser_generator: ParserGenerator = dataclasses.field( default_factory=_initialize_parser_generator ) + auto_detectable_build_systems: Dict[ + Type[BuildSystemRule], PluginProvidedBuildSystemAutoDetection + ] = dataclasses.field(default_factory=dict) def package_processors_in_order(self) -> Sequence[PluginProvidedPackageProcessor]: order = [] diff --git a/src/debputy/plugin/api/impl.py b/src/debputy/plugin/api/impl.py index b0674fb..c2f03d0 100644 --- a/src/debputy/plugin/api/impl.py +++ b/src/debputy/plugin/api/impl.py @@ -31,7 +31,8 @@ from typing import ( Any, Literal, Container, - get_args, + TYPE_CHECKING, + is_typeddict, ) from debputy import DEBPUTY_DOC_ROOT_DIR @@ -43,16 +44,18 @@ from debputy.exceptions import ( PluginInitializationError, PluginAPIViolationError, PluginNotFoundError, + PluginIncorrectRegistrationError, ) from debputy.maintscript_snippet import ( STD_CONTROL_SCRIPTS, MaintscriptSnippetContainer, MaintscriptSnippet, ) -from debputy.manifest_parser.base_types import TypeMapping from debputy.manifest_parser.exceptions import ManifestParseException from debputy.manifest_parser.parser_data import ParserContextData +from debputy.manifest_parser.tagging_types import TypeMapping from debputy.manifest_parser.util import AttributePath +from debputy.manifest_parser.util import resolve_package_type_selectors from debputy.plugin.api.feature_set import PluginProvidedFeatureSet from debputy.plugin.api.impl_types import ( DebputyPluginMetadata, @@ -70,11 +73,12 @@ from debputy.plugin.api.impl_types import ( AutomaticDiscardRuleExample, PPFFormatParam, ServiceManagerDetails, - resolve_package_type_selectors, KnownPackagingFileInfo, PluginProvidedKnownPackagingFile, InstallPatternDHCompatRule, PluginProvidedTypeMapping, + PluginProvidedBuildSystemAutoDetection, + BSR, ) from debputy.plugin.api.plugin_parser import ( PLUGIN_METADATA_PARSER, @@ -108,6 +112,21 @@ from debputy.plugin.api.spec import ( packager_provided_file_reference_documentation, TypeMappingDocumentation, DebputyIntegrationMode, + reference_documentation, + _DEBPUTY_DISPATCH_METADATA_ATTR_NAME, + BuildSystemManifestRuleMetadata, +) +from debputy.plugin.api.std_docs import _STD_ATTR_DOCS +from debputy.plugin.debputy.to_be_api_types import ( + BuildSystemRule, + BuildRuleParsedFormat, + BSPF, + debputy_build_system, +) +from debputy.plugin.plugin_state import ( + run_in_context_of_plugin, + run_in_context_of_plugin_wrap_errors, + wrap_plugin_code, ) from debputy.substitution import ( Substitution, @@ -123,6 +142,9 @@ from debputy.util import ( _warn, ) +if TYPE_CHECKING: + from debputy.highlevel_manifest import HighLevelManifest + PLUGIN_TEST_SUFFIX = re.compile(r"_(?:t|test|check)(?:_([a-z0-9_]+))?[.]py$") @@ -362,7 +384,7 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): all_detectors[self._plugin_name].append( MetadataOrMaintscriptDetector( detector_id=auto_detector_id, - detector=auto_detector, + detector=wrap_plugin_code(self._plugin_name, auto_detector), plugin_metadata=self._plugin_metadata, applies_to_package_types=package_types, enabled=True, @@ -575,7 +597,7 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): package_processors[processor_key] = PluginProvidedPackageProcessor( processor_id, resolve_package_type_selectors(package_type), - processor, + wrap_plugin_code(self._plugin_name, processor), frozenset(dependencies), self._plugin_metadata, ) @@ -704,8 +726,8 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): ) service_managers[service_manager] = ServiceManagerDetails( service_manager, - detector, - integrator, + wrap_plugin_code(self._plugin_name, detector), + wrap_plugin_code(self._plugin_name, integrator), self._plugin_metadata, ) @@ -776,7 +798,7 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): dispatching_parser = parser_generator.dispatchable_table_parsers[rule_type] dispatching_parser.register_keyword( rule_name, - handler, + wrap_plugin_code(self._plugin_name, handler), self._plugin_metadata, inline_reference_documentation=inline_reference_documentation, ) @@ -820,6 +842,10 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): ) parent_dispatcher = dispatchable_object_parsers[rule_type] child_dispatcher = dispatchable_object_parsers[object_parser_key] + + if on_end_parse_step is not None: + on_end_parse_step = wrap_plugin_code(self._plugin_name, on_end_parse_step) + parent_dispatcher.register_child_parser( rule_name, child_dispatcher, @@ -838,7 +864,7 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): def pluggable_manifest_rule( self, rule_type: Union[TTP, str], - rule_name: Union[str, List[str]], + rule_name: Union[str, Sequence[str]], parsed_format: Type[PF], handler: DIPHandler, *, @@ -847,8 +873,15 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): expected_debputy_integration_mode: Optional[ Container[DebputyIntegrationMode] ] = None, + apply_standard_attribute_documentation: bool = False, ) -> None: + # When unrestricted this, consider which types will be unrestricted self._restricted_api() + if apply_standard_attribute_documentation and sys.version_info < (3, 12): + _error( + f"The plugin {self._plugin_metadata.plugin_name} requires python 3.12 due to" + f" its use of apply_standard_attribute_documentation" + ) feature_set = self._feature_set parser_generator = feature_set.manifest_parser_generator if isinstance(rule_type, str): @@ -870,16 +903,22 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): ) dispatching_parser = parser_generator.dispatchable_table_parsers[rule_type] + if apply_standard_attribute_documentation: + docs = _STD_ATTR_DOCS + else: + docs = None + parser = feature_set.manifest_parser_generator.generate_parser( parsed_format, source_content=source_format, inline_reference_documentation=inline_reference_documentation, expected_debputy_integration_mode=expected_debputy_integration_mode, + automatic_docs=docs, ) dispatching_parser.register_parser( rule_name, parser, - handler, + wrap_plugin_code(self._plugin_name, handler), self._plugin_metadata, ) @@ -890,6 +929,108 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): self._unloaders.append(_unload) + def register_build_system( + self, + build_system_definition: type[BSPF], + ) -> None: + self._restricted_api() + if not is_typeddict(build_system_definition): + raise PluginInitializationError( + f"Expected build_system_definition to be a subclass of {BuildRuleParsedFormat.__name__}," + f" but got {build_system_definition.__name__} instead" + ) + metadata = getattr( + build_system_definition, + _DEBPUTY_DISPATCH_METADATA_ATTR_NAME, + None, + ) + if not isinstance(metadata, BuildSystemManifestRuleMetadata): + raise PluginIncorrectRegistrationError( + f"The {build_system_definition.__qualname__} type should have been annotated with" + f" @{debputy_build_system.__name__}." + ) + assert len(metadata.manifest_keywords) == 1 + build_system_impl = metadata.build_system_impl + assert build_system_impl is not None + manifest_keyword = next(iter(metadata.manifest_keywords)) + self.pluggable_manifest_rule( + metadata.dispatched_type, + metadata.manifest_keywords, + build_system_definition, + # pluggable_manifest_rule does the wrapping + metadata.unwrapped_constructor, + source_format=metadata.source_format, + ) + self._auto_detectable_build_system( + manifest_keyword, + build_system_impl, + constructor=wrap_plugin_code( + self._plugin_name, + build_system_impl, + ), + shadowing_build_systems_when_active=metadata.auto_detection_shadow_build_systems, + ) + + def _auto_detectable_build_system( + self, + manifest_keyword: str, + rule_type: type[BSR], + *, + shadowing_build_systems_when_active: FrozenSet[str] = frozenset(), + constructor: Optional[ + Callable[[BuildRuleParsedFormat, AttributePath, "HighLevelManifest"], BSR] + ] = None, + ) -> None: + self._restricted_api() + feature_set = self._feature_set + existing = feature_set.auto_detectable_build_systems.get(rule_type) + if existing is not None: + bs_name = rule_type.__class__.__name__ + if existing.plugin_metadata.plugin_name == self._plugin_name: + message = ( + f"Bug in the plugin {self._plugin_name}: It tried to register the" + f' auto-detection of the build system "{bs_name}" twice.' + ) + else: + message = ( + f"The plugins {existing.plugin_metadata.plugin_name} and {self._plugin_name}" + f' both tried to provide auto-detection of the build system "{bs_name}"' + ) + raise PluginConflictError( + message, existing.plugin_metadata, self._plugin_metadata + ) + + if constructor is None: + + def impl( + attributes: BuildRuleParsedFormat, + attribute_path: AttributePath, + manifest: "HighLevelManifest", + ) -> BSR: + return rule_type(attributes, attribute_path, manifest) + + else: + impl = constructor + + feature_set.auto_detectable_build_systems[rule_type] = ( + PluginProvidedBuildSystemAutoDetection( + manifest_keyword, + rule_type, + wrap_plugin_code(self._plugin_name, rule_type.auto_detect_build_system), + impl, + shadowing_build_systems_when_active, + self._plugin_metadata, + ) + ) + + def _unload() -> None: + try: + del feature_set.auto_detectable_build_systems[rule_type] + except KeyError: + pass + + self._unloaders.append(_unload) + def known_packaging_files( self, packaging_file_details: KnownPackagingFileInfo, @@ -981,6 +1122,7 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): message, existing.plugin_metadata, self._plugin_metadata ) parser_generator = self._feature_set.manifest_parser_generator + # TODO: Wrap the mapper in the plugin context mapped_types[target_type] = PluginProvidedTypeMapping( type_mapping, reference_documentation, self._plugin_metadata ) @@ -1437,6 +1579,10 @@ def load_plugin_features( if plugin_metadata.plugin_name not in unloadable_plugins: raise if debug_mode: + _warn( + f"The optional plugin {plugin_metadata.plugin_name} failed during load. Re-raising due" + f" to --debug/-d." + ) raise try: api.unload_plugin() @@ -1448,11 +1594,6 @@ def load_plugin_features( ) raise e from None else: - if debug_mode: - _warn( - f"The optional plugin {plugin_metadata.plugin_name} failed during load. Re-raising due" - f" to --debug/-d." - ) _warn( f"The optional plugin {plugin_metadata.plugin_name} failed during load. The plugin was" f" deactivated. Use debug mode (--debug) to show the stacktrace (the warning will become an error)" @@ -1635,7 +1776,7 @@ def _resolve_module_initializer( ) sys.modules[module_name] = mod try: - loader.exec_module(mod) + run_in_context_of_plugin(plugin_name, loader.exec_module, mod) except (Exception, GeneratorExit) as e: raise PluginInitializationError( f"Failed to load {plugin_name} (path: {module_fs_path})." @@ -1645,7 +1786,9 @@ def _resolve_module_initializer( if module is None: try: - module = importlib.import_module(module_name) + module = run_in_context_of_plugin( + plugin_name, importlib.import_module, module_name + ) except ModuleNotFoundError as e: if module_fs_path is None: raise PluginMetadataError( @@ -1660,7 +1803,12 @@ def _resolve_module_initializer( f' explicit "module" definition in {json_file_path}.' ) from e - plugin_initializer = getattr(module, plugin_initializer_name) + plugin_initializer = run_in_context_of_plugin_wrap_errors( + plugin_name, + getattr, + module, + plugin_initializer_name, + ) if plugin_initializer is None: raise PluginMetadataError( diff --git a/src/debputy/plugin/api/impl_types.py b/src/debputy/plugin/api/impl_types.py index 1a9bfdf..85beaf8 100644 --- a/src/debputy/plugin/api/impl_types.py +++ b/src/debputy/plugin/api/impl_types.py @@ -1,6 +1,5 @@ import dataclasses import os.path -import textwrap from typing import ( Optional, Callable, @@ -24,22 +23,21 @@ from typing import ( Set, Iterator, Container, + Protocol, ) from weakref import ref -from debputy import DEBPUTY_DOC_ROOT_DIR from debputy.exceptions import ( DebputyFSIsROError, PluginAPIViolationError, PluginConflictError, UnhandledOrUnexpectedErrorFromPluginError, + PluginBaseError, + PluginInitializationError, ) from debputy.filesystem_scan import as_path_def -from debputy.installations import InstallRule -from debputy.maintscript_snippet import DpkgMaintscriptHelperCommand -from debputy.manifest_conditions import ManifestCondition -from debputy.manifest_parser.base_types import DebputyParsedContent, TypeMapping from debputy.manifest_parser.exceptions import ManifestParseException +from debputy.manifest_parser.tagging_types import DebputyParsedContent, TypeMapping from debputy.manifest_parser.util import AttributePath, check_integration_mode from debputy.packages import BinaryPackage from debputy.plugin.api import ( @@ -62,15 +60,16 @@ from debputy.plugin.api.spec import ( TypeMappingDocumentation, DebputyIntegrationMode, ) +from debputy.plugin.plugin_state import ( + run_in_context_of_plugin, +) from debputy.substitution import VariableContext -from debputy.transformation_rules import TransformationRule from debputy.util import _normalize_path, package_cross_check_precheck if TYPE_CHECKING: from debputy.plugin.api.spec import ( ServiceDetector, ServiceIntegrator, - PackageTypeSelector, ) from debputy.manifest_parser.parser_data import ParserContextData from debputy.highlevel_manifest import ( @@ -78,10 +77,10 @@ if TYPE_CHECKING: PackageTransformationDefinition, BinaryPackageData, ) - - -_PACKAGE_TYPE_DEB_ONLY = frozenset(["deb"]) -_ALL_PACKAGE_TYPES = frozenset(["deb", "udeb"]) + from debputy.plugin.debputy.to_be_api_types import ( + BuildSystemRule, + BuildRuleParsedFormat, + ) TD = TypeVar("TD", bound="Union[DebputyParsedContent, List[DebputyParsedContent]]") @@ -89,26 +88,12 @@ PF = TypeVar("PF") SF = TypeVar("SF") TP = TypeVar("TP") TTP = Type[TP] +BSR = TypeVar("BSR", bound="BuildSystemRule") DIPKWHandler = Callable[[str, AttributePath, "ParserContextData"], TP] DIPHandler = Callable[[str, PF, AttributePath, "ParserContextData"], TP] -def resolve_package_type_selectors( - package_type: "PackageTypeSelector", -) -> FrozenSet[str]: - if package_type is _ALL_PACKAGE_TYPES or package_type is _PACKAGE_TYPE_DEB_ONLY: - return cast("FrozenSet[str]", package_type) - if isinstance(package_type, str): - return ( - _PACKAGE_TYPE_DEB_ONLY - if package_type == "deb" - else frozenset([package_type]) - ) - else: - return frozenset(package_type) - - @dataclasses.dataclass(slots=True) class DebputyPluginMetadata: plugin_name: str @@ -143,7 +128,17 @@ class DebputyPluginMetadata: def load_plugin(self) -> None: plugin_loader = self.plugin_loader assert plugin_loader is not None - self.plugin_initializer = plugin_loader() + try: + self.plugin_initializer = run_in_context_of_plugin( + self.plugin_name, + plugin_loader, + ) + except PluginBaseError: + raise + except Exception as e: + raise PluginInitializationError( + f"Initialization of {self.plugin_name} failed due to its initializer raising an exception" + ) from e assert self.plugin_initializer is not None @@ -270,12 +265,10 @@ class MetadataOrMaintscriptDetector: " this stage (file system layout is committed and the attempted changes" " would be lost)." ) from e - except (ChildProcessError, RuntimeError, AttributeError) as e: - nv = f"{self.plugin_metadata.plugin_name}" - raise UnhandledOrUnexpectedErrorFromPluginError( - f"The plugin {nv} threw an unhandled or unexpected exception from its metadata" - f" detector with id {self.detector_id}." - ) from e + except UnhandledOrUnexpectedErrorFromPluginError as e: + e.add_note( + f"The exception was raised by the detector with the ID: {self.detector_id}" + ) class DeclarativeInputParser(Generic[TD]): @@ -605,7 +598,7 @@ class DispatchingObjectParser( ) remaining_valid_attribute_names = ", ".join(remaining_valid_attributes) raise ManifestParseException( - f'The attribute "{first_key}" is not applicable at {attribute_path.path}(with the current set' + f'The attribute "{first_key}" is not applicable at {attribute_path.path} (with the current set' " of plugins). Possible attributes available (and not already used) are:" f" {remaining_valid_attribute_names}.{doc_ref}" ) @@ -615,7 +608,10 @@ class DispatchingObjectParser( if value is None: if isinstance(provided_parser.parser, DispatchingObjectParser): provided_parser.handler( - key, {}, attribute_path[key], parser_context + key, + {}, + attribute_path[key], + parser_context, ) continue value_path = attribute_path[key] @@ -774,64 +770,6 @@ class DeclarativeValuelessKeywordInputParser(DeclarativeInputParser[None]): ) -SUPPORTED_DISPATCHABLE_TABLE_PARSERS = { - InstallRule: "installations", - TransformationRule: "packages.{{PACKAGE}}.transformations", - DpkgMaintscriptHelperCommand: "packages.{{PACKAGE}}.conffile-management", - ManifestCondition: "*.when", -} - -OPARSER_MANIFEST_ROOT = "<ROOT>" -OPARSER_PACKAGES_ROOT = "packages" -OPARSER_PACKAGES = "packages.{{PACKAGE}}" -OPARSER_MANIFEST_DEFINITIONS = "definitions" - -SUPPORTED_DISPATCHABLE_OBJECT_PARSERS = { - OPARSER_MANIFEST_ROOT: reference_documentation( - reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md", - ), - OPARSER_MANIFEST_DEFINITIONS: reference_documentation( - title="Packager provided definitions", - description="Reusable packager provided definitions such as manifest variables.", - reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#packager-provided-definitions", - ), - OPARSER_PACKAGES: reference_documentation( - title="Binary package rules", - description=textwrap.dedent( - """\ - Inside the manifest, the `packages` mapping can be used to define requests for the binary packages - you want `debputy` to produce. Each key inside `packages` must be the name of a binary package - defined in `debian/control`. The value is a dictionary defining which features that `debputy` - should apply to that binary package. An example could be: - - packages: - foo: - transformations: - - create-symlink: - path: usr/share/foo/my-first-symlink - target: /usr/share/bar/symlink-target - - create-symlink: - path: usr/lib/{{DEB_HOST_MULTIARCH}}/my-second-symlink - target: /usr/lib/{{DEB_HOST_MULTIARCH}}/baz/symlink-target - bar: - transformations: - - create-directories: - - some/empty/directory.d - - another/empty/integration-point.d - - create-directories: - path: a/third-empty/directory.d - owner: www-data - group: www-data - - In this case, `debputy` will create some symlinks inside the `foo` package and some directories for - the `bar` package. The following subsections define the keys you can use under each binary package. - """ - ), - reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#binary-package-rules", - ), -} - - @dataclasses.dataclass(slots=True) class PluginProvidedManifestVariable: plugin_metadata: DebputyPluginMetadata @@ -1214,6 +1152,11 @@ class PluginProvidedKnownPackagingFile: plugin_metadata: DebputyPluginMetadata +class BuildSystemAutoDetector(Protocol): + + def __call__(self, source_root: VirtualPath, *args: Any, **kwargs: Any) -> bool: ... + + @dataclasses.dataclass(slots=True, frozen=True) class PluginProvidedTypeMapping: mapped_type: TypeMapping[Any, Any] @@ -1221,6 +1164,19 @@ class PluginProvidedTypeMapping: plugin_metadata: DebputyPluginMetadata +@dataclasses.dataclass(slots=True, frozen=True) +class PluginProvidedBuildSystemAutoDetection(Generic[BSR]): + manifest_keyword: str + build_system_rule_type: Type[BSR] + detector: BuildSystemAutoDetector + constructor: Callable[ + ["BuildRuleParsedFormat", AttributePath, "HighLevelManifest"], + BSR, + ] + auto_detection_shadow_build_systems: FrozenSet[str] + plugin_metadata: DebputyPluginMetadata + + class PackageDataTable: def __init__(self, package_data_table: Mapping[str, "BinaryPackageData"]) -> None: self._package_data_table = package_data_table diff --git a/src/debputy/plugin/api/parser_tables.py b/src/debputy/plugin/api/parser_tables.py new file mode 100644 index 0000000..37d3e37 --- /dev/null +++ b/src/debputy/plugin/api/parser_tables.py @@ -0,0 +1,67 @@ +import textwrap + +from debputy import DEBPUTY_DOC_ROOT_DIR +from debputy.installations import InstallRule +from debputy.maintscript_snippet import DpkgMaintscriptHelperCommand +from debputy.manifest_conditions import ManifestCondition +from debputy.plugin.api import reference_documentation +from debputy.plugin.debputy.to_be_api_types import BuildRule +from debputy.transformation_rules import TransformationRule + +SUPPORTED_DISPATCHABLE_TABLE_PARSERS = { + InstallRule: "installations", + TransformationRule: "packages.{{PACKAGE}}.transformations", + DpkgMaintscriptHelperCommand: "packages.{{PACKAGE}}.conffile-management", + ManifestCondition: "*.when", + BuildRule: "builds", +} + +OPARSER_MANIFEST_ROOT = "<ROOT>" +OPARSER_PACKAGES_ROOT = "packages" +OPARSER_PACKAGES = "packages.{{PACKAGE}}" +OPARSER_MANIFEST_DEFINITIONS = "definitions" + +SUPPORTED_DISPATCHABLE_OBJECT_PARSERS = { + OPARSER_MANIFEST_ROOT: reference_documentation( + reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md", + ), + OPARSER_MANIFEST_DEFINITIONS: reference_documentation( + title="Packager provided definitions", + description="Reusable packager provided definitions such as manifest variables.", + reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#packager-provided-definitions", + ), + OPARSER_PACKAGES: reference_documentation( + title="Binary package rules", + description=textwrap.dedent( + """\ + Inside the manifest, the `packages` mapping can be used to define requests for the binary packages + you want `debputy` to produce. Each key inside `packages` must be the name of a binary package + defined in `debian/control`. The value is a dictionary defining which features that `debputy` + should apply to that binary package. An example could be: + + packages: + foo: + transformations: + - create-symlink: + path: usr/share/foo/my-first-symlink + target: /usr/share/bar/symlink-target + - create-symlink: + path: usr/lib/{{DEB_HOST_MULTIARCH}}/my-second-symlink + target: /usr/lib/{{DEB_HOST_MULTIARCH}}/baz/symlink-target + bar: + transformations: + - create-directories: + - some/empty/directory.d + - another/empty/integration-point.d + - create-directories: + path: a/third-empty/directory.d + owner: www-data + group: www-data + + In this case, `debputy` will create some symlinks inside the `foo` package and some directories for + the `bar` package. The following subsections define the keys you can use under each binary package. + """ + ), + reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#binary-package-rules", + ), +} diff --git a/src/debputy/plugin/api/plugin_parser.py b/src/debputy/plugin/api/plugin_parser.py index dd5c0d0..0e7954b 100644 --- a/src/debputy/plugin/api/plugin_parser.py +++ b/src/debputy/plugin/api/plugin_parser.py @@ -1,10 +1,10 @@ from typing import NotRequired, List, Any, TypedDict -from debputy.manifest_parser.base_types import ( +from debputy.manifest_parser.tagging_types import ( DebputyParsedContent, - OctalMode, TypeMapping, ) +from debputy.manifest_parser.base_types import OctalMode from debputy.manifest_parser.declarative_parser import ParserGenerator from debputy.plugin.api.impl_types import KnownPackagingFileInfo diff --git a/src/debputy/plugin/api/spec.py b/src/debputy/plugin/api/spec.py index b7f19c0..30308f9 100644 --- a/src/debputy/plugin/api/spec.py +++ b/src/debputy/plugin/api/spec.py @@ -25,6 +25,7 @@ from typing import ( Tuple, get_args, Container, + final, ) from debian.substvars import Substvars @@ -32,17 +33,23 @@ from debian.substvars import Substvars from debputy import util from debputy.exceptions import TestPathWithNonExistentFSPathError, PureVirtualPathError from debputy.interpreter import Interpreter, extract_shebang_interpreter_from_file +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.manifest_parser.util import parse_symbolic_mode from debputy.packages import BinaryPackage from debputy.types import S if TYPE_CHECKING: + from debputy.plugin.debputy.to_be_api_types import BuildRule, BSR, BuildSystemRule + from debputy.plugin.api.impl_types import DIPHandler from debputy.manifest_parser.base_types import ( StaticFileSystemOwner, StaticFileSystemGroup, ) +DP = TypeVar("DP", bound=DebputyDispatchableType) + + PluginInitializationEntryPoint = Callable[["DebputyPluginInitializer"], None] MetadataAutoDetector = Callable[ ["VirtualPath", "BinaryCtrlAccessor", "PackageProcessingContext"], None @@ -86,17 +93,20 @@ DebputyIntegrationMode = Literal[ "dh-sequence-zz-debputy-rrr", ] +INTEGRATION_MODE_FULL: DebputyIntegrationMode = "full" INTEGRATION_MODE_DH_DEBPUTY_RRR: DebputyIntegrationMode = "dh-sequence-zz-debputy-rrr" INTEGRATION_MODE_DH_DEBPUTY: DebputyIntegrationMode = "dh-sequence-zz-debputy" ALL_DEBPUTY_INTEGRATION_MODES: FrozenSet[DebputyIntegrationMode] = frozenset( get_args(DebputyIntegrationMode) ) +_DEBPUTY_DISPATCH_METADATA_ATTR_NAME = "_debputy_dispatch_metadata" + def only_integrations( *integrations: DebputyIntegrationMode, ) -> Container[DebputyIntegrationMode]: - return frozenset(*integrations) + return frozenset(integrations) def not_integrations( @@ -212,6 +222,27 @@ class PathDef: materialized_content: Optional[str] = None +@dataclasses.dataclass(slots=True, frozen=True) +class DispatchablePluggableManifestRuleMetadata(Generic[DP]): + """NOT PUBLIC API (used internally by part of the public API)""" + + manifest_keywords: Sequence[str] + dispatched_type: Type[DP] + unwrapped_constructor: "DIPHandler" + expected_debputy_integration_mode: Optional[Container[DebputyIntegrationMode]] = ( + None + ) + online_reference_documentation: Optional["ParserDocumentation"] = None + apply_standard_attribute_documentation: bool = False + source_format: Optional[Any] = None + + +@dataclasses.dataclass(slots=True, frozen=True) +class BuildSystemManifestRuleMetadata(DispatchablePluggableManifestRuleMetadata): + build_system_impl: Optional[Type["BuildSystemRule"]] = (None,) + auto_detection_shadow_build_systems: FrozenSet[str] = frozenset() + + def virtual_path_def( path_name: str, /, @@ -1507,6 +1538,16 @@ class ParserAttributeDocumentation: attributes: FrozenSet[str] description: Optional[str] + @property + def is_hidden(self) -> bool: + return False + + +@final +@dataclasses.dataclass(slots=True, frozen=True) +class StandardParserAttributeDocumentation(ParserAttributeDocumentation): + sort_category: int = 0 + def undocumented_attr(attr: str) -> ParserAttributeDocumentation: """Describe an attribute as undocumented @@ -1514,6 +1555,8 @@ def undocumented_attr(attr: str) -> ParserAttributeDocumentation: If you for some reason do not want to document a particular attribute, you can mark it as undocumented. This is required if you are only documenting a subset of the attributes, because `debputy` assumes any omission to be a mistake. + + :param attr: Name of the attribute """ return ParserAttributeDocumentation( frozenset({attr}), diff --git a/src/debputy/plugin/api/std_docs.py b/src/debputy/plugin/api/std_docs.py new file mode 100644 index 0000000..f07c307 --- /dev/null +++ b/src/debputy/plugin/api/std_docs.py @@ -0,0 +1,142 @@ +import textwrap +from typing import Type, Sequence, Mapping, Container, Iterable, Any + +from debputy.manifest_parser.base_types import DebputyParsedContentStandardConditional +from debputy.manifest_parser.tagging_types import DebputyParsedContent +from debputy.plugin.api.spec import ( + ParserAttributeDocumentation, + StandardParserAttributeDocumentation, +) +from debputy.plugin.debputy.to_be_api_types import ( + OptionalInstallDirectly, + OptionalInSourceBuild, + OptionalBuildDirectory, + BuildRuleParsedFormat, +) + +_STD_ATTR_DOCS: Mapping[ + Type[DebputyParsedContent], + Sequence[ParserAttributeDocumentation], +] = { + BuildRuleParsedFormat: [ + StandardParserAttributeDocumentation( + frozenset(["name"]), + textwrap.dedent( + """\ + The name of the build step. + + The name is used for multiple things, such as: + 1) If you ever need to reference the build elsewhere, the name will be used. + 2) When `debputy` references the build in log output and error, it will use the name. + 3) It is used as defaults for when `debputy` derives build and `DESTDIR` directories + for the build. + """ + ), + # Put in top, + sort_category=-1000, + ), + StandardParserAttributeDocumentation( + frozenset(["for_packages"]), + textwrap.dedent( + """\ + Which package or packages this build step applies to. + + Either a package name or a list of package names. + """ + ), + ), + StandardParserAttributeDocumentation( + frozenset(["environment"]), + textwrap.dedent( + """\ + Specify that this build step uses the named environment + + If omitted, the default environment will be used. If no default environment is present, + then this option is mandatory. + """ + ), + ), + ], + OptionalBuildDirectory: [ + StandardParserAttributeDocumentation( + frozenset(["build_directory"]), + textwrap.dedent( + """\ + The build directory to use for the build. + + By default, `debputy` will derive a build directory automatically if the build system needs + it. However, it can be useful if you need to reference the directory name from other parts + of the manifest or want a "better" name than `debputy` comes up with. + """ + ), + ), + ], + OptionalInSourceBuild: [ + StandardParserAttributeDocumentation( + frozenset(["perform_in_source_build"]), + textwrap.dedent( + """\ + Whether the build system should use "in source" or "out of source" build. + + This is mostly useful for forcing "in source" builds for build systems that default to + "out of source" builds like `autoconf`. + + The default depends on the build system and the value of the `build-directory` attribute + (if supported by the build system). + """ + ), + # Late + sort_category=500, + ), + ], + OptionalInstallDirectly: [ + StandardParserAttributeDocumentation( + frozenset(["install_directly_to_package"]), + textwrap.dedent( + """\ + Whether the build system should install all upstream content directly into the package. + + This option is mostly useful for disabling said behavior by setting the attribute to `false`. + The attribute conditionally defaults to `true` when the build only applies to one package. + If explicitly set to `true`, then this build step must apply to exactly one package (usually + implying that `for` is set to that package when the source builds multiple packages). + + When `true`, this behaves similar to `dh_auto_install --destdir=debian/PACKAGE`. + """ + ), + ), + ], + DebputyParsedContentStandardConditional: [ + StandardParserAttributeDocumentation( + frozenset(["when"]), + textwrap.dedent( + """\ + A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). + + The conditional will disable the entire rule when the conditional evaluates to false. + """ + ), + # Last + sort_category=9999, + ), + ], +} + + +def docs_from( + *ts: Any, + exclude_attributes: Container[str] = frozenset(), +) -> Iterable[ParserAttributeDocumentation]: + """Provide standard attribute documentation from existing types + + This is a work-around for `apply_standard_attribute_documentation` requiring python3.12. + If you can assume python3.12, use `apply_standard_attribute_documentation` instead. + """ + for t in ts: + attrs = _STD_ATTR_DOCS.get(t) + if attrs is None: + raise ValueError(f"No standard documentation for {str(t)}") + for attr in attrs: + if any(a in exclude_attributes for a in attrs): + continue + yield attr diff --git a/src/debputy/plugin/debputy/binary_package_rules.py b/src/debputy/plugin/debputy/binary_package_rules.py index 98da763..45547b9 100644 --- a/src/debputy/plugin/debputy/binary_package_rules.py +++ b/src/debputy/plugin/debputy/binary_package_rules.py @@ -17,14 +17,10 @@ from typing import ( from debputy import DEBPUTY_DOC_ROOT_DIR from debputy.maintscript_snippet import DpkgMaintscriptHelperCommand, MaintscriptSnippet -from debputy.manifest_parser.base_types import ( - DebputyParsedContent, - FileSystemExactMatchRule, -) -from debputy.manifest_parser.declarative_parser import ( - DebputyParseHint, - ParserGenerator, -) +from debputy.manifest_parser.base_types import FileSystemExactMatchRule +from debputy.manifest_parser.tagging_types import DebputyParsedContent +from debputy.manifest_parser.parse_hints import DebputyParseHint +from debputy.manifest_parser.declarative_parser import ParserGenerator from debputy.manifest_parser.exceptions import ManifestParseException from debputy.manifest_parser.parser_data import ParserContextData from debputy.manifest_parser.util import AttributePath @@ -34,7 +30,7 @@ from debputy.plugin.api.impl import ( DebputyPluginInitializerProvider, ServiceDefinitionImpl, ) -from debputy.plugin.api.impl_types import OPARSER_PACKAGES +from debputy.plugin.api.parser_tables import OPARSER_PACKAGES from debputy.plugin.api.spec import ( ServiceUpgradeRule, ServiceDefinition, diff --git a/src/debputy/plugin/debputy/build_system_rules.py b/src/debputy/plugin/debputy/build_system_rules.py new file mode 100644 index 0000000..b7ee898 --- /dev/null +++ b/src/debputy/plugin/debputy/build_system_rules.py @@ -0,0 +1,2319 @@ +import dataclasses +import json +import os +import subprocess +import textwrap +from typing import ( + NotRequired, + TypedDict, + Self, + cast, + Dict, + Mapping, + Sequence, + MutableMapping, + Iterable, + Container, + List, + Tuple, + Union, + Optional, + TYPE_CHECKING, + Literal, +) + +from debian.debian_support import Version + +from debputy import DEBPUTY_DOC_ROOT_DIR +from debputy._manifest_constants import MK_BUILDS +from debputy.manifest_parser.base_types import ( + BuildEnvironmentDefinition, + DebputyParsedContentStandardConditional, + FileSystemExactMatchRule, +) +from debputy.manifest_parser.exceptions import ( + ManifestParseException, + ManifestInvalidUserDataException, +) +from debputy.manifest_parser.parser_data import ParserContextData +from debputy.manifest_parser.util import AttributePath +from debputy.plugin.api import reference_documentation +from debputy.plugin.api.impl import ( + DebputyPluginInitializerProvider, +) +from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT +from debputy.plugin.api.spec import ( + documented_attr, + INTEGRATION_MODE_FULL, + only_integrations, + VirtualPath, +) +from debputy.plugin.api.std_docs import docs_from +from debputy.plugin.debputy.to_be_api_types import ( + BuildRule, + StepBasedBuildSystemRule, + OptionalInstallDirectly, + BuildSystemCharacteristics, + OptionalBuildDirectory, + OptionalInSourceBuild, + MakefileSupport, + BuildRuleParsedFormat, + debputy_build_system, + CleanHelper, + NinjaBuildSupport, +) +from debputy.types import EnvironmentModification +from debputy.util import ( + _warn, + run_build_system_command, + _error, + PerlConfigVars, + resolve_perl_config, + generated_content_dir, +) + +if TYPE_CHECKING: + from debputy.build_support.build_context import BuildContext + from debputy.highlevel_manifest import HighLevelManifest + + +PERL_CMD = "perl" + + +def register_build_system_rules(api: DebputyPluginInitializerProvider) -> None: + register_build_keywords(api) + register_build_rules(api) + + +def register_build_keywords(api: DebputyPluginInitializerProvider) -> None: + + api.pluggable_manifest_rule( + OPARSER_MANIFEST_ROOT, + "build-environments", + List[NamedEnvironmentSourceFormat], + _parse_build_environments, + expected_debputy_integration_mode=only_integrations(INTEGRATION_MODE_FULL), + inline_reference_documentation=reference_documentation( + title="Build Environments (`build-environments`)", + description=textwrap.dedent( + """\ + Define named environments to set the environment for any build commands that needs + a non-default environment. + + The environment definitions can be used to tweak the environment variables used by the + build commands. An example: + + build-environments: + - name: custom-env + set: + ENV_VAR: foo + ANOTHER_ENV_VAR: bar + builds: + - autoconf: + environment: custom-env + + The environment definition has multiple attributes for setting environment variables + which determines when the definition is applied. The resulting environment is the + result of the following order of operations. + + 1. The environment `debputy` received from its parent process. + 2. Apply all the variable definitions from `set` (if the attribute is present) + 3. Apply all computed variables (such as variables from `dpkg-buildflags`). + 4. Apply all the variable definitions from `override` (if the attribute is present) + 5. Remove all variables listed in `unset` (if the attribute is present). + + Accordingly, both `override` and `unset` will overrule any computed variables while + `set` will be overruled by any computed variables. + + Note that these variables are not available via manifest substitution (they are only + visible to build commands). They are only available to build commands. + """ + ), + attributes=[ + documented_attr( + "name", + textwrap.dedent( + """\ + The name of the environment + + The name is used to reference the environment from build rules. + """ + ), + ), + documented_attr( + "set", + textwrap.dedent( + """\ + A mapping of environment variables to be set. + + Note these environment variables are set before computed variables (such + as `dpkg-buildflags`) are provided. They can affect the content of the + computed variables, but they cannot overrule them. If you need to overrule + a computed variable, please use `override` instead. + """ + ), + ), + documented_attr( + "override", + textwrap.dedent( + """\ + A mapping of environment variables to set. + + Similar to `set`, but it can overrule computed variables like those from + `dpkg-buildflags`. + """ + ), + ), + documented_attr( + "unset", + textwrap.dedent( + """\ + A list of environment variables to unset. + + Any environment variable named here will be unset. No warnings or errors + will be raised if a given variable was not set. + """ + ), + ), + ], + reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#build-environment-build-environment", + ), + ) + api.pluggable_manifest_rule( + OPARSER_MANIFEST_ROOT, + "default-build-environment", + EnvironmentSourceFormat, + _parse_default_environment, + expected_debputy_integration_mode=only_integrations(INTEGRATION_MODE_FULL), + inline_reference_documentation=reference_documentation( + title="Default Build Environment (`default-build-environment`)", + description=textwrap.dedent( + """\ + Define the environment variables used in all build commands that uses the default + environment. + + The environment definition can be used to tweak the environment variables used by the + build commands. An example: + + default-build-environment: + set: + ENV_VAR: foo + ANOTHER_ENV_VAR: bar + + The environment definition has multiple attributes for setting environment variables + which determines when the definition is applied. The resulting environment is the + result of the following order of operations. + + 1. The environment `debputy` received from its parent process. + 2. Apply all the variable definitions from `set` (if the attribute is present) + 3. Apply all computed variables (such as variables from `dpkg-buildflags`). + 4. Apply all the variable definitions from `override` (if the attribute is present) + 5. Remove all variables listed in `unset` (if the attribute is present). + + Accordingly, both `override` and `unset` will overrule any computed variables while + `set` will be overruled by any computed variables. + + Note that these variables are not available via manifest substitution (they are only + visible to build commands). They are only available to build commands. + """ + ), + attributes=[ + documented_attr( + "set", + textwrap.dedent( + """\ + A mapping of environment variables to be set. + + Note these environment variables are set before computed variables (such + as `dpkg-buildflags`) are provided. They can affect the content of the + computed variables, but they cannot overrule them. If you need to overrule + a computed variable, please use `override` instead. + """ + ), + ), + documented_attr( + "override", + textwrap.dedent( + """\ + A mapping of environment variables to set. + + Similar to `set`, but it can overrule computed variables like those from + `dpkg-buildflags`. + """ + ), + ), + documented_attr( + "unset", + textwrap.dedent( + """\ + A list of environment variables to unset. + + Any environment variable named here will be unset. No warnings or errors + will be raised if a given variable was not set. + """ + ), + ), + ], + reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#build-environment-build-environment", + ), + ) + api.pluggable_manifest_rule( + OPARSER_MANIFEST_ROOT, + MK_BUILDS, + List[BuildRule], + _handle_build_rules, + expected_debputy_integration_mode=only_integrations(INTEGRATION_MODE_FULL), + inline_reference_documentation=reference_documentation( + title="Build rules", + description=textwrap.dedent( + """\ + Define how to build the upstream part of the package. Usually this is done via "build systems", + which also defines the clean rules. + """ + ), + ), + ) + + +def register_build_rules(api: DebputyPluginInitializerProvider) -> None: + api.register_build_system(ParsedAutoconfBuildRuleDefinition) + api.register_build_system(ParsedMakeBuildRuleDefinition) + + api.register_build_system(ParsedPerlBuildBuildRuleDefinition) + api.register_build_system(ParsedPerlMakeMakerBuildRuleDefinition) + api.register_build_system(ParsedDebhelperBuildRuleDefinition) + + api.register_build_system(ParsedCMakeBuildRuleDefinition) + api.register_build_system(ParsedMesonBuildRuleDefinition) + + api.register_build_system(ParsedQmakeBuildRuleDefinition) + api.register_build_system(ParsedQmake6BuildRuleDefinition) + + +class EnvironmentSourceFormat(TypedDict): + set: NotRequired[Dict[str, str]] + override: NotRequired[Dict[str, str]] + unset: NotRequired[List[str]] + + +class NamedEnvironmentSourceFormat(EnvironmentSourceFormat): + name: str + + +_READ_ONLY_ENV_VARS = { + "DEB_CHECK_COMMAND": None, + "DEB_SIGN_KEYID": None, + "DEB_SIGN_KEYFILE": None, + "DEB_BUILD_OPTIONS": "DEB_BUILD_MAINT_OPTIONS", + "DEB_BUILD_PROFILES": None, + "DEB_RULES_REQUIRES_ROOT": None, + "DEB_GAIN_ROOT_COMMAND": None, + "DH_EXTRA_ADDONS": None, + "DH_NO_ACT": None, +} + + +def _check_variables( + env_vars: Iterable[str], + attribute_path: AttributePath, +) -> None: + for env_var in env_vars: + if env_var not in _READ_ONLY_ENV_VARS: + continue + alt = _READ_ONLY_ENV_VARS.get(env_var) + var_path = attribute_path[env_var].path_key_lc + if alt is None: + raise ManifestParseException( + f"The variable {env_var} cannot be modified by the manifest. This restriction is generally" + f" because the build should not touch those variables or changing them have no effect" + f" (since the consumer will not see the change). The problematic definition was {var_path}" + ) + else: + raise ManifestParseException( + f"The variable {env_var} cannot be modified by the manifest. This restriction is generally" + f" because the build should not touch those variables or changing them have no effect" + f" (since the consumer will not see the change). Depending on what you are trying to" + f' accomplish, the variable "{alt}" might be a suitable alternative.' + f" The problematic definition was {var_path}" + ) + + +def _no_overlap( + lhs: Iterable[Union[str, Tuple[int, str]]], + rhs: Container[str], + lhs_key: str, + rhs_key: str, + redundant_key: str, + attribute_path: AttributePath, +) -> None: + for kt in lhs: + if isinstance(kt, tuple): + lhs_path_key, var = kt + else: + lhs_path_key = var = kt + if var not in rhs: + continue + lhs_path = attribute_path[lhs_key][lhs_path_key].path_key_lc + rhs_path = attribute_path[rhs_key][var].path_key_lc + r_path = lhs_path if redundant_key == rhs_key else rhs_path + raise ManifestParseException( + f"The environment variable {var} was declared in {lhs_path} and {rhs_path}." + f" Due to how the variables are applied, the definition in {r_path} is redundant" + f" and can effectively be removed. Please review the manifest and remove one of" + f" the two definitions." + ) + + +@dataclasses.dataclass(slots=True, frozen=True) +class ManifestProvidedBuildEnvironment(BuildEnvironmentDefinition): + + name: str + is_default: bool + attribute_path: AttributePath + parser_context: ParserContextData + + set_vars: Mapping[str, str] + override_vars: Mapping[str, str] + unset_vars: Sequence[str] + + @classmethod + def from_environment_definition( + cls, + env: EnvironmentSourceFormat, + attribute_path: AttributePath, + parser_context: ParserContextData, + is_default: bool = False, + ) -> Self: + reference_name: Optional[str] + if is_default: + name = "default-env" + reference_name = None + else: + named_env = cast("NamedEnvironmentSourceFormat", env) + name = named_env["name"] + reference_name = name + + set_vars = env.get("set", {}) + override_vars = env.get("override", {}) + unset_vars = env.get("unset", []) + _check_variables(set_vars, attribute_path["set"]) + _check_variables(override_vars, attribute_path["override"]) + _check_variables(unset_vars, attribute_path["unset"]) + + if not set_vars and not override_vars and not unset_vars: + raise ManifestParseException( + f"The environment definition {attribute_path.path_key_lc} was empty. Please provide" + " some content or delete the definition." + ) + + _no_overlap( + enumerate(unset_vars), + set_vars, + "unset", + "set", + "set", + attribute_path, + ) + _no_overlap( + enumerate(unset_vars), + override_vars, + "unset", + "override", + "override", + attribute_path, + ) + _no_overlap( + override_vars, + set_vars, + "override", + "set", + "set", + attribute_path, + ) + + r = cls( + name, + is_default, + attribute_path, + parser_context, + set_vars, + override_vars, + unset_vars, + ) + parser_context._register_build_environment( + reference_name, + r, + attribute_path, + is_default, + ) + + return r + + def update_env(self, env: MutableMapping[str, str]) -> None: + if set_vars := self.set_vars: + env.update(set_vars) + dpkg_env = self.dpkg_buildflags_env(env, self.attribute_path.path_key_lc) + self.log_computed_env(f"dpkg-buildflags [{self.name}]", dpkg_env) + if overlapping_env := dpkg_env.keys() & set_vars.keys(): + for var in overlapping_env: + key_lc = self.attribute_path["set"][var].path_key_lc + _warn( + f'The variable "{var}" defined at {key_lc} is shadowed by a computed variable.' + f" If the manifest definition is more important, please define it via `override` rather than" + f" `set`." + ) + env.update(dpkg_env) + if override_vars := self.override_vars: + env.update(override_vars) + if unset_vars := self.unset_vars: + for var in unset_vars: + try: + del env[var] + except KeyError: + pass + + +_MAKE_DEFAULT_TOOLS = [ + ("CC", "gcc"), + ("CXX", "g++"), + ("PKG_CONFIG", "pkg-config"), +] + + +class MakefileBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ("_make_support", "_build_target", "_install_target", "_directory") + + def __init__( + self, + attributes: "ParsedMakeBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(attributes, attribute_path, parser_context) + directory = attributes.get("directory") + if directory is not None: + self._directory = directory.match_rule.path + else: + self._directory = None + self._make_support = MakefileSupport.from_build_system(self) + self._build_target = attributes.get("build_target") + self._test_target = attributes.get("test_target") + self._install_target = attributes.get("install_target") + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + return any(p in source_root for p in ("Makefile", "makefile", "GNUmakefile")) + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="not-supported", + ) + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + # No configure step + pass + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + extra_vars = [] + build_target = self._build_target + if build_target is not None: + extra_vars.append(build_target) + if context.is_cross_compiling: + for envvar, tool in _MAKE_DEFAULT_TOOLS: + cross_tool = os.environ.get(envvar) + if cross_tool is None: + cross_tool = context.cross_tool(tool) + extra_vars.append(f"{envvar}={cross_tool}") + self._make_support.run_make( + context, + *extra_vars, + "INSTALL=install --strip-program=true", + directory=self._directory, + ) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._run_make_maybe_explicit_target( + context, + self._test_target, + ["test", "check"], + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + self._run_make_maybe_explicit_target( + context, + self._install_target, + ["install"], + f"DESTDIR={dest_dir}", + "AM_UPDATE_INFO_DIR=no", + "INSTALL=install --strip-program=true", + ) + + def _run_make_maybe_explicit_target( + self, + context: "BuildContext", + provided_target: Optional[str], + fallback_targets: Sequence[str], + *make_args: str, + ) -> None: + make_support = self._make_support + if provided_target is not None: + make_support.run_make( + context, + provided_target, + *make_args, + directory=self._directory, + ) + else: + make_support.run_first_existing_target_if_any( + context, + fallback_targets, + *make_args, + directory=self._directory, + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + self._make_support.run_first_existing_target_if_any( + context, + ["distclean", "realclean", "clean"], + ) + + +class PerlBuildBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = "configure_args" + + def __init__( + self, + attributes: "ParsedPerlBuildBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(attributes, attribute_path, parser_context) + self.configure_args = attributes.get("configure_args", []) + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + return "Build.PL" in source_root + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="not-supported", + ) + + @staticmethod + def _perl_cross_build_env( + context: "BuildContext", + ) -> Tuple[PerlConfigVars, Optional[EnvironmentModification]]: + perl_config_data = resolve_perl_config( + context.dpkg_architecture_variables, + None, + ) + if context.is_cross_compiling: + perl5lib_dir = perl_config_data.cross_inc_dir + if perl5lib_dir is not None: + env_perl5lib = os.environ.get("PERL5LIB") + if env_perl5lib is not None: + perl5lib_dir = ( + perl5lib_dir + perl_config_data.path_sep + env_perl5lib + ) + env_mod = EnvironmentModification( + replacements=[ + ("PERL5LIB", perl5lib_dir), + ], + ) + return perl_config_data, env_mod + return perl_config_data, None + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + perl_config_data, cross_env_mod = self._perl_cross_build_env(context) + configure_env = EnvironmentModification( + replacements=[("PERL_MM_USE_DEFAULT", "1")] + ) + if cross_env_mod is not None: + configure_env = configure_env.combine(cross_env_mod) + + configure_cmd = [ + PERL_CMD, + "Build.PL", + "--installdirs", + "vendor", + ] + cflags = os.environ.get("CFLAGS", "") + cppflags = os.environ.get("CPPFLAGS", "") + ldflags = os.environ.get("LDFLAGS", "") + + if cflags != "" or cppflags != "": + configure_cmd.append("--config") + combined = f"{cflags} {cppflags}".strip() + configure_cmd.append(f"optimize={combined}") + + if ldflags != "" or cflags != "" or context.is_cross_compiling: + configure_cmd.append("--config") + combined = f"{perl_config_data.ld} {cflags} {ldflags}".strip() + configure_cmd.append(f"ld={combined}") + if self.configure_args: + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + configure_cmd.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + run_build_system_command(*configure_cmd, env_mod=configure_env) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + _, cross_env_mod = self._perl_cross_build_env(context) + run_build_system_command(PERL_CMD, "Build", env_mod=cross_env_mod) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + _, cross_env_mod = self._perl_cross_build_env(context) + run_build_system_command( + PERL_CMD, + "Build", + "test", + "--verbose", + "1", + env_mod=cross_env_mod, + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + _, cross_env_mod = self._perl_cross_build_env(context) + run_build_system_command( + PERL_CMD, + "Build", + "install", + "--destdir", + dest_dir, + "--create_packlist", + "0", + env_mod=cross_env_mod, + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + _, cross_env_mod = self._perl_cross_build_env(context) + if os.path.lexists("Build"): + run_build_system_command( + PERL_CMD, + "Build", + "realclean", + "--allow_mb_mismatch", + "1", + env_mod=cross_env_mod, + ) + + +class PerlMakeMakerBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ("configure_args", "_make_support") + + def __init__( + self, + attributes: "ParsedPerlBuildBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(attributes, attribute_path, parser_context) + self.configure_args = attributes.get("configure_args", []) + self._make_support = MakefileSupport.from_build_system(self) + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + return "Makefile.PL" in source_root + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="not-supported", + ) + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + configure_env = EnvironmentModification( + replacements=[ + ("PERL_MM_USE_DEFAULT", "1"), + ("PERL_AUTOINSTALL", "--skipdeps"), + ] + ) + perl_args = [] + mm_args = ["INSTALLDIRS=vendor"] + if "CFLAGS" in os.environ: + mm_args.append( + f"OPTIMIZE={os.environ['CFLAGS']} {os.environ['CPPFLAGS']}".rstrip() + ) + + perl_config_data = resolve_perl_config( + context.dpkg_architecture_variables, + None, + ) + + if "LDFLAGS" in os.environ: + mm_args.append( + f"LD={perl_config_data.ld} {os.environ['CFLAGS']} {os.environ['LDFLAGS']}" + ) + + if context.is_cross_compiling: + perl5lib_dir = perl_config_data.cross_inc_dir + if perl5lib_dir is not None: + perl_args.append(f"-I{perl5lib_dir}") + + if self.configure_args: + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + mm_args.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + run_build_system_command( + PERL_CMD, + *perl_args, + "Makefile.PL", + *mm_args, + env_mod=configure_env, + ) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._make_support.run_make(context) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._make_support.run_first_existing_target_if_any( + context, + ["check", "test"], + "TEST_VERBOSE=1", + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + is_mm_makefile = False + with open("Makefile", "rb") as fd: + for line in fd: + if b"generated automatically by MakeMaker" in line: + is_mm_makefile = True + break + + install_args = [f"DESTDIR={dest_dir}"] + + # Special case for Makefile.PL that uses + # Module::Build::Compat. PREFIX should not be passed + # for those; it already installs into /usr by default. + if is_mm_makefile: + install_args.append("PREFIX=/usr") + + self._make_support.run_first_existing_target_if_any( + context, + ["install"], + *install_args, + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + self._make_support.run_first_existing_target_if_any( + context, + ["distclean", "realclean", "clean"], + ) + + +class DebhelperBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ("configure_args", "dh_build_system") + + def __init__( + self, + parsed_data: "ParsedDebhelperBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(parsed_data, attribute_path, parser_context) + self.configure_args = parsed_data.get("configure_args", []) + self.dh_build_system = parsed_data.get("dh_build_system") + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + try: + v = subprocess.check_output( + ["dh_assistant", "which-build-system"], + cwd=source_root.fs_path, + ) + except subprocess.CalledProcessError: + return False + else: + d = json.loads(v) + build_system = d.get("build-system") + return build_system is not None + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="supported-but-not-default", + ) + + def before_first_impl_step( + self, *, stage: Literal["build", "clean"], **kwargs + ) -> None: + dh_build_system = self.dh_build_system + if dh_build_system is None: + return + try: + subprocess.check_call( + ["dh_assistant", "which-build-system", f"-S{dh_build_system}"] + ) + except FileNotFoundError: + _error( + "The debhelper build system assumes `dh_assistant` is available (`debhelper (>= 13.5~)`)" + ) + except subprocess.SubprocessError: + raise ManifestInvalidUserDataException( + f'The debhelper build system "{dh_build_system}" does not seem to' + f" be available according to" + f" `dh_assistant which-build-system -S{dh_build_system}`" + ) from None + + def _default_options(self) -> List[str]: + default_options = [] + if self.dh_build_system is not None: + default_options.append(f"-S{self.dh_build_system}") + if self.build_directory is not None: + default_options.append(f"-B{self.build_directory}") + + return default_options + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + if ( + os.path.lexists("configure.ac") or os.path.lexists("configure.in") + ) and not os.path.lexists("debian/autoreconf.before"): + run_build_system_command("dh_update_autotools_config") + run_build_system_command("dh_autoreconf") + + default_options = self._default_options() + configure_args = default_options.copy() + if self.configure_args: + configure_args.append("--") + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + configure_args.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + run_build_system_command("dh_auto_configure", *configure_args) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + default_options = self._default_options() + run_build_system_command("dh_auto_build", *default_options) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + default_options = self._default_options() + run_build_system_command("dh_auto_test", *default_options) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + default_options = self._default_options() + run_build_system_command( + "dh_auto_install", + *default_options, + f"--destdir={dest_dir}", + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + default_options = self._default_options() + run_build_system_command("dh_auto_clean", *default_options) + # The "global" clean logic takes care of `dh_autoreconf_clean` and `dh_clean` + + +class AutoconfBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ("configure_args", "_make_support") + + def __init__( + self, + parsed_data: "ParsedAutoconfBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(parsed_data, attribute_path, parser_context) + configure_args = [a for a in parsed_data.get("configure_args", [])] + self.configure_args = configure_args + self._make_support = MakefileSupport.from_build_system(self) + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="supported-and-default", + ) + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + if "configure.ac" in source_root: + return True + configure_in = source_root.get("configure.in") + if configure_in is not None and configure_in.is_file: + with configure_in.open(byte_io=True, buffering=4096) as fd: + for no, line in enumerate(fd): + if no > 100: + break + if b"AC_INIT" in line or b"AC_PREREQ" in line: + return True + configure = source_root.get("configure") + if configure is None or not configure.is_executable or not configure.is_file: + return False + with configure.open(byte_io=True, buffering=4096) as fd: + for no, line in enumerate(fd): + if no > 10: + break + if b"GNU Autoconf" in line: + return True + return False + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + if ( + os.path.lexists("configure.ac") or os.path.lexists("configure.in") + ) and not os.path.lexists("debian/autoreconf.before"): + run_build_system_command("dh_update_autotools_config") + run_build_system_command("dh_autoreconf") + + dpkg_architecture_variables = context.dpkg_architecture_variables + multi_arch = dpkg_architecture_variables.current_host_multiarch + silent_rules = ( + "--enable-silent-rules" + if context.is_terse_build + else "--disable-silent-rules" + ) + + configure_args = [ + f"--build={dpkg_architecture_variables['DEB_BUILD_GNU_TYPE']}", + "--prefix=/usr", + "--includedir=${prefix}/include", + "--mandir=${prefix}/share/man", + "--infodir=${prefix}/share/info", + "--sysconfdir=/etc", + "--localstatedir=/var", + "--disable-option-checking", + silent_rules, + f"--libdir=${{prefix}}/{multi_arch}", + "--runstatedir=/run", + "--disable-maintainer-mode", + "--disable-dependency-tracking", + ] + if dpkg_architecture_variables.is_cross_compiling: + configure_args.append( + f"--host={dpkg_architecture_variables['DEB_HOST_GNU_TYPE']}" + ) + if self.configure_args: + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + configure_args.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + self.ensure_build_dir_exists() + configure_script = self.relative_from_builddir_to_source("configure") + with self.dump_logs_on_error("config.log"): + run_build_system_command( + configure_script, + *configure_args, + cwd=self.build_directory, + ) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._make_support.run_make(context) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + limit = context.parallelization_limit(support_zero_as_unlimited=True) + testsuite_flags = [f"-j{limit}"] if limit else ["-j"] + + if not context.is_terse_build: + testsuite_flags.append("--verbose") + self._make_support.run_first_existing_target_if_any( + context, + # Order is deliberately inverse compared to debhelper (#924052) + ["check", "test"], + f"TESTSUITEFLAGS={' '.join(testsuite_flags)}", + "VERBOSE=1", + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + enable_parallelization = not os.path.lexists(self.build_dir_path("libtool")) + self._make_support.run_first_existing_target_if_any( + context, + ["install"], + f"DESTDIR={dest_dir}", + "AM_UPDATE_INFO_DIR=no", + enable_parallelization=enable_parallelization, + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + if self.out_of_source_build: + return + self._make_support.run_first_existing_target_if_any( + context, + ["distclean", "realclean", "clean"], + ) + # The "global" clean logic takes care of `dh_autoreconf_clean` and `dh_clean` + + +class CMakeBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ( + "configure_args", + "target_build_system", + "_make_support", + "_ninja_support", + ) + + def __init__( + self, + parsed_data: "ParsedCMakeBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(parsed_data, attribute_path, parser_context) + configure_args = [a for a in parsed_data.get("configure_args", [])] + self.configure_args = configure_args + self.target_build_system: Literal["make", "ninja"] = parsed_data.get( + "target_build_system", "make" + ) + self._make_support = MakefileSupport.from_build_system(self) + self._ninja_support = NinjaBuildSupport.from_build_system(self) + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="required", + ) + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + return "CMakeLists.txt" in source_root + + @staticmethod + def _default_cmake_env( + build_context: "BuildContext", + ) -> EnvironmentModification: + replacements = {} + if "DEB_PYTHON_INSTALL_LAYOUT" not in os.environ: + replacements["DEB_PYTHON_INSTALL_LAYOUT"] = "deb" + if "PKG_CONFIG" not in os.environ: + replacements["PKG_CONFIG"] = build_context.cross_tool("pkg-config") + return EnvironmentModification(replacements=replacements) + + @classmethod + def cmake_generator(cls, target_build_system: Literal["make", "ninja"]) -> str: + cmake_generators = { + "make": "Unix Makefiles", + "ninja": "Ninja", + } + return cmake_generators[target_build_system] + + @staticmethod + def _compiler_and_cross_flags( + context: "BuildContext", + cmake_flags: List[str], + ) -> None: + + if "CC" in os.environ: + cmake_flags.append(f"-DCMAKE_C_COMPILER={os.environ['CC']}") + elif context.is_cross_compiling: + cmake_flags.append(f"-DCMAKE_C_COMPILER={context.cross_tool('gcc')}") + + if "CXX" in os.environ: + cmake_flags.append(f"-DCMAKE_CXX_COMPILER={os.environ['CXX']}") + elif context.is_cross_compiling: + cmake_flags.append(f"-DCMAKE_CXX_COMPILER={context.cross_tool('g++')}") + + if context.is_cross_compiling: + deb_host2cmake_system = { + "linux": "Linux", + "kfreebsd": "kFreeBSD", + "hurd": "GNU", + } + + gnu_cpu2system_processor = { + "arm": "armv7l", + "misp64el": "mips64", + "powerpc64le": "ppc64le", + } + dpkg_architecture_variables = context.dpkg_architecture_variables + + try: + system_name = deb_host2cmake_system[ + dpkg_architecture_variables["DEB_HOST_ARCH_OS"] + ] + except KeyError as e: + name = e.args[0] + _error( + f"Cannot cross-compile via cmake: Missing CMAKE_SYSTEM_NAME for the DEB_HOST_ARCH_OS {name}" + ) + + gnu_cpu = dpkg_architecture_variables["DEB_HOST_GNU_CPU"] + system_processor = gnu_cpu2system_processor.get(gnu_cpu, gnu_cpu) + + cmake_flags.append(f"-DCMAKE_SYSTEM_NAME={system_name}") + cmake_flags.append(f"-DCMAKE_SYSTEM_PROCESSOR={system_processor}") + + pkg_config = context.cross_tool("pkg-config") + # Historical uses. Current versions of cmake uses the env variable instead. + cmake_flags.append(f"-DPKG_CONFIG_EXECUTABLE=/usr/bin/{pkg_config}") + cmake_flags.append(f"-DPKGCONFIG_EXECUTABLE=/usr/bin/{pkg_config}") + cmake_flags.append( + f"-DQMAKE_EXECUTABLE=/usr/bin/{context.cross_tool('qmake')}" + ) + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + cmake_flags = [ + "-DCMAKE_INSTALL_PREFIX=/usr", + "-DCMAKE_BUILD_TYPE=None", + "-DCMAKE_INSTALL_SYSCONFDIR=/etc", + "-DCMAKE_INSTALL_LOCALSTATEDIR=/var", + "-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON", + "-DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF", + "-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON", + "-DFETCHCONTENT_FULLY_DISCONNECTED=ON", + "-DCMAKE_INSTALL_RUNSTATEDIR=/run", + "-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON", + "-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON", + f"-G{self.cmake_generator(self.target_build_system)}", + ] + if not context.is_terse_build: + cmake_flags.append("-DCMAKE_VERBOSE_MAKEFILE=ON") + + self._compiler_and_cross_flags(context, cmake_flags) + + if self.configure_args: + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + cmake_flags.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + + env_mod = self._default_cmake_env(context) + if "CPPFLAGS" in os.environ: + # CMake doesn't respect CPPFLAGS, see #653916. + cppflags = os.environ["CPPFLAGS"] + cflags = os.environ.get("CFLAGS", "") + f" {cppflags}".lstrip() + cxxflags = os.environ.get("CXXFLAGS", "") + f" {cppflags}".lstrip() + env_mod = env_mod.combine( + # The debhelper build system never showed this delta, so people might find it annoying. + EnvironmentModification( + replacements={ + "CFLAGS": cflags, + "CXXFLAGS": cxxflags, + } + ) + ) + if "ASMFLAGS" not in os.environ and "ASFLAGS" in os.environ: + env_mod = env_mod.combine( + # The debhelper build system never showed this delta, so people might find it annoying. + EnvironmentModification( + replacements={ + "ASMFLAGS": os.environ["ASFLAGS"], + } + ) + ) + self.ensure_build_dir_exists() + source_dir_from_build_dir = self.relative_from_builddir_to_source() + + with self.dump_logs_on_error( + "CMakeCache.txt", + "CMakeFiles/CMakeOutput.log", + "CMakeFiles/CMakeError.log", + ): + run_build_system_command( + "cmake", + *cmake_flags, + source_dir_from_build_dir, + cwd=self.build_directory, + env_mod=env_mod, + ) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + if self.target_build_system == "make": + make_flags = [] + if not context.is_terse_build: + make_flags.append("VERBOSE=1") + self._make_support.run_make(context, *make_flags) + else: + self._ninja_support.run_ninja_build(context) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + env_mod = EnvironmentModification( + replacements={ + "CTEST_OUTPUT_ON_FAILURE": "1", + }, + ) + if self.target_build_system == "make": + # Unlike make, CTest does not have "unlimited parallel" setting (-j implies + # -j1). Therefore, we do not set "allow zero as unlimited" here. + make_flags = [f"ARGS+=-j{context.parallelization_limit()}"] + if not context.is_terse_build: + make_flags.append("ARGS+=--verbose") + self._make_support.run_first_existing_target_if_any( + context, + ["check", "test"], + *make_flags, + env_mod=env_mod, + ) + else: + self._ninja_support.run_ninja_test(context, env_mod=env_mod) + + limit = context.parallelization_limit(support_zero_as_unlimited=True) + testsuite_flags = [f"-j{limit}"] if limit else ["-j"] + + if not context.is_terse_build: + testsuite_flags.append("--verbose") + self._make_support.run_first_existing_target_if_any( + context, + # Order is deliberately inverse compared to debhelper (#924052) + ["check", "test"], + f"TESTSUITEFLAGS={' '.join(testsuite_flags)}", + "VERBOSE=1", + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + env_mod = EnvironmentModification( + replacements={ + "LC_ALL": "C.UTF-8", + "DESTDIR": dest_dir, + } + ).combine(self._default_cmake_env(context)) + run_build_system_command( + "cmake", + "--install", + self.build_directory, + env_mod=env_mod, + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + if self.out_of_source_build: + return + if self.target_build_system == "make": + # Keep it here in case we change the `required` "out of source" to "supported-default" + self._make_support.run_first_existing_target_if_any( + context, + ["distclean", "realclean", "clean"], + ) + else: + self._ninja_support.run_ninja_clean(context) + + +class MesonBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ( + "configure_args", + "_ninja_support", + ) + + def __init__( + self, + parsed_data: "ParsedMesonBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(parsed_data, attribute_path, parser_context) + configure_args = [a for a in parsed_data.get("configure_args", [])] + self.configure_args = configure_args + self._ninja_support = NinjaBuildSupport.from_build_system(self) + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="required", + ) + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + return "meson.build" in source_root + + @staticmethod + def _default_meson_env() -> EnvironmentModification: + replacements = { + "LC_ALL": "C.UTF-8", + } + if "DEB_PYTHON_INSTALL_LAYOUT" not in os.environ: + replacements["DEB_PYTHON_INSTALL_LAYOUT"] = "deb" + return EnvironmentModification(replacements=replacements) + + @classmethod + def cmake_generator(cls, target_build_system: Literal["make", "ninja"]) -> str: + cmake_generators = { + "make": "Unix Makefiles", + "ninja": "Ninja", + } + return cmake_generators[target_build_system] + + @staticmethod + def _cross_flags( + context: "BuildContext", + meson_flags: List[str], + ) -> None: + if not context.is_cross_compiling: + return + # Needs a cross-file http://mesonbuild.com/Cross-compilation.html + cross_files_dir = os.path.abspath( + generated_content_dir( + subdir_key="meson-cross-files", + ) + ) + host_arch = context.dpkg_architecture_variables.current_host_arch + cross_file = os.path.join(cross_files_dir, f"meson-cross-file-{host_arch}.conf") + if not os.path.isfile(cross_file): + env = os.environ + if env.get("LC_ALL") != "C.UTF-8": + env = dict(env) + env["LC_ALL"] = "C.UTF-8" + else: + env = None + subprocess.check_call( + [ + "/usr/share/meson/debcrossgen", + f"--arch={host_arch}", + f"-o{cross_file}", + ], + stdout=subprocess.DEVNULL, + env=env, + ) + + meson_flags.append("--cross-file") + meson_flags.append(cross_file) + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + meson_version = Version( + subprocess.check_output( + ["meson", "--version"], + encoding="utf-8", + ).strip() + ) + dpkg_architecture_variables = context.dpkg_architecture_variables + + meson_flags = [ + "--wrap-mode=odownload", + "--buildtype=plain", + "--sysconfdir=/etc", + "--localstatedir=/var", + f"--libdir=lib/{dpkg_architecture_variables.current_host_multiarch}", + "--auto-features=enabled", + ] + if meson_version >= Version("1.2.0"): + # There was a behaviour change in Meson 1.2.0: previously + # byte-compilation wasn't supported, but since 1.2.0 it is on by + # default. We can only use this option to turn it off in versions + # where the option exists. + meson_flags.append("-Dpython.bytecompile=-1") + + self._cross_flags(context, meson_flags) + + if self.configure_args: + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + meson_flags.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + + env_mod = self._default_meson_env() + + self.ensure_build_dir_exists() + source_dir_from_build_dir = self.relative_from_builddir_to_source() + + with self.dump_logs_on_error("meson-logs/meson-log.txt"): + run_build_system_command( + "meson", + "setup", + source_dir_from_build_dir, + *meson_flags, + cwd=self.build_directory, + env_mod=env_mod, + ) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._ninja_support.run_ninja_build(context) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + env_mod = EnvironmentModification( + replacements={ + "MESON_TESTTHREDS": f"{context.parallelization_limit()}", + }, + ).combine(self._default_meson_env()) + with self.dump_logs_on_error("meson-logs/testlog.txt"): + run_build_system_command( + "meson", + "test", + env_mod=env_mod, + cwd=self.build_directory, + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + run_build_system_command( + "meson", + "install", + "--destdir", + dest_dir, + env_mod=self._default_meson_env(), + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + # `debputy` will handle all the cleanup for us by virtue of "out of source build" + assert self.out_of_source_build + + +def _add_qmake_flag(options: List[str], envvar: str, *, include_cppflags: bool) -> None: + value = os.environ.get(envvar) + if value is None: + return + if include_cppflags: + cppflags = os.environ.get("CPPFLAGS") + if cppflags: + value = f"{value} {cppflags}" + + options.append(f"QMAKE_{envvar}_RELEASE={value}") + options.append(f"QMAKE_{envvar}_DEBUG={value}") + + +class ParsedGenericQmakeBuildRuleDefinition( + OptionalInstallDirectly, + OptionalInSourceBuild, + OptionalBuildDirectory, +): + configure_args: NotRequired[List[str]] + + +class AbstractQmakeBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ("configure_args", "_make_support") + + def __init__( + self, + parsed_data: "ParsedGenericQmakeBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(parsed_data, attribute_path, parser_context) + configure_args = [a for a in parsed_data.get("configure_args", [])] + self.configure_args = configure_args + self._make_support = MakefileSupport.from_build_system(self) + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="supported-and-default", + ) + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + return any(p.name.endswith(".pro") for p in source_root.iterdir) + + @classmethod + def os_mkspec_mapping(cls) -> Mapping[str, str]: + return { + "linux": "linux-g++", + "kfreebsd": "gnukfreebsd-g++", + "hurd": "hurd-g++", + } + + def qmake_command(self) -> str: + raise NotImplementedError + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + + configure_args = [ + "-makefile", + ] + qmake_cmd = context.cross_tool(self.qmake_command()) + + if context.is_cross_compiling: + host_os = context.dpkg_architecture_variables["DEB_HOST_ARCH_OS"] + os2mkspec = self.os_mkspec_mapping() + try: + spec = os2mkspec[host_os] + except KeyError: + _error( + f'Sorry, `debputy` cannot cross build this package for "{host_os}".' + f' Missing a "DEB OS -> qmake -spec <VALUE>" mapping.' + ) + configure_args.append("-spec") + configure_args.append(spec) + + _add_qmake_flag(configure_args, "CFLAGS", include_cppflags=True) + _add_qmake_flag(configure_args, "CXXFLAGS", include_cppflags=True) + _add_qmake_flag(configure_args, "LDFLAGS", include_cppflags=False) + + configure_args.append("QMAKE_STRIP=:") + configure_args.append("PREFIX=/usr") + + if self.configure_args: + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + configure_args.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + + self.ensure_build_dir_exists() + if not self.out_of_source_build: + configure_args.append(self.relative_from_builddir_to_source()) + + with self.dump_logs_on_error("config.log"): + run_build_system_command( + qmake_cmd, + *configure_args, + cwd=self.build_directory, + ) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._make_support.run_make(context) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + limit = context.parallelization_limit(support_zero_as_unlimited=True) + testsuite_flags = [f"-j{limit}"] if limit else ["-j"] + + if not context.is_terse_build: + testsuite_flags.append("--verbose") + self._make_support.run_first_existing_target_if_any( + context, + # Order is deliberately inverse compared to debhelper (#924052) + ["check", "test"], + f"TESTSUITEFLAGS={' '.join(testsuite_flags)}", + "VERBOSE=1", + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + enable_parallelization = not os.path.lexists(self.build_dir_path("libtool")) + self._make_support.run_first_existing_target_if_any( + context, + ["install"], + f"DESTDIR={dest_dir}", + "AM_UPDATE_INFO_DIR=no", + enable_parallelization=enable_parallelization, + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + if self.out_of_source_build: + return + self._make_support.run_first_existing_target_if_any( + context, + ["distclean", "realclean", "clean"], + ) + + +class QmakeBuildSystemRule(AbstractQmakeBuildSystemRule): + + def qmake_command(self) -> str: + return "qmake" + + +class Qmake6BuildSystemRule(AbstractQmakeBuildSystemRule): + + def qmake_command(self) -> str: + return "qmake6" + + +@debputy_build_system( + "make", + MakefileBuildSystemRule, + auto_detection_shadows_build_systems="debhelper", + online_reference_documentation=reference_documentation( + title="Make Build System", + description=textwrap.dedent( + "" + """\ + Run a plain `make` file with nothing else. + + This build system will attempt to use `make` to leverage instructions + in a makefile (such as, `Makefile` or `GNUMakefile`). + + By default, the makefile build system assumes it should use "in-source" + build semantics. If needed be, an explicit `build-directory` can be + provided if the `Makefile` is not in the source folder but instead in + some other directory. + """ + ), + attributes=[ + documented_attr( + "directory", + textwrap.dedent( + """\ + The directory from which to run make if it is not the source root + + This works like using `make -C DIRECTORY ...` (or `cd DIRECTORY && make ...`). + """ + ), + ), + documented_attr( + "build_target", + textwrap.dedent( + """\ + The target name to use for the "build" step. + + If omitted, `make` will be run without any explicit target leaving it to decide + the default. + """ + ), + ), + documented_attr( + "test_target", + textwrap.dedent( + """\ + The target name to use for the "test" step. + + If omitted, `make check` or `make test` will be used if it looks like `make` + will accept one of those targets. Otherwise, the step will be skipped. + """ + ), + ), + documented_attr( + "install_target", + textwrap.dedent( + """\ + The target name to use for the "install" step. + + If omitted, `make install` will be used if it looks like `make` will accept that target. + Otherwise, the step will be skipped. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedMakeBuildRuleDefinition( + OptionalInstallDirectly, +): + directory: NotRequired[FileSystemExactMatchRule] + build_target: NotRequired[str] + test_target: NotRequired[str] + install_target: NotRequired[str] + + +@debputy_build_system( + "autoconf", + AutoconfBuildSystemRule, + auto_detection_shadows_build_systems=["debhelper", "make"], + online_reference_documentation=reference_documentation( + title="Autoconf Build System", + description=textwrap.dedent( + """\ + Run an autoconf-based build system as the upstream build system. + + This build rule will attempt to use autoreconf to update the `configure` + script before running the `configure` script if needed. Otherwise, it + follows the classic `./configure && make && make install` pattern. + + The build rule uses "out of source" builds by default since it is easier + and more reliable for clean and makes it easier to support multiple + builds (that is, two or more build systems for the same source package). + This is in contract to `debhelper`, which defaults to "in source" builds + for `autoconf`. If you need that behavior, please set + `perform-in-source-build: true`. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `configure` script. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + OptionalInSourceBuild, + OptionalBuildDirectory, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedAutoconfBuildRuleDefinition( + OptionalInstallDirectly, + OptionalInSourceBuild, + OptionalBuildDirectory, +): + configure_args: NotRequired[List[str]] + + +@debputy_build_system( + "cmake", + CMakeBuildSystemRule, + auto_detection_shadows_build_systems=["debhelper", "make"], + online_reference_documentation=reference_documentation( + title="CMake Build System", + description=textwrap.dedent( + """\ + Run an cmake-based build system as the upstream build system. + + The build rule uses "out of source" builds. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `cmake` command. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + OptionalBuildDirectory, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedCMakeBuildRuleDefinition( + OptionalInstallDirectly, + OptionalBuildDirectory, +): + configure_args: NotRequired[List[str]] + target_build_system: Literal["make", "ninja"] + + +@debputy_build_system( + "meson", + MesonBuildSystemRule, + auto_detection_shadows_build_systems=["debhelper", "make"], + online_reference_documentation=reference_documentation( + title="Meson Build System", + description=textwrap.dedent( + """\ + Run an meson-based build system as the upstream build system. + + The build rule uses "out of source" builds. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `meson` command. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + OptionalBuildDirectory, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedMesonBuildRuleDefinition( + OptionalInstallDirectly, + OptionalBuildDirectory, +): + configure_args: NotRequired[List[str]] + + +@debputy_build_system( + "perl-build", + PerlBuildBuildSystemRule, + auto_detection_shadows_build_systems=[ + "debhelper", + "make", + "perl-makemaker", + ], + online_reference_documentation=reference_documentation( + title='Perl "Build.PL" Build System', + description=textwrap.dedent( + """\ + Build using the `Build.PL` Build system used by some Perl packages. + + This build rule will attempt to use the `Build.PL` script to build the + upstream code. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `Build.PL` script. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedPerlBuildBuildRuleDefinition( + OptionalInstallDirectly, +): + configure_args: NotRequired[List[str]] + + +@debputy_build_system( + "debhelper", + DebhelperBuildSystemRule, + online_reference_documentation=reference_documentation( + title="Debhelper Build System", + description=textwrap.dedent( + """\ + Delegate to a debhelper provided build system + + This build rule will attempt to use the `dh_auto_*` tools to build the + upstream code. By default, `dh_auto_*` will use auto-detection to determine + which build system they will use. This can be overridden by the + `dh-build-system` attribute. + """ + ), + attributes=[ + documented_attr( + "dh_build_system", + textwrap.dedent( + """\ + Which debhelper build system to use. This attribute is passed to + the `dh_auto_*` commands as the `-S` parameter, so any value valid + for that will be accepted. + + Note that many debhelper build systems require extra build + dependencies before they can be used. Please consult the documentation + of the relevant debhelper build system for details. + """ + ), + ), + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to underlying configuration command + (via `dh_auto_configure -- <configure-args`). + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + OptionalBuildDirectory, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedDebhelperBuildRuleDefinition( + OptionalInstallDirectly, + OptionalBuildDirectory, +): + configure_args: NotRequired[List[str]] + dh_build_system: NotRequired[str] + + +@debputy_build_system( + "perl-makemaker", + PerlMakeMakerBuildSystemRule, + auto_detection_shadows_build_systems=[ + "debhelper", + "make", + ], + online_reference_documentation=reference_documentation( + title='Perl "MakeMaker" Build System', + description=textwrap.dedent( + """\ + Build using the "MakeMaker" Build system used by some Perl packages. + + This build rule will attempt to use the `Makefile.PL` script to build the + upstream code. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `Makefile.PL` script. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedPerlMakeMakerBuildRuleDefinition( + OptionalInstallDirectly, +): + configure_args: NotRequired[List[str]] + + +@debputy_build_system( + "qmake", + QmakeBuildSystemRule, + auto_detection_shadows_build_systems=[ + "debhelper", + "make", + # Open question, should this shadow "qmake6" and later? + ], + online_reference_documentation=reference_documentation( + title='QT "qmake" Build System', + description=textwrap.dedent( + """\ + Build using the "qmake" by QT. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `qmake` command. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + OptionalInSourceBuild, + OptionalBuildDirectory, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedQmakeBuildRuleDefinition(ParsedGenericQmakeBuildRuleDefinition): + pass + + +@debputy_build_system( + "qmake6", + Qmake6BuildSystemRule, + auto_detection_shadows_build_systems=[ + "debhelper", + "make", + ], + online_reference_documentation=reference_documentation( + title='QT "qmake6" Build System', + description=textwrap.dedent( + """\ + Build using the "qmake6" from the `qmake6` package. This is like the `qmake` system + but is specifically for QT6. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `qmake6` command. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + OptionalInSourceBuild, + OptionalBuildDirectory, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedQmake6BuildRuleDefinition(ParsedGenericQmakeBuildRuleDefinition): + pass + + +def _parse_default_environment( + _name: str, + parsed_data: EnvironmentSourceFormat, + attribute_path: AttributePath, + parser_context: ParserContextData, +) -> ManifestProvidedBuildEnvironment: + return ManifestProvidedBuildEnvironment.from_environment_definition( + parsed_data, + attribute_path, + parser_context, + is_default=True, + ) + + +def _parse_build_environments( + _name: str, + parsed_data: List[NamedEnvironmentSourceFormat], + attribute_path: AttributePath, + parser_context: ParserContextData, +) -> List[ManifestProvidedBuildEnvironment]: + return [ + ManifestProvidedBuildEnvironment.from_environment_definition( + value, + attribute_path[idx], + parser_context, + is_default=False, + ) + for idx, value in enumerate(parsed_data) + ] + + +def _handle_build_rules( + _name: str, + parsed_data: List[BuildRule], + _attribute_path: AttributePath, + _parser_context: ParserContextData, +) -> List[BuildRule]: + return parsed_data diff --git a/src/debputy/plugin/debputy/manifest_root_rules.py b/src/debputy/plugin/debputy/manifest_root_rules.py index 1d3b096..f539243 100644 --- a/src/debputy/plugin/debputy/manifest_root_rules.py +++ b/src/debputy/plugin/debputy/manifest_root_rules.py @@ -12,18 +12,22 @@ from debputy._manifest_constants import ( ) from debputy.exceptions import DebputySubstitutionError from debputy.installations import InstallRule -from debputy.manifest_parser.base_types import DebputyParsedContent +from debputy.manifest_parser.tagging_types import DebputyParsedContent from debputy.manifest_parser.exceptions import ManifestParseException from debputy.manifest_parser.parser_data import ParserContextData from debputy.manifest_parser.util import AttributePath from debputy.plugin.api import reference_documentation from debputy.plugin.api.impl import DebputyPluginInitializerProvider -from debputy.plugin.api.impl_types import ( +from debputy.plugin.api.parser_tables import ( OPARSER_MANIFEST_ROOT, OPARSER_MANIFEST_DEFINITIONS, OPARSER_PACKAGES, ) -from debputy.plugin.api.spec import not_integrations, INTEGRATION_MODE_DH_DEBPUTY_RRR +from debputy.plugin.api.spec import ( + not_integrations, + INTEGRATION_MODE_DH_DEBPUTY_RRR, +) +from debputy.plugin.debputy.build_system_rules import register_build_system_rules from debputy.substitution import VariableNameState, SUBST_VAR_RE if TYPE_CHECKING: @@ -166,6 +170,8 @@ def register_manifest_root_rules(api: DebputyPluginInitializerProvider) -> None: nested_in_package_context=True, ) + register_build_system_rules(api) + class ManifestVersionFormat(DebputyParsedContent): manifest_version: ManifestVersion diff --git a/src/debputy/plugin/debputy/private_api.py b/src/debputy/plugin/debputy/private_api.py index d042378..75081a4 100644 --- a/src/debputy/plugin/debputy/private_api.py +++ b/src/debputy/plugin/debputy/private_api.py @@ -43,9 +43,11 @@ from debputy.manifest_conditions import ( BuildProfileMatch, SourceContextArchMatchManifestCondition, ) -from debputy.manifest_parser.base_types import ( +from debputy.manifest_parser.tagging_types import ( DebputyParsedContent, - DebputyParsedContentStandardConditional, + TypeMapping, +) +from debputy.manifest_parser.base_types import ( FileSystemMode, StaticFileSystemOwner, StaticFileSystemGroup, @@ -53,11 +55,12 @@ from debputy.manifest_parser.base_types import ( FileSystemExactMatchRule, FileSystemMatchRule, SymbolicMode, - TypeMapping, OctalMode, FileSystemExactNonDirMatchRule, + BuildEnvironmentDefinition, + DebputyParsedContentStandardConditional, ) -from debputy.manifest_parser.declarative_parser import DebputyParseHint +from debputy.manifest_parser.parse_hints import DebputyParseHint from debputy.manifest_parser.exceptions import ManifestParseException from debputy.manifest_parser.mapper_code import type_mapper_str2package from debputy.manifest_parser.parser_data import ParserContextData @@ -79,7 +82,9 @@ from debputy.plugin.api.spec import ( not_integrations, INTEGRATION_MODE_DH_DEBPUTY_RRR, ) +from debputy.plugin.api.std_docs import docs_from from debputy.plugin.debputy.binary_package_rules import register_binary_package_rules +from debputy.plugin.debputy.build_system_rules import register_build_system_rules from debputy.plugin.debputy.discard_rules import ( _debputy_discard_pyc_files, _debputy_prune_la_files, @@ -587,6 +592,16 @@ def register_type_mappings(api: DebputyPluginInitializerProvider) -> None: ], ), ) + api.register_mapped_type( + TypeMapping( + BuildEnvironmentDefinition, + str, + lambda v, ap, pc: pc.resolve_build_environment(v, ap), + ), + reference_documentation=type_mapping_reference_documentation( + description="Reference to an build environment defined in `build-environments`", + ), + ) def register_service_managers( @@ -897,14 +912,7 @@ def register_install_rules(api: DebputyPluginInitializerProvider) -> None: """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc("generic-install-install"), ), @@ -1193,14 +1201,7 @@ def register_install_rules(api: DebputyPluginInitializerProvider) -> None: """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc( "install-manpages-install-man" @@ -1355,14 +1356,7 @@ def register_install_rules(api: DebputyPluginInitializerProvider) -> None: """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc("generic-install-install"), ), @@ -1408,14 +1402,7 @@ def register_transformation_rules(api: DebputyPluginInitializerProvider) -> None """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc( "move-transformation-rule-move" @@ -1564,14 +1551,7 @@ def register_transformation_rules(api: DebputyPluginInitializerProvider) -> None """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc( "create-symlinks-transformation-rule-create-symlink" @@ -1680,14 +1660,7 @@ def register_transformation_rules(api: DebputyPluginInitializerProvider) -> None """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc( "change-path-ownergroup-or-mode-path-metadata" @@ -1768,14 +1741,7 @@ def register_transformation_rules(api: DebputyPluginInitializerProvider) -> None """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc( "create-directories-transformation-rule-directories" diff --git a/src/debputy/plugin/debputy/to_be_api_types.py b/src/debputy/plugin/debputy/to_be_api_types.py new file mode 100644 index 0000000..d7be694 --- /dev/null +++ b/src/debputy/plugin/debputy/to_be_api_types.py @@ -0,0 +1,1039 @@ +import contextlib +import dataclasses +import os.path +import subprocess +from typing import ( + Optional, + FrozenSet, + final, + TYPE_CHECKING, + Union, + Annotated, + List, + NotRequired, + Literal, + Any, + Type, + TypeVar, + Self, + Sequence, + Callable, + Container, + Iterable, + is_typeddict, +) + +from debputy.exceptions import PluginAPIViolationError, PluginInitializationError +from debputy.manifest_conditions import ManifestCondition +from debputy.manifest_parser.base_types import ( + BuildEnvironmentDefinition, + DebputyParsedContentStandardConditional, + FileSystemExactMatchRule, +) +from debputy.manifest_parser.exceptions import ( + ManifestParseException, + ManifestInvalidUserDataException, +) +from debputy.manifest_parser.parse_hints import DebputyParseHint +from debputy.manifest_parser.parser_data import ParserContextData +from debputy.manifest_parser.tagging_types import DebputyDispatchableType +from debputy.manifest_parser.util import AttributePath +from debputy.packages import BinaryPackage +from debputy.plugin.api.spec import ( + ParserDocumentation, + DebputyIntegrationMode, + BuildSystemManifestRuleMetadata, + _DEBPUTY_DISPATCH_METADATA_ATTR_NAME, + VirtualPath, +) +from debputy.plugin.plugin_state import run_in_context_of_plugin +from debputy.substitution import Substitution +from debputy.types import EnvironmentModification +from debputy.util import run_build_system_command, _debug_log, _info, _warn + +if TYPE_CHECKING: + from debputy.build_support.build_context import BuildContext + from debputy.highlevel_manifest import HighLevelManifest + from debputy.plugin.api.impl_types import DIPHandler + + +AT = TypeVar("AT") +BSR = TypeVar("BSR", bound="BuildSystemRule") +BSPF = TypeVar("BSPF", bound="BuildRuleDefinitionBase") + + +@dataclasses.dataclass(slots=True, frozen=True) +class BuildSystemCharacteristics: + out_of_source_builds: Literal[ + "required", + "supported-and-default", + "supported-but-not-default", + "not-supported", + ] + + +class CleanHelper: + def schedule_removal_of_files(self, *args: str) -> None: + """Schedule removal of these files + + This will remove the provided files in bulk. The files are not guaranteed + to be deleted in any particular order. If anything needs urgent removal, + `os.unlink` can be used directly. + + Note: Symlinks will **not** be followed. If a symlink and target must + be deleted, ensure both are passed. + + + :param args: Path names to remove. Each must be removable with + `os.unlink` + """ + raise NotImplementedError + + def schedule_removal_of_directories(self, *args: str) -> None: + """Schedule removal of these directories + + This will remove the provided dirs in bulk. The dirs are not guaranteed + to be deleted in any particular order. If anything needs urgent removal, + then it can be done directly instead of passing it to this method. + + If anything needs urgent removal, then it can be removed immediately. + + :param args: Path names to remove. + """ + raise NotImplementedError + + +class BuildRuleParsedFormat(DebputyParsedContentStandardConditional): + name: NotRequired[str] + for_packages: NotRequired[ + Annotated[ + Union[BinaryPackage, List[BinaryPackage]], + DebputyParseHint.manifest_attribute("for"), + ] + ] + environment: NotRequired[BuildEnvironmentDefinition] + + +class OptionalBuildDirectory(BuildRuleParsedFormat): + build_directory: NotRequired[FileSystemExactMatchRule] + + +class OptionalInSourceBuild(BuildRuleParsedFormat): + perform_in_source_build: NotRequired[bool] + + +class OptionalInstallDirectly(BuildRuleParsedFormat): + install_directly_to_package: NotRequired[bool] + + +BuildSystemDefinition = Union[ + BuildRuleParsedFormat, + OptionalBuildDirectory, + OptionalInSourceBuild, + OptionalInstallDirectly, +] + + +class BuildRule(DebputyDispatchableType): + __slots__ = ( + "_auto_generated_stem", + "_name", + "_for_packages", + "_manifest_condition", + "_attribute_path", + "_environment", + "_substitution", + ) + + def __init__( + self, + attributes: BuildRuleParsedFormat, + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__() + + self._name = attributes.get("name") + for_packages = attributes.get("for_packages") + + if for_packages is None: + if isinstance(parser_context, ParserContextData): + all_binaries = parser_context.binary_packages.values() + else: + all_binaries = parser_context.all_packages + self._for_packages = frozenset(all_binaries) + else: + self._for_packages = frozenset( + for_packages if isinstance(for_packages, list) else [for_packages] + ) + self._manifest_condition = attributes.get("when") + self._attribute_path = attribute_path + self._substitution = parser_context.substitution + self._auto_generated_stem: Optional[str] = None + environment = attributes.get("environment") + if environment is None: + assert isinstance(parser_context, ParserContextData) + self._environment = parser_context.resolve_build_environment( + None, + attribute_path, + ) + else: + self._environment = environment + + @final + @property + def name(self) -> Optional[str]: + return self._name + + @final + @property + def attribute_path(self) -> AttributePath: + return self._attribute_path + + @final + @property + def manifest_condition(self) -> Optional[ManifestCondition]: + return self._manifest_condition + + @final + @property + def for_packages(self) -> FrozenSet[BinaryPackage]: + return self._for_packages + + @final + @property + def substitution(self) -> Substitution: + return self._substitution + + @final + @property + def environment(self) -> BuildEnvironmentDefinition: + return self._environment + + @final + @property + def auto_generated_stem(self) -> str: + stem = self._auto_generated_stem + if stem is None: + raise AssertionError( + "The auto-generated-stem is not available at this time" + ) + return stem + + @final + @auto_generated_stem.setter + def auto_generated_stem(self, value: str) -> None: + if self._auto_generated_stem is not None: + raise AssertionError("The auto-generated-stem should only be set once") + assert value is not None + self._auto_generated_stem = value + + @final + def run_build( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + run_in_context_of_plugin( + self._debputy_plugin, + self.perform_build, + context, + manifest, + **kwargs, + ) + + def perform_build( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + raise NotImplementedError + + @property + def is_buildsystem(self) -> bool: + return False + + @property + def name_or_tag(self) -> str: + name = self.name + if name is None: + return self.auto_generated_stem + return name + + +def _is_type_or_none(v: Optional[Any], expected_type: Type[AT]) -> Optional[AT]: + if isinstance(v, expected_type): + return v + return None + + +class BuildSystemRule(BuildRule): + + __slots__ = ( + "_build_directory", + "source_directory", + "install_directly_to_package", + "perform_in_source_build", + ) + + def __init__( + self, + attributes: BuildSystemDefinition, + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(attributes, attribute_path, parser_context) + build_directory = _is_type_or_none( + attributes.get("build_directory"), FileSystemExactMatchRule + ) + if build_directory is not None: + self._build_directory = build_directory.match_rule.path + else: + self._build_directory = None + self.source_directory = "." + self.install_directly_to_package = False + self.perform_in_source_build = _is_type_or_none( + attributes.get("perform_in_source_build"), bool + ) + install_directly_to_package = _is_type_or_none( + attributes.get("install_directly_to_package"), bool + ) + if install_directly_to_package is None: + self.install_directly_to_package = len(self.for_packages) == 1 + elif install_directly_to_package and len(self.for_packages) > 1: + idtp_path = attribute_path["install_directly_to_package"].path + raise ManifestParseException( + f'The attribute "install-directly-to-package" ({idtp_path}) cannot' + " be true when the build system applies to multiple packages." + ) + else: + self.install_directly_to_package = install_directly_to_package + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + """Check if the build system apply automatically. + + This class method is called when the manifest does not declare any build rules at + all. + + :param source_root: The source root (the directory containing `debian/`). Usually, + the detection code would look at this for files related to the upstream build system. + :param args: For future compat, new arguments might appear as positional arguments. + :param kwargs: For future compat, new arguments might appear as keyword argument. + :return: True if the build system can be used, False when it would not be useful + to use the build system (at least with all defaults). + Note: Be sure to use proper `bool` return values. The calling code does an + `isinstance` check to ensure that the version of `debputy` supports the + auto-detector (in case the return type is ever expanded in the future). + """ + return False + + @property + def out_of_source_build(self) -> bool: + build_directory = self.build_directory + return build_directory != self.source_directory + + @property + def build_directory(self) -> str: + directory = self._build_directory + if directory is None: + return self.source_directory + return directory + + @contextlib.contextmanager + def dump_logs_on_error(self, *logs: str) -> None: + """Context manager that will dump logs to stdout on error + + :param logs: The logs to be dumped. Relative path names are assumed to be relative to + the build directory. + """ + try: + yield + except (Exception, KeyboardInterrupt, SystemExit): + _warn( + "Error occurred, attempting to provide relevant logs as requested by the build system provider" + ) + found_any = False + for log in logs: + if not os.path.isabs(log): + log = self.build_dir_path(log) + if not os.path.isfile(log): + _info( + f'Would have pushed "{log}" to stdout, but it does not exist.' + ) + continue + subprocess.run(["tail", "-v", "-n", "+0", log]) + found_any = True + if not found_any: + _warn( + f"None of the logs provided were available (relative to build directory): {', '.join(logs)}" + ) + raise + + @final + def run_clean( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: CleanHelper, + **kwargs, + ) -> None: + run_in_context_of_plugin( + self._debputy_plugin, + self.perform_clean, + context, + manifest, + clean_helper, + **kwargs, + ) + + def perform_clean( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: CleanHelper, + **kwargs, + ) -> None: + raise NotImplementedError + + def ensure_build_dir_exists(self) -> None: + build_dir = self.build_directory + source_dir = self.source_directory + if build_dir == source_dir: + return + os.makedirs(build_dir, mode=0o755, exist_ok=True) + + def build_dir_path(self, /, path: str = "") -> str: + build_dir = self.build_directory + if path == "": + return build_dir + return os.path.join(build_dir, path) + + def relative_from_builddir_to_source( + self, + path_in_source_dir: Optional[str] = None, + ) -> str: + build_dir = self.build_directory + source_dir = self.source_directory + if build_dir == source_dir: + return path_in_source_dir + return os.path.relpath(os.path.join(source_dir, path_in_source_dir), build_dir) + + @final + @property + def is_buildsystem(self) -> bool: + return True + + +class StepBasedBuildSystemRule(BuildSystemRule): + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + raise NotImplementedError + + @final + def perform_clean( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: CleanHelper, + **kwargs, + ) -> None: + self._check_characteristics() + self.before_first_impl_step(stage="clean") + self.clean_impl(context, manifest, clean_helper, **kwargs) + if self.out_of_source_build: + build_directory = self.build_directory + assert build_directory is not None + if os.path.lexists(build_directory): + clean_helper.schedule_removal_of_directories(build_directory) + dest_dir = self.resolve_dest_dir() + if not isinstance(dest_dir, BinaryPackage): + clean_helper.schedule_removal_of_directories(dest_dir) + + @final + def perform_build( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._check_characteristics() + self.before_first_impl_step(stage="build") + self.configure_impl(context, manifest, **kwargs) + self.build_impl(context, manifest, **kwargs) + if context.should_run_tests: + self.test_impl(context, manifest, **kwargs) + dest_dir = self.resolve_dest_dir() + if isinstance(dest_dir, BinaryPackage): + dest_dir = f"debian/{dest_dir.name}" + # Make it absolute for everyone (that worked for debhelper). + # At least autoconf's "make install" requires an absolute path, so making is + # relative would have at least one known issue. + abs_dest_dir = os.path.abspath(dest_dir) + self.install_impl(context, manifest, abs_dest_dir, **kwargs) + + def before_first_impl_step( + self, + /, + stage: Literal["build", "clean"], + **kwargs, + ) -> None: + """Called before any `*_impl` method is called. + + This can be used to validate input against data that is not available statically + (that is, it will be checked during build but not in static checks). An example + is that the `debhelper` build system uses this to validate the provided `dh-build-system` + to ensure that `debhelper` knows about the build system. This check cannot be done + statically since the build system is only required to be available in a chroot build + and not on the host system. + + The method can also be used to compute common state for all the `*_impl` methods that + is awkward to do in `__init__`. Note there is no data sharing between the different + stages. This has to do with how `debputy` will be called (usually `clean` followed by + a source package assembly in `dpkg` and then `build`). + + The check is done both on build and on clean before the relevant implementation methods + are invoked. + + Any exception will abort the build. Prefer to raise ManifestInvalidUserDataException + exceptions for issues related to incorrect data. + + The method is not invoked if the steps are skipped, which can happen with build profiles + or arch:any vs. arch:all builds. + + :param stage: A discriminator variable to determine which kind of steps will be invoked + after this method returns. For state initialization, this can be useful if the state + is somewhat expensive and not needed for `clean`. + """ + pass + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + """Called to handle the "configure" and "build" part of the build + + This is basically a mix of `dh_auto_configure` and `dh_auto_build` from `debhelper`. + If the upstream build also runs test as a part of the build, this method should + check `context.should_run_tests` and pass the relevant flags to disable tests when + `context.should_run_tests` is false. + """ + raise NotImplementedError + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + """Called to handle the "configure" and "build" part of the build + + This is basically a mix of `dh_auto_configure` and `dh_auto_build` from `debhelper`. + If the upstream build also runs test as a part of the build, this method should + check `context.should_run_tests` and pass the relevant flags to disable tests when + `context.should_run_tests` is false. + """ + raise NotImplementedError + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + """Called to handle the "test" part of the build + + This is basically `dh_auto_test` from `debhelper`. + + Note: This will be skipped when `context.should_run_tests` is False. Therefore, the + method can assume that when invoked then tests must be run. + + It is always run after `configure_and_build_impl`. + """ + raise NotImplementedError + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + """Called to handle the "install" part of the build + + This is basically `dh_auto_install` from `debhelper`. + + The `dest_dir` attribute is what the upstream should install its data into. It + follows the `DESTDIR` convention from autoconf/make. The `dest_dir` should not + be second-guessed since `debputy` will provide automatically as a search path + for installation rules when relevant. + + It is always run after `configure_and_build_impl` and, if relevant, `test_impl`. + """ + raise NotImplementedError + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + """Called to handle the "clean" part of the build + + This is basically `dh_auto_clean` from `debhelper`. + + For out-of-source builds, `debputy` will remove the build directory for you + if it exists (when this method returns). This method is only "in-source" cleaning + or for "dirty" state left outside the designated build directory. + + Note that state *cannot* be shared between `clean` and other steps due to limitations + of how the Debian build system works in general. + """ + raise NotImplementedError + + def _check_characteristics(self) -> None: + characteristics = self.characteristics() + + _debug_log(f"Characteristics for {self.name_or_tag} {self.__class__.__name__} ") + + if self.out_of_source_build and self.perform_in_source_build: + raise ManifestInvalidUserDataException( + f"Cannot use 'build-directory' with 'perform-in-source-build' at {self.attribute_path.path}" + ) + if ( + characteristics.out_of_source_builds == "required" + and self.perform_in_source_build + ): + path = self.attribute_path["perform_in_source_build"].path_key_lc + + # FIXME: How do I determine the faulty plugin from here. + raise PluginAPIViolationError( + f"The build system {self.__class__.__qualname__} had an perform-in-source-build attribute, but claims" + f" it requires out of source builds. Please file a bug against the provider asking them not to use" + f' "{OptionalInSourceBuild.__name__}" as base for their build system definition or tweak' + f" the characteristics of the build system as the current combination is inconsistent." + f" The offending definition is at {path}." + ) + + if ( + characteristics.out_of_source_builds + in ("required", "supported-and-default") + and not self.out_of_source_build + ): + + if not self.perform_in_source_build: + self._build_directory = self._pick_build_dir() + else: + assert characteristics.out_of_source_builds != "required" + elif ( + characteristics.out_of_source_builds == "not-supported" + and self.out_of_source_build + ): + path = self.attribute_path["build_directory"].path_key_lc + + # FIXME: How do I determine the faulty plugin from here. + raise PluginAPIViolationError( + f"The build system {self.__class__.__qualname__} had a build-directory attribute, but claims it does" + f" not support out of source builds. Please file a bug against the provider asking them not to use" + f' "{OptionalBuildDirectory.__name__}" as base for their build system definition or tweak' + f" the characteristics of the build system as the current combination is inconsistent." + f" The offending definition is at {path}." + ) + + def _pick_build_dir(self) -> str: + tag = self.name if self.name is not None else self.auto_generated_stem + if tag == "": + return "_build" + return f"_build-{tag}" + + @final + def resolve_dest_dir(self) -> Union[str, BinaryPackage]: + auto_generated_stem = self.auto_generated_stem + if self.install_directly_to_package: + assert len(self.for_packages) == 1 + return next(iter(self.for_packages)) + if auto_generated_stem == "": + return "debian/tmp" + return f"debian/tmp-{auto_generated_stem}" + + +# Using the same logic as debhelper for the same reasons. +def _make_target_exists(make_cmd: str, target: str, *, directory: str = ".") -> bool: + cmd = [ + make_cmd, + "-s", + "-n", + "--no-print-directory", + ] + if directory and directory != ".": + cmd.append("-C") + cmd.append(directory) + cmd.append(target) + env = dict(os.environ) + env["LC_ALL"] = "C.UTF-8" + try: + res = subprocess.run( + cmd, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + restore_signals=True, + ) + except FileNotFoundError: + return False + + options = ( + f"*** No rule to make target '{target}", + f"*** No rule to make target `{target}", + ) + + stdout = res.stdout.decode("utf-8") + return not any(o in stdout for o in options) + + +def _find_first_existing_make_target( + make_cmd: str, + targets: Sequence[str], + *, + directory: str = ".", +) -> Optional[str]: + for target in targets: + if _make_target_exists(make_cmd, target, directory=directory): + return target + return None + + +_UNSET = object() + + +class NinjaBuildSupport: + __slots__ = ("_provided_ninja_program", "_build_system_rule") + + def __init__( + self, + provided_ninja_program: str, + build_system_rule: BuildSystemRule, + ) -> None: + self._provided_ninja_program = provided_ninja_program + self._build_system_rule = build_system_rule + + @classmethod + def from_build_system( + cls, + build_system: BuildSystemRule, + *, + ninja_program: Optional[str] = None, + ) -> Self: + if ninja_program is None: + ninja_program = "ninja" + return cls(ninja_program, build_system) + + @property + def _directory(self) -> str: + return self._build_system_rule.build_directory + + def _pick_directory( + self, arg: Union[Optional[str], _UNSET] = _UNSET + ) -> Optional[str]: + if arg is _UNSET: + return self._directory + return arg + + def run_ninja_build( + self, + build_context: "BuildContext", + *ninja_args: str, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + enable_parallelization: bool = True, + ) -> None: + extra_ninja_args = [] + if not build_context.is_terse_build: + extra_ninja_args.append("-v") + self._run_ninja( + build_context, + *extra_ninja_args, + *ninja_args, + env_mod=env_mod, + directory=directory, + enable_parallelization=enable_parallelization, + ) + + def run_ninja_test( + self, + build_context: "BuildContext", + *ninja_args: str, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + enable_parallelization: bool = True, + ) -> None: + self._run_ninja( + build_context, + "test", + *ninja_args, + env_mod=env_mod, + directory=directory, + enable_parallelization=enable_parallelization, + ) + + def run_ninja_install( + self, + build_context: "BuildContext", + dest_dir: str, + *ninja_args: str, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + # debhelper never had parallel installs, so we do not have it either for now. + enable_parallelization: bool = False, + ) -> None: + install_env_mod = EnvironmentModification( + replacements={ + "DESTDIR": dest_dir, + } + ) + if env_mod is not None: + install_env_mod = install_env_mod.combine(env_mod) + self._run_ninja( + build_context, + "install", + *ninja_args, + directory=directory, + env_mod=install_env_mod, + enable_parallelization=enable_parallelization, + ) + + def run_ninja_clean( + self, + build_context: "BuildContext", + *ninja_args: str, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + enable_parallelization: bool = True, + ) -> None: + self._run_ninja( + build_context, + "clean", + *ninja_args, + env_mod=env_mod, + directory=directory, + enable_parallelization=enable_parallelization, + ) + + def _run_ninja( + self, + build_context: "BuildContext", + *ninja_args: str, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + enable_parallelization: bool = True, + ) -> None: + extra_ninja_args = [] + limit = ( + build_context.parallelization_limit(support_zero_as_unlimited=True) + if enable_parallelization + else 1 + ) + extra_ninja_args.append(f"-j{limit}") + ninja_env_mod = EnvironmentModification( + replacements={ + "LC_ALL": "C.UTF-8", + } + ) + if env_mod is not None: + ninja_env_mod = ninja_env_mod.combine(env_mod) + run_build_system_command( + self._provided_ninja_program, + *extra_ninja_args, + *ninja_args, + cwd=self._pick_directory(directory), + env_mod=ninja_env_mod, + ) + + +class MakefileSupport: + + __slots__ = ("_provided_make_program", "_build_system_rule") + + def __init__( + self, + make_program: str, + build_system_rule: BuildSystemRule, + ) -> None: + self._provided_make_program = make_program + self._build_system_rule = build_system_rule + + @classmethod + def from_build_system( + cls, + build_system: BuildSystemRule, + *, + make_program: Optional[str] = None, + ) -> Self: + if make_program is None: + make_program = os.environ.get("MAKE", "make") + return cls(make_program, build_system) + + @property + def _directory(self) -> str: + return self._build_system_rule.build_directory + + @property + def _make_program(self) -> str: + make_program = self._provided_make_program + if self._provided_make_program is None: + return os.environ.get("MAKE", "make") + return make_program + + def _pick_directory( + self, arg: Union[Optional[str], _UNSET] = _UNSET + ) -> Optional[str]: + if arg is _UNSET: + return self._directory + return arg + + def find_first_existing_make_target( + self, + targets: Sequence[str], + *, + directory: Union[Optional[str], _UNSET] = _UNSET, + ) -> Optional[str]: + for target in targets: + if self.make_target_exists(target, directory=directory): + return target + return None + + def make_target_exists( + self, + target: str, + *, + directory: Union[Optional[str], _UNSET] = _UNSET, + ) -> bool: + return _make_target_exists( + self._make_program, + target, + directory=self._pick_directory(directory), + ) + + def run_first_existing_target_if_any( + self, + build_context: "BuildContext", + targets: Sequence[str], + *make_args: str, + enable_parallelization: bool = True, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + ) -> bool: + target = self.find_first_existing_make_target(targets, directory=directory) + if target is None: + return False + + self.run_make( + build_context, + target, + *make_args, + enable_parallelization=enable_parallelization, + directory=directory, + env_mod=env_mod, + ) + return True + + def run_make( + self, + build_context: "BuildContext", + *make_args: str, + enable_parallelization: bool = True, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + ) -> None: + limit = ( + build_context.parallelization_limit(support_zero_as_unlimited=True) + if enable_parallelization + else 1 + ) + extra_make_args = [f"-j{limit}"] if limit else ["-j"] + run_build_system_command( + self._make_program, + *extra_make_args, + *make_args, + cwd=self._pick_directory(directory), + env_mod=env_mod, + ) + + +def debputy_build_system( + # For future self: Before you get ideas about making manifest_keyword accept a list, + # remember it has consequences for shadowing_build_systems_when_active. + manifest_keyword: str, + provider: Type[BSR], + *, + expected_debputy_integration_mode: Optional[ + Container[DebputyIntegrationMode] + ] = None, + auto_detection_shadows_build_systems: Optional[ + Union[str, Iterable[str]] + ] = frozenset(), + online_reference_documentation: Optional[ParserDocumentation] = None, + apply_standard_attribute_documentation: bool = False, + source_format: Optional[Any] = None, +) -> Callable[[Type[BSPF]], Type[BSPF]]: + if not isinstance(provider, type) or not issubclass(provider, BuildSystemRule): + raise PluginInitializationError( + f"The provider for @{debputy_build_system.__name__} must be subclass of {BuildSystemRule.__name__}goes on the TypedDict that defines the parsed" + f" variant of the manifest definition. Not the build system implementation class." + ) + + def _constructor_wrapper( + _rule_used: str, + *args, + **kwargs, + ) -> BSR: + return provider(*args, **kwargs) + + if isinstance(auto_detection_shadows_build_systems, str): + shadows = frozenset([auto_detection_shadows_build_systems]) + else: + shadows = frozenset(auto_detection_shadows_build_systems) + + metadata = BuildSystemManifestRuleMetadata( + (manifest_keyword,), + BuildRule, + _constructor_wrapper, + expected_debputy_integration_mode=expected_debputy_integration_mode, + source_format=source_format, + online_reference_documentation=online_reference_documentation, + apply_standard_attribute_documentation=apply_standard_attribute_documentation, + auto_detection_shadow_build_systems=shadows, + build_system_impl=provider, + ) + + def _decorator_impl(pf_cls: Type[BSPF]) -> Type[BSPF]: + if isinstance(pf_cls, type) and issubclass(pf_cls, BuildSystemRule): + raise PluginInitializationError( + f"The @{debputy_build_system.__name__} annotation goes on the TypedDict that defines the parsed" + f" variant of the manifest definition. Not the build system implementation class." + ) + + # TODO: In python3.12 we can check more than just `is_typeddict`. In python3.11, woe is us and + # is_typeddict is the only thing that reliably works (cpython#103699) + if not is_typeddict(pf_cls): + raise PluginInitializationError( + f"Expected annotated class to be a subclass of {BuildRuleParsedFormat.__name__}," + f" but got {pf_cls.__name__} instead" + ) + + setattr(pf_cls, _DEBPUTY_DISPATCH_METADATA_ATTR_NAME, metadata) + return pf_cls + + return _decorator_impl diff --git a/src/debputy/plugin/plugin_state.py b/src/debputy/plugin/plugin_state.py new file mode 100644 index 0000000..ef4dabb --- /dev/null +++ b/src/debputy/plugin/plugin_state.py @@ -0,0 +1,113 @@ +import contextvars +import functools +import inspect +from contextvars import ContextVar +from typing import Optional, Callable, ParamSpec, TypeVar, NoReturn, Union + +from debputy.exceptions import ( + UnhandledOrUnexpectedErrorFromPluginError, + DebputyRuntimeError, +) +from debputy.util import _debug_log, _is_debug_log_enabled + +_current_debputy_plugin_cxt_var: ContextVar[Optional[str]] = ContextVar( + "current_debputy_plugin", + default=None, +) + +P = ParamSpec("P") +R = TypeVar("R") + + +def current_debputy_plugin_if_present() -> Optional[str]: + return _current_debputy_plugin_cxt_var.get() + + +def current_debputy_plugin_required() -> str: + v = current_debputy_plugin_if_present() + if v is None: + raise AssertionError( + "current_debputy_plugin_required() was called, but no plugin was set." + ) + return v + + +def wrap_plugin_code( + plugin_name: str, + func: Callable[P, R], + *, + non_debputy_exception_handling: Union[bool, Callable[[Exception], NoReturn]] = True, +) -> Callable[P, R]: + if isinstance(non_debputy_exception_handling, bool): + + runner = run_in_context_of_plugin + if non_debputy_exception_handling: + runner = run_in_context_of_plugin_wrap_errors + + def _wrapper(*args: P.args, **kwargs: P.kwargs) -> None: + return runner(plugin_name, func, *args, **kwargs) + + functools.update_wrapper(_wrapper, func) + return _wrapper + + def _wrapper(*args: P.args, **kwargs: P.kwargs) -> None: + try: + return run_in_context_of_plugin(plugin_name, func, *args, **kwargs) + except DebputyRuntimeError: + raise + except Exception as e: + non_debputy_exception_handling(e) + + functools.update_wrapper(_wrapper, func) + return _wrapper + + +def run_in_context_of_plugin( + plugin: str, + func: Callable[P, R], + *args: P.args, + **kwargs: P.kwargs, +) -> R: + context = contextvars.copy_context() + if _is_debug_log_enabled(): + call_stack = inspect.stack() + caller: str = "[N/A]" + for frame in call_stack: + if frame.filename != __file__: + try: + fname = frame.frame.f_code.co_qualname + except AttributeError: + fname = None + if fname is None: + fname = frame.function + caller = f"{frame.filename}:{frame.lineno} ({fname})" + break + # Do not keep the reference longer than necessary + del call_stack + _debug_log( + f"Switching plugin context to {plugin} at {caller} (from context: {current_debputy_plugin_if_present()})" + ) + # Wish we could just do a regular set without wrapping it in `context.run` + context.run(_current_debputy_plugin_cxt_var.set, plugin) + return context.run(func, *args, **kwargs) + + +def run_in_context_of_plugin_wrap_errors( + plugin: str, + func: Callable[P, R], + *args: P.args, + **kwargs: P.kwargs, +) -> R: + try: + return run_in_context_of_plugin(plugin, func, *args, **kwargs) + except DebputyRuntimeError: + raise + except Exception as e: + if plugin != "debputy": + raise UnhandledOrUnexpectedErrorFromPluginError( + f"{func.__qualname__} from the plugin {plugin} raised exception that was not expected here." + ) from e + else: + raise AssertionError( + "Bug in the `debputy` plugin: Unhandled exception." + ) from e diff --git a/src/debputy/transformation_rules.py b/src/debputy/transformation_rules.py index c7f8a2a..6e96c64 100644 --- a/src/debputy/transformation_rules.py +++ b/src/debputy/transformation_rules.py @@ -11,6 +11,7 @@ from typing import ( Dict, TypeVar, cast, + final, ) from debputy.exceptions import ( @@ -27,12 +28,15 @@ from debputy.manifest_parser.base_types import ( FileSystemMode, StaticFileSystemOwner, StaticFileSystemGroup, - DebputyDispatchableType, ) +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.manifest_parser.util import AttributePath from debputy.path_matcher import MatchRule from debputy.plugin.api import VirtualPath from debputy.plugin.debputy.types import DebputyCapability +from debputy.plugin.plugin_state import ( + run_in_context_of_plugin_wrap_errors, +) from debputy.util import _warn @@ -59,10 +63,26 @@ class PreProvidedExclusion: class TransformationRule(DebputyDispatchableType): + __slots__ = () + @final + def run_transform_file_system( + self, + fs_root: FSPath, + condition_context: ConditionContext, + ) -> None: + run_in_context_of_plugin_wrap_errors( + self._debputy_plugin, + self.transform_file_system, + fs_root, + condition_context, + ) + def transform_file_system( - self, fs_root: FSPath, condition_context: ConditionContext + self, + fs_root: FSPath, + condition_context: ConditionContext, ) -> None: raise NotImplementedError @@ -134,6 +154,7 @@ class RemoveTransformationRule(TransformationRule): keep_empty_parent_dirs: bool, definition_source: AttributePath, ) -> None: + super().__init__() self._match_rules = match_rules self._keep_empty_parent_dirs = keep_empty_parent_dirs self._definition_source = definition_source.path @@ -180,6 +201,7 @@ class MoveTransformationRule(TransformationRule): definition_source: AttributePath, condition: Optional[ManifestCondition], ) -> None: + super().__init__() self._match_rule = match_rule self._dest_path = dest_path self._dest_is_dir = dest_is_dir @@ -283,6 +305,7 @@ class CreateSymlinkPathTransformationRule(TransformationRule): definition_source: AttributePath, condition: Optional[ManifestCondition], ) -> None: + super().__init__() self._link_target = link_target self._link_dest = link_dest self._replacement_rule = replacement_rule @@ -550,6 +573,9 @@ class ModeNormalizationTransformationRule(TransformationRule): self, normalizations: Sequence[Tuple[MatchRule, FileSystemMode]], ) -> None: + # A bit of a hack since it is initialized outside `debputy`. It probably should not + # be a "TransformationRule" (hindsight and all) + run_in_context_of_plugin_wrap_errors("debputy", super().__init__) self._normalizations = normalizations def transform_file_system( @@ -575,6 +601,12 @@ class ModeNormalizationTransformationRule(TransformationRule): class NormalizeShebangLineTransformation(TransformationRule): + + def __init__(self) -> None: + # A bit of a hack since it is initialized outside `debputy`. It probably should not + # be a "TransformationRule" (hindsight and all) + run_in_context_of_plugin_wrap_errors("debputy", super().__init__) + def transform_file_system( self, fs_root: VirtualPath, diff --git a/src/debputy/types.py b/src/debputy/types.py index 05e68c9..dc3cbd3 100644 --- a/src/debputy/types.py +++ b/src/debputy/types.py @@ -1,9 +1,138 @@ -from typing import TypeVar, TYPE_CHECKING +import dataclasses +from typing import ( + TypeVar, + TYPE_CHECKING, + Sequence, + Tuple, + Mapping, + Dict, + Optional, + TypedDict, + NotRequired, + List, + MutableMapping, +) if TYPE_CHECKING: from debputy.plugin.api import VirtualPath from debputy.filesystem_scan import FSPath + VP = TypeVar("VP", VirtualPath, FSPath) + S = TypeVar("S", str, bytes) +else: + VP = TypeVar("VP", "VirtualPath", "FSPath") + S = TypeVar("S", str, bytes) -VP = TypeVar("VP", "VirtualPath", "FSPath") -S = TypeVar("S", str, bytes) + +class EnvironmentModificationSerialized(TypedDict): + replacements: NotRequired[Dict[str, str]] + removals: NotRequired[List[str]] + + +@dataclasses.dataclass(slots=True, frozen=True) +class EnvironmentModification: + replacements: Sequence[Tuple[str, str]] = tuple() + removals: Sequence[str] = tuple() + + @staticmethod + def from_serialized_format( + serial_form: EnvironmentModificationSerialized, + ) -> "EnvironmentModification": + replacements_raw = serial_form.get("replacements") + if replacements_raw is not None: + replacements = tuple((k, v) for k, v in replacements_raw.items()) + else: + replacements = tuple() + return EnvironmentModification( + replacements=replacements, removals=serial_form.get("removals", tuple()) + ) + + def __bool__(self) -> bool: + return not self.removals and not self.replacements + + def combine( + self, other: "Optional[EnvironmentModification]" + ) -> "EnvironmentModification": + if not other: + return self + existing_replacements = {k: v for k, v in self.replacements} + extra_replacements = { + k: v + for k, v in other.replacements + if k not in existing_replacements or existing_replacements[k] != v + } + seen_removals = set(self.removals) + extra_removals = [r for r in other.removals if r not in seen_removals] + + if not extra_replacements and isinstance(self.replacements, tuple): + new_replacements = self.replacements + else: + new_replacements = [] + for k, v in existing_replacements: + if k not in extra_replacements: + new_replacements.append((k, v)) + + for k, v in other.replacements: + if k in extra_replacements: + new_replacements.append((k, v)) + + new_replacements = tuple(new_replacements) + + if not extra_removals and isinstance(self.removals, tuple): + new_removals = self.removals + else: + new_removals = list(self.removals) + new_removals.extend(extra_removals) + new_removals = tuple(new_removals) + + if self.replacements is new_replacements and self.removals is new_removals: + return self + + return EnvironmentModification( + new_replacements, + new_removals, + ) + + def serialize(self) -> EnvironmentModificationSerialized: + serial_form = {} + replacements = self.replacements + if replacements: + serial_form["replacements"] = {k: v for k, v in replacements} + removals = self.removals + if removals: + serial_form["removals"] = list(removals) + return serial_form + + def update_inplace(self, env: MutableMapping[str, str]) -> None: + for k, v in self.replacements: + existing_value = env.get(k) + if v == existing_value: + continue + env[k] = v + + for k in self.removals: + if k not in env: + continue + del env[k] + + def compute_env(self, base_env: Mapping[str, str]) -> Mapping[str, str]: + updated_env: Optional[Dict[str, str]] = None + for k, v in self.replacements: + existing_value = base_env.get(k) + if v == existing_value: + continue + + if updated_env is None: + updated_env = dict(base_env) + updated_env[k] = v + + for k in self.removals: + if k not in base_env: + continue + if updated_env is None: + updated_env = dict(base_env) + del updated_env[k] + + if updated_env is not None: + return updated_env + return base_env diff --git a/src/debputy/util.py b/src/debputy/util.py index ebd38c2..911e6fa 100644 --- a/src/debputy/util.py +++ b/src/debputy/util.py @@ -34,7 +34,7 @@ from debian.deb822 import Deb822 from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable from debputy.exceptions import DebputySubstitutionError - +from debputy.types import EnvironmentModification try: from Levenshtein import distance @@ -105,7 +105,12 @@ _PROFILE_GROUP_SPLIT = re.compile(r">\s+<") _DEFAULT_LOGGER: Optional[logging.Logger] = None _STDOUT_HANDLER: Optional[logging.StreamHandler[Any]] = None _STDERR_HANDLER: Optional[logging.StreamHandler[Any]] = None -PRINT_COMMAND = logging.INFO + 5 +PRINT_COMMAND = logging.INFO + 3 +PRINT_BUILD_SYSTEM_COMMAND = PRINT_COMMAND + 3 + +# Map them back to `INFO`. The names must be unique so the prefix is stripped. +logging.addLevelName(PRINT_COMMAND, "__INFO") +logging.addLevelName(PRINT_BUILD_SYSTEM_COMMAND, "_INFO") def assume_not_none(x: Optional[T]) -> T: @@ -116,6 +121,13 @@ def assume_not_none(x: Optional[T]) -> T: return x +def _non_verbose_info(msg: str) -> None: + global _DEFAULT_LOGGER + logger = _DEFAULT_LOGGER + if logger is not None: + logger.log(PRINT_BUILD_SYSTEM_COMMAND, msg) + + def _info(msg: str) -> None: global _DEFAULT_LOGGER logger = _DEFAULT_LOGGER @@ -124,6 +136,20 @@ def _info(msg: str) -> None: # No fallback print for info +def _is_debug_log_enabled() -> bool: + global _DEFAULT_LOGGER + logger = _DEFAULT_LOGGER + return logger is not None and logger.isEnabledFor(logging.DEBUG) + + +def _debug_log(msg: str) -> None: + global _DEFAULT_LOGGER + logger = _DEFAULT_LOGGER + if logger: + logger.debug(msg) + # No fallback print for info + + def _error(msg: str, *, prog: Optional[str] = None) -> "NoReturn": global _DEFAULT_LOGGER logger = _DEFAULT_LOGGER @@ -226,9 +252,88 @@ def escape_shell(*args: str) -> str: return " ".join(_escape_shell_word(w) for w in args) -def print_command(*args: str, print_at_log_level: int = PRINT_COMMAND) -> None: - if _DEFAULT_LOGGER and _DEFAULT_LOGGER.isEnabledFor(print_at_log_level): - print(f" {escape_shell(*args)}") +def render_command( + *args: str, + cwd: Optional[str] = None, + env_mod: Optional[EnvironmentModification] = None, +) -> str: + env_mod_prefix = "" + if env_mod: + env_mod_parts = [] + if bool(env_mod.removals): + env_mod_parts.append("env") + if cwd is not None: + env_mod_parts.append(f"--chdir={escape_shell(cwd)}") + env_mod_parts.extend(f"--unset={escape_shell(v)}" for v in env_mod.removals) + env_mod_parts.extend( + f"{escape_shell(k)}={escape_shell(v)}" for k, v in env_mod.replacements + ) + + chdir_prefix = "" + if cwd is not None and cwd != ".": + chdir_prefix = f"cd {escape_shell(cwd)} && " + return f"{chdir_prefix}{env_mod_prefix}{escape_shell(*args)}" + + +def print_command( + *args: str, + cwd: Optional[str] = None, + env_mod: Optional[EnvironmentModification] = None, + print_at_log_level: int = PRINT_COMMAND, +) -> None: + if _DEFAULT_LOGGER is None or not _DEFAULT_LOGGER.isEnabledFor(print_at_log_level): + return + + rendered_cmd = render_command( + *args, + cwd=cwd, + env_mod=env_mod, + ) + print(f" {rendered_cmd}") + + +def run_command( + *args: str, + cwd: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, + env_mod: Optional[EnvironmentModification] = None, + print_at_log_level: int = PRINT_COMMAND, +) -> None: + print_command( + *args, + cwd=cwd, + env_mod=env_mod, + print_at_log_level=print_at_log_level, + ) + if env_mod: + if env is None: + env = os.environ + env = env_mod.compute_env(env) + if env is os.environ: + env = None + try: + subprocess.check_call(args, cwd=cwd, env=env) + # At least "clean_logic.py" relies on catching FileNotFoundError + except KeyboardInterrupt: + _error(f"Interrupted (SIGINT) while running {escape_shell(*args)}") + except subprocess.CalledProcessError as e: + _error(f"The command {escape_shell(*args)} failed with status: {e.returncode}") + + +def run_build_system_command( + *args: str, + cwd: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, + env_mod: Optional[EnvironmentModification] = None, + print_at_log_level: int = PRINT_BUILD_SYSTEM_COMMAND, +) -> None: + run_command( + *args, + cwd=cwd, + env=env, + env_mod=env_mod, + print_at_log_level=print_at_log_level, + ) def debian_policy_normalize_symlink_target( @@ -398,7 +503,7 @@ def integrated_with_debhelper() -> None: _DH_INTEGRATION_MODE = True -def scratch_dir() -> str: +def scratch_dir(*, create_if_not_exists: bool = True) -> str: global _SCRATCH_DIR if _SCRATCH_DIR is not None: return _SCRATCH_DIR @@ -411,9 +516,10 @@ def scratch_dir() -> str: is_debputy_dir = False else: _SCRATCH_DIR = debputy_scratch_dir - ensure_dir(_SCRATCH_DIR) - if is_debputy_dir: - Path("debian/.debputy/.gitignore").write_text("*\n") + if create_if_not_exists: + ensure_dir(_SCRATCH_DIR) + if is_debputy_dir: + Path("debian/.debputy/.gitignore").write_text("*\n") return _SCRATCH_DIR @@ -455,9 +561,11 @@ def generated_content_dir( return directory -PerlIncDir = collections.namedtuple("PerlIncDir", ["vendorlib", "vendorarch"]) +PerlConfigVars = collections.namedtuple( + "PerlIncDir", ["vendorlib", "vendorarch", "cross_inc_dir", "ld", "path_sep"] +) PerlConfigData = collections.namedtuple("PerlConfigData", ["version", "debian_abi"]) -_PERL_MODULE_DIRS: Dict[str, PerlIncDir] = {} +_PERL_MODULE_DIRS: Dict[str, PerlConfigVars] = {} @functools.lru_cache(1) @@ -490,42 +598,56 @@ def perlxs_api_dependency() -> str: return f"perlapi-{config.version}" -def perl_module_dirs( +def resolve_perl_config( dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable, - dctrl_bin: "BinaryPackage", -) -> PerlIncDir: + dctrl_bin: Optional["BinaryPackage"], +) -> PerlConfigVars: global _PERL_MODULE_DIRS - arch = ( - dctrl_bin.resolved_architecture - if dpkg_architecture_variables.is_cross_compiling - else "_default_" - ) - module_dir = _PERL_MODULE_DIRS.get(arch) - if module_dir is None: + if dpkg_architecture_variables.is_cross_compiling: + arch = ( + dctrl_bin.resolved_architecture + if dctrl_bin is not None + else dpkg_architecture_variables.current_host_arch + ) + else: + arch = "_build_arch_" + config_vars = _PERL_MODULE_DIRS.get(arch) + if config_vars is None: cmd = ["perl"] if dpkg_architecture_variables.is_cross_compiling: version = _perl_version() - inc_dir = f"/usr/lib/{dctrl_bin.deb_multiarch}/perl/cross-config-{version}" + cross_inc_dir = ( + f"/usr/lib/{dctrl_bin.deb_multiarch}/perl/cross-config-{version}" + ) # FIXME: This should not fallback to "build-arch" but on the other hand, we use the perl module dirs # for every package at the moment. So mandating correct perl dirs implies mandating perl-xs-dev in # cross builds... meh. - if os.path.exists(os.path.join(inc_dir, "Config.pm")): - cmd.append(f"-I{inc_dir}") + if os.path.exists(os.path.join(cross_inc_dir, "Config.pm")): + cmd.append(f"-I{cross_inc_dir}") + else: + cross_inc_dir = None cmd.extend( - ["-MConfig", "-e", 'print "$Config{vendorlib}\n$Config{vendorarch}\n"'] + [ + "-MConfig", + "-e", + 'print "$Config{vendorlib}\n$Config{vendorarch}\n$Config{ld}\n$Config{path_sep}\n"', + ] ) output = subprocess.check_output(cmd).decode("utf-8").splitlines(keepends=False) - if len(output) != 2: + if len(output) != 4: raise ValueError( "Internal error: Unable to determine the perl include directories:" f" Raw output from perl snippet: {output}" ) - module_dir = PerlIncDir( - vendorlib=_normalize_path(output[0]), - vendorarch=_normalize_path(output[1]), + config_vars = PerlConfigVars( + vendorlib="/" + _normalize_path(output[0], with_prefix=False), + vendorarch="/" + _normalize_path(output[1], with_prefix=False), + cross_inc_dir=cross_inc_dir, + ld=output[2], + path_sep=output[3], ) - _PERL_MODULE_DIRS[arch] = module_dir - return module_dir + _PERL_MODULE_DIRS[arch] = config_vars + return config_vars @functools.lru_cache(1) @@ -736,6 +858,12 @@ def change_log_level( _DEFAULT_LOGGER.setLevel(log_level) +def current_log_level() -> Optional[int]: + if _DEFAULT_LOGGER is not None: + return _DEFAULT_LOGGER.level + return None + + def setup_logging( *, log_only_to_stderr: bool = False, @@ -748,13 +876,20 @@ def setup_logging( " Use reconfigure_logging=True if you need to reconfigure it" ) stdout_color, stderr_color, bad_request = _check_color() + colors: Optional[Dict[str, str]] = None if stdout_color or stderr_color: try: import colorlog + except ImportError: stdout_color = False stderr_color = False + else: + colors = dict(colorlog.default_log_colors) + # Add our custom levels. + colors["_INFO"] = colors["INFO"] + colors["__INFO"] = colors["INFO"] if log_only_to_stderr: stdout = sys.stderr @@ -785,7 +920,12 @@ def setup_logging( if stdout_color: stdout_handler = colorlog.StreamHandler(stdout) stdout_handler.setFormatter( - colorlog.ColoredFormatter(color_format, style="{", force_color=True) + colorlog.ColoredFormatter( + color_format, + style="{", + force_color=True, + log_colors=colors, + ) ) logger = colorlog.getLogger() if existing_stdout_handler is not None: @@ -804,7 +944,12 @@ def setup_logging( if stderr_color: stderr_handler = colorlog.StreamHandler(sys.stderr) stderr_handler.setFormatter( - colorlog.ColoredFormatter(color_format, style="{", force_color=True) + colorlog.ColoredFormatter( + color_format, + style="{", + force_color=True, + log_colors=colors, + ) ) logger = logging.getLogger() if existing_stderr_handler is not None: @@ -831,6 +976,7 @@ def setup_logging( *args: Any, **kwargs: Any ) -> logging.LogRecord: # pragma: no cover record = old_factory(*args, **kwargs) + record.levelname = record.levelname.lstrip("_") record.levelnamelower = record.levelname.lower() return record diff --git a/tests/lint_tests/lint_tutil.py b/tests/lint_tests/lint_tutil.py index c32ad3c..c16fde3 100644 --- a/tests/lint_tests/lint_tutil.py +++ b/tests/lint_tests/lint_tutil.py @@ -9,7 +9,10 @@ from debputy.linting.lint_util import ( LintStateImpl, LintState, ) -from debputy.lsp.maint_prefs import MaintainerPreferenceTable, EffectivePreference +from debputy.lsp.maint_prefs import ( + MaintainerPreferenceTable, + EffectiveFormattingPreference, +) from debputy.packages import DctrlParser from debputy.plugin.api.feature_set import PluginProvidedFeatureSet @@ -43,7 +46,7 @@ class LintWrapper: self._dctrl_parser = dctrl_parser self.source_root: Optional[VirtualPathBase] = None self.lint_maint_preference_table = MaintainerPreferenceTable({}, {}) - self.effective_preference: Optional[EffectivePreference] = None + self.effective_preference: Optional[EffectiveFormattingPreference] = None def __call__(self, lines: List[str]) -> Optional[List["Diagnostic"]]: source_package = None diff --git a/tests/lint_tests/test_lint_dctrl.py b/tests/lint_tests/test_lint_dctrl.py index 80d7525..229acc1 100644 --- a/tests/lint_tests/test_lint_dctrl.py +++ b/tests/lint_tests/test_lint_dctrl.py @@ -1209,3 +1209,72 @@ def test_dctrl_lint_dep_field_restricted_or_relations( assert problem_text == "|" assert f"{issue.range}" == "13:1-13:2" assert issue.severity == DiagnosticSeverity.Error + + +def test_dctrl_duplicate_key(line_linter: LintWrapper) -> None: + lines = textwrap.dedent( + f"""\ + Source: jquery-tablesorter + Section: javascript + Priority: optional + Maintainer: Debian Javascript Maintainers <pkg-javascript-devel@lists.alioth.de\ + bian.org> + Uploaders: Paul Gevers <elbrus@debian.org> + Build-Depends: + debhelper-compat (=13), + grunt, + libjs-qunit, + node-grunt-contrib-clean, + node-grunt-contrib-copy, + node-grunt-contrib-uglify, + node-grunt-contrib-concat, + Standards-Version: {CURRENT_STANDARDS_VERSION} + Homepage: https://github.com/Mottie/tablesorter + Vcs-Git: https://salsa.debian.org/js-team/jquery-tablesorter.git + Vcs-Browser: https://salsa.debian.org/js-team/jquery-tablesorter + Rules-Requires-Root: no + + Package: libjs-jquery-tablesorter + Architecture: all + Multi-Arch: foreign + Depends: + ${{misc:Depends}}, + libjs-jquery, + libjs-jquery-metadata, + Recommends: javascript-common + Multi-Arch: foreign + Description: jQuery flexible client-side table sorting plugin + Tablesorter is a jQuery plugin for turning a standard HTML table with THEAD + and TBODY tags into a sortable table without page refreshes. Tablesorter can + successfully parse and sort many types of data including linked data in a + cell. It has many useful features including: + . + * Multi-column alphanumeric sorting and filtering. + * Multi-tbody sorting + * Supports Bootstrap v2-4. + * Parsers for sorting text, alphanumeric text, URIs, integers, currency, + floats, IP addresses, dates (ISO, long and short formats) and time. + Add your own easily. + * Inline editing + * Support for ROWSPAN and COLSPAN on TH elements. + * Support secondary "hidden" sorting (e.g., maintain alphabetical sort when + sorting on other criteria). + * Extensibility via widget system. + * Cross-browser: IE 6.0+, FF 2+, Safari 2.0+, Opera 9.0+, Chrome 5.0+. + + """ + ).splitlines(keepends=True) + + diagnostics = line_linter(lines) + assert len(diagnostics) == 1 + + issue = diagnostics[0] + + msg = ( + "The Multi-Arch field name was used multiple times in this stanza." + " Please ensure the field is only used once per stanza. Note that Multi-Arch and" + " X[BCS]-Multi-Arch are considered the same field." + ) + assert issue.message == msg + assert f"{issue.range}" == "27:0-27:10" + assert issue.severity == DiagnosticSeverity.Error diff --git a/tests/lsp_tests/test_lsp_debputy_manifest_completer.py b/tests/lsp_tests/test_lsp_debputy_manifest_completer.py index f052164..92056ed 100644 --- a/tests/lsp_tests/test_lsp_debputy_manifest_completer.py +++ b/tests/lsp_tests/test_lsp_debputy_manifest_completer.py @@ -200,7 +200,7 @@ def test_basic_debputy_completer_manifest_variable_value( ) assert isinstance(completions, list) keywords = {m.label for m in completions} - assert "0.1" in keywords + assert "'0.1'" in keywords cursor_pos = put_doc_with_cursor( ls, @@ -219,7 +219,7 @@ def test_basic_debputy_completer_manifest_variable_value( ) assert isinstance(completions, list) keywords = {m.label for m in completions} - assert "0.1" in keywords + assert "'0.1'" in keywords def test_basic_debputy_completer_install_rule_dispatch_key( diff --git a/tests/test_debputy_plugin.py b/tests/test_debputy_plugin.py index dc60597..8a4cc59 100644 --- a/tests/test_debputy_plugin.py +++ b/tests/test_debputy_plugin.py @@ -19,6 +19,16 @@ from debputy.plugin.api.test_api import ( from debputy.plugin.api.test_api import manifest_variable_resolution_context from debputy.plugin.api.test_api.test_impl import initialize_plugin_under_test_preloaded from debputy.plugin.api.test_api.test_spec import DetectedService +from debputy.plugin.debputy.build_system_rules import ( + AutoconfBuildSystemRule, + MakefileBuildSystemRule, + PerlBuildBuildSystemRule, + PerlMakeMakerBuildSystemRule, + QmakeBuildSystemRule, + Qmake6BuildSystemRule, + CMakeBuildSystemRule, + MesonBuildSystemRule, +) from debputy.plugin.debputy.debputy_plugin import initialize_debputy_features from debputy.plugin.debputy.private_api import load_libcap from debputy.plugin.debputy.service_management import SystemdServiceContext @@ -1252,3 +1262,129 @@ def test_auto_depends_solink() -> None: context=context_too_many_matches, ) assert "misc:Depends" not in sodep_metadata.substvars + + +@pytest.mark.parametrize( + "filename,expected,mode,content", + [ + ("configure.ac", True, 0o0644, None), + ("configure.in", True, 0o0644, "AC_INIT"), + ("configure.in", True, 0o0644, "AC_PREREQ"), + ("configure.in", False, 0o0644, "None of the above"), + ("configure", True, 0o0755, "GNU Autoconf"), + ("configure", False, 0o0644, "GNU Autoconf"), + ("configure", False, 0o0755, "No magic keyword"), + ("random-file", False, 0o0644, "No configure at all"), + ], +) +def test_auto_detect_autoconf_build_system( + filename: str, + expected: bool, + mode: int, + content: Optional[str], +) -> None: + fs_root = build_virtual_file_system( + [virtual_path_def(filename, mode=mode, content=content)] + ) + detected = AutoconfBuildSystemRule.auto_detect_build_system(fs_root) + assert detected == expected + + +@pytest.mark.parametrize( + "filename,expected", + [ + ("GNUmakefile", True), + ("Makefile", True), + ("makefile", True), + ("random-file", False), + ], +) +def test_auto_detect_make_build_system( + filename: str, + expected: bool, +) -> None: + fs_root = build_virtual_file_system([filename]) + detected = MakefileBuildSystemRule.auto_detect_build_system(fs_root) + assert detected == expected + + +@pytest.mark.parametrize( + "filename,expected", + [ + ("Build.PL", True), + ("random-file", False), + ], +) +def test_auto_detect_perl_build_build_system( + filename: str, + expected: bool, +) -> None: + fs_root = build_virtual_file_system([filename]) + detected = PerlBuildBuildSystemRule.auto_detect_build_system(fs_root) + assert detected == expected + + +@pytest.mark.parametrize( + "filename,expected", + [ + ("Makefile.PL", True), + ("random-file", False), + ], +) +def test_auto_detect_perl_makemaker_build_system( + filename: str, + expected: bool, +) -> None: + fs_root = build_virtual_file_system([filename]) + detected = PerlMakeMakerBuildSystemRule.auto_detect_build_system(fs_root) + assert detected == expected + + +@pytest.mark.parametrize( + "filename,expected", + [ + ("foo.pro", True), + ("random-file", False), + ], +) +def test_auto_detect_qmake_build_systems( + filename: str, + expected: bool, +) -> None: + fs_root = build_virtual_file_system([filename]) + detected_qmake = QmakeBuildSystemRule.auto_detect_build_system(fs_root) + detected_qmake6 = Qmake6BuildSystemRule.auto_detect_build_system(fs_root) + assert detected_qmake == expected + assert detected_qmake6 == expected + + +@pytest.mark.parametrize( + "filename,expected", + [ + ("CMakeLists.txt", True), + ("random-file", False), + ], +) +def test_auto_detect_cmake_build_systems( + filename: str, + expected: bool, +) -> None: + fs_root = build_virtual_file_system([filename]) + detected = CMakeBuildSystemRule.auto_detect_build_system(fs_root) + assert detected == expected + + +@pytest.mark.parametrize( + "filename,expected", + [ + ("meson.build", True), + ("random-file", False), + ], +) +def test_auto_detect_meson_build_systems( + filename: str, + expected: bool, +) -> None: + fs_root = build_virtual_file_system([filename]) + detected = MesonBuildSystemRule.auto_detect_build_system(fs_root) + assert detected == expected diff --git a/tests/test_declarative_parser.py b/tests/test_declarative_parser.py index 26291dd..d52f1c3 100644 --- a/tests/test_declarative_parser.py +++ b/tests/test_declarative_parser.py @@ -10,11 +10,12 @@ from typing import ( import pytest from debputy.highlevel_manifest import PackageTransformationDefinition -from debputy.manifest_parser.base_types import DebputyParsedContent, TypeMapping -from debputy.manifest_parser.declarative_parser import ( - DebputyParseHint, - ParserGenerator, +from debputy.manifest_parser.tagging_types import ( + DebputyParsedContent, + TypeMapping, ) +from debputy.manifest_parser.parse_hints import DebputyParseHint +from debputy.manifest_parser.declarative_parser import ParserGenerator from debputy.manifest_parser.mapper_code import type_mapper_str2package from debputy.manifest_parser.parser_data import ParserContextData from debputy.manifest_parser.util import AttributePath diff --git a/tests/test_migrations.py b/tests/test_migrations.py index f53c716..b179cab 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -15,7 +15,7 @@ from debputy.dh_migration.migrators_impl import ( migrate_install_file, migrate_maintscript, migrate_links_files, - detect_dh_addons, + detect_dh_addons_with_zz_integration, migrate_not_installed_file, migrate_installman_file, migrate_bash_completion, @@ -1465,7 +1465,7 @@ def test_detect_dh_addons( accept_no_migration_issues: AcceptableMigrationIssues, accept_any_migration_issues: AcceptableMigrationIssues, ) -> None: - migrator = detect_dh_addons + migrator = detect_dh_addons_with_zz_integration empty_fs = build_virtual_file_system([DEBIAN_DIR_ENTRY]) dctrl_no_addons_content = textwrap.dedent( diff --git a/tests/test_parser.py b/tests/test_parser.py index 1c84445..0b1ed56 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -489,3 +489,86 @@ def test_yaml_clean_after_removal_unsafe_path( else: with pytest.raises(ManifestParseException) as e_info: manifest_parser_pkg_foo.parse_manifest(fd=content) + + +def test_yaml_build_environment_default( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: + content = textwrap.dedent( + ( + """\ + + manifest-version: '0.1' + default-build-environment: + set: + FOO: "bar" + builds: + # FIXME: we should not require an empty dict here + - autoconf: {} + """ + ) + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + envs = manifest.build_environments + assert not envs.environments + base_env = {} + envs.default_environment.update_env(base_env) + assert "FOO" in base_env + build_rule = manifest.build_rules[0] + assert build_rule.environment is envs.default_environment + + +def test_yaml_build_environments_no_default( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: + content = textwrap.dedent( + ( + f"""\ + + manifest-version: '0.1' + build-environments: + - name: custom-env + set: + FOO: "bar" + builds: + - autoconf: + environment: custom-env + """ + ) + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + envs = manifest.build_environments + assert "custom-env" in envs.environments + custom_env = envs.environments["custom-env"] + assert envs.default_environment is None + base_env = {} + custom_env.update_env(base_env) + assert "FOO" in base_env + build_rule = manifest.build_rules[0] + assert build_rule.environment is custom_env + + +def test_yaml_build_environments_no_default_error( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: + content = textwrap.dedent( + ( + """\ + + manifest-version: '0.1' + build-environments: + - name: custom-env + set: + FOO: "bar" + builds: + # FIXME: we should not require an empty dict here + - autoconf: {} + """ + ) + ) + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + expected_msg = "The following named environments were never referenced: custom-env" + msg = e_info.value.args[0] + assert msg == expected_msg diff --git a/tests/test_style.py b/tests/test_style.py index d3cfb14..8f5b6ca 100644 --- a/tests/test_style.py +++ b/tests/test_style.py @@ -7,7 +7,7 @@ from debputy.yaml.compat import CommentedMap from debputy.lsp.maint_prefs import ( MaintainerPreferenceTable, determine_effective_preference, - EffectivePreference, + EffectiveFormattingPreference, _WAS_DEFAULTS, ) from debputy.packages import SourcePackage @@ -16,19 +16,25 @@ from debputy.packages import SourcePackage def test_load_styles() -> None: styles = MaintainerPreferenceTable.load_preferences() assert "niels@thykier.net" in styles.maintainer_preferences - nt_style = styles.maintainer_preferences["niels@thykier.net"] + nt_maint_pref = styles.maintainer_preferences["niels@thykier.net"] # Note this is data dependent; if it fails because the style changes, update the test - assert nt_style.canonical_name == "Niels Thykier" - assert not nt_style.is_packaging_team - assert nt_style.formatting_deb822_normalize_field_content - assert nt_style.formatting_deb822_short_indent - assert nt_style.formatting_deb822_always_wrap - assert nt_style.formatting_deb822_trailing_separator - assert nt_style.formatting_deb822_max_line_length == 79 - assert not nt_style.formatting_deb822_normalize_stanza_order + assert nt_maint_pref.canonical_name == "Niels Thykier" + assert not nt_maint_pref.is_packaging_team + black_style = styles.named_styles["black"] + nt_style = nt_maint_pref.formatting + assert nt_style is not None + assert black_style == black_style - # TODO: Not implemented yet - assert not nt_style.formatting_deb822_normalize_field_order + +def test_load_no_styles() -> None: + styles = MaintainerPreferenceTable.load_preferences() + assert "packages@qa.debian.org" in styles.maintainer_preferences + qa_maint_pref = styles.maintainer_preferences["packages@qa.debian.org"] + assert qa_maint_pref.canonical_name == "Debian QA Group" + assert qa_maint_pref.is_packaging_team + # Orphaned packages do not have a predefined style, since Debian (nor Debian QA) have + # one well-defined style. + assert qa_maint_pref.formatting is None def test_load_named_styles() -> None: @@ -36,15 +42,15 @@ def test_load_named_styles() -> None: assert "black" in styles.named_styles black_style = styles.named_styles["black"] # Note this is data dependent; if it fails because the style changes, update the test - assert black_style.formatting_deb822_normalize_field_content - assert black_style.formatting_deb822_short_indent - assert black_style.formatting_deb822_always_wrap - assert black_style.formatting_deb822_trailing_separator - assert black_style.formatting_deb822_max_line_length == 79 - assert not black_style.formatting_deb822_normalize_stanza_order + assert black_style.deb822_normalize_field_content + assert black_style.deb822_short_indent + assert black_style.deb822_always_wrap + assert black_style.deb822_trailing_separator + assert black_style.deb822_max_line_length == 79 + assert not black_style.deb822_normalize_stanza_order # TODO: Not implemented yet - assert not black_style.formatting_deb822_normalize_field_order + assert not black_style.deb822_normalize_field_order def test_compat_styles() -> None: @@ -56,11 +62,11 @@ def test_compat_styles() -> None: assert "random-package@packages.debian.org" not in styles.maintainer_preferences assert "random@example.org" not in styles.maintainer_preferences - nt_pref = styles.maintainer_preferences["niels@thykier.net"].as_effective_pref() - zeha_pref = styles.maintainer_preferences["zeha@debian.org"].as_effective_pref() + nt_style = styles.maintainer_preferences["niels@thykier.net"].formatting + zeha_style = styles.maintainer_preferences["zeha@debian.org"].formatting # Data dependency - assert nt_pref == zeha_pref + assert nt_style == zeha_style fields = Deb822( { @@ -72,7 +78,7 @@ def test_compat_styles() -> None: src = SourcePackage(fields) effective_style, tool, _ = determine_effective_preference(styles, src, None) - assert effective_style == nt_pref + assert effective_style == nt_style assert tool == "debputy reformat" fields["Uploaders"] = ( @@ -81,8 +87,8 @@ def test_compat_styles() -> None: src = SourcePackage(fields) effective_style, tool, _ = determine_effective_preference(styles, src, None) - assert effective_style == nt_pref - assert effective_style == zeha_pref + assert effective_style == nt_style + assert effective_style == zeha_style assert tool == "debputy reformat" fields["Uploaders"] = ( @@ -112,7 +118,7 @@ def test_compat_styles_team_maint() -> None: team_style = styles.maintainer_preferences["team@lists.debian.org"] assert team_style.is_packaging_team effective_style, tool, _ = determine_effective_preference(styles, src, None) - assert effective_style == team_style.as_effective_pref() + assert effective_style == team_style.formatting assert tool is None @@ -158,7 +164,7 @@ def test_was_from_salsa_ci_style() -> None: {"variables": CommentedMap({"SALSA_CI_DISABLE_WRAP_AND_SORT": "no"})} ) effective_style, tool, _ = determine_effective_preference(styles, src, salsa_ci) - was_style = EffectivePreference(**_WAS_DEFAULTS) + was_style = EffectiveFormattingPreference(**_WAS_DEFAULTS) assert effective_style == was_style assert tool == "wrap-and-sort" @@ -169,37 +175,37 @@ def test_was_from_salsa_ci_style() -> None: ( "-a", { - "formatting_deb822_always_wrap": True, + "deb822_always_wrap": True, }, ), ( "-sa", { - "formatting_deb822_always_wrap": True, - "formatting_deb822_short_indent": True, + "deb822_always_wrap": True, + "deb822_short_indent": True, }, ), ( "-sa --keep-first", { - "formatting_deb822_always_wrap": True, - "formatting_deb822_short_indent": True, + "deb822_always_wrap": True, + "deb822_short_indent": True, }, ), ( "-sab --keep-first", { - "formatting_deb822_always_wrap": True, - "formatting_deb822_short_indent": True, - "formatting_deb822_normalize_stanza_order": True, + "deb822_always_wrap": True, + "deb822_short_indent": True, + "deb822_normalize_stanza_order": True, }, ), ( "-sab --no-keep-first", { - "formatting_deb822_always_wrap": True, - "formatting_deb822_short_indent": True, - "formatting_deb822_normalize_stanza_order": False, + "deb822_always_wrap": True, + "deb822_short_indent": True, + "deb822_normalize_stanza_order": False, }, ), ], @@ -232,7 +238,7 @@ def test_was_from_salsa_ci_style_args( assert effective_style is None assert tool is None else: - was_style = EffectivePreference(**_WAS_DEFAULTS).replace( + was_style = EffectiveFormattingPreference(**_WAS_DEFAULTS).replace( **style_delta, ) |