diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-08-26 10:22:39 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-08-26 10:22:39 +0000 |
commit | 9e85cf35d5b159f75489bfdf9c7348e2466834a8 (patch) | |
tree | 9ef8d4b6db2d7764642051edfe98073c493c3d8a /src | |
parent | Adding debian version 0.1.44. (diff) | |
download | debputy-9e85cf35d5b159f75489bfdf9c7348e2466834a8.tar.xz debputy-9e85cf35d5b159f75489bfdf9c7348e2466834a8.zip |
Merging upstream version 0.1.45.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src')
57 files changed, 6395 insertions, 827 deletions
diff --git a/src/debputy/_manifest_constants.py b/src/debputy/_manifest_constants.py index 3ed992b..974ef7b 100644 --- a/src/debputy/_manifest_constants.py +++ b/src/debputy/_manifest_constants.py @@ -8,6 +8,8 @@ assert DEFAULT_MANIFEST_VERSION in SUPPORTED_MANIFEST_VERSIONS MK_MANIFEST_VERSION = "manifest-version" MK_PACKAGES = "packages" +MK_BUILDS = "builds" + MK_INSTALLATIONS = "installations" MK_INSTALLATIONS_INSTALL = "install" MK_INSTALLATIONS_MULTI_DEST_INSTALL = "multi-dest-install" diff --git a/src/debputy/build_support/__init__.py b/src/debputy/build_support/__init__.py new file mode 100644 index 0000000..8123659 --- /dev/null +++ b/src/debputy/build_support/__init__.py @@ -0,0 +1,7 @@ +from debputy.build_support.build_logic import perform_builds +from debputy.build_support.clean_logic import perform_clean + +__all__ = [ + "perform_clean", + "perform_builds", +] diff --git a/src/debputy/build_support/build_context.py b/src/debputy/build_support/build_context.py new file mode 100644 index 0000000..2eeef66 --- /dev/null +++ b/src/debputy/build_support/build_context.py @@ -0,0 +1,100 @@ +from typing import Mapping, Optional + +from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable +from debputy.commands.debputy_cmd.context import CommandContext +from debputy.highlevel_manifest import HighLevelManifest +from debputy.manifest_conditions import _run_build_time_tests + + +class BuildContext: + @staticmethod + def from_command_context( + cmd_context: CommandContext, + ) -> "BuildContext": + return BuildContextImpl(cmd_context) + + @property + def deb_build_options(self) -> Mapping[str, Optional[str]]: + raise NotImplementedError + + def parallelization_limit(self, *, support_zero_as_unlimited: bool = False) -> int: + """Parallelization limit of the build + + This is accessor that reads the `parallel` option from `DEB_BUILD_OPTIONS` with relevant + fallback behavior. + + :param support_zero_as_unlimited: The debhelper framework allowed `0` to mean unlimited + in some build systems. If the build system supports this, it should set this option + to True, which will allow `0` as a possible return value. WHen this option is False + (which is the default), `0` will be remapped to a high number to preserve the effect + in spirit (said fallback number is also from `debhelper`). + """ + limit = self.deb_build_options.get("parallel") + if limit is None: + return 1 + try: + v = int(limit) + except ValueError: + return 1 + if v == 0 and not support_zero_as_unlimited: + # debhelper allowed "0" to be used as unlimited in some cases. Preserve that feature + # for callers that are prepared for it. For everyone else, remap 0 to an obscene number + # that de facto has the same behaviour + # + # The number is taken out of `cmake.pm` from `debhelper` to be "Bug compatible" with + # debhelper on the fallback as well. + return 999 + return v + + @property + def is_terse_build(self) -> bool: + """Whether the build is terse + + This is a shorthand for testing for `terse` in DEB_BUILD_OPTIONS + """ + return "terse" in self.deb_build_options + + @property + def is_cross_compiling(self) -> bool: + """Whether the build is considered a cross build + + Note: Do **not** use this as indicator for whether tests should run. Use `should_run_tests` instead. + To the naive eye, they seem like they overlap in functionality, but they do not. There are cross + builds where tests can be run. Additionally, there are non-cross-builds where tests should be + skipped. + """ + return self.dpkg_architecture_variables.is_cross_compiling + + def cross_tool(self, command: str) -> str: + if not self.is_cross_compiling: + return command + cross_prefix = self.dpkg_architecture_variables["DEB_HOST_GNU_TYPE"] + return f"{cross_prefix}-{command}" + + @property + def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable: + raise NotImplementedError + + @property + def should_run_tests(self) -> bool: + return _run_build_time_tests(self.deb_build_options) + + +class BuildContextImpl(BuildContext): + def __init__( + self, + cmd_context: CommandContext, + ) -> None: + self._cmd_context = cmd_context + + @property + def deb_build_options(self) -> Mapping[str, Optional[str]]: + return self._cmd_context.deb_build_options + + @property + def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable: + return self._cmd_context.dpkg_architecture_variables() + + @property + def manifest(self) -> HighLevelManifest: + return self._manifest diff --git a/src/debputy/build_support/build_logic.py b/src/debputy/build_support/build_logic.py new file mode 100644 index 0000000..ee247e7 --- /dev/null +++ b/src/debputy/build_support/build_logic.py @@ -0,0 +1,193 @@ +import collections +import contextlib +import os +from typing import ( + Iterator, + Mapping, + List, + Dict, + Optional, +) + +from debputy.build_support.build_context import BuildContext +from debputy.build_support.buildsystem_detection import ( + auto_detect_buildsystem, +) +from debputy.commands.debputy_cmd.context import CommandContext +from debputy.highlevel_manifest import HighLevelManifest +from debputy.manifest_parser.base_types import BuildEnvironmentDefinition +from debputy.plugin.debputy.to_be_api_types import BuildRule +from debputy.util import ( + _error, + _info, + _non_verbose_info, +) + + +@contextlib.contextmanager +def in_build_env(build_env: BuildEnvironmentDefinition): + remove_unnecessary_env() + # Should possibly be per build + with _setup_build_env(build_env): + yield + + +def _set_stem_if_absent(stems: List[Optional[str]], idx: int, stem: str) -> None: + if stems[idx] is None: + stems[idx] = stem + + +def assign_stems( + build_rules: List[BuildRule], + manifest: HighLevelManifest, +) -> None: + if not build_rules: + return + if len(build_rules) == 1: + build_rules[0].auto_generated_stem = "" + return + + debs = {p.name for p in manifest.all_packages if p.package_type == "deb"} + udebs = {p.name for p in manifest.all_packages if p.package_type == "udeb"} + deb_only_builds: List[int] = [] + udeb_only_builds: List[int] = [] + by_name_only_builds: Dict[str, List[int]] = collections.defaultdict(list) + stems = [rule.name for rule in build_rules] + reserved_stems = set(n for n in stems if n is not None) + + for idx, rule in enumerate(build_rules): + stem = stems[idx] + if stem is not None: + continue + pkg_names = {p.name for p in rule.for_packages} + if pkg_names == debs: + deb_only_builds.append(idx) + elif pkg_names == udebs: + udeb_only_builds.append(idx) + + if len(pkg_names) == 1: + pkg_name = next(iter(pkg_names)) + by_name_only_builds[pkg_name].append(idx) + + if "deb" not in reserved_stems and len(deb_only_builds) == 1: + _set_stem_if_absent(stems, deb_only_builds[0], "deb") + + if "udeb" not in reserved_stems and len(udeb_only_builds) == 1: + _set_stem_if_absent(stems, udeb_only_builds[0], "udeb") + + for pkg, idxs in by_name_only_builds.items(): + if len(idxs) != 1 or pkg in reserved_stems: + continue + _set_stem_if_absent(stems, idxs[0], pkg) + + for idx, rule in enumerate(build_rules): + stem = stems[idx] + if stem is None: + stem = f"bno_{idx}" + rule.auto_generated_stem = stem + _info(f"Assigned {rule.auto_generated_stem} [{stem}] to step {idx}") + + +def perform_builds( + context: CommandContext, + manifest: HighLevelManifest, +) -> None: + build_rules = manifest.build_rules + if build_rules is not None: + if not build_rules: + # Defined but empty disables the auto-detected build system + return + active_packages = frozenset(manifest.active_packages) + condition_context = manifest.source_condition_context + build_context = BuildContext.from_command_context(context) + assign_stems(build_rules, manifest) + for step_no, build_rule in enumerate(build_rules): + step_ref = ( + f"step {step_no} [{build_rule.auto_generated_stem}]" + if build_rule.name is None + else f"step {step_no} [{build_rule.name}]" + ) + if build_rule.for_packages.isdisjoint(active_packages): + _info( + f"Skipping build for {step_ref}: None of the relevant packages are being built" + ) + continue + manifest_condition = build_rule.manifest_condition + if manifest_condition is not None and not manifest_condition.evaluate( + condition_context + ): + _info( + f"Skipping build for {step_ref}: The condition clause evaluated to false" + ) + continue + _info(f"Starting build for {step_ref}.") + with in_build_env(build_rule.environment): + try: + build_rule.run_build(build_context, manifest) + except (RuntimeError, AttributeError) as e: + if context.parsed_args.debug_mode: + raise e + _error( + f"An error occurred during build/install at {step_ref} (defined at {build_rule.attribute_path.path}): {str(e)}" + ) + _info(f"Completed build for {step_ref}.") + + else: + build_system = auto_detect_buildsystem(manifest) + if build_system: + _info(f"Auto-detected build system: {build_system.__class__.__name__}") + build_context = BuildContext.from_command_context(context) + with in_build_env(build_system.environment): + with in_build_env(build_system.environment): + build_system.run_build( + build_context, + manifest, + ) + + _non_verbose_info("Upstream builds completed successfully") + else: + _info("No build system was detected from the current plugin set.") + + +def remove_unnecessary_env() -> None: + vs = [ + "XDG_CACHE_HOME", + "XDG_CONFIG_DIRS", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_DATA_DIRS", + "XDG_RUNTIME_DIR", + ] + for v in vs: + if v in os.environ: + del os.environ[v] + + # FIXME: Add custom HOME + XDG_RUNTIME_DIR + + +@contextlib.contextmanager +def _setup_build_env(build_env: BuildEnvironmentDefinition) -> Iterator[None]: + env_backup = dict(os.environ) + env = dict(env_backup) + had_delta = False + build_env.update_env(env) + if env != env_backup: + _set_env(env) + had_delta = True + _info("Updated environment to match build") + yield + if had_delta or env != env_backup: + _set_env(env_backup) + + +def _set_env(desired_env: Mapping[str, str]) -> None: + os_env = os.environ + for key in os_env.keys() | desired_env.keys(): + desired_value = desired_env.get(key) + if desired_value is None: + try: + del os_env[key] + except KeyError: + pass + else: + os_env[key] = desired_value diff --git a/src/debputy/build_support/buildsystem_detection.py b/src/debputy/build_support/buildsystem_detection.py new file mode 100644 index 0000000..47415fd --- /dev/null +++ b/src/debputy/build_support/buildsystem_detection.py @@ -0,0 +1,112 @@ +from typing import ( + Optional, +) + +from debputy.exceptions import ( + DebputyPluginRuntimeError, + PluginBaseError, +) +from debputy.filesystem_scan import FSRootDir, FSROOverlay +from debputy.highlevel_manifest import HighLevelManifest +from debputy.manifest_parser.base_types import BuildEnvironmentDefinition +from debputy.manifest_parser.util import AttributePath +from debputy.plugin.debputy.to_be_api_types import ( + BuildSystemRule, +) +from debputy.plugin.plugin_state import run_in_context_of_plugin_wrap_errors +from debputy.util import ( + _error, + _debug_log, +) + + +def default_build_environment_only( + manifest: HighLevelManifest, +) -> BuildEnvironmentDefinition: + build_envs = manifest.build_environments + if build_envs.environments: + _error( + 'When automatic build system detection is used, the manifest cannot use "build-environments"' + ) + build_env = build_envs.default_environment + assert build_env is not None + return build_env + + +def auto_detect_buildsystem( + manifest: HighLevelManifest, +) -> Optional[BuildSystemRule]: + auto_detectable_build_systems = ( + manifest.plugin_provided_feature_set.auto_detectable_build_systems + ) + excludes = set() + options = [] + _debug_log("Auto-detecting build systems.") + source_root = FSROOverlay.create_root_dir("", ".") + for ppadbs in auto_detectable_build_systems.values(): + detected = ppadbs.detector(source_root) + if not isinstance(detected, bool): + _error( + f'The auto-detector for the build system {ppadbs.manifest_keyword} returned a "non-bool"' + f" ({detected!r}), which could be a bug in the plugin or the plugin relying on a newer" + " version of `debputy` that changed the auto-detection protocol." + ) + if not detected: + _debug_log( + f"Skipping build system {ppadbs.manifest_keyword}: Detector returned False!" + ) + continue + _debug_log( + f"Considering build system {ppadbs.manifest_keyword} as its Detector returned True!" + ) + if ppadbs.auto_detection_shadow_build_systems: + names = ", ".join( + sorted(x for x in ppadbs.auto_detection_shadow_build_systems) + ) + _debug_log(f"Build system {ppadbs.manifest_keyword} excludes: {names}!") + excludes.update(ppadbs.auto_detection_shadow_build_systems) + options.append(ppadbs) + + if not options: + _debug_log("Zero candidates; continuing without a build system") + return None + + if excludes: + names = ", ".join(sorted(x for x in excludes)) + _debug_log(f"The following build systems have been excluded: {names}!") + remaining_options = [o for o in options if o.manifest_keyword not in excludes] + else: + remaining_options = options + + if len(remaining_options) > 1: + names = ", ".join(o.manifest_keyword for o in remaining_options) + # TODO: This means adding an auto-detectable build system to an existing plugin causes FTBFS + # We need a better way of handling this. Probably the build systems should include + # a grace timer based on d/changelog. Anything before the changelog date is in + # "grace mode" and will not conflict with a build system that is. If all choices + # are in "grace mode", "oldest one" wins. + _error( + f"Multiple build systems match, please pick one explicitly (under `builds:`): {names}" + ) + + if not remaining_options: + names = ", ".join(o.build_system_rule_type.__name__ for o in options) + # TODO: Detect at registration time + _error( + f"Multiple build systems matched but they all shadowed each other: {names}." + f" There is a bug in at least one of them!" + ) + + chosen_build_system = remaining_options[0] + environment = default_build_environment_only(manifest) + bs = run_in_context_of_plugin_wrap_errors( + chosen_build_system.plugin_metadata.plugin_name, + chosen_build_system.constructor, + { + "environment": environment, + }, + AttributePath.builtin_path(), + manifest, + ) + bs.auto_generated_stem = "" + return bs diff --git a/src/debputy/build_support/clean_logic.py b/src/debputy/build_support/clean_logic.py new file mode 100644 index 0000000..13347b0 --- /dev/null +++ b/src/debputy/build_support/clean_logic.py @@ -0,0 +1,233 @@ +import os.path +from typing import ( + Set, + cast, + List, +) + +from debputy.build_support.build_context import BuildContext +from debputy.build_support.build_logic import ( + in_build_env, + assign_stems, +) +from debputy.build_support.buildsystem_detection import auto_detect_buildsystem +from debputy.commands.debputy_cmd.context import CommandContext +from debputy.highlevel_manifest import HighLevelManifest +from debputy.plugin.debputy.to_be_api_types import BuildSystemRule, CleanHelper +from debputy.util import _info, print_command, _error, _debug_log, _warn +from debputy.util import ( + run_build_system_command, +) + +_REMOVE_DIRS = frozenset( + [ + "__pycache__", + "autom4te.cache", + ] +) +_IGNORE_DIRS = frozenset( + [ + ".git", + ".svn", + ".bzr", + ".hg", + "CVS", + ".pc", + "_darcs", + ] +) +DELETE_FILE_EXT = ( + "~", + ".orig", + ".rej", + ".bak", +) +DELETE_FILE_BASENAMES = { + "DEADJOE", + ".SUMS", + "TAGS", +} + + +def _debhelper_left_overs() -> bool: + if os.path.lexists("debian/.debhelper") or os.path.lexists( + "debian/debhelper-build-stamp" + ): + return True + with os.scandir(".") as root_dir: + for child in root_dir: + if child.is_file(follow_symlinks=False) and ( + child.name.endswith(".debhelper.log") + or child.name.endswith(".debhelper") + ): + return True + return False + + +class CleanHelperImpl(CleanHelper): + + def __init__(self) -> None: + self.files_to_remove: Set[str] = set() + self.dirs_to_remove: Set[str] = set() + + def schedule_removal_of_files(self, *args: str) -> None: + self.files_to_remove.update(args) + + def schedule_removal_of_directories(self, *args: str) -> None: + if any(p == "/" for p in args): + raise ValueError("Refusing to delete '/'") + self.dirs_to_remove.update(args) + + +def _scan_for_standard_removals(clean_helper: CleanHelperImpl) -> None: + remove_files = clean_helper.files_to_remove + remove_dirs = clean_helper.dirs_to_remove + with os.scandir(".") as root_dir: + for child in root_dir: + if child.is_file(follow_symlinks=False) and child.name.endswith("-stamp"): + remove_files.add(child.path) + for current_dir, subdirs, files in os.walk("."): + for remove_dir in [d for d in subdirs if d in _REMOVE_DIRS]: + path = os.path.join(current_dir, remove_dir) + remove_dirs.add(path) + subdirs.remove(remove_dir) + for skip_dir in [d for d in subdirs if d in _IGNORE_DIRS]: + subdirs.remove(skip_dir) + + for basename in files: + if ( + basename.endswith(DELETE_FILE_EXT) + or basename in DELETE_FILE_BASENAMES + or (basename.startswith("#") and basename.endswith("#")) + ): + path = os.path.join(current_dir, basename) + remove_files.add(path) + + +def perform_clean( + context: CommandContext, + manifest: HighLevelManifest, +) -> None: + clean_helper = CleanHelperImpl() + + build_rules = manifest.build_rules + if build_rules is not None: + if not build_rules: + # Defined but empty disables the auto-detected build system + return + active_packages = frozenset(manifest.active_packages) + condition_context = manifest.source_condition_context + build_context = BuildContext.from_command_context(context) + assign_stems(build_rules, manifest) + for step_no, build_rule in enumerate(build_rules): + step_ref = ( + f"step {step_no} [{build_rule.auto_generated_stem}]" + if build_rule.name is None + else f"step {step_no} [{build_rule.name}]" + ) + if not build_rule.is_buildsystem: + _debug_log(f"Skipping clean for {step_ref}: Not a build system") + continue + build_system_rule: BuildSystemRule = cast("BuildSystemRule", build_rule) + if build_system_rule.for_packages.isdisjoint(active_packages): + _info( + f"Skipping build for {step_ref}: None of the relevant packages are being built" + ) + continue + manifest_condition = build_system_rule.manifest_condition + if manifest_condition is not None and not manifest_condition.evaluate( + condition_context + ): + _info( + f"Skipping clean for {step_ref}: The condition clause evaluated to false" + ) + continue + _info(f"Starting clean for {step_ref}.") + with in_build_env(build_rule.environment): + try: + build_system_rule.run_clean( + build_context, + manifest, + clean_helper, + ) + except (RuntimeError, AttributeError) as e: + if context.parsed_args.debug_mode: + raise e + _error( + f"An error occurred during clean at {step_ref} (defined at {build_rule.attribute_path.path}): {str(e)}" + ) + _info(f"Completed clean for {step_ref}.") + else: + build_system = auto_detect_buildsystem(manifest) + if build_system: + _info(f"Auto-detected build system: {build_system.__class__.__name__}") + build_context = BuildContext.from_command_context(context) + with in_build_env(build_system.environment): + build_system.run_clean( + build_context, + manifest, + clean_helper, + ) + else: + _info("No build system was detected from the current plugin set.") + + dh_autoreconf_used = os.path.lexists("debian/autoreconf.before") + debhelper_used = False + + if dh_autoreconf_used or _debhelper_left_overs(): + debhelper_used = True + + _scan_for_standard_removals(clean_helper) + + for package in manifest.all_packages: + package_staging_dir = os.path.join("debian", package.name) + if os.path.lexists(package_staging_dir): + clean_helper.schedule_removal_of_directories(package_staging_dir) + + remove_files = clean_helper.files_to_remove + remove_dirs = clean_helper.dirs_to_remove + if remove_files: + print_command("rm", "-f", *remove_files) + _remove_files_if_exists(*remove_files) + if remove_dirs: + run_build_system_command("rm", "-fr", *remove_dirs) + + if debhelper_used: + _info( + "Noted traces of debhelper commands being used; invoking dh_clean to clean up after them" + ) + if dh_autoreconf_used: + run_build_system_command("dh_autoreconf_clean") + run_build_system_command("dh_clean") + + try: + run_build_system_command("dpkg-buildtree", "clean") + except FileNotFoundError: + _warn("The dpkg-buildtree command is not present. Emulating it") + # This is from the manpage of dpkg-buildtree for 1.22.11. + _remove_files_if_exists( + "debian/files", + "debian/files.new", + "debian/substvars", + "debian/substvars.new", + ) + run_build_system_command("rm", "-fr", "debian/tmp") + # Remove debian/.debputy as a separate step. While `rm -fr` should process things in order, + # it will continue on error, which could cause our manifests of things to delete to be deleted + # while leaving things half-removed unless we do this extra step. + run_build_system_command("rm", "-fr", "debian/.debputy") + + +def _remove_files_if_exists(*args: str) -> None: + for path in args: + try: + os.unlink(path) + except FileNotFoundError: + continue + except OSError as e: + if os.path.isdir(path): + _error( + f"Failed to remove {path}: It is a directory, but it should have been a non-directory." + " Please verify everything is as expected and, if it is, remove it manually." + ) + _error(f"Failed to remove {path}: {str(e)}") diff --git a/src/debputy/builtin_manifest_rules.py b/src/debputy/builtin_manifest_rules.py index e420cda..e31a50e 100644 --- a/src/debputy/builtin_manifest_rules.py +++ b/src/debputy/builtin_manifest_rules.py @@ -17,7 +17,7 @@ from debputy.path_matcher import ( ) from debputy.substitution import Substitution from debputy.types import VP -from debputy.util import _normalize_path, perl_module_dirs +from debputy.util import _normalize_path, resolve_perl_config # Imported from dh_fixperms _PERMISSION_NORMALIZATION_SOURCE_DEFINITION = "permission normalization" @@ -218,17 +218,19 @@ def builtin_mode_normalization_rules( OctalMode(0o0644), ) + perl_config_data = resolve_perl_config(dpkg_architecture_variables, dctrl_bin) + yield from ( ( BasenameGlobMatch( "*.pm", - only_when_in_directory=perl_dir, + only_when_in_directory=_normalize_path(perl_dir), path_type=PathType.FILE, recursive_match=True, ), _STD_FILE_MODE, ) - for perl_dir in perl_module_dirs(dpkg_architecture_variables, dctrl_bin) + for perl_dir in (perl_config_data.vendorlib, perl_config_data.vendorarch) ) yield ( diff --git a/src/debputy/commands/debputy_cmd/__main__.py b/src/debputy/commands/debputy_cmd/__main__.py index 3270737..05bd135 100644 --- a/src/debputy/commands/debputy_cmd/__main__.py +++ b/src/debputy/commands/debputy_cmd/__main__.py @@ -24,6 +24,7 @@ from typing import ( from debputy import DEBPUTY_ROOT_DIR, DEBPUTY_PLUGIN_ROOT_DIR from debputy.analysis import REFERENCE_DATA_TABLE from debputy.analysis.debian_dir import scan_debian_dir +from debputy.build_support import perform_clean, perform_builds from debputy.commands.debputy_cmd.context import ( CommandContext, add_arg, @@ -40,10 +41,15 @@ from debputy.exceptions import ( UnhandledOrUnexpectedErrorFromPluginError, SymlinkLoopError, ) +from debputy.highlevel_manifest import HighLevelManifest from debputy.package_build.assemble_deb import ( assemble_debs, ) -from debputy.plugin.api.spec import INTEGRATION_MODE_DH_DEBPUTY_RRR +from debputy.plugin.api.spec import ( + INTEGRATION_MODE_DH_DEBPUTY_RRR, + DebputyIntegrationMode, + INTEGRATION_MODE_FULL, +) try: from argcomplete import autocomplete @@ -92,8 +98,9 @@ from debputy.util import ( escape_shell, program_name, integrated_with_debhelper, - change_log_level, + PRINT_BUILD_SYSTEM_COMMAND, PRINT_COMMAND, + change_log_level, ) @@ -255,6 +262,18 @@ def _add_packages_args(parser: argparse.ArgumentParser) -> None: ) +def _build_subcommand_log_level(context: CommandContext) -> int: + parsed_args = context.parsed_args + log_level: Optional[int] = None + if os.environ.get("DH_VERBOSE", "") != "": + log_level = PRINT_COMMAND + if parsed_args.debug_mode: + log_level = logging.INFO + if log_level is not None: + change_log_level(log_level) + return PRINT_BUILD_SYSTEM_COMMAND + + internal_commands = ROOT_COMMAND.add_dispatching_subcommand( "internal-command", dest="internal_command", @@ -630,10 +649,71 @@ def _run_tests_for_plugin(context: CommandContext) -> None: @internal_commands.register_subcommand( + "dpkg-build-driver-run-task", + help_description="[Internal command] Perform a given Dpkg::BuildDriver task (Not stable API)", + requested_plugins_only=True, + default_log_level=_build_subcommand_log_level, + argparser=[ + add_arg( + "task_name", + metavar="task-name", + choices=[ + "clean", + "build", + "build-arch", + "build-indep", + "binary", + "binary-arch", + "binary-indep", + ], + help="The task to run", + ), + add_arg( + "output", + nargs="?", + default="..", + metavar="output", + help="Where to place the resulting packages. Should be a directory", + ), + ], +) +def _dpkg_build_driver_integration(context: CommandContext) -> None: + parsed_args = context.parsed_args + log_level = context.set_log_level_for_build_subcommand() + task_name = parsed_args.task_name + + if task_name.endswith("-indep"): + context.package_set = "indep" + elif task_name.endswith("arch"): + context.package_set = "arch" + + manifest = context.parse_manifest() + + plugins = context.load_plugins().plugin_data + for plugin in plugins.values(): + if not plugin.is_bundled: + _info(f"Loaded plugin {plugin.plugin_name}") + if task_name == "clean": + perform_clean(context, manifest) + elif task_name in ("build", "build-indep", "build-arch"): + perform_builds(context, manifest) + elif task_name in ("binary", "binary-indep", "binary-arch"): + perform_builds(context, manifest) + assemble( + context, + manifest, + INTEGRATION_MODE_FULL, + debug_materialization=log_level is not None, + ) + else: + _error(f"Unsupported Dpkg::BuildDriver task: {task_name}.") + + +@internal_commands.register_subcommand( "dh-integration-generate-debs", help_description="[Internal command] Generate .deb/.udebs packages from debian/<pkg> (Not stable API)", requested_plugins_only=True, - default_log_level=logging.WARN, + default_log_level=_build_subcommand_log_level, argparser=[ _add_packages_args, add_arg( @@ -653,14 +733,7 @@ def _run_tests_for_plugin(context: CommandContext) -> None: ) def _dh_integration_generate_debs(context: CommandContext) -> None: integrated_with_debhelper() - parsed_args = context.parsed_args - log_level: Optional[int] = None - if os.environ.get("DH_VERBOSE", "") != "": - log_level = PRINT_COMMAND - if parsed_args.debug_mode: - log_level = logging.INFO - if log_level is not None: - change_log_level(log_level) + log_level = context.set_log_level_for_build_subcommand() integration_mode = context.resolve_integration_mode() is_dh_rrr_only_mode = integration_mode == INTEGRATION_MODE_DH_DEBPUTY_RRR if is_dh_rrr_only_mode: @@ -678,13 +751,28 @@ def _dh_integration_generate_debs(context: CommandContext) -> None: _info(f"Loaded plugin {plugin.plugin_name}") manifest = context.parse_manifest() - package_data_table = manifest.perform_installations( - enable_manifest_installation_feature=not is_dh_rrr_only_mode + assemble( + context, + manifest, + integration_mode, + debug_materialization=log_level is not None, ) + + +def assemble( + context: CommandContext, + manifest: HighLevelManifest, + integration_mode: DebputyIntegrationMode, + *, + debug_materialization: bool = False, +) -> None: source_fs = FSROOverlay.create_root_dir("..", ".") source_version = manifest.source_version() is_native = "-" not in source_version - + is_dh_rrr_only_mode = integration_mode == INTEGRATION_MODE_DH_DEBPUTY_RRR + package_data_table = manifest.perform_installations( + enable_manifest_installation_feature=not is_dh_rrr_only_mode + ) if not is_dh_rrr_only_mode: for dctrl_bin in manifest.active_packages: package = dctrl_bin.name @@ -702,7 +790,7 @@ def _dh_integration_generate_debs(context: CommandContext) -> None: fs_root, dctrl_data.substvars, ) - if "nostrip" not in manifest.build_env.deb_build_options: + if "nostrip" not in manifest.deb_options_and_profiles.deb_build_options: dbgsym_ids = relocate_dwarves_into_dbgsym_packages( dctrl_bin, fs_root, @@ -714,7 +802,7 @@ def _dh_integration_generate_debs(context: CommandContext) -> None: dctrl_bin, fs_root, is_native, - manifest.build_env, + manifest.deb_options_and_profiles, ) if not is_native: install_upstream_changelog( @@ -739,7 +827,7 @@ def _dh_integration_generate_debs(context: CommandContext) -> None: manifest, package_data_table, is_dh_rrr_only_mode, - debug_materialization=log_level is not None, + debug_materialization=debug_materialization, ) @@ -901,10 +989,10 @@ def _json_output(data: Any) -> None: ], ) def _migrate_from_dh(context: CommandContext) -> None: + context.must_be_called_in_source_root() parsed_args = context.parsed_args - resolved_migration_target = _check_migration_target( - context.debian_dir, + context, parsed_args.migration_target, ) context.debputy_integration_mode = resolved_migration_target diff --git a/src/debputy/commands/debputy_cmd/context.py b/src/debputy/commands/debputy_cmd/context.py index 0c184c7..a9c0a13 100644 --- a/src/debputy/commands/debputy_cmd/context.py +++ b/src/debputy/commands/debputy_cmd/context.py @@ -15,6 +15,7 @@ from typing import ( Callable, Dict, TYPE_CHECKING, + Literal, ) from debian.debian_support import DpkgArchTable @@ -45,7 +46,14 @@ from debputy.substitution import ( SubstitutionImpl, NULL_SUBSTITUTION, ) -from debputy.util import _error, PKGNAME_REGEX, resolve_source_date_epoch, setup_logging +from debputy.util import ( + _error, + PKGNAME_REGEX, + resolve_source_date_epoch, + setup_logging, + PRINT_COMMAND, + change_log_level, +) if TYPE_CHECKING: from argparse import _SubParsersAction @@ -110,6 +118,19 @@ class CommandContext: Mapping[str, "BinaryPackage"], ] ] = None + self._package_set: Literal["both", "arch", "indep"] = "both" + + @property + def package_set(self) -> Literal["both", "arch", "indep"]: + return self._package_set + + @package_set.setter + def package_set(self, new_value: Literal["both", "arch", "indep"]) -> None: + if self._dctrl_parser is not None: + raise TypeError( + "package_set cannot be redefined once the debian/control parser has been initialized" + ) + self._package_set = new_value @property def debian_dir(self) -> VirtualPath: @@ -135,9 +156,10 @@ class CommandContext: parser = DctrlParser( packages, # -p/--package set(), # -N/--no-package - False, # -i - False, # -a - build_env=DebBuildOptionsAndProfiles.instance(), + # binary-indep and binary-indep (dpkg BuildDriver integration only) + self._package_set == "indep", + self._package_set == "arch", + deb_options_and_profiles=DebBuildOptionsAndProfiles.instance(), dpkg_architecture_variables=dpkg_architecture_table(), dpkg_arch_query_table=DpkgArchTable.load_arch_table(), ) @@ -152,6 +174,9 @@ class CommandContext: _, binary_package_table = self._parse_dctrl() return binary_package_table + def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable: + return self.dctrl_parser.dpkg_architecture_variables + def requested_plugins(self) -> Sequence[str]: if self._requested_plugins is None: self._requested_plugins = self._resolve_requested_plugins() @@ -162,7 +187,7 @@ class CommandContext: @property def deb_build_options_and_profiles(self) -> "DebBuildOptionsAndProfiles": - return self.dctrl_parser.build_env + return self.dctrl_parser.deb_options_and_profiles @property def deb_build_options(self) -> Mapping[str, Optional[str]]: @@ -292,20 +317,37 @@ class CommandContext: debian_control = self.debian_dir.get("control") return debian_control is not None - def resolve_integration_mode(self) -> DebputyIntegrationMode: + def resolve_integration_mode( + self, + require_integration: bool = True, + ) -> DebputyIntegrationMode: integration_mode = self.debputy_integration_mode if integration_mode is None: r = read_dh_addon_sequences(self.debian_dir) bd_sequences, dr_sequences, _ = r all_sequences = bd_sequences | dr_sequences - integration_mode = determine_debputy_integration_mode(all_sequences) - if integration_mode is None: + integration_mode = determine_debputy_integration_mode( + self.source_package().fields, + all_sequences, + ) + if integration_mode is None and not require_integration: _error( "Cannot resolve the integration mode expected for this package. Is this package using `debputy`?" ) self.debputy_integration_mode = integration_mode return integration_mode + def set_log_level_for_build_subcommand(self) -> Optional[int]: + parsed_args = self.parsed_args + log_level: Optional[int] = None + if os.environ.get("DH_VERBOSE", "") != "": + log_level = PRINT_COMMAND + if parsed_args.debug_mode or os.environ.get("DEBPUTY_DEBUG", "") != "": + log_level = logging.DEBUG + if log_level is not None: + change_log_level(log_level) + return log_level + def manifest_parser( self, *, @@ -320,7 +362,6 @@ class CommandContext: manifest_path = self.parsed_args.debputy_manifest if manifest_path is None: manifest_path = os.path.join(self.debian_dir.fs_path, "debputy.manifest") - debian_dir = self.debian_dir return YAMLManifestParser( manifest_path, source_package, @@ -328,7 +369,7 @@ class CommandContext: substitution, dctrl_parser.dpkg_architecture_variables, dctrl_parser.dpkg_arch_query_table, - dctrl_parser.build_env, + dctrl_parser.deb_options_and_profiles, self.load_plugins(), self.resolve_integration_mode(), debian_dir=self.debian_dir, @@ -420,7 +461,7 @@ class GenericSubCommand(SubcommandBase): require_substitution: bool = True, requested_plugins_only: bool = False, log_only_to_stderr: bool = False, - default_log_level: int = logging.INFO, + default_log_level: Union[int, Callable[[CommandContext], int]] = logging.INFO, ) -> None: super().__init__(name, aliases=aliases, help_description=help_description) self._handler = handler @@ -452,7 +493,18 @@ class GenericSubCommand(SubcommandBase): ) if self._log_only_to_stderr: setup_logging(reconfigure_logging=True, log_only_to_stderr=True) - logging.getLogger().setLevel(self._default_log_level) + + default_log_level = self._default_log_level + if isinstance(default_log_level, int): + level = default_log_level + else: + assert callable(default_log_level) + level = default_log_level(context) + change_log_level(level) + if level > logging.DEBUG and ( + context.parsed_args.debug_mode or os.environ.get("DEBPUTY_DEBUG", "") != "" + ): + change_log_level(logging.DEBUG) return self._handler(context) @@ -494,7 +546,7 @@ class DispatchingCommandMixin(CommandBase): require_substitution: bool = True, requested_plugins_only: bool = False, log_only_to_stderr: bool = False, - default_log_level: int = logging.INFO, + default_log_level: Union[int, Callable[[CommandContext], int]] = logging.INFO, ) -> Callable[[CommandHandler], GenericSubCommand]: if isinstance(name, str): cmd_name = name diff --git a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py index eaab750..a03126b 100644 --- a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py +++ b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py @@ -267,7 +267,6 @@ def lsp_describe_features(context: CommandContext) -> None: "--spellcheck", dest="spellcheck", action="store_true", - shared=True, help="Enable spellchecking", ), add_arg( diff --git a/src/debputy/commands/debputy_cmd/plugin_cmds.py b/src/debputy/commands/debputy_cmd/plugin_cmds.py index 83bb88f..9721702 100644 --- a/src/debputy/commands/debputy_cmd/plugin_cmds.py +++ b/src/debputy/commands/debputy_cmd/plugin_cmds.py @@ -2,6 +2,7 @@ import argparse import operator import os import sys +import textwrap from itertools import chain from typing import ( Sequence, @@ -28,7 +29,7 @@ from debputy.commands.debputy_cmd.output import ( ) from debputy.exceptions import DebputySubstitutionError from debputy.filesystem_scan import build_virtual_fs -from debputy.manifest_parser.base_types import TypeMapping +from debputy.manifest_parser.tagging_types import TypeMapping from debputy.manifest_parser.declarative_parser import ( BASIC_SIMPLE_TYPES, ) @@ -45,13 +46,15 @@ from debputy.plugin.api.impl_types import ( PackagerProvidedFileClassSpec, PluginProvidedManifestVariable, DispatchingParserBase, - SUPPORTED_DISPATCHABLE_TABLE_PARSERS, - OPARSER_MANIFEST_ROOT, PluginProvidedDiscardRule, AutomaticDiscardRuleExample, MetadataOrMaintscriptDetector, PluginProvidedTypeMapping, ) +from debputy.plugin.api.parser_tables import ( + SUPPORTED_DISPATCHABLE_TABLE_PARSERS, + OPARSER_MANIFEST_ROOT, +) from debputy.plugin.api.spec import ( TypeMappingExample, ) @@ -538,7 +541,16 @@ def _plugin_cmd_show_manifest_variables(context: CommandContext) -> None: variable_value=None, is_context_specific_variable=False, is_documentation_placeholder=True, - variable_reference_documentation=f'Environment variable "{env_var}"', + variable_reference_documentation=textwrap.dedent( + f"""\ + Environment variable "{env_var}" + + Note that uses beneath `builds:` may use the environment variable defined by + `build-environment:` (depends on whether the rule uses eager or lazy + substitution) while uses outside `builds:` will generally not use a definition + from `build-environment:`. + """ + ), ) else: variable = variables.get(variable_name) diff --git a/src/debputy/deb_packaging_support.py b/src/debputy/deb_packaging_support.py index 875b3b1..92f57a2 100644 --- a/src/debputy/deb_packaging_support.py +++ b/src/debputy/deb_packaging_support.py @@ -81,7 +81,7 @@ from debputy.util import ( _error, ensure_dir, assume_not_none, - perl_module_dirs, + resolve_perl_config, perlxs_api_dependency, detect_fakeroot, grouper, @@ -186,11 +186,11 @@ def handle_perl_code( fs_root: FSPath, substvars: FlushableSubstvars, ) -> None: - known_perl_inc_dirs = perl_module_dirs(dpkg_architecture_variables, dctrl_bin) + perl_config_data = resolve_perl_config(dpkg_architecture_variables, dctrl_bin) detected_dep_requirements = 0 # MakeMaker always makes lib and share dirs, but typically only one directory is actually used. - for perl_inc_dir in known_perl_inc_dirs: + for perl_inc_dir in (perl_config_data.vendorarch, perl_config_data.vendorlib): p = fs_root.lookup(perl_inc_dir) if p and p.is_dir: p.prune_if_empty_dir() @@ -198,8 +198,8 @@ def handle_perl_code( # FIXME: 80% of this belongs in a metadata detector, but that requires us to expose .walk() in the public API, # which will not be today. for d, pm_mode in [ - (known_perl_inc_dirs.vendorlib, PERL_DEP_INDEP_PM_MODULE), - (known_perl_inc_dirs.vendorarch, PERL_DEP_ARCH_PM_MODULE), + (perl_config_data.vendorlib, PERL_DEP_INDEP_PM_MODULE), + (perl_config_data.vendorarch, PERL_DEP_ARCH_PM_MODULE), ]: inc_dir = fs_root.lookup(d) if not inc_dir: diff --git a/src/debputy/dh_migration/migration.py b/src/debputy/dh_migration/migration.py index f7b7d9e..62f739e 100644 --- a/src/debputy/dh_migration/migration.py +++ b/src/debputy/dh_migration/migration.py @@ -3,10 +3,11 @@ import os import re import subprocess from itertools import chain -from typing import Optional, List, Callable, Set, Container +from typing import Optional, List, Callable, Set, Container, Mapping, FrozenSet from debian.deb822 import Deb822 +from debputy.commands.debputy_cmd.context import CommandContext from debputy.dh.debhelper_emulation import CannotEmulateExecutableDHConfigFile from debputy.dh_migration.migrators import MIGRATORS from debputy.dh_migration.migrators_impl import ( @@ -24,9 +25,25 @@ from debputy.highlevel_manifest import HighLevelManifest from debputy.integration_detection import determine_debputy_integration_mode from debputy.manifest_parser.exceptions import ManifestParseException from debputy.plugin.api import VirtualPath -from debputy.plugin.api.spec import DebputyIntegrationMode +from debputy.plugin.api.spec import DebputyIntegrationMode, INTEGRATION_MODE_FULL from debputy.util import _error, _warn, _info, escape_shell, assume_not_none +SUPPORTED_MIGRATIONS: Mapping[ + DebputyIntegrationMode, FrozenSet[DebputyIntegrationMode] +] = { + INTEGRATION_MODE_FULL: frozenset([INTEGRATION_MODE_FULL]), + INTEGRATION_MODE_DH_DEBPUTY: frozenset( + [INTEGRATION_MODE_DH_DEBPUTY, INTEGRATION_MODE_FULL] + ), + INTEGRATION_MODE_DH_DEBPUTY_RRR: frozenset( + [ + INTEGRATION_MODE_DH_DEBPUTY_RRR, + INTEGRATION_MODE_DH_DEBPUTY, + INTEGRATION_MODE_FULL, + ] + ), +} + def _print_migration_summary( migrations: List[FeatureMigration], @@ -143,22 +160,33 @@ def _requested_debputy_plugins(debian_dir: VirtualPath) -> Optional[Set[str]]: def _check_migration_target( - debian_dir: VirtualPath, + context: CommandContext, migration_target: Optional[DebputyIntegrationMode], ) -> DebputyIntegrationMode: - r = read_dh_addon_sequences(debian_dir) - if r is None and migration_target is None: - _error("debian/control is missing and no migration target was provided") - bd_sequences, dr_sequences, _ = r - all_sequences = bd_sequences | dr_sequences - - detected_migration_target = determine_debputy_integration_mode(all_sequences) - - if ( - migration_target == INTEGRATION_MODE_DH_DEBPUTY_RRR - and detected_migration_target == INTEGRATION_MODE_DH_DEBPUTY - ): - _error("Cannot migrate from (zz-)debputy to zz-debputy-rrr") + r = read_dh_addon_sequences(context.debian_dir) + if r is not None: + bd_sequences, dr_sequences, _ = r + all_sequences = bd_sequences | dr_sequences + detected_migration_target = determine_debputy_integration_mode( + context.source_package().fields, + all_sequences, + ) + else: + detected_migration_target = None + + if migration_target is not None and detected_migration_target is not None: + supported_migrations = SUPPORTED_MIGRATIONS.get( + detected_migration_target, + frozenset([detected_migration_target]), + ) + + if ( + migration_target != detected_migration_target + and migration_target not in supported_migrations + ): + _error( + f"Cannot migrate from {detected_migration_target} to {migration_target}" + ) if migration_target is not None: resolved_migration_target = migration_target diff --git a/src/debputy/dh_migration/migrators.py b/src/debputy/dh_migration/migrators.py index 8eff679..8a057b7 100644 --- a/src/debputy/dh_migration/migrators.py +++ b/src/debputy/dh_migration/migrators.py @@ -13,7 +13,7 @@ from debputy.dh_migration.migrators_impl import ( migrate_lintian_overrides_files, detect_unsupported_zz_debputy_features, detect_pam_files, - detect_dh_addons, + detect_dh_addons_with_zz_integration, migrate_not_installed_file, migrate_installman_file, migrate_bash_completion, @@ -21,6 +21,7 @@ from debputy.dh_migration.migrators_impl import ( migrate_dh_installsystemd_files, detect_obsolete_substvars, detect_dh_addons_zz_debputy_rrr, + detect_dh_addons_with_full_integration, ) from debputy.dh_migration.models import AcceptableMigrationIssues, FeatureMigration from debputy.highlevel_manifest import HighLevelManifest @@ -29,13 +30,42 @@ from debputy.plugin.api.spec import ( DebputyIntegrationMode, INTEGRATION_MODE_DH_DEBPUTY_RRR, INTEGRATION_MODE_DH_DEBPUTY, + INTEGRATION_MODE_FULL, ) Migrator = Callable[ - [VirtualPath, HighLevelManifest, AcceptableMigrationIssues, FeatureMigration, str], + [ + VirtualPath, + HighLevelManifest, + AcceptableMigrationIssues, + FeatureMigration, + DebputyIntegrationMode, + ], None, ] +_DH_DEBPUTY_MIGRATORS = [ + detect_unsupported_zz_debputy_features, + detect_pam_files, + migrate_dh_hook_targets, + migrate_dh_installsystemd_files, + migrate_install_file, + migrate_installdocs_file, + migrate_installexamples_file, + migrate_installman_file, + migrate_installinfo_file, + migrate_misspelled_readme_debian_files, + migrate_doc_base_files, + migrate_links_files, + migrate_maintscript, + migrate_tmpfile, + migrate_lintian_overrides_files, + migrate_bash_completion, + detect_obsolete_substvars, + # not-installed should go last, so its rules appear after other installations + # It is not perfect, but it is a start. + migrate_not_installed_file, +] MIGRATORS: Mapping[DebputyIntegrationMode, List[Migrator]] = { INTEGRATION_MODE_DH_DEBPUTY_RRR: [ @@ -45,26 +75,12 @@ MIGRATORS: Mapping[DebputyIntegrationMode, List[Migrator]] = { detect_obsolete_substvars, ], INTEGRATION_MODE_DH_DEBPUTY: [ - detect_unsupported_zz_debputy_features, - detect_pam_files, - migrate_dh_hook_targets, - migrate_dh_installsystemd_files, - migrate_install_file, - migrate_installdocs_file, - migrate_installexamples_file, - migrate_installman_file, - migrate_installinfo_file, - migrate_misspelled_readme_debian_files, - migrate_doc_base_files, - migrate_links_files, - migrate_maintscript, - migrate_tmpfile, - migrate_lintian_overrides_files, - migrate_bash_completion, - detect_dh_addons, - detect_obsolete_substvars, - # not-installed should go last, so its rules appear after other installations - # It is not perfect, but it is a start. - migrate_not_installed_file, + *_DH_DEBPUTY_MIGRATORS, + detect_dh_addons_with_zz_integration, + ], + INTEGRATION_MODE_FULL: [ + *_DH_DEBPUTY_MIGRATORS, + detect_dh_addons_with_full_integration, ], } +del _DH_DEBPUTY_MIGRATORS diff --git a/src/debputy/dh_migration/migrators_impl.py b/src/debputy/dh_migration/migrators_impl.py index 91ea8cd..97b0fd2 100644 --- a/src/debputy/dh_migration/migrators_impl.py +++ b/src/debputy/dh_migration/migrators_impl.py @@ -17,6 +17,7 @@ from typing import ( Callable, TypeVar, Dict, + Container, ) from debian.deb822 import Deb822 @@ -51,6 +52,8 @@ from debputy.plugin.api import VirtualPath from debputy.plugin.api.spec import ( INTEGRATION_MODE_DH_DEBPUTY_RRR, INTEGRATION_MODE_DH_DEBPUTY, + DebputyIntegrationMode, + INTEGRATION_MODE_FULL, ) from debputy.util import ( _error, @@ -61,8 +64,15 @@ from debputy.util import ( has_glob_magic, ) + +class ContainsEverything: + + def __contains__(self, item: str) -> bool: + return True + + # Align with debputy.py -DH_COMMANDS_REPLACED = { +DH_COMMANDS_REPLACED: Mapping[DebputyIntegrationMode, Container[str]] = { INTEGRATION_MODE_DH_DEBPUTY_RRR: frozenset( { "dh_fixperms", @@ -124,6 +134,7 @@ DH_COMMANDS_REPLACED = { "dh_builddeb", } ), + INTEGRATION_MODE_FULL: ContainsEverything(), } _GS_DOC = f"{DEBPUTY_DOC_ROOT_DIR}/GETTING-STARTED-WITH-dh-debputy.md" @@ -375,7 +386,7 @@ def migrate_bash_completion( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_bash-completion files" is_single_binary = sum(1 for _ in manifest.all_packages) == 1 @@ -466,7 +477,7 @@ def migrate_dh_installsystemd_files( manifest: HighLevelManifest, _acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installsystemd files" for dctrl_bin in manifest.all_packages: @@ -499,7 +510,7 @@ def migrate_maintscript( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installdeb files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -608,7 +619,7 @@ def migrate_install_file( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_install config files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -799,7 +810,7 @@ def migrate_installdocs_file( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installdocs config files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -846,7 +857,7 @@ def migrate_installexamples_file( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installexamples config files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -900,7 +911,7 @@ def migrate_installinfo_file( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installinfo config files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -975,7 +986,7 @@ def migrate_installman_file( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installman config files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -1095,7 +1106,7 @@ def migrate_not_installed_file( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_missing's not-installed config file" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -1135,7 +1146,7 @@ def detect_pam_files( manifest: HighLevelManifest, _acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "detect dh_installpam files (min dh compat)" for dctrl_bin in manifest.all_packages: @@ -1150,7 +1161,7 @@ def migrate_tmpfile( manifest: HighLevelManifest, _acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_installtmpfiles config files" for dctrl_bin in manifest.all_packages: @@ -1174,7 +1185,7 @@ def migrate_lintian_overrides_files( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_lintian config files" for dctrl_bin in manifest.all_packages: @@ -1198,7 +1209,7 @@ def migrate_links_files( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh_link files" mutable_manifest = assume_not_none(manifest.mutable_manifest) @@ -1272,7 +1283,7 @@ def migrate_misspelled_readme_debian_files( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "misspelled README.Debian files" for dctrl_bin in manifest.all_packages: @@ -1304,7 +1315,7 @@ def migrate_doc_base_files( manifest: HighLevelManifest, _: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "doc-base files" # ignore the dh_make ".EX" file if one should still be present. The dh_installdocs tool ignores it too. @@ -1355,7 +1366,7 @@ def migrate_dh_hook_targets( _: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - migration_target: str, + migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "dh hook targets" source_root = os.path.dirname(debian_dir.fs_path) @@ -1407,7 +1418,7 @@ def detect_unsupported_zz_debputy_features( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "Known unsupported features" @@ -1426,7 +1437,7 @@ def detect_obsolete_substvars( _manifest: HighLevelManifest, _acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = ( "Check for obsolete ${foo:var} variables in debian/control" @@ -1507,7 +1518,7 @@ def detect_dh_addons_zz_debputy_rrr( _manifest: HighLevelManifest, _acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "Check for dh-sequence-addons" r = read_dh_addon_sequences(debian_dir) @@ -1527,12 +1538,28 @@ def detect_dh_addons_zz_debputy_rrr( feature_migration.warn("Missing Build-Depends on dh-sequence-zz-debputy-rrr") -def detect_dh_addons( +def detect_dh_addons_with_full_integration( + _debian_dir: VirtualPath, + _manifest: HighLevelManifest, + _acceptable_migration_issues: AcceptableMigrationIssues, + feature_migration: FeatureMigration, + _migration_target: DebputyIntegrationMode, +) -> None: + feature_migration.tagline = "Check for dh-sequence-addons and Build-Depends" + feature_migration.warn( + "TODO: Not implemented: Please remove any dh-sequence Build-Dependency" + ) + feature_migration.warn( + "TODO: Not implemented: Please ensure there is a Build-Dependency on `debputy (>= 0.1.45~)" + ) + + +def detect_dh_addons_with_zz_integration( debian_dir: VirtualPath, _manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, feature_migration: FeatureMigration, - _migration_target: str, + _migration_target: DebputyIntegrationMode, ) -> None: feature_migration.tagline = "Check for dh-sequence-addons" r = read_dh_addon_sequences(debian_dir) @@ -1544,6 +1571,8 @@ def detect_dh_addons( ) return + assert _migration_target != INTEGRATION_MODE_FULL + bd_sequences, dr_sequences, _ = r remaining_sequences = bd_sequences | dr_sequences diff --git a/src/debputy/exceptions.py b/src/debputy/exceptions.py index a445997..b3ff7d5 100644 --- a/src/debputy/exceptions.py +++ b/src/debputy/exceptions.py @@ -10,6 +10,10 @@ class DebputyRuntimeError(RuntimeError): return cast("str", self.args[0]) +class DebputyBuildStepError(DebputyRuntimeError): + pass + + class DebputySubstitutionError(DebputyRuntimeError): pass @@ -64,6 +68,10 @@ class PluginInitializationError(PluginBaseError): pass +class PluginIncorrectRegistrationError(PluginInitializationError): + pass + + class PluginMetadataError(PluginBaseError): pass diff --git a/src/debputy/highlevel_manifest.py b/src/debputy/highlevel_manifest.py index 9bdc225..6c910ab 100644 --- a/src/debputy/highlevel_manifest.py +++ b/src/debputy/highlevel_manifest.py @@ -49,7 +49,11 @@ from .maintscript_snippet import ( MaintscriptSnippetContainer, ) from .manifest_conditions import ConditionContext -from .manifest_parser.base_types import FileSystemMatchRule, FileSystemExactMatchRule +from .manifest_parser.base_types import ( + FileSystemMatchRule, + FileSystemExactMatchRule, + BuildEnvironments, +) from .manifest_parser.util import AttributePath from .packager_provided_files import PackagerProvidedFile from .packages import BinaryPackage, SourcePackage @@ -61,6 +65,8 @@ from .plugin.api.impl_types import ( ) from .plugin.api.spec import FlushableSubstvars, VirtualPath from .plugin.debputy.binary_package_rules import ServiceRule +from .plugin.debputy.to_be_api_types import BuildRule +from .plugin.plugin_state import run_in_context_of_plugin from .substitution import Substitution from .transformation_rules import ( TransformationRule, @@ -1036,7 +1042,9 @@ def _install_everything_from_source_dir_if_present( ) -> None: attribute_path = AttributePath.builtin_path()[f"installing {source_dir.fs_path}"] pkg_set = frozenset([dctrl_bin]) - install_rule = InstallRule.install_dest( + install_rule = run_in_context_of_plugin( + "debputy", + InstallRule.install_dest, [FileSystemMatchRule.from_path_match("*", attribute_path, substitution)], None, pkg_set, @@ -1086,6 +1094,8 @@ class HighLevelManifest: dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable, dpkg_arch_query_table: DpkgArchTable, build_env: DebBuildOptionsAndProfiles, + build_environments: BuildEnvironments, + build_rules: Optional[List[BuildRule]], plugin_provided_feature_set: PluginProvidedFeatureSet, debian_dir: VirtualPath, ) -> None: @@ -1100,8 +1110,17 @@ class HighLevelManifest: self._dpkg_arch_query_table = dpkg_arch_query_table self._build_env = build_env self._used_for: Set[str] = set() + self.build_environments = build_environments + self.build_rules = build_rules self._plugin_provided_feature_set = plugin_provided_feature_set self._debian_dir = debian_dir + self._source_condition_context = ConditionContext( + binary_package=None, + substitution=self.substitution, + deb_options_and_profiles=self._build_env, + dpkg_architecture_variables=self._dpkg_architecture_variables, + dpkg_arch_query_table=self._dpkg_arch_query_table, + ) def source_version(self, include_binnmu_version: bool = True) -> str: # TODO: There should an easier way to determine the source version; really. @@ -1116,6 +1135,10 @@ class HighLevelManifest: raise AssertionError(f"Could not resolve {version_var}") from e @property + def source_condition_context(self) -> ConditionContext: + return self._source_condition_context + + @property def debian_dir(self) -> VirtualPath: return self._debian_dir @@ -1124,7 +1147,7 @@ class HighLevelManifest: return self._dpkg_architecture_variables @property - def build_env(self) -> DebBuildOptionsAndProfiles: + def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles: return self._build_env @property @@ -1270,13 +1293,7 @@ class HighLevelManifest: ] path_matcher = SourcePathMatcher(discard_rules) - source_condition_context = ConditionContext( - binary_package=None, - substitution=self.substitution, - build_env=self._build_env, - dpkg_architecture_variables=self._dpkg_architecture_variables, - dpkg_arch_query_table=self._dpkg_arch_query_table, - ) + source_condition_context = self._source_condition_context for dctrl_bin in self.active_packages: package = dctrl_bin.name @@ -1416,23 +1433,14 @@ class HighLevelManifest: self, binary_package: Optional[Union[BinaryPackage, str]] ) -> ConditionContext: if binary_package is None: - return ConditionContext( - binary_package=None, - substitution=self.substitution, - build_env=self._build_env, - dpkg_architecture_variables=self._dpkg_architecture_variables, - dpkg_arch_query_table=self._dpkg_arch_query_table, - ) + return self._source_condition_context if not isinstance(binary_package, str): binary_package = binary_package.name package_transformation = self.package_transformations[binary_package] - return ConditionContext( + return self._source_condition_context.replace( binary_package=package_transformation.binary_package, substitution=package_transformation.substitution, - build_env=self._build_env, - dpkg_architecture_variables=self._dpkg_architecture_variables, - dpkg_arch_query_table=self._dpkg_arch_query_table, ) def apply_fs_transformations( @@ -1452,7 +1460,7 @@ class HighLevelManifest: condition_context = ConditionContext( binary_package=package_transformation.binary_package, substitution=package_transformation.substitution, - build_env=self._build_env, + deb_options_and_profiles=self._build_env, dpkg_architecture_variables=self._dpkg_architecture_variables, dpkg_arch_query_table=self._dpkg_arch_query_table, ) @@ -1466,7 +1474,7 @@ class HighLevelManifest: norm_mode_transformation_rule = ModeNormalizationTransformationRule(norm_rules) norm_mode_transformation_rule.transform_file_system(fs_root, condition_context) for transformation in package_transformation.transformations: - transformation.transform_file_system(fs_root, condition_context) + transformation.run_transform_file_system(fs_root, condition_context) interpreter_normalization = NormalizeShebangLineTransformation() interpreter_normalization.transform_file_system(fs_root, condition_context) diff --git a/src/debputy/highlevel_manifest_parser.py b/src/debputy/highlevel_manifest_parser.py index b7a4600..18d9fa7 100644 --- a/src/debputy/highlevel_manifest_parser.py +++ b/src/debputy/highlevel_manifest_parser.py @@ -43,20 +43,22 @@ from ._deb_options_profiles import DebBuildOptionsAndProfiles from .architecture_support import DpkgArchitectureBuildProcessValuesTable from .filesystem_scan import FSROOverlay from .installations import InstallRule, PPFInstallRule +from .manifest_parser.base_types import BuildEnvironments, BuildEnvironmentDefinition from .manifest_parser.exceptions import ManifestParseException from .manifest_parser.parser_data import ParserContextData from .manifest_parser.util import AttributePath from .packager_provided_files import detect_all_packager_provided_files from .plugin.api import VirtualPath +from .plugin.api.feature_set import PluginProvidedFeatureSet from .plugin.api.impl_types import ( TP, TTP, DispatchingTableParser, - OPARSER_MANIFEST_ROOT, PackageContextData, ) -from .plugin.api.feature_set import PluginProvidedFeatureSet +from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT from .plugin.api.spec import DebputyIntegrationMode +from .plugin.debputy.to_be_api_types import BuildRule from .yaml import YAMLError, MANIFEST_YAML try: @@ -131,11 +133,19 @@ class HighLevelManifestParser(ParserContextData): self._substitution = substitution self._dpkg_architecture_variables = dpkg_architecture_variables self._dpkg_arch_query_table = dpkg_arch_query_table - self._build_env = build_env + self._deb_options_and_profiles = build_env self._package_state_stack: List[PackageTransformationDefinition] = [] self._plugin_provided_feature_set = plugin_provided_feature_set self._debputy_integration_mode = debputy_integration_mode self._declared_variables = {} + self._used_named_envs = set() + self._build_environments: Optional[BuildEnvironments] = BuildEnvironments( + {}, + None, + ) + self._has_set_default_build_environment = False + self._read_build_environment = False + self._build_rules: Optional[List[BuildRule]] = None if isinstance(debian_dir, str): debian_dir = FSROOverlay.create_root_dir("debian", debian_dir) @@ -202,10 +212,21 @@ class HighLevelManifestParser(ParserContextData): return self._dpkg_arch_query_table @property - def build_env(self) -> DebBuildOptionsAndProfiles: - return self._build_env + def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles: + return self._deb_options_and_profiles + + def _self_check(self) -> None: + unused_envs = ( + self._build_environments.environments.keys() - self._used_named_envs + ) + if unused_envs: + unused_env_names = ", ".join(unused_envs) + raise ManifestParseException( + f"The following named environments were never referenced: {unused_env_names}" + ) def build_manifest(self) -> HighLevelManifest: + self._self_check() if self._used: raise TypeError("build_manifest can only be called once!") self._used = True @@ -240,6 +261,8 @@ class HighLevelManifestParser(ParserContextData): ppf_result.reserved_only ) self._transform_dpkg_maintscript_helpers_to_snippets() + build_environments = self.build_environments() + assert build_environments is not None return HighLevelManifest( self.manifest_path, @@ -251,7 +274,9 @@ class HighLevelManifestParser(ParserContextData): self._package_states, self._dpkg_architecture_variables, self._dpkg_arch_query_table, - self._build_env, + self._deb_options_and_profiles, + build_environments, + self._build_rules, self._plugin_provided_feature_set, self._debian_dir, ) @@ -325,6 +350,69 @@ class HighLevelManifestParser(ParserContextData): def debputy_integration_mode(self, new_value: DebputyIntegrationMode) -> None: self._debputy_integration_mode = new_value + def _register_build_environment( + self, + name: Optional[str], + build_environment: BuildEnvironmentDefinition, + attribute_path: AttributePath, + is_default: bool = False, + ) -> None: + assert not self._read_build_environment + + # TODO: Reference the paths of the original environments for the error messages where that is relevant. + if is_default: + if self._has_set_default_build_environment: + raise ManifestParseException( + f"There cannot be multiple default environments and" + f" therefore {attribute_path.path} cannot be a default environment" + ) + self._has_set_default_build_environment = True + self._build_environments.default_environment = build_environment + if name is None: + return + elif name is None: + raise ManifestParseException( + f"Useless environment defined at {attribute_path.path}. It is neither the" + " default environment nor does it have a name (so no rules can reference it" + " explicitly)" + ) + + if name in self._build_environments.environments: + raise ManifestParseException( + f'The environment defined at {attribute_path.path} reuse the name "{name}".' + " The environment name must be unique." + ) + self._build_environments.environments[name] = build_environment + + def resolve_build_environment( + self, + name: Optional[str], + attribute_path: AttributePath, + ) -> BuildEnvironmentDefinition: + if name is None: + return self.build_environments().default_environment + try: + env = self.build_environments().environments[name] + except KeyError: + raise ManifestParseException( + f'The environment "{name}" requested at {attribute_path.path} was not' + f" defined in the `build-environments`" + ) + else: + self._used_named_envs.add(name) + return env + + def build_environments(self) -> BuildEnvironments: + v = self._build_environments + if ( + not self._read_build_environment + and not self._build_environments.environments + and self._build_environments.default_environment is None + ): + self._build_environments.default_environment = BuildEnvironmentDefinition() + self._read_build_environment = True + return v + def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None: package_state = self.current_binary_package_state for dmh in package_state.dpkg_maintscript_helper_snippets: @@ -504,6 +592,7 @@ class YAMLManifestParser(HighLevelManifestParser): ) if service_rules: package_state.requested_service_rules.extend(service_rules) + self._build_rules = parsed_data.get("builds") return self.build_manifest() diff --git a/src/debputy/installations.py b/src/debputy/installations.py index b781757..806a964 100644 --- a/src/debputy/installations.py +++ b/src/debputy/installations.py @@ -31,10 +31,11 @@ from debputy.manifest_conditions import ( from debputy.manifest_parser.base_types import ( FileSystemMatchRule, FileSystemExactMatchRule, - DebputyDispatchableType, ) +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.packages import BinaryPackage from debputy.path_matcher import MatchRule, ExactFileSystemPath, MATCH_ANYTHING +from debputy.plugin.plugin_state import run_in_context_of_plugin from debputy.substitution import Substitution from debputy.util import _error, _warn @@ -585,6 +586,7 @@ class InstallRule(DebputyDispatchableType): *, match_filter: Optional[Callable[["VirtualPath"], bool]] = None, ) -> None: + super().__init__() self._condition = condition self._definition_source = definition_source self._match_filter = match_filter @@ -1025,7 +1027,9 @@ class PPFInstallRule(InstallRule): substitution: Substitution, ppfs: Sequence["PackagerProvidedFile"], ) -> None: - super().__init__( + run_in_context_of_plugin( + "debputy", + super().__init__, None, "<built-in; PPF install rule>", ) diff --git a/src/debputy/integration_detection.py b/src/debputy/integration_detection.py index f412268..cc19057 100644 --- a/src/debputy/integration_detection.py +++ b/src/debputy/integration_detection.py @@ -1,16 +1,21 @@ -from typing import Container, Optional +from typing import Container, Optional, Mapping from debputy.plugin.api.spec import ( DebputyIntegrationMode, INTEGRATION_MODE_DH_DEBPUTY_RRR, INTEGRATION_MODE_DH_DEBPUTY, + INTEGRATION_MODE_FULL, ) def determine_debputy_integration_mode( + source_fields: Mapping[str, str], all_sequences: Container[str], ) -> Optional[DebputyIntegrationMode]: + if source_fields.get("Build-Driver", "").lower() == "debputy": + return INTEGRATION_MODE_FULL + has_zz_debputy = "zz-debputy" in all_sequences or "debputy" in all_sequences has_zz_debputy_rrr = "zz-debputy-rrr" in all_sequences has_any_existing = has_zz_debputy or has_zz_debputy_rrr @@ -18,4 +23,7 @@ def determine_debputy_integration_mode( return INTEGRATION_MODE_DH_DEBPUTY_RRR if has_any_existing: return INTEGRATION_MODE_DH_DEBPUTY + if source_fields.get("Source", "") == "debputy": + # Self-hosting. We cannot set the Build-Driver field since that creates a self-circular dependency loop + return INTEGRATION_MODE_FULL return None diff --git a/src/debputy/linting/lint_impl.py b/src/debputy/linting/lint_impl.py index ddc7e93..0f37dce 100644 --- a/src/debputy/linting/lint_impl.py +++ b/src/debputy/linting/lint_impl.py @@ -35,7 +35,7 @@ from debputy.lsp.lsp_debian_tests_control import ( ) from debputy.lsp.maint_prefs import ( MaintainerPreferenceTable, - EffectivePreference, + EffectiveFormattingPreference, determine_effective_preference, ) from debputy.lsp.quickfixes import provide_standard_quickfixes_from_diagnostics @@ -92,7 +92,7 @@ class LintContext: parsed_deb822_file_content: Optional[Deb822FileElement] = None source_package: Optional[SourcePackage] = None binary_packages: Optional[Mapping[str, BinaryPackage]] = None - effective_preference: Optional[EffectivePreference] = None + effective_preference: Optional[EffectiveFormattingPreference] = None style_tool: Optional[str] = None unsupported_preference_reason: Optional[str] = None salsa_ci: Optional[CommentedMap] = None @@ -233,8 +233,6 @@ def perform_reformat( named_style: Optional[str] = None, ) -> None: parsed_args = context.parsed_args - if not parsed_args.spellcheck: - disable_spellchecking() fo = _output_styling(context.parsed_args, sys.stdout) lint_context = gather_lint_info(context) if named_style is not None: diff --git a/src/debputy/linting/lint_util.py b/src/debputy/linting/lint_util.py index 6346508..017a1dc 100644 --- a/src/debputy/linting/lint_util.py +++ b/src/debputy/linting/lint_util.py @@ -43,7 +43,7 @@ if TYPE_CHECKING: from debputy.lsp.text_util import LintCapablePositionCodec from debputy.lsp.maint_prefs import ( MaintainerPreferenceTable, - EffectivePreference, + EffectiveFormattingPreference, ) @@ -56,9 +56,14 @@ class DebputyMetadata: debputy_integration_mode: Optional[DebputyIntegrationMode] @classmethod - def from_data(cls, dh_sequencer_data: DhSequencerData) -> typing.Self: + def from_data( + cls, + source_fields: Mapping[str, str], + dh_sequencer_data: DhSequencerData, + ) -> typing.Self: integration_mode = determine_debputy_integration_mode( - dh_sequencer_data.sequences + source_fields, + dh_sequencer_data.sequences, ) return cls(integration_mode) @@ -114,12 +119,17 @@ class LintState: raise NotImplementedError @property - def effective_preference(self) -> Optional["EffectivePreference"]: + def effective_preference(self) -> Optional["EffectiveFormattingPreference"]: raise NotImplementedError @property def debputy_metadata(self) -> DebputyMetadata: - return DebputyMetadata.from_data(self.dh_sequencer_data) + src_pkg = self.source_package + src_fields = src_pkg.fields if src_pkg else {} + return DebputyMetadata.from_data( + src_fields, + self.dh_sequencer_data, + ) @property def dh_sequencer_data(self) -> DhSequencerData: @@ -137,7 +147,7 @@ class LintStateImpl(LintState): lines: List[str] source_package: Optional[SourcePackage] = None binary_packages: Optional[Mapping[str, BinaryPackage]] = None - effective_preference: Optional["EffectivePreference"] = None + effective_preference: Optional["EffectiveFormattingPreference"] = None _parsed_cache: Optional[Deb822FileElement] = None _dh_sequencer_cache: Optional[DhSequencerData] = None diff --git a/src/debputy/lsp/lsp_debian_control.py b/src/debputy/lsp/lsp_debian_control.py index 2b8f9b0..ac08266 100644 --- a/src/debputy/lsp/lsp_debian_control.py +++ b/src/debputy/lsp/lsp_debian_control.py @@ -677,7 +677,7 @@ def _doc_inlay_hint( stanza_range = stanza.range_in_parent() if stanza_no < 1: continue - pkg_kvpair = stanza.get_kvpair_element("Package", use_get=True) + pkg_kvpair = stanza.get_kvpair_element(("Package", 0), use_get=True) if pkg_kvpair is None: continue @@ -778,7 +778,7 @@ def _binary_package_checks( ) -> None: package_name = stanza.get("Package", "") source_section = source_stanza.get("Section") - section_kvpair = stanza.get_kvpair_element("Section", use_get=True) + section_kvpair = stanza.get_kvpair_element(("Section", 0), use_get=True) section: Optional[str] = None if section_kvpair is not None: section, section_range = extract_first_value_and_position( @@ -1227,7 +1227,7 @@ def _package_range_of_stanza( binary_stanzas: List[Tuple[Deb822ParagraphElement, TEPosition]], ) -> Iterable[Tuple[str, Optional[str], Range]]: for stanza, stanza_position in binary_stanzas: - kvpair = stanza.get_kvpair_element("Package", use_get=True) + kvpair = stanza.get_kvpair_element(("Package", 0), use_get=True) if kvpair is None: continue representation_field_range = kvpair.range_in_parent().relative_to( diff --git a/src/debputy/lsp/lsp_debian_control_reference_data.py b/src/debputy/lsp/lsp_debian_control_reference_data.py index 2ec885b..1d24045 100644 --- a/src/debputy/lsp/lsp_debian_control_reference_data.py +++ b/src/debputy/lsp/lsp_debian_control_reference_data.py @@ -104,7 +104,7 @@ except ImportError: if TYPE_CHECKING: - from debputy.lsp.maint_prefs import EffectivePreference + from debputy.lsp.maint_prefs import EffectiveFormattingPreference F = TypeVar("F", bound="Deb822KnownField") @@ -519,7 +519,7 @@ def _dctrl_ma_field_validation( stanza_position: "TEPosition", lint_state: LintState, ) -> Iterable[Diagnostic]: - ma_kvpair = stanza.get_kvpair_element("Multi-Arch", use_get=True) + ma_kvpair = stanza.get_kvpair_element(("Multi-Arch", 0), use_get=True) arch = stanza.get("Architecture", "any") if arch == "all" and ma_kvpair is not None: ma_value, ma_value_range = extract_first_value_and_position( @@ -1035,6 +1035,36 @@ def _dctrl_validate_dep( ) +def _rrr_build_driver_mismatch( + _known_field: "F", + _deb822_file: Deb822FileElement, + _kvpair: Deb822KeyValuePairElement, + kvpair_range_te: "TERange", + _field_name_range: "TERange", + stanza: Deb822ParagraphElement, + _stanza_position: "TEPosition", + lint_state: LintState, +) -> Iterable[Diagnostic]: + dr = stanza.get("Build-Driver", "debian-rules") + if dr != "debian-rules": + yield Diagnostic( + lint_state.position_codec.range_to_client_units( + lint_state.lines, + te_range_to_lsp(kvpair_range_te), + ), + f"The Rules-Requires-Root field is irrelevant for the `Build-Driver` `{dr}`.", + DiagnosticSeverity.Warning, + source="debputy", + data=DiagnosticData( + quickfixes=[ + propose_remove_range_quick_fix( + proposed_title="Remove Rules-Requires-Root" + ) + ] + ), + ) + + class Dep5Matcher(BasenameGlobMatch): def __init__(self, basename_glob: str) -> None: super().__init__( @@ -1979,7 +2009,7 @@ class Deb822KnownField: def reformat_field( self, - effective_preference: "EffectivePreference", + effective_preference: "EffectiveFormattingPreference", stanza_range: TERange, kvpair: Deb822KeyValuePairElement, formatter: FormatterCallback, @@ -2002,7 +2032,7 @@ class DctrlLikeKnownField(Deb822KnownField): def reformat_field( self, - effective_preference: "EffectivePreference", + effective_preference: "EffectiveFormattingPreference", stanza_range: TERange, kvpair: Deb822KeyValuePairElement, formatter: FormatterCallback, @@ -2011,7 +2041,7 @@ class DctrlLikeKnownField(Deb822KnownField): ) -> Iterable[TextEdit]: interpretation = self.field_value_class.interpreter() if ( - not effective_preference.formatting_deb822_normalize_field_content + not effective_preference.deb822_normalize_field_content or interpretation is None ): yield from super(DctrlLikeKnownField, self).reformat_field( @@ -2133,7 +2163,7 @@ class DctrlKnownField(DctrlLikeKnownField): def reformat_field( self, - effective_preference: "EffectivePreference", + effective_preference: "EffectiveFormattingPreference", stanza_range: TERange, kvpair: Deb822KeyValuePairElement, formatter: FormatterCallback, @@ -2142,7 +2172,7 @@ class DctrlKnownField(DctrlLikeKnownField): ) -> Iterable[TextEdit]: if ( self.name == "Architecture" - and effective_preference.formatting_deb822_normalize_field_content + and effective_preference.deb822_normalize_field_content ): interpretation = self.field_value_class.interpreter() assert interpretation is not None @@ -2317,6 +2347,47 @@ SOURCE_FIELDS = _fields( ), ), DctrlKnownField( + "Build-Driver", + FieldValueClass.SINGLE_VALUE, + default_value="debian-rules", + known_values=allowed_values( + Keyword( + "debian-rules", + synopsis_doc="Build via `debian/rules`", + hover_text=textwrap.dedent( + """\ + Use the `debian/rules` interface for building packages. + + This is the historical default and the interface that Debian Packages have used for + decades to build debs. + """ + ), + ), + Keyword( + "debputy", + synopsis_doc="Build with `debputy`", + hover_text=textwrap.dedent( + """\ + Use the `debputy` interface for building the package. + + This is provides the "full" integration mode with `debputy` where all parts of the + package build is handled by `debputy`. + + This *may* make any `debhelper` build-dependency redundant depending on which build + system is used. Some build systems (such as `autoconf` still use `debhelper` based tools). + """ + ), + ), + ), + synopsis_doc="Which implementation dpkg should use for the build", + hover_text=textwrap.dedent( + """\ + The name of the build driver that dpkg (`dpkg-buildpackage`) will use for assembling the + package. + """ + ), + ), + DctrlKnownField( "Vcs-Browser", FieldValueClass.SINGLE_VALUE, synopsis_doc="URL for browsers to interact with packaging VCS", @@ -2580,6 +2651,7 @@ SOURCE_FIELDS = _fields( "Rules-Requires-Root", FieldValueClass.SPACE_SEPARATED_LIST, unknown_value_diagnostic_severity=None, + custom_field_check=_rrr_build_driver_mismatch, known_values=allowed_values( Keyword( "no", @@ -2642,6 +2714,9 @@ SOURCE_FIELDS = _fields( ` Build-Depends` on `dpkg-build-api (>= 1)` or later, the default is `no`. Otherwise, the default is `binary-target` + This field is only relevant when when the `Build-Driver` is `debian-rules` (which it is by + default). + Note it is **not** possible to require running the package as "true root". """ ), @@ -3905,6 +3980,57 @@ _DEP5_HEADER_FIELDS = _fields( """ ), ), + Deb822KnownField( + "Files-Excluded", + FieldValueClass.FREE_TEXT_FIELD, + hover_text=textwrap.dedent( + """\ + Remove the listed files from the tarball when repacking (commonly via uscan). This can be useful when the + listed files are non-free but not necessary for the Debian package. In this case, the upstream version of + the package should generally end with `~dfsg` or `+dfsg` (to mark the content changed due to the + Debian Free Software Guidelines). The exclusion can also be useful to remove large files or directories + that are not used by Debian or pre-built binaries. In this case, `~ds` or `+ds` should be added to the + version instead of `~dfsg` or `+dfsg` for "Debian Source" to mark it as altered by Debian. If both reasons + are used, the `~dfsg` or `+dfsg` version is used as that is the more important reason for the repacking. + + Example: + ``` + Files-Excluded: exclude-this + exclude-dir + */exclude-dir + .* + */js/jquery.js + ``` + + The `Files-Included` field can be used to "re-include" files matched by `Files-Excluded`. + + It is also possible to exclude files in specific "upstream components" for source packages with multiple + upstream tarballs. This is done by adding a field called `Files-Excluded-<component>`. The `<component>` + part should then match the component name exactly (case sensitive). + + Defined by: mk-origtargz (usually used via uscan) + """ + ), + ), + Deb822KnownField( + "Files-Included", + FieldValueClass.FREE_TEXT_FIELD, + hover_text=textwrap.dedent( + """\ + Re-include files that were marked for exclusion by `Files-Excluded`. This can be useful for "exclude + everything except X" style semantics where `Files-Excluded` has a very broad pattern and + `Files-Included` then marks a few exceptions. + + It is also possible to re-include files in specific "upstream components" for source packages with multiple + upstream tarballs. This is done by adding a field called `Files-Include-<component>` which is then used + in tandem with `Files-Exclude-<component>`. The `<component>` part should then match the component name + exactly (case sensitive). + + + Defined by: mk-origtargz (usually used via uscan) + """ + ), + ), ) _DEP5_FILES_FIELDS = _fields( Deb822KnownField( @@ -4664,7 +4790,7 @@ class StanzaMetadata(Mapping[str, F], Generic[F], ABC): def reformat_stanza( self, - effective_preference: "EffectivePreference", + effective_preference: "EffectiveFormattingPreference", stanza: Deb822ParagraphElement, stanza_range: TERange, formatter: FormatterCallback, @@ -4672,7 +4798,7 @@ class StanzaMetadata(Mapping[str, F], Generic[F], ABC): lines: List[str], ) -> Iterable[TextEdit]: for known_field in self.stanza_fields.values(): - kvpair = stanza.get_kvpair_element(known_field.name, use_get=True) + kvpair = stanza.get_kvpair_element((known_field.name, 0), use_get=True) if kvpair is None: continue yield from known_field.reformat_field( @@ -4744,7 +4870,7 @@ class Deb822FileMetadata(Generic[S]): def reformat( self, - effective_preference: "EffectivePreference", + effective_preference: "EffectiveFormattingPreference", deb822_file: Deb822FileElement, formatter: FormatterCallback, _content: str, @@ -4856,7 +4982,7 @@ class DctrlFileMetadata(Deb822FileMetadata[DctrlStanzaMetadata]): def reformat( self, - effective_preference: "EffectivePreference", + effective_preference: "EffectiveFormattingPreference", deb822_file: Deb822FileElement, formatter: FormatterCallback, content: str, @@ -4875,7 +5001,7 @@ class DctrlFileMetadata(Deb822FileMetadata[DctrlStanzaMetadata]): ) if ( - not effective_preference.formatting_deb822_normalize_stanza_order + not effective_preference.deb822_normalize_stanza_order or deb822_file.find_first_error_element() is not None ): return edits diff --git a/src/debputy/lsp/lsp_debian_debputy_manifest.py b/src/debputy/lsp/lsp_debian_debputy_manifest.py index a8a2fdf..8c7aeac 100644 --- a/src/debputy/lsp/lsp_debian_debputy_manifest.py +++ b/src/debputy/lsp/lsp_debian_debputy_manifest.py @@ -47,7 +47,7 @@ from debputy.lsprotocol.types import ( DiagnosticRelatedInformation, Location, ) -from debputy.manifest_parser.base_types import DebputyDispatchableType +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.manifest_parser.declarative_parser import ( AttributeDescription, ParserGenerator, @@ -57,7 +57,6 @@ from debputy.manifest_parser.declarative_parser import DeclarativeMappingInputPa from debputy.manifest_parser.util import AttributePath from debputy.plugin.api.impl import plugin_metadata_for_debputys_own_plugin from debputy.plugin.api.impl_types import ( - OPARSER_MANIFEST_ROOT, DeclarativeInputParser, DispatchingParserBase, DebputyPluginMetadata, @@ -65,6 +64,7 @@ from debputy.plugin.api.impl_types import ( InPackageContextParser, DeclarativeValuelessKeywordInputParser, ) +from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT from debputy.plugin.api.spec import DebputyIntegrationMode from debputy.plugin.debputy.private_api import Capability, load_libcap from debputy.util import _info, detect_possible_typo @@ -742,6 +742,22 @@ def _insert_snippet(lines: List[str], server_position: Position) -> bool: return True +def _maybe_quote(v: str) -> str: + if v and v[0].isdigit(): + try: + float(v) + return f"'{v}'" + except ValueError: + pass + return v + + +def _complete_value(v: Any) -> str: + if isinstance(v, str): + return _maybe_quote(v) + return str(v) + + @lsp_completer(_LANGUAGE_IDS) def debputy_manifest_completer( ls: "DebputyLanguageServer", @@ -820,7 +836,9 @@ def debputy_manifest_completer( if isinstance(parser, DispatchingParserBase): if matched_key: items = [ - CompletionItem(k if has_colon else f"{k}:") + CompletionItem( + _maybe_quote(k) if has_colon else f"{_maybe_quote(k)}:" + ) for k in parser.registered_keywords() if k not in parent and not isinstance( @@ -830,7 +848,7 @@ def debputy_manifest_completer( ] else: items = [ - CompletionItem(k) + CompletionItem(_maybe_quote(k)) for k in parser.registered_keywords() if k not in parent and isinstance( @@ -842,7 +860,9 @@ def debputy_manifest_completer( binary_packages = ls.lint_state(doc).binary_packages if binary_packages is not None: items = [ - CompletionItem(p if has_colon else f"{p}:") + CompletionItem( + _maybe_quote(p) if has_colon else f"{_maybe_quote(p)}:" + ) for p in binary_packages if p not in parent ] @@ -858,7 +878,9 @@ def debputy_manifest_completer( locked.add(attr_name) break items = [ - CompletionItem(k if has_colon else f"{k}:") + CompletionItem( + _maybe_quote(k) if has_colon else f"{_maybe_quote(k)}:" + ) for k in parser.manifest_attributes if k not in locked ] @@ -913,7 +935,7 @@ def _completion_from_attr( _info(f"Already filled: {matched} is one of {valid_values}") return None if valid_values: - return [CompletionItem(x) for x in valid_values] + return [CompletionItem(_complete_value(x)) for x in valid_values] return None diff --git a/src/debputy/lsp/lsp_generic_yaml.py b/src/debputy/lsp/lsp_generic_yaml.py index 94267f7..5e67428 100644 --- a/src/debputy/lsp/lsp_generic_yaml.py +++ b/src/debputy/lsp/lsp_generic_yaml.py @@ -1,6 +1,6 @@ from typing import Union, Any, Optional, List, Tuple -from debputy.manifest_parser.base_types import DebputyDispatchableType +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.manifest_parser.declarative_parser import DeclarativeMappingInputParser from debputy.manifest_parser.parser_doc import ( render_rule, diff --git a/src/debputy/lsp/maint_prefs.py b/src/debputy/lsp/maint_prefs.py index fa6315b..4cc70d5 100644 --- a/src/debputy/lsp/maint_prefs.py +++ b/src/debputy/lsp/maint_prefs.py @@ -32,37 +32,37 @@ PT = TypeVar("PT", bool, str, int) BUILTIN_STYLES = os.path.join(os.path.dirname(__file__), "maint-preferences.yaml") -_NORMALISE_FIELD_CONTENT_KEY = ["formatting", "deb822", "normalize-field-content"] +_NORMALISE_FIELD_CONTENT_KEY = ["deb822", "normalize-field-content"] _UPLOADER_SPLIT_RE = re.compile(r"(?<=>)\s*,") _WAS_OPTIONS = { - "-a": ("formatting_deb822_always_wrap", True), - "--always-wrap": ("formatting_deb822_always_wrap", True), - "-s": ("formatting_deb822_short_indent", True), - "--short-indent": ("formatting_deb822_short_indent", True), - "-t": ("formatting_deb822_trailing_separator", True), - "--trailing-separator": ("formatting_deb822_trailing_separator", True), + "-a": ("deb822_always_wrap", True), + "--always-wrap": ("deb822_always_wrap", True), + "-s": ("deb822_short_indent", True), + "--short-indent": ("deb822_short_indent", True), + "-t": ("deb822_trailing_separator", True), + "--trailing-separator": ("deb822_trailing_separator", True), # Noise option for us; we do not accept `--no-keep-first` though "-k": (None, True), "--keep-first": (None, True), "--no-keep-first": ("DISABLE_NORMALIZE_STANZA_ORDER", True), - "-b": ("formatting_deb822_normalize_stanza_order", True), - "--sort-binary-packages": ("formatting_deb822_normalize_stanza_order", True), + "-b": ("deb822_normalize_stanza_order", True), + "--sort-binary-packages": ("deb822_normalize_stanza_order", True), } _WAS_DEFAULTS = { - "formatting_deb822_always_wrap": False, - "formatting_deb822_short_indent": False, - "formatting_deb822_trailing_separator": False, - "formatting_deb822_normalize_stanza_order": False, - "formatting_deb822_normalize_field_content": True, + "deb822_always_wrap": False, + "deb822_short_indent": False, + "deb822_trailing_separator": False, + "deb822_normalize_stanza_order": False, + "deb822_normalize_field_content": True, } @dataclasses.dataclass(slots=True, frozen=True, kw_only=True) class PreferenceOption(Generic[PT]): key: Union[str, List[str]] - expected_type: Type[PT] + expected_type: Union[Type[PT], Callable[[Any], Optional[str]]] description: str default_value: Optional[Union[PT, Callable[[CommentedMap], Optional[PT]]]] = None @@ -88,11 +88,17 @@ class PreferenceOption(Generic[PT]): if callable(default_value): return default_value(data) return default_value - if isinstance(v, self.expected_type): + val_issue: Optional[str] = None + expected_type = self.expected_type + if not isinstance(expected_type, type) and callable(self.expected_type): + val_issue = self.expected_type(v) + elif not isinstance(v, self.expected_type): + val_issue = f"It should have been a {self.expected_type} but it was not" + + if val_issue is None: return v raise ValueError( - f'The value "{self.name}" for key {key} in file "{filename}" should have been a' - f" {self.expected_type} but it was not" + f'The value "{self.name}" for key {key} in file "{filename}" was incorrect: {val_issue}' ) @@ -108,7 +114,7 @@ def _false_when_formatting_content(m: CommentedMap) -> Optional[bool]: return m.mlget(_NORMALISE_FIELD_CONTENT_KEY, list_ok=True, default=False) is True -OPTIONS: List[PreferenceOption] = [ +MAINT_OPTIONS: List[PreferenceOption] = [ PreferenceOption( key="canonical-name", expected_type=str, @@ -139,7 +145,25 @@ OPTIONS: List[PreferenceOption] = [ ), ), PreferenceOption( - key=["formatting", "deb822", "short-indent"], + key="formatting", + expected_type=lambda x: ( + None + if isinstance(x, EffectiveFormattingPreference) + else "It should have been a EffectiveFormattingPreference but it was not" + ), + default_value=None, + description=textwrap.dedent( + """\ + The formatting preference of the maintainer. Can either be a string for a named style or an inline + style. + """ + ), + ), +] + +FORMATTING_OPTIONS = [ + PreferenceOption( + key=["deb822", "short-indent"], expected_type=bool, description=textwrap.dedent( """\ @@ -175,7 +199,7 @@ OPTIONS: List[PreferenceOption] = [ ), ), PreferenceOption( - key=["formatting", "deb822", "always-wrap"], + key=["deb822", "always-wrap"], expected_type=bool, description=textwrap.dedent( """\ @@ -210,7 +234,7 @@ OPTIONS: List[PreferenceOption] = [ ), ), PreferenceOption( - key=["formatting", "deb822", "trailing-separator"], + key=["deb822", "trailing-separator"], expected_type=bool, default_value=False, description=textwrap.dedent( @@ -241,7 +265,7 @@ OPTIONS: List[PreferenceOption] = [ ), ), PreferenceOption( - key=["formatting", "deb822", "max-line-length"], + key=["deb822", "max-line-length"], expected_type=int, default_value=79, description=textwrap.dedent( @@ -297,7 +321,7 @@ OPTIONS: List[PreferenceOption] = [ ), ), PreferenceOption( - key=["formatting", "deb822", "normalize-field-order"], + key=["deb822", "normalize-field-order"], expected_type=bool, default_value=False, description=textwrap.dedent( @@ -332,7 +356,7 @@ OPTIONS: List[PreferenceOption] = [ ), ), PreferenceOption( - key=["formatting", "deb822", "normalize-stanza-order"], + key=["deb822", "normalize-stanza-order"], expected_type=bool, default_value=False, description=textwrap.dedent( @@ -385,43 +409,43 @@ OPTIONS: List[PreferenceOption] = [ @dataclasses.dataclass(slots=True, frozen=True) -class EffectivePreference: - formatting_deb822_short_indent: Optional[bool] = None - formatting_deb822_always_wrap: Optional[bool] = None - formatting_deb822_trailing_separator: bool = False - formatting_deb822_normalize_field_content: bool = False - formatting_deb822_normalize_field_order: bool = False - formatting_deb822_normalize_stanza_order: bool = False - formatting_deb822_max_line_length: int = 79 +class EffectiveFormattingPreference: + deb822_short_indent: Optional[bool] = None + deb822_always_wrap: Optional[bool] = None + deb822_trailing_separator: bool = False + deb822_normalize_field_content: bool = False + deb822_normalize_field_order: bool = False + deb822_normalize_stanza_order: bool = False + deb822_max_line_length: int = 79 @classmethod def from_file( cls, filename: str, key: str, - stylees: CommentedMap, + styles: CommentedMap, ) -> Self: attr = {} - for option in OPTIONS: + for option in FORMATTING_OPTIONS: if not hasattr(cls, option.attribute_name): continue - value = option.extract_value(filename, key, stylees) + value = option.extract_value(filename, key, styles) attr[option.attribute_name] = value return cls(**attr) # type: ignore @classmethod def aligned_preference( cls, - a: Optional["EffectivePreference"], - b: Optional["EffectivePreference"], - ) -> Optional["EffectivePreference"]: + a: Optional["EffectiveFormattingPreference"], + b: Optional["EffectiveFormattingPreference"], + ) -> Optional["EffectiveFormattingPreference"]: if a is None or b is None: return None - for option in OPTIONS: + for option in MAINT_OPTIONS: attr_name = option.attribute_name - if not hasattr(EffectivePreference, attr_name): + if not hasattr(EffectiveFormattingPreference, attr_name): continue a_value = getattr(a, attr_name) b_value = getattr(b, attr_name) @@ -430,14 +454,12 @@ class EffectivePreference: return a def deb822_formatter(self) -> FormatterCallback: - line_length = self.formatting_deb822_max_line_length + line_length = self.deb822_max_line_length return wrap_and_sort_formatter( - 1 if self.formatting_deb822_short_indent else "FIELD_NAME_LENGTH", - trailing_separator=self.formatting_deb822_trailing_separator, - immediate_empty_line=self.formatting_deb822_short_indent or False, - max_line_length_one_liner=( - 0 if self.formatting_deb822_always_wrap else line_length - ), + 1 if self.deb822_short_indent else "FIELD_NAME_LENGTH", + trailing_separator=self.deb822_trailing_separator, + immediate_empty_line=self.deb822_short_indent or False, + max_line_length_one_liner=(0 if self.deb822_always_wrap else line_length), ) def replace(self, /, **changes: Any) -> Self: @@ -445,24 +467,33 @@ class EffectivePreference: @dataclasses.dataclass(slots=True, frozen=True) -class MaintainerPreference(EffectivePreference): +class MaintainerPreference: canonical_name: Optional[str] = None is_packaging_team: bool = False + formatting: Optional[EffectiveFormattingPreference] = None - def as_effective_pref(self) -> EffectivePreference: - fields = { - k: v - for k, v in dataclasses.asdict(self).items() - if hasattr(EffectivePreference, k) - } - return EffectivePreference(**fields) + @classmethod + def from_file( + cls, + filename: str, + key: str, + styles: CommentedMap, + ) -> Self: + attr = {} + + for option in MAINT_OPTIONS: + if not hasattr(cls, option.attribute_name): + continue + value = option.extract_value(filename, key, styles) + attr[option.attribute_name] = value + return cls(**attr) # type: ignore class MaintainerPreferenceTable: def __init__( self, - named_styles: Mapping[str, EffectivePreference], + named_styles: Mapping[str, EffectiveFormattingPreference], maintainer_preferences: Mapping[str, MaintainerPreference], ) -> None: self._named_styles = named_styles @@ -470,7 +501,7 @@ class MaintainerPreferenceTable: @classmethod def load_preferences(cls) -> Self: - named_styles: Dict[str, EffectivePreference] = {} + named_styles: Dict[str, EffectiveFormattingPreference] = {} maintainer_preferences: Dict[str, MaintainerPreference] = {} with open(BUILTIN_STYLES) as fd: parse_file(named_styles, maintainer_preferences, BUILTIN_STYLES, fd) @@ -488,7 +519,7 @@ class MaintainerPreferenceTable: return cls(named_styles, maintainer_preferences) @property - def named_styles(self) -> Mapping[str, EffectivePreference]: + def named_styles(self) -> Mapping[str, EffectiveFormattingPreference]: return self._named_styles @property @@ -497,7 +528,7 @@ class MaintainerPreferenceTable: def parse_file( - named_styles: Dict[str, EffectivePreference], + named_styles: Dict[str, EffectiveFormattingPreference], maintainer_preferences: Dict[str, MaintainerPreference], filename: str, fd, @@ -520,38 +551,45 @@ def parse_file( named_styles_raw = {} for style_name, content in named_styles_raw.items(): - wrapped_style = CommentedMap({"formatting": content}) - style = EffectivePreference.from_file( + style = EffectiveFormattingPreference.from_file( filename, style_name, - wrapped_style, + content, ) named_styles[style_name] = style - for maintainer_email, maintainer_styles in maintainer_rules.items(): - if not isinstance(maintainer_styles, CommentedMap): + for maintainer_email, maintainer_pref in maintainer_rules.items(): + if not isinstance(maintainer_pref, CommentedMap): line_no = maintainer_rules.lc.key(maintainer_email).line raise ValueError( f'The value for maintainer "{maintainer_email}" should have been a mapping,' f' but it is not. The problem entry is at line {line_no} in "{filename}"' ) - formatting = maintainer_styles.get("formatting") + formatting = maintainer_pref.get("formatting") if isinstance(formatting, str): try: - style = named_styles_raw[formatting] + style = named_styles[formatting] except KeyError: line_no = maintainer_rules.lc.key(maintainer_email).line raise ValueError( f'The maintainer "{maintainer_email}" requested the named style "{formatting}",' f' but said style was not defined {filename}. The problem entry is at line {line_no} in "{filename}"' ) from None - maintainer_styles["formatting"] = style - maintainer_preferences[maintainer_email] = MaintainerPreference.from_file( + maintainer_pref["formatting"] = style + elif formatting is not None: + maintainer_pref["formatting"] = EffectiveFormattingPreference.from_file( + filename, + "formatting", + formatting, + ) + mp = MaintainerPreference.from_file( filename, maintainer_email, - maintainer_styles, + maintainer_pref, ) + maintainer_preferences[maintainer_email] = mp + @functools.lru_cache(64) def extract_maint_email(maint: str) -> str: @@ -569,7 +607,7 @@ def determine_effective_preference( maint_preference_table: MaintainerPreferenceTable, source_package: Optional[SourcePackage], salsa_ci: Optional[CommentedMap], -) -> Tuple[Optional[EffectivePreference], Optional[str], Optional[str]]: +) -> Tuple[Optional[EffectiveFormattingPreference], Optional[str], Optional[str]]: style = source_package.fields.get("X-Style") if source_package is not None else None if style is not None: if style not in ALL_PUBLIC_NAMED_STYLES: @@ -612,7 +650,6 @@ def determine_effective_preference( else: msg = None return detected_style, tool_w_args, msg - if source_package is None: return None, None, None @@ -620,47 +657,44 @@ def determine_effective_preference( if maint is None: return None, None, None maint_email = extract_maint_email(maint) - maint_style = maint_preference_table.maintainer_preferences.get(maint_email) + maint_pref = maint_preference_table.maintainer_preferences.get(maint_email) # Special-case "@packages.debian.org" when missing, since they are likely to be "ad-hoc" # teams that will not be registered. In that case, we fall back to looking at the uploader # preferences as-if the maintainer had not been listed at all. - if maint_style is None and not maint_email.endswith("@packages.debian.org"): + if maint_pref is None and not maint_email.endswith("@packages.debian.org"): return None, None, None - if maint_style is not None and maint_style.is_packaging_team: + if maint_pref is not None and maint_pref.is_packaging_team: # When the maintainer is registered as a packaging team, then we assume the packaging # team's style applies unconditionally. - effective = maint_style.as_effective_pref() + effective = maint_pref.formatting tool_w_args = _guess_tool_from_style(maint_preference_table, effective) return effective, tool_w_args, None uploaders = source_package.fields.get("Uploaders") if uploaders is None: - detected_style = ( - maint_style.as_effective_pref() if maint_style is not None else None - ) + detected_style = maint_pref.formatting if maint_pref is not None else None tool_w_args = _guess_tool_from_style(maint_preference_table, detected_style) return detected_style, tool_w_args, None - all_styles: List[Optional[EffectivePreference]] = [] - if maint_style is not None: - all_styles.append(maint_style) + all_styles: List[Optional[EffectiveFormattingPreference]] = [] + if maint_pref is not None: + all_styles.append(maint_pref.formatting) for uploader in _UPLOADER_SPLIT_RE.split(uploaders): uploader_email = extract_maint_email(uploader) - uploader_style = maint_preference_table.maintainer_preferences.get( + uploader_pref = maint_preference_table.maintainer_preferences.get( uploader_email ) - all_styles.append(uploader_style) + all_styles.append(uploader_pref.formatting if uploader_pref else None) if not all_styles: return None, None, None - r = functools.reduce(EffectivePreference.aligned_preference, all_styles) - if isinstance(r, MaintainerPreference): - r = r.as_effective_pref() + r = functools.reduce(EffectiveFormattingPreference.aligned_preference, all_styles) + assert not isinstance(r, MaintainerPreference) tool_w_args = _guess_tool_from_style(maint_preference_table, r) return r, tool_w_args, None def _guess_tool_from_style( maint_preference_table: MaintainerPreferenceTable, - pref: Optional[EffectivePreference], + pref: Optional[EffectiveFormattingPreference], ) -> Optional[str]: if pref is None: return None @@ -682,7 +716,9 @@ def _split_options(args: Iterable[str]) -> Iterable[str]: @functools.lru_cache -def parse_salsa_ci_wrap_and_sort_args(args: str) -> Optional[EffectivePreference]: +def parse_salsa_ci_wrap_and_sort_args( + args: str, +) -> Optional[EffectiveFormattingPreference]: options = dict(_WAS_DEFAULTS) for arg in _split_options(args.split()): v = _WAS_OPTIONS.get(arg) @@ -694,6 +730,6 @@ def parse_salsa_ci_wrap_and_sort_args(args: str) -> Optional[EffectivePreference options[varname] = value if "DISABLE_NORMALIZE_STANZA_ORDER" in options: del options["DISABLE_NORMALIZE_STANZA_ORDER"] - options["formatting_deb822_normalize_stanza_order"] = False + options["deb822_normalize_stanza_order"] = False - return EffectivePreference(**options) # type: ignore + return EffectiveFormattingPreference(**options) # type: ignore diff --git a/src/debputy/maintscript_snippet.py b/src/debputy/maintscript_snippet.py index ca81ca5..58a6bba 100644 --- a/src/debputy/maintscript_snippet.py +++ b/src/debputy/maintscript_snippet.py @@ -1,7 +1,7 @@ import dataclasses from typing import Sequence, Optional, List, Literal, Iterable, Dict, Self -from debputy.manifest_parser.base_types import DebputyDispatchableType +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.manifest_parser.util import AttributePath STD_CONTROL_SCRIPTS = frozenset( diff --git a/src/debputy/manifest_conditions.py b/src/debputy/manifest_conditions.py index 0f5c298..3e97b00 100644 --- a/src/debputy/manifest_conditions.py +++ b/src/debputy/manifest_conditions.py @@ -1,12 +1,12 @@ import dataclasses from enum import Enum -from typing import List, Callable, Optional, Sequence +from typing import List, Callable, Optional, Sequence, Any, Self, Mapping from debian.debian_support import DpkgArchTable from debputy._deb_options_profiles import DebBuildOptionsAndProfiles from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable -from debputy.manifest_parser.base_types import DebputyDispatchableType +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.packages import BinaryPackage from debputy.substitution import Substitution from debputy.util import active_profiles_match @@ -15,11 +15,14 @@ from debputy.util import active_profiles_match @dataclasses.dataclass(slots=True, frozen=True) class ConditionContext: binary_package: Optional[BinaryPackage] - build_env: DebBuildOptionsAndProfiles + deb_options_and_profiles: DebBuildOptionsAndProfiles substitution: Substitution dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable dpkg_arch_query_table: DpkgArchTable + def replace(self, /, **changes: Any) -> "Self": + return dataclasses.replace(self, **changes) + class ManifestCondition(DebputyDispatchableType): __slots__ = () @@ -72,6 +75,7 @@ class NegatedManifestCondition(ManifestCondition): __slots__ = ("_condition",) def __init__(self, condition: ManifestCondition) -> None: + super().__init__() self._condition = condition def negated(self) -> "ManifestCondition": @@ -107,6 +111,7 @@ class ManifestConditionGroup(ManifestCondition): match_type: _ConditionGroupMatchType, conditions: Sequence[ManifestCondition], ) -> None: + super().__init__() self.match_type = match_type self._conditions = conditions @@ -132,6 +137,7 @@ class ArchMatchManifestConditionBase(ManifestCondition): __slots__ = ("_arch_spec", "_is_negated") def __init__(self, arch_spec: List[str], *, is_negated: bool = False) -> None: + super().__init__() self._arch_spec = arch_spec self._is_negated = is_negated @@ -177,6 +183,7 @@ class BuildProfileMatch(ManifestCondition): __slots__ = ("_profile_spec", "_is_negated") def __init__(self, profile_spec: str, *, is_negated: bool = False) -> None: + super().__init__() self._profile_spec = profile_spec self._is_negated = is_negated @@ -190,7 +197,7 @@ class BuildProfileMatch(ManifestCondition): def evaluate(self, context: ConditionContext) -> bool: match = active_profiles_match( - self._profile_spec, context.build_env.deb_build_profiles + self._profile_spec, context.deb_options_and_profiles.deb_build_profiles ) return not match if self._is_negated else match @@ -211,7 +218,14 @@ def _can_run_built_binaries(context: ConditionContext) -> bool: if not context.dpkg_architecture_variables.is_cross_compiling: return True # User / Builder asserted that we could even though we are cross-compiling, so we have to assume it is true - return "crossbuildcanrunhostbinaries" in context.build_env.deb_build_options + return ( + "crossbuildcanrunhostbinaries" + in context.deb_options_and_profiles.deb_build_options + ) + + +def _run_build_time_tests(deb_build_options: Mapping[str, Optional[str]]) -> bool: + return "nocheck" not in deb_build_options _IS_CROSS_BUILDING = _SingletonCondition( @@ -226,12 +240,12 @@ _CAN_EXECUTE_COMPILED_BINARIES = _SingletonCondition( _RUN_BUILD_TIME_TESTS = _SingletonCondition( "Run build time tests", - lambda c: "nocheck" not in c.build_env.deb_build_options, + lambda c: _run_build_time_tests(c.deb_options_and_profiles.deb_build_options), ) _BUILD_DOCS_BDO = _SingletonCondition( "Build docs (nodocs not in DEB_BUILD_OPTIONS)", - lambda c: "nodocs" not in c.build_env.deb_build_options, + lambda c: "nodocs" not in c.deb_options_and_profiles.deb_build_options, ) diff --git a/src/debputy/manifest_parser/base_types.py b/src/debputy/manifest_parser/base_types.py index 865e320..106c30e 100644 --- a/src/debputy/manifest_parser/base_types.py +++ b/src/debputy/manifest_parser/base_types.py @@ -1,9 +1,8 @@ import dataclasses import os +import subprocess from functools import lru_cache from typing import ( - TypedDict, - NotRequired, Sequence, Optional, Union, @@ -12,12 +11,14 @@ from typing import ( Mapping, Iterable, TYPE_CHECKING, - Callable, - Type, - Generic, + Dict, + MutableMapping, + NotRequired, ) +from debputy.manifest_conditions import ManifestCondition from debputy.manifest_parser.exceptions import ManifestParseException +from debputy.manifest_parser.tagging_types import DebputyParsedContent from debputy.manifest_parser.util import ( AttributePath, _SymbolicModeSegment, @@ -25,37 +26,20 @@ from debputy.manifest_parser.util import ( ) from debputy.path_matcher import MatchRule, ExactFileSystemPath from debputy.substitution import Substitution -from debputy.types import S -from debputy.util import _normalize_path, T +from debputy.util import _normalize_path, _error, _warn, _debug_log if TYPE_CHECKING: - from debputy.manifest_conditions import ManifestCondition from debputy.manifest_parser.parser_data import ParserContextData -class DebputyParsedContent(TypedDict): - pass - - -class DebputyDispatchableType: - __slots__ = () - - -class DebputyParsedContentStandardConditional(DebputyParsedContent): - when: NotRequired["ManifestCondition"] - - @dataclasses.dataclass(slots=True, frozen=True) class OwnershipDefinition: entity_name: str entity_id: int -@dataclasses.dataclass -class TypeMapping(Generic[S, T]): - target_type: Type[T] - source_type: Type[S] - mapper: Callable[[S, AttributePath, Optional["ParserContextData"]], T] +class DebputyParsedContentStandardConditional(DebputyParsedContent): + when: NotRequired[ManifestCondition] ROOT_DEFINITION = OwnershipDefinition("root", 0) @@ -438,3 +422,76 @@ class FileSystemExactMatchRule(FileSystemMatchRule): class FileSystemExactNonDirMatchRule(FileSystemExactMatchRule): pass + + +class BuildEnvironmentDefinition: + + def dpkg_buildflags_env( + self, + env: Mapping[str, str], + definition_source: Optional[str], + ) -> Dict[str, str]: + dpkg_env = {} + try: + bf_output = subprocess.check_output(["dpkg-buildflags"], env=env) + except FileNotFoundError: + if definition_source is None: + _error( + "The dpkg-buildflags command was not available and is necessary to set the relevant" + "env variables by default." + ) + _error( + "The dpkg-buildflags command was not available and is necessary to set the relevant" + f"env variables for the environment defined at {definition_source}." + ) + except subprocess.CalledProcessError as e: + if definition_source is None: + _error( + f"The dpkg-buildflags command failed with exit code {e.returncode}. Please review the output from" + f" dpkg-buildflags above to resolve the issue." + ) + _error( + f"The dpkg-buildflags command failed with exit code {e.returncode}. Please review the output from" + f" dpkg-buildflags above to resolve the issue. The environment definition that triggered this call" + f" was {definition_source}" + ) + else: + warned = False + for line in bf_output.decode("utf-8").splitlines(keepends=False): + if "=" not in line or line.startswith("="): + if not warned: + _warn( + f"Unexpected output from dpkg-buildflags (not a K=V line): {line}" + ) + continue + k, v = line.split("=", 1) + if k.strip() != k: + if not warned: + _warn( + f'Unexpected output from dpkg-buildflags (Key had spaces): "{line}"' + ) + continue + dpkg_env[k] = v + + return dpkg_env + + def log_computed_env(self, source: str, computed_env: Mapping[str, str]) -> None: + _debug_log(f"Computed environment variables from {source}") + for k, v in computed_env.items(): + _debug_log(f" {k}={v}") + + def update_env(self, env: MutableMapping[str, str]) -> None: + dpkg_env = self.dpkg_buildflags_env(env, None) + self.log_computed_env("dpkg-buildflags", dpkg_env) + env.update(dpkg_env) + + +class BuildEnvironments: + + def __init__( + self, + environments: Dict[str, BuildEnvironmentDefinition], + default_environment: Optional[BuildEnvironmentDefinition], + ) -> None: + self.environments = environments + self.default_environment = default_environment diff --git a/src/debputy/manifest_parser/declarative_parser.py b/src/debputy/manifest_parser/declarative_parser.py index 6cbbce3..2c350a0 100644 --- a/src/debputy/manifest_parser/declarative_parser.py +++ b/src/debputy/manifest_parser/declarative_parser.py @@ -1,5 +1,6 @@ import collections import dataclasses +import typing from typing import ( Any, Callable, @@ -16,7 +17,6 @@ from typing import ( Mapping, Optional, cast, - is_typeddict, Type, Union, List, @@ -28,13 +28,7 @@ from typing import ( Container, ) -from debputy.manifest_parser.base_types import ( - DebputyParsedContent, - FileSystemMatchRule, - FileSystemExactMatchRule, - DebputyDispatchableType, - TypeMapping, -) +from debputy.manifest_parser.base_types import FileSystemMatchRule from debputy.manifest_parser.exceptions import ( ManifestParseException, ) @@ -43,7 +37,20 @@ from debputy.manifest_parser.mapper_code import ( wrap_into_list, map_each_element, ) +from debputy.manifest_parser.parse_hints import ( + ConditionalRequired, + DebputyParseHint, + TargetAttribute, + ManifestAttribute, + ConflictWithSourceAttribute, + NotPathHint, +) from debputy.manifest_parser.parser_data import ParserContextData +from debputy.manifest_parser.tagging_types import ( + DebputyParsedContent, + DebputyDispatchableType, + TypeMapping, +) from debputy.manifest_parser.util import ( AttributePath, unpack_type, @@ -53,8 +60,6 @@ from debputy.manifest_parser.util import ( from debputy.plugin.api.impl_types import ( DeclarativeInputParser, TD, - _ALL_PACKAGE_TYPES, - resolve_package_type_selectors, ListWrappedDeclarativeInputParser, DispatchingObjectParser, DispatchingTableParser, @@ -64,8 +69,11 @@ from debputy.plugin.api.impl_types import ( ) from debputy.plugin.api.spec import ( ParserDocumentation, - PackageTypeSelector, DebputyIntegrationMode, + StandardParserAttributeDocumentation, + undocumented_attr, + ParserAttributeDocumentation, + reference_documentation, ) from debputy.util import _info, _warn, assume_not_none @@ -478,242 +486,6 @@ class DeclarativeMappingInputParser(DeclarativeInputParser[TD], Generic[TD, SF]) return self._per_attribute_conflicts_cache -class DebputyParseHint: - @classmethod - def target_attribute(cls, target_attribute: str) -> "DebputyParseHint": - """Define this source attribute to have a different target attribute name - - As an example: - - >>> class SourceType(TypedDict): - ... source: Annotated[NotRequired[str], DebputyParseHint.target_attribute("sources")] - ... sources: NotRequired[List[str]] - >>> class TargetType(TypedDict): - ... sources: List[str] - >>> pg = ParserGenerator() - >>> parser = pg.generate_parser(TargetType, source_content=SourceType) - - In this example, the user can provide either `source` or `sources` and the parser will - map them to the `sources` attribute in the `TargetType`. Note this example relies on - the builtin mapping of `str` to `List[str]` to align the types between `source` (from - SourceType) and `sources` (from TargetType). - - The following rules apply: - - * All source attributes that map to the same target attribute will be mutually exclusive - (that is, the user cannot give `source` *and* `sources` as input). - * When the target attribute is required, the source attributes are conditionally - mandatory requiring the user to provide exactly one of them. - * When multiple source attributes point to a single target attribute, none of the source - attributes can be Required. - * The annotation can only be used for the source type specification and the source type - specification must be different from the target type specification. - - The `target_attribute` annotation can be used without having multiple source attributes. This - can be useful if the source attribute name is not valid as a python variable identifier to - rename it to a valid python identifier. - - :param target_attribute: The attribute name in the target content - :return: The annotation. - """ - return TargetAttribute(target_attribute) - - @classmethod - def conflicts_with_source_attributes( - cls, - *conflicting_source_attributes: str, - ) -> "DebputyParseHint": - """Declare a conflict with one or more source attributes - - Example: - - >>> class SourceType(TypedDict): - ... source: Annotated[NotRequired[str], DebputyParseHint.target_attribute("sources")] - ... sources: NotRequired[List[str]] - ... into_dir: NotRequired[str] - ... renamed_to: Annotated[ - ... NotRequired[str], - ... DebputyParseHint.conflicts_with_source_attributes("sources", "into_dir") - ... ] - >>> class TargetType(TypedDict): - ... sources: List[str] - ... into_dir: NotRequired[str] - ... renamed_to: NotRequired[str] - >>> pg = ParserGenerator() - >>> parser = pg.generate_parser(TargetType, source_content=SourceType) - - In this example, if the user was to provide `renamed_to` with `sources` or `into_dir` the parser would report - an error. However, the parser will allow `renamed_to` with `source` as the conflict is considered only for - the input source. That is, it is irrelevant that `sources` and `source´ happens to "map" to the same target - attribute. - - The following rules apply: - * It is not possible for a target attribute to declare conflicts unless the target type spec is reused as - source type spec. - * All attributes involved in a conflict must be NotRequired. If any of the attributes are Required, then - the parser generator will reject the input. - * All attributes listed in the conflict must be valid attributes in the source type spec. - - Note you do not have to specify conflicts between two attributes with the same target attribute name. The - `target_attribute` annotation will handle that for you. - - :param conflicting_source_attributes: All source attributes that cannot be used with this attribute. - :return: The annotation. - """ - if len(conflicting_source_attributes) < 1: - raise ValueError( - "DebputyParseHint.conflicts_with_source_attributes requires at least one attribute as input" - ) - return ConflictWithSourceAttribute(frozenset(conflicting_source_attributes)) - - @classmethod - def required_when_single_binary( - cls, - *, - package_type: PackageTypeSelector = _ALL_PACKAGE_TYPES, - ) -> "DebputyParseHint": - """Declare a source attribute as required when the source package produces exactly one binary package - - The attribute in question must always be declared as `NotRequired` in the TypedDict and this condition - can only be used for source attributes. - """ - resolved_package_types = resolve_package_type_selectors(package_type) - reason = "The field is required for source packages producing exactly one binary package" - if resolved_package_types != _ALL_PACKAGE_TYPES: - types = ", ".join(sorted(resolved_package_types)) - reason += f" of type {types}" - return ConditionalRequired( - reason, - lambda c: len( - [ - p - for p in c.binary_packages.values() - if p.package_type in package_type - ] - ) - == 1, - ) - return ConditionalRequired( - reason, - lambda c: c.is_single_binary_package, - ) - - @classmethod - def required_when_multi_binary( - cls, - *, - package_type: PackageTypeSelector = _ALL_PACKAGE_TYPES, - ) -> "DebputyParseHint": - """Declare a source attribute as required when the source package produces two or more binary package - - The attribute in question must always be declared as `NotRequired` in the TypedDict and this condition - can only be used for source attributes. - """ - resolved_package_types = resolve_package_type_selectors(package_type) - reason = "The field is required for source packages producing two or more binary packages" - if resolved_package_types != _ALL_PACKAGE_TYPES: - types = ", ".join(sorted(resolved_package_types)) - reason = ( - "The field is required for source packages producing not producing exactly one binary packages" - f" of type {types}" - ) - return ConditionalRequired( - reason, - lambda c: len( - [ - p - for p in c.binary_packages.values() - if p.package_type in package_type - ] - ) - != 1, - ) - return ConditionalRequired( - reason, - lambda c: not c.is_single_binary_package, - ) - - @classmethod - def manifest_attribute(cls, attribute: str) -> "DebputyParseHint": - """Declare what the attribute name (as written in the manifest) should be - - By default, debputy will do an attribute normalizing that will take valid python identifiers such - as `dest_dir` and remap it to the manifest variant (such as `dest-dir`) automatically. If you have - a special case, where this built-in normalization is insufficient or the python name is considerably - different from what the user would write in the manifest, you can use this parse hint to set the - name that the user would have to write in the manifest for this attribute. - - >>> class SourceType(TypedDict): - ... source: List[FileSystemMatchRule] - ... # Use "as" in the manifest because "as_" was not pretty enough - ... install_as: Annotated[NotRequired[FileSystemExactMatchRule], DebputyParseHint.manifest_attribute("as")] - - In this example, we use the parse hint to use "as" as the name in the manifest, because we cannot - use "as" a valid python identifier (it is a keyword). While debputy would map `as_` to `as` for us, - we have chosen to use `install_as` as a python identifier. - """ - return ManifestAttribute(attribute) - - @classmethod - def not_path_error_hint(cls) -> "DebputyParseHint": - """Mark this attribute as not a "path hint" when it comes to reporting errors - - By default, `debputy` will pick up attributes that uses path names (FileSystemMatchRule) as - candidates for parse error hints (the little "<Search for: VALUE>" in error messages). - - Most rules only have one active path-based attribute and paths tends to be unique enough - that it helps people spot the issue faster. However, in rare cases, you can have multiple - attributes that fit the bill. In this case, this hint can be used to "hide" the suboptimal - choice. As an example: - - >>> class SourceType(TypedDict): - ... source: List[FileSystemMatchRule] - ... install_as: Annotated[NotRequired[FileSystemExactMatchRule], DebputyParseHint.not_path_error_hint()] - - In this case, without the hint, `debputy` might pick up `install_as` as the attribute to - use as hint for error reporting. However, here we have decided that we never want `install_as` - leaving `source` as the only option. - - Generally, this type hint must be placed on the **source** format. Any source attribute matching - the parsed format will be ignored. - - Mind the asymmetry: The annotation is placed in the **source** format while `debputy` looks at - the type of the target attribute to determine if it counts as path. - """ - return NOT_PATH_HINT - - -@dataclasses.dataclass(frozen=True, slots=True) -class TargetAttribute(DebputyParseHint): - attribute: str - - -@dataclasses.dataclass(frozen=True, slots=True) -class ConflictWithSourceAttribute(DebputyParseHint): - conflicting_attributes: FrozenSet[str] - - -@dataclasses.dataclass(frozen=True, slots=True) -class ConditionalRequired(DebputyParseHint): - reason: str - condition: Callable[["ParserContextData"], bool] - - def condition_applies(self, context: "ParserContextData") -> bool: - return self.condition(context) - - -@dataclasses.dataclass(frozen=True, slots=True) -class ManifestAttribute(DebputyParseHint): - attribute: str - - -class NotPathHint(DebputyParseHint): - pass - - -NOT_PATH_HINT = NotPathHint() - - def _is_path_attribute_candidate( source_attribute: AttributeDescription, target_attribute: AttributeDescription ) -> bool: @@ -730,6 +502,16 @@ def _is_path_attribute_candidate( return isinstance(match_type, type) and issubclass(match_type, FileSystemMatchRule) +if typing.is_typeddict(DebputyParsedContent): + is_typeddict = typing.is_typeddict +else: + + def is_typeddict(t: Any) -> bool: + if typing.is_typeddict(t): + return True + return isinstance(t, type) and issubclass(t, DebputyParsedContent) + + class ParserGenerator: def __init__(self) -> None: self._registered_types: Dict[Any, TypeMapping[Any, Any]] = {} @@ -811,6 +593,9 @@ class ParserGenerator: expected_debputy_integration_mode: Optional[ Container[DebputyIntegrationMode] ] = None, + automatic_docs: Optional[ + Mapping[Type[Any], Sequence[StandardParserAttributeDocumentation]] + ] = None, ) -> DeclarativeInputParser[TD]: """Derive a parser from a TypedDict @@ -978,7 +763,7 @@ class ParserGenerator: f"Unsupported parsed_content descriptor: {parsed_content.__qualname__}." ' Only "TypedDict"-based types and a subset of "DebputyDispatchableType" are supported.' ) - if is_list_wrapped: + if is_list_wrapped and source_content is not None: if get_origin(source_content) != list: raise ValueError( "If the parsed_content is a List type, then source_format must be a List type as well." @@ -1133,10 +918,15 @@ class ParserGenerator: parsed_alt_form.type_validator.combine_mapper(bridge_mapper) ) - _verify_inline_reference_documentation( - source_content_attributes, - inline_reference_documentation, - parsed_alt_form is not None, + inline_reference_documentation = ( + _verify_and_auto_correct_inline_reference_documentation( + parsed_content, + source_typed_dict, + source_content_attributes, + inline_reference_documentation, + parsed_alt_form is not None, + automatic_docs, + ) ) if non_mapping_source_only: parser = DeclarativeNonMappingInputParser( @@ -1700,45 +1490,133 @@ class ParserGenerator: return orig_td -def _verify_inline_reference_documentation( +def _sort_key(attr: StandardParserAttributeDocumentation) -> Any: + key = next(iter(attr.attributes)) + return attr.sort_category, key + + +def _apply_std_docs( + std_doc_table: Optional[ + Mapping[Type[Any], Sequence[StandardParserAttributeDocumentation]] + ], + source_format_typed_dict: Type[Any], + attribute_docs: Optional[Sequence[ParserAttributeDocumentation]], +) -> Optional[Sequence[ParserAttributeDocumentation]]: + if std_doc_table is None or not std_doc_table: + return attribute_docs + + has_docs_for = set() + if attribute_docs: + for attribute_doc in attribute_docs: + has_docs_for.update(attribute_doc.attributes) + + base_seen = set() + std_docs_used = [] + + remaining_bases = set(getattr(source_format_typed_dict, "__orig_bases__", [])) + base_seen.update(remaining_bases) + while remaining_bases: + base = remaining_bases.pop() + new_bases_to_check = { + x for x in getattr(base, "__orig_bases__", []) if x not in base_seen + } + remaining_bases.update(new_bases_to_check) + base_seen.update(new_bases_to_check) + std_docs = std_doc_table.get(base) + if std_docs: + for std_doc in std_docs: + if any(a in has_docs_for for a in std_doc.attributes): + # If there is any overlap, do not add the docs + continue + has_docs_for.update(std_doc.attributes) + std_docs_used.append(std_doc) + + if not std_docs_used: + return attribute_docs + docs = sorted(std_docs_used, key=_sort_key) + if attribute_docs: + # Plugin provided attributes first + c = list(attribute_docs) + c.extend(docs) + docs = c + return tuple(docs) + + +def _verify_and_auto_correct_inline_reference_documentation( + parsed_content: Type[TD], + source_typed_dict: Type[Any], source_content_attributes: Mapping[str, AttributeDescription], inline_reference_documentation: Optional[ParserDocumentation], has_alt_form: bool, -) -> None: - if inline_reference_documentation is None: - return - attribute_doc = inline_reference_documentation.attribute_doc - if attribute_doc: + automatic_docs: Optional[ + Mapping[Type[Any], Sequence[StandardParserAttributeDocumentation]] + ] = None, +) -> Optional[ParserDocumentation]: + orig_attribute_docs = ( + inline_reference_documentation.attribute_doc + if inline_reference_documentation + else None + ) + attribute_docs = _apply_std_docs( + automatic_docs, + source_typed_dict, + orig_attribute_docs, + ) + if inline_reference_documentation is None and attribute_docs is None: + return None + changes = {} + if attribute_docs: seen = set() - for attr_doc in attribute_doc: + had_any_custom_docs = False + for attr_doc in attribute_docs: + if not isinstance(attr_doc, StandardParserAttributeDocumentation): + had_any_custom_docs = True for attr_name in attr_doc.attributes: attr = source_content_attributes.get(attr_name) if attr is None: raise ValueError( - f'The inline_reference_documentation references an attribute "{attr_name}", which does not' - f" exist in the source format." + f"The inline_reference_documentation for the source format of {parsed_content.__qualname__}" + f' references an attribute "{attr_name}", which does not exist in the source format.' ) if attr_name in seen: raise ValueError( - f'The inline_reference_documentation has documentation for "{attr_name}" twice,' - f" which is not supported. Please document it at most once" + f"The inline_reference_documentation for the source format of {parsed_content.__qualname__}" + f' has documentation for "{attr_name}" twice, which is not supported.' + f" Please document it at most once" ) seen.add(attr_name) - undocumented = source_content_attributes.keys() - seen if undocumented: - undocumented_attrs = ", ".join(undocumented) - raise ValueError( - "The following attributes were not documented. If this is deliberate, then please" - ' declare each them as undocumented (via undocumented_attr("foo")):' - f" {undocumented_attrs}" - ) + if had_any_custom_docs: + undocumented_attrs = ", ".join(undocumented) + raise ValueError( + f"The following attributes were not documented for the source format of" + f" {parsed_content.__qualname__}. If this is deliberate, then please" + ' declare each them as undocumented (via undocumented_attr("foo")):' + f" {undocumented_attrs}" + ) + combined_docs = list(attribute_docs) + combined_docs.extend(undocumented_attr(a) for a in sorted(undocumented)) + attribute_docs = combined_docs + + if attribute_docs and orig_attribute_docs != attribute_docs: + assert attribute_docs is not None + changes["attribute_doc"] = tuple(attribute_docs) - if inline_reference_documentation.alt_parser_description and not has_alt_form: + if ( + inline_reference_documentation is not None + and inline_reference_documentation.alt_parser_description + and not has_alt_form + ): raise ValueError( "The inline_reference_documentation had documentation for an non-mapping format," " but the source format does not have a non-mapping format." ) + if changes: + if inline_reference_documentation is None: + inline_reference_documentation = reference_documentation() + return inline_reference_documentation.replace(**changes) + return inline_reference_documentation def _check_conflicts( diff --git a/src/debputy/manifest_parser/exceptions.py b/src/debputy/manifest_parser/exceptions.py index 671ec1b..f058458 100644 --- a/src/debputy/manifest_parser/exceptions.py +++ b/src/debputy/manifest_parser/exceptions.py @@ -1,9 +1,17 @@ from debputy.exceptions import DebputyRuntimeError -class ManifestParseException(DebputyRuntimeError): +class ManifestException(DebputyRuntimeError): + pass + + +class ManifestParseException(ManifestException): pass class ManifestTypeException(ManifestParseException): pass + + +class ManifestInvalidUserDataException(ManifestException): + pass diff --git a/src/debputy/manifest_parser/mapper_code.py b/src/debputy/manifest_parser/mapper_code.py index d7a08c3..f206af9 100644 --- a/src/debputy/manifest_parser/mapper_code.py +++ b/src/debputy/manifest_parser/mapper_code.py @@ -4,22 +4,25 @@ from typing import ( Union, List, Callable, + TYPE_CHECKING, ) from debputy.manifest_parser.exceptions import ManifestTypeException -from debputy.manifest_parser.parser_data import ParserContextData -from debputy.manifest_parser.util import AttributePath from debputy.packages import BinaryPackage from debputy.util import assume_not_none +if TYPE_CHECKING: + from debputy.manifest_parser.util import AttributePath + from debputy.manifest_parser.parser_data import ParserContextData + S = TypeVar("S") T = TypeVar("T") def type_mapper_str2package( raw_package_name: str, - ap: AttributePath, - opc: Optional[ParserContextData], + ap: "AttributePath", + opc: Optional["ParserContextData"], ) -> BinaryPackage: pc = assume_not_none(opc) if "{{" in raw_package_name: @@ -50,7 +53,7 @@ def type_mapper_str2package( def wrap_into_list( x: T, - _ap: AttributePath, + _ap: "AttributePath", _pc: Optional["ParserContextData"], ) -> List[T]: return [x] @@ -58,18 +61,18 @@ def wrap_into_list( def normalize_into_list( x: Union[T, List[T]], - _ap: AttributePath, + _ap: "AttributePath", _pc: Optional["ParserContextData"], ) -> List[T]: return x if isinstance(x, list) else [x] def map_each_element( - mapper: Callable[[S, AttributePath, Optional["ParserContextData"]], T], -) -> Callable[[List[S], AttributePath, Optional["ParserContextData"]], List[T]]: + mapper: Callable[[S, "AttributePath", Optional["ParserContextData"]], T], +) -> Callable[[List[S], "AttributePath", Optional["ParserContextData"]], List[T]]: def _generated_mapper( xs: List[S], - ap: AttributePath, + ap: "AttributePath", pc: Optional["ParserContextData"], ) -> List[T]: return [mapper(s, ap[i], pc) for i, s in enumerate(xs)] diff --git a/src/debputy/manifest_parser/parse_hints.py b/src/debputy/manifest_parser/parse_hints.py new file mode 100644 index 0000000..30b8aca --- /dev/null +++ b/src/debputy/manifest_parser/parse_hints.py @@ -0,0 +1,259 @@ +import dataclasses +from typing import ( + NotRequired, + TypedDict, + TYPE_CHECKING, + Callable, + FrozenSet, + Annotated, + List, +) + +from debputy.manifest_parser.util import ( + resolve_package_type_selectors, + _ALL_PACKAGE_TYPES, +) +from debputy.plugin.api.spec import PackageTypeSelector + +if TYPE_CHECKING: + from debputy.manifest_parser.parser_data import ParserContextData + + +class DebputyParseHint: + @classmethod + def target_attribute(cls, target_attribute: str) -> "DebputyParseHint": + """Define this source attribute to have a different target attribute name + + As an example: + + >>> from debputy.manifest_parser.declarative_parser import ParserGenerator + >>> class SourceType(TypedDict): + ... source: Annotated[NotRequired[str], DebputyParseHint.target_attribute("sources")] + ... sources: NotRequired[List[str]] + >>> class TargetType(TypedDict): + ... sources: List[str] + >>> pg = ParserGenerator() + >>> parser = pg.generate_parser(TargetType, source_content=SourceType) + + In this example, the user can provide either `source` or `sources` and the parser will + map them to the `sources` attribute in the `TargetType`. Note this example relies on + the builtin mapping of `str` to `List[str]` to align the types between `source` (from + SourceType) and `sources` (from TargetType). + + The following rules apply: + + * All source attributes that map to the same target attribute will be mutually exclusive + (that is, the user cannot give `source` *and* `sources` as input). + * When the target attribute is required, the source attributes are conditionally + mandatory requiring the user to provide exactly one of them. + * When multiple source attributes point to a single target attribute, none of the source + attributes can be Required. + * The annotation can only be used for the source type specification and the source type + specification must be different from the target type specification. + + The `target_attribute` annotation can be used without having multiple source attributes. This + can be useful if the source attribute name is not valid as a python variable identifier to + rename it to a valid python identifier. + + :param target_attribute: The attribute name in the target content + :return: The annotation. + """ + return TargetAttribute(target_attribute) + + @classmethod + def conflicts_with_source_attributes( + cls, + *conflicting_source_attributes: str, + ) -> "DebputyParseHint": + """Declare a conflict with one or more source attributes + + Example: + + >>> from debputy.manifest_parser.declarative_parser import ParserGenerator + >>> class SourceType(TypedDict): + ... source: Annotated[NotRequired[str], DebputyParseHint.target_attribute("sources")] + ... sources: NotRequired[List[str]] + ... into_dir: NotRequired[str] + ... renamed_to: Annotated[ + ... NotRequired[str], + ... DebputyParseHint.conflicts_with_source_attributes("sources", "into_dir") + ... ] + >>> class TargetType(TypedDict): + ... sources: List[str] + ... into_dir: NotRequired[str] + ... renamed_to: NotRequired[str] + >>> pg = ParserGenerator() + >>> parser = pg.generate_parser(TargetType, source_content=SourceType) + + In this example, if the user was to provide `renamed_to` with `sources` or `into_dir` the parser would report + an error. However, the parser will allow `renamed_to` with `source` as the conflict is considered only for + the input source. That is, it is irrelevant that `sources` and `source´ happens to "map" to the same target + attribute. + + The following rules apply: + * It is not possible for a target attribute to declare conflicts unless the target type spec is reused as + source type spec. + * All attributes involved in a conflict must be NotRequired. If any of the attributes are Required, then + the parser generator will reject the input. + * All attributes listed in the conflict must be valid attributes in the source type spec. + + Note you do not have to specify conflicts between two attributes with the same target attribute name. The + `target_attribute` annotation will handle that for you. + + :param conflicting_source_attributes: All source attributes that cannot be used with this attribute. + :return: The annotation. + """ + if len(conflicting_source_attributes) < 1: + raise ValueError( + "DebputyParseHint.conflicts_with_source_attributes requires at least one attribute as input" + ) + return ConflictWithSourceAttribute(frozenset(conflicting_source_attributes)) + + @classmethod + def required_when_single_binary( + cls, + *, + package_type: PackageTypeSelector = _ALL_PACKAGE_TYPES, + ) -> "DebputyParseHint": + """Declare a source attribute as required when the source package produces exactly one binary package + + The attribute in question must always be declared as `NotRequired` in the TypedDict and this condition + can only be used for source attributes. + """ + resolved_package_types = resolve_package_type_selectors(package_type) + reason = "The field is required for source packages producing exactly one binary package" + if resolved_package_types != _ALL_PACKAGE_TYPES: + types = ", ".join(sorted(resolved_package_types)) + reason += f" of type {types}" + return ConditionalRequired( + reason, + lambda c: len( + [ + p + for p in c.binary_packages.values() + if p.package_type in package_type + ] + ) + == 1, + ) + return ConditionalRequired( + reason, + lambda c: c.is_single_binary_package, + ) + + @classmethod + def required_when_multi_binary( + cls, + *, + package_type: PackageTypeSelector = _ALL_PACKAGE_TYPES, + ) -> "DebputyParseHint": + """Declare a source attribute as required when the source package produces two or more binary package + + The attribute in question must always be declared as `NotRequired` in the TypedDict and this condition + can only be used for source attributes. + """ + resolved_package_types = resolve_package_type_selectors(package_type) + reason = "The field is required for source packages producing two or more binary packages" + if resolved_package_types != _ALL_PACKAGE_TYPES: + types = ", ".join(sorted(resolved_package_types)) + reason = ( + "The field is required for source packages producing not producing exactly one binary packages" + f" of type {types}" + ) + return ConditionalRequired( + reason, + lambda c: len( + [ + p + for p in c.binary_packages.values() + if p.package_type in package_type + ] + ) + != 1, + ) + return ConditionalRequired( + reason, + lambda c: not c.is_single_binary_package, + ) + + @classmethod + def manifest_attribute(cls, attribute: str) -> "DebputyParseHint": + """Declare what the attribute name (as written in the manifest) should be + + By default, debputy will do an attribute normalizing that will take valid python identifiers such + as `dest_dir` and remap it to the manifest variant (such as `dest-dir`) automatically. If you have + a special case, where this built-in normalization is insufficient or the python name is considerably + different from what the user would write in the manifest, you can use this parse hint to set the + name that the user would have to write in the manifest for this attribute. + + >>> from debputy.manifest_parser.base_types import FileSystemMatchRule, FileSystemExactMatchRule + >>> class SourceType(TypedDict): + ... source: List[FileSystemMatchRule] + ... # Use "as" in the manifest because "as_" was not pretty enough + ... install_as: Annotated[NotRequired[FileSystemExactMatchRule], DebputyParseHint.manifest_attribute("as")] + + In this example, we use the parse hint to use "as" as the name in the manifest, because we cannot + use "as" a valid python identifier (it is a keyword). While debputy would map `as_` to `as` for us, + we have chosen to use `install_as` as a python identifier. + """ + return ManifestAttribute(attribute) + + @classmethod + def not_path_error_hint(cls) -> "DebputyParseHint": + """Mark this attribute as not a "path hint" when it comes to reporting errors + + By default, `debputy` will pick up attributes that uses path names (FileSystemMatchRule) as + candidates for parse error hints (the little "<Search for: VALUE>" in error messages). + + Most rules only have one active path-based attribute and paths tends to be unique enough + that it helps people spot the issue faster. However, in rare cases, you can have multiple + attributes that fit the bill. In this case, this hint can be used to "hide" the suboptimal + choice. As an example: + + >>> from debputy.manifest_parser.base_types import FileSystemMatchRule, FileSystemExactMatchRule + >>> class SourceType(TypedDict): + ... source: List[FileSystemMatchRule] + ... install_as: Annotated[NotRequired[FileSystemExactMatchRule], DebputyParseHint.not_path_error_hint()] + + In this case, without the hint, `debputy` might pick up `install_as` as the attribute to + use as hint for error reporting. However, here we have decided that we never want `install_as` + leaving `source` as the only option. + + Generally, this type hint must be placed on the **source** format. Any source attribute matching + the parsed format will be ignored. + + Mind the asymmetry: The annotation is placed in the **source** format while `debputy` looks at + the type of the target attribute to determine if it counts as path. + """ + return NOT_PATH_HINT + + +@dataclasses.dataclass(frozen=True, slots=True) +class TargetAttribute(DebputyParseHint): + attribute: str + + +@dataclasses.dataclass(frozen=True, slots=True) +class ConflictWithSourceAttribute(DebputyParseHint): + conflicting_attributes: FrozenSet[str] + + +@dataclasses.dataclass(frozen=True, slots=True) +class ConditionalRequired(DebputyParseHint): + reason: str + condition: Callable[["ParserContextData"], bool] + + def condition_applies(self, context: "ParserContextData") -> bool: + return self.condition(context) + + +@dataclasses.dataclass(frozen=True, slots=True) +class ManifestAttribute(DebputyParseHint): + attribute: str + + +class NotPathHint(DebputyParseHint): + pass + + +NOT_PATH_HINT = NotPathHint() diff --git a/src/debputy/manifest_parser/parser_data.py b/src/debputy/manifest_parser/parser_data.py index 30d9ce0..acc5c67 100644 --- a/src/debputy/manifest_parser/parser_data.py +++ b/src/debputy/manifest_parser/parser_data.py @@ -11,12 +11,15 @@ from debian.debian_support import DpkgArchTable from debputy._deb_options_profiles import DebBuildOptionsAndProfiles from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable +from debputy.manifest_parser.base_types import BuildEnvironmentDefinition from debputy.manifest_parser.exceptions import ManifestParseException from debputy.manifest_parser.util import AttributePath -from debputy.packages import BinaryPackage -from debputy.plugin.api.impl_types import ( +from debputy.manifest_parser.util import ( _ALL_PACKAGE_TYPES, resolve_package_type_selectors, +) +from debputy.packages import BinaryPackage +from debputy.plugin.api.impl_types import ( TP, DispatchingTableParser, TTP, @@ -101,7 +104,7 @@ class ParserContextData: raise NotImplementedError @property - def build_env(self) -> DebBuildOptionsAndProfiles: + def deb_options_and_profiles(self) -> DebBuildOptionsAndProfiles: raise NotImplementedError @contextlib.contextmanager @@ -129,3 +132,8 @@ class ParserContextData: @property def debputy_integration_mode(self) -> DebputyIntegrationMode: raise NotImplementedError + + def resolve_build_environment( + self, name: Optional[str], attribute_path: AttributePath + ) -> BuildEnvironmentDefinition: + raise NotImplementedError diff --git a/src/debputy/manifest_parser/tagging_types.py b/src/debputy/manifest_parser/tagging_types.py new file mode 100644 index 0000000..83030f0 --- /dev/null +++ b/src/debputy/manifest_parser/tagging_types.py @@ -0,0 +1,36 @@ +import dataclasses +from typing import ( + TypedDict, + TYPE_CHECKING, + Generic, + Type, + Callable, + Optional, +) + +from debputy.plugin.plugin_state import current_debputy_plugin_required +from debputy.types import S +from debputy.util import T + +if TYPE_CHECKING: + from debputy.manifest_parser.parser_data import ParserContextData + + from debputy.manifest_parser.util import AttributePath + + +class DebputyParsedContent(TypedDict): + pass + + +class DebputyDispatchableType: + __slots__ = ("_debputy_plugin",) + + def __init__(self) -> None: + self._debputy_plugin = current_debputy_plugin_required() + + +@dataclasses.dataclass +class TypeMapping(Generic[S, T]): + target_type: Type[T] + source_type: Type[S] + mapper: Callable[[S, "AttributePath", Optional["ParserContextData"]], T] diff --git a/src/debputy/manifest_parser/util.py b/src/debputy/manifest_parser/util.py index bcaa617..4e8fd7c 100644 --- a/src/debputy/manifest_parser/util.py +++ b/src/debputy/manifest_parser/util.py @@ -15,6 +15,8 @@ from typing import ( Iterable, Container, Literal, + FrozenSet, + cast, ) from debputy.yaml.compat import CommentedBase @@ -23,7 +25,7 @@ from debputy.manifest_parser.exceptions import ManifestParseException if TYPE_CHECKING: from debputy.manifest_parser.parser_data import ParserContextData - from debputy.plugin.api.spec import DebputyIntegrationMode + from debputy.plugin.api.spec import DebputyIntegrationMode, PackageTypeSelector MP = TypeVar("MP", bound="DebputyParseHint") @@ -34,6 +36,25 @@ AttributePathAliasMapping = Mapping[ LineReportKind = Literal["key", "value", "container"] +_PACKAGE_TYPE_DEB_ONLY = frozenset(["deb"]) +_ALL_PACKAGE_TYPES = frozenset(["deb", "udeb"]) + + +def resolve_package_type_selectors( + package_type: "PackageTypeSelector", +) -> FrozenSet[str]: + if package_type is _ALL_PACKAGE_TYPES or package_type is _PACKAGE_TYPE_DEB_ONLY: + return cast("FrozenSet[str]", package_type) + if isinstance(package_type, str): + return ( + _PACKAGE_TYPE_DEB_ONLY + if package_type == "deb" + else frozenset([package_type]) + ) + else: + return frozenset(package_type) + + class AttributePath: __slots__ = ("parent", "container", "name", "alias_mapping", "path_hint") @@ -94,7 +115,7 @@ class AttributePath: lc_data = lc.value(key) else: lc_data = lc.item(key) - except (AttributeError, RuntimeError, LookupError): + except (AttributeError, RuntimeError, LookupError, TypeError): lc_data = None else: lc_data = None @@ -161,7 +182,7 @@ class AttributePath: if container is not None: try: child_container = self.container[item] - except (AttributeError, RuntimeError, LookupError): + except (AttributeError, RuntimeError, LookupError, TypeError): child_container = None else: child_container = None diff --git a/src/debputy/package_build/assemble_deb.py b/src/debputy/package_build/assemble_deb.py index fd92f37..21fe0d6 100644 --- a/src/debputy/package_build/assemble_deb.py +++ b/src/debputy/package_build/assemble_deb.py @@ -95,8 +95,8 @@ def assemble_debs( package_metadata_context = dctrl_data.package_metadata_context if ( dbgsym_package_name in package_data_table - or "noautodbgsym" in manifest.build_env.deb_build_options - or "noddebs" in manifest.build_env.deb_build_options + or "noautodbgsym" in manifest.deb_options_and_profiles.deb_build_options + or "noddebs" in manifest.deb_options_and_profiles.deb_build_options ): # Discard the dbgsym part if it conflicts with a real package, or # we were asked not to build it. diff --git a/src/debputy/packager_provided_files.py b/src/debputy/packager_provided_files.py index a35beec..5657ad2 100644 --- a/src/debputy/packager_provided_files.py +++ b/src/debputy/packager_provided_files.py @@ -35,6 +35,8 @@ _KNOWN_NON_TYPO_EXTENSIONS = frozenset( "bash", "pl", "py", + # Fairly common image format in older packages + "xpm", } ) diff --git a/src/debputy/packages.py b/src/debputy/packages.py index 3a6ee16..0a3876a 100644 --- a/src/debputy/packages.py +++ b/src/debputy/packages.py @@ -44,15 +44,15 @@ class DctrlParser: DpkgArchitectureBuildProcessValuesTable ] = None, dpkg_arch_query_table: Optional[DpkgArchTable] = None, - build_env: Optional[DebBuildOptionsAndProfiles] = None, + deb_options_and_profiles: Optional[DebBuildOptionsAndProfiles] = None, ignore_errors: bool = False, ) -> None: if dpkg_architecture_variables is None: dpkg_architecture_variables = dpkg_architecture_table() if dpkg_arch_query_table is None: dpkg_arch_query_table = DpkgArchTable.load_arch_table() - if build_env is None: - build_env = DebBuildOptionsAndProfiles.instance() + if deb_options_and_profiles is None: + deb_options_and_profiles = DebBuildOptionsAndProfiles.instance() # If no selection option is set, then all packages are acted on (except the # excluded ones) @@ -66,7 +66,7 @@ class DctrlParser: self.select_arch_any = select_arch_any self.dpkg_architecture_variables = dpkg_architecture_variables self.dpkg_arch_query_table = dpkg_arch_query_table - self.build_env = build_env + self.deb_options_and_profiles = deb_options_and_profiles self.ignore_errors = ignore_errors @overload @@ -138,7 +138,7 @@ class DctrlParser: self.select_arch_any, self.dpkg_architecture_variables, self.dpkg_arch_query_table, - self.build_env, + self.deb_options_and_profiles, i, ) ) diff --git a/src/debputy/path_matcher.py b/src/debputy/path_matcher.py index 2917b14..a7b8356 100644 --- a/src/debputy/path_matcher.py +++ b/src/debputy/path_matcher.py @@ -161,7 +161,7 @@ class MatchRule: _error( f'The pattern "{path_or_glob}" (defined in {definition_source}) looks like it contains a' f' brace expansion (such as "{{a,b}}" or "{{a..b}}"). Brace expansions are not supported.' - " If you wanted to match the literal path a brace in it, please use a substitution to insert" + " If you wanted to match the literal path with a brace in it, please use a substitution to insert" f' the opening brace. As an example: "{replacement}"' ) diff --git a/src/debputy/plugin/api/feature_set.py b/src/debputy/plugin/api/feature_set.py index a56f37b..30d79be 100644 --- a/src/debputy/plugin/api/feature_set.py +++ b/src/debputy/plugin/api/feature_set.py @@ -1,29 +1,24 @@ import dataclasses -import textwrap -from typing import Dict, List, Tuple, Sequence, Any +from typing import Dict, List, Tuple, Sequence, Any, Optional, Type -from debputy import DEBPUTY_DOC_ROOT_DIR from debputy.manifest_parser.declarative_parser import ParserGenerator -from debputy.plugin.api import reference_documentation from debputy.plugin.api.impl_types import ( DebputyPluginMetadata, PackagerProvidedFileClassSpec, MetadataOrMaintscriptDetector, - TTP, - DispatchingTableParser, - TP, - SUPPORTED_DISPATCHABLE_TABLE_PARSERS, - DispatchingObjectParser, - SUPPORTED_DISPATCHABLE_OBJECT_PARSERS, PluginProvidedManifestVariable, PluginProvidedPackageProcessor, PluginProvidedDiscardRule, ServiceManagerDetails, PluginProvidedKnownPackagingFile, PluginProvidedTypeMapping, - OPARSER_PACKAGES, - OPARSER_PACKAGES_ROOT, + PluginProvidedBuildSystemAutoDetection, +) +from debputy.plugin.api.parser_tables import ( + SUPPORTED_DISPATCHABLE_OBJECT_PARSERS, + SUPPORTED_DISPATCHABLE_TABLE_PARSERS, ) +from debputy.plugin.debputy.to_be_api_types import BuildSystemRule def _initialize_parser_generator() -> ParserGenerator: @@ -70,6 +65,9 @@ class PluginProvidedFeatureSet: manifest_parser_generator: ParserGenerator = dataclasses.field( default_factory=_initialize_parser_generator ) + auto_detectable_build_systems: Dict[ + Type[BuildSystemRule], PluginProvidedBuildSystemAutoDetection + ] = dataclasses.field(default_factory=dict) def package_processors_in_order(self) -> Sequence[PluginProvidedPackageProcessor]: order = [] diff --git a/src/debputy/plugin/api/impl.py b/src/debputy/plugin/api/impl.py index b0674fb..c2f03d0 100644 --- a/src/debputy/plugin/api/impl.py +++ b/src/debputy/plugin/api/impl.py @@ -31,7 +31,8 @@ from typing import ( Any, Literal, Container, - get_args, + TYPE_CHECKING, + is_typeddict, ) from debputy import DEBPUTY_DOC_ROOT_DIR @@ -43,16 +44,18 @@ from debputy.exceptions import ( PluginInitializationError, PluginAPIViolationError, PluginNotFoundError, + PluginIncorrectRegistrationError, ) from debputy.maintscript_snippet import ( STD_CONTROL_SCRIPTS, MaintscriptSnippetContainer, MaintscriptSnippet, ) -from debputy.manifest_parser.base_types import TypeMapping from debputy.manifest_parser.exceptions import ManifestParseException from debputy.manifest_parser.parser_data import ParserContextData +from debputy.manifest_parser.tagging_types import TypeMapping from debputy.manifest_parser.util import AttributePath +from debputy.manifest_parser.util import resolve_package_type_selectors from debputy.plugin.api.feature_set import PluginProvidedFeatureSet from debputy.plugin.api.impl_types import ( DebputyPluginMetadata, @@ -70,11 +73,12 @@ from debputy.plugin.api.impl_types import ( AutomaticDiscardRuleExample, PPFFormatParam, ServiceManagerDetails, - resolve_package_type_selectors, KnownPackagingFileInfo, PluginProvidedKnownPackagingFile, InstallPatternDHCompatRule, PluginProvidedTypeMapping, + PluginProvidedBuildSystemAutoDetection, + BSR, ) from debputy.plugin.api.plugin_parser import ( PLUGIN_METADATA_PARSER, @@ -108,6 +112,21 @@ from debputy.plugin.api.spec import ( packager_provided_file_reference_documentation, TypeMappingDocumentation, DebputyIntegrationMode, + reference_documentation, + _DEBPUTY_DISPATCH_METADATA_ATTR_NAME, + BuildSystemManifestRuleMetadata, +) +from debputy.plugin.api.std_docs import _STD_ATTR_DOCS +from debputy.plugin.debputy.to_be_api_types import ( + BuildSystemRule, + BuildRuleParsedFormat, + BSPF, + debputy_build_system, +) +from debputy.plugin.plugin_state import ( + run_in_context_of_plugin, + run_in_context_of_plugin_wrap_errors, + wrap_plugin_code, ) from debputy.substitution import ( Substitution, @@ -123,6 +142,9 @@ from debputy.util import ( _warn, ) +if TYPE_CHECKING: + from debputy.highlevel_manifest import HighLevelManifest + PLUGIN_TEST_SUFFIX = re.compile(r"_(?:t|test|check)(?:_([a-z0-9_]+))?[.]py$") @@ -362,7 +384,7 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): all_detectors[self._plugin_name].append( MetadataOrMaintscriptDetector( detector_id=auto_detector_id, - detector=auto_detector, + detector=wrap_plugin_code(self._plugin_name, auto_detector), plugin_metadata=self._plugin_metadata, applies_to_package_types=package_types, enabled=True, @@ -575,7 +597,7 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): package_processors[processor_key] = PluginProvidedPackageProcessor( processor_id, resolve_package_type_selectors(package_type), - processor, + wrap_plugin_code(self._plugin_name, processor), frozenset(dependencies), self._plugin_metadata, ) @@ -704,8 +726,8 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): ) service_managers[service_manager] = ServiceManagerDetails( service_manager, - detector, - integrator, + wrap_plugin_code(self._plugin_name, detector), + wrap_plugin_code(self._plugin_name, integrator), self._plugin_metadata, ) @@ -776,7 +798,7 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): dispatching_parser = parser_generator.dispatchable_table_parsers[rule_type] dispatching_parser.register_keyword( rule_name, - handler, + wrap_plugin_code(self._plugin_name, handler), self._plugin_metadata, inline_reference_documentation=inline_reference_documentation, ) @@ -820,6 +842,10 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): ) parent_dispatcher = dispatchable_object_parsers[rule_type] child_dispatcher = dispatchable_object_parsers[object_parser_key] + + if on_end_parse_step is not None: + on_end_parse_step = wrap_plugin_code(self._plugin_name, on_end_parse_step) + parent_dispatcher.register_child_parser( rule_name, child_dispatcher, @@ -838,7 +864,7 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): def pluggable_manifest_rule( self, rule_type: Union[TTP, str], - rule_name: Union[str, List[str]], + rule_name: Union[str, Sequence[str]], parsed_format: Type[PF], handler: DIPHandler, *, @@ -847,8 +873,15 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): expected_debputy_integration_mode: Optional[ Container[DebputyIntegrationMode] ] = None, + apply_standard_attribute_documentation: bool = False, ) -> None: + # When unrestricted this, consider which types will be unrestricted self._restricted_api() + if apply_standard_attribute_documentation and sys.version_info < (3, 12): + _error( + f"The plugin {self._plugin_metadata.plugin_name} requires python 3.12 due to" + f" its use of apply_standard_attribute_documentation" + ) feature_set = self._feature_set parser_generator = feature_set.manifest_parser_generator if isinstance(rule_type, str): @@ -870,16 +903,22 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): ) dispatching_parser = parser_generator.dispatchable_table_parsers[rule_type] + if apply_standard_attribute_documentation: + docs = _STD_ATTR_DOCS + else: + docs = None + parser = feature_set.manifest_parser_generator.generate_parser( parsed_format, source_content=source_format, inline_reference_documentation=inline_reference_documentation, expected_debputy_integration_mode=expected_debputy_integration_mode, + automatic_docs=docs, ) dispatching_parser.register_parser( rule_name, parser, - handler, + wrap_plugin_code(self._plugin_name, handler), self._plugin_metadata, ) @@ -890,6 +929,108 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): self._unloaders.append(_unload) + def register_build_system( + self, + build_system_definition: type[BSPF], + ) -> None: + self._restricted_api() + if not is_typeddict(build_system_definition): + raise PluginInitializationError( + f"Expected build_system_definition to be a subclass of {BuildRuleParsedFormat.__name__}," + f" but got {build_system_definition.__name__} instead" + ) + metadata = getattr( + build_system_definition, + _DEBPUTY_DISPATCH_METADATA_ATTR_NAME, + None, + ) + if not isinstance(metadata, BuildSystemManifestRuleMetadata): + raise PluginIncorrectRegistrationError( + f"The {build_system_definition.__qualname__} type should have been annotated with" + f" @{debputy_build_system.__name__}." + ) + assert len(metadata.manifest_keywords) == 1 + build_system_impl = metadata.build_system_impl + assert build_system_impl is not None + manifest_keyword = next(iter(metadata.manifest_keywords)) + self.pluggable_manifest_rule( + metadata.dispatched_type, + metadata.manifest_keywords, + build_system_definition, + # pluggable_manifest_rule does the wrapping + metadata.unwrapped_constructor, + source_format=metadata.source_format, + ) + self._auto_detectable_build_system( + manifest_keyword, + build_system_impl, + constructor=wrap_plugin_code( + self._plugin_name, + build_system_impl, + ), + shadowing_build_systems_when_active=metadata.auto_detection_shadow_build_systems, + ) + + def _auto_detectable_build_system( + self, + manifest_keyword: str, + rule_type: type[BSR], + *, + shadowing_build_systems_when_active: FrozenSet[str] = frozenset(), + constructor: Optional[ + Callable[[BuildRuleParsedFormat, AttributePath, "HighLevelManifest"], BSR] + ] = None, + ) -> None: + self._restricted_api() + feature_set = self._feature_set + existing = feature_set.auto_detectable_build_systems.get(rule_type) + if existing is not None: + bs_name = rule_type.__class__.__name__ + if existing.plugin_metadata.plugin_name == self._plugin_name: + message = ( + f"Bug in the plugin {self._plugin_name}: It tried to register the" + f' auto-detection of the build system "{bs_name}" twice.' + ) + else: + message = ( + f"The plugins {existing.plugin_metadata.plugin_name} and {self._plugin_name}" + f' both tried to provide auto-detection of the build system "{bs_name}"' + ) + raise PluginConflictError( + message, existing.plugin_metadata, self._plugin_metadata + ) + + if constructor is None: + + def impl( + attributes: BuildRuleParsedFormat, + attribute_path: AttributePath, + manifest: "HighLevelManifest", + ) -> BSR: + return rule_type(attributes, attribute_path, manifest) + + else: + impl = constructor + + feature_set.auto_detectable_build_systems[rule_type] = ( + PluginProvidedBuildSystemAutoDetection( + manifest_keyword, + rule_type, + wrap_plugin_code(self._plugin_name, rule_type.auto_detect_build_system), + impl, + shadowing_build_systems_when_active, + self._plugin_metadata, + ) + ) + + def _unload() -> None: + try: + del feature_set.auto_detectable_build_systems[rule_type] + except KeyError: + pass + + self._unloaders.append(_unload) + def known_packaging_files( self, packaging_file_details: KnownPackagingFileInfo, @@ -981,6 +1122,7 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): message, existing.plugin_metadata, self._plugin_metadata ) parser_generator = self._feature_set.manifest_parser_generator + # TODO: Wrap the mapper in the plugin context mapped_types[target_type] = PluginProvidedTypeMapping( type_mapping, reference_documentation, self._plugin_metadata ) @@ -1437,6 +1579,10 @@ def load_plugin_features( if plugin_metadata.plugin_name not in unloadable_plugins: raise if debug_mode: + _warn( + f"The optional plugin {plugin_metadata.plugin_name} failed during load. Re-raising due" + f" to --debug/-d." + ) raise try: api.unload_plugin() @@ -1448,11 +1594,6 @@ def load_plugin_features( ) raise e from None else: - if debug_mode: - _warn( - f"The optional plugin {plugin_metadata.plugin_name} failed during load. Re-raising due" - f" to --debug/-d." - ) _warn( f"The optional plugin {plugin_metadata.plugin_name} failed during load. The plugin was" f" deactivated. Use debug mode (--debug) to show the stacktrace (the warning will become an error)" @@ -1635,7 +1776,7 @@ def _resolve_module_initializer( ) sys.modules[module_name] = mod try: - loader.exec_module(mod) + run_in_context_of_plugin(plugin_name, loader.exec_module, mod) except (Exception, GeneratorExit) as e: raise PluginInitializationError( f"Failed to load {plugin_name} (path: {module_fs_path})." @@ -1645,7 +1786,9 @@ def _resolve_module_initializer( if module is None: try: - module = importlib.import_module(module_name) + module = run_in_context_of_plugin( + plugin_name, importlib.import_module, module_name + ) except ModuleNotFoundError as e: if module_fs_path is None: raise PluginMetadataError( @@ -1660,7 +1803,12 @@ def _resolve_module_initializer( f' explicit "module" definition in {json_file_path}.' ) from e - plugin_initializer = getattr(module, plugin_initializer_name) + plugin_initializer = run_in_context_of_plugin_wrap_errors( + plugin_name, + getattr, + module, + plugin_initializer_name, + ) if plugin_initializer is None: raise PluginMetadataError( diff --git a/src/debputy/plugin/api/impl_types.py b/src/debputy/plugin/api/impl_types.py index 1a9bfdf..85beaf8 100644 --- a/src/debputy/plugin/api/impl_types.py +++ b/src/debputy/plugin/api/impl_types.py @@ -1,6 +1,5 @@ import dataclasses import os.path -import textwrap from typing import ( Optional, Callable, @@ -24,22 +23,21 @@ from typing import ( Set, Iterator, Container, + Protocol, ) from weakref import ref -from debputy import DEBPUTY_DOC_ROOT_DIR from debputy.exceptions import ( DebputyFSIsROError, PluginAPIViolationError, PluginConflictError, UnhandledOrUnexpectedErrorFromPluginError, + PluginBaseError, + PluginInitializationError, ) from debputy.filesystem_scan import as_path_def -from debputy.installations import InstallRule -from debputy.maintscript_snippet import DpkgMaintscriptHelperCommand -from debputy.manifest_conditions import ManifestCondition -from debputy.manifest_parser.base_types import DebputyParsedContent, TypeMapping from debputy.manifest_parser.exceptions import ManifestParseException +from debputy.manifest_parser.tagging_types import DebputyParsedContent, TypeMapping from debputy.manifest_parser.util import AttributePath, check_integration_mode from debputy.packages import BinaryPackage from debputy.plugin.api import ( @@ -62,15 +60,16 @@ from debputy.plugin.api.spec import ( TypeMappingDocumentation, DebputyIntegrationMode, ) +from debputy.plugin.plugin_state import ( + run_in_context_of_plugin, +) from debputy.substitution import VariableContext -from debputy.transformation_rules import TransformationRule from debputy.util import _normalize_path, package_cross_check_precheck if TYPE_CHECKING: from debputy.plugin.api.spec import ( ServiceDetector, ServiceIntegrator, - PackageTypeSelector, ) from debputy.manifest_parser.parser_data import ParserContextData from debputy.highlevel_manifest import ( @@ -78,10 +77,10 @@ if TYPE_CHECKING: PackageTransformationDefinition, BinaryPackageData, ) - - -_PACKAGE_TYPE_DEB_ONLY = frozenset(["deb"]) -_ALL_PACKAGE_TYPES = frozenset(["deb", "udeb"]) + from debputy.plugin.debputy.to_be_api_types import ( + BuildSystemRule, + BuildRuleParsedFormat, + ) TD = TypeVar("TD", bound="Union[DebputyParsedContent, List[DebputyParsedContent]]") @@ -89,26 +88,12 @@ PF = TypeVar("PF") SF = TypeVar("SF") TP = TypeVar("TP") TTP = Type[TP] +BSR = TypeVar("BSR", bound="BuildSystemRule") DIPKWHandler = Callable[[str, AttributePath, "ParserContextData"], TP] DIPHandler = Callable[[str, PF, AttributePath, "ParserContextData"], TP] -def resolve_package_type_selectors( - package_type: "PackageTypeSelector", -) -> FrozenSet[str]: - if package_type is _ALL_PACKAGE_TYPES or package_type is _PACKAGE_TYPE_DEB_ONLY: - return cast("FrozenSet[str]", package_type) - if isinstance(package_type, str): - return ( - _PACKAGE_TYPE_DEB_ONLY - if package_type == "deb" - else frozenset([package_type]) - ) - else: - return frozenset(package_type) - - @dataclasses.dataclass(slots=True) class DebputyPluginMetadata: plugin_name: str @@ -143,7 +128,17 @@ class DebputyPluginMetadata: def load_plugin(self) -> None: plugin_loader = self.plugin_loader assert plugin_loader is not None - self.plugin_initializer = plugin_loader() + try: + self.plugin_initializer = run_in_context_of_plugin( + self.plugin_name, + plugin_loader, + ) + except PluginBaseError: + raise + except Exception as e: + raise PluginInitializationError( + f"Initialization of {self.plugin_name} failed due to its initializer raising an exception" + ) from e assert self.plugin_initializer is not None @@ -270,12 +265,10 @@ class MetadataOrMaintscriptDetector: " this stage (file system layout is committed and the attempted changes" " would be lost)." ) from e - except (ChildProcessError, RuntimeError, AttributeError) as e: - nv = f"{self.plugin_metadata.plugin_name}" - raise UnhandledOrUnexpectedErrorFromPluginError( - f"The plugin {nv} threw an unhandled or unexpected exception from its metadata" - f" detector with id {self.detector_id}." - ) from e + except UnhandledOrUnexpectedErrorFromPluginError as e: + e.add_note( + f"The exception was raised by the detector with the ID: {self.detector_id}" + ) class DeclarativeInputParser(Generic[TD]): @@ -605,7 +598,7 @@ class DispatchingObjectParser( ) remaining_valid_attribute_names = ", ".join(remaining_valid_attributes) raise ManifestParseException( - f'The attribute "{first_key}" is not applicable at {attribute_path.path}(with the current set' + f'The attribute "{first_key}" is not applicable at {attribute_path.path} (with the current set' " of plugins). Possible attributes available (and not already used) are:" f" {remaining_valid_attribute_names}.{doc_ref}" ) @@ -615,7 +608,10 @@ class DispatchingObjectParser( if value is None: if isinstance(provided_parser.parser, DispatchingObjectParser): provided_parser.handler( - key, {}, attribute_path[key], parser_context + key, + {}, + attribute_path[key], + parser_context, ) continue value_path = attribute_path[key] @@ -774,64 +770,6 @@ class DeclarativeValuelessKeywordInputParser(DeclarativeInputParser[None]): ) -SUPPORTED_DISPATCHABLE_TABLE_PARSERS = { - InstallRule: "installations", - TransformationRule: "packages.{{PACKAGE}}.transformations", - DpkgMaintscriptHelperCommand: "packages.{{PACKAGE}}.conffile-management", - ManifestCondition: "*.when", -} - -OPARSER_MANIFEST_ROOT = "<ROOT>" -OPARSER_PACKAGES_ROOT = "packages" -OPARSER_PACKAGES = "packages.{{PACKAGE}}" -OPARSER_MANIFEST_DEFINITIONS = "definitions" - -SUPPORTED_DISPATCHABLE_OBJECT_PARSERS = { - OPARSER_MANIFEST_ROOT: reference_documentation( - reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md", - ), - OPARSER_MANIFEST_DEFINITIONS: reference_documentation( - title="Packager provided definitions", - description="Reusable packager provided definitions such as manifest variables.", - reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#packager-provided-definitions", - ), - OPARSER_PACKAGES: reference_documentation( - title="Binary package rules", - description=textwrap.dedent( - """\ - Inside the manifest, the `packages` mapping can be used to define requests for the binary packages - you want `debputy` to produce. Each key inside `packages` must be the name of a binary package - defined in `debian/control`. The value is a dictionary defining which features that `debputy` - should apply to that binary package. An example could be: - - packages: - foo: - transformations: - - create-symlink: - path: usr/share/foo/my-first-symlink - target: /usr/share/bar/symlink-target - - create-symlink: - path: usr/lib/{{DEB_HOST_MULTIARCH}}/my-second-symlink - target: /usr/lib/{{DEB_HOST_MULTIARCH}}/baz/symlink-target - bar: - transformations: - - create-directories: - - some/empty/directory.d - - another/empty/integration-point.d - - create-directories: - path: a/third-empty/directory.d - owner: www-data - group: www-data - - In this case, `debputy` will create some symlinks inside the `foo` package and some directories for - the `bar` package. The following subsections define the keys you can use under each binary package. - """ - ), - reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#binary-package-rules", - ), -} - - @dataclasses.dataclass(slots=True) class PluginProvidedManifestVariable: plugin_metadata: DebputyPluginMetadata @@ -1214,6 +1152,11 @@ class PluginProvidedKnownPackagingFile: plugin_metadata: DebputyPluginMetadata +class BuildSystemAutoDetector(Protocol): + + def __call__(self, source_root: VirtualPath, *args: Any, **kwargs: Any) -> bool: ... + + @dataclasses.dataclass(slots=True, frozen=True) class PluginProvidedTypeMapping: mapped_type: TypeMapping[Any, Any] @@ -1221,6 +1164,19 @@ class PluginProvidedTypeMapping: plugin_metadata: DebputyPluginMetadata +@dataclasses.dataclass(slots=True, frozen=True) +class PluginProvidedBuildSystemAutoDetection(Generic[BSR]): + manifest_keyword: str + build_system_rule_type: Type[BSR] + detector: BuildSystemAutoDetector + constructor: Callable[ + ["BuildRuleParsedFormat", AttributePath, "HighLevelManifest"], + BSR, + ] + auto_detection_shadow_build_systems: FrozenSet[str] + plugin_metadata: DebputyPluginMetadata + + class PackageDataTable: def __init__(self, package_data_table: Mapping[str, "BinaryPackageData"]) -> None: self._package_data_table = package_data_table diff --git a/src/debputy/plugin/api/parser_tables.py b/src/debputy/plugin/api/parser_tables.py new file mode 100644 index 0000000..37d3e37 --- /dev/null +++ b/src/debputy/plugin/api/parser_tables.py @@ -0,0 +1,67 @@ +import textwrap + +from debputy import DEBPUTY_DOC_ROOT_DIR +from debputy.installations import InstallRule +from debputy.maintscript_snippet import DpkgMaintscriptHelperCommand +from debputy.manifest_conditions import ManifestCondition +from debputy.plugin.api import reference_documentation +from debputy.plugin.debputy.to_be_api_types import BuildRule +from debputy.transformation_rules import TransformationRule + +SUPPORTED_DISPATCHABLE_TABLE_PARSERS = { + InstallRule: "installations", + TransformationRule: "packages.{{PACKAGE}}.transformations", + DpkgMaintscriptHelperCommand: "packages.{{PACKAGE}}.conffile-management", + ManifestCondition: "*.when", + BuildRule: "builds", +} + +OPARSER_MANIFEST_ROOT = "<ROOT>" +OPARSER_PACKAGES_ROOT = "packages" +OPARSER_PACKAGES = "packages.{{PACKAGE}}" +OPARSER_MANIFEST_DEFINITIONS = "definitions" + +SUPPORTED_DISPATCHABLE_OBJECT_PARSERS = { + OPARSER_MANIFEST_ROOT: reference_documentation( + reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md", + ), + OPARSER_MANIFEST_DEFINITIONS: reference_documentation( + title="Packager provided definitions", + description="Reusable packager provided definitions such as manifest variables.", + reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#packager-provided-definitions", + ), + OPARSER_PACKAGES: reference_documentation( + title="Binary package rules", + description=textwrap.dedent( + """\ + Inside the manifest, the `packages` mapping can be used to define requests for the binary packages + you want `debputy` to produce. Each key inside `packages` must be the name of a binary package + defined in `debian/control`. The value is a dictionary defining which features that `debputy` + should apply to that binary package. An example could be: + + packages: + foo: + transformations: + - create-symlink: + path: usr/share/foo/my-first-symlink + target: /usr/share/bar/symlink-target + - create-symlink: + path: usr/lib/{{DEB_HOST_MULTIARCH}}/my-second-symlink + target: /usr/lib/{{DEB_HOST_MULTIARCH}}/baz/symlink-target + bar: + transformations: + - create-directories: + - some/empty/directory.d + - another/empty/integration-point.d + - create-directories: + path: a/third-empty/directory.d + owner: www-data + group: www-data + + In this case, `debputy` will create some symlinks inside the `foo` package and some directories for + the `bar` package. The following subsections define the keys you can use under each binary package. + """ + ), + reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#binary-package-rules", + ), +} diff --git a/src/debputy/plugin/api/plugin_parser.py b/src/debputy/plugin/api/plugin_parser.py index dd5c0d0..0e7954b 100644 --- a/src/debputy/plugin/api/plugin_parser.py +++ b/src/debputy/plugin/api/plugin_parser.py @@ -1,10 +1,10 @@ from typing import NotRequired, List, Any, TypedDict -from debputy.manifest_parser.base_types import ( +from debputy.manifest_parser.tagging_types import ( DebputyParsedContent, - OctalMode, TypeMapping, ) +from debputy.manifest_parser.base_types import OctalMode from debputy.manifest_parser.declarative_parser import ParserGenerator from debputy.plugin.api.impl_types import KnownPackagingFileInfo diff --git a/src/debputy/plugin/api/spec.py b/src/debputy/plugin/api/spec.py index b7f19c0..30308f9 100644 --- a/src/debputy/plugin/api/spec.py +++ b/src/debputy/plugin/api/spec.py @@ -25,6 +25,7 @@ from typing import ( Tuple, get_args, Container, + final, ) from debian.substvars import Substvars @@ -32,17 +33,23 @@ from debian.substvars import Substvars from debputy import util from debputy.exceptions import TestPathWithNonExistentFSPathError, PureVirtualPathError from debputy.interpreter import Interpreter, extract_shebang_interpreter_from_file +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.manifest_parser.util import parse_symbolic_mode from debputy.packages import BinaryPackage from debputy.types import S if TYPE_CHECKING: + from debputy.plugin.debputy.to_be_api_types import BuildRule, BSR, BuildSystemRule + from debputy.plugin.api.impl_types import DIPHandler from debputy.manifest_parser.base_types import ( StaticFileSystemOwner, StaticFileSystemGroup, ) +DP = TypeVar("DP", bound=DebputyDispatchableType) + + PluginInitializationEntryPoint = Callable[["DebputyPluginInitializer"], None] MetadataAutoDetector = Callable[ ["VirtualPath", "BinaryCtrlAccessor", "PackageProcessingContext"], None @@ -86,17 +93,20 @@ DebputyIntegrationMode = Literal[ "dh-sequence-zz-debputy-rrr", ] +INTEGRATION_MODE_FULL: DebputyIntegrationMode = "full" INTEGRATION_MODE_DH_DEBPUTY_RRR: DebputyIntegrationMode = "dh-sequence-zz-debputy-rrr" INTEGRATION_MODE_DH_DEBPUTY: DebputyIntegrationMode = "dh-sequence-zz-debputy" ALL_DEBPUTY_INTEGRATION_MODES: FrozenSet[DebputyIntegrationMode] = frozenset( get_args(DebputyIntegrationMode) ) +_DEBPUTY_DISPATCH_METADATA_ATTR_NAME = "_debputy_dispatch_metadata" + def only_integrations( *integrations: DebputyIntegrationMode, ) -> Container[DebputyIntegrationMode]: - return frozenset(*integrations) + return frozenset(integrations) def not_integrations( @@ -212,6 +222,27 @@ class PathDef: materialized_content: Optional[str] = None +@dataclasses.dataclass(slots=True, frozen=True) +class DispatchablePluggableManifestRuleMetadata(Generic[DP]): + """NOT PUBLIC API (used internally by part of the public API)""" + + manifest_keywords: Sequence[str] + dispatched_type: Type[DP] + unwrapped_constructor: "DIPHandler" + expected_debputy_integration_mode: Optional[Container[DebputyIntegrationMode]] = ( + None + ) + online_reference_documentation: Optional["ParserDocumentation"] = None + apply_standard_attribute_documentation: bool = False + source_format: Optional[Any] = None + + +@dataclasses.dataclass(slots=True, frozen=True) +class BuildSystemManifestRuleMetadata(DispatchablePluggableManifestRuleMetadata): + build_system_impl: Optional[Type["BuildSystemRule"]] = (None,) + auto_detection_shadow_build_systems: FrozenSet[str] = frozenset() + + def virtual_path_def( path_name: str, /, @@ -1507,6 +1538,16 @@ class ParserAttributeDocumentation: attributes: FrozenSet[str] description: Optional[str] + @property + def is_hidden(self) -> bool: + return False + + +@final +@dataclasses.dataclass(slots=True, frozen=True) +class StandardParserAttributeDocumentation(ParserAttributeDocumentation): + sort_category: int = 0 + def undocumented_attr(attr: str) -> ParserAttributeDocumentation: """Describe an attribute as undocumented @@ -1514,6 +1555,8 @@ def undocumented_attr(attr: str) -> ParserAttributeDocumentation: If you for some reason do not want to document a particular attribute, you can mark it as undocumented. This is required if you are only documenting a subset of the attributes, because `debputy` assumes any omission to be a mistake. + + :param attr: Name of the attribute """ return ParserAttributeDocumentation( frozenset({attr}), diff --git a/src/debputy/plugin/api/std_docs.py b/src/debputy/plugin/api/std_docs.py new file mode 100644 index 0000000..f07c307 --- /dev/null +++ b/src/debputy/plugin/api/std_docs.py @@ -0,0 +1,142 @@ +import textwrap +from typing import Type, Sequence, Mapping, Container, Iterable, Any + +from debputy.manifest_parser.base_types import DebputyParsedContentStandardConditional +from debputy.manifest_parser.tagging_types import DebputyParsedContent +from debputy.plugin.api.spec import ( + ParserAttributeDocumentation, + StandardParserAttributeDocumentation, +) +from debputy.plugin.debputy.to_be_api_types import ( + OptionalInstallDirectly, + OptionalInSourceBuild, + OptionalBuildDirectory, + BuildRuleParsedFormat, +) + +_STD_ATTR_DOCS: Mapping[ + Type[DebputyParsedContent], + Sequence[ParserAttributeDocumentation], +] = { + BuildRuleParsedFormat: [ + StandardParserAttributeDocumentation( + frozenset(["name"]), + textwrap.dedent( + """\ + The name of the build step. + + The name is used for multiple things, such as: + 1) If you ever need to reference the build elsewhere, the name will be used. + 2) When `debputy` references the build in log output and error, it will use the name. + 3) It is used as defaults for when `debputy` derives build and `DESTDIR` directories + for the build. + """ + ), + # Put in top, + sort_category=-1000, + ), + StandardParserAttributeDocumentation( + frozenset(["for_packages"]), + textwrap.dedent( + """\ + Which package or packages this build step applies to. + + Either a package name or a list of package names. + """ + ), + ), + StandardParserAttributeDocumentation( + frozenset(["environment"]), + textwrap.dedent( + """\ + Specify that this build step uses the named environment + + If omitted, the default environment will be used. If no default environment is present, + then this option is mandatory. + """ + ), + ), + ], + OptionalBuildDirectory: [ + StandardParserAttributeDocumentation( + frozenset(["build_directory"]), + textwrap.dedent( + """\ + The build directory to use for the build. + + By default, `debputy` will derive a build directory automatically if the build system needs + it. However, it can be useful if you need to reference the directory name from other parts + of the manifest or want a "better" name than `debputy` comes up with. + """ + ), + ), + ], + OptionalInSourceBuild: [ + StandardParserAttributeDocumentation( + frozenset(["perform_in_source_build"]), + textwrap.dedent( + """\ + Whether the build system should use "in source" or "out of source" build. + + This is mostly useful for forcing "in source" builds for build systems that default to + "out of source" builds like `autoconf`. + + The default depends on the build system and the value of the `build-directory` attribute + (if supported by the build system). + """ + ), + # Late + sort_category=500, + ), + ], + OptionalInstallDirectly: [ + StandardParserAttributeDocumentation( + frozenset(["install_directly_to_package"]), + textwrap.dedent( + """\ + Whether the build system should install all upstream content directly into the package. + + This option is mostly useful for disabling said behavior by setting the attribute to `false`. + The attribute conditionally defaults to `true` when the build only applies to one package. + If explicitly set to `true`, then this build step must apply to exactly one package (usually + implying that `for` is set to that package when the source builds multiple packages). + + When `true`, this behaves similar to `dh_auto_install --destdir=debian/PACKAGE`. + """ + ), + ), + ], + DebputyParsedContentStandardConditional: [ + StandardParserAttributeDocumentation( + frozenset(["when"]), + textwrap.dedent( + """\ + A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). + + The conditional will disable the entire rule when the conditional evaluates to false. + """ + ), + # Last + sort_category=9999, + ), + ], +} + + +def docs_from( + *ts: Any, + exclude_attributes: Container[str] = frozenset(), +) -> Iterable[ParserAttributeDocumentation]: + """Provide standard attribute documentation from existing types + + This is a work-around for `apply_standard_attribute_documentation` requiring python3.12. + If you can assume python3.12, use `apply_standard_attribute_documentation` instead. + """ + for t in ts: + attrs = _STD_ATTR_DOCS.get(t) + if attrs is None: + raise ValueError(f"No standard documentation for {str(t)}") + for attr in attrs: + if any(a in exclude_attributes for a in attrs): + continue + yield attr diff --git a/src/debputy/plugin/debputy/binary_package_rules.py b/src/debputy/plugin/debputy/binary_package_rules.py index 98da763..45547b9 100644 --- a/src/debputy/plugin/debputy/binary_package_rules.py +++ b/src/debputy/plugin/debputy/binary_package_rules.py @@ -17,14 +17,10 @@ from typing import ( from debputy import DEBPUTY_DOC_ROOT_DIR from debputy.maintscript_snippet import DpkgMaintscriptHelperCommand, MaintscriptSnippet -from debputy.manifest_parser.base_types import ( - DebputyParsedContent, - FileSystemExactMatchRule, -) -from debputy.manifest_parser.declarative_parser import ( - DebputyParseHint, - ParserGenerator, -) +from debputy.manifest_parser.base_types import FileSystemExactMatchRule +from debputy.manifest_parser.tagging_types import DebputyParsedContent +from debputy.manifest_parser.parse_hints import DebputyParseHint +from debputy.manifest_parser.declarative_parser import ParserGenerator from debputy.manifest_parser.exceptions import ManifestParseException from debputy.manifest_parser.parser_data import ParserContextData from debputy.manifest_parser.util import AttributePath @@ -34,7 +30,7 @@ from debputy.plugin.api.impl import ( DebputyPluginInitializerProvider, ServiceDefinitionImpl, ) -from debputy.plugin.api.impl_types import OPARSER_PACKAGES +from debputy.plugin.api.parser_tables import OPARSER_PACKAGES from debputy.plugin.api.spec import ( ServiceUpgradeRule, ServiceDefinition, diff --git a/src/debputy/plugin/debputy/build_system_rules.py b/src/debputy/plugin/debputy/build_system_rules.py new file mode 100644 index 0000000..b7ee898 --- /dev/null +++ b/src/debputy/plugin/debputy/build_system_rules.py @@ -0,0 +1,2319 @@ +import dataclasses +import json +import os +import subprocess +import textwrap +from typing import ( + NotRequired, + TypedDict, + Self, + cast, + Dict, + Mapping, + Sequence, + MutableMapping, + Iterable, + Container, + List, + Tuple, + Union, + Optional, + TYPE_CHECKING, + Literal, +) + +from debian.debian_support import Version + +from debputy import DEBPUTY_DOC_ROOT_DIR +from debputy._manifest_constants import MK_BUILDS +from debputy.manifest_parser.base_types import ( + BuildEnvironmentDefinition, + DebputyParsedContentStandardConditional, + FileSystemExactMatchRule, +) +from debputy.manifest_parser.exceptions import ( + ManifestParseException, + ManifestInvalidUserDataException, +) +from debputy.manifest_parser.parser_data import ParserContextData +from debputy.manifest_parser.util import AttributePath +from debputy.plugin.api import reference_documentation +from debputy.plugin.api.impl import ( + DebputyPluginInitializerProvider, +) +from debputy.plugin.api.parser_tables import OPARSER_MANIFEST_ROOT +from debputy.plugin.api.spec import ( + documented_attr, + INTEGRATION_MODE_FULL, + only_integrations, + VirtualPath, +) +from debputy.plugin.api.std_docs import docs_from +from debputy.plugin.debputy.to_be_api_types import ( + BuildRule, + StepBasedBuildSystemRule, + OptionalInstallDirectly, + BuildSystemCharacteristics, + OptionalBuildDirectory, + OptionalInSourceBuild, + MakefileSupport, + BuildRuleParsedFormat, + debputy_build_system, + CleanHelper, + NinjaBuildSupport, +) +from debputy.types import EnvironmentModification +from debputy.util import ( + _warn, + run_build_system_command, + _error, + PerlConfigVars, + resolve_perl_config, + generated_content_dir, +) + +if TYPE_CHECKING: + from debputy.build_support.build_context import BuildContext + from debputy.highlevel_manifest import HighLevelManifest + + +PERL_CMD = "perl" + + +def register_build_system_rules(api: DebputyPluginInitializerProvider) -> None: + register_build_keywords(api) + register_build_rules(api) + + +def register_build_keywords(api: DebputyPluginInitializerProvider) -> None: + + api.pluggable_manifest_rule( + OPARSER_MANIFEST_ROOT, + "build-environments", + List[NamedEnvironmentSourceFormat], + _parse_build_environments, + expected_debputy_integration_mode=only_integrations(INTEGRATION_MODE_FULL), + inline_reference_documentation=reference_documentation( + title="Build Environments (`build-environments`)", + description=textwrap.dedent( + """\ + Define named environments to set the environment for any build commands that needs + a non-default environment. + + The environment definitions can be used to tweak the environment variables used by the + build commands. An example: + + build-environments: + - name: custom-env + set: + ENV_VAR: foo + ANOTHER_ENV_VAR: bar + builds: + - autoconf: + environment: custom-env + + The environment definition has multiple attributes for setting environment variables + which determines when the definition is applied. The resulting environment is the + result of the following order of operations. + + 1. The environment `debputy` received from its parent process. + 2. Apply all the variable definitions from `set` (if the attribute is present) + 3. Apply all computed variables (such as variables from `dpkg-buildflags`). + 4. Apply all the variable definitions from `override` (if the attribute is present) + 5. Remove all variables listed in `unset` (if the attribute is present). + + Accordingly, both `override` and `unset` will overrule any computed variables while + `set` will be overruled by any computed variables. + + Note that these variables are not available via manifest substitution (they are only + visible to build commands). They are only available to build commands. + """ + ), + attributes=[ + documented_attr( + "name", + textwrap.dedent( + """\ + The name of the environment + + The name is used to reference the environment from build rules. + """ + ), + ), + documented_attr( + "set", + textwrap.dedent( + """\ + A mapping of environment variables to be set. + + Note these environment variables are set before computed variables (such + as `dpkg-buildflags`) are provided. They can affect the content of the + computed variables, but they cannot overrule them. If you need to overrule + a computed variable, please use `override` instead. + """ + ), + ), + documented_attr( + "override", + textwrap.dedent( + """\ + A mapping of environment variables to set. + + Similar to `set`, but it can overrule computed variables like those from + `dpkg-buildflags`. + """ + ), + ), + documented_attr( + "unset", + textwrap.dedent( + """\ + A list of environment variables to unset. + + Any environment variable named here will be unset. No warnings or errors + will be raised if a given variable was not set. + """ + ), + ), + ], + reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#build-environment-build-environment", + ), + ) + api.pluggable_manifest_rule( + OPARSER_MANIFEST_ROOT, + "default-build-environment", + EnvironmentSourceFormat, + _parse_default_environment, + expected_debputy_integration_mode=only_integrations(INTEGRATION_MODE_FULL), + inline_reference_documentation=reference_documentation( + title="Default Build Environment (`default-build-environment`)", + description=textwrap.dedent( + """\ + Define the environment variables used in all build commands that uses the default + environment. + + The environment definition can be used to tweak the environment variables used by the + build commands. An example: + + default-build-environment: + set: + ENV_VAR: foo + ANOTHER_ENV_VAR: bar + + The environment definition has multiple attributes for setting environment variables + which determines when the definition is applied. The resulting environment is the + result of the following order of operations. + + 1. The environment `debputy` received from its parent process. + 2. Apply all the variable definitions from `set` (if the attribute is present) + 3. Apply all computed variables (such as variables from `dpkg-buildflags`). + 4. Apply all the variable definitions from `override` (if the attribute is present) + 5. Remove all variables listed in `unset` (if the attribute is present). + + Accordingly, both `override` and `unset` will overrule any computed variables while + `set` will be overruled by any computed variables. + + Note that these variables are not available via manifest substitution (they are only + visible to build commands). They are only available to build commands. + """ + ), + attributes=[ + documented_attr( + "set", + textwrap.dedent( + """\ + A mapping of environment variables to be set. + + Note these environment variables are set before computed variables (such + as `dpkg-buildflags`) are provided. They can affect the content of the + computed variables, but they cannot overrule them. If you need to overrule + a computed variable, please use `override` instead. + """ + ), + ), + documented_attr( + "override", + textwrap.dedent( + """\ + A mapping of environment variables to set. + + Similar to `set`, but it can overrule computed variables like those from + `dpkg-buildflags`. + """ + ), + ), + documented_attr( + "unset", + textwrap.dedent( + """\ + A list of environment variables to unset. + + Any environment variable named here will be unset. No warnings or errors + will be raised if a given variable was not set. + """ + ), + ), + ], + reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#build-environment-build-environment", + ), + ) + api.pluggable_manifest_rule( + OPARSER_MANIFEST_ROOT, + MK_BUILDS, + List[BuildRule], + _handle_build_rules, + expected_debputy_integration_mode=only_integrations(INTEGRATION_MODE_FULL), + inline_reference_documentation=reference_documentation( + title="Build rules", + description=textwrap.dedent( + """\ + Define how to build the upstream part of the package. Usually this is done via "build systems", + which also defines the clean rules. + """ + ), + ), + ) + + +def register_build_rules(api: DebputyPluginInitializerProvider) -> None: + api.register_build_system(ParsedAutoconfBuildRuleDefinition) + api.register_build_system(ParsedMakeBuildRuleDefinition) + + api.register_build_system(ParsedPerlBuildBuildRuleDefinition) + api.register_build_system(ParsedPerlMakeMakerBuildRuleDefinition) + api.register_build_system(ParsedDebhelperBuildRuleDefinition) + + api.register_build_system(ParsedCMakeBuildRuleDefinition) + api.register_build_system(ParsedMesonBuildRuleDefinition) + + api.register_build_system(ParsedQmakeBuildRuleDefinition) + api.register_build_system(ParsedQmake6BuildRuleDefinition) + + +class EnvironmentSourceFormat(TypedDict): + set: NotRequired[Dict[str, str]] + override: NotRequired[Dict[str, str]] + unset: NotRequired[List[str]] + + +class NamedEnvironmentSourceFormat(EnvironmentSourceFormat): + name: str + + +_READ_ONLY_ENV_VARS = { + "DEB_CHECK_COMMAND": None, + "DEB_SIGN_KEYID": None, + "DEB_SIGN_KEYFILE": None, + "DEB_BUILD_OPTIONS": "DEB_BUILD_MAINT_OPTIONS", + "DEB_BUILD_PROFILES": None, + "DEB_RULES_REQUIRES_ROOT": None, + "DEB_GAIN_ROOT_COMMAND": None, + "DH_EXTRA_ADDONS": None, + "DH_NO_ACT": None, +} + + +def _check_variables( + env_vars: Iterable[str], + attribute_path: AttributePath, +) -> None: + for env_var in env_vars: + if env_var not in _READ_ONLY_ENV_VARS: + continue + alt = _READ_ONLY_ENV_VARS.get(env_var) + var_path = attribute_path[env_var].path_key_lc + if alt is None: + raise ManifestParseException( + f"The variable {env_var} cannot be modified by the manifest. This restriction is generally" + f" because the build should not touch those variables or changing them have no effect" + f" (since the consumer will not see the change). The problematic definition was {var_path}" + ) + else: + raise ManifestParseException( + f"The variable {env_var} cannot be modified by the manifest. This restriction is generally" + f" because the build should not touch those variables or changing them have no effect" + f" (since the consumer will not see the change). Depending on what you are trying to" + f' accomplish, the variable "{alt}" might be a suitable alternative.' + f" The problematic definition was {var_path}" + ) + + +def _no_overlap( + lhs: Iterable[Union[str, Tuple[int, str]]], + rhs: Container[str], + lhs_key: str, + rhs_key: str, + redundant_key: str, + attribute_path: AttributePath, +) -> None: + for kt in lhs: + if isinstance(kt, tuple): + lhs_path_key, var = kt + else: + lhs_path_key = var = kt + if var not in rhs: + continue + lhs_path = attribute_path[lhs_key][lhs_path_key].path_key_lc + rhs_path = attribute_path[rhs_key][var].path_key_lc + r_path = lhs_path if redundant_key == rhs_key else rhs_path + raise ManifestParseException( + f"The environment variable {var} was declared in {lhs_path} and {rhs_path}." + f" Due to how the variables are applied, the definition in {r_path} is redundant" + f" and can effectively be removed. Please review the manifest and remove one of" + f" the two definitions." + ) + + +@dataclasses.dataclass(slots=True, frozen=True) +class ManifestProvidedBuildEnvironment(BuildEnvironmentDefinition): + + name: str + is_default: bool + attribute_path: AttributePath + parser_context: ParserContextData + + set_vars: Mapping[str, str] + override_vars: Mapping[str, str] + unset_vars: Sequence[str] + + @classmethod + def from_environment_definition( + cls, + env: EnvironmentSourceFormat, + attribute_path: AttributePath, + parser_context: ParserContextData, + is_default: bool = False, + ) -> Self: + reference_name: Optional[str] + if is_default: + name = "default-env" + reference_name = None + else: + named_env = cast("NamedEnvironmentSourceFormat", env) + name = named_env["name"] + reference_name = name + + set_vars = env.get("set", {}) + override_vars = env.get("override", {}) + unset_vars = env.get("unset", []) + _check_variables(set_vars, attribute_path["set"]) + _check_variables(override_vars, attribute_path["override"]) + _check_variables(unset_vars, attribute_path["unset"]) + + if not set_vars and not override_vars and not unset_vars: + raise ManifestParseException( + f"The environment definition {attribute_path.path_key_lc} was empty. Please provide" + " some content or delete the definition." + ) + + _no_overlap( + enumerate(unset_vars), + set_vars, + "unset", + "set", + "set", + attribute_path, + ) + _no_overlap( + enumerate(unset_vars), + override_vars, + "unset", + "override", + "override", + attribute_path, + ) + _no_overlap( + override_vars, + set_vars, + "override", + "set", + "set", + attribute_path, + ) + + r = cls( + name, + is_default, + attribute_path, + parser_context, + set_vars, + override_vars, + unset_vars, + ) + parser_context._register_build_environment( + reference_name, + r, + attribute_path, + is_default, + ) + + return r + + def update_env(self, env: MutableMapping[str, str]) -> None: + if set_vars := self.set_vars: + env.update(set_vars) + dpkg_env = self.dpkg_buildflags_env(env, self.attribute_path.path_key_lc) + self.log_computed_env(f"dpkg-buildflags [{self.name}]", dpkg_env) + if overlapping_env := dpkg_env.keys() & set_vars.keys(): + for var in overlapping_env: + key_lc = self.attribute_path["set"][var].path_key_lc + _warn( + f'The variable "{var}" defined at {key_lc} is shadowed by a computed variable.' + f" If the manifest definition is more important, please define it via `override` rather than" + f" `set`." + ) + env.update(dpkg_env) + if override_vars := self.override_vars: + env.update(override_vars) + if unset_vars := self.unset_vars: + for var in unset_vars: + try: + del env[var] + except KeyError: + pass + + +_MAKE_DEFAULT_TOOLS = [ + ("CC", "gcc"), + ("CXX", "g++"), + ("PKG_CONFIG", "pkg-config"), +] + + +class MakefileBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ("_make_support", "_build_target", "_install_target", "_directory") + + def __init__( + self, + attributes: "ParsedMakeBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(attributes, attribute_path, parser_context) + directory = attributes.get("directory") + if directory is not None: + self._directory = directory.match_rule.path + else: + self._directory = None + self._make_support = MakefileSupport.from_build_system(self) + self._build_target = attributes.get("build_target") + self._test_target = attributes.get("test_target") + self._install_target = attributes.get("install_target") + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + return any(p in source_root for p in ("Makefile", "makefile", "GNUmakefile")) + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="not-supported", + ) + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + # No configure step + pass + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + extra_vars = [] + build_target = self._build_target + if build_target is not None: + extra_vars.append(build_target) + if context.is_cross_compiling: + for envvar, tool in _MAKE_DEFAULT_TOOLS: + cross_tool = os.environ.get(envvar) + if cross_tool is None: + cross_tool = context.cross_tool(tool) + extra_vars.append(f"{envvar}={cross_tool}") + self._make_support.run_make( + context, + *extra_vars, + "INSTALL=install --strip-program=true", + directory=self._directory, + ) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._run_make_maybe_explicit_target( + context, + self._test_target, + ["test", "check"], + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + self._run_make_maybe_explicit_target( + context, + self._install_target, + ["install"], + f"DESTDIR={dest_dir}", + "AM_UPDATE_INFO_DIR=no", + "INSTALL=install --strip-program=true", + ) + + def _run_make_maybe_explicit_target( + self, + context: "BuildContext", + provided_target: Optional[str], + fallback_targets: Sequence[str], + *make_args: str, + ) -> None: + make_support = self._make_support + if provided_target is not None: + make_support.run_make( + context, + provided_target, + *make_args, + directory=self._directory, + ) + else: + make_support.run_first_existing_target_if_any( + context, + fallback_targets, + *make_args, + directory=self._directory, + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + self._make_support.run_first_existing_target_if_any( + context, + ["distclean", "realclean", "clean"], + ) + + +class PerlBuildBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = "configure_args" + + def __init__( + self, + attributes: "ParsedPerlBuildBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(attributes, attribute_path, parser_context) + self.configure_args = attributes.get("configure_args", []) + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + return "Build.PL" in source_root + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="not-supported", + ) + + @staticmethod + def _perl_cross_build_env( + context: "BuildContext", + ) -> Tuple[PerlConfigVars, Optional[EnvironmentModification]]: + perl_config_data = resolve_perl_config( + context.dpkg_architecture_variables, + None, + ) + if context.is_cross_compiling: + perl5lib_dir = perl_config_data.cross_inc_dir + if perl5lib_dir is not None: + env_perl5lib = os.environ.get("PERL5LIB") + if env_perl5lib is not None: + perl5lib_dir = ( + perl5lib_dir + perl_config_data.path_sep + env_perl5lib + ) + env_mod = EnvironmentModification( + replacements=[ + ("PERL5LIB", perl5lib_dir), + ], + ) + return perl_config_data, env_mod + return perl_config_data, None + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + perl_config_data, cross_env_mod = self._perl_cross_build_env(context) + configure_env = EnvironmentModification( + replacements=[("PERL_MM_USE_DEFAULT", "1")] + ) + if cross_env_mod is not None: + configure_env = configure_env.combine(cross_env_mod) + + configure_cmd = [ + PERL_CMD, + "Build.PL", + "--installdirs", + "vendor", + ] + cflags = os.environ.get("CFLAGS", "") + cppflags = os.environ.get("CPPFLAGS", "") + ldflags = os.environ.get("LDFLAGS", "") + + if cflags != "" or cppflags != "": + configure_cmd.append("--config") + combined = f"{cflags} {cppflags}".strip() + configure_cmd.append(f"optimize={combined}") + + if ldflags != "" or cflags != "" or context.is_cross_compiling: + configure_cmd.append("--config") + combined = f"{perl_config_data.ld} {cflags} {ldflags}".strip() + configure_cmd.append(f"ld={combined}") + if self.configure_args: + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + configure_cmd.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + run_build_system_command(*configure_cmd, env_mod=configure_env) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + _, cross_env_mod = self._perl_cross_build_env(context) + run_build_system_command(PERL_CMD, "Build", env_mod=cross_env_mod) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + _, cross_env_mod = self._perl_cross_build_env(context) + run_build_system_command( + PERL_CMD, + "Build", + "test", + "--verbose", + "1", + env_mod=cross_env_mod, + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + _, cross_env_mod = self._perl_cross_build_env(context) + run_build_system_command( + PERL_CMD, + "Build", + "install", + "--destdir", + dest_dir, + "--create_packlist", + "0", + env_mod=cross_env_mod, + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + _, cross_env_mod = self._perl_cross_build_env(context) + if os.path.lexists("Build"): + run_build_system_command( + PERL_CMD, + "Build", + "realclean", + "--allow_mb_mismatch", + "1", + env_mod=cross_env_mod, + ) + + +class PerlMakeMakerBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ("configure_args", "_make_support") + + def __init__( + self, + attributes: "ParsedPerlBuildBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(attributes, attribute_path, parser_context) + self.configure_args = attributes.get("configure_args", []) + self._make_support = MakefileSupport.from_build_system(self) + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + return "Makefile.PL" in source_root + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="not-supported", + ) + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + configure_env = EnvironmentModification( + replacements=[ + ("PERL_MM_USE_DEFAULT", "1"), + ("PERL_AUTOINSTALL", "--skipdeps"), + ] + ) + perl_args = [] + mm_args = ["INSTALLDIRS=vendor"] + if "CFLAGS" in os.environ: + mm_args.append( + f"OPTIMIZE={os.environ['CFLAGS']} {os.environ['CPPFLAGS']}".rstrip() + ) + + perl_config_data = resolve_perl_config( + context.dpkg_architecture_variables, + None, + ) + + if "LDFLAGS" in os.environ: + mm_args.append( + f"LD={perl_config_data.ld} {os.environ['CFLAGS']} {os.environ['LDFLAGS']}" + ) + + if context.is_cross_compiling: + perl5lib_dir = perl_config_data.cross_inc_dir + if perl5lib_dir is not None: + perl_args.append(f"-I{perl5lib_dir}") + + if self.configure_args: + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + mm_args.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + run_build_system_command( + PERL_CMD, + *perl_args, + "Makefile.PL", + *mm_args, + env_mod=configure_env, + ) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._make_support.run_make(context) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._make_support.run_first_existing_target_if_any( + context, + ["check", "test"], + "TEST_VERBOSE=1", + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + is_mm_makefile = False + with open("Makefile", "rb") as fd: + for line in fd: + if b"generated automatically by MakeMaker" in line: + is_mm_makefile = True + break + + install_args = [f"DESTDIR={dest_dir}"] + + # Special case for Makefile.PL that uses + # Module::Build::Compat. PREFIX should not be passed + # for those; it already installs into /usr by default. + if is_mm_makefile: + install_args.append("PREFIX=/usr") + + self._make_support.run_first_existing_target_if_any( + context, + ["install"], + *install_args, + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + self._make_support.run_first_existing_target_if_any( + context, + ["distclean", "realclean", "clean"], + ) + + +class DebhelperBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ("configure_args", "dh_build_system") + + def __init__( + self, + parsed_data: "ParsedDebhelperBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(parsed_data, attribute_path, parser_context) + self.configure_args = parsed_data.get("configure_args", []) + self.dh_build_system = parsed_data.get("dh_build_system") + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + try: + v = subprocess.check_output( + ["dh_assistant", "which-build-system"], + cwd=source_root.fs_path, + ) + except subprocess.CalledProcessError: + return False + else: + d = json.loads(v) + build_system = d.get("build-system") + return build_system is not None + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="supported-but-not-default", + ) + + def before_first_impl_step( + self, *, stage: Literal["build", "clean"], **kwargs + ) -> None: + dh_build_system = self.dh_build_system + if dh_build_system is None: + return + try: + subprocess.check_call( + ["dh_assistant", "which-build-system", f"-S{dh_build_system}"] + ) + except FileNotFoundError: + _error( + "The debhelper build system assumes `dh_assistant` is available (`debhelper (>= 13.5~)`)" + ) + except subprocess.SubprocessError: + raise ManifestInvalidUserDataException( + f'The debhelper build system "{dh_build_system}" does not seem to' + f" be available according to" + f" `dh_assistant which-build-system -S{dh_build_system}`" + ) from None + + def _default_options(self) -> List[str]: + default_options = [] + if self.dh_build_system is not None: + default_options.append(f"-S{self.dh_build_system}") + if self.build_directory is not None: + default_options.append(f"-B{self.build_directory}") + + return default_options + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + if ( + os.path.lexists("configure.ac") or os.path.lexists("configure.in") + ) and not os.path.lexists("debian/autoreconf.before"): + run_build_system_command("dh_update_autotools_config") + run_build_system_command("dh_autoreconf") + + default_options = self._default_options() + configure_args = default_options.copy() + if self.configure_args: + configure_args.append("--") + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + configure_args.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + run_build_system_command("dh_auto_configure", *configure_args) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + default_options = self._default_options() + run_build_system_command("dh_auto_build", *default_options) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + default_options = self._default_options() + run_build_system_command("dh_auto_test", *default_options) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + default_options = self._default_options() + run_build_system_command( + "dh_auto_install", + *default_options, + f"--destdir={dest_dir}", + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + default_options = self._default_options() + run_build_system_command("dh_auto_clean", *default_options) + # The "global" clean logic takes care of `dh_autoreconf_clean` and `dh_clean` + + +class AutoconfBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ("configure_args", "_make_support") + + def __init__( + self, + parsed_data: "ParsedAutoconfBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(parsed_data, attribute_path, parser_context) + configure_args = [a for a in parsed_data.get("configure_args", [])] + self.configure_args = configure_args + self._make_support = MakefileSupport.from_build_system(self) + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="supported-and-default", + ) + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + if "configure.ac" in source_root: + return True + configure_in = source_root.get("configure.in") + if configure_in is not None and configure_in.is_file: + with configure_in.open(byte_io=True, buffering=4096) as fd: + for no, line in enumerate(fd): + if no > 100: + break + if b"AC_INIT" in line or b"AC_PREREQ" in line: + return True + configure = source_root.get("configure") + if configure is None or not configure.is_executable or not configure.is_file: + return False + with configure.open(byte_io=True, buffering=4096) as fd: + for no, line in enumerate(fd): + if no > 10: + break + if b"GNU Autoconf" in line: + return True + return False + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + if ( + os.path.lexists("configure.ac") or os.path.lexists("configure.in") + ) and not os.path.lexists("debian/autoreconf.before"): + run_build_system_command("dh_update_autotools_config") + run_build_system_command("dh_autoreconf") + + dpkg_architecture_variables = context.dpkg_architecture_variables + multi_arch = dpkg_architecture_variables.current_host_multiarch + silent_rules = ( + "--enable-silent-rules" + if context.is_terse_build + else "--disable-silent-rules" + ) + + configure_args = [ + f"--build={dpkg_architecture_variables['DEB_BUILD_GNU_TYPE']}", + "--prefix=/usr", + "--includedir=${prefix}/include", + "--mandir=${prefix}/share/man", + "--infodir=${prefix}/share/info", + "--sysconfdir=/etc", + "--localstatedir=/var", + "--disable-option-checking", + silent_rules, + f"--libdir=${{prefix}}/{multi_arch}", + "--runstatedir=/run", + "--disable-maintainer-mode", + "--disable-dependency-tracking", + ] + if dpkg_architecture_variables.is_cross_compiling: + configure_args.append( + f"--host={dpkg_architecture_variables['DEB_HOST_GNU_TYPE']}" + ) + if self.configure_args: + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + configure_args.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + self.ensure_build_dir_exists() + configure_script = self.relative_from_builddir_to_source("configure") + with self.dump_logs_on_error("config.log"): + run_build_system_command( + configure_script, + *configure_args, + cwd=self.build_directory, + ) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._make_support.run_make(context) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + limit = context.parallelization_limit(support_zero_as_unlimited=True) + testsuite_flags = [f"-j{limit}"] if limit else ["-j"] + + if not context.is_terse_build: + testsuite_flags.append("--verbose") + self._make_support.run_first_existing_target_if_any( + context, + # Order is deliberately inverse compared to debhelper (#924052) + ["check", "test"], + f"TESTSUITEFLAGS={' '.join(testsuite_flags)}", + "VERBOSE=1", + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + enable_parallelization = not os.path.lexists(self.build_dir_path("libtool")) + self._make_support.run_first_existing_target_if_any( + context, + ["install"], + f"DESTDIR={dest_dir}", + "AM_UPDATE_INFO_DIR=no", + enable_parallelization=enable_parallelization, + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + if self.out_of_source_build: + return + self._make_support.run_first_existing_target_if_any( + context, + ["distclean", "realclean", "clean"], + ) + # The "global" clean logic takes care of `dh_autoreconf_clean` and `dh_clean` + + +class CMakeBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ( + "configure_args", + "target_build_system", + "_make_support", + "_ninja_support", + ) + + def __init__( + self, + parsed_data: "ParsedCMakeBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(parsed_data, attribute_path, parser_context) + configure_args = [a for a in parsed_data.get("configure_args", [])] + self.configure_args = configure_args + self.target_build_system: Literal["make", "ninja"] = parsed_data.get( + "target_build_system", "make" + ) + self._make_support = MakefileSupport.from_build_system(self) + self._ninja_support = NinjaBuildSupport.from_build_system(self) + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="required", + ) + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + return "CMakeLists.txt" in source_root + + @staticmethod + def _default_cmake_env( + build_context: "BuildContext", + ) -> EnvironmentModification: + replacements = {} + if "DEB_PYTHON_INSTALL_LAYOUT" not in os.environ: + replacements["DEB_PYTHON_INSTALL_LAYOUT"] = "deb" + if "PKG_CONFIG" not in os.environ: + replacements["PKG_CONFIG"] = build_context.cross_tool("pkg-config") + return EnvironmentModification(replacements=replacements) + + @classmethod + def cmake_generator(cls, target_build_system: Literal["make", "ninja"]) -> str: + cmake_generators = { + "make": "Unix Makefiles", + "ninja": "Ninja", + } + return cmake_generators[target_build_system] + + @staticmethod + def _compiler_and_cross_flags( + context: "BuildContext", + cmake_flags: List[str], + ) -> None: + + if "CC" in os.environ: + cmake_flags.append(f"-DCMAKE_C_COMPILER={os.environ['CC']}") + elif context.is_cross_compiling: + cmake_flags.append(f"-DCMAKE_C_COMPILER={context.cross_tool('gcc')}") + + if "CXX" in os.environ: + cmake_flags.append(f"-DCMAKE_CXX_COMPILER={os.environ['CXX']}") + elif context.is_cross_compiling: + cmake_flags.append(f"-DCMAKE_CXX_COMPILER={context.cross_tool('g++')}") + + if context.is_cross_compiling: + deb_host2cmake_system = { + "linux": "Linux", + "kfreebsd": "kFreeBSD", + "hurd": "GNU", + } + + gnu_cpu2system_processor = { + "arm": "armv7l", + "misp64el": "mips64", + "powerpc64le": "ppc64le", + } + dpkg_architecture_variables = context.dpkg_architecture_variables + + try: + system_name = deb_host2cmake_system[ + dpkg_architecture_variables["DEB_HOST_ARCH_OS"] + ] + except KeyError as e: + name = e.args[0] + _error( + f"Cannot cross-compile via cmake: Missing CMAKE_SYSTEM_NAME for the DEB_HOST_ARCH_OS {name}" + ) + + gnu_cpu = dpkg_architecture_variables["DEB_HOST_GNU_CPU"] + system_processor = gnu_cpu2system_processor.get(gnu_cpu, gnu_cpu) + + cmake_flags.append(f"-DCMAKE_SYSTEM_NAME={system_name}") + cmake_flags.append(f"-DCMAKE_SYSTEM_PROCESSOR={system_processor}") + + pkg_config = context.cross_tool("pkg-config") + # Historical uses. Current versions of cmake uses the env variable instead. + cmake_flags.append(f"-DPKG_CONFIG_EXECUTABLE=/usr/bin/{pkg_config}") + cmake_flags.append(f"-DPKGCONFIG_EXECUTABLE=/usr/bin/{pkg_config}") + cmake_flags.append( + f"-DQMAKE_EXECUTABLE=/usr/bin/{context.cross_tool('qmake')}" + ) + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + cmake_flags = [ + "-DCMAKE_INSTALL_PREFIX=/usr", + "-DCMAKE_BUILD_TYPE=None", + "-DCMAKE_INSTALL_SYSCONFDIR=/etc", + "-DCMAKE_INSTALL_LOCALSTATEDIR=/var", + "-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON", + "-DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF", + "-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON", + "-DFETCHCONTENT_FULLY_DISCONNECTED=ON", + "-DCMAKE_INSTALL_RUNSTATEDIR=/run", + "-DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON", + "-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON", + f"-G{self.cmake_generator(self.target_build_system)}", + ] + if not context.is_terse_build: + cmake_flags.append("-DCMAKE_VERBOSE_MAKEFILE=ON") + + self._compiler_and_cross_flags(context, cmake_flags) + + if self.configure_args: + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + cmake_flags.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + + env_mod = self._default_cmake_env(context) + if "CPPFLAGS" in os.environ: + # CMake doesn't respect CPPFLAGS, see #653916. + cppflags = os.environ["CPPFLAGS"] + cflags = os.environ.get("CFLAGS", "") + f" {cppflags}".lstrip() + cxxflags = os.environ.get("CXXFLAGS", "") + f" {cppflags}".lstrip() + env_mod = env_mod.combine( + # The debhelper build system never showed this delta, so people might find it annoying. + EnvironmentModification( + replacements={ + "CFLAGS": cflags, + "CXXFLAGS": cxxflags, + } + ) + ) + if "ASMFLAGS" not in os.environ and "ASFLAGS" in os.environ: + env_mod = env_mod.combine( + # The debhelper build system never showed this delta, so people might find it annoying. + EnvironmentModification( + replacements={ + "ASMFLAGS": os.environ["ASFLAGS"], + } + ) + ) + self.ensure_build_dir_exists() + source_dir_from_build_dir = self.relative_from_builddir_to_source() + + with self.dump_logs_on_error( + "CMakeCache.txt", + "CMakeFiles/CMakeOutput.log", + "CMakeFiles/CMakeError.log", + ): + run_build_system_command( + "cmake", + *cmake_flags, + source_dir_from_build_dir, + cwd=self.build_directory, + env_mod=env_mod, + ) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + if self.target_build_system == "make": + make_flags = [] + if not context.is_terse_build: + make_flags.append("VERBOSE=1") + self._make_support.run_make(context, *make_flags) + else: + self._ninja_support.run_ninja_build(context) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + env_mod = EnvironmentModification( + replacements={ + "CTEST_OUTPUT_ON_FAILURE": "1", + }, + ) + if self.target_build_system == "make": + # Unlike make, CTest does not have "unlimited parallel" setting (-j implies + # -j1). Therefore, we do not set "allow zero as unlimited" here. + make_flags = [f"ARGS+=-j{context.parallelization_limit()}"] + if not context.is_terse_build: + make_flags.append("ARGS+=--verbose") + self._make_support.run_first_existing_target_if_any( + context, + ["check", "test"], + *make_flags, + env_mod=env_mod, + ) + else: + self._ninja_support.run_ninja_test(context, env_mod=env_mod) + + limit = context.parallelization_limit(support_zero_as_unlimited=True) + testsuite_flags = [f"-j{limit}"] if limit else ["-j"] + + if not context.is_terse_build: + testsuite_flags.append("--verbose") + self._make_support.run_first_existing_target_if_any( + context, + # Order is deliberately inverse compared to debhelper (#924052) + ["check", "test"], + f"TESTSUITEFLAGS={' '.join(testsuite_flags)}", + "VERBOSE=1", + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + env_mod = EnvironmentModification( + replacements={ + "LC_ALL": "C.UTF-8", + "DESTDIR": dest_dir, + } + ).combine(self._default_cmake_env(context)) + run_build_system_command( + "cmake", + "--install", + self.build_directory, + env_mod=env_mod, + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + if self.out_of_source_build: + return + if self.target_build_system == "make": + # Keep it here in case we change the `required` "out of source" to "supported-default" + self._make_support.run_first_existing_target_if_any( + context, + ["distclean", "realclean", "clean"], + ) + else: + self._ninja_support.run_ninja_clean(context) + + +class MesonBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ( + "configure_args", + "_ninja_support", + ) + + def __init__( + self, + parsed_data: "ParsedMesonBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(parsed_data, attribute_path, parser_context) + configure_args = [a for a in parsed_data.get("configure_args", [])] + self.configure_args = configure_args + self._ninja_support = NinjaBuildSupport.from_build_system(self) + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="required", + ) + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + return "meson.build" in source_root + + @staticmethod + def _default_meson_env() -> EnvironmentModification: + replacements = { + "LC_ALL": "C.UTF-8", + } + if "DEB_PYTHON_INSTALL_LAYOUT" not in os.environ: + replacements["DEB_PYTHON_INSTALL_LAYOUT"] = "deb" + return EnvironmentModification(replacements=replacements) + + @classmethod + def cmake_generator(cls, target_build_system: Literal["make", "ninja"]) -> str: + cmake_generators = { + "make": "Unix Makefiles", + "ninja": "Ninja", + } + return cmake_generators[target_build_system] + + @staticmethod + def _cross_flags( + context: "BuildContext", + meson_flags: List[str], + ) -> None: + if not context.is_cross_compiling: + return + # Needs a cross-file http://mesonbuild.com/Cross-compilation.html + cross_files_dir = os.path.abspath( + generated_content_dir( + subdir_key="meson-cross-files", + ) + ) + host_arch = context.dpkg_architecture_variables.current_host_arch + cross_file = os.path.join(cross_files_dir, f"meson-cross-file-{host_arch}.conf") + if not os.path.isfile(cross_file): + env = os.environ + if env.get("LC_ALL") != "C.UTF-8": + env = dict(env) + env["LC_ALL"] = "C.UTF-8" + else: + env = None + subprocess.check_call( + [ + "/usr/share/meson/debcrossgen", + f"--arch={host_arch}", + f"-o{cross_file}", + ], + stdout=subprocess.DEVNULL, + env=env, + ) + + meson_flags.append("--cross-file") + meson_flags.append(cross_file) + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + meson_version = Version( + subprocess.check_output( + ["meson", "--version"], + encoding="utf-8", + ).strip() + ) + dpkg_architecture_variables = context.dpkg_architecture_variables + + meson_flags = [ + "--wrap-mode=odownload", + "--buildtype=plain", + "--sysconfdir=/etc", + "--localstatedir=/var", + f"--libdir=lib/{dpkg_architecture_variables.current_host_multiarch}", + "--auto-features=enabled", + ] + if meson_version >= Version("1.2.0"): + # There was a behaviour change in Meson 1.2.0: previously + # byte-compilation wasn't supported, but since 1.2.0 it is on by + # default. We can only use this option to turn it off in versions + # where the option exists. + meson_flags.append("-Dpython.bytecompile=-1") + + self._cross_flags(context, meson_flags) + + if self.configure_args: + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + meson_flags.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + + env_mod = self._default_meson_env() + + self.ensure_build_dir_exists() + source_dir_from_build_dir = self.relative_from_builddir_to_source() + + with self.dump_logs_on_error("meson-logs/meson-log.txt"): + run_build_system_command( + "meson", + "setup", + source_dir_from_build_dir, + *meson_flags, + cwd=self.build_directory, + env_mod=env_mod, + ) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._ninja_support.run_ninja_build(context) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + env_mod = EnvironmentModification( + replacements={ + "MESON_TESTTHREDS": f"{context.parallelization_limit()}", + }, + ).combine(self._default_meson_env()) + with self.dump_logs_on_error("meson-logs/testlog.txt"): + run_build_system_command( + "meson", + "test", + env_mod=env_mod, + cwd=self.build_directory, + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + run_build_system_command( + "meson", + "install", + "--destdir", + dest_dir, + env_mod=self._default_meson_env(), + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + # `debputy` will handle all the cleanup for us by virtue of "out of source build" + assert self.out_of_source_build + + +def _add_qmake_flag(options: List[str], envvar: str, *, include_cppflags: bool) -> None: + value = os.environ.get(envvar) + if value is None: + return + if include_cppflags: + cppflags = os.environ.get("CPPFLAGS") + if cppflags: + value = f"{value} {cppflags}" + + options.append(f"QMAKE_{envvar}_RELEASE={value}") + options.append(f"QMAKE_{envvar}_DEBUG={value}") + + +class ParsedGenericQmakeBuildRuleDefinition( + OptionalInstallDirectly, + OptionalInSourceBuild, + OptionalBuildDirectory, +): + configure_args: NotRequired[List[str]] + + +class AbstractQmakeBuildSystemRule(StepBasedBuildSystemRule): + + __slots__ = ("configure_args", "_make_support") + + def __init__( + self, + parsed_data: "ParsedGenericQmakeBuildRuleDefinition", + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(parsed_data, attribute_path, parser_context) + configure_args = [a for a in parsed_data.get("configure_args", [])] + self.configure_args = configure_args + self._make_support = MakefileSupport.from_build_system(self) + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + return BuildSystemCharacteristics( + out_of_source_builds="supported-and-default", + ) + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + return any(p.name.endswith(".pro") for p in source_root.iterdir) + + @classmethod + def os_mkspec_mapping(cls) -> Mapping[str, str]: + return { + "linux": "linux-g++", + "kfreebsd": "gnukfreebsd-g++", + "hurd": "hurd-g++", + } + + def qmake_command(self) -> str: + raise NotImplementedError + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + + configure_args = [ + "-makefile", + ] + qmake_cmd = context.cross_tool(self.qmake_command()) + + if context.is_cross_compiling: + host_os = context.dpkg_architecture_variables["DEB_HOST_ARCH_OS"] + os2mkspec = self.os_mkspec_mapping() + try: + spec = os2mkspec[host_os] + except KeyError: + _error( + f'Sorry, `debputy` cannot cross build this package for "{host_os}".' + f' Missing a "DEB OS -> qmake -spec <VALUE>" mapping.' + ) + configure_args.append("-spec") + configure_args.append(spec) + + _add_qmake_flag(configure_args, "CFLAGS", include_cppflags=True) + _add_qmake_flag(configure_args, "CXXFLAGS", include_cppflags=True) + _add_qmake_flag(configure_args, "LDFLAGS", include_cppflags=False) + + configure_args.append("QMAKE_STRIP=:") + configure_args.append("PREFIX=/usr") + + if self.configure_args: + substitution = self.substitution + attr_path = self.attribute_path["configure_args"] + configure_args.extend( + substitution.substitute(v, attr_path[i].path) + for i, v in enumerate(self.configure_args) + ) + + self.ensure_build_dir_exists() + if not self.out_of_source_build: + configure_args.append(self.relative_from_builddir_to_source()) + + with self.dump_logs_on_error("config.log"): + run_build_system_command( + qmake_cmd, + *configure_args, + cwd=self.build_directory, + ) + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._make_support.run_make(context) + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + limit = context.parallelization_limit(support_zero_as_unlimited=True) + testsuite_flags = [f"-j{limit}"] if limit else ["-j"] + + if not context.is_terse_build: + testsuite_flags.append("--verbose") + self._make_support.run_first_existing_target_if_any( + context, + # Order is deliberately inverse compared to debhelper (#924052) + ["check", "test"], + f"TESTSUITEFLAGS={' '.join(testsuite_flags)}", + "VERBOSE=1", + ) + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + enable_parallelization = not os.path.lexists(self.build_dir_path("libtool")) + self._make_support.run_first_existing_target_if_any( + context, + ["install"], + f"DESTDIR={dest_dir}", + "AM_UPDATE_INFO_DIR=no", + enable_parallelization=enable_parallelization, + ) + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + if self.out_of_source_build: + return + self._make_support.run_first_existing_target_if_any( + context, + ["distclean", "realclean", "clean"], + ) + + +class QmakeBuildSystemRule(AbstractQmakeBuildSystemRule): + + def qmake_command(self) -> str: + return "qmake" + + +class Qmake6BuildSystemRule(AbstractQmakeBuildSystemRule): + + def qmake_command(self) -> str: + return "qmake6" + + +@debputy_build_system( + "make", + MakefileBuildSystemRule, + auto_detection_shadows_build_systems="debhelper", + online_reference_documentation=reference_documentation( + title="Make Build System", + description=textwrap.dedent( + "" + """\ + Run a plain `make` file with nothing else. + + This build system will attempt to use `make` to leverage instructions + in a makefile (such as, `Makefile` or `GNUMakefile`). + + By default, the makefile build system assumes it should use "in-source" + build semantics. If needed be, an explicit `build-directory` can be + provided if the `Makefile` is not in the source folder but instead in + some other directory. + """ + ), + attributes=[ + documented_attr( + "directory", + textwrap.dedent( + """\ + The directory from which to run make if it is not the source root + + This works like using `make -C DIRECTORY ...` (or `cd DIRECTORY && make ...`). + """ + ), + ), + documented_attr( + "build_target", + textwrap.dedent( + """\ + The target name to use for the "build" step. + + If omitted, `make` will be run without any explicit target leaving it to decide + the default. + """ + ), + ), + documented_attr( + "test_target", + textwrap.dedent( + """\ + The target name to use for the "test" step. + + If omitted, `make check` or `make test` will be used if it looks like `make` + will accept one of those targets. Otherwise, the step will be skipped. + """ + ), + ), + documented_attr( + "install_target", + textwrap.dedent( + """\ + The target name to use for the "install" step. + + If omitted, `make install` will be used if it looks like `make` will accept that target. + Otherwise, the step will be skipped. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedMakeBuildRuleDefinition( + OptionalInstallDirectly, +): + directory: NotRequired[FileSystemExactMatchRule] + build_target: NotRequired[str] + test_target: NotRequired[str] + install_target: NotRequired[str] + + +@debputy_build_system( + "autoconf", + AutoconfBuildSystemRule, + auto_detection_shadows_build_systems=["debhelper", "make"], + online_reference_documentation=reference_documentation( + title="Autoconf Build System", + description=textwrap.dedent( + """\ + Run an autoconf-based build system as the upstream build system. + + This build rule will attempt to use autoreconf to update the `configure` + script before running the `configure` script if needed. Otherwise, it + follows the classic `./configure && make && make install` pattern. + + The build rule uses "out of source" builds by default since it is easier + and more reliable for clean and makes it easier to support multiple + builds (that is, two or more build systems for the same source package). + This is in contract to `debhelper`, which defaults to "in source" builds + for `autoconf`. If you need that behavior, please set + `perform-in-source-build: true`. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `configure` script. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + OptionalInSourceBuild, + OptionalBuildDirectory, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedAutoconfBuildRuleDefinition( + OptionalInstallDirectly, + OptionalInSourceBuild, + OptionalBuildDirectory, +): + configure_args: NotRequired[List[str]] + + +@debputy_build_system( + "cmake", + CMakeBuildSystemRule, + auto_detection_shadows_build_systems=["debhelper", "make"], + online_reference_documentation=reference_documentation( + title="CMake Build System", + description=textwrap.dedent( + """\ + Run an cmake-based build system as the upstream build system. + + The build rule uses "out of source" builds. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `cmake` command. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + OptionalBuildDirectory, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedCMakeBuildRuleDefinition( + OptionalInstallDirectly, + OptionalBuildDirectory, +): + configure_args: NotRequired[List[str]] + target_build_system: Literal["make", "ninja"] + + +@debputy_build_system( + "meson", + MesonBuildSystemRule, + auto_detection_shadows_build_systems=["debhelper", "make"], + online_reference_documentation=reference_documentation( + title="Meson Build System", + description=textwrap.dedent( + """\ + Run an meson-based build system as the upstream build system. + + The build rule uses "out of source" builds. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `meson` command. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + OptionalBuildDirectory, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedMesonBuildRuleDefinition( + OptionalInstallDirectly, + OptionalBuildDirectory, +): + configure_args: NotRequired[List[str]] + + +@debputy_build_system( + "perl-build", + PerlBuildBuildSystemRule, + auto_detection_shadows_build_systems=[ + "debhelper", + "make", + "perl-makemaker", + ], + online_reference_documentation=reference_documentation( + title='Perl "Build.PL" Build System', + description=textwrap.dedent( + """\ + Build using the `Build.PL` Build system used by some Perl packages. + + This build rule will attempt to use the `Build.PL` script to build the + upstream code. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `Build.PL` script. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedPerlBuildBuildRuleDefinition( + OptionalInstallDirectly, +): + configure_args: NotRequired[List[str]] + + +@debputy_build_system( + "debhelper", + DebhelperBuildSystemRule, + online_reference_documentation=reference_documentation( + title="Debhelper Build System", + description=textwrap.dedent( + """\ + Delegate to a debhelper provided build system + + This build rule will attempt to use the `dh_auto_*` tools to build the + upstream code. By default, `dh_auto_*` will use auto-detection to determine + which build system they will use. This can be overridden by the + `dh-build-system` attribute. + """ + ), + attributes=[ + documented_attr( + "dh_build_system", + textwrap.dedent( + """\ + Which debhelper build system to use. This attribute is passed to + the `dh_auto_*` commands as the `-S` parameter, so any value valid + for that will be accepted. + + Note that many debhelper build systems require extra build + dependencies before they can be used. Please consult the documentation + of the relevant debhelper build system for details. + """ + ), + ), + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to underlying configuration command + (via `dh_auto_configure -- <configure-args`). + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + OptionalBuildDirectory, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedDebhelperBuildRuleDefinition( + OptionalInstallDirectly, + OptionalBuildDirectory, +): + configure_args: NotRequired[List[str]] + dh_build_system: NotRequired[str] + + +@debputy_build_system( + "perl-makemaker", + PerlMakeMakerBuildSystemRule, + auto_detection_shadows_build_systems=[ + "debhelper", + "make", + ], + online_reference_documentation=reference_documentation( + title='Perl "MakeMaker" Build System', + description=textwrap.dedent( + """\ + Build using the "MakeMaker" Build system used by some Perl packages. + + This build rule will attempt to use the `Makefile.PL` script to build the + upstream code. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `Makefile.PL` script. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedPerlMakeMakerBuildRuleDefinition( + OptionalInstallDirectly, +): + configure_args: NotRequired[List[str]] + + +@debputy_build_system( + "qmake", + QmakeBuildSystemRule, + auto_detection_shadows_build_systems=[ + "debhelper", + "make", + # Open question, should this shadow "qmake6" and later? + ], + online_reference_documentation=reference_documentation( + title='QT "qmake" Build System', + description=textwrap.dedent( + """\ + Build using the "qmake" by QT. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `qmake` command. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + OptionalInSourceBuild, + OptionalBuildDirectory, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedQmakeBuildRuleDefinition(ParsedGenericQmakeBuildRuleDefinition): + pass + + +@debputy_build_system( + "qmake6", + Qmake6BuildSystemRule, + auto_detection_shadows_build_systems=[ + "debhelper", + "make", + ], + online_reference_documentation=reference_documentation( + title='QT "qmake6" Build System', + description=textwrap.dedent( + """\ + Build using the "qmake6" from the `qmake6` package. This is like the `qmake` system + but is specifically for QT6. + """ + ), + attributes=[ + documented_attr( + "configure_args", + textwrap.dedent( + """\ + Arguments to be passed to the `qmake6` command. + """ + ), + ), + *docs_from( + DebputyParsedContentStandardConditional, + OptionalInstallDirectly, + OptionalInSourceBuild, + OptionalBuildDirectory, + BuildRuleParsedFormat, + ), + ], + ), +) +class ParsedQmake6BuildRuleDefinition(ParsedGenericQmakeBuildRuleDefinition): + pass + + +def _parse_default_environment( + _name: str, + parsed_data: EnvironmentSourceFormat, + attribute_path: AttributePath, + parser_context: ParserContextData, +) -> ManifestProvidedBuildEnvironment: + return ManifestProvidedBuildEnvironment.from_environment_definition( + parsed_data, + attribute_path, + parser_context, + is_default=True, + ) + + +def _parse_build_environments( + _name: str, + parsed_data: List[NamedEnvironmentSourceFormat], + attribute_path: AttributePath, + parser_context: ParserContextData, +) -> List[ManifestProvidedBuildEnvironment]: + return [ + ManifestProvidedBuildEnvironment.from_environment_definition( + value, + attribute_path[idx], + parser_context, + is_default=False, + ) + for idx, value in enumerate(parsed_data) + ] + + +def _handle_build_rules( + _name: str, + parsed_data: List[BuildRule], + _attribute_path: AttributePath, + _parser_context: ParserContextData, +) -> List[BuildRule]: + return parsed_data diff --git a/src/debputy/plugin/debputy/manifest_root_rules.py b/src/debputy/plugin/debputy/manifest_root_rules.py index 1d3b096..f539243 100644 --- a/src/debputy/plugin/debputy/manifest_root_rules.py +++ b/src/debputy/plugin/debputy/manifest_root_rules.py @@ -12,18 +12,22 @@ from debputy._manifest_constants import ( ) from debputy.exceptions import DebputySubstitutionError from debputy.installations import InstallRule -from debputy.manifest_parser.base_types import DebputyParsedContent +from debputy.manifest_parser.tagging_types import DebputyParsedContent from debputy.manifest_parser.exceptions import ManifestParseException from debputy.manifest_parser.parser_data import ParserContextData from debputy.manifest_parser.util import AttributePath from debputy.plugin.api import reference_documentation from debputy.plugin.api.impl import DebputyPluginInitializerProvider -from debputy.plugin.api.impl_types import ( +from debputy.plugin.api.parser_tables import ( OPARSER_MANIFEST_ROOT, OPARSER_MANIFEST_DEFINITIONS, OPARSER_PACKAGES, ) -from debputy.plugin.api.spec import not_integrations, INTEGRATION_MODE_DH_DEBPUTY_RRR +from debputy.plugin.api.spec import ( + not_integrations, + INTEGRATION_MODE_DH_DEBPUTY_RRR, +) +from debputy.plugin.debputy.build_system_rules import register_build_system_rules from debputy.substitution import VariableNameState, SUBST_VAR_RE if TYPE_CHECKING: @@ -166,6 +170,8 @@ def register_manifest_root_rules(api: DebputyPluginInitializerProvider) -> None: nested_in_package_context=True, ) + register_build_system_rules(api) + class ManifestVersionFormat(DebputyParsedContent): manifest_version: ManifestVersion diff --git a/src/debputy/plugin/debputy/private_api.py b/src/debputy/plugin/debputy/private_api.py index d042378..75081a4 100644 --- a/src/debputy/plugin/debputy/private_api.py +++ b/src/debputy/plugin/debputy/private_api.py @@ -43,9 +43,11 @@ from debputy.manifest_conditions import ( BuildProfileMatch, SourceContextArchMatchManifestCondition, ) -from debputy.manifest_parser.base_types import ( +from debputy.manifest_parser.tagging_types import ( DebputyParsedContent, - DebputyParsedContentStandardConditional, + TypeMapping, +) +from debputy.manifest_parser.base_types import ( FileSystemMode, StaticFileSystemOwner, StaticFileSystemGroup, @@ -53,11 +55,12 @@ from debputy.manifest_parser.base_types import ( FileSystemExactMatchRule, FileSystemMatchRule, SymbolicMode, - TypeMapping, OctalMode, FileSystemExactNonDirMatchRule, + BuildEnvironmentDefinition, + DebputyParsedContentStandardConditional, ) -from debputy.manifest_parser.declarative_parser import DebputyParseHint +from debputy.manifest_parser.parse_hints import DebputyParseHint from debputy.manifest_parser.exceptions import ManifestParseException from debputy.manifest_parser.mapper_code import type_mapper_str2package from debputy.manifest_parser.parser_data import ParserContextData @@ -79,7 +82,9 @@ from debputy.plugin.api.spec import ( not_integrations, INTEGRATION_MODE_DH_DEBPUTY_RRR, ) +from debputy.plugin.api.std_docs import docs_from from debputy.plugin.debputy.binary_package_rules import register_binary_package_rules +from debputy.plugin.debputy.build_system_rules import register_build_system_rules from debputy.plugin.debputy.discard_rules import ( _debputy_discard_pyc_files, _debputy_prune_la_files, @@ -587,6 +592,16 @@ def register_type_mappings(api: DebputyPluginInitializerProvider) -> None: ], ), ) + api.register_mapped_type( + TypeMapping( + BuildEnvironmentDefinition, + str, + lambda v, ap, pc: pc.resolve_build_environment(v, ap), + ), + reference_documentation=type_mapping_reference_documentation( + description="Reference to an build environment defined in `build-environments`", + ), + ) def register_service_managers( @@ -897,14 +912,7 @@ def register_install_rules(api: DebputyPluginInitializerProvider) -> None: """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc("generic-install-install"), ), @@ -1193,14 +1201,7 @@ def register_install_rules(api: DebputyPluginInitializerProvider) -> None: """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc( "install-manpages-install-man" @@ -1355,14 +1356,7 @@ def register_install_rules(api: DebputyPluginInitializerProvider) -> None: """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc("generic-install-install"), ), @@ -1408,14 +1402,7 @@ def register_transformation_rules(api: DebputyPluginInitializerProvider) -> None """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc( "move-transformation-rule-move" @@ -1564,14 +1551,7 @@ def register_transformation_rules(api: DebputyPluginInitializerProvider) -> None """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc( "create-symlinks-transformation-rule-create-symlink" @@ -1680,14 +1660,7 @@ def register_transformation_rules(api: DebputyPluginInitializerProvider) -> None """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc( "change-path-ownergroup-or-mode-path-metadata" @@ -1768,14 +1741,7 @@ def register_transformation_rules(api: DebputyPluginInitializerProvider) -> None """ ), ), - documented_attr( - "when", - textwrap.dedent( - """\ - A condition as defined in [Conditional rules]({MANIFEST_FORMAT_DOC}#Conditional rules). - """ - ), - ), + *docs_from(DebputyParsedContentStandardConditional), ], reference_documentation_url=_manifest_format_doc( "create-directories-transformation-rule-directories" diff --git a/src/debputy/plugin/debputy/to_be_api_types.py b/src/debputy/plugin/debputy/to_be_api_types.py new file mode 100644 index 0000000..d7be694 --- /dev/null +++ b/src/debputy/plugin/debputy/to_be_api_types.py @@ -0,0 +1,1039 @@ +import contextlib +import dataclasses +import os.path +import subprocess +from typing import ( + Optional, + FrozenSet, + final, + TYPE_CHECKING, + Union, + Annotated, + List, + NotRequired, + Literal, + Any, + Type, + TypeVar, + Self, + Sequence, + Callable, + Container, + Iterable, + is_typeddict, +) + +from debputy.exceptions import PluginAPIViolationError, PluginInitializationError +from debputy.manifest_conditions import ManifestCondition +from debputy.manifest_parser.base_types import ( + BuildEnvironmentDefinition, + DebputyParsedContentStandardConditional, + FileSystemExactMatchRule, +) +from debputy.manifest_parser.exceptions import ( + ManifestParseException, + ManifestInvalidUserDataException, +) +from debputy.manifest_parser.parse_hints import DebputyParseHint +from debputy.manifest_parser.parser_data import ParserContextData +from debputy.manifest_parser.tagging_types import DebputyDispatchableType +from debputy.manifest_parser.util import AttributePath +from debputy.packages import BinaryPackage +from debputy.plugin.api.spec import ( + ParserDocumentation, + DebputyIntegrationMode, + BuildSystemManifestRuleMetadata, + _DEBPUTY_DISPATCH_METADATA_ATTR_NAME, + VirtualPath, +) +from debputy.plugin.plugin_state import run_in_context_of_plugin +from debputy.substitution import Substitution +from debputy.types import EnvironmentModification +from debputy.util import run_build_system_command, _debug_log, _info, _warn + +if TYPE_CHECKING: + from debputy.build_support.build_context import BuildContext + from debputy.highlevel_manifest import HighLevelManifest + from debputy.plugin.api.impl_types import DIPHandler + + +AT = TypeVar("AT") +BSR = TypeVar("BSR", bound="BuildSystemRule") +BSPF = TypeVar("BSPF", bound="BuildRuleDefinitionBase") + + +@dataclasses.dataclass(slots=True, frozen=True) +class BuildSystemCharacteristics: + out_of_source_builds: Literal[ + "required", + "supported-and-default", + "supported-but-not-default", + "not-supported", + ] + + +class CleanHelper: + def schedule_removal_of_files(self, *args: str) -> None: + """Schedule removal of these files + + This will remove the provided files in bulk. The files are not guaranteed + to be deleted in any particular order. If anything needs urgent removal, + `os.unlink` can be used directly. + + Note: Symlinks will **not** be followed. If a symlink and target must + be deleted, ensure both are passed. + + + :param args: Path names to remove. Each must be removable with + `os.unlink` + """ + raise NotImplementedError + + def schedule_removal_of_directories(self, *args: str) -> None: + """Schedule removal of these directories + + This will remove the provided dirs in bulk. The dirs are not guaranteed + to be deleted in any particular order. If anything needs urgent removal, + then it can be done directly instead of passing it to this method. + + If anything needs urgent removal, then it can be removed immediately. + + :param args: Path names to remove. + """ + raise NotImplementedError + + +class BuildRuleParsedFormat(DebputyParsedContentStandardConditional): + name: NotRequired[str] + for_packages: NotRequired[ + Annotated[ + Union[BinaryPackage, List[BinaryPackage]], + DebputyParseHint.manifest_attribute("for"), + ] + ] + environment: NotRequired[BuildEnvironmentDefinition] + + +class OptionalBuildDirectory(BuildRuleParsedFormat): + build_directory: NotRequired[FileSystemExactMatchRule] + + +class OptionalInSourceBuild(BuildRuleParsedFormat): + perform_in_source_build: NotRequired[bool] + + +class OptionalInstallDirectly(BuildRuleParsedFormat): + install_directly_to_package: NotRequired[bool] + + +BuildSystemDefinition = Union[ + BuildRuleParsedFormat, + OptionalBuildDirectory, + OptionalInSourceBuild, + OptionalInstallDirectly, +] + + +class BuildRule(DebputyDispatchableType): + __slots__ = ( + "_auto_generated_stem", + "_name", + "_for_packages", + "_manifest_condition", + "_attribute_path", + "_environment", + "_substitution", + ) + + def __init__( + self, + attributes: BuildRuleParsedFormat, + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__() + + self._name = attributes.get("name") + for_packages = attributes.get("for_packages") + + if for_packages is None: + if isinstance(parser_context, ParserContextData): + all_binaries = parser_context.binary_packages.values() + else: + all_binaries = parser_context.all_packages + self._for_packages = frozenset(all_binaries) + else: + self._for_packages = frozenset( + for_packages if isinstance(for_packages, list) else [for_packages] + ) + self._manifest_condition = attributes.get("when") + self._attribute_path = attribute_path + self._substitution = parser_context.substitution + self._auto_generated_stem: Optional[str] = None + environment = attributes.get("environment") + if environment is None: + assert isinstance(parser_context, ParserContextData) + self._environment = parser_context.resolve_build_environment( + None, + attribute_path, + ) + else: + self._environment = environment + + @final + @property + def name(self) -> Optional[str]: + return self._name + + @final + @property + def attribute_path(self) -> AttributePath: + return self._attribute_path + + @final + @property + def manifest_condition(self) -> Optional[ManifestCondition]: + return self._manifest_condition + + @final + @property + def for_packages(self) -> FrozenSet[BinaryPackage]: + return self._for_packages + + @final + @property + def substitution(self) -> Substitution: + return self._substitution + + @final + @property + def environment(self) -> BuildEnvironmentDefinition: + return self._environment + + @final + @property + def auto_generated_stem(self) -> str: + stem = self._auto_generated_stem + if stem is None: + raise AssertionError( + "The auto-generated-stem is not available at this time" + ) + return stem + + @final + @auto_generated_stem.setter + def auto_generated_stem(self, value: str) -> None: + if self._auto_generated_stem is not None: + raise AssertionError("The auto-generated-stem should only be set once") + assert value is not None + self._auto_generated_stem = value + + @final + def run_build( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + run_in_context_of_plugin( + self._debputy_plugin, + self.perform_build, + context, + manifest, + **kwargs, + ) + + def perform_build( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + raise NotImplementedError + + @property + def is_buildsystem(self) -> bool: + return False + + @property + def name_or_tag(self) -> str: + name = self.name + if name is None: + return self.auto_generated_stem + return name + + +def _is_type_or_none(v: Optional[Any], expected_type: Type[AT]) -> Optional[AT]: + if isinstance(v, expected_type): + return v + return None + + +class BuildSystemRule(BuildRule): + + __slots__ = ( + "_build_directory", + "source_directory", + "install_directly_to_package", + "perform_in_source_build", + ) + + def __init__( + self, + attributes: BuildSystemDefinition, + attribute_path: AttributePath, + parser_context: Union[ParserContextData, "HighLevelManifest"], + ) -> None: + super().__init__(attributes, attribute_path, parser_context) + build_directory = _is_type_or_none( + attributes.get("build_directory"), FileSystemExactMatchRule + ) + if build_directory is not None: + self._build_directory = build_directory.match_rule.path + else: + self._build_directory = None + self.source_directory = "." + self.install_directly_to_package = False + self.perform_in_source_build = _is_type_or_none( + attributes.get("perform_in_source_build"), bool + ) + install_directly_to_package = _is_type_or_none( + attributes.get("install_directly_to_package"), bool + ) + if install_directly_to_package is None: + self.install_directly_to_package = len(self.for_packages) == 1 + elif install_directly_to_package and len(self.for_packages) > 1: + idtp_path = attribute_path["install_directly_to_package"].path + raise ManifestParseException( + f'The attribute "install-directly-to-package" ({idtp_path}) cannot' + " be true when the build system applies to multiple packages." + ) + else: + self.install_directly_to_package = install_directly_to_package + + @classmethod + def auto_detect_build_system( + cls, + source_root: VirtualPath, + *args, + **kwargs, + ) -> bool: + """Check if the build system apply automatically. + + This class method is called when the manifest does not declare any build rules at + all. + + :param source_root: The source root (the directory containing `debian/`). Usually, + the detection code would look at this for files related to the upstream build system. + :param args: For future compat, new arguments might appear as positional arguments. + :param kwargs: For future compat, new arguments might appear as keyword argument. + :return: True if the build system can be used, False when it would not be useful + to use the build system (at least with all defaults). + Note: Be sure to use proper `bool` return values. The calling code does an + `isinstance` check to ensure that the version of `debputy` supports the + auto-detector (in case the return type is ever expanded in the future). + """ + return False + + @property + def out_of_source_build(self) -> bool: + build_directory = self.build_directory + return build_directory != self.source_directory + + @property + def build_directory(self) -> str: + directory = self._build_directory + if directory is None: + return self.source_directory + return directory + + @contextlib.contextmanager + def dump_logs_on_error(self, *logs: str) -> None: + """Context manager that will dump logs to stdout on error + + :param logs: The logs to be dumped. Relative path names are assumed to be relative to + the build directory. + """ + try: + yield + except (Exception, KeyboardInterrupt, SystemExit): + _warn( + "Error occurred, attempting to provide relevant logs as requested by the build system provider" + ) + found_any = False + for log in logs: + if not os.path.isabs(log): + log = self.build_dir_path(log) + if not os.path.isfile(log): + _info( + f'Would have pushed "{log}" to stdout, but it does not exist.' + ) + continue + subprocess.run(["tail", "-v", "-n", "+0", log]) + found_any = True + if not found_any: + _warn( + f"None of the logs provided were available (relative to build directory): {', '.join(logs)}" + ) + raise + + @final + def run_clean( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: CleanHelper, + **kwargs, + ) -> None: + run_in_context_of_plugin( + self._debputy_plugin, + self.perform_clean, + context, + manifest, + clean_helper, + **kwargs, + ) + + def perform_clean( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: CleanHelper, + **kwargs, + ) -> None: + raise NotImplementedError + + def ensure_build_dir_exists(self) -> None: + build_dir = self.build_directory + source_dir = self.source_directory + if build_dir == source_dir: + return + os.makedirs(build_dir, mode=0o755, exist_ok=True) + + def build_dir_path(self, /, path: str = "") -> str: + build_dir = self.build_directory + if path == "": + return build_dir + return os.path.join(build_dir, path) + + def relative_from_builddir_to_source( + self, + path_in_source_dir: Optional[str] = None, + ) -> str: + build_dir = self.build_directory + source_dir = self.source_directory + if build_dir == source_dir: + return path_in_source_dir + return os.path.relpath(os.path.join(source_dir, path_in_source_dir), build_dir) + + @final + @property + def is_buildsystem(self) -> bool: + return True + + +class StepBasedBuildSystemRule(BuildSystemRule): + + @classmethod + def characteristics(cls) -> BuildSystemCharacteristics: + raise NotImplementedError + + @final + def perform_clean( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: CleanHelper, + **kwargs, + ) -> None: + self._check_characteristics() + self.before_first_impl_step(stage="clean") + self.clean_impl(context, manifest, clean_helper, **kwargs) + if self.out_of_source_build: + build_directory = self.build_directory + assert build_directory is not None + if os.path.lexists(build_directory): + clean_helper.schedule_removal_of_directories(build_directory) + dest_dir = self.resolve_dest_dir() + if not isinstance(dest_dir, BinaryPackage): + clean_helper.schedule_removal_of_directories(dest_dir) + + @final + def perform_build( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + self._check_characteristics() + self.before_first_impl_step(stage="build") + self.configure_impl(context, manifest, **kwargs) + self.build_impl(context, manifest, **kwargs) + if context.should_run_tests: + self.test_impl(context, manifest, **kwargs) + dest_dir = self.resolve_dest_dir() + if isinstance(dest_dir, BinaryPackage): + dest_dir = f"debian/{dest_dir.name}" + # Make it absolute for everyone (that worked for debhelper). + # At least autoconf's "make install" requires an absolute path, so making is + # relative would have at least one known issue. + abs_dest_dir = os.path.abspath(dest_dir) + self.install_impl(context, manifest, abs_dest_dir, **kwargs) + + def before_first_impl_step( + self, + /, + stage: Literal["build", "clean"], + **kwargs, + ) -> None: + """Called before any `*_impl` method is called. + + This can be used to validate input against data that is not available statically + (that is, it will be checked during build but not in static checks). An example + is that the `debhelper` build system uses this to validate the provided `dh-build-system` + to ensure that `debhelper` knows about the build system. This check cannot be done + statically since the build system is only required to be available in a chroot build + and not on the host system. + + The method can also be used to compute common state for all the `*_impl` methods that + is awkward to do in `__init__`. Note there is no data sharing between the different + stages. This has to do with how `debputy` will be called (usually `clean` followed by + a source package assembly in `dpkg` and then `build`). + + The check is done both on build and on clean before the relevant implementation methods + are invoked. + + Any exception will abort the build. Prefer to raise ManifestInvalidUserDataException + exceptions for issues related to incorrect data. + + The method is not invoked if the steps are skipped, which can happen with build profiles + or arch:any vs. arch:all builds. + + :param stage: A discriminator variable to determine which kind of steps will be invoked + after this method returns. For state initialization, this can be useful if the state + is somewhat expensive and not needed for `clean`. + """ + pass + + def configure_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + """Called to handle the "configure" and "build" part of the build + + This is basically a mix of `dh_auto_configure` and `dh_auto_build` from `debhelper`. + If the upstream build also runs test as a part of the build, this method should + check `context.should_run_tests` and pass the relevant flags to disable tests when + `context.should_run_tests` is false. + """ + raise NotImplementedError + + def build_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + """Called to handle the "configure" and "build" part of the build + + This is basically a mix of `dh_auto_configure` and `dh_auto_build` from `debhelper`. + If the upstream build also runs test as a part of the build, this method should + check `context.should_run_tests` and pass the relevant flags to disable tests when + `context.should_run_tests` is false. + """ + raise NotImplementedError + + def test_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + **kwargs, + ) -> None: + """Called to handle the "test" part of the build + + This is basically `dh_auto_test` from `debhelper`. + + Note: This will be skipped when `context.should_run_tests` is False. Therefore, the + method can assume that when invoked then tests must be run. + + It is always run after `configure_and_build_impl`. + """ + raise NotImplementedError + + def install_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + dest_dir: str, + **kwargs, + ) -> None: + """Called to handle the "install" part of the build + + This is basically `dh_auto_install` from `debhelper`. + + The `dest_dir` attribute is what the upstream should install its data into. It + follows the `DESTDIR` convention from autoconf/make. The `dest_dir` should not + be second-guessed since `debputy` will provide automatically as a search path + for installation rules when relevant. + + It is always run after `configure_and_build_impl` and, if relevant, `test_impl`. + """ + raise NotImplementedError + + def clean_impl( + self, + context: "BuildContext", + manifest: "HighLevelManifest", + clean_helper: "CleanHelper", + **kwargs, + ) -> None: + """Called to handle the "clean" part of the build + + This is basically `dh_auto_clean` from `debhelper`. + + For out-of-source builds, `debputy` will remove the build directory for you + if it exists (when this method returns). This method is only "in-source" cleaning + or for "dirty" state left outside the designated build directory. + + Note that state *cannot* be shared between `clean` and other steps due to limitations + of how the Debian build system works in general. + """ + raise NotImplementedError + + def _check_characteristics(self) -> None: + characteristics = self.characteristics() + + _debug_log(f"Characteristics for {self.name_or_tag} {self.__class__.__name__} ") + + if self.out_of_source_build and self.perform_in_source_build: + raise ManifestInvalidUserDataException( + f"Cannot use 'build-directory' with 'perform-in-source-build' at {self.attribute_path.path}" + ) + if ( + characteristics.out_of_source_builds == "required" + and self.perform_in_source_build + ): + path = self.attribute_path["perform_in_source_build"].path_key_lc + + # FIXME: How do I determine the faulty plugin from here. + raise PluginAPIViolationError( + f"The build system {self.__class__.__qualname__} had an perform-in-source-build attribute, but claims" + f" it requires out of source builds. Please file a bug against the provider asking them not to use" + f' "{OptionalInSourceBuild.__name__}" as base for their build system definition or tweak' + f" the characteristics of the build system as the current combination is inconsistent." + f" The offending definition is at {path}." + ) + + if ( + characteristics.out_of_source_builds + in ("required", "supported-and-default") + and not self.out_of_source_build + ): + + if not self.perform_in_source_build: + self._build_directory = self._pick_build_dir() + else: + assert characteristics.out_of_source_builds != "required" + elif ( + characteristics.out_of_source_builds == "not-supported" + and self.out_of_source_build + ): + path = self.attribute_path["build_directory"].path_key_lc + + # FIXME: How do I determine the faulty plugin from here. + raise PluginAPIViolationError( + f"The build system {self.__class__.__qualname__} had a build-directory attribute, but claims it does" + f" not support out of source builds. Please file a bug against the provider asking them not to use" + f' "{OptionalBuildDirectory.__name__}" as base for their build system definition or tweak' + f" the characteristics of the build system as the current combination is inconsistent." + f" The offending definition is at {path}." + ) + + def _pick_build_dir(self) -> str: + tag = self.name if self.name is not None else self.auto_generated_stem + if tag == "": + return "_build" + return f"_build-{tag}" + + @final + def resolve_dest_dir(self) -> Union[str, BinaryPackage]: + auto_generated_stem = self.auto_generated_stem + if self.install_directly_to_package: + assert len(self.for_packages) == 1 + return next(iter(self.for_packages)) + if auto_generated_stem == "": + return "debian/tmp" + return f"debian/tmp-{auto_generated_stem}" + + +# Using the same logic as debhelper for the same reasons. +def _make_target_exists(make_cmd: str, target: str, *, directory: str = ".") -> bool: + cmd = [ + make_cmd, + "-s", + "-n", + "--no-print-directory", + ] + if directory and directory != ".": + cmd.append("-C") + cmd.append(directory) + cmd.append(target) + env = dict(os.environ) + env["LC_ALL"] = "C.UTF-8" + try: + res = subprocess.run( + cmd, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + restore_signals=True, + ) + except FileNotFoundError: + return False + + options = ( + f"*** No rule to make target '{target}", + f"*** No rule to make target `{target}", + ) + + stdout = res.stdout.decode("utf-8") + return not any(o in stdout for o in options) + + +def _find_first_existing_make_target( + make_cmd: str, + targets: Sequence[str], + *, + directory: str = ".", +) -> Optional[str]: + for target in targets: + if _make_target_exists(make_cmd, target, directory=directory): + return target + return None + + +_UNSET = object() + + +class NinjaBuildSupport: + __slots__ = ("_provided_ninja_program", "_build_system_rule") + + def __init__( + self, + provided_ninja_program: str, + build_system_rule: BuildSystemRule, + ) -> None: + self._provided_ninja_program = provided_ninja_program + self._build_system_rule = build_system_rule + + @classmethod + def from_build_system( + cls, + build_system: BuildSystemRule, + *, + ninja_program: Optional[str] = None, + ) -> Self: + if ninja_program is None: + ninja_program = "ninja" + return cls(ninja_program, build_system) + + @property + def _directory(self) -> str: + return self._build_system_rule.build_directory + + def _pick_directory( + self, arg: Union[Optional[str], _UNSET] = _UNSET + ) -> Optional[str]: + if arg is _UNSET: + return self._directory + return arg + + def run_ninja_build( + self, + build_context: "BuildContext", + *ninja_args: str, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + enable_parallelization: bool = True, + ) -> None: + extra_ninja_args = [] + if not build_context.is_terse_build: + extra_ninja_args.append("-v") + self._run_ninja( + build_context, + *extra_ninja_args, + *ninja_args, + env_mod=env_mod, + directory=directory, + enable_parallelization=enable_parallelization, + ) + + def run_ninja_test( + self, + build_context: "BuildContext", + *ninja_args: str, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + enable_parallelization: bool = True, + ) -> None: + self._run_ninja( + build_context, + "test", + *ninja_args, + env_mod=env_mod, + directory=directory, + enable_parallelization=enable_parallelization, + ) + + def run_ninja_install( + self, + build_context: "BuildContext", + dest_dir: str, + *ninja_args: str, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + # debhelper never had parallel installs, so we do not have it either for now. + enable_parallelization: bool = False, + ) -> None: + install_env_mod = EnvironmentModification( + replacements={ + "DESTDIR": dest_dir, + } + ) + if env_mod is not None: + install_env_mod = install_env_mod.combine(env_mod) + self._run_ninja( + build_context, + "install", + *ninja_args, + directory=directory, + env_mod=install_env_mod, + enable_parallelization=enable_parallelization, + ) + + def run_ninja_clean( + self, + build_context: "BuildContext", + *ninja_args: str, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + enable_parallelization: bool = True, + ) -> None: + self._run_ninja( + build_context, + "clean", + *ninja_args, + env_mod=env_mod, + directory=directory, + enable_parallelization=enable_parallelization, + ) + + def _run_ninja( + self, + build_context: "BuildContext", + *ninja_args: str, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + enable_parallelization: bool = True, + ) -> None: + extra_ninja_args = [] + limit = ( + build_context.parallelization_limit(support_zero_as_unlimited=True) + if enable_parallelization + else 1 + ) + extra_ninja_args.append(f"-j{limit}") + ninja_env_mod = EnvironmentModification( + replacements={ + "LC_ALL": "C.UTF-8", + } + ) + if env_mod is not None: + ninja_env_mod = ninja_env_mod.combine(env_mod) + run_build_system_command( + self._provided_ninja_program, + *extra_ninja_args, + *ninja_args, + cwd=self._pick_directory(directory), + env_mod=ninja_env_mod, + ) + + +class MakefileSupport: + + __slots__ = ("_provided_make_program", "_build_system_rule") + + def __init__( + self, + make_program: str, + build_system_rule: BuildSystemRule, + ) -> None: + self._provided_make_program = make_program + self._build_system_rule = build_system_rule + + @classmethod + def from_build_system( + cls, + build_system: BuildSystemRule, + *, + make_program: Optional[str] = None, + ) -> Self: + if make_program is None: + make_program = os.environ.get("MAKE", "make") + return cls(make_program, build_system) + + @property + def _directory(self) -> str: + return self._build_system_rule.build_directory + + @property + def _make_program(self) -> str: + make_program = self._provided_make_program + if self._provided_make_program is None: + return os.environ.get("MAKE", "make") + return make_program + + def _pick_directory( + self, arg: Union[Optional[str], _UNSET] = _UNSET + ) -> Optional[str]: + if arg is _UNSET: + return self._directory + return arg + + def find_first_existing_make_target( + self, + targets: Sequence[str], + *, + directory: Union[Optional[str], _UNSET] = _UNSET, + ) -> Optional[str]: + for target in targets: + if self.make_target_exists(target, directory=directory): + return target + return None + + def make_target_exists( + self, + target: str, + *, + directory: Union[Optional[str], _UNSET] = _UNSET, + ) -> bool: + return _make_target_exists( + self._make_program, + target, + directory=self._pick_directory(directory), + ) + + def run_first_existing_target_if_any( + self, + build_context: "BuildContext", + targets: Sequence[str], + *make_args: str, + enable_parallelization: bool = True, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + ) -> bool: + target = self.find_first_existing_make_target(targets, directory=directory) + if target is None: + return False + + self.run_make( + build_context, + target, + *make_args, + enable_parallelization=enable_parallelization, + directory=directory, + env_mod=env_mod, + ) + return True + + def run_make( + self, + build_context: "BuildContext", + *make_args: str, + enable_parallelization: bool = True, + directory: Union[Optional[str], _UNSET] = _UNSET, + env_mod: Optional[EnvironmentModification] = None, + ) -> None: + limit = ( + build_context.parallelization_limit(support_zero_as_unlimited=True) + if enable_parallelization + else 1 + ) + extra_make_args = [f"-j{limit}"] if limit else ["-j"] + run_build_system_command( + self._make_program, + *extra_make_args, + *make_args, + cwd=self._pick_directory(directory), + env_mod=env_mod, + ) + + +def debputy_build_system( + # For future self: Before you get ideas about making manifest_keyword accept a list, + # remember it has consequences for shadowing_build_systems_when_active. + manifest_keyword: str, + provider: Type[BSR], + *, + expected_debputy_integration_mode: Optional[ + Container[DebputyIntegrationMode] + ] = None, + auto_detection_shadows_build_systems: Optional[ + Union[str, Iterable[str]] + ] = frozenset(), + online_reference_documentation: Optional[ParserDocumentation] = None, + apply_standard_attribute_documentation: bool = False, + source_format: Optional[Any] = None, +) -> Callable[[Type[BSPF]], Type[BSPF]]: + if not isinstance(provider, type) or not issubclass(provider, BuildSystemRule): + raise PluginInitializationError( + f"The provider for @{debputy_build_system.__name__} must be subclass of {BuildSystemRule.__name__}goes on the TypedDict that defines the parsed" + f" variant of the manifest definition. Not the build system implementation class." + ) + + def _constructor_wrapper( + _rule_used: str, + *args, + **kwargs, + ) -> BSR: + return provider(*args, **kwargs) + + if isinstance(auto_detection_shadows_build_systems, str): + shadows = frozenset([auto_detection_shadows_build_systems]) + else: + shadows = frozenset(auto_detection_shadows_build_systems) + + metadata = BuildSystemManifestRuleMetadata( + (manifest_keyword,), + BuildRule, + _constructor_wrapper, + expected_debputy_integration_mode=expected_debputy_integration_mode, + source_format=source_format, + online_reference_documentation=online_reference_documentation, + apply_standard_attribute_documentation=apply_standard_attribute_documentation, + auto_detection_shadow_build_systems=shadows, + build_system_impl=provider, + ) + + def _decorator_impl(pf_cls: Type[BSPF]) -> Type[BSPF]: + if isinstance(pf_cls, type) and issubclass(pf_cls, BuildSystemRule): + raise PluginInitializationError( + f"The @{debputy_build_system.__name__} annotation goes on the TypedDict that defines the parsed" + f" variant of the manifest definition. Not the build system implementation class." + ) + + # TODO: In python3.12 we can check more than just `is_typeddict`. In python3.11, woe is us and + # is_typeddict is the only thing that reliably works (cpython#103699) + if not is_typeddict(pf_cls): + raise PluginInitializationError( + f"Expected annotated class to be a subclass of {BuildRuleParsedFormat.__name__}," + f" but got {pf_cls.__name__} instead" + ) + + setattr(pf_cls, _DEBPUTY_DISPATCH_METADATA_ATTR_NAME, metadata) + return pf_cls + + return _decorator_impl diff --git a/src/debputy/plugin/plugin_state.py b/src/debputy/plugin/plugin_state.py new file mode 100644 index 0000000..ef4dabb --- /dev/null +++ b/src/debputy/plugin/plugin_state.py @@ -0,0 +1,113 @@ +import contextvars +import functools +import inspect +from contextvars import ContextVar +from typing import Optional, Callable, ParamSpec, TypeVar, NoReturn, Union + +from debputy.exceptions import ( + UnhandledOrUnexpectedErrorFromPluginError, + DebputyRuntimeError, +) +from debputy.util import _debug_log, _is_debug_log_enabled + +_current_debputy_plugin_cxt_var: ContextVar[Optional[str]] = ContextVar( + "current_debputy_plugin", + default=None, +) + +P = ParamSpec("P") +R = TypeVar("R") + + +def current_debputy_plugin_if_present() -> Optional[str]: + return _current_debputy_plugin_cxt_var.get() + + +def current_debputy_plugin_required() -> str: + v = current_debputy_plugin_if_present() + if v is None: + raise AssertionError( + "current_debputy_plugin_required() was called, but no plugin was set." + ) + return v + + +def wrap_plugin_code( + plugin_name: str, + func: Callable[P, R], + *, + non_debputy_exception_handling: Union[bool, Callable[[Exception], NoReturn]] = True, +) -> Callable[P, R]: + if isinstance(non_debputy_exception_handling, bool): + + runner = run_in_context_of_plugin + if non_debputy_exception_handling: + runner = run_in_context_of_plugin_wrap_errors + + def _wrapper(*args: P.args, **kwargs: P.kwargs) -> None: + return runner(plugin_name, func, *args, **kwargs) + + functools.update_wrapper(_wrapper, func) + return _wrapper + + def _wrapper(*args: P.args, **kwargs: P.kwargs) -> None: + try: + return run_in_context_of_plugin(plugin_name, func, *args, **kwargs) + except DebputyRuntimeError: + raise + except Exception as e: + non_debputy_exception_handling(e) + + functools.update_wrapper(_wrapper, func) + return _wrapper + + +def run_in_context_of_plugin( + plugin: str, + func: Callable[P, R], + *args: P.args, + **kwargs: P.kwargs, +) -> R: + context = contextvars.copy_context() + if _is_debug_log_enabled(): + call_stack = inspect.stack() + caller: str = "[N/A]" + for frame in call_stack: + if frame.filename != __file__: + try: + fname = frame.frame.f_code.co_qualname + except AttributeError: + fname = None + if fname is None: + fname = frame.function + caller = f"{frame.filename}:{frame.lineno} ({fname})" + break + # Do not keep the reference longer than necessary + del call_stack + _debug_log( + f"Switching plugin context to {plugin} at {caller} (from context: {current_debputy_plugin_if_present()})" + ) + # Wish we could just do a regular set without wrapping it in `context.run` + context.run(_current_debputy_plugin_cxt_var.set, plugin) + return context.run(func, *args, **kwargs) + + +def run_in_context_of_plugin_wrap_errors( + plugin: str, + func: Callable[P, R], + *args: P.args, + **kwargs: P.kwargs, +) -> R: + try: + return run_in_context_of_plugin(plugin, func, *args, **kwargs) + except DebputyRuntimeError: + raise + except Exception as e: + if plugin != "debputy": + raise UnhandledOrUnexpectedErrorFromPluginError( + f"{func.__qualname__} from the plugin {plugin} raised exception that was not expected here." + ) from e + else: + raise AssertionError( + "Bug in the `debputy` plugin: Unhandled exception." + ) from e diff --git a/src/debputy/transformation_rules.py b/src/debputy/transformation_rules.py index c7f8a2a..6e96c64 100644 --- a/src/debputy/transformation_rules.py +++ b/src/debputy/transformation_rules.py @@ -11,6 +11,7 @@ from typing import ( Dict, TypeVar, cast, + final, ) from debputy.exceptions import ( @@ -27,12 +28,15 @@ from debputy.manifest_parser.base_types import ( FileSystemMode, StaticFileSystemOwner, StaticFileSystemGroup, - DebputyDispatchableType, ) +from debputy.manifest_parser.tagging_types import DebputyDispatchableType from debputy.manifest_parser.util import AttributePath from debputy.path_matcher import MatchRule from debputy.plugin.api import VirtualPath from debputy.plugin.debputy.types import DebputyCapability +from debputy.plugin.plugin_state import ( + run_in_context_of_plugin_wrap_errors, +) from debputy.util import _warn @@ -59,10 +63,26 @@ class PreProvidedExclusion: class TransformationRule(DebputyDispatchableType): + __slots__ = () + @final + def run_transform_file_system( + self, + fs_root: FSPath, + condition_context: ConditionContext, + ) -> None: + run_in_context_of_plugin_wrap_errors( + self._debputy_plugin, + self.transform_file_system, + fs_root, + condition_context, + ) + def transform_file_system( - self, fs_root: FSPath, condition_context: ConditionContext + self, + fs_root: FSPath, + condition_context: ConditionContext, ) -> None: raise NotImplementedError @@ -134,6 +154,7 @@ class RemoveTransformationRule(TransformationRule): keep_empty_parent_dirs: bool, definition_source: AttributePath, ) -> None: + super().__init__() self._match_rules = match_rules self._keep_empty_parent_dirs = keep_empty_parent_dirs self._definition_source = definition_source.path @@ -180,6 +201,7 @@ class MoveTransformationRule(TransformationRule): definition_source: AttributePath, condition: Optional[ManifestCondition], ) -> None: + super().__init__() self._match_rule = match_rule self._dest_path = dest_path self._dest_is_dir = dest_is_dir @@ -283,6 +305,7 @@ class CreateSymlinkPathTransformationRule(TransformationRule): definition_source: AttributePath, condition: Optional[ManifestCondition], ) -> None: + super().__init__() self._link_target = link_target self._link_dest = link_dest self._replacement_rule = replacement_rule @@ -550,6 +573,9 @@ class ModeNormalizationTransformationRule(TransformationRule): self, normalizations: Sequence[Tuple[MatchRule, FileSystemMode]], ) -> None: + # A bit of a hack since it is initialized outside `debputy`. It probably should not + # be a "TransformationRule" (hindsight and all) + run_in_context_of_plugin_wrap_errors("debputy", super().__init__) self._normalizations = normalizations def transform_file_system( @@ -575,6 +601,12 @@ class ModeNormalizationTransformationRule(TransformationRule): class NormalizeShebangLineTransformation(TransformationRule): + + def __init__(self) -> None: + # A bit of a hack since it is initialized outside `debputy`. It probably should not + # be a "TransformationRule" (hindsight and all) + run_in_context_of_plugin_wrap_errors("debputy", super().__init__) + def transform_file_system( self, fs_root: VirtualPath, diff --git a/src/debputy/types.py b/src/debputy/types.py index 05e68c9..dc3cbd3 100644 --- a/src/debputy/types.py +++ b/src/debputy/types.py @@ -1,9 +1,138 @@ -from typing import TypeVar, TYPE_CHECKING +import dataclasses +from typing import ( + TypeVar, + TYPE_CHECKING, + Sequence, + Tuple, + Mapping, + Dict, + Optional, + TypedDict, + NotRequired, + List, + MutableMapping, +) if TYPE_CHECKING: from debputy.plugin.api import VirtualPath from debputy.filesystem_scan import FSPath + VP = TypeVar("VP", VirtualPath, FSPath) + S = TypeVar("S", str, bytes) +else: + VP = TypeVar("VP", "VirtualPath", "FSPath") + S = TypeVar("S", str, bytes) -VP = TypeVar("VP", "VirtualPath", "FSPath") -S = TypeVar("S", str, bytes) + +class EnvironmentModificationSerialized(TypedDict): + replacements: NotRequired[Dict[str, str]] + removals: NotRequired[List[str]] + + +@dataclasses.dataclass(slots=True, frozen=True) +class EnvironmentModification: + replacements: Sequence[Tuple[str, str]] = tuple() + removals: Sequence[str] = tuple() + + @staticmethod + def from_serialized_format( + serial_form: EnvironmentModificationSerialized, + ) -> "EnvironmentModification": + replacements_raw = serial_form.get("replacements") + if replacements_raw is not None: + replacements = tuple((k, v) for k, v in replacements_raw.items()) + else: + replacements = tuple() + return EnvironmentModification( + replacements=replacements, removals=serial_form.get("removals", tuple()) + ) + + def __bool__(self) -> bool: + return not self.removals and not self.replacements + + def combine( + self, other: "Optional[EnvironmentModification]" + ) -> "EnvironmentModification": + if not other: + return self + existing_replacements = {k: v for k, v in self.replacements} + extra_replacements = { + k: v + for k, v in other.replacements + if k not in existing_replacements or existing_replacements[k] != v + } + seen_removals = set(self.removals) + extra_removals = [r for r in other.removals if r not in seen_removals] + + if not extra_replacements and isinstance(self.replacements, tuple): + new_replacements = self.replacements + else: + new_replacements = [] + for k, v in existing_replacements: + if k not in extra_replacements: + new_replacements.append((k, v)) + + for k, v in other.replacements: + if k in extra_replacements: + new_replacements.append((k, v)) + + new_replacements = tuple(new_replacements) + + if not extra_removals and isinstance(self.removals, tuple): + new_removals = self.removals + else: + new_removals = list(self.removals) + new_removals.extend(extra_removals) + new_removals = tuple(new_removals) + + if self.replacements is new_replacements and self.removals is new_removals: + return self + + return EnvironmentModification( + new_replacements, + new_removals, + ) + + def serialize(self) -> EnvironmentModificationSerialized: + serial_form = {} + replacements = self.replacements + if replacements: + serial_form["replacements"] = {k: v for k, v in replacements} + removals = self.removals + if removals: + serial_form["removals"] = list(removals) + return serial_form + + def update_inplace(self, env: MutableMapping[str, str]) -> None: + for k, v in self.replacements: + existing_value = env.get(k) + if v == existing_value: + continue + env[k] = v + + for k in self.removals: + if k not in env: + continue + del env[k] + + def compute_env(self, base_env: Mapping[str, str]) -> Mapping[str, str]: + updated_env: Optional[Dict[str, str]] = None + for k, v in self.replacements: + existing_value = base_env.get(k) + if v == existing_value: + continue + + if updated_env is None: + updated_env = dict(base_env) + updated_env[k] = v + + for k in self.removals: + if k not in base_env: + continue + if updated_env is None: + updated_env = dict(base_env) + del updated_env[k] + + if updated_env is not None: + return updated_env + return base_env diff --git a/src/debputy/util.py b/src/debputy/util.py index ebd38c2..911e6fa 100644 --- a/src/debputy/util.py +++ b/src/debputy/util.py @@ -34,7 +34,7 @@ from debian.deb822 import Deb822 from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable from debputy.exceptions import DebputySubstitutionError - +from debputy.types import EnvironmentModification try: from Levenshtein import distance @@ -105,7 +105,12 @@ _PROFILE_GROUP_SPLIT = re.compile(r">\s+<") _DEFAULT_LOGGER: Optional[logging.Logger] = None _STDOUT_HANDLER: Optional[logging.StreamHandler[Any]] = None _STDERR_HANDLER: Optional[logging.StreamHandler[Any]] = None -PRINT_COMMAND = logging.INFO + 5 +PRINT_COMMAND = logging.INFO + 3 +PRINT_BUILD_SYSTEM_COMMAND = PRINT_COMMAND + 3 + +# Map them back to `INFO`. The names must be unique so the prefix is stripped. +logging.addLevelName(PRINT_COMMAND, "__INFO") +logging.addLevelName(PRINT_BUILD_SYSTEM_COMMAND, "_INFO") def assume_not_none(x: Optional[T]) -> T: @@ -116,6 +121,13 @@ def assume_not_none(x: Optional[T]) -> T: return x +def _non_verbose_info(msg: str) -> None: + global _DEFAULT_LOGGER + logger = _DEFAULT_LOGGER + if logger is not None: + logger.log(PRINT_BUILD_SYSTEM_COMMAND, msg) + + def _info(msg: str) -> None: global _DEFAULT_LOGGER logger = _DEFAULT_LOGGER @@ -124,6 +136,20 @@ def _info(msg: str) -> None: # No fallback print for info +def _is_debug_log_enabled() -> bool: + global _DEFAULT_LOGGER + logger = _DEFAULT_LOGGER + return logger is not None and logger.isEnabledFor(logging.DEBUG) + + +def _debug_log(msg: str) -> None: + global _DEFAULT_LOGGER + logger = _DEFAULT_LOGGER + if logger: + logger.debug(msg) + # No fallback print for info + + def _error(msg: str, *, prog: Optional[str] = None) -> "NoReturn": global _DEFAULT_LOGGER logger = _DEFAULT_LOGGER @@ -226,9 +252,88 @@ def escape_shell(*args: str) -> str: return " ".join(_escape_shell_word(w) for w in args) -def print_command(*args: str, print_at_log_level: int = PRINT_COMMAND) -> None: - if _DEFAULT_LOGGER and _DEFAULT_LOGGER.isEnabledFor(print_at_log_level): - print(f" {escape_shell(*args)}") +def render_command( + *args: str, + cwd: Optional[str] = None, + env_mod: Optional[EnvironmentModification] = None, +) -> str: + env_mod_prefix = "" + if env_mod: + env_mod_parts = [] + if bool(env_mod.removals): + env_mod_parts.append("env") + if cwd is not None: + env_mod_parts.append(f"--chdir={escape_shell(cwd)}") + env_mod_parts.extend(f"--unset={escape_shell(v)}" for v in env_mod.removals) + env_mod_parts.extend( + f"{escape_shell(k)}={escape_shell(v)}" for k, v in env_mod.replacements + ) + + chdir_prefix = "" + if cwd is not None and cwd != ".": + chdir_prefix = f"cd {escape_shell(cwd)} && " + return f"{chdir_prefix}{env_mod_prefix}{escape_shell(*args)}" + + +def print_command( + *args: str, + cwd: Optional[str] = None, + env_mod: Optional[EnvironmentModification] = None, + print_at_log_level: int = PRINT_COMMAND, +) -> None: + if _DEFAULT_LOGGER is None or not _DEFAULT_LOGGER.isEnabledFor(print_at_log_level): + return + + rendered_cmd = render_command( + *args, + cwd=cwd, + env_mod=env_mod, + ) + print(f" {rendered_cmd}") + + +def run_command( + *args: str, + cwd: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, + env_mod: Optional[EnvironmentModification] = None, + print_at_log_level: int = PRINT_COMMAND, +) -> None: + print_command( + *args, + cwd=cwd, + env_mod=env_mod, + print_at_log_level=print_at_log_level, + ) + if env_mod: + if env is None: + env = os.environ + env = env_mod.compute_env(env) + if env is os.environ: + env = None + try: + subprocess.check_call(args, cwd=cwd, env=env) + # At least "clean_logic.py" relies on catching FileNotFoundError + except KeyboardInterrupt: + _error(f"Interrupted (SIGINT) while running {escape_shell(*args)}") + except subprocess.CalledProcessError as e: + _error(f"The command {escape_shell(*args)} failed with status: {e.returncode}") + + +def run_build_system_command( + *args: str, + cwd: Optional[str] = None, + env: Optional[Mapping[str, str]] = None, + env_mod: Optional[EnvironmentModification] = None, + print_at_log_level: int = PRINT_BUILD_SYSTEM_COMMAND, +) -> None: + run_command( + *args, + cwd=cwd, + env=env, + env_mod=env_mod, + print_at_log_level=print_at_log_level, + ) def debian_policy_normalize_symlink_target( @@ -398,7 +503,7 @@ def integrated_with_debhelper() -> None: _DH_INTEGRATION_MODE = True -def scratch_dir() -> str: +def scratch_dir(*, create_if_not_exists: bool = True) -> str: global _SCRATCH_DIR if _SCRATCH_DIR is not None: return _SCRATCH_DIR @@ -411,9 +516,10 @@ def scratch_dir() -> str: is_debputy_dir = False else: _SCRATCH_DIR = debputy_scratch_dir - ensure_dir(_SCRATCH_DIR) - if is_debputy_dir: - Path("debian/.debputy/.gitignore").write_text("*\n") + if create_if_not_exists: + ensure_dir(_SCRATCH_DIR) + if is_debputy_dir: + Path("debian/.debputy/.gitignore").write_text("*\n") return _SCRATCH_DIR @@ -455,9 +561,11 @@ def generated_content_dir( return directory -PerlIncDir = collections.namedtuple("PerlIncDir", ["vendorlib", "vendorarch"]) +PerlConfigVars = collections.namedtuple( + "PerlIncDir", ["vendorlib", "vendorarch", "cross_inc_dir", "ld", "path_sep"] +) PerlConfigData = collections.namedtuple("PerlConfigData", ["version", "debian_abi"]) -_PERL_MODULE_DIRS: Dict[str, PerlIncDir] = {} +_PERL_MODULE_DIRS: Dict[str, PerlConfigVars] = {} @functools.lru_cache(1) @@ -490,42 +598,56 @@ def perlxs_api_dependency() -> str: return f"perlapi-{config.version}" -def perl_module_dirs( +def resolve_perl_config( dpkg_architecture_variables: DpkgArchitectureBuildProcessValuesTable, - dctrl_bin: "BinaryPackage", -) -> PerlIncDir: + dctrl_bin: Optional["BinaryPackage"], +) -> PerlConfigVars: global _PERL_MODULE_DIRS - arch = ( - dctrl_bin.resolved_architecture - if dpkg_architecture_variables.is_cross_compiling - else "_default_" - ) - module_dir = _PERL_MODULE_DIRS.get(arch) - if module_dir is None: + if dpkg_architecture_variables.is_cross_compiling: + arch = ( + dctrl_bin.resolved_architecture + if dctrl_bin is not None + else dpkg_architecture_variables.current_host_arch + ) + else: + arch = "_build_arch_" + config_vars = _PERL_MODULE_DIRS.get(arch) + if config_vars is None: cmd = ["perl"] if dpkg_architecture_variables.is_cross_compiling: version = _perl_version() - inc_dir = f"/usr/lib/{dctrl_bin.deb_multiarch}/perl/cross-config-{version}" + cross_inc_dir = ( + f"/usr/lib/{dctrl_bin.deb_multiarch}/perl/cross-config-{version}" + ) # FIXME: This should not fallback to "build-arch" but on the other hand, we use the perl module dirs # for every package at the moment. So mandating correct perl dirs implies mandating perl-xs-dev in # cross builds... meh. - if os.path.exists(os.path.join(inc_dir, "Config.pm")): - cmd.append(f"-I{inc_dir}") + if os.path.exists(os.path.join(cross_inc_dir, "Config.pm")): + cmd.append(f"-I{cross_inc_dir}") + else: + cross_inc_dir = None cmd.extend( - ["-MConfig", "-e", 'print "$Config{vendorlib}\n$Config{vendorarch}\n"'] + [ + "-MConfig", + "-e", + 'print "$Config{vendorlib}\n$Config{vendorarch}\n$Config{ld}\n$Config{path_sep}\n"', + ] ) output = subprocess.check_output(cmd).decode("utf-8").splitlines(keepends=False) - if len(output) != 2: + if len(output) != 4: raise ValueError( "Internal error: Unable to determine the perl include directories:" f" Raw output from perl snippet: {output}" ) - module_dir = PerlIncDir( - vendorlib=_normalize_path(output[0]), - vendorarch=_normalize_path(output[1]), + config_vars = PerlConfigVars( + vendorlib="/" + _normalize_path(output[0], with_prefix=False), + vendorarch="/" + _normalize_path(output[1], with_prefix=False), + cross_inc_dir=cross_inc_dir, + ld=output[2], + path_sep=output[3], ) - _PERL_MODULE_DIRS[arch] = module_dir - return module_dir + _PERL_MODULE_DIRS[arch] = config_vars + return config_vars @functools.lru_cache(1) @@ -736,6 +858,12 @@ def change_log_level( _DEFAULT_LOGGER.setLevel(log_level) +def current_log_level() -> Optional[int]: + if _DEFAULT_LOGGER is not None: + return _DEFAULT_LOGGER.level + return None + + def setup_logging( *, log_only_to_stderr: bool = False, @@ -748,13 +876,20 @@ def setup_logging( " Use reconfigure_logging=True if you need to reconfigure it" ) stdout_color, stderr_color, bad_request = _check_color() + colors: Optional[Dict[str, str]] = None if stdout_color or stderr_color: try: import colorlog + except ImportError: stdout_color = False stderr_color = False + else: + colors = dict(colorlog.default_log_colors) + # Add our custom levels. + colors["_INFO"] = colors["INFO"] + colors["__INFO"] = colors["INFO"] if log_only_to_stderr: stdout = sys.stderr @@ -785,7 +920,12 @@ def setup_logging( if stdout_color: stdout_handler = colorlog.StreamHandler(stdout) stdout_handler.setFormatter( - colorlog.ColoredFormatter(color_format, style="{", force_color=True) + colorlog.ColoredFormatter( + color_format, + style="{", + force_color=True, + log_colors=colors, + ) ) logger = colorlog.getLogger() if existing_stdout_handler is not None: @@ -804,7 +944,12 @@ def setup_logging( if stderr_color: stderr_handler = colorlog.StreamHandler(sys.stderr) stderr_handler.setFormatter( - colorlog.ColoredFormatter(color_format, style="{", force_color=True) + colorlog.ColoredFormatter( + color_format, + style="{", + force_color=True, + log_colors=colors, + ) ) logger = logging.getLogger() if existing_stderr_handler is not None: @@ -831,6 +976,7 @@ def setup_logging( *args: Any, **kwargs: Any ) -> logging.LogRecord: # pragma: no cover record = old_factory(*args, **kwargs) + record.levelname = record.levelname.lstrip("_") record.levelnamelower = record.levelname.lower() return record |