Merging upstream version 0.1.46.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

11 files changed, 117 insertions, 85 deletions

diff --git a/src/debputy/build_support/build_context.py b/src/debputy/build_support/build_context.py
index 2eeef66..cdaf4b4 100644
--- a/src/debputy/build_support/build_context.py
+++ b/src/debputy/build_support/build_context.py
@@ -2,7 +2,6 @@ from typing import Mapping, Optional
 
 from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable
 from debputy.commands.debputy_cmd.context import CommandContext
-from debputy.highlevel_manifest import HighLevelManifest
 from debputy.manifest_conditions import _run_build_time_tests
 
 
@@ -94,7 +93,3 @@ class BuildContextImpl(BuildContext):
     @property
     def dpkg_architecture_variables(self) -> DpkgArchitectureBuildProcessValuesTable:
         return self._cmd_context.dpkg_architecture_variables()
-
-    @property
-    def manifest(self) -> HighLevelManifest:
-        return self._manifest
diff --git a/src/debputy/commands/debputy_cmd/__main__.py b/src/debputy/commands/debputy_cmd/__main__.py
index 05bd135..1432327 100644
--- a/src/debputy/commands/debputy_cmd/__main__.py
+++ b/src/debputy/commands/debputy_cmd/__main__.py
@@ -22,6 +22,7 @@ from typing import (
 )
 
 from debputy import DEBPUTY_ROOT_DIR, DEBPUTY_PLUGIN_ROOT_DIR
+from debputy._deb_options_profiles import DebBuildOptionsAndProfiles
 from debputy.analysis import REFERENCE_DATA_TABLE
 from debputy.analysis.debian_dir import scan_debian_dir
 from debputy.build_support import perform_clean, perform_builds
@@ -770,9 +771,7 @@ def assemble(
     source_version = manifest.source_version()
     is_native = "-" not in source_version
     is_dh_rrr_only_mode = integration_mode == INTEGRATION_MODE_DH_DEBPUTY_RRR
-    package_data_table = manifest.perform_installations(
-        enable_manifest_installation_feature=not is_dh_rrr_only_mode
-    )
+    package_data_table = manifest.perform_installations(integration_mode)
     if not is_dh_rrr_only_mode:
         for dctrl_bin in manifest.active_packages:
             package = dctrl_bin.name
diff --git a/src/debputy/commands/debputy_cmd/context.py b/src/debputy/commands/debputy_cmd/context.py
index a9c0a13..d0a6cf7 100644
--- a/src/debputy/commands/debputy_cmd/context.py
+++ b/src/debputy/commands/debputy_cmd/context.py
@@ -53,6 +53,7 @@ from debputy.util import (
     setup_logging,
     PRINT_COMMAND,
     change_log_level,
+    _warn,
 )
 
 if TYPE_CHECKING:
@@ -89,6 +90,22 @@ class Command:
     requested_plugins_only: bool = False
 
 
+def _host_dpo_to_dbo(opt_and_profiles: "DebBuildOptionsAndProfiles", v: str) -> bool:
+
+    if (
+        v in opt_and_profiles.deb_build_profiles
+        and v not in opt_and_profiles.deb_build_options
+    ):
+        val = os.environ.get("DEB_BUILD_OPTIONS", "") + " " + v
+        _warn(
+            f'Copying "{v}" into DEB_BUILD_OPTIONS: It was in DEB_BUILD_PROFILES but not in DEB_BUILD_OPTIONS'
+        )
+        os.environ["DEB_BUILD_OPTIONS"] = val.lstrip()
+        # Note: It will not be immediately visible since `DebBuildOptionsAndProfiles` caches the result
+        return True
+    return False
+
+
 class CommandContext:
     def __init__(
         self,
@@ -153,13 +170,21 @@ class CommandContext:
             if hasattr(self.parsed_args, "packages"):
                 packages = self.parsed_args.packages
 
+            instance = DebBuildOptionsAndProfiles(environ=os.environ)
+
+            dirty = _host_dpo_to_dbo(instance, "nodoc")
+            dirty = _host_dpo_to_dbo(instance, "nocheck") or dirty
+
+            if dirty:
+                instance = DebBuildOptionsAndProfiles(environ=os.environ)
+
             parser = DctrlParser(
                 packages,  # -p/--package
                 set(),  # -N/--no-package
                 # binary-indep and binary-indep (dpkg BuildDriver integration only)
                 self._package_set == "indep",
                 self._package_set == "arch",
-                deb_options_and_profiles=DebBuildOptionsAndProfiles.instance(),
+                deb_options_and_profiles=instance,
                 dpkg_architecture_variables=dpkg_architecture_table(),
                 dpkg_arch_query_table=DpkgArchTable.load_arch_table(),
             )
diff --git a/src/debputy/highlevel_manifest.py b/src/debputy/highlevel_manifest.py
index 6c910ab..ef47d45 100644
--- a/src/debputy/highlevel_manifest.py
+++ b/src/debputy/highlevel_manifest.py
@@ -63,7 +63,12 @@ from .plugin.api.impl_types import (
     PackageProcessingContextProvider,
     PackageDataTable,
 )
-from .plugin.api.spec import FlushableSubstvars, VirtualPath
+from .plugin.api.spec import (
+    FlushableSubstvars,
+    VirtualPath,
+    DebputyIntegrationMode,
+    INTEGRATION_MODE_DH_DEBPUTY_RRR,
+)
 from .plugin.debputy.binary_package_rules import ServiceRule
 from .plugin.debputy.to_be_api_types import BuildRule
 from .plugin.plugin_state import run_in_context_of_plugin
@@ -1185,12 +1190,15 @@ class HighLevelManifest:
 
     def perform_installations(
         self,
+        integration_mode: DebputyIntegrationMode,
         *,
         install_request_context: Optional[InstallSearchDirContext] = None,
-        enable_manifest_installation_feature: bool = True,
     ) -> PackageDataTable:
         package_data_dict = {}
         package_data_table = PackageDataTable(package_data_dict)
+        enable_manifest_installation_feature = (
+            integration_mode != INTEGRATION_MODE_DH_DEBPUTY_RRR
+        )
         if install_request_context is None:
 
             @functools.lru_cache(None)
@@ -1201,6 +1209,7 @@ class HighLevelManifest:
             source_root_dir = _as_path(".")
             into = frozenset(self._binary_packages.values())
             default_search_dirs = [dtmp_dir]
+            # TODO: In integration-mode full use build systems to define the per_package_search_dirs
             per_package_search_dirs = {
                 t.binary_package: [_as_path(f.match_rule.path) for f in t.search_dirs]
                 for t in self.package_transformations.values()
diff --git a/src/debputy/lsp/lsp_debian_control_reference_data.py b/src/debputy/lsp/lsp_debian_control_reference_data.py
index 1d24045..91eb6c2 100644
--- a/src/debputy/lsp/lsp_debian_control_reference_data.py
+++ b/src/debputy/lsp/lsp_debian_control_reference_data.py
@@ -1363,7 +1363,7 @@ _PKGNAME_VS_SECTION_RULES = [
     ),
     _package_name_section_rule(
         "localization",
-        lambda n: n.startswith("hypen-"),
+        lambda n: n.startswith("hyphen-"),
         confirm_re=re.compile(r"^hyphen-[a-z]{2}(?:-[a-z]{2})?$"),
     ),
     _package_name_section_rule(
@@ -2981,10 +2981,10 @@ BINARY_FIELDS = _fields(
                    (APT and dpkg) will refuse to uninstall it without some very insisting force options and warnings.
 
                  * Other packages are not required to declare explicit dependencies on essential packages as a
-                   side-effect of the above except as to ensure a that the given essential package is upgraded
+                   side-effect of the above except as to ensure that the given essential package is upgraded
                    to a given minimum version.
 
-                 * Once installed, essential packages function must at all time no matter where dpkg is in its
+                 * Once installed, essential packages must function at all times no matter where dpkg is in its
                    installation or upgrade process. During bootstrapping or installation, this requirement is
                    relaxed.
             """
@@ -3324,7 +3324,7 @@ BINARY_FIELDS = _fields(
       *advised to leave it at its default until you have a working basic package or lots of time to understand*
       *this topic.*
 
-      Declare that the package will only built when the given build-profiles are satisfied.
+      Declare that the package will only be built when the given build-profiles are satisfied.
 
       This field is primarily used in combination with build profiles inside the build dependency related fields
       to reduce the number of build dependencies required during bootstrapping of a new architecture.
@@ -3512,8 +3512,9 @@ BINARY_FIELDS = _fields(
 
        * If you have an architecture dependent package, where everything is installed in
          `/usr/lib/${DEB_HOST_MULTIARCH}` (plus a bit of standard documentation in `/usr/share/doc`), then
-         you *probably* want `Multi-Arch: same`. Note that `debputy` automatically detects the most common
-         variants of this case and sets the field for you.
+         you *probably* want `Multi-Arch: same`. Note that when using `debputy` as the build helper, `debputy`
+         will automatically detect the most common variants of this case and sets the field for you when
+          relevant.
 
        * If none of the above applies, then omit the field unless you know what you are doing or you are
          receiving advice from a Multi-Arch expert.
@@ -3539,11 +3540,11 @@ BINARY_FIELDS = _fields(
           solely providing data or binaries that have "Multi-Arch neutral interfaces". Sadly, describing
           a "Multi-Arch neutral interface" is hard and often only done by Multi-Arch experts on a case-by-case
           basis. Among other, scripts despite being the same on all architectures can still have a "non-neutral"
-          "Multi-Arch" interface if their output is architecture dependent or if they dependencies force them
+          "Multi-Arch" interface if their output is architecture dependent or if their dependencies force them
           out of the `foreign` role. The dependency issue usually happens when depending indirectly on an
           `Multi-Arch: allowed` package.
 
-          Some programs are have "Multi-Arch dependent interfaces" and are not safe to declare as
+          Some programs have "Multi-Arch dependent interfaces" and are not safe to declare as
           `Multi-Arch: foreign`. The name `foreign` refers to the fact that the package can satisfy relations
           for native *and foreign* architectures at the same time.
 
@@ -3560,7 +3561,7 @@ BINARY_FIELDS = _fields(
           Note: This value **cannot** be used with `Architecture: all`.
 
 
-        * `allowed` - **Advanced value**. This value is for a complex use-case that most people does not
+        * `allowed` - **Advanced value**. This value is for a complex use-case that most people do not
           need. Consider it only if none of the other values seem to do the trick.
 
           The package is **NOT** co-installable with itself but can satisfy Multi-Arch foreign and Multi-Arch same
diff --git a/src/debputy/packager_provided_files.py b/src/debputy/packager_provided_files.py
index 5657ad2..95d1b17 100644
--- a/src/debputy/packager_provided_files.py
+++ b/src/debputy/packager_provided_files.py
@@ -140,7 +140,9 @@ def _find_definition(
             continue
         # If the stem is also the extension and a known one at that, then
         # we do not consider it a typo match (to avoid false positives).
-        if not had_arch and stem in _KNOWN_NON_TYPO_EXTENSIONS:
+        #
+        # We also ignore "foo.1" since manpages are kind of common.
+        if not had_arch and (stem in _KNOWN_NON_TYPO_EXTENSIONS or stem.isdigit()):
             continue
         matches = detect_possible_typo(stem, stems)
         if matches is not None and len(matches) == 1:
diff --git a/src/debputy/plugin/debputy/build_system_rules.py b/src/debputy/plugin/debputy/build_system_rules.py
index b7ee898..454b2db 100644
--- a/src/debputy/plugin/debputy/build_system_rules.py
+++ b/src/debputy/plugin/debputy/build_system_rules.py
@@ -657,9 +657,7 @@ class PerlBuildBuildSystemRule(StepBasedBuildSystemRule):
                         perl5lib_dir + perl_config_data.path_sep + env_perl5lib
                     )
                 env_mod = EnvironmentModification(
-                    replacements=[
-                        ("PERL5LIB", perl5lib_dir),
-                    ],
+                    replacements=(("PERL5LIB", perl5lib_dir),),
                 )
                 return perl_config_data, env_mod
         return perl_config_data, None
@@ -672,7 +670,7 @@ class PerlBuildBuildSystemRule(StepBasedBuildSystemRule):
     ) -> None:
         perl_config_data, cross_env_mod = self._perl_cross_build_env(context)
         configure_env = EnvironmentModification(
-            replacements=[("PERL_MM_USE_DEFAULT", "1")]
+            replacements=(("PERL_MM_USE_DEFAULT", "1"),)
         )
         if cross_env_mod is not None:
             configure_env = configure_env.combine(cross_env_mod)
@@ -804,10 +802,11 @@ class PerlMakeMakerBuildSystemRule(StepBasedBuildSystemRule):
         **kwargs,
     ) -> None:
         configure_env = EnvironmentModification(
-            replacements=[
+            replacements=(
                 ("PERL_MM_USE_DEFAULT", "1"),
                 ("PERL_AUTOINSTALL", "--skipdeps"),
-            ]
+                ("PKG_CONFIG", context.cross_tool("pkg-config")),
+            )
         )
         perl_args = []
         mm_args = ["INSTALLDIRS=vendor"]
@@ -1255,7 +1254,9 @@ class CMakeBuildSystemRule(StepBasedBuildSystemRule):
             replacements["DEB_PYTHON_INSTALL_LAYOUT"] = "deb"
         if "PKG_CONFIG" not in os.environ:
             replacements["PKG_CONFIG"] = build_context.cross_tool("pkg-config")
-        return EnvironmentModification(replacements=replacements)
+        return EnvironmentModification(
+            replacements=tuple((k, v) for k, v in replacements.items())
+        )
 
     @classmethod
     def cmake_generator(cls, target_build_system: Literal["make", "ninja"]) -> str:
@@ -1361,19 +1362,17 @@ class CMakeBuildSystemRule(StepBasedBuildSystemRule):
             env_mod = env_mod.combine(
                 # The debhelper build system never showed this delta, so people might find it annoying.
                 EnvironmentModification(
-                    replacements={
-                        "CFLAGS": cflags,
-                        "CXXFLAGS": cxxflags,
-                    }
+                    replacements=(
+                        ("CFLAGS", cflags),
+                        ("CXXFLAGS", cxxflags),
+                    )
                 )
             )
         if "ASMFLAGS" not in os.environ and "ASFLAGS" in os.environ:
             env_mod = env_mod.combine(
                 # The debhelper build system never showed this delta, so people might find it annoying.
                 EnvironmentModification(
-                    replacements={
-                        "ASMFLAGS": os.environ["ASFLAGS"],
-                    }
+                    replacements=(("ASMFLAGS", os.environ["ASFLAGS"]),),
                 )
             )
         self.ensure_build_dir_exists()
@@ -1413,9 +1412,7 @@ class CMakeBuildSystemRule(StepBasedBuildSystemRule):
         **kwargs,
     ) -> None:
         env_mod = EnvironmentModification(
-            replacements={
-                "CTEST_OUTPUT_ON_FAILURE": "1",
-            },
+            replacements=(("CTEST_OUTPUT_ON_FAILURE", "1"),),
         )
         if self.target_build_system == "make":
             # Unlike make, CTest does not have "unlimited parallel" setting (-j implies
@@ -1453,10 +1450,10 @@ class CMakeBuildSystemRule(StepBasedBuildSystemRule):
         **kwargs,
     ) -> None:
         env_mod = EnvironmentModification(
-            replacements={
-                "LC_ALL": "C.UTF-8",
-                "DESTDIR": dest_dir,
-            }
+            replacements=(
+                ("LC_ALL", "C.UTF-8"),
+                ("DESTDIR", dest_dir),
+            )
         ).combine(self._default_cmake_env(context))
         run_build_system_command(
             "cmake",
@@ -1524,7 +1521,9 @@ class MesonBuildSystemRule(StepBasedBuildSystemRule):
         }
         if "DEB_PYTHON_INSTALL_LAYOUT" not in os.environ:
             replacements["DEB_PYTHON_INSTALL_LAYOUT"] = "deb"
-        return EnvironmentModification(replacements=replacements)
+        return EnvironmentModification(
+            replacements=tuple((k, v) for k, v in replacements.items())
+        )
 
     @classmethod
     def cmake_generator(cls, target_build_system: Literal["make", "ninja"]) -> str:
@@ -1638,9 +1637,7 @@ class MesonBuildSystemRule(StepBasedBuildSystemRule):
         **kwargs,
     ) -> None:
         env_mod = EnvironmentModification(
-            replacements={
-                "MESON_TESTTHREDS": f"{context.parallelization_limit()}",
-            },
+            replacements=(("MESON_TESTTHREDS", f"{context.parallelization_limit()}"),),
         ).combine(self._default_meson_env())
         with self.dump_logs_on_error("meson-logs/testlog.txt"):
             run_build_system_command(
diff --git a/src/debputy/plugin/debputy/package_processors.py b/src/debputy/plugin/debputy/package_processors.py
index 1d19b66..0099c3b 100644
--- a/src/debputy/plugin/debputy/package_processors.py
+++ b/src/debputy/plugin/debputy/package_processors.py
@@ -8,7 +8,15 @@ from contextlib import ExitStack
 from typing import Optional, Iterator, IO, Any, List, Dict, Callable, Union
 
 from debputy.plugin.api import VirtualPath
-from debputy.util import _error, xargs, escape_shell, _info, assume_not_none
+from debputy.util import (
+    _error,
+    xargs,
+    escape_shell,
+    _info,
+    assume_not_none,
+    print_command,
+    _debug_log,
+)
 
 
 @contextlib.contextmanager
@@ -119,8 +127,27 @@ def process_manpages(fs_root: VirtualPath, _unused1: Any, _unused2: Any) -> None
         for manpage in manpages:
             dest_name = manpage
             if dest_name.endswith(".gz"):
-                dest_name = dest_name[:-3]
-            os.rename(f"{dest_name}.encoded", manpage)
+                encoded_name = dest_name[:-3] + ".encoded"
+                with open(dest_name, "wb") as out:
+                    _debug_log(
+                        f"Recompressing {dest_name} via gzip -9nc {escape_shell(encoded_name)}"
+                    )
+                    try:
+                        subprocess.check_call(
+                            [
+                                "gzip",
+                                "-9nc",
+                                encoded_name,
+                            ],
+                            stdin=subprocess.DEVNULL,
+                            stdout=out,
+                        )
+                    except subprocess.CalledProcessError:
+                        _error(
+                            f"The command {escape_shell('gzip', '-nc', f'{encoded_name}')} > {dest_name} failed!"
+                        )
+            else:
+                os.rename(f"{dest_name}.encoded", manpage)
 
 
 def _filter_compress_paths() -> Callable[[VirtualPath], Iterator[VirtualPath]]:
diff --git a/src/debputy/plugin/debputy/to_be_api_types.py b/src/debputy/plugin/debputy/to_be_api_types.py
index d7be694..720d1d2 100644
--- a/src/debputy/plugin/debputy/to_be_api_types.py
+++ b/src/debputy/plugin/debputy/to_be_api_types.py
@@ -798,11 +798,7 @@ class NinjaBuildSupport:
         # debhelper never had parallel installs, so we do not have it either for now.
         enable_parallelization: bool = False,
     ) -> None:
-        install_env_mod = EnvironmentModification(
-            replacements={
-                "DESTDIR": dest_dir,
-            }
-        )
+        install_env_mod = EnvironmentModification(replacements=(("DESTDIR", dest_dir),))
         if env_mod is not None:
             install_env_mod = install_env_mod.combine(env_mod)
         self._run_ninja(
@@ -846,11 +842,7 @@ class NinjaBuildSupport:
             else 1
         )
         extra_ninja_args.append(f"-j{limit}")
-        ninja_env_mod = EnvironmentModification(
-            replacements={
-                "LC_ALL": "C.UTF-8",
-            }
-        )
+        ninja_env_mod = EnvironmentModification(replacements=(("LC_ALL", "C.UTF-8"),))
         if env_mod is not None:
             ninja_env_mod = ninja_env_mod.combine(env_mod)
         run_build_system_command(
diff --git a/src/debputy/types.py b/src/debputy/types.py
index dc3cbd3..5f0bde0 100644
--- a/src/debputy/types.py
+++ b/src/debputy/types.py
@@ -34,24 +34,12 @@ class EnvironmentModification:
     replacements: Sequence[Tuple[str, str]] = tuple()
     removals: Sequence[str] = tuple()
 
-    @staticmethod
-    def from_serialized_format(
-        serial_form: EnvironmentModificationSerialized,
-    ) -> "EnvironmentModification":
-        replacements_raw = serial_form.get("replacements")
-        if replacements_raw is not None:
-            replacements = tuple((k, v) for k, v in replacements_raw.items())
-        else:
-            replacements = tuple()
-        return EnvironmentModification(
-            replacements=replacements, removals=serial_form.get("removals", tuple())
-        )
-
     def __bool__(self) -> bool:
         return not self.removals and not self.replacements
 
     def combine(
-        self, other: "Optional[EnvironmentModification]"
+        self,
+        other: "Optional[EnvironmentModification]",
     ) -> "EnvironmentModification":
         if not other:
             return self
@@ -93,16 +81,6 @@ class EnvironmentModification:
             new_removals,
         )
 
-    def serialize(self) -> EnvironmentModificationSerialized:
-        serial_form = {}
-        replacements = self.replacements
-        if replacements:
-            serial_form["replacements"] = {k: v for k, v in replacements}
-        removals = self.removals
-        if removals:
-            serial_form["removals"] = list(removals)
-        return serial_form
-
     def update_inplace(self, env: MutableMapping[str, str]) -> None:
         for k, v in self.replacements:
             existing_value = env.get(k)
diff --git a/src/debputy/util.py b/src/debputy/util.py
index 911e6fa..0824569 100644
--- a/src/debputy/util.py
+++ b/src/debputy/util.py
@@ -99,8 +99,9 @@ POSTINST_DEFAULT_CONDITION = (
 
 
 _SPACE_RE = re.compile(r"\s")
+_WORD_EQUAL = re.compile(r"^-*[\w_\-]+=")
 _DOUBLE_ESCAPEES = re.compile(r'([\n`$"\\])')
-_REGULAR_ESCAPEES = re.compile(r'([\s!"$()*+#;<>?@\[\]\\`|~])')
+_REGULAR_ESCAPEES = re.compile(r"""([\s!"$()*+#;<>?@'\[\]\\`|~])""")
 _PROFILE_GROUP_SPLIT = re.compile(r">\s+<")
 _DEFAULT_LOGGER: Optional[logging.Logger] = None
 _STDOUT_HANDLER: Optional[logging.StreamHandler[Any]] = None
@@ -242,7 +243,13 @@ def _backslash_escape(m: re.Match[str]) -> str:
 
 
 def _escape_shell_word(w: str) -> str:
-    if _SPACE_RE.match(w):
+    if _SPACE_RE.search(w):
+        if "=" in w and (m := _WORD_EQUAL.search(w)) is not None:
+            s = m.span(0)
+            assert s[0] == 0
+            prefix = w[0 : s[1]]
+            escaped_value = _DOUBLE_ESCAPEES.sub(_backslash_escape, w[s[1] :])
+            return f'{prefix}"{escaped_value}"'
         w = _DOUBLE_ESCAPEES.sub(_backslash_escape, w)
         return f'"{w}"'
     return _REGULAR_ESCAPEES.sub(_backslash_escape, w)