Coverage for src/debputy/package_build/assemble_deb.py: 14%
98 statements
« prev ^ index » next coverage.py v7.2.7, created at 2024-04-07 12:14 +0200
1import json
2import os
3import subprocess
4from typing import Optional, Sequence, List, Tuple
6from debputy import DEBPUTY_ROOT_DIR
7from debputy.commands.debputy_cmd.context import CommandContext
8from debputy.deb_packaging_support import setup_control_files
9from debputy.debhelper_emulation import dhe_dbgsym_root_dir
10from debputy.filesystem_scan import FSRootDir
11from debputy.highlevel_manifest import HighLevelManifest
12from debputy.intermediate_manifest import IntermediateManifest
13from debputy.plugin.api.impl_types import PackageDataTable
14from debputy.util import (
15 escape_shell,
16 _error,
17 compute_output_filename,
18 scratch_dir,
19 ensure_dir,
20 _warn,
21 assume_not_none,
22)
25_RRR_DEB_ASSEMBLY_KEYWORD = "debputy/deb-assembly"
26_WARNED_ABOUT_FALLBACK_ASSEMBLY = False
29def _serialize_intermediate_manifest(members: IntermediateManifest) -> str:
30 serial_format = [m.to_manifest() for m in members]
31 return json.dumps(serial_format)
34def determine_assembly_method(
35 package: str,
36 intermediate_manifest: IntermediateManifest,
37) -> Tuple[bool, bool, List[str]]:
38 paths_needing_root = (
39 tm for tm in intermediate_manifest if tm.owner != "root" or tm.group != "root"
40 )
41 matched_path = next(paths_needing_root, None)
42 if matched_path is None:
43 return False, False, []
44 rrr = os.environ.get("DEB_RULES_REQUIRES_ROOT")
45 if rrr and _RRR_DEB_ASSEMBLY_KEYWORD in rrr:
46 gain_root_cmd = os.environ.get("DEB_GAIN_ROOT_CMD")
47 if not gain_root_cmd:
48 _error(
49 "DEB_RULES_REQUIRES_ROOT contains a debputy keyword but DEB_GAIN_ROOT_CMD does not contain a "
50 '"gain root" command'
51 )
52 return True, False, gain_root_cmd.split()
53 if rrr == "no":
54 global _WARNED_ABOUT_FALLBACK_ASSEMBLY
55 if not _WARNED_ABOUT_FALLBACK_ASSEMBLY:
56 _warn(
57 'Using internal assembly method due to "Rules-Requires-Root" being "no" and dpkg-deb assembly would'
58 " require (fake)root for binary packages that needs it."
59 )
60 _WARNED_ABOUT_FALLBACK_ASSEMBLY = True
61 return True, True, []
63 _error(
64 f'Due to the path "{matched_path.member_path}" in {package}, the package assembly will require (fake)root.'
65 " However, this command is not run as root nor was debputy requested to use a root command via"
66 f' "Rules-Requires-Root". Please consider adding "{_RRR_DEB_ASSEMBLY_KEYWORD}" to "Rules-Requires-Root"'
67 " in debian/control. Though, due to #1036865, you may have to revert to"
68 ' "Rules-Requires-Root: binary-targets" depending on which version of dpkg you need to support.'
69 ' Alternatively, you can set "Rules-Requires-Root: no" in debian/control and debputy will assemble'
70 " the package anyway. In this case, dpkg-deb will not be used, but the output should be bit-for-bit"
71 " compatible with what debputy would have produced with dpkg-deb (and root/fakeroot)."
72 )
75def assemble_debs(
76 context: CommandContext,
77 manifest: HighLevelManifest,
78 package_data_table: PackageDataTable,
79 is_dh_rrr_only_mode: bool,
80) -> None:
81 parsed_args = context.parsed_args
82 output_path = parsed_args.output
83 upstream_args = parsed_args.upstream_args
84 deb_materialize = str(DEBPUTY_ROOT_DIR / "deb_materialization.py")
85 mtime = context.mtime
87 for dctrl_bin in manifest.active_packages:
88 package = dctrl_bin.name
89 dbgsym_package_name = f"{package}-dbgsym"
90 dctrl_data = package_data_table[package]
91 fs_root = dctrl_data.fs_root
92 control_output_dir = assume_not_none(dctrl_data.control_output_dir)
93 package_metadata_context = dctrl_data.package_metadata_context
94 if (
95 dbgsym_package_name in package_data_table
96 or "noautodbgsym" in manifest.build_env.deb_build_options
97 or "noddebs" in manifest.build_env.deb_build_options
98 ):
99 # Discard the dbgsym part if it conflicts with a real package, or
100 # we were asked not to build it.
101 dctrl_data.dbgsym_info.dbgsym_fs_root = FSRootDir()
102 dctrl_data.dbgsym_info.dbgsym_ids.clear()
103 dbgsym_fs_root = dctrl_data.dbgsym_info.dbgsym_fs_root
104 dbgsym_ids = dctrl_data.dbgsym_info.dbgsym_ids
105 intermediate_manifest = manifest.finalize_data_tar_contents(
106 package, fs_root, mtime
107 )
109 setup_control_files(
110 dctrl_data,
111 manifest,
112 dbgsym_fs_root,
113 dbgsym_ids,
114 package_metadata_context,
115 allow_ctrl_file_management=not is_dh_rrr_only_mode,
116 )
118 needs_root, use_fallback_assembly, gain_root_cmd = determine_assembly_method(
119 package, intermediate_manifest
120 )
122 if not dctrl_bin.is_udeb and any(
123 f for f in dbgsym_fs_root.all_paths() if f.is_file
124 ):
125 # We never built udebs due to #797391. We currently do not generate a control
126 # file for it either for the same reason.
127 dbgsym_root = dhe_dbgsym_root_dir(dctrl_bin)
128 if not os.path.isdir(output_path):
129 _error(
130 "Cannot produce a dbgsym package when output path is not a directory."
131 )
132 dbgsym_intermediate_manifest = manifest.finalize_data_tar_contents(
133 dbgsym_package_name,
134 dbgsym_fs_root,
135 mtime,
136 )
137 _assemble_deb(
138 dbgsym_package_name,
139 deb_materialize,
140 dbgsym_intermediate_manifest,
141 mtime,
142 os.path.join(dbgsym_root, "DEBIAN"),
143 output_path,
144 upstream_args,
145 is_udeb=dctrl_bin.is_udeb, # Review this if we ever do dbgsyms for udebs
146 use_fallback_assembly=False,
147 needs_root=False,
148 )
150 _assemble_deb(
151 package,
152 deb_materialize,
153 intermediate_manifest,
154 mtime,
155 control_output_dir,
156 output_path,
157 upstream_args,
158 is_udeb=dctrl_bin.is_udeb,
159 use_fallback_assembly=use_fallback_assembly,
160 needs_root=needs_root,
161 gain_root_cmd=gain_root_cmd,
162 )
165def _assemble_deb(
166 package: str,
167 deb_materialize_cmd: str,
168 intermediate_manifest: IntermediateManifest,
169 mtime: int,
170 control_output_dir: str,
171 output_path: str,
172 upstream_args: Optional[List[str]],
173 is_udeb: bool = False,
174 use_fallback_assembly: bool = False,
175 needs_root: bool = False,
176 gain_root_cmd: Optional[Sequence[str]] = None,
177) -> None:
178 scratch_root_dir = scratch_dir()
179 materialization_dir = os.path.join(
180 scratch_root_dir, "materialization-dirs", package
181 )
182 ensure_dir(os.path.dirname(materialization_dir))
183 materialize_cmd: List[str] = []
184 assert not use_fallback_assembly or not gain_root_cmd
185 if needs_root and gain_root_cmd:
186 # Only use the gain_root_cmd if we absolutely need it.
187 # Note that gain_root_cmd will be empty unless R³ is set to the relevant keyword
188 # that would make us use targeted promotion. Therefore, we do not need to check other
189 # conditions than the package needing root. (R³: binary-targets implies `needs_root=True`
190 # without a gain_root_cmd)
191 materialize_cmd.extend(gain_root_cmd)
192 materialize_cmd.extend(
193 [
194 deb_materialize_cmd,
195 "materialize-deb",
196 "--intermediate-package-manifest",
197 "-",
198 "--may-move-control-files",
199 "--may-move-data-files",
200 "--source-date-epoch",
201 str(mtime),
202 "--discard-existing-output",
203 control_output_dir,
204 materialization_dir,
205 ]
206 )
207 output = output_path
208 if is_udeb:
209 materialize_cmd.append("--udeb")
210 output = os.path.join(
211 output_path, compute_output_filename(control_output_dir, True)
212 )
214 assembly_method = "debputy" if needs_root and use_fallback_assembly else "dpkg-deb"
215 combined_materialization_and_assembly = not needs_root
216 if combined_materialization_and_assembly:
217 materialize_cmd.extend(
218 ["--build-method", assembly_method, "--assembled-deb-output", output]
219 )
221 if upstream_args:
222 materialize_cmd.append("--")
223 materialize_cmd.extend(upstream_args)
225 if combined_materialization_and_assembly:
226 print(
227 f"Materializing and assembling {package} via: {escape_shell(*materialize_cmd)}"
228 )
229 else:
230 print(f"Materializing {package} via: {escape_shell(*materialize_cmd)}")
231 proc = subprocess.Popen(materialize_cmd, stdin=subprocess.PIPE)
232 proc.communicate(
233 _serialize_intermediate_manifest(intermediate_manifest).encode("utf-8")
234 )
235 if proc.returncode != 0:
236 _error(f"{escape_shell(deb_materialize_cmd)} exited with a non-zero exit code!")
238 if not combined_materialization_and_assembly:
239 build_materialization = [
240 deb_materialize_cmd,
241 "build-materialized-deb",
242 materialization_dir,
243 assembly_method,
244 "--output",
245 output,
246 ]
247 print(f"Assembling {package} via: {escape_shell(*build_materialization)}")
248 try:
249 subprocess.check_call(build_materialization)
250 except subprocess.CalledProcessError as e:
251 exit_code = f" with exit code {e.returncode}" if e.returncode else ""
252 _error(
253 f"Assembly command for {package} failed{exit_code}. Please review the output of the command"
254 f" for more details on the problem."
255 )