diff options
39 files changed, 988 insertions, 339 deletions
@@ -81,7 +81,6 @@ my (@plugins, $integration_mode); init(options => { "destdir=s" => \$dh{DESTDIR}, "plugin=s" => \@plugins, - "integration-mode=s" => \$integration_mode, }); # Set the default destination directory. @@ -104,8 +103,6 @@ push(@debputy_cmdline, 'internal-command', 'dh-integration-generate-debs', ); -push(@debputy_cmdline, "--integration-mode=${integration_mode}") - if defined($integration_mode); for my $package (@{$dh{DOPACKAGES}}) { push(@debputy_cmdline, '-p', $package); } diff --git a/lib/Debian/Debhelper/Sequence/zz_debputy_rrr.pm b/lib/Debian/Debhelper/Sequence/zz_debputy_rrr.pm index 22d61e3..c912cc3 100644 --- a/lib/Debian/Debhelper/Sequence/zz_debputy_rrr.pm +++ b/lib/Debian/Debhelper/Sequence/zz_debputy_rrr.pm @@ -4,7 +4,6 @@ insert_after('dh_builddeb', 'dh_debputy'); if (exists($INC{"Debian/Debhelper/Sequence/debputy.pm"})) { error("The zz-debputy-rrr sequence cannot be used with the (zz-)debputy sequence"); } -add_command_options('dh_debputy', '--integration-mode=rrr'); remove_command('dh_fixperms'); remove_command('dh_shlibdeps'); diff --git a/src/debputy/analysis/debian_dir.py b/src/debputy/analysis/debian_dir.py index d63b0c9..1e88b14 100644 --- a/src/debputy/analysis/debian_dir.py +++ b/src/debputy/analysis/debian_dir.py @@ -3,6 +3,7 @@ import os import stat import subprocess from typing import ( + AbstractSet, List, Mapping, Iterable, @@ -15,12 +16,14 @@ from typing import ( Iterator, TypedDict, NotRequired, - FrozenSet, ) from debputy.analysis import REFERENCE_DATA_TABLE from debputy.analysis.analysis_util import flatten_ppfs -from debputy.dh_migration.migrators_impl import read_dh_addon_sequences +from debputy.dh.dh_assistant import ( + resolve_active_and_inactive_dh_commands, + read_dh_addon_sequences, +) from debputy.packager_provided_files import ( PackagerProvidedFile, detect_all_packager_provided_files, @@ -71,6 +74,9 @@ def scan_debian_dir( feature_set: PluginProvidedFeatureSet, binary_packages: Mapping[str, BinaryPackage], debian_dir: VirtualPath, + *, + uses_dh_sequencer: bool = True, + dh_sequences: Optional[AbstractSet[str]] = None, ) -> Tuple[List[PackagingFileInfo], List[str], int, Optional[object]]: known_packaging_files = feature_set.known_packaging_files debputy_plugin_metadata = plugin_metadata_for_debputys_own_plugin() @@ -85,18 +91,19 @@ def scan_debian_dir( annotated: List[PackagingFileInfo] = [] seen_paths: Dict[str, PackagingFileInfo] = {} - r = read_dh_addon_sequences(debian_dir) - if r is not None: - bd_sequences, dr_sequences, saw_dh = r - drules_sequences = bd_sequences | dr_sequences - else: - drules_sequences = set() - saw_dh = False + if dh_sequences is None: + r = read_dh_addon_sequences(debian_dir) + if r is not None: + bd_sequences, dr_sequences, uses_dh_sequencer = r + dh_sequences = bd_sequences | dr_sequences + else: + dh_sequences = set() + uses_dh_sequencer = False is_debputy_package = ( - "debputy" in drules_sequences - or "zz-debputy" in drules_sequences - or "zz_debputy" in drules_sequences - or "zz-debputy-rrr" in drules_sequences + "debputy" in dh_sequences + or "zz-debputy" in dh_sequences + or "zz_debputy" in dh_sequences + or "zz-debputy-rrr" in dh_sequences ) dh_compat_level, dh_assistant_exit_code = _extract_dh_compat_level() dh_issues = [] @@ -137,9 +144,9 @@ def scan_debian_dir( binary_packages, debputy_plugin_metadata, dh_pkgfile_docs, - drules_sequences, + dh_sequences, dh_compat_level, - saw_dh, + uses_dh_sequencer, ) else: @@ -278,49 +285,12 @@ def _kpf_install_pattern( return ppkpf.info.get("install_pattern") -def _parse_dh_cmd_list( - cmd_list: Optional[List[Union[Mapping[str, Any], object]]] -) -> Iterable[str]: - if not isinstance(cmd_list, list): - return - - for command in cmd_list: - if not isinstance(command, dict): - continue - command_name = command.get("command") - if isinstance(command_name, str): - yield command_name - - -def _resolve_active_and_inactive_dh_commands( - dh_rules_addons: Iterable[str], -) -> Tuple[FrozenSet[str], FrozenSet[str]]: - cmd = ["dh_assistant", "list-commands", "--output-format=json"] - if dh_rules_addons: - addons = ",".join(dh_rules_addons) - cmd.append(f"--with={addons}") - try: - output = subprocess.check_output( - cmd, - stderr=subprocess.DEVNULL, - ) - except (subprocess.CalledProcessError, FileNotFoundError): - return frozenset(), frozenset() - else: - result = json.loads(output) - active_commands = frozenset(_parse_dh_cmd_list(result.get("commands"))) - disabled_commands = frozenset( - _parse_dh_cmd_list(result.get("disabled-commands")) - ) - return active_commands, disabled_commands - - def _resolve_debhelper_config_files( debian_dir: VirtualPath, binary_packages: Mapping[str, BinaryPackage], debputy_plugin_metadata: DebputyPluginMetadata, dh_ppf_docs: Dict[str, PluginProvidedKnownPackagingFile], - dh_rules_addons: Sequence[str], + dh_rules_addons: AbstractSet[str], dh_compat_level: int, saw_dh: bool, ) -> Tuple[List[PackagerProvidedFile], Optional[object], int]: @@ -349,7 +319,7 @@ def _resolve_debhelper_config_files( "config-files", [] ) issues = result.get("issues") - active_commands, _ = _resolve_active_and_inactive_dh_commands(dh_rules_addons) + dh_commands = resolve_active_and_inactive_dh_commands(dh_rules_addons) for config_file in config_files: if not isinstance(config_file, dict): continue @@ -408,7 +378,7 @@ def _resolve_debhelper_config_files( else: continue is_active = command.get("is-active", True) - if is_active is None and command_name in active_commands: + if is_active is None and command_name in dh_commands.active_commands: is_active = True if not isinstance(is_active, bool): continue @@ -443,7 +413,9 @@ def _resolve_debhelper_config_files( if not has_active_command: dh_cmds = ppkpf.info.get("debhelper_commands") if dh_cmds: - has_active_command = any(c in active_commands for c in dh_cmds) + has_active_command = any( + c in dh_commands.active_commands for c in dh_cmds + ) dh_ppfs[stem] = _fake_PPFClassSpec( debputy_plugin_metadata, stem, diff --git a/src/debputy/commands/debputy_cmd/__main__.py b/src/debputy/commands/debputy_cmd/__main__.py index 218ca4f..731576e 100644 --- a/src/debputy/commands/debputy_cmd/__main__.py +++ b/src/debputy/commands/debputy_cmd/__main__.py @@ -43,6 +43,7 @@ from debputy.exceptions import ( from debputy.package_build.assemble_deb import ( assemble_debs, ) +from debputy.plugin.api.spec import INTEGRATION_MODE_DH_DEBPUTY_RRR try: from argcomplete import autocomplete @@ -66,9 +67,9 @@ from debputy.plugin.api.impl import ( find_related_implementation_files_for_plugin, parse_json_plugin_desc, ) -from debputy.dh_migration.migration import migrate_from_dh +from debputy.dh_migration.migration import migrate_from_dh, _check_migration_target from debputy.dh_migration.models import AcceptableMigrationIssues -from debputy.debhelper_emulation import ( +from debputy.dh.debhelper_emulation import ( dhe_pkgdir, ) @@ -634,12 +635,6 @@ def _run_tests_for_plugin(context: CommandContext) -> None: argparser=[ _add_packages_args, add_arg( - "--integration-mode", - dest="integration_mode", - default=None, - choices=["rrr"], - ), - add_arg( "output", metavar="output", help="Where to place the resulting packages. Should be a directory", @@ -657,7 +652,8 @@ def _run_tests_for_plugin(context: CommandContext) -> None: def _dh_integration_generate_debs(context: CommandContext) -> None: integrated_with_debhelper() parsed_args = context.parsed_args - is_dh_rrr_only_mode = parsed_args.integration_mode == "rrr" + integration_mode = context.resolve_integration_mode() + is_dh_rrr_only_mode = integration_mode == INTEGRATION_MODE_DH_DEBPUTY_RRR if is_dh_rrr_only_mode: problematic_plugins = list(context.requested_plugins()) problematic_plugins.extend(context.required_plugins()) @@ -900,6 +896,12 @@ def _json_output(data: Any) -> None: ) def _migrate_from_dh(context: CommandContext) -> None: parsed_args = context.parsed_args + + resolved_migration_target = _check_migration_target( + context.debian_dir, + parsed_args.migration_target, + ) + context.debputy_integration_mode = resolved_migration_target manifest = context.parse_manifest() acceptable_migration_issues = AcceptableMigrationIssues( frozenset( @@ -910,7 +912,7 @@ def _migrate_from_dh(context: CommandContext) -> None: manifest, acceptable_migration_issues, parsed_args.destructive, - parsed_args.migration_target, + resolved_migration_target, lambda p: context.parse_manifest(manifest_path=p), ) diff --git a/src/debputy/commands/debputy_cmd/context.py b/src/debputy/commands/debputy_cmd/context.py index e3cf501..0c184c7 100644 --- a/src/debputy/commands/debputy_cmd/context.py +++ b/src/debputy/commands/debputy_cmd/context.py @@ -24,10 +24,12 @@ from debputy.architecture_support import ( DpkgArchitectureBuildProcessValuesTable, dpkg_architecture_table, ) +from debputy.dh.dh_assistant import read_dh_addon_sequences from debputy.exceptions import DebputyRuntimeError from debputy.filesystem_scan import FSROOverlay from debputy.highlevel_manifest import HighLevelManifest from debputy.highlevel_manifest_parser import YAMLManifestParser +from debputy.integration_detection import determine_debputy_integration_mode from debputy.packages import ( SourcePackage, BinaryPackage, @@ -36,6 +38,7 @@ from debputy.packages import ( from debputy.plugin.api import VirtualPath from debputy.plugin.api.impl import load_plugin_features from debputy.plugin.api.feature_set import PluginProvidedFeatureSet +from debputy.plugin.api.spec import DebputyIntegrationMode from debputy.substitution import ( Substitution, VariableContext, @@ -100,6 +103,7 @@ class CommandContext: self._requested_plugins: Optional[Sequence[str]] = None self._plugins_loaded = False self._dctrl_parser: Optional[DctrlParser] = None + self.debputy_integration_mode: Optional[DebputyIntegrationMode] = None self._dctrl_data: Optional[ Tuple[ "SourcePackage", @@ -288,6 +292,20 @@ class CommandContext: debian_control = self.debian_dir.get("control") return debian_control is not None + def resolve_integration_mode(self) -> DebputyIntegrationMode: + integration_mode = self.debputy_integration_mode + if integration_mode is None: + r = read_dh_addon_sequences(self.debian_dir) + bd_sequences, dr_sequences, _ = r + all_sequences = bd_sequences | dr_sequences + integration_mode = determine_debputy_integration_mode(all_sequences) + if integration_mode is None: + _error( + "Cannot resolve the integration mode expected for this package. Is this package using `debputy`?" + ) + self.debputy_integration_mode = integration_mode + return integration_mode + def manifest_parser( self, *, @@ -302,6 +320,7 @@ class CommandContext: manifest_path = self.parsed_args.debputy_manifest if manifest_path is None: manifest_path = os.path.join(self.debian_dir.fs_path, "debputy.manifest") + debian_dir = self.debian_dir return YAMLManifestParser( manifest_path, source_package, @@ -311,6 +330,7 @@ class CommandContext: dctrl_parser.dpkg_arch_query_table, dctrl_parser.build_env, self.load_plugins(), + self.resolve_integration_mode(), debian_dir=self.debian_dir, ) diff --git a/src/debputy/deb_packaging_support.py b/src/debputy/deb_packaging_support.py index c88841f..897c2b6 100644 --- a/src/debputy/deb_packaging_support.py +++ b/src/debputy/deb_packaging_support.py @@ -38,7 +38,7 @@ from debian.deb822 import Deb822 from debputy._deb_options_profiles import DebBuildOptionsAndProfiles from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable -from debputy.debhelper_emulation import ( +from debputy.dh.debhelper_emulation import ( dhe_install_pkg_file_as_ctrl_file_if_present, dhe_dbgsym_root_dir, ) diff --git a/src/debputy/dh/__init__.py b/src/debputy/dh/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/debputy/dh/__init__.py diff --git a/src/debputy/debhelper_emulation.py b/src/debputy/dh/debhelper_emulation.py index 8242a32..b41bbff 100644 --- a/src/debputy/debhelper_emulation.py +++ b/src/debputy/dh/debhelper_emulation.py @@ -13,12 +13,9 @@ from typing import ( cast, Mapping, Any, - Set, List, ) -from debian.deb822 import Deb822 - from debputy.packages import BinaryPackage from debputy.plugin.api import VirtualPath from debputy.substitution import Substitution @@ -237,38 +234,3 @@ def dhe_install_path(source: str, dest: str, mode: int) -> None: print_command("install", "-p", f"-m{oct(mode)[2:]}", source, dest) shutil.copyfile(source, dest) os.chmod(dest, mode) - - -_FIND_DH_WITH = re.compile(r"--with(?:\s+|=)(\S+)") -_DEP_REGEX = re.compile("^([a-z0-9][-+.a-z0-9]+)", re.ASCII) - - -def parse_drules_for_addons(lines: Iterable[str], sequences: Set[str]) -> bool: - saw_dh = False - for line in lines: - if not line.startswith("\tdh "): - continue - saw_dh = True - for match in _FIND_DH_WITH.finditer(line): - sequence_def = match.group(1) - sequences.update(sequence_def.split(",")) - return saw_dh - - -def extract_dh_addons_from_control( - source_paragraph: Union[Mapping[str, str], Deb822], - sequences: Set[str], -) -> None: - for f in ("Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch"): - field = source_paragraph.get(f) - if not field: - continue - - for dep_clause in (d.strip() for d in field.split(",")): - match = _DEP_REGEX.match(dep_clause.strip()) - if not match: - continue - dep = match.group(1) - if not dep.startswith("dh-sequence-"): - continue - sequences.add(dep[12:]) diff --git a/src/debputy/dh/dh_assistant.py b/src/debputy/dh/dh_assistant.py new file mode 100644 index 0000000..ba8c14f --- /dev/null +++ b/src/debputy/dh/dh_assistant.py @@ -0,0 +1,125 @@ +import dataclasses +import json +import re +import subprocess +from typing import Iterable, FrozenSet, Optional, List, Union, Mapping, Any, Set, Tuple + +from debian.deb822 import Deb822 + +from debputy.plugin.api import VirtualPath +from debputy.util import _info + +_FIND_DH_WITH = re.compile(r"--with(?:\s+|=)(\S+)") +_DEP_REGEX = re.compile("^([a-z0-9][-+.a-z0-9]+)", re.ASCII) + + +@dataclasses.dataclass(frozen=True, slots=True) +class DhListCommands: + active_commands: FrozenSet[str] + disabled_commands: FrozenSet[str] + + +@dataclasses.dataclass(frozen=True, slots=True) +class DhSequencerData: + sequences: FrozenSet[str] + uses_dh_sequencer: bool + + +def _parse_dh_cmd_list( + cmd_list: Optional[List[Union[Mapping[str, Any], object]]] +) -> Iterable[str]: + if not isinstance(cmd_list, list): + return + + for command in cmd_list: + if not isinstance(command, dict): + continue + command_name = command.get("command") + if isinstance(command_name, str): + yield command_name + + +def resolve_active_and_inactive_dh_commands( + dh_rules_addons: Iterable[str], + *, + source_root: Optional[str] = None, +) -> DhListCommands: + cmd = ["dh_assistant", "list-commands", "--output-format=json"] + if dh_rules_addons: + addons = ",".join(dh_rules_addons) + cmd.append(f"--with={addons}") + try: + output = subprocess.check_output( + cmd, + stderr=subprocess.DEVNULL, + cwd=source_root, + ) + except (subprocess.CalledProcessError, FileNotFoundError): + return DhListCommands( + frozenset(), + frozenset(), + ) + else: + result = json.loads(output) + active_commands = frozenset(_parse_dh_cmd_list(result.get("commands"))) + disabled_commands = frozenset( + _parse_dh_cmd_list(result.get("disabled-commands")) + ) + return DhListCommands( + active_commands, + disabled_commands, + ) + + +def parse_drules_for_addons(lines: Iterable[str], sequences: Set[str]) -> bool: + saw_dh = False + for line in lines: + if not line.startswith("\tdh "): + continue + saw_dh = True + for match in _FIND_DH_WITH.finditer(line): + sequence_def = match.group(1) + sequences.update(sequence_def.split(",")) + return saw_dh + + +def extract_dh_addons_from_control( + source_paragraph: Union[Mapping[str, str], Deb822], + sequences: Set[str], +) -> None: + for f in ("Build-Depends", "Build-Depends-Indep", "Build-Depends-Arch"): + field = source_paragraph.get(f) + if not field: + continue + + for dep_clause in (d.strip() for d in field.split(",")): + match = _DEP_REGEX.match(dep_clause.strip()) + if not match: + continue + dep = match.group(1) + if not dep.startswith("dh-sequence-"): + continue + sequences.add(dep[12:]) + + +def read_dh_addon_sequences( + debian_dir: VirtualPath, +) -> Optional[Tuple[Set[str], Set[str], bool]]: + ctrl_file = debian_dir.get("control") + if ctrl_file: + dr_sequences: Set[str] = set() + bd_sequences: Set[str] = set() + + drules = debian_dir.get("rules") + saw_dh = False + if drules and drules.is_file: + with drules.open() as fd: + saw_dh = parse_drules_for_addons(fd, dr_sequences) + + with ctrl_file.open() as fd: + ctrl = list(Deb822.iter_paragraphs(fd)) + source_paragraph = ctrl[0] if ctrl else {} + + extract_dh_addons_from_control(source_paragraph, bd_sequences) + return bd_sequences, dr_sequences, saw_dh + return None diff --git a/src/debputy/dh_migration/migration.py b/src/debputy/dh_migration/migration.py index 59a7ee4..f7b7d9e 100644 --- a/src/debputy/dh_migration/migration.py +++ b/src/debputy/dh_migration/migration.py @@ -7,13 +7,13 @@ from typing import Optional, List, Callable, Set, Container from debian.deb822 import Deb822 -from debputy.debhelper_emulation import CannotEmulateExecutableDHConfigFile +from debputy.dh.debhelper_emulation import CannotEmulateExecutableDHConfigFile from debputy.dh_migration.migrators import MIGRATORS from debputy.dh_migration.migrators_impl import ( - read_dh_addon_sequences, - MIGRATION_TARGET_DH_DEBPUTY, - MIGRATION_TARGET_DH_DEBPUTY_RRR, + INTEGRATION_MODE_DH_DEBPUTY, + INTEGRATION_MODE_DH_DEBPUTY_RRR, ) +from debputy.dh.dh_assistant import read_dh_addon_sequences from debputy.dh_migration.models import ( FeatureMigration, AcceptableMigrationIssues, @@ -21,8 +21,10 @@ from debputy.dh_migration.models import ( ConflictingChange, ) from debputy.highlevel_manifest import HighLevelManifest +from debputy.integration_detection import determine_debputy_integration_mode from debputy.manifest_parser.exceptions import ManifestParseException from debputy.plugin.api import VirtualPath +from debputy.plugin.api.spec import DebputyIntegrationMode from debputy.util import _error, _warn, _info, escape_shell, assume_not_none @@ -140,49 +142,38 @@ def _requested_debputy_plugins(debian_dir: VirtualPath) -> Optional[Set[str]]: return plugins -def determine_debputy_integration_level(sequences: Container[str]) -> Optional[str]: - has_zz_debputy = "zz-debputy" in sequences or "debputy" in sequences - has_zz_debputy_rrr = "zz-debputy-rrr" in sequences - if has_zz_debputy: - return MIGRATION_TARGET_DH_DEBPUTY - if has_zz_debputy_rrr: - return MIGRATION_TARGET_DH_DEBPUTY_RRR - return None - - def _check_migration_target( debian_dir: VirtualPath, - migration_target: Optional[str], -) -> str: + migration_target: Optional[DebputyIntegrationMode], +) -> DebputyIntegrationMode: r = read_dh_addon_sequences(debian_dir) if r is None and migration_target is None: _error("debian/control is missing and no migration target was provided") bd_sequences, dr_sequences, _ = r all_sequences = bd_sequences | dr_sequences - has_zz_debputy = "zz-debputy" in all_sequences or "debputy" in all_sequences - has_zz_debputy_rrr = "zz-debputy-rrr" in all_sequences - has_any_existing = has_zz_debputy or has_zz_debputy_rrr + detected_migration_target = determine_debputy_integration_mode(all_sequences) - if migration_target == "dh-sequence-zz-debputy-rrr" and has_zz_debputy: + if ( + migration_target == INTEGRATION_MODE_DH_DEBPUTY_RRR + and detected_migration_target == INTEGRATION_MODE_DH_DEBPUTY + ): _error("Cannot migrate from (zz-)debputy to zz-debputy-rrr") - if has_zz_debputy_rrr and not has_zz_debputy: - resolved_migration_target = MIGRATION_TARGET_DH_DEBPUTY_RRR - else: - resolved_migration_target = MIGRATION_TARGET_DH_DEBPUTY - if migration_target is not None: resolved_migration_target = migration_target - - if has_any_existing: - _info( - f'Using "{resolved_migration_target}" as migration target based on the packaging' - ) + _info(f'Using "{resolved_migration_target}" as migration target as requested') else: - _info( - f'Using "{resolved_migration_target}" as default migration target. Use --migration-target to choose!' - ) + if detected_migration_target is not None: + _info( + f'Using "{detected_migration_target}" as migration target based on the packaging' + ) + else: + detected_migration_target = INTEGRATION_MODE_DH_DEBPUTY + _info( + f'Using "{detected_migration_target}" as default migration target. Use --migration-target to choose!' + ) + resolved_migration_target = detected_migration_target return resolved_migration_target @@ -191,7 +182,7 @@ def migrate_from_dh( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, permit_destructive_changes: Optional[bool], - migration_target: Optional[str], + migration_target: DebputyIntegrationMode, manifest_parser_factory: Callable[[str], HighLevelManifest], ) -> None: migrations = [] @@ -204,17 +195,15 @@ def migrate_from_dh( debian_dir = manifest.debian_dir mutable_manifest = assume_not_none(manifest.mutable_manifest) - resolved_migration_target = _check_migration_target(debian_dir, migration_target) - try: - for migrator in MIGRATORS[resolved_migration_target]: + for migrator in MIGRATORS[migration_target]: feature_migration = FeatureMigration(migrator.__name__) migrator( debian_dir, manifest, acceptable_migration_issues, feature_migration, - resolved_migration_target, + migration_target, ) migrations.append(feature_migration) except CannotEmulateExecutableDHConfigFile as e: diff --git a/src/debputy/dh_migration/migrators.py b/src/debputy/dh_migration/migrators.py index 7e056ae..8eff679 100644 --- a/src/debputy/dh_migration/migrators.py +++ b/src/debputy/dh_migration/migrators.py @@ -21,12 +21,15 @@ from debputy.dh_migration.migrators_impl import ( migrate_dh_installsystemd_files, detect_obsolete_substvars, detect_dh_addons_zz_debputy_rrr, - MIGRATION_TARGET_DH_DEBPUTY, - MIGRATION_TARGET_DH_DEBPUTY_RRR, ) from debputy.dh_migration.models import AcceptableMigrationIssues, FeatureMigration from debputy.highlevel_manifest import HighLevelManifest from debputy.plugin.api import VirtualPath +from debputy.plugin.api.spec import ( + DebputyIntegrationMode, + INTEGRATION_MODE_DH_DEBPUTY_RRR, + INTEGRATION_MODE_DH_DEBPUTY, +) Migrator = Callable[ [VirtualPath, HighLevelManifest, AcceptableMigrationIssues, FeatureMigration, str], @@ -34,14 +37,14 @@ Migrator = Callable[ ] -MIGRATORS: Mapping[str, List[Migrator]] = { - MIGRATION_TARGET_DH_DEBPUTY_RRR: [ +MIGRATORS: Mapping[DebputyIntegrationMode, List[Migrator]] = { + INTEGRATION_MODE_DH_DEBPUTY_RRR: [ migrate_dh_hook_targets, migrate_misspelled_readme_debian_files, detect_dh_addons_zz_debputy_rrr, detect_obsolete_substvars, ], - MIGRATION_TARGET_DH_DEBPUTY: [ + INTEGRATION_MODE_DH_DEBPUTY: [ detect_unsupported_zz_debputy_features, detect_pam_files, migrate_dh_hook_targets, diff --git a/src/debputy/dh_migration/migrators_impl.py b/src/debputy/dh_migration/migrators_impl.py index 48ec1e0..91ea8cd 100644 --- a/src/debputy/dh_migration/migrators_impl.py +++ b/src/debputy/dh_migration/migrators_impl.py @@ -24,12 +24,13 @@ from debian.deb822 import Deb822 from debputy import DEBPUTY_DOC_ROOT_DIR from debputy.architecture_support import dpkg_architecture_table from debputy.deb_packaging_support import dpkg_field_list_pkg_dep -from debputy.debhelper_emulation import ( +from debputy.dh.debhelper_emulation import ( dhe_filedoublearray, DHConfigFileLine, dhe_pkgfile, - parse_drules_for_addons, - extract_dh_addons_from_control, +) +from debputy.dh.dh_assistant import ( + read_dh_addon_sequences, ) from debputy.dh_migration.models import ( ConflictingChange, @@ -47,6 +48,10 @@ from debputy.highlevel_manifest import ( from debputy.installations import MAN_GUESS_FROM_BASENAME, MAN_GUESS_LANG_FROM_PATH from debputy.packages import BinaryPackage from debputy.plugin.api import VirtualPath +from debputy.plugin.api.spec import ( + INTEGRATION_MODE_DH_DEBPUTY_RRR, + INTEGRATION_MODE_DH_DEBPUTY, +) from debputy.util import ( _error, PKGVERSION_REGEX, @@ -56,13 +61,9 @@ from debputy.util import ( has_glob_magic, ) -MIGRATION_TARGET_DH_DEBPUTY_RRR = "dh-sequence-zz-debputy-rrr" -MIGRATION_TARGET_DH_DEBPUTY = "dh-sequence-zz-debputy" - - # Align with debputy.py DH_COMMANDS_REPLACED = { - MIGRATION_TARGET_DH_DEBPUTY_RRR: frozenset( + INTEGRATION_MODE_DH_DEBPUTY_RRR: frozenset( { "dh_fixperms", "dh_shlibdeps", @@ -71,7 +72,7 @@ DH_COMMANDS_REPLACED = { "dh_builddeb", } ), - MIGRATION_TARGET_DH_DEBPUTY: frozenset( + INTEGRATION_MODE_DH_DEBPUTY: frozenset( { "dh_install", "dh_installdocs", @@ -1501,29 +1502,6 @@ def detect_obsolete_substvars( ) -def read_dh_addon_sequences( - debian_dir: VirtualPath, -) -> Optional[Tuple[Set[str], Set[str], bool]]: - ctrl_file = debian_dir.get("control") - if ctrl_file: - dr_sequences: Set[str] = set() - bd_sequences: Set[str] = set() - - drules = debian_dir.get("rules") - saw_dh = False - if drules and drules.is_file: - with drules.open() as fd: - saw_dh = parse_drules_for_addons(fd, dr_sequences) - - with ctrl_file.open() as fd: - ctrl = list(Deb822.iter_paragraphs(fd)) - source_paragraph = ctrl[0] if ctrl else {} - - extract_dh_addons_from_control(source_paragraph, bd_sequences) - return bd_sequences, dr_sequences, saw_dh - return None - - def detect_dh_addons_zz_debputy_rrr( debian_dir: VirtualPath, _manifest: HighLevelManifest, diff --git a/src/debputy/highlevel_manifest.py b/src/debputy/highlevel_manifest.py index d3898ad..9bdc225 100644 --- a/src/debputy/highlevel_manifest.py +++ b/src/debputy/highlevel_manifest.py @@ -26,7 +26,7 @@ from ._deb_options_profiles import DebBuildOptionsAndProfiles from ._manifest_constants import * from .architecture_support import DpkgArchitectureBuildProcessValuesTable from .builtin_manifest_rules import builtin_mode_normalization_rules -from .debhelper_emulation import ( +from debputy.dh.debhelper_emulation import ( dhe_dbgsym_root_dir, assert_no_dbgsym_migration, read_dbgsym_file, diff --git a/src/debputy/highlevel_manifest_parser.py b/src/debputy/highlevel_manifest_parser.py index c5fb410..dd97d58 100644 --- a/src/debputy/highlevel_manifest_parser.py +++ b/src/debputy/highlevel_manifest_parser.py @@ -56,6 +56,7 @@ from .plugin.api.impl_types import ( PackageContextData, ) from .plugin.api.feature_set import PluginProvidedFeatureSet +from .plugin.api.spec import DebputyIntegrationMode from .yaml import YAMLError, MANIFEST_YAML try: @@ -116,6 +117,7 @@ class HighLevelManifestParser(ParserContextData): dpkg_arch_query_table: DpkgArchTable, build_env: DebBuildOptionsAndProfiles, plugin_provided_feature_set: PluginProvidedFeatureSet, + debputy_integration_mode: DebputyIntegrationMode, *, # Available for testing purposes only debian_dir: Union[str, VirtualPath] = "./debian", @@ -132,6 +134,7 @@ class HighLevelManifestParser(ParserContextData): self._build_env = build_env self._package_state_stack: List[PackageTransformationDefinition] = [] self._plugin_provided_feature_set = plugin_provided_feature_set + self._debputy_integration_mode = debputy_integration_mode self._declared_variables = {} if isinstance(debian_dir, str): @@ -314,6 +317,14 @@ class HighLevelManifestParser(ParserContextData): def is_in_binary_package_state(self) -> bool: return bool(self._package_state_stack) + @property + def debputy_integration_mode(self) -> DebputyIntegrationMode: + return self._debputy_integration_mode + + @debputy_integration_mode.setter + def debputy_integration_mode(self, new_value: DebputyIntegrationMode) -> None: + self._debputy_integration_mode = new_value + def _transform_dpkg_maintscript_helpers_to_snippets(self) -> None: package_state = self.current_binary_package_state for dmh in package_state.dpkg_maintscript_helper_snippets: diff --git a/src/debputy/integration_detection.py b/src/debputy/integration_detection.py new file mode 100644 index 0000000..f412268 --- /dev/null +++ b/src/debputy/integration_detection.py @@ -0,0 +1,21 @@ +from typing import Container, Optional + +from debputy.plugin.api.spec import ( + DebputyIntegrationMode, + INTEGRATION_MODE_DH_DEBPUTY_RRR, + INTEGRATION_MODE_DH_DEBPUTY, +) + + +def determine_debputy_integration_mode( + all_sequences: Container[str], +) -> Optional[DebputyIntegrationMode]: + + has_zz_debputy = "zz-debputy" in all_sequences or "debputy" in all_sequences + has_zz_debputy_rrr = "zz-debputy-rrr" in all_sequences + has_any_existing = has_zz_debputy or has_zz_debputy_rrr + if has_zz_debputy_rrr: + return INTEGRATION_MODE_DH_DEBPUTY_RRR + if has_any_existing: + return INTEGRATION_MODE_DH_DEBPUTY + return None diff --git a/src/debputy/linting/lint_util.py b/src/debputy/linting/lint_util.py index 30e1177..ddce7c2 100644 --- a/src/debputy/linting/lint_util.py +++ b/src/debputy/linting/lint_util.py @@ -17,6 +17,16 @@ from typing import ( cast, ) +from debputy.commands.debputy_cmd.output import OutputStylingBase +from debputy.dh.dh_assistant import ( + extract_dh_addons_from_control, + DhSequencerData, + parse_drules_for_addons, +) +from debputy.filesystem_scan import VirtualPathBase +from debputy.integration_detection import determine_debputy_integration_mode +from debputy.lsp.diagnostics import LintSeverity +from debputy.lsp.vendoring._deb822_repro import Deb822FileElement, parse_deb822_file from debputy.lsprotocol.types import ( Position, Range, @@ -24,15 +34,9 @@ from debputy.lsprotocol.types import ( DiagnosticSeverity, TextEdit, ) - -from debputy.commands.debputy_cmd.output import OutputStylingBase -from debputy.debhelper_emulation import extract_dh_addons_from_control -from debputy.dh_migration.migration import determine_debputy_integration_level -from debputy.filesystem_scan import VirtualPathBase -from debputy.lsp.diagnostics import LintSeverity -from debputy.lsp.vendoring._deb822_repro import Deb822FileElement, parse_deb822_file from debputy.packages import SourcePackage, BinaryPackage from debputy.plugin.api.feature_set import PluginProvidedFeatureSet +from debputy.plugin.api.spec import DebputyIntegrationMode from debputy.util import _warn if TYPE_CHECKING: @@ -49,15 +53,14 @@ FormatterImpl = Callable[["LintState"], Optional[Sequence[TextEdit]]] @dataclasses.dataclass(slots=True) class DebputyMetadata: - debputy_integration_level: Optional[str] + debputy_integration_mode: Optional[DebputyIntegrationMode] @classmethod - def from_data(cls, source_data: Optional[SourcePackage]) -> typing.Self: - sequences = set() - if source_data: - extract_dh_addons_from_control(source_data.fields, sequences) - integration_level = determine_debputy_integration_level(sequences) - return cls(integration_level) + def from_data(cls, dh_sequencer_data: DhSequencerData) -> typing.Self: + integration_mode = determine_debputy_integration_mode( + dh_sequencer_data.sequences + ) + return cls(integration_mode) class LintState: @@ -116,7 +119,11 @@ class LintState: @property def debputy_metadata(self) -> DebputyMetadata: - return DebputyMetadata.from_data(self.source_package) + return DebputyMetadata.from_data(self.dh_sequencer_data) + + @property + def dh_sequencer_data(self) -> DhSequencerData: + raise NotImplementedError @dataclasses.dataclass(slots=True) @@ -132,6 +139,7 @@ class LintStateImpl(LintState): binary_packages: Optional[Mapping[str, BinaryPackage]] = None effective_preference: Optional["EffectivePreference"] = None _parsed_cache: Optional[Deb822FileElement] = None + _dh_sequencer_cache: Optional[DhSequencerData] = None @property def doc_uri(self) -> str: @@ -155,6 +163,28 @@ class LintStateImpl(LintState): self._parsed_cache = cache return cache + @property + def dh_sequencer_data(self) -> DhSequencerData: + dh_sequencer_cache = self._dh_sequencer_cache + if dh_sequencer_cache is None: + debian_dir = self.debian_dir + dh_sequences: typing.Set[str] = set() + saw_dh = False + src_pkg = self.source_package + drules = debian_dir.get("rules") if debian_dir is not None else None + if drules and drules.is_file: + with drules.open() as fd: + saw_dh = parse_drules_for_addons(fd, dh_sequences) + if src_pkg: + extract_dh_addons_from_control(src_pkg.fields, dh_sequences) + + dh_sequencer_cache = DhSequencerData( + frozenset(dh_sequences), + saw_dh, + ) + self._dh_sequencer_cache = dh_sequencer_cache + return dh_sequencer_cache + class LintDiagnosticResultState(IntEnum): REPORTED = 1 diff --git a/src/debputy/lsp/debputy_ls.py b/src/debputy/lsp/debputy_ls.py index ed076cd..eb4162f 100644 --- a/src/debputy/lsp/debputy_ls.py +++ b/src/debputy/lsp/debputy_ls.py @@ -9,8 +9,14 @@ from typing import ( TYPE_CHECKING, Tuple, Literal, + Set, ) +from debputy.dh.dh_assistant import ( + parse_drules_for_addons, + DhSequencerData, + extract_dh_addons_from_control, +) from debputy.lsprotocol.types import MarkupKind from debputy.filesystem_scan import FSROOverlay, VirtualPathBase @@ -163,6 +169,24 @@ class SalsaCICache(FileCache): self.parsed_content = None +@dataclasses.dataclass(slots=True) +class DebianRulesCache(FileCache): + sequences: Optional[Set[str]] = None + saw_dh: bool = False + + def _update_cache(self, doc: "TextDocument", source: str) -> None: + sequences = set() + self.saw_dh = parse_drules_for_addons( + source.splitlines(), + sequences, + ) + self.sequences = sequences + + def _clear_cache(self) -> None: + self.sequences = None + self.saw_dh = False + + class LSProvidedLintState(LintState): def __init__( self, @@ -208,6 +232,11 @@ class LSProvidedLintState(LintState): ) for p in ("salsa-ci.yml", os.path.join("..", ".gitlab-ci.yml")) ] + drules_path = os.path.join(debian_dir_path, "rules") + self._drules_cache = DebianRulesCache( + from_fs_path(drules_path) if doc.path != drules_path else doc.uri, + drules_path, + ) @property def plugin_feature_set(self) -> PluginProvidedFeatureSet: @@ -287,6 +316,24 @@ class LSProvidedLintState(LintState): def salsa_ci(self) -> Optional[CommentedMap]: return None + @property + def dh_sequencer_data(self) -> DhSequencerData: + dh_sequences: Set[str] = set() + saw_dh = False + src_pkg = self.source_package + drules_cache = self._drules_cache + if drules_cache.resolve_cache(self._ls): + saw_dh = drules_cache.saw_dh + if drules_cache.sequences: + dh_sequences.update(drules_cache.sequences) + if src_pkg: + extract_dh_addons_from_control(src_pkg.fields, dh_sequences) + + return DhSequencerData( + frozenset(dh_sequences), + saw_dh, + ) + def _preference( client_preference: Optional[List[MarkupKind]], diff --git a/src/debputy/lsp/lsp_debian_control.py b/src/debputy/lsp/lsp_debian_control.py index e91a43e..5bb6265 100644 --- a/src/debputy/lsp/lsp_debian_control.py +++ b/src/debputy/lsp/lsp_debian_control.py @@ -36,6 +36,7 @@ from debputy.lsp.lsp_features import ( lsp_will_save_wait_until, lsp_format_document, LanguageDispatch, + lsp_text_doc_inlay_hints, ) from debputy.lsp.lsp_generic_deb822 import ( deb822_completer, @@ -57,6 +58,7 @@ from debputy.lsp.text_util import ( normalize_dctrl_field_name, LintCapablePositionCodec, te_range_to_lsp, + te_position_to_lsp, ) from debputy.lsp.vendoring._deb822_repro import ( Deb822FileElement, @@ -86,6 +88,9 @@ from debputy.lsprotocol.types import ( WillSaveTextDocumentParams, TextEdit, DocumentFormattingParams, + InlayHintParams, + InlayHint, + InlayHintLabelPart, ) from debputy.util import detect_possible_typo @@ -454,6 +459,69 @@ def _debian_control_folding_ranges( return deb822_folding_ranges(ls, params, _DCTRL_FILE_METADATA) +@lsp_text_doc_inlay_hints(_LANGUAGE_IDS) +def _doc_inlay_hint( + ls: "DebputyLanguageServer", + params: InlayHintParams, +) -> Optional[List[InlayHint]]: + doc = ls.workspace.get_text_document(params.text_document.uri) + lint_state = ls.lint_state(doc) + deb822_file = lint_state.parsed_deb822_file_content + if not deb822_file: + return None + inlay_hints = [] + stanzas = list(deb822_file) + if len(stanzas) < 2: + return None + source_stanza = stanzas[0] + source_stanza_pos = source_stanza.position_in_file() + for stanza_no, stanza in enumerate(deb822_file): + stanza_range = stanza.range_in_parent() + if stanza_no < 1: + continue + pkg_kvpair = stanza.get_kvpair_element("Package", use_get=True) + if pkg_kvpair is None: + continue + + inlay_hint_pos_te = pkg_kvpair.range_in_parent().end_pos.relative_to( + stanza_range.start_pos + ) + inlay_hint_pos = doc.position_codec.position_to_client_units( + lint_state.lines, + te_position_to_lsp(inlay_hint_pos_te), + ) + stanza_def = _DCTRL_FILE_METADATA.classify_stanza(stanza, stanza_no) + for known_field in stanza_def.stanza_fields.values(): + if not known_field.inherits_from_source or known_field.name in stanza: + continue + + inherited_value = source_stanza.get(known_field.name) + if inherited_value is not None: + kvpair = source_stanza.get_kvpair_element(known_field.name) + value_range_te = kvpair.range_in_parent().relative_to(source_stanza_pos) + value_range = doc.position_codec.range_to_client_units( + lint_state.lines, + te_range_to_lsp(value_range_te), + ) + inlay_hints.append( + InlayHint( + inlay_hint_pos, + [ + InlayHintLabelPart( + f"{known_field.name}: {inherited_value}\n", + tooltip="Inherited from Source stanza", + location=Location( + params.text_document.uri, + value_range, + ), + ), + ], + ) + ) + + return inlay_hints + + def _paragraph_representation_field( paragraph: Deb822ParagraphElement, ) -> Deb822KeyValuePairElement: @@ -983,10 +1051,15 @@ def _detect_misspelled_packaging_files( binary_packages = lint_state.binary_packages if debian_dir is None or binary_packages is None: return + + dh_sequencer_data = lint_state.dh_sequencer_data + all_pkg_file_data, _, _, _ = scan_debian_dir( lint_state.plugin_feature_set, binary_packages, debian_dir, + uses_dh_sequencer=dh_sequencer_data.uses_dh_sequencer, + dh_sequences=dh_sequencer_data.sequences, ) stanza_ranges = { p: (a, r) diff --git a/src/debputy/lsp/lsp_debian_debputy_manifest.py b/src/debputy/lsp/lsp_debian_debputy_manifest.py index fd7e6b0..15e9aa6 100644 --- a/src/debputy/lsp/lsp_debian_debputy_manifest.py +++ b/src/debputy/lsp/lsp_debian_debputy_manifest.py @@ -9,22 +9,7 @@ from typing import ( Literal, get_args, get_origin, -) - -from debputy.lsprotocol.types import ( - Diagnostic, - TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, - Position, - Range, - DiagnosticSeverity, - HoverParams, - Hover, - TEXT_DOCUMENT_CODE_ACTION, - CompletionParams, - CompletionList, - CompletionItem, - DiagnosticRelatedInformation, - Location, + Container, ) from debputy.highlevel_manifest import MANIFEST_YAML @@ -47,6 +32,21 @@ from debputy.lsp.quickfixes import propose_correct_text_quick_fix from debputy.lsp.text_util import ( LintCapablePositionCodec, ) +from debputy.lsprotocol.types import ( + Diagnostic, + TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, + Position, + Range, + DiagnosticSeverity, + HoverParams, + Hover, + TEXT_DOCUMENT_CODE_ACTION, + CompletionParams, + CompletionList, + CompletionItem, + DiagnosticRelatedInformation, + Location, +) from debputy.manifest_parser.base_types import DebputyDispatchableType from debputy.manifest_parser.declarative_parser import ( AttributeDescription, @@ -65,6 +65,8 @@ from debputy.plugin.api.impl_types import ( InPackageContextParser, DeclarativeValuelessKeywordInputParser, ) +from debputy.plugin.api.spec import DebputyIntegrationMode +from debputy.plugin.debputy.private_api import Capability, load_libcap from debputy.util import _info, detect_possible_typo from debputy.yaml.compat import ( Node, @@ -147,29 +149,51 @@ def _lint_debian_debputy_manifest( feature_set = lint_state.plugin_feature_set pg = feature_set.manifest_parser_generator root_parser = pg.dispatchable_object_parsers[OPARSER_MANIFEST_ROOT] + debputy_integration_mode = lint_state.debputy_metadata.debputy_integration_mode + diagnostics.extend( _lint_content( lint_state, pg, root_parser, + debputy_integration_mode, content, ) ) return diagnostics -def _unknown_key( - key: Optional[str], - expected_keys: Iterable[str], +def _integration_mode_allows_key( + debputy_integration_mode: Optional[DebputyIntegrationMode], + expected_debputy_integration_modes: Optional[Container[DebputyIntegrationMode]], + key: str, line: int, col: int, lines: List[str], position_codec: LintCapablePositionCodec, - *, - message_format: str = 'Unknown or unsupported key "{key}".', -) -> Tuple["Diagnostic", Optional[str]]: +) -> Iterable["Diagnostic"]: + if debputy_integration_mode is None or expected_debputy_integration_modes is None: + return + if debputy_integration_mode in expected_debputy_integration_modes: + return + key_range = _key_range(key, line, col, lines, position_codec) + yield Diagnostic( + key_range, + f'Feature "{key}" not supported in integration mode {debputy_integration_mode}', + DiagnosticSeverity.Error, + source="debputy", + ) + + +def _key_range( + key: str, + line: int, + col: int, + lines: List[str], + position_codec: LintCapablePositionCodec, +) -> Range: key_len = len(key) if key else 1 - key_range = position_codec.range_to_client_units( + return position_codec.range_to_client_units( lines, Range( Position( @@ -183,6 +207,19 @@ def _unknown_key( ), ) + +def _unknown_key( + key: Optional[str], + expected_keys: Iterable[str], + line: int, + col: int, + lines: List[str], + position_codec: LintCapablePositionCodec, + *, + message_format: str = 'Unknown or unsupported key "{key}".', +) -> Tuple["Diagnostic", Optional[str]]: + key_range = _key_range(key, line, col, lines, position_codec) + candidates = detect_possible_typo(key, expected_keys) if key is not None else () extra = "" corrected_key = None @@ -276,42 +313,96 @@ def _conflicting_key( ) +def _remaining_line(lint_state: LintState, line_no: int, pos_start: int) -> Range: + raw_line = lint_state.lines[line_no].rstrip() + pos_end = len(raw_line) + return lint_state.position_codec.range_to_client_units( + lint_state.lines, + Range( + Position( + line_no, + pos_start, + ), + Position( + line_no, + pos_end, + ), + ), + ) + + def _lint_attr_value( lint_state: LintState, attr: AttributeDescription, pg: ParserGenerator, + debputy_integration_mode: Optional[DebputyIntegrationMode], + key: str, value: Any, + pos: Tuple[int, int], ) -> Iterable["Diagnostic"]: - attr_type = attr.attribute_type - type_mapping = pg.get_mapped_type_from_target_type(attr_type) + target_attr_type = attr.attribute_type + type_mapping = pg.get_mapped_type_from_target_type(target_attr_type) + source_attr_type = target_attr_type if type_mapping is not None: - attr_type = type_mapping.source_type - orig = get_origin(attr_type) - valid_values: Sequence[Any] = tuple() + source_attr_type = type_mapping.source_type + orig = get_origin(source_attr_type) + valid_values: Optional[Sequence[Any]] = None if orig == Literal: valid_values = get_args(attr.attribute_type) elif orig == bool or attr.attribute_type == bool: - valid_values = ("true", "false") - elif isinstance(attr_type, type) and issubclass(attr_type, DebputyDispatchableType): - parser = pg.dispatch_parser_table_for(attr_type) - yield from _lint_content( - lint_state, - pg, - parser, - value, - ) - return + valid_values = (True, False) + elif isinstance(target_attr_type, type): + if issubclass(target_attr_type, Capability): + has_libcap, _, is_valid_cap = load_libcap() + if has_libcap and not is_valid_cap(value): + line_no, cursor_pos = pos + cap_range = _remaining_line(lint_state, line_no, cursor_pos) + yield Diagnostic( + cap_range, + "The value could not be parsed as a capability via cap_from_text on this system", + DiagnosticSeverity.Warning, + source="debputy", + ) + return + if issubclass(target_attr_type, DebputyDispatchableType): + parser = pg.dispatch_parser_table_for(target_attr_type) + yield from _lint_content( + lint_state, + pg, + parser, + debputy_integration_mode, + value, + ) + return - if value in valid_values: + if valid_values is None or value in valid_values: return - # TODO: Emit diagnostic for broken values - return + line_no, cursor_pos = pos + value_range = _remaining_line(lint_state, line_no, cursor_pos) + yield Diagnostic( + value_range, + f'Not a supported value for "{key}"', + DiagnosticSeverity.Error, + source="debputy", + data=DiagnosticData( + quickfixes=[ + propose_correct_text_quick_fix(_as_yaml_value(m)) for m in valid_values + ] + ), + ) + + +def _as_yaml_value(v: Any) -> str: + if isinstance(v, bool): + return str(v).lower() + return str(v) def _lint_declarative_mapping_input_parser( lint_state: LintState, pg: ParserGenerator, parser: DeclarativeMappingInputParser, + debputy_integration_mode: Optional[DebputyIntegrationMode], content: Any, ) -> Iterable["Diagnostic"]: if not isinstance(content, CommentedMap): @@ -340,7 +431,10 @@ def _lint_declarative_mapping_input_parser( lint_state, attr, pg, + debputy_integration_mode, + key, value, + lc.value(key), ) for forbidden_key in attr.conflicting_attributes: @@ -382,6 +476,7 @@ def _lint_content( lint_state: LintState, pg: ParserGenerator, parser: DeclarativeInputParser[Any], + debputy_integration_mode: Optional[DebputyIntegrationMode], content: Any, ) -> Iterable["Diagnostic"]: if isinstance(parser, DispatchingParserBase): @@ -390,8 +485,9 @@ def _lint_content( lc = content.lc for key, value in content.items(): is_known = parser.is_known_keyword(key) + line, col = lc.key(key) + orig_key = key if not is_known: - line, col = lc.key(key) diag, corrected_key = _unknown_key( key, parser.registered_keywords(), @@ -408,10 +504,20 @@ def _lint_content( if is_known: subparser = parser.parser_for(key) assert subparser is not None + yield from _integration_mode_allows_key( + debputy_integration_mode, + subparser.parser.expected_debputy_integration_mode, + orig_key, + line, + col, + lint_state.lines, + lint_state.position_codec, + ) yield from _lint_content( lint_state, pg, subparser.parser, + debputy_integration_mode, value, ) elif isinstance(parser, ListWrappedDeclarativeInputParser): @@ -419,7 +525,9 @@ def _lint_content( return subparser = parser.delegate for value in content: - yield from _lint_content(lint_state, pg, subparser, value) + yield from _lint_content( + lint_state, pg, subparser, debputy_integration_mode, value + ) elif isinstance(parser, InPackageContextParser): if not isinstance(content, CommentedMap): return @@ -438,12 +546,19 @@ def _lint_content( message_format='Unknown package "{key}".', ) yield diag - yield from _lint_content(lint_state, pg, parser.delegate, v) + yield from _lint_content( + lint_state, + pg, + parser.delegate, + debputy_integration_mode, + v, + ) elif isinstance(parser, DeclarativeMappingInputParser): yield from _lint_declarative_mapping_input_parser( lint_state, pg, parser, + debputy_integration_mode, content, ) diff --git a/src/debputy/lsp/lsp_debian_rules.py b/src/debputy/lsp/lsp_debian_rules.py index 7c6d627..bc31d52 100644 --- a/src/debputy/lsp/lsp_debian_rules.py +++ b/src/debputy/lsp/lsp_debian_rules.py @@ -1,6 +1,5 @@ import functools import itertools -import json import os import re import subprocess @@ -12,25 +11,15 @@ from typing import ( List, Iterator, Tuple, - Set, FrozenSet, ) -from debputy.lsprotocol.types import ( - CompletionItem, - Diagnostic, - Range, - Position, - DiagnosticSeverity, - CompletionList, - CompletionParams, - TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, - TEXT_DOCUMENT_CODE_ACTION, +from debputy.dh.dh_assistant import ( + resolve_active_and_inactive_dh_commands, + DhListCommands, ) - -from debputy.debhelper_emulation import parse_drules_for_addons -from debputy.dh_migration.migrators_impl import DH_COMMANDS_REPLACED from debputy.linting.lint_util import LintState +from debputy.lsp.debputy_ls import DebputyLanguageServer from debputy.lsp.diagnostics import DiagnosticData from debputy.lsp.lsp_features import ( lint_diagnostics, @@ -43,7 +32,18 @@ from debputy.lsp.spellchecking import spellcheck_line from debputy.lsp.text_util import ( LintCapablePositionCodec, ) -from debputy.util import _warn, detect_possible_typo +from debputy.lsprotocol.types import ( + CompletionItem, + Diagnostic, + Range, + Position, + DiagnosticSeverity, + CompletionList, + CompletionParams, + TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, + TEXT_DOCUMENT_CODE_ACTION, +) +from debputy.util import detect_possible_typo try: from debputy.lsp.vendoring._deb822_repro.locatable import ( @@ -200,15 +200,13 @@ def iter_make_lines( yield line_no, line -def _forbidden_hook_targets(lint_state: LintState) -> FrozenSet[str]: - debputy_integration_level = lint_state.debputy_metadata.debputy_integration_level - if debputy_integration_level is None: - return frozenset() - commands = DH_COMMANDS_REPLACED.get(debputy_integration_level) - if not commands: +def _forbidden_hook_targets(dh_commands: DhListCommands) -> FrozenSet[str]: + if not dh_commands.disabled_commands: return frozenset() return frozenset( - itertools.chain.from_iterable(_as_hook_targets(c) for c in commands) + itertools.chain.from_iterable( + _as_hook_targets(c) for c in dh_commands.disabled_commands + ) ) @@ -226,10 +224,16 @@ def _lint_debian_rules_impl( make_error = _run_make_dryrun(source_root, lines) if make_error is not None: diagnostics.append(make_error) - - all_dh_commands = _all_dh_commands(source_root, lines) - if all_dh_commands: - all_hook_targets = {ht for c in all_dh_commands for ht in _as_hook_targets(c)} + dh_sequencer_data = lint_state.dh_sequencer_data + dh_sequences = dh_sequencer_data.sequences + dh_commands = resolve_active_and_inactive_dh_commands( + dh_sequences, + source_root=source_root, + ) + if dh_commands.active_commands: + all_hook_targets = { + ht for c in dh_commands.active_commands for ht in _as_hook_targets(c) + } all_hook_targets.update(_KNOWN_TARGETS) source = "debputy (dh_assistant)" else: @@ -237,7 +241,7 @@ def _lint_debian_rules_impl( source = "debputy" missing_targets = {} - forbidden_hook_targets = _forbidden_hook_targets(lint_state) + forbidden_hook_targets = _forbidden_hook_targets(dh_commands) all_allowed_hook_targets = all_hook_targets - forbidden_hook_targets for line_no, line in iter_make_lines(lines, position_codec, diagnostics): @@ -323,42 +327,9 @@ def _lint_debian_rules_impl( return diagnostics -def _all_dh_commands(source_root: str, lines: List[str]) -> Optional[Sequence[str]]: - drules_sequences: Set[str] = set() - parse_drules_for_addons(lines, drules_sequences) - cmd = ["dh_assistant", "list-commands", "--output-format=json"] - if drules_sequences: - cmd.append(f"--with={','.join(drules_sequences)}") - try: - output = subprocess.check_output( - cmd, - stderr=subprocess.DEVNULL, - cwd=source_root, - ) - except (FileNotFoundError, subprocess.CalledProcessError) as e: - _warn(f"dh_assistant failed (dir: {source_root}): {str(e)}") - return None - data = json.loads(output) - commands_raw = data.get("commands") if isinstance(data, dict) else None - if not isinstance(commands_raw, list): - return None - - commands = [] - - for command in commands_raw: - if not isinstance(command, dict): - return None - command_name = command.get("command") - if not command_name: - return None - commands.append(command_name) - - return commands - - @lsp_completer(_LANGUAGE_IDS) def _debian_rules_completions( - ls: "LanguageServer", + ls: "DebputyLanguageServer", params: CompletionParams, ) -> Optional[Union[CompletionList, Sequence[CompletionItem]]]: doc = ls.workspace.get_text_document(params.text_document.uri) @@ -374,9 +345,18 @@ def _debian_rules_completions( return None source_root = os.path.dirname(os.path.dirname(doc.path)) - all_commands = _all_dh_commands(source_root, lines) - if all_commands is None: + dh_sequencer_data = ls.lint_state(doc).dh_sequencer_data + dh_sequences = dh_sequencer_data.sequences + dh_commands = resolve_active_and_inactive_dh_commands( + dh_sequences, + source_root=source_root, + ) + if not dh_commands.active_commands: return None - items = [CompletionItem(ht) for c in all_commands for ht in _as_hook_targets(c)] + items = [ + CompletionItem(ht) + for c in dh_commands.active_commands + for ht in _as_hook_targets(c) + ] return items diff --git a/src/debputy/lsp/lsp_dispatch.py b/src/debputy/lsp/lsp_dispatch.py index e5def62..27170d0 100644 --- a/src/debputy/lsp/lsp_dispatch.py +++ b/src/debputy/lsp/lsp_dispatch.py @@ -22,6 +22,7 @@ from debputy.lsp.lsp_features import ( FORMAT_FILE_HANDLERS, _DispatchRule, C, + TEXT_DOC_INLAY_HANDLERS, ) from debputy.util import _info from debputy.lsprotocol.types import ( @@ -30,9 +31,12 @@ from debputy.lsprotocol.types import ( TEXT_DOCUMENT_DID_CHANGE, TEXT_DOCUMENT_DID_OPEN, TEXT_DOCUMENT_COMPLETION, + TEXT_DOCUMENT_INLAY_HINT, CompletionList, CompletionItem, CompletionParams, + InlayHintParams, + InlayHint, TEXT_DOCUMENT_HOVER, TEXT_DOCUMENT_FOLDING_RANGE, FoldingRange, @@ -175,6 +179,20 @@ def _hover( ) +@DEBPUTY_LANGUAGE_SERVER.feature(TEXT_DOCUMENT_INLAY_HINT) +def _doc_inlay_hint( + ls: "DebputyLanguageServer", + params: InlayHintParams, +) -> Optional[List[InlayHint]]: + return _dispatch_standard_handler( + ls, + params.text_document.uri, + params, + TEXT_DOC_INLAY_HANDLERS, + "Inlay hint (doc) request", + ) + + @DEBPUTY_LANGUAGE_SERVER.feature(TEXT_DOCUMENT_CODE_ACTION) def _code_actions( ls: "DebputyLanguageServer", diff --git a/src/debputy/lsp/lsp_features.py b/src/debputy/lsp/lsp_features.py index c513523..2604959 100644 --- a/src/debputy/lsp/lsp_features.py +++ b/src/debputy/lsp/lsp_features.py @@ -73,6 +73,7 @@ FOLDING_RANGE_HANDLERS = {} SEMANTIC_TOKENS_FULL_HANDLERS = {} WILL_SAVE_WAIT_UNTIL_HANDLERS = {} FORMAT_FILE_HANDLERS = {} +TEXT_DOC_INLAY_HANDLERS = {} _ALIAS_OF = {} @@ -197,6 +198,12 @@ def lsp_hover( return _registering_wrapper(file_formats, HOVER_HANDLERS) +def lsp_text_doc_inlay_hints( + file_formats: Union[LanguageDispatch, Sequence[LanguageDispatch]] +) -> Callable[[C], C]: + return _registering_wrapper(file_formats, TEXT_DOC_INLAY_HANDLERS) + + def lsp_folding_ranges( file_formats: Union[LanguageDispatch, Sequence[LanguageDispatch]] ) -> Callable[[C], C]: @@ -292,6 +299,7 @@ def describe_lsp_features(context: CommandContext) -> None: ("folding ranges", FOLDING_RANGE_HANDLERS), ("semantic tokens", SEMANTIC_TOKENS_FULL_HANDLERS), ("on-save handler", WILL_SAVE_WAIT_UNTIL_HANDLERS), + ("inlay hint (doc)", TEXT_DOC_INLAY_HANDLERS), ("format file handler", FORMAT_FILE_HANDLERS), ] print("LSP language IDs and their features:") diff --git a/src/debputy/manifest_parser/declarative_parser.py b/src/debputy/manifest_parser/declarative_parser.py index 72beec3..4a32368 100644 --- a/src/debputy/manifest_parser/declarative_parser.py +++ b/src/debputy/manifest_parser/declarative_parser.py @@ -44,7 +44,12 @@ from debputy.manifest_parser.mapper_code import ( map_each_element, ) from debputy.manifest_parser.parser_data import ParserContextData -from debputy.manifest_parser.util import AttributePath, unpack_type, find_annotation +from debputy.manifest_parser.util import ( + AttributePath, + unpack_type, + find_annotation, + check_integration_mode, +) from debputy.plugin.api.impl_types import ( DeclarativeInputParser, TD, @@ -57,7 +62,11 @@ from debputy.plugin.api.impl_types import ( TP, InPackageContextParser, ) -from debputy.plugin.api.spec import ParserDocumentation, PackageTypeSelector +from debputy.plugin.api.spec import ( + ParserDocumentation, + PackageTypeSelector, + DebputyIntegrationMode, +) from debputy.util import _info, _warn, assume_not_none try: @@ -242,6 +251,9 @@ def _extract_path_hint(v: Any, attribute_path: AttributePath) -> bool: class DeclarativeNonMappingInputParser(DeclarativeInputParser[TD], Generic[TD, SF]): alt_form_parser: AttributeDescription inline_reference_documentation: Optional[ParserDocumentation] = None + expected_debputy_integration_mode: Optional[Container[DebputyIntegrationMode]] = ( + None + ) def parse_input( self, @@ -250,6 +262,11 @@ class DeclarativeNonMappingInputParser(DeclarativeInputParser[TD], Generic[TD, S *, parser_context: Optional["ParserContextData"] = None, ) -> TD: + check_integration_mode( + path, + parser_context, + self.expected_debputy_integration_mode, + ) if self.reference_documentation_url is not None: doc_ref = f" (Documentation: {self.reference_documentation_url})" else: @@ -286,6 +303,9 @@ class DeclarativeMappingInputParser(DeclarativeInputParser[TD], Generic[TD, SF]) _per_attribute_conflicts_cache: Optional[Mapping[str, FrozenSet[str]]] = None inline_reference_documentation: Optional[ParserDocumentation] = None path_hint_source_attributes: Sequence[str] = tuple() + expected_debputy_integration_mode: Optional[Container[DebputyIntegrationMode]] = ( + None + ) def _parse_alt_form( self, @@ -422,6 +442,11 @@ class DeclarativeMappingInputParser(DeclarativeInputParser[TD], Generic[TD, SF]) *, parser_context: Optional["ParserContextData"] = None, ) -> TD: + check_integration_mode( + path, + parser_context, + self.expected_debputy_integration_mode, + ) if value is None: form_note = " The attribute must be a mapping." if self.alt_form_parser is not None: @@ -738,11 +763,16 @@ class ParserGenerator: path: str, *, parser_documentation: Optional[ParserDocumentation] = None, + expected_debputy_integration_mode: Optional[ + Container[DebputyIntegrationMode] + ] = None, ) -> None: assert path not in self._in_package_context_parser assert path not in self._object_parsers self._object_parsers[path] = DispatchingObjectParser( - path, parser_documentation=parser_documentation + path, + parser_documentation=parser_documentation, + expected_debputy_integration_mode=expected_debputy_integration_mode, ) def add_in_package_context_parser( @@ -778,6 +808,9 @@ class ParserGenerator: source_content: Optional[SF] = None, allow_optional: bool = False, inline_reference_documentation: Optional[ParserDocumentation] = None, + expected_debputy_integration_mode: Optional[ + Container[DebputyIntegrationMode] + ] = None, ) -> DeclarativeInputParser[TD]: """Derive a parser from a TypedDict @@ -906,6 +939,10 @@ class ParserGenerator: should set this to True. Though, in 99.9% of all cases, you want `NotRequired` rather than `Optional` (and can keep this False). :param inline_reference_documentation: Optionally, programmatic documentation + :param expected_debputy_integration_mode: If provided, this declares the integration modes where the + result of the parser can be used. This is primarily useful for "fail-fast" on incorrect usage. + When the restriction is not satisfiable, the generated parser will trigger a parse error immediately + (resulting in a "compile time" failure rather than a "runtime" failure). :return: An input parser capable of reading input matching the TypedDict(s) used as reference. """ orig_parsed_content = parsed_content @@ -932,6 +969,7 @@ class ParserGenerator: parser = ListWrappedDeclarativeInputParser( parser, inline_reference_documentation=inline_reference_documentation, + expected_debputy_integration_mode=expected_debputy_integration_mode, ) return parser @@ -1104,6 +1142,7 @@ class ParserGenerator: parser = DeclarativeNonMappingInputParser( assume_not_none(parsed_alt_form), inline_reference_documentation=inline_reference_documentation, + expected_debputy_integration_mode=expected_debputy_integration_mode, ) else: parser = DeclarativeMappingInputParser( @@ -1116,9 +1155,13 @@ class ParserGenerator: at_least_one_of=at_least_one_of, inline_reference_documentation=inline_reference_documentation, path_hint_source_attributes=tuple(path_hint_source_attributes), + expected_debputy_integration_mode=expected_debputy_integration_mode, ) if is_list_wrapped: - parser = ListWrappedDeclarativeInputParser(parser) + parser = ListWrappedDeclarativeInputParser( + parser, + expected_debputy_integration_mode=expected_debputy_integration_mode, + ) return parser def _as_type_validator( diff --git a/src/debputy/manifest_parser/parser_data.py b/src/debputy/manifest_parser/parser_data.py index 3c36815..30d9ce0 100644 --- a/src/debputy/manifest_parser/parser_data.py +++ b/src/debputy/manifest_parser/parser_data.py @@ -4,17 +4,13 @@ from typing import ( Optional, Mapping, NoReturn, - Union, - Any, TYPE_CHECKING, - Tuple, ) from debian.debian_support import DpkgArchTable from debputy._deb_options_profiles import DebBuildOptionsAndProfiles from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable -from debputy.manifest_conditions import ManifestCondition from debputy.manifest_parser.exceptions import ManifestParseException from debputy.manifest_parser.util import AttributePath from debputy.packages import BinaryPackage @@ -24,12 +20,10 @@ from debputy.plugin.api.impl_types import ( TP, DispatchingTableParser, TTP, - DispatchingObjectParser, ) -from debputy.plugin.api.spec import PackageTypeSelector +from debputy.plugin.api.spec import PackageTypeSelector, DebputyIntegrationMode from debputy.substitution import Substitution - if TYPE_CHECKING: from debputy.highlevel_manifest import PackageTransformationDefinition @@ -131,3 +125,7 @@ class ParserContextData: def dispatch_parser_table_for(self, rule_type: TTP) -> DispatchingTableParser[TP]: raise NotImplementedError + + @property + def debputy_integration_mode(self) -> DebputyIntegrationMode: + raise NotImplementedError diff --git a/src/debputy/manifest_parser/util.py b/src/debputy/manifest_parser/util.py index ad214e2..a9cbbe8 100644 --- a/src/debputy/manifest_parser/util.py +++ b/src/debputy/manifest_parser/util.py @@ -2,7 +2,6 @@ import dataclasses from typing import ( Iterator, Union, - Self, Optional, List, Tuple, @@ -14,10 +13,14 @@ from typing import ( TypeVar, TYPE_CHECKING, Iterable, + Container, ) +from debputy.manifest_parser.exceptions import ManifestParseException + if TYPE_CHECKING: - from debputy.manifest_parser.declarative_parser import DebputyParseHint + from debputy.manifest_parser.parser_data import ParserContextData + from debputy.plugin.api.spec import DebputyIntegrationMode MP = TypeVar("MP", bound="DebputyParseHint") @@ -118,6 +121,27 @@ class AttributePath(object): yield current +def check_integration_mode( + path: AttributePath, + parser_context: Optional["ParserContextData"] = None, + expected_debputy_integration_mode: Optional[ + Container["DebputyIntegrationMode"] + ] = None, +) -> None: + if expected_debputy_integration_mode is None: + return + if parser_context is None: + raise AssertionError( + f"Cannot use integration mode restriction when parsing {path.path} since it is not parsed in the manifest context" + ) + if parser_context.debputy_integration_mode not in expected_debputy_integration_mode: + raise ManifestParseException( + f"The attribute {path.path} cannot be used as it is not allowed for" + f" the current debputy integration mode ({parser_context.debputy_integration_mode})." + f" Please remove the manifest definition or change the integration mode" + ) + + @dataclasses.dataclass(slots=True, frozen=True) class _SymbolicModeSegment: base_mode: int diff --git a/src/debputy/package_build/assemble_deb.py b/src/debputy/package_build/assemble_deb.py index 6f0d873..fd92f37 100644 --- a/src/debputy/package_build/assemble_deb.py +++ b/src/debputy/package_build/assemble_deb.py @@ -6,7 +6,7 @@ from typing import Optional, Sequence, List, Tuple from debputy import DEBPUTY_ROOT_DIR from debputy.commands.debputy_cmd.context import CommandContext from debputy.deb_packaging_support import setup_control_files -from debputy.debhelper_emulation import dhe_dbgsym_root_dir +from debputy.dh.debhelper_emulation import dhe_dbgsym_root_dir from debputy.filesystem_scan import FSRootDir from debputy.highlevel_manifest import HighLevelManifest from debputy.intermediate_manifest import IntermediateManifest @@ -17,7 +17,6 @@ from debputy.util import ( compute_output_filename, scratch_dir, ensure_dir, - _warn, assume_not_none, _info, ) diff --git a/src/debputy/plugin/api/impl.py b/src/debputy/plugin/api/impl.py index c951c2f..6369663 100644 --- a/src/debputy/plugin/api/impl.py +++ b/src/debputy/plugin/api/impl.py @@ -30,6 +30,8 @@ from typing import ( FrozenSet, Any, Literal, + Container, + get_args, ) from debputy import DEBPUTY_DOC_ROOT_DIR @@ -105,6 +107,7 @@ from debputy.plugin.api.spec import ( PackagerProvidedFileReferenceDocumentation, packager_provided_file_reference_documentation, TypeMappingDocumentation, + DebputyIntegrationMode, ) from debputy.substitution import ( Substitution, @@ -841,6 +844,9 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): *, source_format: Optional[SF] = None, inline_reference_documentation: Optional[ParserDocumentation] = None, + expected_debputy_integration_mode: Optional[ + Container[DebputyIntegrationMode] + ] = None, ) -> None: self._restricted_api() feature_set = self._feature_set @@ -868,6 +874,7 @@ class DebputyPluginInitializerProvider(DebputyPluginInitializer): parsed_format, source_content=source_format, inline_reference_documentation=inline_reference_documentation, + expected_debputy_integration_mode=expected_debputy_integration_mode, ) dispatching_parser.register_parser( rule_name, diff --git a/src/debputy/plugin/api/impl_types.py b/src/debputy/plugin/api/impl_types.py index d62ccbc..77e96ea 100644 --- a/src/debputy/plugin/api/impl_types.py +++ b/src/debputy/plugin/api/impl_types.py @@ -23,6 +23,7 @@ from typing import ( Literal, Set, Iterator, + Container, ) from weakref import ref @@ -39,7 +40,7 @@ from debputy.maintscript_snippet import DpkgMaintscriptHelperCommand from debputy.manifest_conditions import ManifestCondition from debputy.manifest_parser.base_types import DebputyParsedContent, TypeMapping from debputy.manifest_parser.exceptions import ManifestParseException -from debputy.manifest_parser.util import AttributePath +from debputy.manifest_parser.util import AttributePath, check_integration_mode from debputy.packages import BinaryPackage from debputy.plugin.api import ( VirtualPath, @@ -59,6 +60,7 @@ from debputy.plugin.api.spec import ( reference_documentation, PackagerProvidedFileReferenceDocumentation, TypeMappingDocumentation, + DebputyIntegrationMode, ) from debputy.substitution import VariableContext from debputy.transformation_rules import TransformationRule @@ -282,6 +284,12 @@ class DeclarativeInputParser(Generic[TD]): return None @property + def expected_debputy_integration_mode( + self, + ) -> Optional[Container[DebputyIntegrationMode]]: + return None + + @property def reference_documentation_url(self) -> Optional[str]: doc = self.inline_reference_documentation return doc.documentation_reference_url if doc is not None else None @@ -297,16 +305,30 @@ class DeclarativeInputParser(Generic[TD]): class DelegatingDeclarativeInputParser(DeclarativeInputParser[TD]): - __slots__ = ("delegate", "_reference_documentation") + __slots__ = ( + "delegate", + "_reference_documentation", + "_expected_debputy_integration_mode", + ) def __init__( self, delegate: DeclarativeInputParser[TD], *, inline_reference_documentation: Optional[ParserDocumentation] = None, + expected_debputy_integration_mode: Optional[ + Container[DebputyIntegrationMode] + ] = None, ) -> None: self.delegate = delegate self._reference_documentation = inline_reference_documentation + self._expected_debputy_integration_mode = expected_debputy_integration_mode + + @property + def expected_debputy_integration_mode( + self, + ) -> Optional[Container[DebputyIntegrationMode]]: + return self._expected_debputy_integration_mode @property def inline_reference_documentation(self) -> Optional[ParserDocumentation]: @@ -334,6 +356,9 @@ class ListWrappedDeclarativeInputParser(DelegatingDeclarativeInputParser[TD]): *, parser_context: Optional["ParserContextData"] = None, ) -> TD: + check_integration_mode( + path, parser_context, self._expected_debputy_integration_mode + ) if not isinstance(value, list): doc_ref = self._doc_url_error_suffix(see_url_version=True) raise ManifestParseException( @@ -466,12 +491,22 @@ class DispatchingObjectParser( manifest_attribute_path_template: str, *, parser_documentation: Optional[ParserDocumentation] = None, + expected_debputy_integration_mode: Optional[ + Container[DebputyIntegrationMode] + ] = None, ) -> None: super().__init__(manifest_attribute_path_template) self._attribute_documentation: List[ParserAttributeDocumentation] = [] if parser_documentation is None: parser_documentation = reference_documentation() self._parser_documentation = parser_documentation + self._expected_debputy_integration_mode = expected_debputy_integration_mode + + @property + def expected_debputy_integration_mode( + self, + ) -> Optional[Container[DebputyIntegrationMode]]: + return self._expected_debputy_integration_mode @property def reference_documentation_url(self) -> Optional[str]: @@ -540,6 +575,11 @@ class DispatchingObjectParser( *, parser_context: "ParserContextData", ) -> TP: + check_integration_mode( + attribute_path, + parser_context, + self._expected_debputy_integration_mode, + ) doc_ref = "" if self.reference_documentation_url is not None: doc_ref = ( @@ -602,6 +642,8 @@ class PackageContextData(Generic[TP]): class InPackageContextParser( DelegatingDeclarativeInputParser[Mapping[str, PackageContextData[TP]]] ): + __slots__ = () + def __init__( self, manifest_attribute_path_template: str, @@ -621,6 +663,11 @@ class InPackageContextParser( parser_context: Optional["ParserContextData"] = None, ) -> TP: assert parser_context is not None + check_integration_mode( + attribute_path, + parser_context, + self._expected_debputy_integration_mode, + ) doc_ref = "" if self.reference_documentation_url is not None: doc_ref = ( diff --git a/src/debputy/plugin/api/spec.py b/src/debputy/plugin/api/spec.py index 07954e6..b7f19c0 100644 --- a/src/debputy/plugin/api/spec.py +++ b/src/debputy/plugin/api/spec.py @@ -23,6 +23,8 @@ from typing import ( List, Type, Tuple, + get_args, + Container, ) from debian.substvars import Substvars @@ -78,6 +80,29 @@ ServiceIntegrator = Callable[ ] PMT = TypeVar("PMT") +DebputyIntegrationMode = Literal[ + "full", + "dh-sequence-zz-debputy", + "dh-sequence-zz-debputy-rrr", +] + +INTEGRATION_MODE_DH_DEBPUTY_RRR: DebputyIntegrationMode = "dh-sequence-zz-debputy-rrr" +INTEGRATION_MODE_DH_DEBPUTY: DebputyIntegrationMode = "dh-sequence-zz-debputy" +ALL_DEBPUTY_INTEGRATION_MODES: FrozenSet[DebputyIntegrationMode] = frozenset( + get_args(DebputyIntegrationMode) +) + + +def only_integrations( + *integrations: DebputyIntegrationMode, +) -> Container[DebputyIntegrationMode]: + return frozenset(*integrations) + + +def not_integrations( + *integrations: DebputyIntegrationMode, +) -> Container[DebputyIntegrationMode]: + return ALL_DEBPUTY_INTEGRATION_MODES - frozenset(integrations) @dataclasses.dataclass(slots=True, frozen=True) diff --git a/src/debputy/plugin/debputy/binary_package_rules.py b/src/debputy/plugin/debputy/binary_package_rules.py index 29c6136..98da763 100644 --- a/src/debputy/plugin/debputy/binary_package_rules.py +++ b/src/debputy/plugin/debputy/binary_package_rules.py @@ -40,6 +40,8 @@ from debputy.plugin.api.spec import ( ServiceDefinition, DSD, documented_attr, + INTEGRATION_MODE_DH_DEBPUTY_RRR, + not_integrations, ) from debputy.transformation_rules import TransformationRule from debputy.util import _error @@ -148,6 +150,9 @@ def register_binary_package_rules(api: DebputyPluginInitializerProvider) -> None "conffile-management", List[DpkgMaintscriptHelperCommand], _unpack_list, + expected_debputy_integration_mode=not_integrations( + INTEGRATION_MODE_DH_DEBPUTY_RRR + ), ) api.pluggable_manifest_rule( @@ -156,6 +161,9 @@ def register_binary_package_rules(api: DebputyPluginInitializerProvider) -> None List[ServiceRuleParsedFormat], _process_service_rules, source_format=List[ServiceRuleSourceFormat], + expected_debputy_integration_mode=not_integrations( + INTEGRATION_MODE_DH_DEBPUTY_RRR + ), inline_reference_documentation=reference_documentation( title="Define how services in the package will be handled (`services`)", description=textwrap.dedent( @@ -273,6 +281,9 @@ def register_binary_package_rules(api: DebputyPluginInitializerProvider) -> None ListParsedFormat, _parse_clean_after_removal, source_format=List[Any], + expected_debputy_integration_mode=not_integrations( + INTEGRATION_MODE_DH_DEBPUTY_RRR + ), # FIXME: debputy won't see the attributes for this one :'( inline_reference_documentation=reference_documentation( title="Remove runtime created paths on purge or post removal (`clean-after-removal`)", @@ -337,6 +348,9 @@ def register_binary_package_rules(api: DebputyPluginInitializerProvider) -> None InstallationSearchDirsParsedFormat, _parse_installation_search_dirs, source_format=List[FileSystemExactMatchRule], + expected_debputy_integration_mode=not_integrations( + INTEGRATION_MODE_DH_DEBPUTY_RRR + ), inline_reference_documentation=reference_documentation( title="Custom installation time search directories (`installation-search-dirs`)", description=textwrap.dedent( diff --git a/src/debputy/plugin/debputy/manifest_root_rules.py b/src/debputy/plugin/debputy/manifest_root_rules.py index ca8cf1e..80a4799 100644 --- a/src/debputy/plugin/debputy/manifest_root_rules.py +++ b/src/debputy/plugin/debputy/manifest_root_rules.py @@ -21,9 +21,9 @@ from debputy.plugin.api.impl import DebputyPluginInitializerProvider from debputy.plugin.api.impl_types import ( OPARSER_MANIFEST_ROOT, OPARSER_MANIFEST_DEFINITIONS, - SUPPORTED_DISPATCHABLE_OBJECT_PARSERS, OPARSER_PACKAGES, ) +from debputy.plugin.api.spec import not_integrations, INTEGRATION_MODE_DH_DEBPUTY_RRR from debputy.substitution import VariableNameState, SUBST_VAR_RE if TYPE_CHECKING: @@ -110,6 +110,9 @@ def register_manifest_root_rules(api: DebputyPluginInitializerProvider) -> None: MK_INSTALLATIONS, List[InstallRule], _handle_installation_rules, + expected_debputy_integration_mode=not_integrations( + INTEGRATION_MODE_DH_DEBPUTY_RRR + ), inline_reference_documentation=reference_documentation( title="Installations", description=textwrap.dedent( diff --git a/src/debputy/plugin/debputy/private_api.py b/src/debputy/plugin/debputy/private_api.py index 37c9318..d042378 100644 --- a/src/debputy/plugin/debputy/private_api.py +++ b/src/debputy/plugin/debputy/private_api.py @@ -1,7 +1,7 @@ import ctypes import ctypes.util +import dataclasses import functools -import itertools import textwrap import time from datetime import datetime @@ -61,7 +61,7 @@ from debputy.manifest_parser.declarative_parser import DebputyParseHint from debputy.manifest_parser.exceptions import ManifestParseException from debputy.manifest_parser.mapper_code import type_mapper_str2package from debputy.manifest_parser.parser_data import ParserContextData -from debputy.manifest_parser.util import AttributePath +from debputy.manifest_parser.util import AttributePath, check_integration_mode from debputy.packages import BinaryPackage from debputy.path_matcher import ExactFileSystemPath from debputy.plugin.api import ( @@ -76,6 +76,8 @@ from debputy.plugin.api.impl_types import automatic_discard_rule_example, PPFFor from debputy.plugin.api.spec import ( type_mapping_reference_documentation, type_mapping_example, + not_integrations, + INTEGRATION_MODE_DH_DEBPUTY_RRR, ) from debputy.plugin.debputy.binary_package_rules import register_binary_package_rules from debputy.plugin.debputy.discard_rules import ( @@ -153,10 +155,27 @@ _DOCUMENTED_DPKG_ARCH_VARS = { } +_NOT_INTEGRATION_RRR = not_integrations(INTEGRATION_MODE_DH_DEBPUTY_RRR) + + def _manifest_format_doc(anchor: str) -> str: return f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#{anchor}" +@dataclasses.dataclass(slots=True, frozen=True) +class Capability: + value: str + + @classmethod + def parse( + cls, + raw_value: str, + _attribute_path: AttributePath, + _parser_context: "ParserContextData", + ) -> "Capability": + return cls(raw_value) + + @functools.lru_cache def load_libcap() -> Tuple[bool, Optional[str], Callable[[str], bool]]: cap_library_path = ctypes.util.find_library("cap.so") @@ -313,6 +332,28 @@ def initialize_via_private_api(public_api: DebputyPluginInitializer) -> None: def register_type_mappings(api: DebputyPluginInitializerProvider) -> None: api.register_mapped_type( + TypeMapping(Capability, str, Capability.parse), + reference_documentation=type_mapping_reference_documentation( + description=textwrap.dedent( + """\ + A Linux capability + + The value is a Linux capability parsable by cap_from_text on the host system. + + With `libcap2` installed, `debputy` will attempt to parse the value and provide + warnings if the value cannot be parsed by `libcap2`. However, `debputy` will + currently never emit hard errors for unknown capabilities. + """, + ), + examples=[ + type_mapping_example("cap_chown=p"), + type_mapping_example("cap_chown=ep"), + type_mapping_example("cap_kill-pe"), + type_mapping_example("=ep cap_chown-e cap_kill-ep"), + ], + ), + ) + api.register_mapped_type( TypeMapping( FileSystemMatchRule, str, @@ -2072,14 +2113,14 @@ class PathManifestSourceDictFormat(_ModeOwnerBase): ] paths: NotRequired[List[FileSystemMatchRule]] recursive: NotRequired[bool] - capabilities: NotRequired[str] + capabilities: NotRequired[Capability] capability_mode: NotRequired[FileSystemMode] class PathManifestRule(_ModeOwnerBase): paths: List[FileSystemMatchRule] recursive: NotRequired[bool] - capabilities: NotRequired[str] + capabilities: NotRequired[Capability] capability_mode: NotRequired[FileSystemMode] @@ -2729,7 +2770,7 @@ def _transformation_path_metadata( _name: str, parsed_data: PathManifestRule, attribute_path: AttributePath, - _context: ParserContextData, + context: ParserContextData, ) -> TransformationRule: match_rules = parsed_data["paths"] owner = parsed_data.get("owner") @@ -2738,16 +2779,28 @@ def _transformation_path_metadata( recursive = parsed_data.get("recursive", False) capabilities = parsed_data.get("capabilities") capability_mode = parsed_data.get("capability_mode") + cap: Optional[str] = None if capabilities is not None: + check_integration_mode( + attribute_path["capabilities"], + context, + _NOT_INTEGRATION_RRR, + ) if capability_mode is None: capability_mode = SymbolicMode.parse_filesystem_mode( "a-s", attribute_path["capability-mode"], ) + cap = capabilities.value validate_cap = check_cap_checker() - validate_cap(capabilities, attribute_path["capabilities"].path) + validate_cap(cap, attribute_path["capabilities"].path) elif capability_mode is not None and capabilities is None: + check_integration_mode( + attribute_path["capability_mode"], + context, + _NOT_INTEGRATION_RRR, + ) raise ManifestParseException( "The attribute capability-mode cannot be provided without capabilities" f" in {attribute_path.path}" @@ -2765,7 +2818,7 @@ def _transformation_path_metadata( group, mode, recursive, - capabilities, + cap, capability_mode, attribute_path.path, condition, diff --git a/src/debputy/version.py b/src/debputy/version.py index de56318..eb69c8f 100644 --- a/src/debputy/version.py +++ b/src/debputy/version.py @@ -55,6 +55,9 @@ if __version__ in ("N/A",): except (subprocess.CalledProcessError, FileNotFoundError): v = "N/A" + if v.startswith("archive/"): + v = v[8:] + if v.startswith("debian/"): v = v[7:] return v diff --git a/tests/lint_tests/test_lint_debputy.py b/tests/lint_tests/test_lint_debputy.py index 2af34ae..7464cdf 100644 --- a/tests/lint_tests/test_lint_debputy.py +++ b/tests/lint_tests/test_lint_debputy.py @@ -183,3 +183,78 @@ def test_debputy_lint_check_package_names(line_linter: LintWrapper) -> None: msg = 'Unknown package "unknown-package".' assert diag.message == msg assert f"{diag.range}" == "2:4-2:19" + + +def test_debputy_lint_integration_mode(line_linter: LintWrapper) -> None: + lines = textwrap.dedent( + """\ + manifest-version: 0.1 + installations: [] + packages: + foo: + services: + - service: foo + """ + ).splitlines(keepends=True) + line_linter.dctrl_lines = textwrap.dedent( + """\ + Source: foo + Build-Depends: dh-sequence-zz-debputy-rrr, + + Package: foo + """ + ).splitlines(keepends=True) + + diagnostics = line_linter(lines) + assert diagnostics and len(diagnostics) == 2 + first_issue, second_issue = diagnostics + + msg = 'Feature "installations" not supported in integration mode dh-sequence-zz-debputy-rrr' + assert first_issue.message == msg + assert f"{first_issue.range}" == "1:0-1:13" + assert first_issue.severity == DiagnosticSeverity.Error + + msg = 'Feature "services" not supported in integration mode dh-sequence-zz-debputy-rrr' + assert second_issue.message == msg + assert f"{second_issue.range}" == "4:8-4:16" + assert second_issue.severity == DiagnosticSeverity.Error + + # Changing the integration mode should fix both + line_linter.dctrl_lines = textwrap.dedent( + """\ + Source: foo + Build-Depends: dh-sequence-zz-debputy, + + Package: foo + """ + ).splitlines(keepends=True) + diagnostics = line_linter(lines) + assert not diagnostics + + +def test_debputy_lint_attr_value_checks(line_linter: LintWrapper) -> None: + lines = textwrap.dedent( + """\ + manifest-version: 0.1 + packages: + foo: + services: + - service: foo + enable-on-install: "true" + on-upgrade: "bar" + """ + ).splitlines(keepends=True) + + diagnostics = line_linter(lines) + assert diagnostics and len(diagnostics) == 2 + first_issue, second_issue = diagnostics + + msg = 'Not a supported value for "enable-on-install"' + assert first_issue.message == msg + assert f"{first_issue.range}" == "5:29-5:35" + assert first_issue.severity == DiagnosticSeverity.Error + + msg = 'Not a supported value for "on-upgrade"' + assert second_issue.message == msg + assert f"{second_issue.range}" == "6:22-6:27" + assert second_issue.severity == DiagnosticSeverity.Error diff --git a/tests/test_fs_metadata.py b/tests/test_fs_metadata.py index f32afb0..7dd3d55 100644 --- a/tests/test_fs_metadata.py +++ b/tests/test_fs_metadata.py @@ -35,6 +35,7 @@ def manifest_parser_pkg_foo( dpkg_arch_query, no_profiles_or_build_options, debputy_plugin_feature_set, + "full", debian_dir=debian_dir, ) diff --git a/tests/test_install_rules.py b/tests/test_install_rules.py index c8ffb84..a361864 100644 --- a/tests/test_install_rules.py +++ b/tests/test_install_rules.py @@ -33,6 +33,7 @@ def manifest_parser_pkg_foo( dpkg_arch_query, no_profiles_or_build_options, debputy_plugin_feature_set, + "full", debian_dir=debian_dir, ) @@ -58,6 +59,7 @@ def manifest_parser_pkg_foo_w_udeb( dpkg_arch_query, no_profiles_or_build_options, debputy_plugin_feature_set, + "full", debian_dir=debian_dir, ) diff --git a/tests/test_interpreter.py b/tests/test_interpreter.py index 154ee4a..be8ec8a 100644 --- a/tests/test_interpreter.py +++ b/tests/test_interpreter.py @@ -127,6 +127,7 @@ def empty_manifest( dpkg_arch_query, no_profiles_or_build_options, debputy_plugin_feature_set, + "full", debian_dir=debian_dir, ).build_manifest() diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 9d43549..f53c716 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -1,6 +1,6 @@ import io import textwrap -from typing import Iterable, Callable, Optional, List, Tuple, Sequence +from typing import Callable, Optional, List, Tuple, Sequence import pytest @@ -22,8 +22,6 @@ from debputy.dh_migration.migrators_impl import ( migrate_installinfo_file, migrate_dh_installsystemd_files, detect_obsolete_substvars, - MIGRATION_TARGET_DH_DEBPUTY, - MIGRATION_TARGET_DH_DEBPUTY_RRR, detect_dh_addons_zz_debputy_rrr, ) from debputy.dh_migration.models import ( @@ -34,6 +32,10 @@ from debputy.dh_migration.models import ( from debputy.highlevel_manifest import HighLevelManifest from debputy.highlevel_manifest_parser import YAMLManifestParser from debputy.plugin.api import virtual_path_def, VirtualPath +from debputy.plugin.api.spec import ( + INTEGRATION_MODE_DH_DEBPUTY_RRR, + INTEGRATION_MODE_DH_DEBPUTY, +) from debputy.plugin.api.test_api import ( build_virtual_file_system, ) @@ -65,6 +67,7 @@ def manifest_parser_pkg_foo_factory( dpkg_arch_query, no_profiles_or_build_options, debputy_plugin_feature_set, + "full", debian_dir=debian_dir, ) @@ -94,7 +97,7 @@ def run_migrator( manifest: HighLevelManifest, acceptable_migration_issues: AcceptableMigrationIssues, *, - migration_target=MIGRATION_TARGET_DH_DEBPUTY, + migration_target=INTEGRATION_MODE_DH_DEBPUTY, ) -> FeatureMigration: feature_migration = FeatureMigration(migrator.__name__) migrator( @@ -1661,7 +1664,7 @@ def test_detect_dh_addons_rrr( empty_fs, empty_manifest_pkg_foo, accept_no_migration_issues, - migration_target=MIGRATION_TARGET_DH_DEBPUTY_RRR, + migration_target=INTEGRATION_MODE_DH_DEBPUTY_RRR, ) assert migration.anything_to_do assert migration.warnings == [no_ctrl_msg] diff --git a/tests/test_parser.py b/tests/test_parser.py index 4792842..4aee024 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -34,6 +34,7 @@ def manifest_parser_pkg_foo( dpkg_arch_query, no_profiles_or_build_options, debputy_plugin_feature_set, + "full", debian_dir=debian_dir, ) |