diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-17 16:40:52 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-17 16:40:52 +0000 |
commit | 8b8344c3982326d9c9b99e7bca8e667746b77691 (patch) | |
tree | a16bb8a7cbd05b1cdf5085afe13e74b3d15ce4e2 /src | |
parent | Releasing progress-linux version 0.1.36-0.0~progress7.99u1. (diff) | |
download | debputy-8b8344c3982326d9c9b99e7bca8e667746b77691.tar.xz debputy-8b8344c3982326d9c9b99e7bca8e667746b77691.zip |
Merging upstream version 0.1.37.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src')
26 files changed, 1152 insertions, 678 deletions
diff --git a/src/debputy/analysis/__init__.py b/src/debputy/analysis/__init__.py new file mode 100644 index 0000000..931c574 --- /dev/null +++ b/src/debputy/analysis/__init__.py @@ -0,0 +1,9 @@ +from debputy.plugin.api.impl_types import ( + KNOWN_PACKAGING_FILE_CATEGORY_DESCRIPTIONS, + KNOWN_PACKAGING_FILE_CONFIG_FEATURE_DESCRIPTION, +) + +REFERENCE_DATA_TABLE = { + "config-features": KNOWN_PACKAGING_FILE_CONFIG_FEATURE_DESCRIPTION, + "file-categories": KNOWN_PACKAGING_FILE_CATEGORY_DESCRIPTIONS, +} diff --git a/src/debputy/commands/debputy_cmd/dc_util.py b/src/debputy/analysis/analysis_util.py index f54a4d1..f54a4d1 100644 --- a/src/debputy/commands/debputy_cmd/dc_util.py +++ b/src/debputy/analysis/analysis_util.py diff --git a/src/debputy/analysis/debian_dir.py b/src/debputy/analysis/debian_dir.py new file mode 100644 index 0000000..7e5b37a --- /dev/null +++ b/src/debputy/analysis/debian_dir.py @@ -0,0 +1,617 @@ +import json +import os +import stat +import subprocess +from typing import ( + List, + Mapping, + Iterable, + Tuple, + Optional, + Sequence, + Dict, + Any, + Union, + Iterator, + TypedDict, + NotRequired, +) + +from debputy.analysis import REFERENCE_DATA_TABLE +from debputy.analysis.analysis_util import flatten_ppfs +from debputy.dh_migration.migrators_impl import read_dh_addon_sequences +from debputy.packager_provided_files import ( + PackagerProvidedFile, + detect_all_packager_provided_files, +) +from debputy.packages import BinaryPackage +from debputy.plugin.api import ( + VirtualPath, + packager_provided_file_reference_documentation, +) +from debputy.plugin.api.feature_set import PluginProvidedFeatureSet +from debputy.plugin.api.impl import plugin_metadata_for_debputys_own_plugin +from debputy.plugin.api.impl_types import ( + PluginProvidedKnownPackagingFile, + DebputyPluginMetadata, + KnownPackagingFileInfo, + InstallPatternDHCompatRule, + PackagerProvidedFileClassSpec, + expand_known_packaging_config_features, +) +from debputy.util import assume_not_none, escape_shell + +PackagingFileInfo = TypedDict( + "PackagingFileInfo", + { + "path": str, + "binary-package": NotRequired[str], + "install-path": NotRequired[str], + "install-pattern": NotRequired[str], + "file-categories": NotRequired[List[str]], + "config-features": NotRequired[List[str]], + "pkgfile-is-active-in-build": NotRequired[bool], + "pkgfile-stem": NotRequired[str], + "pkgfile-explicit-package-name": NotRequired[bool], + "pkgfile-name-segment": NotRequired[str], + "pkgfile-architecture-restriction": NotRequired[str], + "likely-typo-of": NotRequired[str], + "likely-generated-from": NotRequired[List[str]], + "related-tools": NotRequired[List[str]], + "documentation-uris": NotRequired[List[str]], + "debputy-cmd-templates": NotRequired[List[List[str]]], + "generates": NotRequired[str], + "generated-from": NotRequired[str], + }, +) + + +def scan_debian_dir( + feature_set: PluginProvidedFeatureSet, + binary_packages: Mapping[str, BinaryPackage], + debian_dir: VirtualPath, +) -> Tuple[List[PackagingFileInfo], List[str], int, Optional[object]]: + known_packaging_files = feature_set.known_packaging_files + debputy_plugin_metadata = plugin_metadata_for_debputys_own_plugin() + + reference_data_set_names = [ + "config-features", + "file-categories", + ] + for n in reference_data_set_names: + assert n in REFERENCE_DATA_TABLE + + annotated: List[PackagingFileInfo] = [] + seen_paths: Dict[str, PackagingFileInfo] = {} + + r = read_dh_addon_sequences(debian_dir) + if r is not None: + bd_sequences, dr_sequences, saw_dh = r + drules_sequences = bd_sequences | dr_sequences + else: + drules_sequences = set() + saw_dh = False + is_debputy_package = ( + "debputy" in drules_sequences + or "zz-debputy" in drules_sequences + or "zz_debputy" in drules_sequences + or "zz-debputy-rrr" in drules_sequences + ) + dh_compat_level, dh_assistant_exit_code = _extract_dh_compat_level() + dh_issues = [] + + static_packaging_files = { + kpf.detection_value: kpf + for kpf in known_packaging_files.values() + if kpf.detection_method == "path" + } + dh_pkgfile_docs = { + kpf.detection_value: kpf + for kpf in known_packaging_files.values() + if kpf.detection_method == "dh.pkgfile" + } + + if is_debputy_package: + all_debputy_ppfs = list( + flatten_ppfs( + detect_all_packager_provided_files( + feature_set.packager_provided_files, + debian_dir, + binary_packages, + allow_fuzzy_matches=True, + detect_typos=True, + ) + ) + ) + else: + all_debputy_ppfs = [] + + if dh_compat_level is not None: + ( + all_dh_ppfs, + dh_issues, + dh_assistant_exit_code, + ) = _resolve_debhelper_config_files( + debian_dir, + binary_packages, + debputy_plugin_metadata, + dh_pkgfile_docs, + drules_sequences, + dh_compat_level, + saw_dh, + ) + + else: + all_dh_ppfs = [] + + for ppf in all_debputy_ppfs: + key = ppf.path.path + ref_doc = ppf.definition.reference_documentation + documentation_uris = ( + ref_doc.format_documentation_uris if ref_doc is not None else None + ) + details: PackagingFileInfo = { + "path": key, + "pkgfile-stem": ppf.definition.stem, + "pkgfile-explicit-package-name": ppf.uses_explicit_package_name, + "pkgfile-is-active-in-build": ppf.definition.has_active_command, + "debputy-cmd-templates": [ + ["debputy", "plugin", "show", "p-p-f", ppf.definition.stem] + ], + } + if ppf.fuzzy_match and key.endswith(".in"): + _merge_list(details, "file-categories", ["generic-template"]) + details["generates"] = key[:-3] + elif assume_not_none(ppf.path.parent_dir).get(ppf.path.name + ".in"): + _merge_list(details, "file-categories", ["generated"]) + details["generated-from"] = key + ".in" + name_segment = ppf.name_segment + arch_restriction = ppf.architecture_restriction + if name_segment is not None: + details["pkgfile-name-segment"] = name_segment + if arch_restriction: + details["pkgfile-architecture-restriction"] = arch_restriction + seen_paths[key] = details + annotated.append(details) + static_details = static_packaging_files.get(key) + if static_details is not None: + # debhelper compat rules does not apply to debputy files + _add_known_packaging_data(details, static_details, None) + if documentation_uris: + details["documentation-uris"] = list(documentation_uris) + + _merge_ppfs(annotated, seen_paths, all_dh_ppfs, dh_pkgfile_docs, dh_compat_level) + + for virtual_path in _scan_debian_dir(debian_dir): + key = virtual_path.path + if key in seen_paths: + continue + if virtual_path.is_symlink: + try: + st = os.stat(virtual_path.fs_path) + except FileNotFoundError: + continue + else: + if not stat.S_ISREG(st.st_mode): + continue + elif not virtual_path.is_file: + continue + + static_match = static_packaging_files.get(virtual_path.path) + if static_match is not None: + details: PackagingFileInfo = { + "path": key, + } + annotated.append(details) + if assume_not_none(virtual_path.parent_dir).get(virtual_path.name + ".in"): + details["generated-from"] = key + ".in" + _merge_list(details, "file-categories", ["generated"]) + _add_known_packaging_data(details, static_match, dh_compat_level) + + return annotated, reference_data_set_names, dh_assistant_exit_code, dh_issues + + +def _fake_PPFClassSpec( + debputy_plugin_metadata: DebputyPluginMetadata, + stem: str, + doc_uris: Optional[Sequence[str]], + install_pattern: Optional[str], + *, + default_priority: Optional[int] = None, + packageless_is_fallback_for_all_packages: bool = False, + post_formatting_rewrite: Optional[str] = None, + bug_950723: bool = False, + has_active_command: bool = False, +) -> PackagerProvidedFileClassSpec: + if install_pattern is None: + install_pattern = "not-a-real-ppf" + if post_formatting_rewrite is not None: + formatting_hook = _POST_FORMATTING_REWRITE[post_formatting_rewrite] + else: + formatting_hook = None + return PackagerProvidedFileClassSpec( + debputy_plugin_metadata, + stem, + install_pattern, + allow_architecture_segment=True, + allow_name_segment=True, + default_priority=default_priority, + default_mode=0o644, + post_formatting_rewrite=formatting_hook, + packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages, + reservation_only=False, + formatting_callback=None, + bug_950723=bug_950723, + has_active_command=has_active_command, + reference_documentation=packager_provided_file_reference_documentation( + format_documentation_uris=doc_uris, + ), + ) + + +def _relevant_dh_compat_rules( + compat_level: Optional[int], + info: KnownPackagingFileInfo, +) -> Iterable[InstallPatternDHCompatRule]: + if compat_level is None: + return + dh_compat_rules = info.get("dh_compat_rules") + if not dh_compat_rules: + return + for dh_compat_rule in dh_compat_rules: + rule_compat_level = dh_compat_rule.get("starting_with_compat_level") + if rule_compat_level is not None and compat_level < rule_compat_level: + continue + yield dh_compat_rule + + +def _kpf_install_pattern( + compat_level: Optional[int], + ppkpf: PluginProvidedKnownPackagingFile, +) -> Optional[str]: + for compat_rule in _relevant_dh_compat_rules(compat_level, ppkpf.info): + install_pattern = compat_rule.get("install_pattern") + if install_pattern is not None: + return install_pattern + return ppkpf.info.get("install_pattern") + + +def _resolve_debhelper_config_files( + debian_dir: VirtualPath, + binary_packages: Mapping[str, BinaryPackage], + debputy_plugin_metadata: DebputyPluginMetadata, + dh_ppf_docs: Dict[str, PluginProvidedKnownPackagingFile], + dh_rules_addons: Iterable[str], + dh_compat_level: int, + saw_dh: bool, +) -> Tuple[List[PackagerProvidedFile], Optional[object], int]: + dh_ppfs = {} + commands, exit_code = _relevant_dh_commands(dh_rules_addons) + + cmd = ["dh_assistant", "list-guessed-dh-config-files"] + if dh_rules_addons: + addons = ",".join(dh_rules_addons) + cmd.append(f"--with={addons}") + try: + output = subprocess.check_output( + cmd, + stderr=subprocess.DEVNULL, + ) + except (subprocess.CalledProcessError, FileNotFoundError) as e: + config_files = [] + issues = None + if isinstance(e, subprocess.CalledProcessError): + exit_code = e.returncode + else: + exit_code = 127 + else: + result = json.loads(output) + config_files: List[Union[Mapping[str, Any], object]] = result.get( + "config-files", [] + ) + issues = result.get("issues") + for config_file in config_files: + if not isinstance(config_file, dict): + continue + if config_file.get("file-type") != "pkgfile": + continue + stem = config_file.get("pkgfile") + if stem is None: + continue + internal = config_file.get("internal") + if isinstance(internal, dict): + bug_950723 = internal.get("bug#950723", False) is True + else: + bug_950723 = False + commands = config_file.get("commands") + documentation_uris = [] + related_tools = [] + seen_commands = set() + seen_docs = set() + ppkpf = dh_ppf_docs.get(stem) + + if ppkpf: + dh_cmds = ppkpf.info.get("debhelper_commands") + doc_uris = ppkpf.info.get("documentation_uris") + default_priority = ppkpf.info.get("default_priority") + if doc_uris is not None: + seen_docs.update(doc_uris) + documentation_uris.extend(doc_uris) + if dh_cmds is not None: + seen_commands.update(dh_cmds) + related_tools.extend(dh_cmds) + install_pattern = _kpf_install_pattern(dh_compat_level, ppkpf) + post_formatting_rewrite = ppkpf.info.get("post_formatting_rewrite") + packageless_is_fallback_for_all_packages = ppkpf.info.get( + "packageless_is_fallback_for_all_packages", + False, + ) + # If it is a debhelper PPF, then `has_active_command` is false by default. + has_active_command = ppkpf.info.get("has_active_command", False) + else: + install_pattern = None + default_priority = None + post_formatting_rewrite = None + packageless_is_fallback_for_all_packages = False + has_active_command = False + for command in commands: + if isinstance(command, dict): + command_name = command.get("command") + if isinstance(command_name, str) and command_name: + if command_name not in seen_commands: + related_tools.append(command_name) + seen_commands.add(command_name) + manpage = f"man:{command_name}(1)" + if manpage not in seen_docs: + documentation_uris.append(manpage) + seen_docs.add(manpage) + else: + continue + is_active = command.get("is-active", True) + if not isinstance(is_active, bool): + continue + if is_active: + has_active_command = True + dh_ppfs[stem] = _fake_PPFClassSpec( + debputy_plugin_metadata, + stem, + documentation_uris, + install_pattern, + default_priority=default_priority, + post_formatting_rewrite=post_formatting_rewrite, + packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages, + bug_950723=bug_950723, + has_active_command=has_active_command if saw_dh else True, + ) + for ppkpf in dh_ppf_docs.values(): + stem = ppkpf.detection_value + if stem in dh_ppfs: + continue + + default_priority = ppkpf.info.get("default_priority") + install_pattern = _kpf_install_pattern(dh_compat_level, ppkpf) + post_formatting_rewrite = ppkpf.info.get("post_formatting_rewrite") + packageless_is_fallback_for_all_packages = ppkpf.info.get( + "packageless_is_fallback_for_all_packages", + False, + ) + dh_ppfs[stem] = _fake_PPFClassSpec( + debputy_plugin_metadata, + stem, + ppkpf.info.get("documentation_uris"), + install_pattern, + default_priority=default_priority, + post_formatting_rewrite=post_formatting_rewrite, + packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages, + has_active_command=( + ppkpf.info.get("has_active_command", False) if saw_dh else False + ), + ) + all_dh_ppfs = list( + flatten_ppfs( + detect_all_packager_provided_files( + dh_ppfs, + debian_dir, + binary_packages, + allow_fuzzy_matches=True, + detect_typos=True, + ) + ) + ) + return all_dh_ppfs, issues, exit_code + + +def _merge_list( + existing_table: Dict[str, Any], + key: str, + new_data: Optional[Sequence[str]], +) -> None: + if not new_data: + return + existing_values = existing_table.get(key, []) + if isinstance(existing_values, tuple): + existing_values = list(existing_values) + assert isinstance(existing_values, list) + seen = set(existing_values) + existing_values.extend(x for x in new_data if x not in seen) + existing_table[key] = existing_values + + +def _merge_ppfs( + identified: List[PackagingFileInfo], + seen_paths: Dict[str, PackagingFileInfo], + ppfs: List[PackagerProvidedFile], + context: Mapping[str, PluginProvidedKnownPackagingFile], + dh_compat_level: Optional[int], +) -> None: + for ppf in ppfs: + key = ppf.path.path + ref_doc = ppf.definition.reference_documentation + documentation_uris = ( + ref_doc.format_documentation_uris if ref_doc is not None else None + ) + if not ppf.definition.installed_as_format.startswith("not-a-real-ppf"): + try: + parts = ppf.compute_dest() + except RuntimeError: + dest = None + else: + dest = "/".join(parts).lstrip(".") + else: + dest = None + orig_details = seen_paths.get(key) + if orig_details is None: + details: PackagingFileInfo = { + "path": key, + "pkgfile-stem": ppf.definition.stem, + "pkgfile-is-active-in-build": ppf.definition.has_active_command, + "pkgfile-explicit-package-name": ppf.uses_explicit_package_name, + "binary-package": ppf.package_name, + } + if ppf.expected_path is not None: + details["likely-typo-of"] = ppf.expected_path + identified.append(details) + else: + details = orig_details + # We do not merge the "is typo" field; if the original + for k, v in [ + ("pkgfile-stem", ppf.definition.stem), + ("pkgfile-explicit-package-name", ppf.definition.has_active_command), + ("binary-package", ppf.package_name), + ]: + if k not in details: + details[k] = v + if ppf.definition.has_active_command and details.get( + "pkgfile-is-active-in-build", False + ): + details["pkgfile-is-active-in-build"] = True + if ppf.expected_path is None and "likely-typo-of" in details: + del details["likely-typo-of"] + + name_segment = ppf.name_segment + arch_restriction = ppf.architecture_restriction + if name_segment is not None and "pkgfile-name-segment" not in details: + details["pkgfile-name-segment"] = name_segment + if ( + arch_restriction is not None + and "pkgfile-architecture-restriction" not in details + ): + details["pkgfile-architecture-restriction"] = arch_restriction + if ppf.fuzzy_match and key.endswith(".in"): + _merge_list(details, "file-categories", ["generic-template"]) + details["generates"] = key[:-3] + elif assume_not_none(ppf.path.parent_dir).get(ppf.path.name + ".in"): + _merge_list(details, "file-categories", ["generated"]) + details["generated-from"] = key + ".in" + if dest is not None and "install-path" not in details: + details["install-path"] = dest + + extra_details = context.get(ppf.definition.stem) + if extra_details is not None: + _add_known_packaging_data(details, extra_details, dh_compat_level) + + _merge_list(details, "documentation-uris", documentation_uris) + + +def _extract_dh_compat_level() -> Tuple[Optional[int], int]: + try: + output = subprocess.check_output( + ["dh_assistant", "active-compat-level"], + stderr=subprocess.DEVNULL, + ) + except (FileNotFoundError, subprocess.CalledProcessError) as e: + exit_code = 127 + if isinstance(e, subprocess.CalledProcessError): + exit_code = e.returncode + return None, exit_code + else: + data = json.loads(output) + active_compat_level = data.get("active-compat-level") + exit_code = 0 + if not isinstance(active_compat_level, int) or active_compat_level < 1: + active_compat_level = None + exit_code = 255 + return active_compat_level, exit_code + + +def _relevant_dh_commands(dh_rules_addons: Iterable[str]) -> Tuple[List[str], int]: + cmd = ["dh_assistant", "list-commands", "--output-format=json"] + if dh_rules_addons: + addons = ",".join(dh_rules_addons) + cmd.append(f"--with={addons}") + try: + output = subprocess.check_output( + cmd, + stderr=subprocess.DEVNULL, + ) + except (FileNotFoundError, subprocess.CalledProcessError) as e: + exit_code = 127 + if isinstance(e, subprocess.CalledProcessError): + exit_code = e.returncode + return [], exit_code + else: + data = json.loads(output) + commands_json = data.get("commands") + commands = [] + for command in commands_json: + if isinstance(command, dict): + command_name = command.get("command") + if isinstance(command_name, str) and command_name: + commands.append(command_name) + return commands, 0 + + +def _add_known_packaging_data( + details: PackagingFileInfo, + plugin_data: PluginProvidedKnownPackagingFile, + dh_compat_level: Optional[int], +): + install_pattern = _kpf_install_pattern( + dh_compat_level, + plugin_data, + ) + config_features = plugin_data.info.get("config_features") + if config_features: + config_features = expand_known_packaging_config_features( + dh_compat_level or 0, + config_features, + ) + _merge_list(details, "config-features", config_features) + + if dh_compat_level is not None: + extra_config_features = [] + for dh_compat_rule in _relevant_dh_compat_rules( + dh_compat_level, plugin_data.info + ): + cf = dh_compat_rule.get("add_config_features") + if cf: + extra_config_features.extend(cf) + if extra_config_features: + extra_config_features = expand_known_packaging_config_features( + dh_compat_level, + extra_config_features, + ) + _merge_list(details, "config-features", extra_config_features) + if "install-pattern" not in details and install_pattern is not None: + details["install-pattern"] = install_pattern + for mk, ok in [ + ("file_categories", "file-categories"), + ("documentation_uris", "documentation-uris"), + ("debputy_cmd_templates", "debputy-cmd-templates"), + ]: + value = plugin_data.info.get(mk) + if value and ok == "debputy-cmd-templates": + value = [escape_shell(*c) for c in value] + _merge_list(details, ok, value) + + +def _scan_debian_dir(debian_dir: VirtualPath) -> Iterator[VirtualPath]: + for p in debian_dir.iterdir: + yield p + if p.is_dir and p.path in ("debian/source", "debian/tests"): + yield from p.iterdir + + +_POST_FORMATTING_REWRITE = { + "period-to-underscore": lambda n: n.replace(".", "_"), +} diff --git a/src/debputy/commands/debputy_cmd/__main__.py b/src/debputy/commands/debputy_cmd/__main__.py index 37f89cd..218ca4f 100644 --- a/src/debputy/commands/debputy_cmd/__main__.py +++ b/src/debputy/commands/debputy_cmd/__main__.py @@ -4,7 +4,6 @@ import json import logging import os import shutil -import stat import subprocess import sys import textwrap @@ -13,34 +12,26 @@ from tempfile import TemporaryDirectory from typing import ( List, Dict, - Iterable, Any, Tuple, - Sequence, Optional, NoReturn, - Mapping, - Union, NamedTuple, Literal, - Set, - Iterator, - TypedDict, - NotRequired, cast, ) from debputy import DEBPUTY_ROOT_DIR, DEBPUTY_PLUGIN_ROOT_DIR +from debputy.analysis import REFERENCE_DATA_TABLE +from debputy.analysis.debian_dir import scan_debian_dir from debputy.commands.debputy_cmd.context import ( CommandContext, add_arg, ROOT_COMMAND, CommandArg, ) -from debputy.commands.debputy_cmd.dc_util import flatten_ppfs from debputy.commands.debputy_cmd.output import _stream_to_pager from debputy.dh_migration.migrators import MIGRATORS -from debputy.dh_migration.migrators_impl import read_dh_addon_sequences from debputy.exceptions import ( DebputyRuntimeError, PluginNotFoundError, @@ -52,14 +43,6 @@ from debputy.exceptions import ( from debputy.package_build.assemble_deb import ( assemble_debs, ) -from debputy.packager_provided_files import ( - detect_all_packager_provided_files, - PackagerProvidedFile, -) -from debputy.plugin.api.spec import ( - VirtualPath, - packager_provided_file_reference_documentation, -) try: from argcomplete import autocomplete @@ -75,25 +58,16 @@ from debputy.filesystem_scan import ( FSRootDir, ) from debputy.plugin.api.impl_types import ( - PackagerProvidedFileClassSpec, DebputyPluginMetadata, - PluginProvidedKnownPackagingFile, - KNOWN_PACKAGING_FILE_CATEGORY_DESCRIPTIONS, - KNOWN_PACKAGING_FILE_CONFIG_FEATURE_DESCRIPTION, - expand_known_packaging_config_features, - InstallPatternDHCompatRule, - KnownPackagingFileInfo, ) from debputy.plugin.api.impl import ( find_json_plugin, find_tests_for_plugin, find_related_implementation_files_for_plugin, parse_json_plugin_desc, - plugin_metadata_for_debputys_own_plugin, ) from debputy.dh_migration.migration import migrate_from_dh from debputy.dh_migration.models import AcceptableMigrationIssues -from debputy.packages import BinaryPackage from debputy.debhelper_emulation import ( dhe_pkgdir, ) @@ -117,14 +91,8 @@ from debputy.util import ( escape_shell, program_name, integrated_with_debhelper, - assume_not_none, ) -REFERENCE_DATA_TABLE = { - "config-features": KNOWN_PACKAGING_FILE_CONFIG_FEATURE_DESCRIPTION, - "file-categories": KNOWN_PACKAGING_FILE_CATEGORY_DESCRIPTIONS, -} - class SharedArgument(NamedTuple): """ @@ -773,341 +741,6 @@ def _dh_integration_generate_debs(context: CommandContext) -> None: ) -PackagingFileInfo = TypedDict( - "PackagingFileInfo", - { - "path": str, - "binary-package": NotRequired[str], - "install-path": NotRequired[str], - "install-pattern": NotRequired[str], - "file-categories": NotRequired[List[str]], - "config-features": NotRequired[List[str]], - "likely-generated-from": NotRequired[List[str]], - "related-tools": NotRequired[List[str]], - "documentation-uris": NotRequired[List[str]], - "debputy-cmd-templates": NotRequired[List[List[str]]], - "generates": NotRequired[str], - "generated-from": NotRequired[str], - }, -) - - -def _scan_debian_dir(debian_dir: VirtualPath) -> Iterator[VirtualPath]: - for p in debian_dir.iterdir: - yield p - if p.is_dir and p.path in ("debian/source", "debian/tests"): - yield from p.iterdir - - -_POST_FORMATTING_REWRITE = { - "period-to-underscore": lambda n: n.replace(".", "_"), -} - - -def _fake_PPFClassSpec( - debputy_plugin_metadata: DebputyPluginMetadata, - stem: str, - doc_uris: Optional[Sequence[str]], - install_pattern: Optional[str], - *, - default_priority: Optional[int] = None, - packageless_is_fallback_for_all_packages: bool = False, - post_formatting_rewrite: Optional[str] = None, - bug_950723: bool = False, -) -> PackagerProvidedFileClassSpec: - if install_pattern is None: - install_pattern = "not-a-real-ppf" - if post_formatting_rewrite is not None: - formatting_hook = _POST_FORMATTING_REWRITE[post_formatting_rewrite] - else: - formatting_hook = None - return PackagerProvidedFileClassSpec( - debputy_plugin_metadata, - stem, - install_pattern, - allow_architecture_segment=True, - allow_name_segment=True, - default_priority=default_priority, - default_mode=0o644, - post_formatting_rewrite=formatting_hook, - packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages, - reservation_only=False, - formatting_callback=None, - bug_950723=bug_950723, - reference_documentation=packager_provided_file_reference_documentation( - format_documentation_uris=doc_uris, - ), - ) - - -def _relevant_dh_compat_rules( - compat_level: Optional[int], - info: KnownPackagingFileInfo, -) -> Iterable[InstallPatternDHCompatRule]: - if compat_level is None: - return - dh_compat_rules = info.get("dh_compat_rules") - if not dh_compat_rules: - return - for dh_compat_rule in dh_compat_rules: - rule_compat_level = dh_compat_rule.get("starting_with_compat_level") - if rule_compat_level is not None and compat_level < rule_compat_level: - continue - yield dh_compat_rule - - -def _kpf_install_pattern( - compat_level: Optional[int], - ppkpf: PluginProvidedKnownPackagingFile, -) -> Optional[str]: - for compat_rule in _relevant_dh_compat_rules(compat_level, ppkpf.info): - install_pattern = compat_rule.get("install_pattern") - if install_pattern is not None: - return install_pattern - return ppkpf.info.get("install_pattern") - - -def _resolve_debhelper_config_files( - debian_dir: VirtualPath, - binary_packages: Mapping[str, BinaryPackage], - debputy_plugin_metadata: DebputyPluginMetadata, - dh_ppf_docs: Dict[str, PluginProvidedKnownPackagingFile], - dh_rules_addons: Iterable[str], - dh_compat_level: int, -) -> Tuple[List[PackagerProvidedFile], Optional[object], int]: - dh_ppfs = {} - commands, exit_code = _relevant_dh_commands(dh_rules_addons) - dh_commands = set(commands) - - cmd = ["dh_assistant", "list-guessed-dh-config-files"] - if dh_rules_addons: - addons = ",".join(dh_rules_addons) - cmd.append(f"--with={addons}") - try: - output = subprocess.check_output( - cmd, - stderr=subprocess.DEVNULL, - ) - except (subprocess.CalledProcessError, FileNotFoundError) as e: - config_files = [] - issues = None - if isinstance(e, subprocess.CalledProcessError): - exit_code = e.returncode - else: - exit_code = 127 - else: - result = json.loads(output) - config_files: List[Union[Mapping[str, Any], object]] = result.get( - "config-files", [] - ) - issues = result.get("issues") - for config_file in config_files: - if not isinstance(config_file, dict): - continue - if config_file.get("file-type") != "pkgfile": - continue - stem = config_file.get("pkgfile") - if stem is None: - continue - internal = config_file.get("internal") - if isinstance(internal, dict): - bug_950723 = internal.get("bug#950723", False) is True - else: - bug_950723 = False - commands = config_file.get("commands") - documentation_uris = [] - related_tools = [] - seen_commands = set() - seen_docs = set() - ppkpf = dh_ppf_docs.get(stem) - if ppkpf: - dh_cmds = ppkpf.info.get("debhelper_commands") - doc_uris = ppkpf.info.get("documentation_uris") - default_priority = ppkpf.info.get("default_priority") - if doc_uris is not None: - seen_docs.update(doc_uris) - documentation_uris.extend(doc_uris) - if dh_cmds is not None: - seen_commands.update(dh_cmds) - related_tools.extend(dh_cmds) - install_pattern = _kpf_install_pattern(dh_compat_level, ppkpf) - post_formatting_rewrite = ppkpf.info.get("post_formatting_rewrite") - packageless_is_fallback_for_all_packages = ppkpf.info.get( - "packageless_is_fallback_for_all_packages", - False, - ) - else: - install_pattern = None - default_priority = None - post_formatting_rewrite = None - packageless_is_fallback_for_all_packages = False - for command in commands: - if isinstance(command, dict): - command_name = command.get("command") - if isinstance(command_name, str) and command_name: - if command_name not in seen_commands: - related_tools.append(command_name) - seen_commands.add(command_name) - manpage = f"man:{command_name}(1)" - if manpage not in seen_docs: - documentation_uris.append(manpage) - seen_docs.add(manpage) - dh_ppfs[stem] = _fake_PPFClassSpec( - debputy_plugin_metadata, - stem, - documentation_uris, - install_pattern, - default_priority=default_priority, - post_formatting_rewrite=post_formatting_rewrite, - packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages, - bug_950723=bug_950723, - ) - for ppkpf in dh_ppf_docs.values(): - stem = ppkpf.detection_value - if stem in dh_ppfs: - continue - - default_priority = ppkpf.info.get("default_priority") - commands = ppkpf.info.get("debhelper_commands") - install_pattern = _kpf_install_pattern(dh_compat_level, ppkpf) - post_formatting_rewrite = ppkpf.info.get("post_formatting_rewrite") - packageless_is_fallback_for_all_packages = ppkpf.info.get( - "packageless_is_fallback_for_all_packages", - False, - ) - if commands and not any(c in dh_commands for c in commands): - continue - dh_ppfs[stem] = _fake_PPFClassSpec( - debputy_plugin_metadata, - stem, - ppkpf.info.get("documentation_uris"), - install_pattern, - default_priority=default_priority, - post_formatting_rewrite=post_formatting_rewrite, - packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages, - ) - all_dh_ppfs = list( - flatten_ppfs( - detect_all_packager_provided_files( - dh_ppfs, - debian_dir, - binary_packages, - allow_fuzzy_matches=True, - ) - ) - ) - return all_dh_ppfs, issues, exit_code - - -def _merge_list( - existing_table: Dict[str, Any], - key: str, - new_data: Optional[Sequence[str]], -) -> None: - if not new_data: - return - existing_values = existing_table.get(key, []) - if isinstance(existing_values, tuple): - existing_values = list(existing_values) - assert isinstance(existing_values, list) - seen = set(existing_values) - existing_values.extend(x for x in new_data if x not in seen) - existing_table[key] = existing_values - - -def _merge_ppfs( - identified: List[PackagingFileInfo], - seen_paths: Set[str], - ppfs: List[PackagerProvidedFile], - context: Mapping[str, PluginProvidedKnownPackagingFile], - dh_compat_level: Optional[int], -) -> None: - for ppf in ppfs: - key = ppf.path.path - ref_doc = ppf.definition.reference_documentation - documentation_uris = ( - ref_doc.format_documentation_uris if ref_doc is not None else None - ) - - if not ppf.definition.installed_as_format.startswith("not-a-real-ppf"): - try: - parts = ppf.compute_dest() - except RuntimeError: - dest = None - else: - dest = "/".join(parts).lstrip(".") - else: - dest = None - seen_paths.add(key) - details: PackagingFileInfo = { - "path": key, - "binary-package": ppf.package_name, - } - if ppf.fuzzy_match and key.endswith(".in"): - _merge_list(details, "file-categories", ["generic-template"]) - details["generates"] = key[:-3] - elif assume_not_none(ppf.path.parent_dir).get(ppf.path.name + ".in"): - _merge_list(details, "file-categories", ["generated"]) - details["generated-from"] = key + ".in" - if dest is not None: - details["install-path"] = dest - identified.append(details) - - extra_details = context.get(ppf.definition.stem) - if extra_details is not None: - _add_known_packaging_data(details, extra_details, dh_compat_level) - - _merge_list(details, "documentation-uris", documentation_uris) - - -def _extract_dh_compat_level() -> Tuple[Optional[int], int]: - try: - output = subprocess.check_output( - ["dh_assistant", "active-compat-level"], - stderr=subprocess.DEVNULL, - ) - except (FileNotFoundError, subprocess.CalledProcessError) as e: - exit_code = 127 - if isinstance(e, subprocess.CalledProcessError): - exit_code = e.returncode - return None, exit_code - else: - data = json.loads(output) - active_compat_level = data.get("active-compat-level") - exit_code = 0 - if not isinstance(active_compat_level, int) or active_compat_level < 1: - active_compat_level = None - exit_code = 255 - return active_compat_level, exit_code - - -def _relevant_dh_commands(dh_rules_addons: Iterable[str]) -> Tuple[List[str], int]: - cmd = ["dh_assistant", "list-commands", "--output-format=json"] - if dh_rules_addons: - addons = ",".join(dh_rules_addons) - cmd.append(f"--with={addons}") - try: - output = subprocess.check_output( - cmd, - stderr=subprocess.DEVNULL, - ) - except (FileNotFoundError, subprocess.CalledProcessError) as e: - exit_code = 127 - if isinstance(e, subprocess.CalledProcessError): - exit_code = e.returncode - return [], exit_code - else: - data = json.loads(output) - commands_json = data.get("commands") - commands = [] - for command in commands_json: - if isinstance(command, dict): - command_name = command.get("command") - if isinstance(command_name, str) and command_name: - commands.append(command_name) - return commands, 0 - - @tool_support_commands.register_subcommand( "supports-tool-command", help_description="Test where a given tool-support command exists", @@ -1180,50 +813,6 @@ def _export_reference_data(context: CommandContext) -> None: raise AssertionError(f"Unsupported output format {output_format}") -def _add_known_packaging_data( - details: PackagingFileInfo, - plugin_data: PluginProvidedKnownPackagingFile, - dh_compat_level: Optional[int], -): - install_pattern = _kpf_install_pattern( - dh_compat_level, - plugin_data, - ) - config_features = plugin_data.info.get("config_features") - if config_features: - config_features = expand_known_packaging_config_features( - dh_compat_level or 0, - config_features, - ) - _merge_list(details, "config-features", config_features) - - if dh_compat_level is not None: - extra_config_features = [] - for dh_compat_rule in _relevant_dh_compat_rules( - dh_compat_level, plugin_data.info - ): - cf = dh_compat_rule.get("add_config_features") - if cf: - extra_config_features.extend(cf) - if extra_config_features: - extra_config_features = expand_known_packaging_config_features( - dh_compat_level, - extra_config_features, - ) - _merge_list(details, "config-features", extra_config_features) - if "install-pattern" not in details and install_pattern is not None: - details["install-pattern"] = install_pattern - for mk, ok in [ - ("file_categories", "file-categories"), - ("documentation_uris", "documentation-uris"), - ("debputy_cmd_templates", "debputy-cmd-templates"), - ]: - value = plugin_data.info.get(mk) - if value and ok == "debputy-cmd-templates": - value = [escape_shell(*c) for c in value] - _merge_list(details, ok, value) - - @tool_support_commands.register_subcommand( "annotate-debian-directory", log_only_to_stderr=True, @@ -1235,130 +824,13 @@ def _annotate_debian_directory(context: CommandContext) -> None: # Validates that we are run from a debian directory as a side effect binary_packages = context.binary_packages() feature_set = context.load_plugins() - known_packaging_files = feature_set.known_packaging_files - debputy_plugin_metadata = plugin_metadata_for_debputys_own_plugin() - - reference_data_set_names = [ - "config-features", - "file-categories", - ] - for n in reference_data_set_names: - assert n in REFERENCE_DATA_TABLE - - annotated: List[PackagingFileInfo] = [] - seen_paths = set() - - r = read_dh_addon_sequences(context.debian_dir) - if r is not None: - bd_sequences, dr_sequences = r - drules_sequences = bd_sequences | dr_sequences - else: - drules_sequences = set() - is_debputy_package = ( - "debputy" in drules_sequences - or "zz-debputy" in drules_sequences - or "zz_debputy" in drules_sequences - or "zz-debputy-rrr" in drules_sequences - ) - dh_compat_level, dh_assistant_exit_code = _extract_dh_compat_level() - dh_issues = [] - - static_packaging_files = { - kpf.detection_value: kpf - for kpf in known_packaging_files.values() - if kpf.detection_method == "path" - } - dh_pkgfile_docs = { - kpf.detection_value: kpf - for kpf in known_packaging_files.values() - if kpf.detection_method == "dh.pkgfile" - } - - if is_debputy_package: - all_debputy_ppfs = list( - flatten_ppfs( - detect_all_packager_provided_files( - feature_set.packager_provided_files, - context.debian_dir, - binary_packages, - allow_fuzzy_matches=True, - ) - ) - ) - else: - all_debputy_ppfs = [] - - if dh_compat_level is not None: - ( - all_dh_ppfs, - dh_issues, - dh_assistant_exit_code, - ) = _resolve_debhelper_config_files( - context.debian_dir, - binary_packages, - debputy_plugin_metadata, - dh_pkgfile_docs, - drules_sequences, - dh_compat_level, - ) - - else: - all_dh_ppfs = [] - for ppf in all_debputy_ppfs: - key = ppf.path.path - ref_doc = ppf.definition.reference_documentation - documentation_uris = ( - ref_doc.format_documentation_uris if ref_doc is not None else None - ) - details: PackagingFileInfo = { - "path": key, - "debputy-cmd-templates": [ - ["debputy", "plugin", "show", "p-p-f", ppf.definition.stem] - ], - } - if ppf.fuzzy_match and key.endswith(".in"): - _merge_list(details, "file-categories", ["generic-template"]) - details["generates"] = key[:-3] - elif assume_not_none(ppf.path.parent_dir).get(ppf.path.name + ".in"): - _merge_list(details, "file-categories", ["generated"]) - details["generated-from"] = key + ".in" - seen_paths.add(key) - annotated.append(details) - static_details = static_packaging_files.get(key) - if static_details is not None: - # debhelper compat rules does not apply to debputy files - _add_known_packaging_data(details, static_details, None) - if documentation_uris: - details["documentation-uris"] = list(documentation_uris) - - _merge_ppfs(annotated, seen_paths, all_dh_ppfs, dh_pkgfile_docs, dh_compat_level) - - for virtual_path in _scan_debian_dir(context.debian_dir): - key = virtual_path.path - if key in seen_paths: - continue - if virtual_path.is_symlink: - try: - st = os.stat(virtual_path.fs_path) - except FileNotFoundError: - continue - else: - if not stat.S_ISREG(st.st_mode): - continue - elif not virtual_path.is_file: - continue - - static_match = static_packaging_files.get(virtual_path.path) - if static_match is not None: - details: PackagingFileInfo = { - "path": key, - } - annotated.append(details) - if assume_not_none(virtual_path.parent_dir).get(virtual_path.name + ".in"): - details["generated-from"] = key + ".in" - _merge_list(details, "file-categories", ["generated"]) - _add_known_packaging_data(details, static_match, dh_compat_level) + result = scan_debian_dir( + feature_set, + binary_packages, + context.debian_dir, + ) + annotated, reference_data_set_names, dh_assistant_exit_code, dh_issues = result data = { "result": annotated, diff --git a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py index ee64a84..d715e5a 100644 --- a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py +++ b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py @@ -247,6 +247,7 @@ def lsp_describe_features(context: CommandContext) -> None: @ROOT_COMMAND.register_subcommand( "lint", log_only_to_stderr=True, + help_description="Provide diagnostics for the packaging (like `lsp server` except no editor is needed)", argparser=[ add_arg( "--spellcheck", @@ -305,6 +306,7 @@ def lint_cmd(context: CommandContext) -> None: @ROOT_COMMAND.register_subcommand( "reformat", + help_description="Reformat the packaging files based on the packaging/maintainer rules", argparser=[ add_arg( "--style", @@ -361,7 +363,7 @@ def reformat_cmd(context: CommandContext) -> None: perform_reformat(context, named_style=context.parsed_args.named_style) -def ensure_lint_and_lsp_commands_are_loaded(): +def ensure_lint_and_lsp_commands_are_loaded() -> None: # Loading the module does the heavy lifting # However, having this function means that we do not have an "unused" import that some tool # gets tempted to remove diff --git a/src/debputy/commands/debputy_cmd/plugin_cmds.py b/src/debputy/commands/debputy_cmd/plugin_cmds.py index 60d3c70..83bb88f 100644 --- a/src/debputy/commands/debputy_cmd/plugin_cmds.py +++ b/src/debputy/commands/debputy_cmd/plugin_cmds.py @@ -1,5 +1,4 @@ import argparse -import itertools import operator import os import sys @@ -12,17 +11,16 @@ from typing import ( Any, Optional, Type, - Mapping, Callable, ) from debputy import DEBPUTY_DOC_ROOT_DIR +from debputy.analysis.analysis_util import flatten_ppfs from debputy.commands.debputy_cmd.context import ( CommandContext, add_arg, ROOT_COMMAND, ) -from debputy.commands.debputy_cmd.dc_util import flatten_ppfs from debputy.commands.debputy_cmd.output import ( _stream_to_pager, _output_styling, @@ -32,8 +30,6 @@ from debputy.exceptions import DebputySubstitutionError from debputy.filesystem_scan import build_virtual_fs from debputy.manifest_parser.base_types import TypeMapping from debputy.manifest_parser.declarative_parser import ( - DeclarativeMappingInputParser, - DeclarativeNonMappingInputParser, BASIC_SIMPLE_TYPES, ) from debputy.manifest_parser.parser_data import ParserContextData @@ -49,9 +45,6 @@ from debputy.plugin.api.impl_types import ( PackagerProvidedFileClassSpec, PluginProvidedManifestVariable, DispatchingParserBase, - DeclarativeInputParser, - DebputyPluginMetadata, - DispatchingObjectParser, SUPPORTED_DISPATCHABLE_TABLE_PARSERS, OPARSER_MANIFEST_ROOT, PluginProvidedDiscardRule, @@ -60,13 +53,10 @@ from debputy.plugin.api.impl_types import ( PluginProvidedTypeMapping, ) from debputy.plugin.api.spec import ( - ParserDocumentation, - reference_documentation, - undocumented_attr, TypeMappingExample, ) from debputy.substitution import Substitution -from debputy.util import _error, assume_not_none, _warn +from debputy.util import _error, _warn plugin_dispatcher = ROOT_COMMAND.add_dispatching_subcommand( "plugin", diff --git a/src/debputy/debhelper_emulation.py b/src/debputy/debhelper_emulation.py index 65a26f8..8242a32 100644 --- a/src/debputy/debhelper_emulation.py +++ b/src/debputy/debhelper_emulation.py @@ -243,13 +243,16 @@ _FIND_DH_WITH = re.compile(r"--with(?:\s+|=)(\S+)") _DEP_REGEX = re.compile("^([a-z0-9][-+.a-z0-9]+)", re.ASCII) -def parse_drules_for_addons(lines: Iterable[str], sequences: Set[str]) -> None: +def parse_drules_for_addons(lines: Iterable[str], sequences: Set[str]) -> bool: + saw_dh = False for line in lines: if not line.startswith("\tdh "): continue + saw_dh = True for match in _FIND_DH_WITH.finditer(line): sequence_def = match.group(1) sequences.update(sequence_def.split(",")) + return saw_dh def extract_dh_addons_from_control( diff --git a/src/debputy/dh_migration/migration.py b/src/debputy/dh_migration/migration.py index a1bd15a..59a7ee4 100644 --- a/src/debputy/dh_migration/migration.py +++ b/src/debputy/dh_migration/migration.py @@ -157,7 +157,7 @@ def _check_migration_target( r = read_dh_addon_sequences(debian_dir) if r is None and migration_target is None: _error("debian/control is missing and no migration target was provided") - bd_sequences, dr_sequences = r + bd_sequences, dr_sequences, _ = r all_sequences = bd_sequences | dr_sequences has_zz_debputy = "zz-debputy" in all_sequences or "debputy" in all_sequences diff --git a/src/debputy/dh_migration/migrators_impl.py b/src/debputy/dh_migration/migrators_impl.py index d68768c..48ec1e0 100644 --- a/src/debputy/dh_migration/migrators_impl.py +++ b/src/debputy/dh_migration/migrators_impl.py @@ -1503,23 +1503,24 @@ def detect_obsolete_substvars( def read_dh_addon_sequences( debian_dir: VirtualPath, -) -> Optional[Tuple[Set[str], Set[str]]]: +) -> Optional[Tuple[Set[str], Set[str], bool]]: ctrl_file = debian_dir.get("control") if ctrl_file: dr_sequences: Set[str] = set() bd_sequences: Set[str] = set() drules = debian_dir.get("rules") + saw_dh = False if drules and drules.is_file: with drules.open() as fd: - parse_drules_for_addons(fd, dr_sequences) + saw_dh = parse_drules_for_addons(fd, dr_sequences) with ctrl_file.open() as fd: ctrl = list(Deb822.iter_paragraphs(fd)) source_paragraph = ctrl[0] if ctrl else {} extract_dh_addons_from_control(source_paragraph, bd_sequences) - return bd_sequences, dr_sequences + return bd_sequences, dr_sequences, saw_dh return None @@ -1539,7 +1540,7 @@ def detect_dh_addons_zz_debputy_rrr( ) return - bd_sequences, dr_sequences = r + bd_sequences, dr_sequences, _ = r remaining_sequences = bd_sequences | dr_sequences saw_dh_debputy = "zz-debputy-rrr" in remaining_sequences @@ -1565,7 +1566,7 @@ def detect_dh_addons( ) return - bd_sequences, dr_sequences = r + bd_sequences, dr_sequences, _ = r remaining_sequences = bd_sequences | dr_sequences saw_dh_debputy = ( diff --git a/src/debputy/linting/lint_util.py b/src/debputy/linting/lint_util.py index 0c293f0..30e1177 100644 --- a/src/debputy/linting/lint_util.py +++ b/src/debputy/linting/lint_util.py @@ -170,6 +170,8 @@ class LintDiagnosticResult: is_file_level_diagnostic: bool has_broken_range: bool missing_severity: bool + discovered_in: str + report_for_related_file: Optional[str] class LintReport: @@ -183,7 +185,7 @@ class LintReport: self.number_of_broken_diagnostics: int = 0 self.lint_state: Optional[LintState] = None self.start_timestamp = datetime.datetime.now() - self.durations: Mapping[str, float] = collections.defaultdict(lambda: 0.0) + self.durations: typing.Dict[str, float] = collections.defaultdict(lambda: 0.0) self._timer = time.perf_counter() @contextlib.contextmanager @@ -217,6 +219,7 @@ class LintReport: assert lint_state is not None if in_file is None: in_file = lint_state.path + discovered_in_file = in_file severity = diagnostic.severity missing_severity = False error_marker: Optional[RuntimeError] = None @@ -230,13 +233,30 @@ class LintReport: diag_range = diagnostic.range start_pos = diag_range.start end_pos = diag_range.start - is_file_level_diagnostic = _is_file_level_diagnostic( - lines, - start_pos.line, - start_pos.character, - end_pos.line, - end_pos.character, - ) + diag_data = diagnostic.data + if isinstance(diag_data, dict): + report_for_related_file = diag_data.get("report_for_related_file") + if report_for_related_file is None or not isinstance( + report_for_related_file, str + ): + report_for_related_file = None + else: + in_file = report_for_related_file + # Force it to exist in self.durations, since subclasses can use .items() or "foo" in self.durations. + if in_file not in self.durations: + self.durations[in_file] = 0 + else: + report_for_related_file = None + if report_for_related_file is not None: + is_file_level_diagnostic = True + else: + is_file_level_diagnostic = _is_file_level_diagnostic( + lines, + start_pos.line, + start_pos.character, + end_pos.line, + end_pos.character, + ) has_broken_range = not is_file_level_diagnostic and ( end_pos.line > len(lines) or start_pos.line < 0 ) @@ -251,6 +271,8 @@ class LintReport: is_file_level_diagnostic, has_broken_range, missing_severity, + report_for_related_file=report_for_related_file, + discovered_in=discovered_in_file, ) self.diagnostics_by_file[in_file].append(diagnostic_result) @@ -351,13 +373,22 @@ class TermLintReport(LintReport): bg="black", style="bold", ) - start_line = diagnostic.range.start.line - start_position = diagnostic.range.start.character - end_line = diagnostic.range.end.line - end_position = diagnostic.range.end.character + + if diagnostic_result.is_file_level_diagnostic: + start_line = 0 + start_position = 0 + end_line = 0 + end_position = 0 + else: + start_line = diagnostic.range.start.line + start_position = diagnostic.range.start.character + end_line = diagnostic.range.end.line + end_position = diagnostic.range.end.character + has_fixit = "" lines = lint_state.lines line_no_width = len(str(len(lines))) + if diagnostic_result.result_state == LintDiagnosticResultState.FIXABLE: has_fixit = " [Correctable via --auto-fix]" print( diff --git a/src/debputy/lsp/debputy_ls.py b/src/debputy/lsp/debputy_ls.py index 2009fd7..e195208 100644 --- a/src/debputy/lsp/debputy_ls.py +++ b/src/debputy/lsp/debputy_ls.py @@ -29,7 +29,6 @@ from debputy.packages import ( BinaryPackage, DctrlParser, ) -from debputy.plugin.api import VirtualPath from debputy.plugin.api.feature_set import PluginProvidedFeatureSet from debputy.util import _info from debputy.yaml import MANIFEST_YAML, YAMLError diff --git a/src/debputy/lsp/diagnostics.py b/src/debputy/lsp/diagnostics.py index 618b91d..5c1907e 100644 --- a/src/debputy/lsp/diagnostics.py +++ b/src/debputy/lsp/diagnostics.py @@ -6,3 +6,4 @@ LintSeverity = Literal["error", "warning", "informational", "pedantic", "spellin class DiagnosticData(TypedDict): quickfixes: NotRequired[Optional[List[Any]]] lint_severity: NotRequired[Optional[LintSeverity]] + report_for_related_file: NotRequired[str] diff --git a/src/debputy/lsp/lsp_debian_control.py b/src/debputy/lsp/lsp_debian_control.py index 6e775c5..370a14f 100644 --- a/src/debputy/lsp/lsp_debian_control.py +++ b/src/debputy/lsp/lsp_debian_control.py @@ -1,4 +1,5 @@ import dataclasses +import os.path import re import textwrap from typing import ( @@ -9,30 +10,10 @@ from typing import ( Mapping, List, Dict, + Iterable, ) -from debputy.lsprotocol.types import ( - DiagnosticSeverity, - Range, - Diagnostic, - Position, - FoldingRange, - FoldingRangeParams, - CompletionItem, - CompletionList, - CompletionParams, - DiagnosticRelatedInformation, - Location, - HoverParams, - Hover, - TEXT_DOCUMENT_CODE_ACTION, - SemanticTokens, - SemanticTokensParams, - WillSaveTextDocumentParams, - TextEdit, - DocumentFormattingParams, -) - +from debputy.analysis.debian_dir import scan_debian_dir from debputy.linting.lint_util import LintState from debputy.lsp.debputy_ls import DebputyLanguageServer from debputy.lsp.diagnostics import DiagnosticData @@ -75,7 +56,6 @@ from debputy.lsp.spellchecking import default_spellchecker from debputy.lsp.text_util import ( normalize_dctrl_field_name, LintCapablePositionCodec, - detect_possible_typo, te_range_to_lsp, ) from debputy.lsp.vendoring._deb822_repro import ( @@ -86,6 +66,28 @@ from debputy.lsp.vendoring._deb822_repro.parsing import ( Deb822KeyValuePairElement, LIST_SPACE_SEPARATED_INTERPRETATION, ) +from debputy.lsprotocol.types import ( + DiagnosticSeverity, + Range, + Diagnostic, + Position, + FoldingRange, + FoldingRangeParams, + CompletionItem, + CompletionList, + CompletionParams, + DiagnosticRelatedInformation, + Location, + HoverParams, + Hover, + TEXT_DOCUMENT_CODE_ACTION, + SemanticTokens, + SemanticTokensParams, + WillSaveTextDocumentParams, + TextEdit, + DocumentFormattingParams, +) +from debputy.util import detect_possible_typo try: from debputy.lsp.vendoring._deb822_repro.locatable import ( @@ -94,7 +96,6 @@ try: START_POSITION, ) - from pygls.server import LanguageServer from pygls.workspace import TextDocument except ImportError: pass @@ -919,6 +920,7 @@ def _lint_debian_control( paragraphs = list(deb822_file) source_paragraph = paragraphs[0] if paragraphs else None + binary_stanzas_w_pos = [] for paragraph_no, paragraph in enumerate(paragraphs, start=1): paragraph_pos = paragraph.position_in_file() @@ -928,6 +930,7 @@ def _lint_debian_control( if is_binary_paragraph: known_fields = BINARY_FIELDS other_known_fields = SOURCE_FIELDS + binary_stanzas_w_pos.append((paragraph, paragraph_pos)) else: known_fields = SOURCE_FIELDS other_known_fields = BINARY_FIELDS @@ -944,9 +947,113 @@ def _lint_debian_control( diagnostics, ) + _detect_misspelled_packaging_files( + lint_state, + binary_stanzas_w_pos, + diagnostics, + ) + return diagnostics +def _package_range_of_stanza( + lint_state: LintState, + binary_stanzas: List[Tuple[Deb822ParagraphElement, TEPosition]], +) -> Iterable[Tuple[str, Range]]: + for stanza, stanza_position in binary_stanzas: + kvpair = stanza.get_kvpair_element("Package") + if kvpair is None: + continue + representation_field_range = kvpair.range_in_parent().relative_to( + stanza_position + ) + representation_field_range = lint_state.position_codec.range_to_client_units( + lint_state.lines, + te_range_to_lsp(representation_field_range), + ) + yield stanza["Package"], representation_field_range + + +def _detect_misspelled_packaging_files( + lint_state: LintState, + binary_stanzas_w_pos: List[Tuple[Deb822ParagraphElement, TEPosition]], + diagnostics: List[Diagnostic], +) -> None: + debian_dir = lint_state.debian_dir + binary_packages = lint_state.binary_packages + if debian_dir is None or binary_packages is None: + return + all_pkg_file_data, _, _, _ = scan_debian_dir( + lint_state.plugin_feature_set, + binary_packages, + debian_dir, + ) + stanza_ranges = { + p: r for p, r in _package_range_of_stanza(lint_state, binary_stanzas_w_pos) + } + + for pkg_file_data in all_pkg_file_data: + binary_package = pkg_file_data.get("binary-package") + explicit_package = pkg_file_data.get("pkgfile-explicit-package-name", True) + name_segment = pkg_file_data.get("pkgfile-name-segment") + stem = pkg_file_data.get("pkgfile-stem") + if binary_package is None or stem is None: + continue + diag_range = stanza_ranges.get(binary_package) + if diag_range is None: + continue + + path = pkg_file_data["path"] + likely_typo_of = pkg_file_data.get("likely-typo-of") + if likely_typo_of is not None: + diagnostics.append( + Diagnostic( + diag_range, + f'The file "{path}" is likely a typo of "{likely_typo_of}"', + severity=DiagnosticSeverity.Warning, + source="debputy", + data=DiagnosticData( + report_for_related_file=path, + ), + ) + ) + continue + + if not pkg_file_data.get("pkgfile-is-active-in-build", True): + diagnostics.append( + Diagnostic( + diag_range, + f"The file {path} is related to a command that is not active in the dh sequence" + " with the current addons", + severity=DiagnosticSeverity.Warning, + source="debputy", + data=DiagnosticData( + report_for_related_file=path, + ), + ) + ) + continue + + if not explicit_package and name_segment is not None: + basename = os.path.basename(path) + alt_name = f"{binary_package}.{stem}" + arch_restriction = pkg_file_data.get("pkgfile-architecture-restriction") + if arch_restriction is not None: + alt_name = f"{alt_name}.{arch_restriction}" + diagnostics.append( + Diagnostic( + diag_range, + f'Possible typo in "{path}". Consider renaming the file to "debian/{binary_package}.{basename}"' + f' or "debian/{alt_name} if it is intended for {binary_package}', + severity=DiagnosticSeverity.Warning, + source="debputy", + data=DiagnosticData( + report_for_related_file=path, + ), + ) + ) + + @lsp_will_save_wait_until(_LANGUAGE_IDS) def _debian_control_on_save_formatting( ls: "DebputyLanguageServer", diff --git a/src/debputy/lsp/lsp_debian_control_reference_data.py b/src/debputy/lsp/lsp_debian_control_reference_data.py index f4a962a..9a589bf 100644 --- a/src/debputy/lsp/lsp_debian_control_reference_data.py +++ b/src/debputy/lsp/lsp_debian_control_reference_data.py @@ -26,6 +26,7 @@ from typing import ( ) from debian.debian_support import DpkgArchTable, Version + from debputy.lsprotocol.types import ( DiagnosticSeverity, Diagnostic, @@ -38,6 +39,7 @@ from debputy.lsprotocol.types import ( CompletionItemTag, MarkupKind, CompletionItemKind, + CompletionItemLabelDetails, ) from debputy.filesystem_scan import VirtualPathBase @@ -57,7 +59,6 @@ from debputy.lsp.text_edit import apply_text_edits from debputy.lsp.text_util import ( normalize_dctrl_field_name, LintCapablePositionCodec, - detect_possible_typo, te_range_to_lsp, trim_end_of_line_whitespace, ) @@ -90,7 +91,7 @@ from debputy.lsp.vendoring._deb822_repro.types import TE from debputy.lsp.vendoring.wrap_and_sort import _sort_packages_key from debputy.path_matcher import BasenameGlobMatch from debputy.plugin.api import VirtualPath -from debputy.util import PKGNAME_REGEX, _info +from debputy.util import PKGNAME_REGEX, _info, detect_possible_typo try: from debputy.lsp.vendoring._deb822_repro.locatable import ( @@ -298,6 +299,7 @@ ALL_SECTIONS = allowed_values( ALL_PRIORITIES = allowed_values( Keyword( "required", + synopsis_doc="[RARE]: Package is Essential or an Essential package needs it (and is not a library)", hover_text=textwrap.dedent( """\ The package is necessary for the proper functioning of the system (read: dpkg needs it). @@ -311,6 +313,7 @@ ALL_PRIORITIES = allowed_values( ), Keyword( "important", + synopsis_doc="[RARE]: Bare minimum of bare minimum of commonly-expected and necessary tools", hover_text=textwrap.dedent( """\ The *important* packages are a bare minimum of commonly-expected and necessary tools. @@ -324,6 +327,7 @@ ALL_PRIORITIES = allowed_values( ), Keyword( "standard", + synopsis_doc="[RARE]: If your distribution installer would install this by default (not for libraries)", hover_text=textwrap.dedent( """\ These packages provide a reasonable small but not too limited character-mode system. This is @@ -340,6 +344,8 @@ ALL_PRIORITIES = allowed_values( ), Keyword( "optional", + synopsis_doc="The default priority.", + sort_text="aa-optional", hover_text="This is the default priority and used by the majority of all packages" " in the Debian archive", ), @@ -347,6 +353,8 @@ ALL_PRIORITIES = allowed_values( "extra", is_obsolete=True, replaced_by="optional", + sort_text="zz-extra", + synopsis_doc="Obsolete alias of `optional`", hover_text="Obsolete alias of `optional`.", ), ) @@ -359,6 +367,7 @@ def all_architectures_and_wildcards( yield Keyword( "any", is_exclusive=True, + synopsis_doc="Build once per machine architecture (native code, such as C/C++, interpreter to C bindings)", hover_text=textwrap.dedent( """\ The package is an architecture dependent package and need to be compiled for each and every @@ -372,6 +381,7 @@ def all_architectures_and_wildcards( yield Keyword( "all", is_exclusive=True, + synopsis_doc="Independent of machine architecture (scripts, Java without JNI, data or documentation)", hover_text=textwrap.dedent( """\ The package is an architecture independent package. This is typically fitting for packages containing @@ -1237,6 +1247,7 @@ class Deb822KnownField: lint_state, stanza_parts, "", + markdown_kind, is_completion_for_field=True, ) if options is not None and len(options) == 1: @@ -1288,7 +1299,7 @@ class Deb822KnownField: if value_being_completed == "": current_dir = base_dir - unmatched_parts: Sequence[str] = tuple() + unmatched_parts: Sequence[str] = () else: current_dir, unmatched_parts = base_dir.attempt_lookup( value_being_completed @@ -1311,6 +1322,9 @@ class Deb822KnownField: " " in child.name or "\t" in child.name ): continue + sort_text = ( + f"z-{child.name}" if child.name.startswith(".") else f"a-{child.name}" + ) if child.is_dir: if _should_ignore_dir( child, @@ -1321,7 +1335,12 @@ class Deb822KnownField: items.append( CompletionItem( f"{child.path}/", + label_details=CompletionItemLabelDetails( + description=child.path, + ), insert_text=path_escaper(f"{child.path}/"), + filter_text=f"{child.path}/", + sort_text=sort_text, kind=CompletionItemKind.Folder, ) ) @@ -1329,7 +1348,12 @@ class Deb822KnownField: items.append( CompletionItem( child.path, + label_details=CompletionItemLabelDetails( + description=child.path, + ), insert_text=path_escaper(child.path), + filter_text=child.path, + sort_text=sort_text, kind=CompletionItemKind.File, ) ) @@ -1340,6 +1364,7 @@ class Deb822KnownField: lint_state: LintState, stanza_parts: Sequence[Deb822ParagraphElement], value_being_completed: str, + markdown_kind: MarkupKind, *, is_completion_for_field: bool = False, ) -> Optional[Sequence[CompletionItem]]: @@ -1374,6 +1399,13 @@ class Deb822KnownField: CompletionItem( keyword.value, insert_text=keyword.value, + sort_text=keyword.sort_text, + detail=keyword.synopsis_doc, + documentation=( + MarkupContent(value=keyword.hover_text, kind=markdown_kind) + if keyword.hover_text + else None + ), ) for keyword in known_values.values() if keyword.is_keyword_valid_completion_in_stanza(stanza_parts) @@ -1430,6 +1462,7 @@ class Deb822KnownField: stanza_position, lint_state, ) + yield from self._dep5_file_list_diagnostics(kvpair, kvpair_position, lint_state) if not self.spellcheck_value: yield from self._known_value_diagnostics( kvpair, @@ -1484,6 +1517,56 @@ class Deb822KnownField: ), ) + def _dep5_file_list_diagnostics( + self, + kvpair: Deb822KeyValuePairElement, + kvpair_position: "TEPosition", + lint_state: LintState, + ) -> Iterable[Diagnostic]: + source_root = lint_state.source_root + if ( + self.field_value_class != FieldValueClass.DEP5_FILE_LIST + or source_root is None + ): + return + interpreter = self.field_value_class.interpreter() + values = kvpair.interpret_as(interpreter) + value_off = kvpair.value_element.position_in_parent().relative_to( + kvpair_position + ) + + assert interpreter is not None + + for token in values.iter_parts(): + if token.is_whitespace: + continue + text = token.convert_to_text() + if "?" in text or "*" in text: + # TODO: We should validate these as well + continue + matched_path, missing_part = source_root.attempt_lookup(text) + # It is common practice to delete "dirty" files during clean. This causes files listed + # in `debian/copyright` to go missing and as a consequence, we do not validate whether + # they are present (that would require us to check the `.orig.tar`, which we could but + # do not have the infrastructure for). + if not missing_part and matched_path.is_dir: + path_range_te = token.range_in_parent().relative_to(value_off) + path_range = lint_state.position_codec.range_to_client_units( + lint_state.lines, + te_range_to_lsp(path_range_te), + ) + yield Diagnostic( + path_range, + "Directories cannot be a match. Use `dir/*` to match everything in it", + severity=DiagnosticSeverity.Warning, + source="debputy", + data=DiagnosticData( + quickfixes=[ + propose_correct_text_quick_fix(f"{matched_path.path}/*") + ] + ), + ) + def _known_value_diagnostics( self, kvpair: Deb822KeyValuePairElement, @@ -2962,6 +3045,9 @@ BINARY_FIELDS = _fields( known_values=allowed_values( Keyword( "no", + # Show `no` after `foreign` and `same`. Often, you want `same` or `foreign`. + sort_text="zz-no", + synopsis_doc='No Multi-Arch support (often used for "reviewed and no support possible/needed")', hover_text=textwrap.dedent( """\ The default. The package can be installed for at most one architecture at the time. It can @@ -2977,6 +3063,7 @@ BINARY_FIELDS = _fields( ), Keyword( "foreign", + synopsis_doc="Can satisfy dependencies for other architectures (common for data and *some* scripts)", hover_text=textwrap.dedent( """\ The package can be installed for at most one architecture at the time. However, it can @@ -2995,6 +3082,7 @@ BINARY_FIELDS = _fields( Keyword( "same", can_complete_keyword_in_stanza=_complete_only_in_arch_dep_pkgs, + synopsis_doc="Co-installable with itself for different architectures (common for native libraries)", hover_text=textwrap.dedent( """\ The same version of the package can be co-installed for multiple architecture. However, @@ -3010,6 +3098,9 @@ BINARY_FIELDS = _fields( ), Keyword( "allowed", + # Never show `allowed` first, it is the absolute least likely candidate. + sort_text="zzzz-allowed", + synopsis_doc="[RARE]: Consumer decides whether it is `same` and `foreign`", hover_text=textwrap.dedent( """\ **Advanced and very rare value**. This value is exceedingly rare to the point that less diff --git a/src/debputy/lsp/lsp_debian_copyright.py b/src/debputy/lsp/lsp_debian_copyright.py index a6d5573..9cbac26 100644 --- a/src/debputy/lsp/lsp_debian_copyright.py +++ b/src/debputy/lsp/lsp_debian_copyright.py @@ -67,7 +67,6 @@ from debputy.lsp.spellchecking import default_spellchecker from debputy.lsp.text_util import ( normalize_dctrl_field_name, LintCapablePositionCodec, - detect_possible_typo, te_range_to_lsp, ) from debputy.lsp.vendoring._deb822_repro import ( @@ -78,6 +77,7 @@ from debputy.lsp.vendoring._deb822_repro.parsing import ( Deb822KeyValuePairElement, LIST_SPACE_SEPARATED_INTERPRETATION, ) +from debputy.util import detect_possible_typo try: from debputy.lsp.vendoring._deb822_repro.locatable import ( diff --git a/src/debputy/lsp/lsp_debian_debputy_manifest.py b/src/debputy/lsp/lsp_debian_debputy_manifest.py index 78575cb..fd7e6b0 100644 --- a/src/debputy/lsp/lsp_debian_debputy_manifest.py +++ b/src/debputy/lsp/lsp_debian_debputy_manifest.py @@ -46,7 +46,6 @@ from debputy.lsp.lsp_generic_yaml import ( from debputy.lsp.quickfixes import propose_correct_text_quick_fix from debputy.lsp.text_util import ( LintCapablePositionCodec, - detect_possible_typo, ) from debputy.manifest_parser.base_types import DebputyDispatchableType from debputy.manifest_parser.declarative_parser import ( @@ -66,7 +65,7 @@ from debputy.plugin.api.impl_types import ( InPackageContextParser, DeclarativeValuelessKeywordInputParser, ) -from debputy.util import _info +from debputy.util import _info, detect_possible_typo from debputy.yaml.compat import ( Node, CommentedMap, diff --git a/src/debputy/lsp/lsp_debian_rules.py b/src/debputy/lsp/lsp_debian_rules.py index 072b785..7c6d627 100644 --- a/src/debputy/lsp/lsp_debian_rules.py +++ b/src/debputy/lsp/lsp_debian_rules.py @@ -43,7 +43,7 @@ from debputy.lsp.spellchecking import spellcheck_line from debputy.lsp.text_util import ( LintCapablePositionCodec, ) -from debputy.util import _warn +from debputy.util import _warn, detect_possible_typo try: from debputy.lsp.vendoring._deb822_repro.locatable import ( @@ -58,34 +58,6 @@ except ImportError: pass -try: - from Levenshtein import distance -except ImportError: - - def _detect_possible_typo( - provided_value: str, - known_values: Iterable[str], - ) -> Sequence[str]: - return tuple() - -else: - - def _detect_possible_typo( - provided_value: str, - known_values: Iterable[str], - ) -> Sequence[str]: - k_len = len(provided_value) - candidates = [] - for known_value in known_values: - if abs(k_len - len(known_value)) > 2: - continue - d = distance(provided_value, known_value) - if d > 2: - continue - candidates.append(known_value) - return candidates - - _CONTAINS_TAB_OR_COLON = re.compile(r"[\t:]") _WORDS_RE = re.compile("([a-zA-Z0-9_-]+)") _MAKE_ERROR_RE = re.compile(r"^[^:]+:(\d+):\s*(\S.+)") @@ -314,7 +286,7 @@ def _lint_debian_rules_impl( missing_targets[target] = hook_location for target, (line_no, pos, endpos) in missing_targets.items(): - candidates = _detect_possible_typo(target, all_allowed_hook_targets) + candidates = detect_possible_typo(target, all_allowed_hook_targets) if not candidates and not target.startswith( ("override_", "execute_before_", "execute_after_") ): diff --git a/src/debputy/lsp/lsp_debian_tests_control.py b/src/debputy/lsp/lsp_debian_tests_control.py index 3562917..f188762 100644 --- a/src/debputy/lsp/lsp_debian_tests_control.py +++ b/src/debputy/lsp/lsp_debian_tests_control.py @@ -65,7 +65,6 @@ from debputy.lsp.spellchecking import default_spellchecker from debputy.lsp.text_util import ( normalize_dctrl_field_name, LintCapablePositionCodec, - detect_possible_typo, te_range_to_lsp, ) from debputy.lsp.vendoring._deb822_repro import ( @@ -76,6 +75,7 @@ from debputy.lsp.vendoring._deb822_repro.parsing import ( Deb822KeyValuePairElement, LIST_SPACE_SEPARATED_INTERPRETATION, ) +from debputy.util import detect_possible_typo try: from debputy.lsp.vendoring._deb822_repro.locatable import ( diff --git a/src/debputy/lsp/lsp_generic_deb822.py b/src/debputy/lsp/lsp_generic_deb822.py index d8b9596..895a3e0 100644 --- a/src/debputy/lsp/lsp_generic_deb822.py +++ b/src/debputy/lsp/lsp_generic_deb822.py @@ -272,6 +272,9 @@ def deb822_completer( ) items: Optional[Sequence[CompletionItem]] + markdown_kind = ls.completion_item_document_markup( + MarkupKind.Markdown, MarkupKind.PlainText + ) if in_value: _info(f"Completion for field value {current_field} -- {word_at_position}") if known_field is None: @@ -281,15 +284,16 @@ def deb822_completer( lint_state, list(matched_stanzas), value_being_completed, + markdown_kind, ) else: _info("Completing field name") assert stanza_metadata is not None items = _complete_field_name( - ls, lint_state, stanza_metadata, matched_stanzas, + markdown_kind, ) _info( @@ -638,15 +642,12 @@ def deb822_semantic_tokens_full( def _complete_field_name( - ls: "DebputyLanguageServer", lint_state: LintState, fields: StanzaMetadata[Any], matched_stanzas: Iterable[Deb822ParagraphElement], + markdown_kind: MarkupKind, ) -> Sequence[CompletionItem]: items = [] - markdown_kind = ls.completion_item_document_markup( - MarkupKind.Markdown, MarkupKind.PlainText - ) matched_stanzas = list(matched_stanzas) # TODO: Normalize fields according to file rules (X[BCS]- should be stripped in some files) seen_fields = set( diff --git a/src/debputy/lsp/lsp_reference_keyword.py b/src/debputy/lsp/lsp_reference_keyword.py index c71352a..4046aaa 100644 --- a/src/debputy/lsp/lsp_reference_keyword.py +++ b/src/debputy/lsp/lsp_reference_keyword.py @@ -8,10 +8,12 @@ from debputy.lsp.vendoring._deb822_repro import Deb822ParagraphElement @dataclasses.dataclass(slots=True, frozen=True) class Keyword: value: str + synopsis_doc: Optional[str] = None hover_text: Optional[str] = None is_obsolete: bool = False replaced_by: Optional[str] = None is_exclusive: bool = False + sort_text: Optional[str] = None can_complete_keyword_in_stanza: Optional[ Callable[[Iterable[Deb822ParagraphElement]], bool] ] = None diff --git a/src/debputy/lsp/lsp_self_check.py b/src/debputy/lsp/lsp_self_check.py index 3c7d2e4..aa28a56 100644 --- a/src/debputy/lsp/lsp_self_check.py +++ b/src/debputy/lsp/lsp_self_check.py @@ -1,7 +1,10 @@ import dataclasses import os.path +import subprocess from typing import Callable, Sequence, List, Optional, TypeVar +from debian.debian_support import Version + from debputy.util import _error @@ -24,7 +27,7 @@ def lsp_import_check( *, feature_name: Optional[str] = None, is_mandatory: bool = False, -): +) -> Callable[[C], C]: def _wrapper(func: C) -> C: @@ -51,6 +54,29 @@ def lsp_import_check( return _wrapper +def lsp_generic_check( + problem: str, + how_to_fix: str, + *, + feature_name: Optional[str] = None, + is_mandatory: bool = False, +) -> Callable[[C], C]: + + def _wrapper(func: C) -> C: + LSP_CHECKS.append( + LSPSelfCheck( + _feature_name(feature_name, func), + func, + problem, + how_to_fix, + is_mandatory=is_mandatory, + ) + ) + return func + + return _wrapper + + def _feature_name(feature: Optional[str], func: Callable[[], None]) -> str: if feature is not None: return feature @@ -83,6 +109,32 @@ def spell_checking() -> bool: ) +@lsp_generic_check( + feature_name="Extra dh support", + problem="Missing dependencies", + how_to_fix="Run `apt satisfy debhelper (>= 13.16~)` to enable this feature", +) +def check_dh_version() -> bool: + try: + output = subprocess.check_output( + [ + "dpkg-query", + "-W", + "--showformat=${Version} ${db:Status-Status}\n", + "debhelper", + ] + ).decode("utf-8") + except (FileNotFoundError, subprocess.CalledProcessError): + return False + else: + parts = output.split() + if len(parts) != 2: + return False + if parts[1] != "installed": + return False + return Version(parts[0]) >= Version("13.16~") + + def assert_can_start_lsp() -> None: for self_check in LSP_CHECKS: if self_check.is_mandatory and not self_check.test(): diff --git a/src/debputy/lsp/text_util.py b/src/debputy/lsp/text_util.py index 8228fef..4550001 100644 --- a/src/debputy/lsp/text_util.py +++ b/src/debputy/lsp/text_util.py @@ -31,34 +31,6 @@ else: LintCapablePositionCodec = LinterPositionCodec -try: - from Levenshtein import distance -except ImportError: - - def detect_possible_typo( - provided_value: str, - known_values: Iterable[str], - ) -> Sequence[str]: - return tuple() - -else: - - def detect_possible_typo( - provided_value: str, - known_values: Iterable[str], - ) -> Sequence[str]: - k_len = len(provided_value) - candidates = [] - for known_value in known_values: - if abs(k_len - len(known_value)) > 2: - continue - d = distance(provided_value, known_value) - if d > 2: - continue - candidates.append(known_value) - return candidates - - def normalize_dctrl_field_name(f: str) -> str: if not f or not f.startswith(("x", "X")): return f diff --git a/src/debputy/lsprotocol/types.py b/src/debputy/lsprotocol/types.py index a57c4e0..e32b932 100644 --- a/src/debputy/lsprotocol/types.py +++ b/src/debputy/lsprotocol/types.py @@ -24,10 +24,25 @@ else: except ImportError: - class StubModule: + stub_attr = { + "__name__": __name__, + "__file__": __file__, + "__doc__": __doc__, + } + bad_attr = frozenset( + [ + "pytestmark", + "pytest_plugins", + ] + ) + class StubModule: @staticmethod def __getattr__(item: Any) -> Any: + if item in stub_attr: + return stub_attr[item] + if item in bad_attr: + raise AttributeError(item) return types def __call__(self, *args, **kwargs) -> Any: diff --git a/src/debputy/packager_provided_files.py b/src/debputy/packager_provided_files.py index 6d74999..85fff11 100644 --- a/src/debputy/packager_provided_files.py +++ b/src/debputy/packager_provided_files.py @@ -1,11 +1,25 @@ import collections import dataclasses -from typing import Mapping, Iterable, Dict, List, Optional, Tuple +from typing import Mapping, Iterable, Dict, List, Optional, Tuple, Sequence from debputy.packages import BinaryPackage from debputy.plugin.api import VirtualPath from debputy.plugin.api.impl_types import PackagerProvidedFileClassSpec -from debputy.util import _error +from debputy.util import _error, CAN_DETECT_TYPOS, detect_possible_typo + + +_KNOWN_NON_PPFS = frozenset( + { + "gbp.conf", # Typo matches with `gbp.config` (dh_installdebconf) in two edits steps + # No reason to check any of these as they are never PPFs + "clean", + "control", + "compat", + "debputy.manifest", + "rules", + # NB: changelog and copyright are (de facto) ppfs, so they are deliberately omitted + } +) @dataclasses.dataclass(frozen=True, slots=True) @@ -17,6 +31,10 @@ class PackagerProvidedFile: definition: PackagerProvidedFileClassSpec match_priority: int = 0 fuzzy_match: bool = False + uses_explicit_package_name: bool = False + name_segment: Optional[str] = None + architecture_restriction: Optional[str] = None + expected_path: Optional[str] = None def compute_dest(self) -> Tuple[str, str]: return self.definition.compute_dest( @@ -71,21 +89,40 @@ def _find_package_name_prefix( def _find_definition( packager_provided_files: Mapping[str, PackagerProvidedFileClassSpec], basename: str, -) -> Tuple[Optional[str], Optional[PackagerProvidedFileClassSpec]]: + *, + period2stems: Optional[Mapping[int, Sequence[str]]] = None, +) -> Tuple[Optional[str], Optional[PackagerProvidedFileClassSpec], Optional[str]]: definition = packager_provided_files.get(basename) if definition is not None: - return None, definition + return None, definition, None + if period2stems and "." not in basename: + stems = period2stems.get(0) + matches = detect_possible_typo(basename, stems) if stems else None + if matches and len(matches) == 1: + definition = packager_provided_files[matches[0]] + return None, definition, basename install_as_name = basename file_class = "" + period_count = -1 while "." in install_as_name: + period_count += 1 install_as_name, file_class_part = install_as_name.rsplit(".", 1) file_class = ( file_class_part + "." + file_class if file_class != "" else file_class_part ) definition = packager_provided_files.get(file_class) if definition is not None: - return install_as_name, definition - return None, None + return install_as_name, definition, None + if not period2stems: + continue + stems = period2stems.get(period_count) + if not stems: + continue + matches = detect_possible_typo(file_class, stems) + if matches and len(matches) == 1: + definition = packager_provided_files[matches[0]] + return install_as_name, definition, file_class + return None, None, None def _check_mismatches( @@ -129,6 +166,7 @@ def _split_path( path: VirtualPath, *, allow_fuzzy_matches: bool = False, + period2stems: Optional[Mapping[int, Sequence[str]]] = None, ) -> Iterable[PackagerProvidedFile]: owning_package_name = main_binary_package basename = path.name @@ -148,6 +186,9 @@ def _split_path( definition=definition, match_priority=match_priority, fuzzy_match=False, + uses_explicit_package_name=False, + name_segment=None, + architecture_restriction=None, ) for n in binary_packages ) @@ -160,6 +201,9 @@ def _split_path( definition=definition, match_priority=match_priority, fuzzy_match=False, + uses_explicit_package_name=False, + name_segment=None, + architecture_restriction=None, ) return @@ -178,6 +222,7 @@ def _split_path( owning_package = binary_packages[owning_package_name] match_priority = 1 if explicit_package else 0 fuzzy_match = False + arch_restriction: Optional[str] = None if allow_fuzzy_matches and basename.endswith(".in") and len(basename) > 3: basename = basename[:-3] @@ -189,22 +234,24 @@ def _split_path( if last_word == owning_package.package_deb_architecture_variable("ARCH"): match_priority = 3 basename = remaining - had_arch = True + arch_restriction = last_word elif last_word == owning_package.package_deb_architecture_variable( "ARCH_OS" ): match_priority = 2 basename = remaining - had_arch = True + arch_restriction = last_word elif last_word == "all" and owning_package.is_arch_all: - # This case does not make sense, but we detect it so we can report an error + # This case does not make sense, but we detect it, so we can report an error # via _check_mismatches. match_priority = -1 basename = remaining - had_arch = True + arch_restriction = last_word - install_as_name, definition = _find_definition( - packager_provided_files, basename + install_as_name, definition, typoed_stem = _find_definition( + packager_provided_files, + basename, + period2stems=period2stems, ) if definition is None: continue @@ -218,14 +265,24 @@ def _split_path( definition, owning_package, install_as_name, - had_arch, + arch_restriction is not None, ) + + expected_path: Optional[str] = None if ( definition.packageless_is_fallback_for_all_packages and install_as_name is None and not had_arch and not explicit_package + and arch_restriction is None ): + if typoed_stem is not None: + parent_path = ( + path.parent_dir.path + "/" if path.parent_dir is not None else "" + ) + expected_path = f"{parent_path}{definition.stem}" + if fuzzy_match and path.name.endswith(".in"): + expected_path += ".in" yield from ( PackagerProvidedFile( path=path, @@ -235,6 +292,10 @@ def _split_path( definition=definition, match_priority=match_priority, fuzzy_match=fuzzy_match, + uses_explicit_package_name=False, + name_segment=None, + architecture_restriction=None, + expected_path=expected_path, ) for n in binary_packages ) @@ -248,6 +309,23 @@ def _split_path( if bug_950723: provided_key = f"{provided_key}@" basename = f"{basename}@" + package_prefix = f"{owning_package_name}@" + else: + package_prefix = owning_package_name + if typoed_stem: + parent_path = ( + path.parent_dir.path + "/" if path.parent_dir is not None else "" + ) + basename = definition.stem + if install_as_name is not None: + basename = f"{install_as_name}.{basename}" + if explicit_package: + basename = f"{package_prefix}.{basename}" + if arch_restriction is not None: + basename = f"{basename}.{arch_restriction}" + expected_path = f"{parent_path}{basename}" + if fuzzy_match and path.name.endswith(".in"): + expected_path += ".in" yield PackagerProvidedFile( path=path, package_name=owning_package_name, @@ -256,16 +334,34 @@ def _split_path( definition=definition, match_priority=match_priority, fuzzy_match=fuzzy_match, + uses_explicit_package_name=bool(explicit_package), + name_segment=install_as_name, + architecture_restriction=arch_restriction, + expected_path=expected_path, ) return +def _period_stem(stems: Iterable[str]) -> Mapping[int, Sequence[str]]: + result: Dict[int, List[str]] = {} + for stem in stems: + period_count = stem.count(".") + matched_stems = result.get(period_count) + if not matched_stems: + matched_stems = [stem] + result[period_count] = matched_stems + else: + matched_stems.append(stem) + return result + + def detect_all_packager_provided_files( packager_provided_files: Mapping[str, PackagerProvidedFileClassSpec], debian_dir: VirtualPath, binary_packages: Mapping[str, BinaryPackage], *, allow_fuzzy_matches: bool = False, + detect_typos: bool = False, ) -> Dict[str, PerPackagePackagerProvidedResult]: main_binary_package = [ p.name for p in binary_packages.values() if p.is_main_package @@ -274,10 +370,16 @@ def detect_all_packager_provided_files( n: {} for n in binary_packages } max_periods_in_package_name = max(name.count(".") for name in binary_packages) + if detect_typos and CAN_DETECT_TYPOS: + period2stems = _period_stem(packager_provided_files.keys()) + else: + period2stems = {} for entry in debian_dir.iterdir: if entry.is_dir: continue + if entry.name in _KNOWN_NON_PPFS: + continue matching_ppfs = _split_path( packager_provided_files, binary_packages, @@ -285,6 +387,7 @@ def detect_all_packager_provided_files( max_periods_in_package_name, entry, allow_fuzzy_matches=allow_fuzzy_matches, + period2stems=period2stems, ) for packager_provided_file in matching_ppfs: provided_files_for_package = provided_files[ diff --git a/src/debputy/plugin/api/impl_types.py b/src/debputy/plugin/api/impl_types.py index 99f589c..d62ccbc 100644 --- a/src/debputy/plugin/api/impl_types.py +++ b/src/debputy/plugin/api/impl_types.py @@ -188,6 +188,7 @@ class PackagerProvidedFileClassSpec: ) reference_documentation: Optional[PackagerProvidedFileReferenceDocumentation] = None bug_950723: bool = False + has_active_command: bool = True @property def supports_priority(self) -> bool: @@ -1155,6 +1156,7 @@ class KnownPackagingFileInfo(DebputyParsedContent): default_priority: NotRequired[int] post_formatting_rewrite: NotRequired[Literal["period-to-underscore"]] packageless_is_fallback_for_all_packages: NotRequired[bool] + has_active_command: NotRequired[bool] @dataclasses.dataclass(slots=True) diff --git a/src/debputy/util.py b/src/debputy/util.py index 01ffaa0..860280e 100644 --- a/src/debputy/util.py +++ b/src/debputy/util.py @@ -35,6 +35,39 @@ from debian.deb822 import Deb822 from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable from debputy.exceptions import DebputySubstitutionError + +try: + from Levenshtein import distance +except ImportError: + + CAN_DETECT_TYPOS = False + + def detect_possible_typo( + provided_value: str, + known_values: Iterable[str], + ) -> Sequence[str]: + return () + +else: + + CAN_DETECT_TYPOS = True + + def detect_possible_typo( + provided_value: str, + known_values: Iterable[str], + ) -> Sequence[str]: + k_len = len(provided_value) + candidates = [] + for known_value in known_values: + if abs(k_len - len(known_value)) > 2: + continue + d = distance(provided_value, known_value) + if d > 2: + continue + candidates.append(known_value) + return candidates + + if TYPE_CHECKING: from debputy.packages import BinaryPackage from debputy.substitution import Substitution |