summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/debputy/analysis/__init__.py9
-rw-r--r--src/debputy/analysis/analysis_util.py (renamed from src/debputy/commands/debputy_cmd/dc_util.py)0
-rw-r--r--src/debputy/analysis/debian_dir.py617
-rw-r--r--src/debputy/commands/debputy_cmd/__main__.py544
-rw-r--r--src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py4
-rw-r--r--src/debputy/commands/debputy_cmd/plugin_cmds.py14
-rw-r--r--src/debputy/debhelper_emulation.py5
-rw-r--r--src/debputy/dh_migration/migration.py2
-rw-r--r--src/debputy/dh_migration/migrators_impl.py11
-rw-r--r--src/debputy/linting/lint_util.py55
-rw-r--r--src/debputy/lsp/debputy_ls.py1
-rw-r--r--src/debputy/lsp/diagnostics.py1
-rw-r--r--src/debputy/lsp/lsp_debian_control.py155
-rw-r--r--src/debputy/lsp/lsp_debian_control_reference_data.py97
-rw-r--r--src/debputy/lsp/lsp_debian_copyright.py2
-rw-r--r--src/debputy/lsp/lsp_debian_debputy_manifest.py3
-rw-r--r--src/debputy/lsp/lsp_debian_rules.py32
-rw-r--r--src/debputy/lsp/lsp_debian_tests_control.py2
-rw-r--r--src/debputy/lsp/lsp_generic_deb822.py11
-rw-r--r--src/debputy/lsp/lsp_reference_keyword.py2
-rw-r--r--src/debputy/lsp/lsp_self_check.py54
-rw-r--r--src/debputy/lsp/text_util.py28
-rw-r--r--src/debputy/lsprotocol/types.py17
-rw-r--r--src/debputy/packager_provided_files.py129
-rw-r--r--src/debputy/plugin/api/impl_types.py2
-rw-r--r--src/debputy/util.py33
26 files changed, 1152 insertions, 678 deletions
diff --git a/src/debputy/analysis/__init__.py b/src/debputy/analysis/__init__.py
new file mode 100644
index 0000000..931c574
--- /dev/null
+++ b/src/debputy/analysis/__init__.py
@@ -0,0 +1,9 @@
+from debputy.plugin.api.impl_types import (
+ KNOWN_PACKAGING_FILE_CATEGORY_DESCRIPTIONS,
+ KNOWN_PACKAGING_FILE_CONFIG_FEATURE_DESCRIPTION,
+)
+
+REFERENCE_DATA_TABLE = {
+ "config-features": KNOWN_PACKAGING_FILE_CONFIG_FEATURE_DESCRIPTION,
+ "file-categories": KNOWN_PACKAGING_FILE_CATEGORY_DESCRIPTIONS,
+}
diff --git a/src/debputy/commands/debputy_cmd/dc_util.py b/src/debputy/analysis/analysis_util.py
index f54a4d1..f54a4d1 100644
--- a/src/debputy/commands/debputy_cmd/dc_util.py
+++ b/src/debputy/analysis/analysis_util.py
diff --git a/src/debputy/analysis/debian_dir.py b/src/debputy/analysis/debian_dir.py
new file mode 100644
index 0000000..7e5b37a
--- /dev/null
+++ b/src/debputy/analysis/debian_dir.py
@@ -0,0 +1,617 @@
+import json
+import os
+import stat
+import subprocess
+from typing import (
+ List,
+ Mapping,
+ Iterable,
+ Tuple,
+ Optional,
+ Sequence,
+ Dict,
+ Any,
+ Union,
+ Iterator,
+ TypedDict,
+ NotRequired,
+)
+
+from debputy.analysis import REFERENCE_DATA_TABLE
+from debputy.analysis.analysis_util import flatten_ppfs
+from debputy.dh_migration.migrators_impl import read_dh_addon_sequences
+from debputy.packager_provided_files import (
+ PackagerProvidedFile,
+ detect_all_packager_provided_files,
+)
+from debputy.packages import BinaryPackage
+from debputy.plugin.api import (
+ VirtualPath,
+ packager_provided_file_reference_documentation,
+)
+from debputy.plugin.api.feature_set import PluginProvidedFeatureSet
+from debputy.plugin.api.impl import plugin_metadata_for_debputys_own_plugin
+from debputy.plugin.api.impl_types import (
+ PluginProvidedKnownPackagingFile,
+ DebputyPluginMetadata,
+ KnownPackagingFileInfo,
+ InstallPatternDHCompatRule,
+ PackagerProvidedFileClassSpec,
+ expand_known_packaging_config_features,
+)
+from debputy.util import assume_not_none, escape_shell
+
+PackagingFileInfo = TypedDict(
+ "PackagingFileInfo",
+ {
+ "path": str,
+ "binary-package": NotRequired[str],
+ "install-path": NotRequired[str],
+ "install-pattern": NotRequired[str],
+ "file-categories": NotRequired[List[str]],
+ "config-features": NotRequired[List[str]],
+ "pkgfile-is-active-in-build": NotRequired[bool],
+ "pkgfile-stem": NotRequired[str],
+ "pkgfile-explicit-package-name": NotRequired[bool],
+ "pkgfile-name-segment": NotRequired[str],
+ "pkgfile-architecture-restriction": NotRequired[str],
+ "likely-typo-of": NotRequired[str],
+ "likely-generated-from": NotRequired[List[str]],
+ "related-tools": NotRequired[List[str]],
+ "documentation-uris": NotRequired[List[str]],
+ "debputy-cmd-templates": NotRequired[List[List[str]]],
+ "generates": NotRequired[str],
+ "generated-from": NotRequired[str],
+ },
+)
+
+
+def scan_debian_dir(
+ feature_set: PluginProvidedFeatureSet,
+ binary_packages: Mapping[str, BinaryPackage],
+ debian_dir: VirtualPath,
+) -> Tuple[List[PackagingFileInfo], List[str], int, Optional[object]]:
+ known_packaging_files = feature_set.known_packaging_files
+ debputy_plugin_metadata = plugin_metadata_for_debputys_own_plugin()
+
+ reference_data_set_names = [
+ "config-features",
+ "file-categories",
+ ]
+ for n in reference_data_set_names:
+ assert n in REFERENCE_DATA_TABLE
+
+ annotated: List[PackagingFileInfo] = []
+ seen_paths: Dict[str, PackagingFileInfo] = {}
+
+ r = read_dh_addon_sequences(debian_dir)
+ if r is not None:
+ bd_sequences, dr_sequences, saw_dh = r
+ drules_sequences = bd_sequences | dr_sequences
+ else:
+ drules_sequences = set()
+ saw_dh = False
+ is_debputy_package = (
+ "debputy" in drules_sequences
+ or "zz-debputy" in drules_sequences
+ or "zz_debputy" in drules_sequences
+ or "zz-debputy-rrr" in drules_sequences
+ )
+ dh_compat_level, dh_assistant_exit_code = _extract_dh_compat_level()
+ dh_issues = []
+
+ static_packaging_files = {
+ kpf.detection_value: kpf
+ for kpf in known_packaging_files.values()
+ if kpf.detection_method == "path"
+ }
+ dh_pkgfile_docs = {
+ kpf.detection_value: kpf
+ for kpf in known_packaging_files.values()
+ if kpf.detection_method == "dh.pkgfile"
+ }
+
+ if is_debputy_package:
+ all_debputy_ppfs = list(
+ flatten_ppfs(
+ detect_all_packager_provided_files(
+ feature_set.packager_provided_files,
+ debian_dir,
+ binary_packages,
+ allow_fuzzy_matches=True,
+ detect_typos=True,
+ )
+ )
+ )
+ else:
+ all_debputy_ppfs = []
+
+ if dh_compat_level is not None:
+ (
+ all_dh_ppfs,
+ dh_issues,
+ dh_assistant_exit_code,
+ ) = _resolve_debhelper_config_files(
+ debian_dir,
+ binary_packages,
+ debputy_plugin_metadata,
+ dh_pkgfile_docs,
+ drules_sequences,
+ dh_compat_level,
+ saw_dh,
+ )
+
+ else:
+ all_dh_ppfs = []
+
+ for ppf in all_debputy_ppfs:
+ key = ppf.path.path
+ ref_doc = ppf.definition.reference_documentation
+ documentation_uris = (
+ ref_doc.format_documentation_uris if ref_doc is not None else None
+ )
+ details: PackagingFileInfo = {
+ "path": key,
+ "pkgfile-stem": ppf.definition.stem,
+ "pkgfile-explicit-package-name": ppf.uses_explicit_package_name,
+ "pkgfile-is-active-in-build": ppf.definition.has_active_command,
+ "debputy-cmd-templates": [
+ ["debputy", "plugin", "show", "p-p-f", ppf.definition.stem]
+ ],
+ }
+ if ppf.fuzzy_match and key.endswith(".in"):
+ _merge_list(details, "file-categories", ["generic-template"])
+ details["generates"] = key[:-3]
+ elif assume_not_none(ppf.path.parent_dir).get(ppf.path.name + ".in"):
+ _merge_list(details, "file-categories", ["generated"])
+ details["generated-from"] = key + ".in"
+ name_segment = ppf.name_segment
+ arch_restriction = ppf.architecture_restriction
+ if name_segment is not None:
+ details["pkgfile-name-segment"] = name_segment
+ if arch_restriction:
+ details["pkgfile-architecture-restriction"] = arch_restriction
+ seen_paths[key] = details
+ annotated.append(details)
+ static_details = static_packaging_files.get(key)
+ if static_details is not None:
+ # debhelper compat rules does not apply to debputy files
+ _add_known_packaging_data(details, static_details, None)
+ if documentation_uris:
+ details["documentation-uris"] = list(documentation_uris)
+
+ _merge_ppfs(annotated, seen_paths, all_dh_ppfs, dh_pkgfile_docs, dh_compat_level)
+
+ for virtual_path in _scan_debian_dir(debian_dir):
+ key = virtual_path.path
+ if key in seen_paths:
+ continue
+ if virtual_path.is_symlink:
+ try:
+ st = os.stat(virtual_path.fs_path)
+ except FileNotFoundError:
+ continue
+ else:
+ if not stat.S_ISREG(st.st_mode):
+ continue
+ elif not virtual_path.is_file:
+ continue
+
+ static_match = static_packaging_files.get(virtual_path.path)
+ if static_match is not None:
+ details: PackagingFileInfo = {
+ "path": key,
+ }
+ annotated.append(details)
+ if assume_not_none(virtual_path.parent_dir).get(virtual_path.name + ".in"):
+ details["generated-from"] = key + ".in"
+ _merge_list(details, "file-categories", ["generated"])
+ _add_known_packaging_data(details, static_match, dh_compat_level)
+
+ return annotated, reference_data_set_names, dh_assistant_exit_code, dh_issues
+
+
+def _fake_PPFClassSpec(
+ debputy_plugin_metadata: DebputyPluginMetadata,
+ stem: str,
+ doc_uris: Optional[Sequence[str]],
+ install_pattern: Optional[str],
+ *,
+ default_priority: Optional[int] = None,
+ packageless_is_fallback_for_all_packages: bool = False,
+ post_formatting_rewrite: Optional[str] = None,
+ bug_950723: bool = False,
+ has_active_command: bool = False,
+) -> PackagerProvidedFileClassSpec:
+ if install_pattern is None:
+ install_pattern = "not-a-real-ppf"
+ if post_formatting_rewrite is not None:
+ formatting_hook = _POST_FORMATTING_REWRITE[post_formatting_rewrite]
+ else:
+ formatting_hook = None
+ return PackagerProvidedFileClassSpec(
+ debputy_plugin_metadata,
+ stem,
+ install_pattern,
+ allow_architecture_segment=True,
+ allow_name_segment=True,
+ default_priority=default_priority,
+ default_mode=0o644,
+ post_formatting_rewrite=formatting_hook,
+ packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages,
+ reservation_only=False,
+ formatting_callback=None,
+ bug_950723=bug_950723,
+ has_active_command=has_active_command,
+ reference_documentation=packager_provided_file_reference_documentation(
+ format_documentation_uris=doc_uris,
+ ),
+ )
+
+
+def _relevant_dh_compat_rules(
+ compat_level: Optional[int],
+ info: KnownPackagingFileInfo,
+) -> Iterable[InstallPatternDHCompatRule]:
+ if compat_level is None:
+ return
+ dh_compat_rules = info.get("dh_compat_rules")
+ if not dh_compat_rules:
+ return
+ for dh_compat_rule in dh_compat_rules:
+ rule_compat_level = dh_compat_rule.get("starting_with_compat_level")
+ if rule_compat_level is not None and compat_level < rule_compat_level:
+ continue
+ yield dh_compat_rule
+
+
+def _kpf_install_pattern(
+ compat_level: Optional[int],
+ ppkpf: PluginProvidedKnownPackagingFile,
+) -> Optional[str]:
+ for compat_rule in _relevant_dh_compat_rules(compat_level, ppkpf.info):
+ install_pattern = compat_rule.get("install_pattern")
+ if install_pattern is not None:
+ return install_pattern
+ return ppkpf.info.get("install_pattern")
+
+
+def _resolve_debhelper_config_files(
+ debian_dir: VirtualPath,
+ binary_packages: Mapping[str, BinaryPackage],
+ debputy_plugin_metadata: DebputyPluginMetadata,
+ dh_ppf_docs: Dict[str, PluginProvidedKnownPackagingFile],
+ dh_rules_addons: Iterable[str],
+ dh_compat_level: int,
+ saw_dh: bool,
+) -> Tuple[List[PackagerProvidedFile], Optional[object], int]:
+ dh_ppfs = {}
+ commands, exit_code = _relevant_dh_commands(dh_rules_addons)
+
+ cmd = ["dh_assistant", "list-guessed-dh-config-files"]
+ if dh_rules_addons:
+ addons = ",".join(dh_rules_addons)
+ cmd.append(f"--with={addons}")
+ try:
+ output = subprocess.check_output(
+ cmd,
+ stderr=subprocess.DEVNULL,
+ )
+ except (subprocess.CalledProcessError, FileNotFoundError) as e:
+ config_files = []
+ issues = None
+ if isinstance(e, subprocess.CalledProcessError):
+ exit_code = e.returncode
+ else:
+ exit_code = 127
+ else:
+ result = json.loads(output)
+ config_files: List[Union[Mapping[str, Any], object]] = result.get(
+ "config-files", []
+ )
+ issues = result.get("issues")
+ for config_file in config_files:
+ if not isinstance(config_file, dict):
+ continue
+ if config_file.get("file-type") != "pkgfile":
+ continue
+ stem = config_file.get("pkgfile")
+ if stem is None:
+ continue
+ internal = config_file.get("internal")
+ if isinstance(internal, dict):
+ bug_950723 = internal.get("bug#950723", False) is True
+ else:
+ bug_950723 = False
+ commands = config_file.get("commands")
+ documentation_uris = []
+ related_tools = []
+ seen_commands = set()
+ seen_docs = set()
+ ppkpf = dh_ppf_docs.get(stem)
+
+ if ppkpf:
+ dh_cmds = ppkpf.info.get("debhelper_commands")
+ doc_uris = ppkpf.info.get("documentation_uris")
+ default_priority = ppkpf.info.get("default_priority")
+ if doc_uris is not None:
+ seen_docs.update(doc_uris)
+ documentation_uris.extend(doc_uris)
+ if dh_cmds is not None:
+ seen_commands.update(dh_cmds)
+ related_tools.extend(dh_cmds)
+ install_pattern = _kpf_install_pattern(dh_compat_level, ppkpf)
+ post_formatting_rewrite = ppkpf.info.get("post_formatting_rewrite")
+ packageless_is_fallback_for_all_packages = ppkpf.info.get(
+ "packageless_is_fallback_for_all_packages",
+ False,
+ )
+ # If it is a debhelper PPF, then `has_active_command` is false by default.
+ has_active_command = ppkpf.info.get("has_active_command", False)
+ else:
+ install_pattern = None
+ default_priority = None
+ post_formatting_rewrite = None
+ packageless_is_fallback_for_all_packages = False
+ has_active_command = False
+ for command in commands:
+ if isinstance(command, dict):
+ command_name = command.get("command")
+ if isinstance(command_name, str) and command_name:
+ if command_name not in seen_commands:
+ related_tools.append(command_name)
+ seen_commands.add(command_name)
+ manpage = f"man:{command_name}(1)"
+ if manpage not in seen_docs:
+ documentation_uris.append(manpage)
+ seen_docs.add(manpage)
+ else:
+ continue
+ is_active = command.get("is-active", True)
+ if not isinstance(is_active, bool):
+ continue
+ if is_active:
+ has_active_command = True
+ dh_ppfs[stem] = _fake_PPFClassSpec(
+ debputy_plugin_metadata,
+ stem,
+ documentation_uris,
+ install_pattern,
+ default_priority=default_priority,
+ post_formatting_rewrite=post_formatting_rewrite,
+ packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages,
+ bug_950723=bug_950723,
+ has_active_command=has_active_command if saw_dh else True,
+ )
+ for ppkpf in dh_ppf_docs.values():
+ stem = ppkpf.detection_value
+ if stem in dh_ppfs:
+ continue
+
+ default_priority = ppkpf.info.get("default_priority")
+ install_pattern = _kpf_install_pattern(dh_compat_level, ppkpf)
+ post_formatting_rewrite = ppkpf.info.get("post_formatting_rewrite")
+ packageless_is_fallback_for_all_packages = ppkpf.info.get(
+ "packageless_is_fallback_for_all_packages",
+ False,
+ )
+ dh_ppfs[stem] = _fake_PPFClassSpec(
+ debputy_plugin_metadata,
+ stem,
+ ppkpf.info.get("documentation_uris"),
+ install_pattern,
+ default_priority=default_priority,
+ post_formatting_rewrite=post_formatting_rewrite,
+ packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages,
+ has_active_command=(
+ ppkpf.info.get("has_active_command", False) if saw_dh else False
+ ),
+ )
+ all_dh_ppfs = list(
+ flatten_ppfs(
+ detect_all_packager_provided_files(
+ dh_ppfs,
+ debian_dir,
+ binary_packages,
+ allow_fuzzy_matches=True,
+ detect_typos=True,
+ )
+ )
+ )
+ return all_dh_ppfs, issues, exit_code
+
+
+def _merge_list(
+ existing_table: Dict[str, Any],
+ key: str,
+ new_data: Optional[Sequence[str]],
+) -> None:
+ if not new_data:
+ return
+ existing_values = existing_table.get(key, [])
+ if isinstance(existing_values, tuple):
+ existing_values = list(existing_values)
+ assert isinstance(existing_values, list)
+ seen = set(existing_values)
+ existing_values.extend(x for x in new_data if x not in seen)
+ existing_table[key] = existing_values
+
+
+def _merge_ppfs(
+ identified: List[PackagingFileInfo],
+ seen_paths: Dict[str, PackagingFileInfo],
+ ppfs: List[PackagerProvidedFile],
+ context: Mapping[str, PluginProvidedKnownPackagingFile],
+ dh_compat_level: Optional[int],
+) -> None:
+ for ppf in ppfs:
+ key = ppf.path.path
+ ref_doc = ppf.definition.reference_documentation
+ documentation_uris = (
+ ref_doc.format_documentation_uris if ref_doc is not None else None
+ )
+ if not ppf.definition.installed_as_format.startswith("not-a-real-ppf"):
+ try:
+ parts = ppf.compute_dest()
+ except RuntimeError:
+ dest = None
+ else:
+ dest = "/".join(parts).lstrip(".")
+ else:
+ dest = None
+ orig_details = seen_paths.get(key)
+ if orig_details is None:
+ details: PackagingFileInfo = {
+ "path": key,
+ "pkgfile-stem": ppf.definition.stem,
+ "pkgfile-is-active-in-build": ppf.definition.has_active_command,
+ "pkgfile-explicit-package-name": ppf.uses_explicit_package_name,
+ "binary-package": ppf.package_name,
+ }
+ if ppf.expected_path is not None:
+ details["likely-typo-of"] = ppf.expected_path
+ identified.append(details)
+ else:
+ details = orig_details
+ # We do not merge the "is typo" field; if the original
+ for k, v in [
+ ("pkgfile-stem", ppf.definition.stem),
+ ("pkgfile-explicit-package-name", ppf.definition.has_active_command),
+ ("binary-package", ppf.package_name),
+ ]:
+ if k not in details:
+ details[k] = v
+ if ppf.definition.has_active_command and details.get(
+ "pkgfile-is-active-in-build", False
+ ):
+ details["pkgfile-is-active-in-build"] = True
+ if ppf.expected_path is None and "likely-typo-of" in details:
+ del details["likely-typo-of"]
+
+ name_segment = ppf.name_segment
+ arch_restriction = ppf.architecture_restriction
+ if name_segment is not None and "pkgfile-name-segment" not in details:
+ details["pkgfile-name-segment"] = name_segment
+ if (
+ arch_restriction is not None
+ and "pkgfile-architecture-restriction" not in details
+ ):
+ details["pkgfile-architecture-restriction"] = arch_restriction
+ if ppf.fuzzy_match and key.endswith(".in"):
+ _merge_list(details, "file-categories", ["generic-template"])
+ details["generates"] = key[:-3]
+ elif assume_not_none(ppf.path.parent_dir).get(ppf.path.name + ".in"):
+ _merge_list(details, "file-categories", ["generated"])
+ details["generated-from"] = key + ".in"
+ if dest is not None and "install-path" not in details:
+ details["install-path"] = dest
+
+ extra_details = context.get(ppf.definition.stem)
+ if extra_details is not None:
+ _add_known_packaging_data(details, extra_details, dh_compat_level)
+
+ _merge_list(details, "documentation-uris", documentation_uris)
+
+
+def _extract_dh_compat_level() -> Tuple[Optional[int], int]:
+ try:
+ output = subprocess.check_output(
+ ["dh_assistant", "active-compat-level"],
+ stderr=subprocess.DEVNULL,
+ )
+ except (FileNotFoundError, subprocess.CalledProcessError) as e:
+ exit_code = 127
+ if isinstance(e, subprocess.CalledProcessError):
+ exit_code = e.returncode
+ return None, exit_code
+ else:
+ data = json.loads(output)
+ active_compat_level = data.get("active-compat-level")
+ exit_code = 0
+ if not isinstance(active_compat_level, int) or active_compat_level < 1:
+ active_compat_level = None
+ exit_code = 255
+ return active_compat_level, exit_code
+
+
+def _relevant_dh_commands(dh_rules_addons: Iterable[str]) -> Tuple[List[str], int]:
+ cmd = ["dh_assistant", "list-commands", "--output-format=json"]
+ if dh_rules_addons:
+ addons = ",".join(dh_rules_addons)
+ cmd.append(f"--with={addons}")
+ try:
+ output = subprocess.check_output(
+ cmd,
+ stderr=subprocess.DEVNULL,
+ )
+ except (FileNotFoundError, subprocess.CalledProcessError) as e:
+ exit_code = 127
+ if isinstance(e, subprocess.CalledProcessError):
+ exit_code = e.returncode
+ return [], exit_code
+ else:
+ data = json.loads(output)
+ commands_json = data.get("commands")
+ commands = []
+ for command in commands_json:
+ if isinstance(command, dict):
+ command_name = command.get("command")
+ if isinstance(command_name, str) and command_name:
+ commands.append(command_name)
+ return commands, 0
+
+
+def _add_known_packaging_data(
+ details: PackagingFileInfo,
+ plugin_data: PluginProvidedKnownPackagingFile,
+ dh_compat_level: Optional[int],
+):
+ install_pattern = _kpf_install_pattern(
+ dh_compat_level,
+ plugin_data,
+ )
+ config_features = plugin_data.info.get("config_features")
+ if config_features:
+ config_features = expand_known_packaging_config_features(
+ dh_compat_level or 0,
+ config_features,
+ )
+ _merge_list(details, "config-features", config_features)
+
+ if dh_compat_level is not None:
+ extra_config_features = []
+ for dh_compat_rule in _relevant_dh_compat_rules(
+ dh_compat_level, plugin_data.info
+ ):
+ cf = dh_compat_rule.get("add_config_features")
+ if cf:
+ extra_config_features.extend(cf)
+ if extra_config_features:
+ extra_config_features = expand_known_packaging_config_features(
+ dh_compat_level,
+ extra_config_features,
+ )
+ _merge_list(details, "config-features", extra_config_features)
+ if "install-pattern" not in details and install_pattern is not None:
+ details["install-pattern"] = install_pattern
+ for mk, ok in [
+ ("file_categories", "file-categories"),
+ ("documentation_uris", "documentation-uris"),
+ ("debputy_cmd_templates", "debputy-cmd-templates"),
+ ]:
+ value = plugin_data.info.get(mk)
+ if value and ok == "debputy-cmd-templates":
+ value = [escape_shell(*c) for c in value]
+ _merge_list(details, ok, value)
+
+
+def _scan_debian_dir(debian_dir: VirtualPath) -> Iterator[VirtualPath]:
+ for p in debian_dir.iterdir:
+ yield p
+ if p.is_dir and p.path in ("debian/source", "debian/tests"):
+ yield from p.iterdir
+
+
+_POST_FORMATTING_REWRITE = {
+ "period-to-underscore": lambda n: n.replace(".", "_"),
+}
diff --git a/src/debputy/commands/debputy_cmd/__main__.py b/src/debputy/commands/debputy_cmd/__main__.py
index 37f89cd..218ca4f 100644
--- a/src/debputy/commands/debputy_cmd/__main__.py
+++ b/src/debputy/commands/debputy_cmd/__main__.py
@@ -4,7 +4,6 @@ import json
import logging
import os
import shutil
-import stat
import subprocess
import sys
import textwrap
@@ -13,34 +12,26 @@ from tempfile import TemporaryDirectory
from typing import (
List,
Dict,
- Iterable,
Any,
Tuple,
- Sequence,
Optional,
NoReturn,
- Mapping,
- Union,
NamedTuple,
Literal,
- Set,
- Iterator,
- TypedDict,
- NotRequired,
cast,
)
from debputy import DEBPUTY_ROOT_DIR, DEBPUTY_PLUGIN_ROOT_DIR
+from debputy.analysis import REFERENCE_DATA_TABLE
+from debputy.analysis.debian_dir import scan_debian_dir
from debputy.commands.debputy_cmd.context import (
CommandContext,
add_arg,
ROOT_COMMAND,
CommandArg,
)
-from debputy.commands.debputy_cmd.dc_util import flatten_ppfs
from debputy.commands.debputy_cmd.output import _stream_to_pager
from debputy.dh_migration.migrators import MIGRATORS
-from debputy.dh_migration.migrators_impl import read_dh_addon_sequences
from debputy.exceptions import (
DebputyRuntimeError,
PluginNotFoundError,
@@ -52,14 +43,6 @@ from debputy.exceptions import (
from debputy.package_build.assemble_deb import (
assemble_debs,
)
-from debputy.packager_provided_files import (
- detect_all_packager_provided_files,
- PackagerProvidedFile,
-)
-from debputy.plugin.api.spec import (
- VirtualPath,
- packager_provided_file_reference_documentation,
-)
try:
from argcomplete import autocomplete
@@ -75,25 +58,16 @@ from debputy.filesystem_scan import (
FSRootDir,
)
from debputy.plugin.api.impl_types import (
- PackagerProvidedFileClassSpec,
DebputyPluginMetadata,
- PluginProvidedKnownPackagingFile,
- KNOWN_PACKAGING_FILE_CATEGORY_DESCRIPTIONS,
- KNOWN_PACKAGING_FILE_CONFIG_FEATURE_DESCRIPTION,
- expand_known_packaging_config_features,
- InstallPatternDHCompatRule,
- KnownPackagingFileInfo,
)
from debputy.plugin.api.impl import (
find_json_plugin,
find_tests_for_plugin,
find_related_implementation_files_for_plugin,
parse_json_plugin_desc,
- plugin_metadata_for_debputys_own_plugin,
)
from debputy.dh_migration.migration import migrate_from_dh
from debputy.dh_migration.models import AcceptableMigrationIssues
-from debputy.packages import BinaryPackage
from debputy.debhelper_emulation import (
dhe_pkgdir,
)
@@ -117,14 +91,8 @@ from debputy.util import (
escape_shell,
program_name,
integrated_with_debhelper,
- assume_not_none,
)
-REFERENCE_DATA_TABLE = {
- "config-features": KNOWN_PACKAGING_FILE_CONFIG_FEATURE_DESCRIPTION,
- "file-categories": KNOWN_PACKAGING_FILE_CATEGORY_DESCRIPTIONS,
-}
-
class SharedArgument(NamedTuple):
"""
@@ -773,341 +741,6 @@ def _dh_integration_generate_debs(context: CommandContext) -> None:
)
-PackagingFileInfo = TypedDict(
- "PackagingFileInfo",
- {
- "path": str,
- "binary-package": NotRequired[str],
- "install-path": NotRequired[str],
- "install-pattern": NotRequired[str],
- "file-categories": NotRequired[List[str]],
- "config-features": NotRequired[List[str]],
- "likely-generated-from": NotRequired[List[str]],
- "related-tools": NotRequired[List[str]],
- "documentation-uris": NotRequired[List[str]],
- "debputy-cmd-templates": NotRequired[List[List[str]]],
- "generates": NotRequired[str],
- "generated-from": NotRequired[str],
- },
-)
-
-
-def _scan_debian_dir(debian_dir: VirtualPath) -> Iterator[VirtualPath]:
- for p in debian_dir.iterdir:
- yield p
- if p.is_dir and p.path in ("debian/source", "debian/tests"):
- yield from p.iterdir
-
-
-_POST_FORMATTING_REWRITE = {
- "period-to-underscore": lambda n: n.replace(".", "_"),
-}
-
-
-def _fake_PPFClassSpec(
- debputy_plugin_metadata: DebputyPluginMetadata,
- stem: str,
- doc_uris: Optional[Sequence[str]],
- install_pattern: Optional[str],
- *,
- default_priority: Optional[int] = None,
- packageless_is_fallback_for_all_packages: bool = False,
- post_formatting_rewrite: Optional[str] = None,
- bug_950723: bool = False,
-) -> PackagerProvidedFileClassSpec:
- if install_pattern is None:
- install_pattern = "not-a-real-ppf"
- if post_formatting_rewrite is not None:
- formatting_hook = _POST_FORMATTING_REWRITE[post_formatting_rewrite]
- else:
- formatting_hook = None
- return PackagerProvidedFileClassSpec(
- debputy_plugin_metadata,
- stem,
- install_pattern,
- allow_architecture_segment=True,
- allow_name_segment=True,
- default_priority=default_priority,
- default_mode=0o644,
- post_formatting_rewrite=formatting_hook,
- packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages,
- reservation_only=False,
- formatting_callback=None,
- bug_950723=bug_950723,
- reference_documentation=packager_provided_file_reference_documentation(
- format_documentation_uris=doc_uris,
- ),
- )
-
-
-def _relevant_dh_compat_rules(
- compat_level: Optional[int],
- info: KnownPackagingFileInfo,
-) -> Iterable[InstallPatternDHCompatRule]:
- if compat_level is None:
- return
- dh_compat_rules = info.get("dh_compat_rules")
- if not dh_compat_rules:
- return
- for dh_compat_rule in dh_compat_rules:
- rule_compat_level = dh_compat_rule.get("starting_with_compat_level")
- if rule_compat_level is not None and compat_level < rule_compat_level:
- continue
- yield dh_compat_rule
-
-
-def _kpf_install_pattern(
- compat_level: Optional[int],
- ppkpf: PluginProvidedKnownPackagingFile,
-) -> Optional[str]:
- for compat_rule in _relevant_dh_compat_rules(compat_level, ppkpf.info):
- install_pattern = compat_rule.get("install_pattern")
- if install_pattern is not None:
- return install_pattern
- return ppkpf.info.get("install_pattern")
-
-
-def _resolve_debhelper_config_files(
- debian_dir: VirtualPath,
- binary_packages: Mapping[str, BinaryPackage],
- debputy_plugin_metadata: DebputyPluginMetadata,
- dh_ppf_docs: Dict[str, PluginProvidedKnownPackagingFile],
- dh_rules_addons: Iterable[str],
- dh_compat_level: int,
-) -> Tuple[List[PackagerProvidedFile], Optional[object], int]:
- dh_ppfs = {}
- commands, exit_code = _relevant_dh_commands(dh_rules_addons)
- dh_commands = set(commands)
-
- cmd = ["dh_assistant", "list-guessed-dh-config-files"]
- if dh_rules_addons:
- addons = ",".join(dh_rules_addons)
- cmd.append(f"--with={addons}")
- try:
- output = subprocess.check_output(
- cmd,
- stderr=subprocess.DEVNULL,
- )
- except (subprocess.CalledProcessError, FileNotFoundError) as e:
- config_files = []
- issues = None
- if isinstance(e, subprocess.CalledProcessError):
- exit_code = e.returncode
- else:
- exit_code = 127
- else:
- result = json.loads(output)
- config_files: List[Union[Mapping[str, Any], object]] = result.get(
- "config-files", []
- )
- issues = result.get("issues")
- for config_file in config_files:
- if not isinstance(config_file, dict):
- continue
- if config_file.get("file-type") != "pkgfile":
- continue
- stem = config_file.get("pkgfile")
- if stem is None:
- continue
- internal = config_file.get("internal")
- if isinstance(internal, dict):
- bug_950723 = internal.get("bug#950723", False) is True
- else:
- bug_950723 = False
- commands = config_file.get("commands")
- documentation_uris = []
- related_tools = []
- seen_commands = set()
- seen_docs = set()
- ppkpf = dh_ppf_docs.get(stem)
- if ppkpf:
- dh_cmds = ppkpf.info.get("debhelper_commands")
- doc_uris = ppkpf.info.get("documentation_uris")
- default_priority = ppkpf.info.get("default_priority")
- if doc_uris is not None:
- seen_docs.update(doc_uris)
- documentation_uris.extend(doc_uris)
- if dh_cmds is not None:
- seen_commands.update(dh_cmds)
- related_tools.extend(dh_cmds)
- install_pattern = _kpf_install_pattern(dh_compat_level, ppkpf)
- post_formatting_rewrite = ppkpf.info.get("post_formatting_rewrite")
- packageless_is_fallback_for_all_packages = ppkpf.info.get(
- "packageless_is_fallback_for_all_packages",
- False,
- )
- else:
- install_pattern = None
- default_priority = None
- post_formatting_rewrite = None
- packageless_is_fallback_for_all_packages = False
- for command in commands:
- if isinstance(command, dict):
- command_name = command.get("command")
- if isinstance(command_name, str) and command_name:
- if command_name not in seen_commands:
- related_tools.append(command_name)
- seen_commands.add(command_name)
- manpage = f"man:{command_name}(1)"
- if manpage not in seen_docs:
- documentation_uris.append(manpage)
- seen_docs.add(manpage)
- dh_ppfs[stem] = _fake_PPFClassSpec(
- debputy_plugin_metadata,
- stem,
- documentation_uris,
- install_pattern,
- default_priority=default_priority,
- post_formatting_rewrite=post_formatting_rewrite,
- packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages,
- bug_950723=bug_950723,
- )
- for ppkpf in dh_ppf_docs.values():
- stem = ppkpf.detection_value
- if stem in dh_ppfs:
- continue
-
- default_priority = ppkpf.info.get("default_priority")
- commands = ppkpf.info.get("debhelper_commands")
- install_pattern = _kpf_install_pattern(dh_compat_level, ppkpf)
- post_formatting_rewrite = ppkpf.info.get("post_formatting_rewrite")
- packageless_is_fallback_for_all_packages = ppkpf.info.get(
- "packageless_is_fallback_for_all_packages",
- False,
- )
- if commands and not any(c in dh_commands for c in commands):
- continue
- dh_ppfs[stem] = _fake_PPFClassSpec(
- debputy_plugin_metadata,
- stem,
- ppkpf.info.get("documentation_uris"),
- install_pattern,
- default_priority=default_priority,
- post_formatting_rewrite=post_formatting_rewrite,
- packageless_is_fallback_for_all_packages=packageless_is_fallback_for_all_packages,
- )
- all_dh_ppfs = list(
- flatten_ppfs(
- detect_all_packager_provided_files(
- dh_ppfs,
- debian_dir,
- binary_packages,
- allow_fuzzy_matches=True,
- )
- )
- )
- return all_dh_ppfs, issues, exit_code
-
-
-def _merge_list(
- existing_table: Dict[str, Any],
- key: str,
- new_data: Optional[Sequence[str]],
-) -> None:
- if not new_data:
- return
- existing_values = existing_table.get(key, [])
- if isinstance(existing_values, tuple):
- existing_values = list(existing_values)
- assert isinstance(existing_values, list)
- seen = set(existing_values)
- existing_values.extend(x for x in new_data if x not in seen)
- existing_table[key] = existing_values
-
-
-def _merge_ppfs(
- identified: List[PackagingFileInfo],
- seen_paths: Set[str],
- ppfs: List[PackagerProvidedFile],
- context: Mapping[str, PluginProvidedKnownPackagingFile],
- dh_compat_level: Optional[int],
-) -> None:
- for ppf in ppfs:
- key = ppf.path.path
- ref_doc = ppf.definition.reference_documentation
- documentation_uris = (
- ref_doc.format_documentation_uris if ref_doc is not None else None
- )
-
- if not ppf.definition.installed_as_format.startswith("not-a-real-ppf"):
- try:
- parts = ppf.compute_dest()
- except RuntimeError:
- dest = None
- else:
- dest = "/".join(parts).lstrip(".")
- else:
- dest = None
- seen_paths.add(key)
- details: PackagingFileInfo = {
- "path": key,
- "binary-package": ppf.package_name,
- }
- if ppf.fuzzy_match and key.endswith(".in"):
- _merge_list(details, "file-categories", ["generic-template"])
- details["generates"] = key[:-3]
- elif assume_not_none(ppf.path.parent_dir).get(ppf.path.name + ".in"):
- _merge_list(details, "file-categories", ["generated"])
- details["generated-from"] = key + ".in"
- if dest is not None:
- details["install-path"] = dest
- identified.append(details)
-
- extra_details = context.get(ppf.definition.stem)
- if extra_details is not None:
- _add_known_packaging_data(details, extra_details, dh_compat_level)
-
- _merge_list(details, "documentation-uris", documentation_uris)
-
-
-def _extract_dh_compat_level() -> Tuple[Optional[int], int]:
- try:
- output = subprocess.check_output(
- ["dh_assistant", "active-compat-level"],
- stderr=subprocess.DEVNULL,
- )
- except (FileNotFoundError, subprocess.CalledProcessError) as e:
- exit_code = 127
- if isinstance(e, subprocess.CalledProcessError):
- exit_code = e.returncode
- return None, exit_code
- else:
- data = json.loads(output)
- active_compat_level = data.get("active-compat-level")
- exit_code = 0
- if not isinstance(active_compat_level, int) or active_compat_level < 1:
- active_compat_level = None
- exit_code = 255
- return active_compat_level, exit_code
-
-
-def _relevant_dh_commands(dh_rules_addons: Iterable[str]) -> Tuple[List[str], int]:
- cmd = ["dh_assistant", "list-commands", "--output-format=json"]
- if dh_rules_addons:
- addons = ",".join(dh_rules_addons)
- cmd.append(f"--with={addons}")
- try:
- output = subprocess.check_output(
- cmd,
- stderr=subprocess.DEVNULL,
- )
- except (FileNotFoundError, subprocess.CalledProcessError) as e:
- exit_code = 127
- if isinstance(e, subprocess.CalledProcessError):
- exit_code = e.returncode
- return [], exit_code
- else:
- data = json.loads(output)
- commands_json = data.get("commands")
- commands = []
- for command in commands_json:
- if isinstance(command, dict):
- command_name = command.get("command")
- if isinstance(command_name, str) and command_name:
- commands.append(command_name)
- return commands, 0
-
-
@tool_support_commands.register_subcommand(
"supports-tool-command",
help_description="Test where a given tool-support command exists",
@@ -1180,50 +813,6 @@ def _export_reference_data(context: CommandContext) -> None:
raise AssertionError(f"Unsupported output format {output_format}")
-def _add_known_packaging_data(
- details: PackagingFileInfo,
- plugin_data: PluginProvidedKnownPackagingFile,
- dh_compat_level: Optional[int],
-):
- install_pattern = _kpf_install_pattern(
- dh_compat_level,
- plugin_data,
- )
- config_features = plugin_data.info.get("config_features")
- if config_features:
- config_features = expand_known_packaging_config_features(
- dh_compat_level or 0,
- config_features,
- )
- _merge_list(details, "config-features", config_features)
-
- if dh_compat_level is not None:
- extra_config_features = []
- for dh_compat_rule in _relevant_dh_compat_rules(
- dh_compat_level, plugin_data.info
- ):
- cf = dh_compat_rule.get("add_config_features")
- if cf:
- extra_config_features.extend(cf)
- if extra_config_features:
- extra_config_features = expand_known_packaging_config_features(
- dh_compat_level,
- extra_config_features,
- )
- _merge_list(details, "config-features", extra_config_features)
- if "install-pattern" not in details and install_pattern is not None:
- details["install-pattern"] = install_pattern
- for mk, ok in [
- ("file_categories", "file-categories"),
- ("documentation_uris", "documentation-uris"),
- ("debputy_cmd_templates", "debputy-cmd-templates"),
- ]:
- value = plugin_data.info.get(mk)
- if value and ok == "debputy-cmd-templates":
- value = [escape_shell(*c) for c in value]
- _merge_list(details, ok, value)
-
-
@tool_support_commands.register_subcommand(
"annotate-debian-directory",
log_only_to_stderr=True,
@@ -1235,130 +824,13 @@ def _annotate_debian_directory(context: CommandContext) -> None:
# Validates that we are run from a debian directory as a side effect
binary_packages = context.binary_packages()
feature_set = context.load_plugins()
- known_packaging_files = feature_set.known_packaging_files
- debputy_plugin_metadata = plugin_metadata_for_debputys_own_plugin()
-
- reference_data_set_names = [
- "config-features",
- "file-categories",
- ]
- for n in reference_data_set_names:
- assert n in REFERENCE_DATA_TABLE
-
- annotated: List[PackagingFileInfo] = []
- seen_paths = set()
-
- r = read_dh_addon_sequences(context.debian_dir)
- if r is not None:
- bd_sequences, dr_sequences = r
- drules_sequences = bd_sequences | dr_sequences
- else:
- drules_sequences = set()
- is_debputy_package = (
- "debputy" in drules_sequences
- or "zz-debputy" in drules_sequences
- or "zz_debputy" in drules_sequences
- or "zz-debputy-rrr" in drules_sequences
- )
- dh_compat_level, dh_assistant_exit_code = _extract_dh_compat_level()
- dh_issues = []
-
- static_packaging_files = {
- kpf.detection_value: kpf
- for kpf in known_packaging_files.values()
- if kpf.detection_method == "path"
- }
- dh_pkgfile_docs = {
- kpf.detection_value: kpf
- for kpf in known_packaging_files.values()
- if kpf.detection_method == "dh.pkgfile"
- }
-
- if is_debputy_package:
- all_debputy_ppfs = list(
- flatten_ppfs(
- detect_all_packager_provided_files(
- feature_set.packager_provided_files,
- context.debian_dir,
- binary_packages,
- allow_fuzzy_matches=True,
- )
- )
- )
- else:
- all_debputy_ppfs = []
-
- if dh_compat_level is not None:
- (
- all_dh_ppfs,
- dh_issues,
- dh_assistant_exit_code,
- ) = _resolve_debhelper_config_files(
- context.debian_dir,
- binary_packages,
- debputy_plugin_metadata,
- dh_pkgfile_docs,
- drules_sequences,
- dh_compat_level,
- )
-
- else:
- all_dh_ppfs = []
- for ppf in all_debputy_ppfs:
- key = ppf.path.path
- ref_doc = ppf.definition.reference_documentation
- documentation_uris = (
- ref_doc.format_documentation_uris if ref_doc is not None else None
- )
- details: PackagingFileInfo = {
- "path": key,
- "debputy-cmd-templates": [
- ["debputy", "plugin", "show", "p-p-f", ppf.definition.stem]
- ],
- }
- if ppf.fuzzy_match and key.endswith(".in"):
- _merge_list(details, "file-categories", ["generic-template"])
- details["generates"] = key[:-3]
- elif assume_not_none(ppf.path.parent_dir).get(ppf.path.name + ".in"):
- _merge_list(details, "file-categories", ["generated"])
- details["generated-from"] = key + ".in"
- seen_paths.add(key)
- annotated.append(details)
- static_details = static_packaging_files.get(key)
- if static_details is not None:
- # debhelper compat rules does not apply to debputy files
- _add_known_packaging_data(details, static_details, None)
- if documentation_uris:
- details["documentation-uris"] = list(documentation_uris)
-
- _merge_ppfs(annotated, seen_paths, all_dh_ppfs, dh_pkgfile_docs, dh_compat_level)
-
- for virtual_path in _scan_debian_dir(context.debian_dir):
- key = virtual_path.path
- if key in seen_paths:
- continue
- if virtual_path.is_symlink:
- try:
- st = os.stat(virtual_path.fs_path)
- except FileNotFoundError:
- continue
- else:
- if not stat.S_ISREG(st.st_mode):
- continue
- elif not virtual_path.is_file:
- continue
-
- static_match = static_packaging_files.get(virtual_path.path)
- if static_match is not None:
- details: PackagingFileInfo = {
- "path": key,
- }
- annotated.append(details)
- if assume_not_none(virtual_path.parent_dir).get(virtual_path.name + ".in"):
- details["generated-from"] = key + ".in"
- _merge_list(details, "file-categories", ["generated"])
- _add_known_packaging_data(details, static_match, dh_compat_level)
+ result = scan_debian_dir(
+ feature_set,
+ binary_packages,
+ context.debian_dir,
+ )
+ annotated, reference_data_set_names, dh_assistant_exit_code, dh_issues = result
data = {
"result": annotated,
diff --git a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py
index ee64a84..d715e5a 100644
--- a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py
+++ b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py
@@ -247,6 +247,7 @@ def lsp_describe_features(context: CommandContext) -> None:
@ROOT_COMMAND.register_subcommand(
"lint",
log_only_to_stderr=True,
+ help_description="Provide diagnostics for the packaging (like `lsp server` except no editor is needed)",
argparser=[
add_arg(
"--spellcheck",
@@ -305,6 +306,7 @@ def lint_cmd(context: CommandContext) -> None:
@ROOT_COMMAND.register_subcommand(
"reformat",
+ help_description="Reformat the packaging files based on the packaging/maintainer rules",
argparser=[
add_arg(
"--style",
@@ -361,7 +363,7 @@ def reformat_cmd(context: CommandContext) -> None:
perform_reformat(context, named_style=context.parsed_args.named_style)
-def ensure_lint_and_lsp_commands_are_loaded():
+def ensure_lint_and_lsp_commands_are_loaded() -> None:
# Loading the module does the heavy lifting
# However, having this function means that we do not have an "unused" import that some tool
# gets tempted to remove
diff --git a/src/debputy/commands/debputy_cmd/plugin_cmds.py b/src/debputy/commands/debputy_cmd/plugin_cmds.py
index 60d3c70..83bb88f 100644
--- a/src/debputy/commands/debputy_cmd/plugin_cmds.py
+++ b/src/debputy/commands/debputy_cmd/plugin_cmds.py
@@ -1,5 +1,4 @@
import argparse
-import itertools
import operator
import os
import sys
@@ -12,17 +11,16 @@ from typing import (
Any,
Optional,
Type,
- Mapping,
Callable,
)
from debputy import DEBPUTY_DOC_ROOT_DIR
+from debputy.analysis.analysis_util import flatten_ppfs
from debputy.commands.debputy_cmd.context import (
CommandContext,
add_arg,
ROOT_COMMAND,
)
-from debputy.commands.debputy_cmd.dc_util import flatten_ppfs
from debputy.commands.debputy_cmd.output import (
_stream_to_pager,
_output_styling,
@@ -32,8 +30,6 @@ from debputy.exceptions import DebputySubstitutionError
from debputy.filesystem_scan import build_virtual_fs
from debputy.manifest_parser.base_types import TypeMapping
from debputy.manifest_parser.declarative_parser import (
- DeclarativeMappingInputParser,
- DeclarativeNonMappingInputParser,
BASIC_SIMPLE_TYPES,
)
from debputy.manifest_parser.parser_data import ParserContextData
@@ -49,9 +45,6 @@ from debputy.plugin.api.impl_types import (
PackagerProvidedFileClassSpec,
PluginProvidedManifestVariable,
DispatchingParserBase,
- DeclarativeInputParser,
- DebputyPluginMetadata,
- DispatchingObjectParser,
SUPPORTED_DISPATCHABLE_TABLE_PARSERS,
OPARSER_MANIFEST_ROOT,
PluginProvidedDiscardRule,
@@ -60,13 +53,10 @@ from debputy.plugin.api.impl_types import (
PluginProvidedTypeMapping,
)
from debputy.plugin.api.spec import (
- ParserDocumentation,
- reference_documentation,
- undocumented_attr,
TypeMappingExample,
)
from debputy.substitution import Substitution
-from debputy.util import _error, assume_not_none, _warn
+from debputy.util import _error, _warn
plugin_dispatcher = ROOT_COMMAND.add_dispatching_subcommand(
"plugin",
diff --git a/src/debputy/debhelper_emulation.py b/src/debputy/debhelper_emulation.py
index 65a26f8..8242a32 100644
--- a/src/debputy/debhelper_emulation.py
+++ b/src/debputy/debhelper_emulation.py
@@ -243,13 +243,16 @@ _FIND_DH_WITH = re.compile(r"--with(?:\s+|=)(\S+)")
_DEP_REGEX = re.compile("^([a-z0-9][-+.a-z0-9]+)", re.ASCII)
-def parse_drules_for_addons(lines: Iterable[str], sequences: Set[str]) -> None:
+def parse_drules_for_addons(lines: Iterable[str], sequences: Set[str]) -> bool:
+ saw_dh = False
for line in lines:
if not line.startswith("\tdh "):
continue
+ saw_dh = True
for match in _FIND_DH_WITH.finditer(line):
sequence_def = match.group(1)
sequences.update(sequence_def.split(","))
+ return saw_dh
def extract_dh_addons_from_control(
diff --git a/src/debputy/dh_migration/migration.py b/src/debputy/dh_migration/migration.py
index a1bd15a..59a7ee4 100644
--- a/src/debputy/dh_migration/migration.py
+++ b/src/debputy/dh_migration/migration.py
@@ -157,7 +157,7 @@ def _check_migration_target(
r = read_dh_addon_sequences(debian_dir)
if r is None and migration_target is None:
_error("debian/control is missing and no migration target was provided")
- bd_sequences, dr_sequences = r
+ bd_sequences, dr_sequences, _ = r
all_sequences = bd_sequences | dr_sequences
has_zz_debputy = "zz-debputy" in all_sequences or "debputy" in all_sequences
diff --git a/src/debputy/dh_migration/migrators_impl.py b/src/debputy/dh_migration/migrators_impl.py
index d68768c..48ec1e0 100644
--- a/src/debputy/dh_migration/migrators_impl.py
+++ b/src/debputy/dh_migration/migrators_impl.py
@@ -1503,23 +1503,24 @@ def detect_obsolete_substvars(
def read_dh_addon_sequences(
debian_dir: VirtualPath,
-) -> Optional[Tuple[Set[str], Set[str]]]:
+) -> Optional[Tuple[Set[str], Set[str], bool]]:
ctrl_file = debian_dir.get("control")
if ctrl_file:
dr_sequences: Set[str] = set()
bd_sequences: Set[str] = set()
drules = debian_dir.get("rules")
+ saw_dh = False
if drules and drules.is_file:
with drules.open() as fd:
- parse_drules_for_addons(fd, dr_sequences)
+ saw_dh = parse_drules_for_addons(fd, dr_sequences)
with ctrl_file.open() as fd:
ctrl = list(Deb822.iter_paragraphs(fd))
source_paragraph = ctrl[0] if ctrl else {}
extract_dh_addons_from_control(source_paragraph, bd_sequences)
- return bd_sequences, dr_sequences
+ return bd_sequences, dr_sequences, saw_dh
return None
@@ -1539,7 +1540,7 @@ def detect_dh_addons_zz_debputy_rrr(
)
return
- bd_sequences, dr_sequences = r
+ bd_sequences, dr_sequences, _ = r
remaining_sequences = bd_sequences | dr_sequences
saw_dh_debputy = "zz-debputy-rrr" in remaining_sequences
@@ -1565,7 +1566,7 @@ def detect_dh_addons(
)
return
- bd_sequences, dr_sequences = r
+ bd_sequences, dr_sequences, _ = r
remaining_sequences = bd_sequences | dr_sequences
saw_dh_debputy = (
diff --git a/src/debputy/linting/lint_util.py b/src/debputy/linting/lint_util.py
index 0c293f0..30e1177 100644
--- a/src/debputy/linting/lint_util.py
+++ b/src/debputy/linting/lint_util.py
@@ -170,6 +170,8 @@ class LintDiagnosticResult:
is_file_level_diagnostic: bool
has_broken_range: bool
missing_severity: bool
+ discovered_in: str
+ report_for_related_file: Optional[str]
class LintReport:
@@ -183,7 +185,7 @@ class LintReport:
self.number_of_broken_diagnostics: int = 0
self.lint_state: Optional[LintState] = None
self.start_timestamp = datetime.datetime.now()
- self.durations: Mapping[str, float] = collections.defaultdict(lambda: 0.0)
+ self.durations: typing.Dict[str, float] = collections.defaultdict(lambda: 0.0)
self._timer = time.perf_counter()
@contextlib.contextmanager
@@ -217,6 +219,7 @@ class LintReport:
assert lint_state is not None
if in_file is None:
in_file = lint_state.path
+ discovered_in_file = in_file
severity = diagnostic.severity
missing_severity = False
error_marker: Optional[RuntimeError] = None
@@ -230,13 +233,30 @@ class LintReport:
diag_range = diagnostic.range
start_pos = diag_range.start
end_pos = diag_range.start
- is_file_level_diagnostic = _is_file_level_diagnostic(
- lines,
- start_pos.line,
- start_pos.character,
- end_pos.line,
- end_pos.character,
- )
+ diag_data = diagnostic.data
+ if isinstance(diag_data, dict):
+ report_for_related_file = diag_data.get("report_for_related_file")
+ if report_for_related_file is None or not isinstance(
+ report_for_related_file, str
+ ):
+ report_for_related_file = None
+ else:
+ in_file = report_for_related_file
+ # Force it to exist in self.durations, since subclasses can use .items() or "foo" in self.durations.
+ if in_file not in self.durations:
+ self.durations[in_file] = 0
+ else:
+ report_for_related_file = None
+ if report_for_related_file is not None:
+ is_file_level_diagnostic = True
+ else:
+ is_file_level_diagnostic = _is_file_level_diagnostic(
+ lines,
+ start_pos.line,
+ start_pos.character,
+ end_pos.line,
+ end_pos.character,
+ )
has_broken_range = not is_file_level_diagnostic and (
end_pos.line > len(lines) or start_pos.line < 0
)
@@ -251,6 +271,8 @@ class LintReport:
is_file_level_diagnostic,
has_broken_range,
missing_severity,
+ report_for_related_file=report_for_related_file,
+ discovered_in=discovered_in_file,
)
self.diagnostics_by_file[in_file].append(diagnostic_result)
@@ -351,13 +373,22 @@ class TermLintReport(LintReport):
bg="black",
style="bold",
)
- start_line = diagnostic.range.start.line
- start_position = diagnostic.range.start.character
- end_line = diagnostic.range.end.line
- end_position = diagnostic.range.end.character
+
+ if diagnostic_result.is_file_level_diagnostic:
+ start_line = 0
+ start_position = 0
+ end_line = 0
+ end_position = 0
+ else:
+ start_line = diagnostic.range.start.line
+ start_position = diagnostic.range.start.character
+ end_line = diagnostic.range.end.line
+ end_position = diagnostic.range.end.character
+
has_fixit = ""
lines = lint_state.lines
line_no_width = len(str(len(lines)))
+
if diagnostic_result.result_state == LintDiagnosticResultState.FIXABLE:
has_fixit = " [Correctable via --auto-fix]"
print(
diff --git a/src/debputy/lsp/debputy_ls.py b/src/debputy/lsp/debputy_ls.py
index 2009fd7..e195208 100644
--- a/src/debputy/lsp/debputy_ls.py
+++ b/src/debputy/lsp/debputy_ls.py
@@ -29,7 +29,6 @@ from debputy.packages import (
BinaryPackage,
DctrlParser,
)
-from debputy.plugin.api import VirtualPath
from debputy.plugin.api.feature_set import PluginProvidedFeatureSet
from debputy.util import _info
from debputy.yaml import MANIFEST_YAML, YAMLError
diff --git a/src/debputy/lsp/diagnostics.py b/src/debputy/lsp/diagnostics.py
index 618b91d..5c1907e 100644
--- a/src/debputy/lsp/diagnostics.py
+++ b/src/debputy/lsp/diagnostics.py
@@ -6,3 +6,4 @@ LintSeverity = Literal["error", "warning", "informational", "pedantic", "spellin
class DiagnosticData(TypedDict):
quickfixes: NotRequired[Optional[List[Any]]]
lint_severity: NotRequired[Optional[LintSeverity]]
+ report_for_related_file: NotRequired[str]
diff --git a/src/debputy/lsp/lsp_debian_control.py b/src/debputy/lsp/lsp_debian_control.py
index 6e775c5..370a14f 100644
--- a/src/debputy/lsp/lsp_debian_control.py
+++ b/src/debputy/lsp/lsp_debian_control.py
@@ -1,4 +1,5 @@
import dataclasses
+import os.path
import re
import textwrap
from typing import (
@@ -9,30 +10,10 @@ from typing import (
Mapping,
List,
Dict,
+ Iterable,
)
-from debputy.lsprotocol.types import (
- DiagnosticSeverity,
- Range,
- Diagnostic,
- Position,
- FoldingRange,
- FoldingRangeParams,
- CompletionItem,
- CompletionList,
- CompletionParams,
- DiagnosticRelatedInformation,
- Location,
- HoverParams,
- Hover,
- TEXT_DOCUMENT_CODE_ACTION,
- SemanticTokens,
- SemanticTokensParams,
- WillSaveTextDocumentParams,
- TextEdit,
- DocumentFormattingParams,
-)
-
+from debputy.analysis.debian_dir import scan_debian_dir
from debputy.linting.lint_util import LintState
from debputy.lsp.debputy_ls import DebputyLanguageServer
from debputy.lsp.diagnostics import DiagnosticData
@@ -75,7 +56,6 @@ from debputy.lsp.spellchecking import default_spellchecker
from debputy.lsp.text_util import (
normalize_dctrl_field_name,
LintCapablePositionCodec,
- detect_possible_typo,
te_range_to_lsp,
)
from debputy.lsp.vendoring._deb822_repro import (
@@ -86,6 +66,28 @@ from debputy.lsp.vendoring._deb822_repro.parsing import (
Deb822KeyValuePairElement,
LIST_SPACE_SEPARATED_INTERPRETATION,
)
+from debputy.lsprotocol.types import (
+ DiagnosticSeverity,
+ Range,
+ Diagnostic,
+ Position,
+ FoldingRange,
+ FoldingRangeParams,
+ CompletionItem,
+ CompletionList,
+ CompletionParams,
+ DiagnosticRelatedInformation,
+ Location,
+ HoverParams,
+ Hover,
+ TEXT_DOCUMENT_CODE_ACTION,
+ SemanticTokens,
+ SemanticTokensParams,
+ WillSaveTextDocumentParams,
+ TextEdit,
+ DocumentFormattingParams,
+)
+from debputy.util import detect_possible_typo
try:
from debputy.lsp.vendoring._deb822_repro.locatable import (
@@ -94,7 +96,6 @@ try:
START_POSITION,
)
- from pygls.server import LanguageServer
from pygls.workspace import TextDocument
except ImportError:
pass
@@ -919,6 +920,7 @@ def _lint_debian_control(
paragraphs = list(deb822_file)
source_paragraph = paragraphs[0] if paragraphs else None
+ binary_stanzas_w_pos = []
for paragraph_no, paragraph in enumerate(paragraphs, start=1):
paragraph_pos = paragraph.position_in_file()
@@ -928,6 +930,7 @@ def _lint_debian_control(
if is_binary_paragraph:
known_fields = BINARY_FIELDS
other_known_fields = SOURCE_FIELDS
+ binary_stanzas_w_pos.append((paragraph, paragraph_pos))
else:
known_fields = SOURCE_FIELDS
other_known_fields = BINARY_FIELDS
@@ -944,9 +947,113 @@ def _lint_debian_control(
diagnostics,
)
+ _detect_misspelled_packaging_files(
+ lint_state,
+ binary_stanzas_w_pos,
+ diagnostics,
+ )
+
return diagnostics
+def _package_range_of_stanza(
+ lint_state: LintState,
+ binary_stanzas: List[Tuple[Deb822ParagraphElement, TEPosition]],
+) -> Iterable[Tuple[str, Range]]:
+ for stanza, stanza_position in binary_stanzas:
+ kvpair = stanza.get_kvpair_element("Package")
+ if kvpair is None:
+ continue
+ representation_field_range = kvpair.range_in_parent().relative_to(
+ stanza_position
+ )
+ representation_field_range = lint_state.position_codec.range_to_client_units(
+ lint_state.lines,
+ te_range_to_lsp(representation_field_range),
+ )
+ yield stanza["Package"], representation_field_range
+
+
+def _detect_misspelled_packaging_files(
+ lint_state: LintState,
+ binary_stanzas_w_pos: List[Tuple[Deb822ParagraphElement, TEPosition]],
+ diagnostics: List[Diagnostic],
+) -> None:
+ debian_dir = lint_state.debian_dir
+ binary_packages = lint_state.binary_packages
+ if debian_dir is None or binary_packages is None:
+ return
+ all_pkg_file_data, _, _, _ = scan_debian_dir(
+ lint_state.plugin_feature_set,
+ binary_packages,
+ debian_dir,
+ )
+ stanza_ranges = {
+ p: r for p, r in _package_range_of_stanza(lint_state, binary_stanzas_w_pos)
+ }
+
+ for pkg_file_data in all_pkg_file_data:
+ binary_package = pkg_file_data.get("binary-package")
+ explicit_package = pkg_file_data.get("pkgfile-explicit-package-name", True)
+ name_segment = pkg_file_data.get("pkgfile-name-segment")
+ stem = pkg_file_data.get("pkgfile-stem")
+ if binary_package is None or stem is None:
+ continue
+ diag_range = stanza_ranges.get(binary_package)
+ if diag_range is None:
+ continue
+
+ path = pkg_file_data["path"]
+ likely_typo_of = pkg_file_data.get("likely-typo-of")
+ if likely_typo_of is not None:
+ diagnostics.append(
+ Diagnostic(
+ diag_range,
+ f'The file "{path}" is likely a typo of "{likely_typo_of}"',
+ severity=DiagnosticSeverity.Warning,
+ source="debputy",
+ data=DiagnosticData(
+ report_for_related_file=path,
+ ),
+ )
+ )
+ continue
+
+ if not pkg_file_data.get("pkgfile-is-active-in-build", True):
+ diagnostics.append(
+ Diagnostic(
+ diag_range,
+ f"The file {path} is related to a command that is not active in the dh sequence"
+ " with the current addons",
+ severity=DiagnosticSeverity.Warning,
+ source="debputy",
+ data=DiagnosticData(
+ report_for_related_file=path,
+ ),
+ )
+ )
+ continue
+
+ if not explicit_package and name_segment is not None:
+ basename = os.path.basename(path)
+ alt_name = f"{binary_package}.{stem}"
+ arch_restriction = pkg_file_data.get("pkgfile-architecture-restriction")
+ if arch_restriction is not None:
+ alt_name = f"{alt_name}.{arch_restriction}"
+ diagnostics.append(
+ Diagnostic(
+ diag_range,
+ f'Possible typo in "{path}". Consider renaming the file to "debian/{binary_package}.{basename}"'
+ f' or "debian/{alt_name} if it is intended for {binary_package}',
+ severity=DiagnosticSeverity.Warning,
+ source="debputy",
+ data=DiagnosticData(
+ report_for_related_file=path,
+ ),
+ )
+ )
+
+
@lsp_will_save_wait_until(_LANGUAGE_IDS)
def _debian_control_on_save_formatting(
ls: "DebputyLanguageServer",
diff --git a/src/debputy/lsp/lsp_debian_control_reference_data.py b/src/debputy/lsp/lsp_debian_control_reference_data.py
index f4a962a..9a589bf 100644
--- a/src/debputy/lsp/lsp_debian_control_reference_data.py
+++ b/src/debputy/lsp/lsp_debian_control_reference_data.py
@@ -26,6 +26,7 @@ from typing import (
)
from debian.debian_support import DpkgArchTable, Version
+
from debputy.lsprotocol.types import (
DiagnosticSeverity,
Diagnostic,
@@ -38,6 +39,7 @@ from debputy.lsprotocol.types import (
CompletionItemTag,
MarkupKind,
CompletionItemKind,
+ CompletionItemLabelDetails,
)
from debputy.filesystem_scan import VirtualPathBase
@@ -57,7 +59,6 @@ from debputy.lsp.text_edit import apply_text_edits
from debputy.lsp.text_util import (
normalize_dctrl_field_name,
LintCapablePositionCodec,
- detect_possible_typo,
te_range_to_lsp,
trim_end_of_line_whitespace,
)
@@ -90,7 +91,7 @@ from debputy.lsp.vendoring._deb822_repro.types import TE
from debputy.lsp.vendoring.wrap_and_sort import _sort_packages_key
from debputy.path_matcher import BasenameGlobMatch
from debputy.plugin.api import VirtualPath
-from debputy.util import PKGNAME_REGEX, _info
+from debputy.util import PKGNAME_REGEX, _info, detect_possible_typo
try:
from debputy.lsp.vendoring._deb822_repro.locatable import (
@@ -298,6 +299,7 @@ ALL_SECTIONS = allowed_values(
ALL_PRIORITIES = allowed_values(
Keyword(
"required",
+ synopsis_doc="[RARE]: Package is Essential or an Essential package needs it (and is not a library)",
hover_text=textwrap.dedent(
"""\
The package is necessary for the proper functioning of the system (read: dpkg needs it).
@@ -311,6 +313,7 @@ ALL_PRIORITIES = allowed_values(
),
Keyword(
"important",
+ synopsis_doc="[RARE]: Bare minimum of bare minimum of commonly-expected and necessary tools",
hover_text=textwrap.dedent(
"""\
The *important* packages are a bare minimum of commonly-expected and necessary tools.
@@ -324,6 +327,7 @@ ALL_PRIORITIES = allowed_values(
),
Keyword(
"standard",
+ synopsis_doc="[RARE]: If your distribution installer would install this by default (not for libraries)",
hover_text=textwrap.dedent(
"""\
These packages provide a reasonable small but not too limited character-mode system. This is
@@ -340,6 +344,8 @@ ALL_PRIORITIES = allowed_values(
),
Keyword(
"optional",
+ synopsis_doc="The default priority.",
+ sort_text="aa-optional",
hover_text="This is the default priority and used by the majority of all packages"
" in the Debian archive",
),
@@ -347,6 +353,8 @@ ALL_PRIORITIES = allowed_values(
"extra",
is_obsolete=True,
replaced_by="optional",
+ sort_text="zz-extra",
+ synopsis_doc="Obsolete alias of `optional`",
hover_text="Obsolete alias of `optional`.",
),
)
@@ -359,6 +367,7 @@ def all_architectures_and_wildcards(
yield Keyword(
"any",
is_exclusive=True,
+ synopsis_doc="Build once per machine architecture (native code, such as C/C++, interpreter to C bindings)",
hover_text=textwrap.dedent(
"""\
The package is an architecture dependent package and need to be compiled for each and every
@@ -372,6 +381,7 @@ def all_architectures_and_wildcards(
yield Keyword(
"all",
is_exclusive=True,
+ synopsis_doc="Independent of machine architecture (scripts, Java without JNI, data or documentation)",
hover_text=textwrap.dedent(
"""\
The package is an architecture independent package. This is typically fitting for packages containing
@@ -1237,6 +1247,7 @@ class Deb822KnownField:
lint_state,
stanza_parts,
"",
+ markdown_kind,
is_completion_for_field=True,
)
if options is not None and len(options) == 1:
@@ -1288,7 +1299,7 @@ class Deb822KnownField:
if value_being_completed == "":
current_dir = base_dir
- unmatched_parts: Sequence[str] = tuple()
+ unmatched_parts: Sequence[str] = ()
else:
current_dir, unmatched_parts = base_dir.attempt_lookup(
value_being_completed
@@ -1311,6 +1322,9 @@ class Deb822KnownField:
" " in child.name or "\t" in child.name
):
continue
+ sort_text = (
+ f"z-{child.name}" if child.name.startswith(".") else f"a-{child.name}"
+ )
if child.is_dir:
if _should_ignore_dir(
child,
@@ -1321,7 +1335,12 @@ class Deb822KnownField:
items.append(
CompletionItem(
f"{child.path}/",
+ label_details=CompletionItemLabelDetails(
+ description=child.path,
+ ),
insert_text=path_escaper(f"{child.path}/"),
+ filter_text=f"{child.path}/",
+ sort_text=sort_text,
kind=CompletionItemKind.Folder,
)
)
@@ -1329,7 +1348,12 @@ class Deb822KnownField:
items.append(
CompletionItem(
child.path,
+ label_details=CompletionItemLabelDetails(
+ description=child.path,
+ ),
insert_text=path_escaper(child.path),
+ filter_text=child.path,
+ sort_text=sort_text,
kind=CompletionItemKind.File,
)
)
@@ -1340,6 +1364,7 @@ class Deb822KnownField:
lint_state: LintState,
stanza_parts: Sequence[Deb822ParagraphElement],
value_being_completed: str,
+ markdown_kind: MarkupKind,
*,
is_completion_for_field: bool = False,
) -> Optional[Sequence[CompletionItem]]:
@@ -1374,6 +1399,13 @@ class Deb822KnownField:
CompletionItem(
keyword.value,
insert_text=keyword.value,
+ sort_text=keyword.sort_text,
+ detail=keyword.synopsis_doc,
+ documentation=(
+ MarkupContent(value=keyword.hover_text, kind=markdown_kind)
+ if keyword.hover_text
+ else None
+ ),
)
for keyword in known_values.values()
if keyword.is_keyword_valid_completion_in_stanza(stanza_parts)
@@ -1430,6 +1462,7 @@ class Deb822KnownField:
stanza_position,
lint_state,
)
+ yield from self._dep5_file_list_diagnostics(kvpair, kvpair_position, lint_state)
if not self.spellcheck_value:
yield from self._known_value_diagnostics(
kvpair,
@@ -1484,6 +1517,56 @@ class Deb822KnownField:
),
)
+ def _dep5_file_list_diagnostics(
+ self,
+ kvpair: Deb822KeyValuePairElement,
+ kvpair_position: "TEPosition",
+ lint_state: LintState,
+ ) -> Iterable[Diagnostic]:
+ source_root = lint_state.source_root
+ if (
+ self.field_value_class != FieldValueClass.DEP5_FILE_LIST
+ or source_root is None
+ ):
+ return
+ interpreter = self.field_value_class.interpreter()
+ values = kvpair.interpret_as(interpreter)
+ value_off = kvpair.value_element.position_in_parent().relative_to(
+ kvpair_position
+ )
+
+ assert interpreter is not None
+
+ for token in values.iter_parts():
+ if token.is_whitespace:
+ continue
+ text = token.convert_to_text()
+ if "?" in text or "*" in text:
+ # TODO: We should validate these as well
+ continue
+ matched_path, missing_part = source_root.attempt_lookup(text)
+ # It is common practice to delete "dirty" files during clean. This causes files listed
+ # in `debian/copyright` to go missing and as a consequence, we do not validate whether
+ # they are present (that would require us to check the `.orig.tar`, which we could but
+ # do not have the infrastructure for).
+ if not missing_part and matched_path.is_dir:
+ path_range_te = token.range_in_parent().relative_to(value_off)
+ path_range = lint_state.position_codec.range_to_client_units(
+ lint_state.lines,
+ te_range_to_lsp(path_range_te),
+ )
+ yield Diagnostic(
+ path_range,
+ "Directories cannot be a match. Use `dir/*` to match everything in it",
+ severity=DiagnosticSeverity.Warning,
+ source="debputy",
+ data=DiagnosticData(
+ quickfixes=[
+ propose_correct_text_quick_fix(f"{matched_path.path}/*")
+ ]
+ ),
+ )
+
def _known_value_diagnostics(
self,
kvpair: Deb822KeyValuePairElement,
@@ -2962,6 +3045,9 @@ BINARY_FIELDS = _fields(
known_values=allowed_values(
Keyword(
"no",
+ # Show `no` after `foreign` and `same`. Often, you want `same` or `foreign`.
+ sort_text="zz-no",
+ synopsis_doc='No Multi-Arch support (often used for "reviewed and no support possible/needed")',
hover_text=textwrap.dedent(
"""\
The default. The package can be installed for at most one architecture at the time. It can
@@ -2977,6 +3063,7 @@ BINARY_FIELDS = _fields(
),
Keyword(
"foreign",
+ synopsis_doc="Can satisfy dependencies for other architectures (common for data and *some* scripts)",
hover_text=textwrap.dedent(
"""\
The package can be installed for at most one architecture at the time. However, it can
@@ -2995,6 +3082,7 @@ BINARY_FIELDS = _fields(
Keyword(
"same",
can_complete_keyword_in_stanza=_complete_only_in_arch_dep_pkgs,
+ synopsis_doc="Co-installable with itself for different architectures (common for native libraries)",
hover_text=textwrap.dedent(
"""\
The same version of the package can be co-installed for multiple architecture. However,
@@ -3010,6 +3098,9 @@ BINARY_FIELDS = _fields(
),
Keyword(
"allowed",
+ # Never show `allowed` first, it is the absolute least likely candidate.
+ sort_text="zzzz-allowed",
+ synopsis_doc="[RARE]: Consumer decides whether it is `same` and `foreign`",
hover_text=textwrap.dedent(
"""\
**Advanced and very rare value**. This value is exceedingly rare to the point that less
diff --git a/src/debputy/lsp/lsp_debian_copyright.py b/src/debputy/lsp/lsp_debian_copyright.py
index a6d5573..9cbac26 100644
--- a/src/debputy/lsp/lsp_debian_copyright.py
+++ b/src/debputy/lsp/lsp_debian_copyright.py
@@ -67,7 +67,6 @@ from debputy.lsp.spellchecking import default_spellchecker
from debputy.lsp.text_util import (
normalize_dctrl_field_name,
LintCapablePositionCodec,
- detect_possible_typo,
te_range_to_lsp,
)
from debputy.lsp.vendoring._deb822_repro import (
@@ -78,6 +77,7 @@ from debputy.lsp.vendoring._deb822_repro.parsing import (
Deb822KeyValuePairElement,
LIST_SPACE_SEPARATED_INTERPRETATION,
)
+from debputy.util import detect_possible_typo
try:
from debputy.lsp.vendoring._deb822_repro.locatable import (
diff --git a/src/debputy/lsp/lsp_debian_debputy_manifest.py b/src/debputy/lsp/lsp_debian_debputy_manifest.py
index 78575cb..fd7e6b0 100644
--- a/src/debputy/lsp/lsp_debian_debputy_manifest.py
+++ b/src/debputy/lsp/lsp_debian_debputy_manifest.py
@@ -46,7 +46,6 @@ from debputy.lsp.lsp_generic_yaml import (
from debputy.lsp.quickfixes import propose_correct_text_quick_fix
from debputy.lsp.text_util import (
LintCapablePositionCodec,
- detect_possible_typo,
)
from debputy.manifest_parser.base_types import DebputyDispatchableType
from debputy.manifest_parser.declarative_parser import (
@@ -66,7 +65,7 @@ from debputy.plugin.api.impl_types import (
InPackageContextParser,
DeclarativeValuelessKeywordInputParser,
)
-from debputy.util import _info
+from debputy.util import _info, detect_possible_typo
from debputy.yaml.compat import (
Node,
CommentedMap,
diff --git a/src/debputy/lsp/lsp_debian_rules.py b/src/debputy/lsp/lsp_debian_rules.py
index 072b785..7c6d627 100644
--- a/src/debputy/lsp/lsp_debian_rules.py
+++ b/src/debputy/lsp/lsp_debian_rules.py
@@ -43,7 +43,7 @@ from debputy.lsp.spellchecking import spellcheck_line
from debputy.lsp.text_util import (
LintCapablePositionCodec,
)
-from debputy.util import _warn
+from debputy.util import _warn, detect_possible_typo
try:
from debputy.lsp.vendoring._deb822_repro.locatable import (
@@ -58,34 +58,6 @@ except ImportError:
pass
-try:
- from Levenshtein import distance
-except ImportError:
-
- def _detect_possible_typo(
- provided_value: str,
- known_values: Iterable[str],
- ) -> Sequence[str]:
- return tuple()
-
-else:
-
- def _detect_possible_typo(
- provided_value: str,
- known_values: Iterable[str],
- ) -> Sequence[str]:
- k_len = len(provided_value)
- candidates = []
- for known_value in known_values:
- if abs(k_len - len(known_value)) > 2:
- continue
- d = distance(provided_value, known_value)
- if d > 2:
- continue
- candidates.append(known_value)
- return candidates
-
-
_CONTAINS_TAB_OR_COLON = re.compile(r"[\t:]")
_WORDS_RE = re.compile("([a-zA-Z0-9_-]+)")
_MAKE_ERROR_RE = re.compile(r"^[^:]+:(\d+):\s*(\S.+)")
@@ -314,7 +286,7 @@ def _lint_debian_rules_impl(
missing_targets[target] = hook_location
for target, (line_no, pos, endpos) in missing_targets.items():
- candidates = _detect_possible_typo(target, all_allowed_hook_targets)
+ candidates = detect_possible_typo(target, all_allowed_hook_targets)
if not candidates and not target.startswith(
("override_", "execute_before_", "execute_after_")
):
diff --git a/src/debputy/lsp/lsp_debian_tests_control.py b/src/debputy/lsp/lsp_debian_tests_control.py
index 3562917..f188762 100644
--- a/src/debputy/lsp/lsp_debian_tests_control.py
+++ b/src/debputy/lsp/lsp_debian_tests_control.py
@@ -65,7 +65,6 @@ from debputy.lsp.spellchecking import default_spellchecker
from debputy.lsp.text_util import (
normalize_dctrl_field_name,
LintCapablePositionCodec,
- detect_possible_typo,
te_range_to_lsp,
)
from debputy.lsp.vendoring._deb822_repro import (
@@ -76,6 +75,7 @@ from debputy.lsp.vendoring._deb822_repro.parsing import (
Deb822KeyValuePairElement,
LIST_SPACE_SEPARATED_INTERPRETATION,
)
+from debputy.util import detect_possible_typo
try:
from debputy.lsp.vendoring._deb822_repro.locatable import (
diff --git a/src/debputy/lsp/lsp_generic_deb822.py b/src/debputy/lsp/lsp_generic_deb822.py
index d8b9596..895a3e0 100644
--- a/src/debputy/lsp/lsp_generic_deb822.py
+++ b/src/debputy/lsp/lsp_generic_deb822.py
@@ -272,6 +272,9 @@ def deb822_completer(
)
items: Optional[Sequence[CompletionItem]]
+ markdown_kind = ls.completion_item_document_markup(
+ MarkupKind.Markdown, MarkupKind.PlainText
+ )
if in_value:
_info(f"Completion for field value {current_field} -- {word_at_position}")
if known_field is None:
@@ -281,15 +284,16 @@ def deb822_completer(
lint_state,
list(matched_stanzas),
value_being_completed,
+ markdown_kind,
)
else:
_info("Completing field name")
assert stanza_metadata is not None
items = _complete_field_name(
- ls,
lint_state,
stanza_metadata,
matched_stanzas,
+ markdown_kind,
)
_info(
@@ -638,15 +642,12 @@ def deb822_semantic_tokens_full(
def _complete_field_name(
- ls: "DebputyLanguageServer",
lint_state: LintState,
fields: StanzaMetadata[Any],
matched_stanzas: Iterable[Deb822ParagraphElement],
+ markdown_kind: MarkupKind,
) -> Sequence[CompletionItem]:
items = []
- markdown_kind = ls.completion_item_document_markup(
- MarkupKind.Markdown, MarkupKind.PlainText
- )
matched_stanzas = list(matched_stanzas)
# TODO: Normalize fields according to file rules (X[BCS]- should be stripped in some files)
seen_fields = set(
diff --git a/src/debputy/lsp/lsp_reference_keyword.py b/src/debputy/lsp/lsp_reference_keyword.py
index c71352a..4046aaa 100644
--- a/src/debputy/lsp/lsp_reference_keyword.py
+++ b/src/debputy/lsp/lsp_reference_keyword.py
@@ -8,10 +8,12 @@ from debputy.lsp.vendoring._deb822_repro import Deb822ParagraphElement
@dataclasses.dataclass(slots=True, frozen=True)
class Keyword:
value: str
+ synopsis_doc: Optional[str] = None
hover_text: Optional[str] = None
is_obsolete: bool = False
replaced_by: Optional[str] = None
is_exclusive: bool = False
+ sort_text: Optional[str] = None
can_complete_keyword_in_stanza: Optional[
Callable[[Iterable[Deb822ParagraphElement]], bool]
] = None
diff --git a/src/debputy/lsp/lsp_self_check.py b/src/debputy/lsp/lsp_self_check.py
index 3c7d2e4..aa28a56 100644
--- a/src/debputy/lsp/lsp_self_check.py
+++ b/src/debputy/lsp/lsp_self_check.py
@@ -1,7 +1,10 @@
import dataclasses
import os.path
+import subprocess
from typing import Callable, Sequence, List, Optional, TypeVar
+from debian.debian_support import Version
+
from debputy.util import _error
@@ -24,7 +27,7 @@ def lsp_import_check(
*,
feature_name: Optional[str] = None,
is_mandatory: bool = False,
-):
+) -> Callable[[C], C]:
def _wrapper(func: C) -> C:
@@ -51,6 +54,29 @@ def lsp_import_check(
return _wrapper
+def lsp_generic_check(
+ problem: str,
+ how_to_fix: str,
+ *,
+ feature_name: Optional[str] = None,
+ is_mandatory: bool = False,
+) -> Callable[[C], C]:
+
+ def _wrapper(func: C) -> C:
+ LSP_CHECKS.append(
+ LSPSelfCheck(
+ _feature_name(feature_name, func),
+ func,
+ problem,
+ how_to_fix,
+ is_mandatory=is_mandatory,
+ )
+ )
+ return func
+
+ return _wrapper
+
+
def _feature_name(feature: Optional[str], func: Callable[[], None]) -> str:
if feature is not None:
return feature
@@ -83,6 +109,32 @@ def spell_checking() -> bool:
)
+@lsp_generic_check(
+ feature_name="Extra dh support",
+ problem="Missing dependencies",
+ how_to_fix="Run `apt satisfy debhelper (>= 13.16~)` to enable this feature",
+)
+def check_dh_version() -> bool:
+ try:
+ output = subprocess.check_output(
+ [
+ "dpkg-query",
+ "-W",
+ "--showformat=${Version} ${db:Status-Status}\n",
+ "debhelper",
+ ]
+ ).decode("utf-8")
+ except (FileNotFoundError, subprocess.CalledProcessError):
+ return False
+ else:
+ parts = output.split()
+ if len(parts) != 2:
+ return False
+ if parts[1] != "installed":
+ return False
+ return Version(parts[0]) >= Version("13.16~")
+
+
def assert_can_start_lsp() -> None:
for self_check in LSP_CHECKS:
if self_check.is_mandatory and not self_check.test():
diff --git a/src/debputy/lsp/text_util.py b/src/debputy/lsp/text_util.py
index 8228fef..4550001 100644
--- a/src/debputy/lsp/text_util.py
+++ b/src/debputy/lsp/text_util.py
@@ -31,34 +31,6 @@ else:
LintCapablePositionCodec = LinterPositionCodec
-try:
- from Levenshtein import distance
-except ImportError:
-
- def detect_possible_typo(
- provided_value: str,
- known_values: Iterable[str],
- ) -> Sequence[str]:
- return tuple()
-
-else:
-
- def detect_possible_typo(
- provided_value: str,
- known_values: Iterable[str],
- ) -> Sequence[str]:
- k_len = len(provided_value)
- candidates = []
- for known_value in known_values:
- if abs(k_len - len(known_value)) > 2:
- continue
- d = distance(provided_value, known_value)
- if d > 2:
- continue
- candidates.append(known_value)
- return candidates
-
-
def normalize_dctrl_field_name(f: str) -> str:
if not f or not f.startswith(("x", "X")):
return f
diff --git a/src/debputy/lsprotocol/types.py b/src/debputy/lsprotocol/types.py
index a57c4e0..e32b932 100644
--- a/src/debputy/lsprotocol/types.py
+++ b/src/debputy/lsprotocol/types.py
@@ -24,10 +24,25 @@ else:
except ImportError:
- class StubModule:
+ stub_attr = {
+ "__name__": __name__,
+ "__file__": __file__,
+ "__doc__": __doc__,
+ }
+ bad_attr = frozenset(
+ [
+ "pytestmark",
+ "pytest_plugins",
+ ]
+ )
+ class StubModule:
@staticmethod
def __getattr__(item: Any) -> Any:
+ if item in stub_attr:
+ return stub_attr[item]
+ if item in bad_attr:
+ raise AttributeError(item)
return types
def __call__(self, *args, **kwargs) -> Any:
diff --git a/src/debputy/packager_provided_files.py b/src/debputy/packager_provided_files.py
index 6d74999..85fff11 100644
--- a/src/debputy/packager_provided_files.py
+++ b/src/debputy/packager_provided_files.py
@@ -1,11 +1,25 @@
import collections
import dataclasses
-from typing import Mapping, Iterable, Dict, List, Optional, Tuple
+from typing import Mapping, Iterable, Dict, List, Optional, Tuple, Sequence
from debputy.packages import BinaryPackage
from debputy.plugin.api import VirtualPath
from debputy.plugin.api.impl_types import PackagerProvidedFileClassSpec
-from debputy.util import _error
+from debputy.util import _error, CAN_DETECT_TYPOS, detect_possible_typo
+
+
+_KNOWN_NON_PPFS = frozenset(
+ {
+ "gbp.conf", # Typo matches with `gbp.config` (dh_installdebconf) in two edits steps
+ # No reason to check any of these as they are never PPFs
+ "clean",
+ "control",
+ "compat",
+ "debputy.manifest",
+ "rules",
+ # NB: changelog and copyright are (de facto) ppfs, so they are deliberately omitted
+ }
+)
@dataclasses.dataclass(frozen=True, slots=True)
@@ -17,6 +31,10 @@ class PackagerProvidedFile:
definition: PackagerProvidedFileClassSpec
match_priority: int = 0
fuzzy_match: bool = False
+ uses_explicit_package_name: bool = False
+ name_segment: Optional[str] = None
+ architecture_restriction: Optional[str] = None
+ expected_path: Optional[str] = None
def compute_dest(self) -> Tuple[str, str]:
return self.definition.compute_dest(
@@ -71,21 +89,40 @@ def _find_package_name_prefix(
def _find_definition(
packager_provided_files: Mapping[str, PackagerProvidedFileClassSpec],
basename: str,
-) -> Tuple[Optional[str], Optional[PackagerProvidedFileClassSpec]]:
+ *,
+ period2stems: Optional[Mapping[int, Sequence[str]]] = None,
+) -> Tuple[Optional[str], Optional[PackagerProvidedFileClassSpec], Optional[str]]:
definition = packager_provided_files.get(basename)
if definition is not None:
- return None, definition
+ return None, definition, None
+ if period2stems and "." not in basename:
+ stems = period2stems.get(0)
+ matches = detect_possible_typo(basename, stems) if stems else None
+ if matches and len(matches) == 1:
+ definition = packager_provided_files[matches[0]]
+ return None, definition, basename
install_as_name = basename
file_class = ""
+ period_count = -1
while "." in install_as_name:
+ period_count += 1
install_as_name, file_class_part = install_as_name.rsplit(".", 1)
file_class = (
file_class_part + "." + file_class if file_class != "" else file_class_part
)
definition = packager_provided_files.get(file_class)
if definition is not None:
- return install_as_name, definition
- return None, None
+ return install_as_name, definition, None
+ if not period2stems:
+ continue
+ stems = period2stems.get(period_count)
+ if not stems:
+ continue
+ matches = detect_possible_typo(file_class, stems)
+ if matches and len(matches) == 1:
+ definition = packager_provided_files[matches[0]]
+ return install_as_name, definition, file_class
+ return None, None, None
def _check_mismatches(
@@ -129,6 +166,7 @@ def _split_path(
path: VirtualPath,
*,
allow_fuzzy_matches: bool = False,
+ period2stems: Optional[Mapping[int, Sequence[str]]] = None,
) -> Iterable[PackagerProvidedFile]:
owning_package_name = main_binary_package
basename = path.name
@@ -148,6 +186,9 @@ def _split_path(
definition=definition,
match_priority=match_priority,
fuzzy_match=False,
+ uses_explicit_package_name=False,
+ name_segment=None,
+ architecture_restriction=None,
)
for n in binary_packages
)
@@ -160,6 +201,9 @@ def _split_path(
definition=definition,
match_priority=match_priority,
fuzzy_match=False,
+ uses_explicit_package_name=False,
+ name_segment=None,
+ architecture_restriction=None,
)
return
@@ -178,6 +222,7 @@ def _split_path(
owning_package = binary_packages[owning_package_name]
match_priority = 1 if explicit_package else 0
fuzzy_match = False
+ arch_restriction: Optional[str] = None
if allow_fuzzy_matches and basename.endswith(".in") and len(basename) > 3:
basename = basename[:-3]
@@ -189,22 +234,24 @@ def _split_path(
if last_word == owning_package.package_deb_architecture_variable("ARCH"):
match_priority = 3
basename = remaining
- had_arch = True
+ arch_restriction = last_word
elif last_word == owning_package.package_deb_architecture_variable(
"ARCH_OS"
):
match_priority = 2
basename = remaining
- had_arch = True
+ arch_restriction = last_word
elif last_word == "all" and owning_package.is_arch_all:
- # This case does not make sense, but we detect it so we can report an error
+ # This case does not make sense, but we detect it, so we can report an error
# via _check_mismatches.
match_priority = -1
basename = remaining
- had_arch = True
+ arch_restriction = last_word
- install_as_name, definition = _find_definition(
- packager_provided_files, basename
+ install_as_name, definition, typoed_stem = _find_definition(
+ packager_provided_files,
+ basename,
+ period2stems=period2stems,
)
if definition is None:
continue
@@ -218,14 +265,24 @@ def _split_path(
definition,
owning_package,
install_as_name,
- had_arch,
+ arch_restriction is not None,
)
+
+ expected_path: Optional[str] = None
if (
definition.packageless_is_fallback_for_all_packages
and install_as_name is None
and not had_arch
and not explicit_package
+ and arch_restriction is None
):
+ if typoed_stem is not None:
+ parent_path = (
+ path.parent_dir.path + "/" if path.parent_dir is not None else ""
+ )
+ expected_path = f"{parent_path}{definition.stem}"
+ if fuzzy_match and path.name.endswith(".in"):
+ expected_path += ".in"
yield from (
PackagerProvidedFile(
path=path,
@@ -235,6 +292,10 @@ def _split_path(
definition=definition,
match_priority=match_priority,
fuzzy_match=fuzzy_match,
+ uses_explicit_package_name=False,
+ name_segment=None,
+ architecture_restriction=None,
+ expected_path=expected_path,
)
for n in binary_packages
)
@@ -248,6 +309,23 @@ def _split_path(
if bug_950723:
provided_key = f"{provided_key}@"
basename = f"{basename}@"
+ package_prefix = f"{owning_package_name}@"
+ else:
+ package_prefix = owning_package_name
+ if typoed_stem:
+ parent_path = (
+ path.parent_dir.path + "/" if path.parent_dir is not None else ""
+ )
+ basename = definition.stem
+ if install_as_name is not None:
+ basename = f"{install_as_name}.{basename}"
+ if explicit_package:
+ basename = f"{package_prefix}.{basename}"
+ if arch_restriction is not None:
+ basename = f"{basename}.{arch_restriction}"
+ expected_path = f"{parent_path}{basename}"
+ if fuzzy_match and path.name.endswith(".in"):
+ expected_path += ".in"
yield PackagerProvidedFile(
path=path,
package_name=owning_package_name,
@@ -256,16 +334,34 @@ def _split_path(
definition=definition,
match_priority=match_priority,
fuzzy_match=fuzzy_match,
+ uses_explicit_package_name=bool(explicit_package),
+ name_segment=install_as_name,
+ architecture_restriction=arch_restriction,
+ expected_path=expected_path,
)
return
+def _period_stem(stems: Iterable[str]) -> Mapping[int, Sequence[str]]:
+ result: Dict[int, List[str]] = {}
+ for stem in stems:
+ period_count = stem.count(".")
+ matched_stems = result.get(period_count)
+ if not matched_stems:
+ matched_stems = [stem]
+ result[period_count] = matched_stems
+ else:
+ matched_stems.append(stem)
+ return result
+
+
def detect_all_packager_provided_files(
packager_provided_files: Mapping[str, PackagerProvidedFileClassSpec],
debian_dir: VirtualPath,
binary_packages: Mapping[str, BinaryPackage],
*,
allow_fuzzy_matches: bool = False,
+ detect_typos: bool = False,
) -> Dict[str, PerPackagePackagerProvidedResult]:
main_binary_package = [
p.name for p in binary_packages.values() if p.is_main_package
@@ -274,10 +370,16 @@ def detect_all_packager_provided_files(
n: {} for n in binary_packages
}
max_periods_in_package_name = max(name.count(".") for name in binary_packages)
+ if detect_typos and CAN_DETECT_TYPOS:
+ period2stems = _period_stem(packager_provided_files.keys())
+ else:
+ period2stems = {}
for entry in debian_dir.iterdir:
if entry.is_dir:
continue
+ if entry.name in _KNOWN_NON_PPFS:
+ continue
matching_ppfs = _split_path(
packager_provided_files,
binary_packages,
@@ -285,6 +387,7 @@ def detect_all_packager_provided_files(
max_periods_in_package_name,
entry,
allow_fuzzy_matches=allow_fuzzy_matches,
+ period2stems=period2stems,
)
for packager_provided_file in matching_ppfs:
provided_files_for_package = provided_files[
diff --git a/src/debputy/plugin/api/impl_types.py b/src/debputy/plugin/api/impl_types.py
index 99f589c..d62ccbc 100644
--- a/src/debputy/plugin/api/impl_types.py
+++ b/src/debputy/plugin/api/impl_types.py
@@ -188,6 +188,7 @@ class PackagerProvidedFileClassSpec:
)
reference_documentation: Optional[PackagerProvidedFileReferenceDocumentation] = None
bug_950723: bool = False
+ has_active_command: bool = True
@property
def supports_priority(self) -> bool:
@@ -1155,6 +1156,7 @@ class KnownPackagingFileInfo(DebputyParsedContent):
default_priority: NotRequired[int]
post_formatting_rewrite: NotRequired[Literal["period-to-underscore"]]
packageless_is_fallback_for_all_packages: NotRequired[bool]
+ has_active_command: NotRequired[bool]
@dataclasses.dataclass(slots=True)
diff --git a/src/debputy/util.py b/src/debputy/util.py
index 01ffaa0..860280e 100644
--- a/src/debputy/util.py
+++ b/src/debputy/util.py
@@ -35,6 +35,39 @@ from debian.deb822 import Deb822
from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable
from debputy.exceptions import DebputySubstitutionError
+
+try:
+ from Levenshtein import distance
+except ImportError:
+
+ CAN_DETECT_TYPOS = False
+
+ def detect_possible_typo(
+ provided_value: str,
+ known_values: Iterable[str],
+ ) -> Sequence[str]:
+ return ()
+
+else:
+
+ CAN_DETECT_TYPOS = True
+
+ def detect_possible_typo(
+ provided_value: str,
+ known_values: Iterable[str],
+ ) -> Sequence[str]:
+ k_len = len(provided_value)
+ candidates = []
+ for known_value in known_values:
+ if abs(k_len - len(known_value)) > 2:
+ continue
+ d = distance(provided_value, known_value)
+ if d > 2:
+ continue
+ candidates.append(known_value)
+ return candidates
+
+
if TYPE_CHECKING:
from debputy.packages import BinaryPackage
from debputy.substitution import Substitution