diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-08-26 10:22:27 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-08-26 10:22:27 +0000 |
commit | 1432689ccec99c30eaf509690bb98549b602fbe9 (patch) | |
tree | 62ec1e027617df8761ee9ec9be4b75474f476350 | |
parent | Adding debian version 0.1.43. (diff) | |
download | debputy-1432689ccec99c30eaf509690bb98549b602fbe9.tar.xz debputy-1432689ccec99c30eaf509690bb98549b602fbe9.zip |
Merging upstream version 0.1.44.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
26 files changed, 815 insertions, 224 deletions
diff --git a/src/debputy/builtin_manifest_rules.py b/src/debputy/builtin_manifest_rules.py index c8e6557..e420cda 100644 --- a/src/debputy/builtin_manifest_rules.py +++ b/src/debputy/builtin_manifest_rules.py @@ -226,10 +226,7 @@ def builtin_mode_normalization_rules( path_type=PathType.FILE, recursive_match=True, ), - SymbolicMode.parse_filesystem_mode( - "a-x", - attribute_path['"*.pm'], - ), + _STD_FILE_MODE, ) for perl_dir in perl_module_dirs(dpkg_architecture_variables, dctrl_bin) ) @@ -241,10 +238,7 @@ def builtin_mode_normalization_rules( path_type=PathType.FILE, recursive_match=True, ), - SymbolicMode.parse_filesystem_mode( - "a-w", - attribute_path['"*.ali"'], - ), + OctalMode(0o444), ) yield ( diff --git a/src/debputy/commands/debputy_cmd/__main__.py b/src/debputy/commands/debputy_cmd/__main__.py index 27d52ca..3270737 100644 --- a/src/debputy/commands/debputy_cmd/__main__.py +++ b/src/debputy/commands/debputy_cmd/__main__.py @@ -660,9 +660,6 @@ def _dh_integration_generate_debs(context: CommandContext) -> None: if parsed_args.debug_mode: log_level = logging.INFO if log_level is not None: - _warn( - f"LOG LEVEL: {log_level} -- {logging.WARNING} -- {PRINT_COMMAND} -- {logging.INFO}" - ) change_log_level(log_level) integration_mode = context.resolve_integration_mode() is_dh_rrr_only_mode = integration_mode == INTEGRATION_MODE_DH_DEBPUTY_RRR diff --git a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py index 46b536b..eaab750 100644 --- a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py +++ b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py @@ -22,21 +22,22 @@ _EDITOR_SNIPPETS = { (add-to-list 'auto-mode-alist '("/debian/debputy.manifest\\'" . yaml-mode)) ;; Inform eglot about the debputy LSP (with-eval-after-load 'eglot - (add-to-list 'eglot-server-programs - '(debian-control-mode . ("debputy" "lsp" "server" "--ignore-language-ids"))) - (add-to-list 'eglot-server-programs - '(debian-changelog-mode . ("debputy" "lsp" "server" "--ignore-language-ids"))) - (add-to-list 'eglot-server-programs - '(debian-copyright-mode . ("debputy" "lsp" "server" "--ignore-language-ids"))) - ;; Requires elpa-dpkg-dev-el (>= 37.12) - (add-to-list 'eglot-server-programs - '(debian-autopkgtest-control-mode . ("debputy" "lsp" "server" "--ignore-language-ids"))) - ;; The debian/rules file uses the qmake mode. - (add-to-list 'eglot-server-programs - '(makefile-gmake-mode . ("debputy" "lsp" "server" "--ignore-language-ids"))) - (add-to-list 'eglot-server-programs - '(yaml-mode . ("debputy" "lsp" "server" "--ignore-language-ids"))) - ) + (add-to-list 'eglot-server-programs + '( + ( + ;; Requires elpa-dpkg-dev-el (>= 37.12) + (debian-autopkgtest-control-mode :language-id "debian/tests/control") + ;; Requires elpa-dpkg-dev-el + (debian-control-mode :language-id "debian/control") + (debian-changelog-mode :language-id "debian/changelog") + (debian-copyright-mode :language-id "debian/copyright") + ;; No language id for these atm. + makefile-gmake-mode + ;; Requires elpa-yaml-mode + yaml-mode + ) + . ("debputy" "lsp" "server") + ))) ;; Auto-start eglot for the relevant modes. (add-hook 'debian-control-mode-hook 'eglot-ensure) @@ -182,7 +183,7 @@ def lsp_server_cmd(context: CommandContext) -> None: debputy_language_server.dctrl_parser = context.dctrl_parser debputy_language_server.trust_language_ids = parsed_args.trust_language_ids - debputy_language_server.finish_initialization() + debputy_language_server.finish_startup_initialization() if parsed_args.tcp and parsed_args.ws: _error("Sorry, --tcp and --ws are mutually exclusive") diff --git a/src/debputy/highlevel_manifest_parser.py b/src/debputy/highlevel_manifest_parser.py index dd97d58..b7a4600 100644 --- a/src/debputy/highlevel_manifest_parser.py +++ b/src/debputy/highlevel_manifest_parser.py @@ -214,7 +214,7 @@ class HighLevelManifestParser(ParserContextData): if not self.substitution.is_used(var): raise ManifestParseException( f'The variable "{var}" is unused. Either use it or remove it.' - f" The variable was declared at {attribute_path.path}." + f" The variable was declared at {attribute_path.path_key_lc}." ) if isinstance(self, YAMLManifestParser) and self._mutable_yaml_manifest is None: self._mutable_yaml_manifest = MutableYAMLManifest.empty_manifest() @@ -451,7 +451,7 @@ class YAMLManifestParser(HighLevelManifestParser): return v def from_yaml_dict(self, yaml_data: object) -> "HighLevelManifest": - attribute_path = AttributePath.root_path() + attribute_path = AttributePath.root_path(yaml_data) parser_generator = self._plugin_provided_feature_set.manifest_parser_generator dispatchable_object_parsers = parser_generator.dispatchable_object_parsers manifest_root_parser = dispatchable_object_parsers[OPARSER_MANIFEST_ROOT] diff --git a/src/debputy/linting/lint_impl.py b/src/debputy/linting/lint_impl.py index ddafd4c..ddc7e93 100644 --- a/src/debputy/linting/lint_impl.py +++ b/src/debputy/linting/lint_impl.py @@ -6,18 +6,6 @@ import sys import textwrap from typing import Optional, List, Union, NoReturn, Mapping -from debputy.lsprotocol.types import ( - CodeAction, - Command, - CodeActionParams, - CodeActionContext, - TextDocumentIdentifier, - TextEdit, - Position, - DiagnosticSeverity, - Diagnostic, -) - from debputy.commands.debputy_cmd.context import CommandContext from debputy.commands.debputy_cmd.output import _output_styling, OutputStylingBase from debputy.filesystem_scan import FSROOverlay @@ -45,13 +33,13 @@ from debputy.lsp.lsp_debian_tests_control import ( _lint_debian_tests_control, _reformat_debian_tests_control, ) -from debputy.lsp.quickfixes import provide_standard_quickfixes_from_diagnostics -from debputy.lsp.spellchecking import disable_spellchecking -from debputy.lsp.style_prefs import ( - StylePreferenceTable, +from debputy.lsp.maint_prefs import ( + MaintainerPreferenceTable, EffectivePreference, - determine_effective_style, + determine_effective_preference, ) +from debputy.lsp.quickfixes import provide_standard_quickfixes_from_diagnostics +from debputy.lsp.spellchecking import disable_spellchecking from debputy.lsp.text_edit import ( get_well_formatted_edit, merge_sort_text_edits, @@ -59,6 +47,17 @@ from debputy.lsp.text_edit import ( OverLappingTextEditException, ) from debputy.lsp.vendoring._deb822_repro import Deb822FileElement +from debputy.lsprotocol.types import ( + CodeAction, + Command, + CodeActionParams, + CodeActionContext, + TextDocumentIdentifier, + TextEdit, + Position, + DiagnosticSeverity, + Diagnostic, +) from debputy.packages import SourcePackage, BinaryPackage from debputy.plugin.api import VirtualPath from debputy.plugin.api.feature_set import PluginProvidedFeatureSet @@ -87,20 +86,21 @@ REFORMAT_FORMATS = { @dataclasses.dataclass(slots=True) class LintContext: plugin_feature_set: PluginProvidedFeatureSet - style_preference_table: StylePreferenceTable + maint_preference_table: MaintainerPreferenceTable source_root: Optional[VirtualPath] debian_dir: Optional[VirtualPath] parsed_deb822_file_content: Optional[Deb822FileElement] = None source_package: Optional[SourcePackage] = None binary_packages: Optional[Mapping[str, BinaryPackage]] = None effective_preference: Optional[EffectivePreference] = None + style_tool: Optional[str] = None unsupported_preference_reason: Optional[str] = None salsa_ci: Optional[CommentedMap] = None def state_for(self, path: str, content: str, lines: List[str]) -> LintStateImpl: return LintStateImpl( self.plugin_feature_set, - self.style_preference_table, + self.maint_preference_table, self.source_root, self.debian_dir, path, @@ -119,7 +119,7 @@ def gather_lint_info(context: CommandContext) -> LintContext: debian_dir = None lint_context = LintContext( context.load_plugins(), - StylePreferenceTable.load_styles(), + MaintainerPreferenceTable.load_preferences(), source_root, debian_dir, ) @@ -147,12 +147,13 @@ def gather_lint_info(context: CommandContext) -> LintContext: except YAMLError: break if source_package is not None or salsa_ci_map is not None: - pref, pref_reason = determine_effective_style( - lint_context.style_preference_table, + pref, tool, pref_reason = determine_effective_preference( + lint_context.maint_preference_table, source_package, salsa_ci_map, ) lint_context.effective_preference = pref + lint_context.style_tool = tool lint_context.unsupported_preference_reason = pref_reason return lint_context @@ -237,9 +238,9 @@ def perform_reformat( fo = _output_styling(context.parsed_args, sys.stdout) lint_context = gather_lint_info(context) if named_style is not None: - style = lint_context.style_preference_table.named_styles.get(named_style) + style = lint_context.maint_preference_table.named_styles.get(named_style) if style is None: - styles = ", ".join(lint_context.style_preference_table.named_styles) + styles = ", ".join(lint_context.maint_preference_table.named_styles) _error(f'There is no style named "{style}". Options include: {styles}') if ( lint_context.effective_preference is not None @@ -257,10 +258,15 @@ def perform_reformat( "While `debputy` could identify a formatting for this package, it does not support it." ) _warn(f"{lint_context.unsupported_preference_reason}") + if lint_context.style_tool is not None: + _info( + f"The following tool might be able to apply the style: {lint_context.style_tool}" + ) if parsed_args.supported_style_required: _error( "Sorry; `debputy` does not support the style. Use --unknown-or-unsupported-style-is-ok to make" - " this a non-error." + " this a non-error (note that `debputy` will not reformat the packaging in this case; just not" + " exit with an error code)." ) else: print( @@ -293,6 +299,11 @@ def perform_reformat( ) ) if parsed_args.supported_style_required: + if lint_context.style_tool is not None: + _error( + "Sorry, `debputy reformat` does not support the packaging style. However, the" + f" formatting is supposedly handled by: {lint_context.style_tool}" + ) _error( "Sorry; `debputy` does not know which style to use for this package. Please either set a" "style or use --unknown-or-unsupported-style-is-ok to make this a non-error" diff --git a/src/debputy/linting/lint_util.py b/src/debputy/linting/lint_util.py index 1ed881c..6346508 100644 --- a/src/debputy/linting/lint_util.py +++ b/src/debputy/linting/lint_util.py @@ -41,8 +41,8 @@ from debputy.util import _warn if TYPE_CHECKING: from debputy.lsp.text_util import LintCapablePositionCodec - from debputy.lsp.style_prefs import ( - StylePreferenceTable, + from debputy.lsp.maint_prefs import ( + MaintainerPreferenceTable, EffectivePreference, ) @@ -110,7 +110,7 @@ class LintState: raise NotImplementedError @property - def style_preference_table(self) -> "StylePreferenceTable": + def maint_preference_table(self) -> "MaintainerPreferenceTable": raise NotImplementedError @property @@ -129,7 +129,7 @@ class LintState: @dataclasses.dataclass(slots=True) class LintStateImpl(LintState): plugin_feature_set: PluginProvidedFeatureSet = dataclasses.field(repr=False) - style_preference_table: "StylePreferenceTable" = dataclasses.field(repr=False) + maint_preference_table: "MaintainerPreferenceTable" = dataclasses.field(repr=False) source_root: Optional[VirtualPathBase] debian_dir: Optional[VirtualPathBase] path: str diff --git a/src/debputy/lsp/apt_cache.py b/src/debputy/lsp/apt_cache.py new file mode 100644 index 0000000..45988a7 --- /dev/null +++ b/src/debputy/lsp/apt_cache.py @@ -0,0 +1,167 @@ +import asyncio +import dataclasses +import subprocess +import sys +from collections import defaultdict +from typing import Literal, Optional, Sequence, Iterable, Mapping + +from debian.deb822 import Deb822 +from debian.debian_support import Version + +AptCacheState = Literal[ + "not-loaded", + "loading", + "loaded", + "failed", + "tooling-not-available", + "empty-cache", +] + + +@dataclasses.dataclass(slots=True) +class PackageInformation: + name: str + architecture: str + version: Version + multi_arch: str + # suites: Sequence[Tuple[str, ...]] + synopsis: str + section: str + provides: Optional[str] + upstream_homepage: Optional[str] + + +@dataclasses.dataclass(slots=True, frozen=True) +class PackageLookup: + name: str + package: Optional[PackageInformation] + provided_by: Sequence[PackageInformation] + + +class AptCache: + + def __init__(self) -> None: + self._state: AptCacheState = "not-loaded" + self._load_error: Optional[str] = None + self._lookups: Mapping[str, PackageLookup] = {} + + @property + def state(self) -> AptCacheState: + return self._state + + @property + def load_error(self) -> Optional[str]: + return self._load_error + + def lookup(self, name: str) -> Optional[PackageLookup]: + return self._lookups.get(name) + + async def load(self) -> None: + if self._state in ("loading", "loaded"): + raise RuntimeError(f"Already {self._state}") + self._load_error = None + self._state = "loading" + try: + files_raw = subprocess.check_output( + [ + "apt-get", + "indextargets", + "--format", + "$(IDENTIFIER)\x1f$(FILENAME)", + ] + ).decode("utf-8") + except FileNotFoundError: + self._state = "tooling-not-available" + self._load_error = "apt-get not available in PATH" + return + except subprocess.CalledProcessError as e: + self._state = "failed" + self._load_error = f"apt-get exited with {e.returncode}" + return + packages = {} + for raw_file_line in files_raw.split("\n"): + if not raw_file_line or raw_file_line.isspace(): + continue + identifier, filename = raw_file_line.split("\x1f") + if identifier not in ("Packages",): + continue + try: + for package_info in parse_apt_file(filename): + # Let other computations happen if needed. + await asyncio.sleep(0) + existing = packages.get(package_info.name) + if existing and package_info.version < existing.version: + continue + packages[package_info.name] = package_info + except FileNotFoundError: + self._state = "tooling-not-available" + self._load_error = "/usr/lib/apt/apt-helper not available" + return + except (AttributeError, RuntimeError, IndexError) as e: + self._state = "failed" + self._load_error = str(e) + return + provides = defaultdict(list) + for package_info in packages.values(): + if not package_info.provides: + continue + # Some packages (`debhelper`) provides the same package multiple times (`debhelper-compat`). + # Normalize that into one. + deps = { + clause.split("(")[0].strip() + for clause in package_info.provides.split(",") + } + for dep in sorted(deps): + provides[dep].append(package_info) + + self._lookups = { + name: PackageLookup( + name, + packages.get(name), + tuple(provides.get(name, [])), + ) + for name in packages.keys() | provides.keys() + } + self._state = "loaded" + + +def parse_apt_file(filename: str) -> Iterable[PackageInformation]: + proc = subprocess.Popen( + ["/usr/lib/apt/apt-helper", "cat-file", filename], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + ) + with proc: + for stanza in Deb822.iter_paragraphs(proc.stdout): + pkg_info = stanza_to_package_info(stanza) + if pkg_info is not None: + yield pkg_info + + +def stanza_to_package_info(stanza: Deb822) -> Optional[PackageInformation]: + try: + name = stanza["Package"] + architecture = sys.intern(stanza["Architecture"]) + version = Version(stanza["Version"]) + multi_arch = sys.intern(stanza.get("Multi-Arch", "no")) + synopsis = stanza["Description"] + section = sys.intern(stanza["Section"]) + provides = stanza.get("Provides") + homepage = stanza.get("Homepage") + except KeyError: + return None + if "\n" in synopsis: + # "Modern" Packages files do not have the full description. But in case we see a (very old one) + # have consistent behavior with the modern ones. + synopsis = synopsis.split("\n")[0] + + return PackageInformation( + name, + architecture, + version, + multi_arch, + synopsis, + section, + provides, + homepage, + ) diff --git a/src/debputy/lsp/debputy_ls.py b/src/debputy/lsp/debputy_ls.py index eb4162f..a1475b2 100644 --- a/src/debputy/lsp/debputy_ls.py +++ b/src/debputy/lsp/debputy_ls.py @@ -1,5 +1,6 @@ import dataclasses import os +import time from typing import ( Optional, List, @@ -17,19 +18,19 @@ from debputy.dh.dh_assistant import ( DhSequencerData, extract_dh_addons_from_control, ) -from debputy.lsprotocol.types import MarkupKind - from debputy.filesystem_scan import FSROOverlay, VirtualPathBase from debputy.linting.lint_util import ( LintState, ) -from debputy.lsp.style_prefs import ( - StylePreferenceTable, +from debputy.lsp.apt_cache import AptCache +from debputy.lsp.maint_prefs import ( + MaintainerPreferenceTable, MaintainerPreference, - determine_effective_style, + determine_effective_preference, ) from debputy.lsp.text_util import LintCapablePositionCodec from debputy.lsp.vendoring._deb822_repro import Deb822FileElement, parse_deb822_file +from debputy.lsprotocol.types import MarkupKind from debputy.packages import ( SourcePackage, BinaryPackage, @@ -301,16 +302,16 @@ class LSProvidedLintState(LintState): salsa_ci = self._resolve_salsa_ci() if source_package is None and salsa_ci is None: return None - style, _ = determine_effective_style( - self.style_preference_table, + style, _, _ = determine_effective_preference( + self.maint_preference_table, source_package, salsa_ci, ) return style @property - def style_preference_table(self) -> StylePreferenceTable: - return self._ls.style_preferences + def maint_preference_table(self) -> MaintainerPreferenceTable: + return self._ls.maint_preferences @property def salsa_ci(self) -> Optional[CommentedMap]: @@ -360,20 +361,42 @@ class DebputyLanguageServer(LanguageServer): self._plugin_feature_set: Optional[PluginProvidedFeatureSet] = None self._trust_language_ids: Optional[bool] = None self._finished_initialization = False - self.style_preferences = StylePreferenceTable({}, {}) + self.maint_preferences = MaintainerPreferenceTable({}, {}) + self.apt_cache = AptCache() + self.background_tasks = set() - def finish_initialization(self) -> None: + def finish_startup_initialization(self) -> None: if self._finished_initialization: return assert self._dctrl_parser is not None assert self._plugin_feature_set is not None assert self._trust_language_ids is not None - self.style_preferences = self.style_preferences.load_styles() + self.maint_preferences = self.maint_preferences.load_preferences() _info( - f"Loaded style preferences: {len(self.style_preferences.maintainer_preferences)} unique maintainer preferences recorded" + f"Loaded style preferences: {len(self.maint_preferences.maintainer_preferences)} unique maintainer preferences recorded" ) self._finished_initialization = True + async def on_initialize(self) -> None: + task = self.loop.create_task(self._load_apt_cache(), name="Index apt cache") + self.background_tasks.add(task) + task.add_done_callback(self.background_tasks.discard) + + def shutdown(self) -> None: + for task in self.background_tasks: + _info(f"Cancelling task: {task.get_name()}") + self.loop.call_soon_threadsafe(task.cancel) + return super().shutdown() + + async def _load_apt_cache(self) -> None: + _info("Starting load of apt cache data") + start = time.time() + await self.apt_cache.load() + end = time.time() + _info( + f"Loading apt cache finished after {end-start} seconds and is now in state {self.apt_cache.state}" + ) + @property def plugin_feature_set(self) -> PluginProvidedFeatureSet: res = self._plugin_feature_set diff --git a/src/debputy/lsp/lsp_debian_control.py b/src/debputy/lsp/lsp_debian_control.py index 5a72222..2b8f9b0 100644 --- a/src/debputy/lsp/lsp_debian_control.py +++ b/src/debputy/lsp/lsp_debian_control.py @@ -2,6 +2,7 @@ import dataclasses import os.path import re import textwrap +from itertools import chain from typing import ( Union, Sequence, @@ -17,6 +18,7 @@ from debputy.analysis.analysis_util import flatten_ppfs from debputy.analysis.debian_dir import resolve_debhelper_config_files from debputy.dh.dh_assistant import extract_dh_compat_level from debputy.linting.lint_util import LintState +from debputy.lsp.apt_cache import PackageLookup from debputy.lsp.debputy_ls import DebputyLanguageServer from debputy.lsp.diagnostics import DiagnosticData from debputy.lsp.lsp_debian_control_reference_data import ( @@ -27,6 +29,7 @@ from debputy.lsp.lsp_debian_control_reference_data import ( package_name_to_section, all_package_relationship_fields, extract_first_value_and_position, + all_source_relationship_fields, ) from debputy.lsp.lsp_features import ( lint_diagnostics, @@ -99,7 +102,7 @@ from debputy.packager_provided_files import ( detect_all_packager_provided_files, ) from debputy.plugin.api.impl import plugin_metadata_for_debputys_own_plugin -from debputy.util import detect_possible_typo +from debputy.util import detect_possible_typo, PKGNAME_REGEX, _info try: from debputy.lsp.vendoring._deb822_repro.locatable import ( @@ -311,58 +314,12 @@ def _debian_control_hover( return deb822_hover(ls, params, _DCTRL_FILE_METADATA, custom_handler=_custom_hover) -def _custom_hover( - server_position: Position, - _current_field: Optional[str], +def _custom_hover_description( + _ls: "DebputyLanguageServer", + _known_field: DctrlKnownField, + line: str, _word_at_position: str, - known_field: Optional[DctrlKnownField], - in_value: bool, - _doc: "TextDocument", - lines: List[str], ) -> Optional[Union[Hover, str]]: - if not in_value: - return None - - line_no = server_position.line - line = lines[line_no] - substvar_search_ref = server_position.character - substvar = "" - try: - if line and line[substvar_search_ref] in ("$", "{"): - substvar_search_ref += 2 - substvar_start = line.rindex("${", 0, substvar_search_ref) - substvar_end = line.index("}", substvar_start) - if server_position.character <= substvar_end: - substvar = line[substvar_start : substvar_end + 1] - except (ValueError, IndexError): - pass - - if substvar == "${}" or _SUBSTVAR_RE.fullmatch(substvar): - substvar_md = _SUBSTVARS_DOC.get(substvar) - - computed_doc = "" - for_field = relationship_substvar_for_field(substvar) - if for_field: - # Leading empty line is intentional! - computed_doc = textwrap.dedent( - f""" - This substvar is a relationship substvar for the field {for_field}. - Relationship substvars are automatically added in the field they - are named after in `debhelper-compat (= 14)` or later, or with - `debputy` (any integration mode after 0.1.21). - """ - ) - - if substvar_md is None: - doc = f"No documentation for {substvar}.\n" - md_fields = "" - else: - doc = substvar_md.description - md_fields = "\n" + substvar_md.render_metadata_fields() - return f"# Substvar `{substvar}`\n\n{doc}{computed_doc}{md_fields}" - - if known_field is None or known_field.name != "Description": - return None if line[0].isspace(): return None try: @@ -405,7 +362,7 @@ def _custom_hover( package name already and it generally does not help the user understand what they are looking at. * In many situations, the user will only see the package name - and its synopsis. The synopsis must standalone. + and its synopsis. The synopsis must be able to stand alone. **Example renderings in various terminal UIs**: ``` @@ -450,6 +407,240 @@ def _custom_hover( ) +def _render_package_lookup( + package_lookup: PackageLookup, + known_field: DctrlKnownField, +) -> str: + name = package_lookup.name + provider = package_lookup.package + if package_lookup.package is None and len(package_lookup.provided_by) == 1: + provider = package_lookup.provided_by[0] + + if provider: + segments = [ + f"# {name} ({provider.version}, {provider.architecture}) ", + "", + ] + + if ( + _is_bd_field(known_field) + and name.startswith("dh-sequence-") + and len(name) > 12 + ): + sequence = name[12:] + segments.append( + f"This build-dependency will activate the `dh` sequence called `{sequence}`." + ) + segments.append("") + segments.extend( + [ + f"Synopsis: {provider.synopsis}", + f"Multi-Arch: {provider.multi_arch}", + f"Section: {provider.section}", + ] + ) + if provider.upstream_homepage is not None: + segments.append(f"Upstream homepage: {provider.upstream_homepage}") + segments.append("") + segments.append( + "Data is from the system's APT cache, which may not match the target distribution." + ) + return "\n".join(segments) + + segments = [ + f"# {name} [virtual]", + "", + "The package {name} is a virtual package provided by one of:", + ] + segments.extend(f" * {p.name}" for p in package_lookup.provided_by) + segments.append("") + segments.append( + "Data is from the system's APT cache, which may not match the target distribution." + ) + return "\n".join(segments) + + +def _disclaimer(is_empty: bool) -> str: + if is_empty: + return textwrap.dedent( + """\ + The system's APT cache is empty, so it was not possible to verify that the + package exist. +""" + ) + return textwrap.dedent( + """\ + The package is not known by the APT cache on this system, so there may be typo + or the package may not be available in the version of your distribution. +""" + ) + + +def _render_package_by_name( + name: str, known_field: DctrlKnownField, is_empty: bool +) -> Optional[str]: + if _is_bd_field(known_field) and name.startswith("dh-sequence-") and len(name) > 12: + sequence = name[12:] + return ( + textwrap.dedent( + f"""\ + # {name} + + This build-dependency will activate the `dh` sequence called `{sequence}`. + + """ + ) + + _disclaimer(is_empty) + ) + return ( + textwrap.dedent( + f"""\ + # {name} + + """ + ) + + _disclaimer(is_empty) + ) + + +def _is_bd_field(known_field: DctrlKnownField) -> bool: + return known_field.name in ( + "Build-Depends", + "Build-Depends-Arch", + "Build-Depends-Indep", + ) + + +def _custom_hover_relationship_field( + ls: "DebputyLanguageServer", + known_field: DctrlKnownField, + _line: str, + word_at_position: str, +) -> Optional[Union[Hover, str]]: + apt_cache = ls.apt_cache + state = apt_cache.state + is_empty = False + _info(f"Rel field: {known_field.name} - {word_at_position} - {state}") + if "|" in word_at_position: + return textwrap.dedent( + f"""\ + Sorry, no hover docs for OR relations at the moment. + + The relation being matched: `{word_at_position}` + + The code is missing logic to determine which side of the OR the lookup is happening. + """ + ) + match = next(iter(PKGNAME_REGEX.finditer(word_at_position)), None) + if match is None: + return + package = match.group() + if state == "empty-cache": + state = "loaded" + is_empty = True + if state == "loaded": + result = apt_cache.lookup(package) + if result is None: + return _render_package_by_name( + package, + known_field, + is_empty=is_empty, + ) + return _render_package_lookup(result, known_field) + + if state in ( + "not-loaded", + "failed", + "tooling-not-available", + ): + details = apt_cache.load_error if apt_cache.load_error else "N/A" + return textwrap.dedent( + f"""\ + Sorry, the APT cache data is not available due to an error or missing tool. + + Details: {details} + """ + ) + + if state == "empty-cache": + return f"Cannot lookup {package}: APT cache data was empty" + + if state == "loading": + return f"Cannot lookup {package}: APT cache data is still being indexed. Please try again in a moment." + return None + + +_CUSTOM_FIELD_HOVER = { + field: _custom_hover_relationship_field + for field in chain( + all_package_relationship_fields().values(), + all_source_relationship_fields().values(), + ) + if field != "Provides" +} + +_CUSTOM_FIELD_HOVER["Description"] = _custom_hover_description + + +def _custom_hover( + ls: "DebputyLanguageServer", + server_position: Position, + _current_field: Optional[str], + word_at_position: str, + known_field: Optional[DctrlKnownField], + in_value: bool, + _doc: "TextDocument", + lines: List[str], +) -> Optional[Union[Hover, str]]: + if not in_value: + return None + + line_no = server_position.line + line = lines[line_no] + substvar_search_ref = server_position.character + substvar = "" + try: + if line and line[substvar_search_ref] in ("$", "{"): + substvar_search_ref += 2 + substvar_start = line.rindex("${", 0, substvar_search_ref) + substvar_end = line.index("}", substvar_start) + if server_position.character <= substvar_end: + substvar = line[substvar_start : substvar_end + 1] + except (ValueError, IndexError): + pass + + if substvar == "${}" or _SUBSTVAR_RE.fullmatch(substvar): + substvar_md = _SUBSTVARS_DOC.get(substvar) + + computed_doc = "" + for_field = relationship_substvar_for_field(substvar) + if for_field: + # Leading empty line is intentional! + computed_doc = textwrap.dedent( + f""" + This substvar is a relationship substvar for the field {for_field}. + Relationship substvars are automatically added in the field they + are named after in `debhelper-compat (= 14)` or later, or with + `debputy` (any integration mode after 0.1.21). + """ + ) + + if substvar_md is None: + doc = f"No documentation for {substvar}.\n" + md_fields = "" + else: + doc = substvar_md.description + md_fields = "\n" + substvar_md.render_metadata_fields() + return f"# Substvar `{substvar}`\n\n{doc}{computed_doc}{md_fields}" + + if known_field is None: + return None + dispatch = _CUSTOM_FIELD_HOVER.get(known_field.name) + if dispatch is None: + return None + return dispatch(ls, known_field, line, word_at_position) + + @lsp_completer(_LANGUAGE_IDS) def _debian_control_completions( ls: "DebputyLanguageServer", diff --git a/src/debputy/lsp/lsp_debian_control_reference_data.py b/src/debputy/lsp/lsp_debian_control_reference_data.py index 007c0dd..2ec885b 100644 --- a/src/debputy/lsp/lsp_debian_control_reference_data.py +++ b/src/debputy/lsp/lsp_debian_control_reference_data.py @@ -104,7 +104,7 @@ except ImportError: if TYPE_CHECKING: - from debputy.lsp.style_prefs import EffectivePreference + from debputy.lsp.maint_prefs import EffectivePreference F = TypeVar("F", bound="Deb822KnownField") diff --git a/src/debputy/lsp/lsp_debian_debputy_manifest.py b/src/debputy/lsp/lsp_debian_debputy_manifest.py index 15e9aa6..a8a2fdf 100644 --- a/src/debputy/lsp/lsp_debian_debputy_manifest.py +++ b/src/debputy/lsp/lsp_debian_debputy_manifest.py @@ -752,7 +752,6 @@ def debputy_manifest_completer( server_position = doc.position_codec.position_from_client_units( lines, params.position ) - attribute_root_path = AttributePath.root_path() orig_line = lines[server_position.line].rstrip() has_colon = ":" in orig_line added_key = _insert_snippet(lines, server_position) @@ -791,6 +790,7 @@ def debputy_manifest_completer( context = lines[server_position.line].replace("\n", "\\n") _info(f"Completion failed: parse error: Line in question: {context}") return None + attribute_root_path = AttributePath.root_path(content) m = _trace_cursor(content, attribute_root_path, server_position) if m is None: @@ -925,13 +925,13 @@ def debputy_manifest_hover( doc = ls.workspace.get_text_document(params.text_document.uri) lines = doc.lines position_codec = doc.position_codec - attribute_root_path = AttributePath.root_path() server_position = position_codec.position_from_client_units(lines, params.position) try: content = MANIFEST_YAML.load("".join(lines)) except YAMLError: return None + attribute_root_path = AttributePath.root_path(content) m = _trace_cursor(content, attribute_root_path, server_position) if m is None: _info("No match") diff --git a/src/debputy/lsp/lsp_dispatch.py b/src/debputy/lsp/lsp_dispatch.py index 27170d0..b5e61dc 100644 --- a/src/debputy/lsp/lsp_dispatch.py +++ b/src/debputy/lsp/lsp_dispatch.py @@ -54,6 +54,8 @@ from debputy.lsprotocol.types import ( TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, WillSaveTextDocumentParams, TEXT_DOCUMENT_FORMATTING, + INITIALIZE, + InitializeParams, ) _DOCUMENT_VERSION_TABLE: Dict[str, int] = {} @@ -94,6 +96,14 @@ def is_doc_at_version(uri: str, version: int) -> bool: return dv == version +@DEBPUTY_LANGUAGE_SERVER.feature(INITIALIZE) +async def _on_initialize( + ls: "DebputyLanguageServer", + _: InitializeParams, +) -> None: + await ls.on_initialize() + + @DEBPUTY_LANGUAGE_SERVER.feature(TEXT_DOCUMENT_DID_OPEN) async def _open_document( ls: "DebputyLanguageServer", diff --git a/src/debputy/lsp/lsp_generic_deb822.py b/src/debputy/lsp/lsp_generic_deb822.py index 895a3e0..4340abc 100644 --- a/src/debputy/lsp/lsp_generic_deb822.py +++ b/src/debputy/lsp/lsp_generic_deb822.py @@ -311,6 +311,7 @@ def deb822_hover( custom_handler: Optional[ Callable[ [ + "DebputyLanguageServer", Position, Optional[str], str, @@ -351,6 +352,7 @@ def deb822_hover( hover_text = None if custom_handler is not None: res = custom_handler( + ls, server_pos, current_field, word_at_position, diff --git a/src/debputy/lsp/lsp_self_check.py b/src/debputy/lsp/lsp_self_check.py index aa28a56..ec0c7e7 100644 --- a/src/debputy/lsp/lsp_self_check.py +++ b/src/debputy/lsp/lsp_self_check.py @@ -110,7 +110,7 @@ def spell_checking() -> bool: @lsp_generic_check( - feature_name="Extra dh support", + feature_name="extra dh support", problem="Missing dependencies", how_to_fix="Run `apt satisfy debhelper (>= 13.16~)` to enable this feature", ) @@ -135,6 +135,30 @@ def check_dh_version() -> bool: return Version(parts[0]) >= Version("13.16~") +@lsp_generic_check( + feature_name="apt cache packages", + problem="Missing apt or empty apt cache", + how_to_fix="", +) +def check_apt_cache() -> bool: + try: + output = subprocess.check_output( + [ + "apt-get", + "indextargets", + "--format", + "$(IDENTIFIER)", + ] + ).decode("utf-8") + except (FileNotFoundError, subprocess.CalledProcessError): + return False + for line in output.splitlines(): + if line.strip() == "Packages": + return True + + return False + + def assert_can_start_lsp() -> None: for self_check in LSP_CHECKS: if self_check.is_mandatory and not self_check.test(): diff --git a/src/debputy/lsp/style-preferences.yaml b/src/debputy/lsp/maint-preferences.yaml index 982f242..982f242 100644 --- a/src/debputy/lsp/style-preferences.yaml +++ b/src/debputy/lsp/maint-preferences.yaml diff --git a/src/debputy/lsp/style_prefs.py b/src/debputy/lsp/maint_prefs.py index 1bcd800..fa6315b 100644 --- a/src/debputy/lsp/style_prefs.py +++ b/src/debputy/lsp/maint_prefs.py @@ -23,14 +23,14 @@ from debputy.lsp.lsp_reference_keyword import ALL_PUBLIC_NAMED_STYLES from debputy.lsp.vendoring._deb822_repro.types import FormatterCallback from debputy.lsp.vendoring.wrap_and_sort import wrap_and_sort_formatter from debputy.packages import SourcePackage -from debputy.util import _error, _info +from debputy.util import _error from debputy.yaml import MANIFEST_YAML from debputy.yaml.compat import CommentedMap PT = TypeVar("PT", bool, str, int) -BUILTIN_STYLES = os.path.join(os.path.dirname(__file__), "style-preferences.yaml") +BUILTIN_STYLES = os.path.join(os.path.dirname(__file__), "maint-preferences.yaml") _NORMALISE_FIELD_CONTENT_KEY = ["formatting", "deb822", "normalize-field-content"] _UPLOADER_SPLIT_RE = re.compile(r"(?<=>)\s*,") @@ -458,7 +458,7 @@ class MaintainerPreference(EffectivePreference): return EffectivePreference(**fields) -class StylePreferenceTable: +class MaintainerPreferenceTable: def __init__( self, @@ -469,7 +469,7 @@ class StylePreferenceTable: self._maintainer_preferences = maintainer_preferences @classmethod - def load_styles(cls) -> Self: + def load_preferences(cls) -> Self: named_styles: Dict[str, EffectivePreference] = {} maintainer_preferences: Dict[str, MaintainerPreference] = {} with open(BUILTIN_STYLES) as fd: @@ -565,16 +565,16 @@ def extract_maint_email(maint: str) -> str: return maint[idx + 1 : -1] -def determine_effective_style( - style_preference_table: StylePreferenceTable, +def determine_effective_preference( + maint_preference_table: MaintainerPreferenceTable, source_package: Optional[SourcePackage], salsa_ci: Optional[CommentedMap], -) -> Tuple[Optional[EffectivePreference], Optional[str]]: +) -> Tuple[Optional[EffectivePreference], Optional[str], Optional[str]]: style = source_package.fields.get("X-Style") if source_package is not None else None if style is not None: if style not in ALL_PUBLIC_NAMED_STYLES: - return None, "X-Style contained an unknown/unsupported style" - return style_preference_table.named_styles.get(style), None + return None, None, "X-Style contained an unknown/unsupported style" + return maint_preference_table.named_styles.get(style), "debputy reformat", None if salsa_ci: disable_wrap_and_sort = salsa_ci.mlget( @@ -600,53 +600,73 @@ def determine_effective_style( if wrap_and_sort_options is None: wrap_and_sort_options = "" elif not isinstance(wrap_and_sort_options, str): - return None, "The salsa-ci had a non-string option for wrap-and-sort" + return ( + None, + None, + "The salsa-ci had a non-string option for wrap-and-sort", + ) detected_style = parse_salsa_ci_wrap_and_sort_args(wrap_and_sort_options) + tool_w_args = f"wrap-and-sort {wrap_and_sort_options}".strip() if detected_style is None: msg = "One or more of the wrap-and-sort options in the salsa-ci file was not supported" else: msg = None - return detected_style, msg + return detected_style, tool_w_args, msg if source_package is None: - return None, None + return None, None, None maint = source_package.fields.get("Maintainer") if maint is None: - return None, None + return None, None, None maint_email = extract_maint_email(maint) - maint_style = style_preference_table.maintainer_preferences.get(maint_email) + maint_style = maint_preference_table.maintainer_preferences.get(maint_email) # Special-case "@packages.debian.org" when missing, since they are likely to be "ad-hoc" # teams that will not be registered. In that case, we fall back to looking at the uploader # preferences as-if the maintainer had not been listed at all. if maint_style is None and not maint_email.endswith("@packages.debian.org"): - return None, None + return None, None, None if maint_style is not None and maint_style.is_packaging_team: # When the maintainer is registered as a packaging team, then we assume the packaging # team's style applies unconditionally. - return maint_style.as_effective_pref(), None + effective = maint_style.as_effective_pref() + tool_w_args = _guess_tool_from_style(maint_preference_table, effective) + return effective, tool_w_args, None uploaders = source_package.fields.get("Uploaders") if uploaders is None: detected_style = ( maint_style.as_effective_pref() if maint_style is not None else None ) - return detected_style, None + tool_w_args = _guess_tool_from_style(maint_preference_table, detected_style) + return detected_style, tool_w_args, None all_styles: List[Optional[EffectivePreference]] = [] if maint_style is not None: all_styles.append(maint_style) for uploader in _UPLOADER_SPLIT_RE.split(uploaders): uploader_email = extract_maint_email(uploader) - uploader_style = style_preference_table.maintainer_preferences.get( + uploader_style = maint_preference_table.maintainer_preferences.get( uploader_email ) all_styles.append(uploader_style) if not all_styles: - return None, None + return None, None, None r = functools.reduce(EffectivePreference.aligned_preference, all_styles) if isinstance(r, MaintainerPreference): - return r.as_effective_pref(), None - return r, None + r = r.as_effective_pref() + tool_w_args = _guess_tool_from_style(maint_preference_table, r) + return r, tool_w_args, None + + +def _guess_tool_from_style( + maint_preference_table: MaintainerPreferenceTable, + pref: Optional[EffectivePreference], +) -> Optional[str]: + if pref is None: + return None + if maint_preference_table.named_styles["black"] == pref: + return "debputy reformat" + return None def _split_options(args: Iterable[str]) -> Iterable[str]: diff --git a/src/debputy/manifest_parser/declarative_parser.py b/src/debputy/manifest_parser/declarative_parser.py index 4a32368..6cbbce3 100644 --- a/src/debputy/manifest_parser/declarative_parser.py +++ b/src/debputy/manifest_parser/declarative_parser.py @@ -349,16 +349,16 @@ class DeclarativeMappingInputParser(DeclarativeInputParser[TD], Generic[TD, SF]) if unused_keys: k = ", ".join(unused_keys) raise ManifestParseException( - f'Unknown keys "{unknown_keys}" at {path.path}". Keys that could be used here are: {k}.{doc_ref}' + f'Unknown keys "{unknown_keys}" at {path.path_container_lc}". Keys that could be used here are: {k}.{doc_ref}' ) raise ManifestParseException( - f'Unknown keys "{unknown_keys}" at {path.path}". Please remove them.{doc_ref}' + f'Unknown keys "{unknown_keys}" at {path.path_container_lc}". Please remove them.{doc_ref}' ) missing_keys = self.input_time_required_parameters - value.keys() if missing_keys: required = ", ".join(repr(k) for k in sorted(missing_keys)) raise ManifestParseException( - f"The following keys were required but not present at {path.path}: {required}{doc_ref}" + f"The following keys were required but not present at {path.path_container_lc}: {required}{doc_ref}" ) for maybe_required in self.all_parameters - value.keys(): attr = self.manifest_attributes[maybe_required] @@ -371,14 +371,14 @@ class DeclarativeMappingInputParser(DeclarativeInputParser[TD], Generic[TD, SF]) ): reason = attr.conditional_required.reason raise ManifestParseException( - f'Missing the *conditionally* required attribute "{maybe_required}" at {path.path}. {reason}{doc_ref}' + f'Missing the *conditionally* required attribute "{maybe_required}" at {path.path_container_lc}. {reason}{doc_ref}' ) for keyset in self.at_least_one_of: matched_keys = value.keys() & keyset if not matched_keys: conditionally_required = ", ".join(repr(k) for k in sorted(keyset)) raise ManifestParseException( - f"At least one of the following keys must be present at {path.path}:" + f"At least one of the following keys must be present at {path.path_container_lc}:" f" {conditionally_required}{doc_ref}" ) for group in self.mutually_exclusive_attributes: @@ -386,7 +386,7 @@ class DeclarativeMappingInputParser(DeclarativeInputParser[TD], Generic[TD, SF]) if len(matched) > 1: ck = ", ".join(repr(k) for k in sorted(matched)) raise ManifestParseException( - f"Could not parse {path.path}: The following attributes are" + f"Could not parse {path.path_container_lc}: The following attributes are" f" mutually exclusive: {ck}{doc_ref}" ) diff --git a/src/debputy/manifest_parser/util.py b/src/debputy/manifest_parser/util.py index a9cbbe8..bcaa617 100644 --- a/src/debputy/manifest_parser/util.py +++ b/src/debputy/manifest_parser/util.py @@ -14,8 +14,11 @@ from typing import ( TYPE_CHECKING, Iterable, Container, + Literal, ) +from debputy.yaml.compat import CommentedBase + from debputy.manifest_parser.exceptions import ManifestParseException if TYPE_CHECKING: @@ -28,26 +31,29 @@ StrOrInt = Union[str, int] AttributePathAliasMapping = Mapping[ StrOrInt, Tuple[StrOrInt, Optional["AttributePathAliasMapping"]] ] +LineReportKind = Literal["key", "value", "container"] -class AttributePath(object): - __slots__ = ("parent", "name", "alias_mapping", "path_hint") +class AttributePath: + __slots__ = ("parent", "container", "name", "alias_mapping", "path_hint") def __init__( self, parent: Optional["AttributePath"], key: Optional[Union[str, int]], *, + container: Optional[Any] = None, alias_mapping: Optional[AttributePathAliasMapping] = None, ) -> None: self.parent = parent + self.container = container self.name = key self.path_hint: Optional[str] = None self.alias_mapping = alias_mapping @classmethod - def root_path(cls) -> "AttributePath": - return AttributePath(None, None) + def root_path(cls, container: Optional[Any]) -> "AttributePath": + return AttributePath(None, None, container=container) @classmethod def builtin_path(cls) -> "AttributePath": @@ -70,8 +76,29 @@ class AttributePath(object): segments.reverse() yield from (s.name for s in segments) - @property - def path(self) -> str: + def _resolve_path(self, report_kind: LineReportKind) -> str: + parent = self.parent + key = self.name + if report_kind == "container": + key = parent.name if parent else None + parent = parent.parent if parent else None + container = parent.container if parent is not None else None + + if isinstance(container, CommentedBase): + lc = container.lc + try: + if isinstance(key, str): + if report_kind == "key": + lc_data = lc.key(key) + else: + lc_data = lc.value(key) + else: + lc_data = lc.item(key) + except (AttributeError, RuntimeError, LookupError): + lc_data = None + else: + lc_data = None + segments = list(self._iter_path()) segments.reverse() parts: List[str] = [] @@ -88,12 +115,31 @@ class AttributePath(object): if parts: parts.append(".") parts.append(k) - if path_hint: + + if lc_data is not None: + line_pos, col = lc_data + # Translate 0-based (index) to 1-based (line number) + line_pos += 1 + parts.append(f" [Line {line_pos} column {col}]") + + elif path_hint: parts.append(f" <Search for: {path_hint}>") if not parts: return "document root" return "".join(parts) + @property + def path_container_lc(self) -> str: + return self._resolve_path("container") + + @property + def path_key_lc(self) -> str: + return self._resolve_path("key") + + @property + def path(self) -> str: + return self._resolve_path("value") + def __str__(self) -> str: return self.path @@ -106,9 +152,25 @@ class AttributePath(object): if item == "": # Support `sources[0]` mapping to `source` by `sources -> source` and `0 -> ""`. return AttributePath( - self.parent, self.name, alias_mapping=alias_mapping + self.parent, + self.name, + alias_mapping=alias_mapping, + container=self.container, ) - return AttributePath(self, item, alias_mapping=alias_mapping) + container = self.container + if container is not None: + try: + child_container = self.container[item] + except (AttributeError, RuntimeError, LookupError): + child_container = None + else: + child_container = None + return AttributePath( + self, + item, + alias_mapping=alias_mapping, + container=child_container, + ) def _iter_path(self) -> Iterator["AttributePath"]: current = self diff --git a/src/debputy/plugin/api/impl.py b/src/debputy/plugin/api/impl.py index 6369663..b0674fb 100644 --- a/src/debputy/plugin/api/impl.py +++ b/src/debputy/plugin/api/impl.py @@ -1867,7 +1867,7 @@ def parse_json_plugin_desc( f" clash with the bundled plugin of same name." ) - attribute_path = AttributePath.root_path() + attribute_path = AttributePath.root_path(raw) try: plugin_json_metadata = PLUGIN_METADATA_PARSER.parse_input( diff --git a/src/debputy/plugin/api/impl_types.py b/src/debputy/plugin/api/impl_types.py index 77e96ea..1a9bfdf 100644 --- a/src/debputy/plugin/api/impl_types.py +++ b/src/debputy/plugin/api/impl_types.py @@ -587,11 +587,11 @@ class DispatchingObjectParser( ) if not isinstance(orig_value, dict): raise ManifestParseException( - f"The attribute {attribute_path.path} must be a non-empty mapping.{doc_ref}" + f"The attribute {attribute_path.path_container_lc} must be a non-empty mapping.{doc_ref}" ) if not orig_value: raise ManifestParseException( - f"The attribute {attribute_path.path} must be a non-empty mapping.{doc_ref}" + f"The attribute {attribute_path.path_container_lc} must be a non-empty mapping.{doc_ref}" ) result = {} unknown_keys = orig_value.keys() - self._parsers.keys() @@ -675,7 +675,7 @@ class InPackageContextParser( ) if not isinstance(orig_value, dict) or not orig_value: raise ManifestParseException( - f"The attribute {attribute_path.path} must be a non-empty mapping.{doc_ref}" + f"The attribute {attribute_path.path_container_lc} must be a non-empty mapping.{doc_ref}" ) delegate = self.delegate result = {} diff --git a/src/debputy/plugin/debputy/manifest_root_rules.py b/src/debputy/plugin/debputy/manifest_root_rules.py index 80a4799..1d3b096 100644 --- a/src/debputy/plugin/debputy/manifest_root_rules.py +++ b/src/debputy/plugin/debputy/manifest_root_rules.py @@ -212,13 +212,13 @@ def _handle_manifest_variables( key_path = variables_path[key] if not SUBST_VAR_RE.match("{{" + key + "}}"): raise ManifestParseException( - f"The variable at {key_path.path} has an invalid name and therefore cannot" + f"The variable at {key_path.path_key_lc} has an invalid name and therefore cannot" " be used." ) if substitution.variable_state(key) != VariableNameState.UNDEFINED: raise ManifestParseException( f'The variable "{key}" is already reserved/defined. Error triggered by' - f" {key_path.path}." + f" {key_path.path_key_lc}." ) try: value = substitution.substitute(value_raw, key_path.path) diff --git a/tests/lint_tests/lint_tutil.py b/tests/lint_tests/lint_tutil.py index 74e08db..c32ad3c 100644 --- a/tests/lint_tests/lint_tutil.py +++ b/tests/lint_tests/lint_tutil.py @@ -9,7 +9,7 @@ from debputy.linting.lint_util import ( LintStateImpl, LintState, ) -from debputy.lsp.style_prefs import StylePreferenceTable, EffectivePreference +from debputy.lsp.maint_prefs import MaintainerPreferenceTable, EffectivePreference from debputy.packages import DctrlParser from debputy.plugin.api.feature_set import PluginProvidedFeatureSet @@ -42,7 +42,7 @@ class LintWrapper: self.path = path self._dctrl_parser = dctrl_parser self.source_root: Optional[VirtualPathBase] = None - self.lint_style_preference_table = StylePreferenceTable({}, {}) + self.lint_maint_preference_table = MaintainerPreferenceTable({}, {}) self.effective_preference: Optional[EffectivePreference] = None def __call__(self, lines: List[str]) -> Optional[List["Diagnostic"]]: @@ -59,7 +59,7 @@ class LintWrapper: debian_dir = source_root.get("debian") if source_root is not None else None state = LintStateImpl( self._debputy_plugin_feature_set, - self.lint_style_preference_table, + self.lint_maint_preference_table, source_root, debian_dir, self.path, diff --git a/tests/test_fs_metadata.py b/tests/test_fs_metadata.py index 7dd3d55..3cbbb03 100644 --- a/tests/test_fs_metadata.py +++ b/tests/test_fs_metadata.py @@ -132,7 +132,9 @@ def test_mtime_clamp_and_builtin_dir_mode( verify_paths(intermediate_manifest, path_defs) -def test_transformations_create_symlink(manifest_parser_pkg_foo): +def test_transformations_create_symlink( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: content = textwrap.dedent( """\ manifest-version: '0.1' @@ -178,7 +180,9 @@ def test_transformations_create_symlink(manifest_parser_pkg_foo): verify_paths(intermediate_manifest, expected_results) -def test_transformations_create_symlink_replace_success(manifest_parser_pkg_foo): +def test_transformations_create_symlink_replace_success( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: content = textwrap.dedent( """\ manifest-version: '0.1' @@ -230,8 +234,10 @@ def test_transformations_create_symlink_replace_success(manifest_parser_pkg_foo) ], ) def test_transformations_create_symlink_replace_failure( - manifest_parser_pkg_foo, replacement_rule, reason -): + manifest_parser_pkg_foo: YAMLManifestParser, + replacement_rule: str, + reason: str, +) -> None: content = textwrap.dedent( f"""\ manifest-version: '0.1' @@ -257,14 +263,15 @@ def test_transformations_create_symlink_replace_failure( f"Refusing to replace ./usr/share/foo with a symlink; {reason} and the active" f" replacement-rule was {replacement_rule}. You can set the replacement-rule to" ' "discard-existing", if you are not interested in the contents of ./usr/share/foo. This error' - " was triggered by packages.foo.transformations[0].create-symlink <Search for: usr/share/foo>." + # Ideally, this would be reported for line 5. + " was triggered by packages.foo.transformations[0].create-symlink [Line 6 column 18]." ) assert e_info.value.args[0] == msg def test_transformations_create_symlink_replace_with_explicit_remove( - manifest_parser_pkg_foo, -): + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: content = textwrap.dedent( """\ manifest-version: '0.1' @@ -299,8 +306,8 @@ def test_transformations_create_symlink_replace_with_explicit_remove( def test_transformations_create_symlink_replace_with_replacement_rule( - manifest_parser_pkg_foo, -): + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: content = textwrap.dedent( """\ manifest-version: '0.1' @@ -335,7 +342,9 @@ def test_transformations_create_symlink_replace_with_replacement_rule( verify_paths(intermediate_manifest, expected_results) -def test_transformations_path_metadata(manifest_parser_pkg_foo): +def test_transformations_path_metadata( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: content = textwrap.dedent( """\ manifest-version: '0.1' @@ -382,7 +391,9 @@ def test_transformations_path_metadata(manifest_parser_pkg_foo): verify_paths(intermediate_manifest, expected_results) -def test_transformations_directories(manifest_parser_pkg_foo): +def test_transformations_directories( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: content = textwrap.dedent( """\ manifest-version: '0.1' @@ -453,7 +464,9 @@ def test_transformations_directories(manifest_parser_pkg_foo): verify_paths(intermediate_manifest, expected_results) -def test_transformation_remove(manifest_parser_pkg_foo): +def test_transformation_remove( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: content = textwrap.dedent( """\ manifest-version: '0.1' @@ -489,7 +502,9 @@ def test_transformation_remove(manifest_parser_pkg_foo): verify_paths(intermediate_manifest, expected_results) -def test_transformation_remove_keep_empty(manifest_parser_pkg_foo): +def test_transformation_remove_keep_empty( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: content = textwrap.dedent( """\ manifest-version: '0.1' @@ -529,7 +544,9 @@ def test_transformation_remove_keep_empty(manifest_parser_pkg_foo): verify_paths(intermediate_manifest, expected_results) -def test_transformation_remove_glob(manifest_parser_pkg_foo): +def test_transformation_remove_glob( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: content = textwrap.dedent( """\ manifest-version: '0.1' @@ -581,7 +598,9 @@ def test_transformation_remove_glob(manifest_parser_pkg_foo): verify_paths(intermediate_manifest, expected_results) -def test_transformation_remove_no_match(manifest_parser_pkg_foo): +def test_transformation_remove_no_match( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: content = textwrap.dedent( """\ manifest-version: '0.1' @@ -612,13 +631,15 @@ def test_transformation_remove_no_match(manifest_parser_pkg_foo): manifest.apply_to_binary_staging_directory("foo", fs_root, claim_mtime_to) expected = ( 'The match rule "./some/non-existing-path" in transformation' - ' "packages.foo.transformations[0].remove <Search for: some/non-existing-path>" did not match any paths. Either' + ' "packages.foo.transformations[0].remove [Line 5 column 18]" did not match any paths. Either' " the definition is redundant (and can be omitted) or the match rule is incorrect." ) assert expected == e_info.value.args[0] -def test_transformation_move_basic(manifest_parser_pkg_foo): +def test_transformation_move_basic( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: content = textwrap.dedent( """\ manifest-version: '0.1' @@ -682,7 +703,9 @@ def test_transformation_move_basic(manifest_parser_pkg_foo): verify_paths(intermediate_manifest, expected_results) -def test_transformation_move_no_match(manifest_parser_pkg_foo): +def test_transformation_move_no_match( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: content = textwrap.dedent( """\ manifest-version: '0.1' @@ -715,13 +738,15 @@ def test_transformation_move_no_match(manifest_parser_pkg_foo): manifest.apply_to_binary_staging_directory("foo", fs_root, claim_mtime_to) expected = ( 'The match rule "./some/non-existing-path" in transformation' - ' "packages.foo.transformations[0].move <Search for: some/non-existing-path>" did not match any paths. Either' + ' "packages.foo.transformations[0].move [Line 6 column 12]" did not match any paths. Either' " the definition is redundant (and can be omitted) or the match rule is incorrect." ) assert expected == e_info.value.args[0] -def test_builtin_mode_normalization(manifest_parser_pkg_foo): +def test_builtin_mode_normalization_shell_scripts( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: manifest = manifest_parser_pkg_foo.build_manifest() claim_mtime_to = 255 sh_script_content = "#!/bin/sh" @@ -773,3 +798,53 @@ def test_builtin_mode_normalization(manifest_parser_pkg_foo): print(intermediate_manifest) verify_paths(intermediate_manifest, expected_results) + + +def test_builtin_mode_normalization( + manifest_parser_pkg_foo: YAMLManifestParser, +) -> None: + manifest = manifest_parser_pkg_foo.build_manifest() + claim_mtime_to = 255 + + paths = [ + virtual_path_def("usr/", mode=0o755, mtime=10, fs_path="/nowhere/usr"), + virtual_path_def( + "usr/share/", mode=0o755, mtime=10, fs_path="/nowhere/usr/share" + ), + virtual_path_def( + "usr/share/perl5/", mode=0o755, mtime=10, fs_path="/nowhere/usr/share/perl5" + ), + virtual_path_def( + "usr/share/perl5/Foo.pm", + # #1076346 + mode=0o444, + mtime=10, + fs_path="/nowhere/Foo.pm", + ), + virtual_path_def( + "usr/share/perl5/Bar.pm", + mode=0o755, + mtime=10, + fs_path="/nowhere/Bar.pm", + ), + ] + + fs_root = build_virtual_fs(paths, read_write_fs=True) + assert [p.name for p in manifest.all_packages] == ["foo"] + + expected_results = [ + ("usr/", Expected(mode=0o755, mtime=10)), + ("usr/share/", Expected(mode=0o755, mtime=10)), + ("usr/share/perl5/", Expected(mode=0o755, mtime=10)), + ("usr/share/perl5/Bar.pm", Expected(mode=0o644, mtime=10)), + ("usr/share/perl5/Foo.pm", Expected(mode=0o644, mtime=10)), + ] + assert [p.name for p in manifest.all_packages] == ["foo"] + + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, claim_mtime_to + ) + + print(intermediate_manifest) + + verify_paths(intermediate_manifest, expected_results) diff --git a/tests/test_install_rules.py b/tests/test_install_rules.py index a361864..e94e8bc 100644 --- a/tests/test_install_rules.py +++ b/tests/test_install_rules.py @@ -620,7 +620,7 @@ def test_install_rules_no_matches(manifest_parser_pkg_foo) -> None: ) expected_msg = ( "There were no matches for build/private-arch-tool in /nowhere/debian/tmp, /nowhere" - " (definition: installations[0].install <Search for: build/private-arch-tool>)." + " (definition: installations[0].install [Line 5 column 6])." " Match rule: ./build/private-arch-tool (the exact path / no globbing)" ) assert e_info.value.message == expected_msg diff --git a/tests/test_parser.py b/tests/test_parser.py index 4aee024..1c84445 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -124,7 +124,10 @@ def test_parsing_variables_reserved(manifest_parser_pkg_foo, varname): with pytest.raises(ManifestParseException) as e_info: manifest_parser_pkg_foo.parse_manifest(fd=content) - msg = f'The variable "{varname}" is already reserved/defined. Error triggered by definitions.variables.{varname}.' + msg = ( + f'The variable "{varname}" is already reserved/defined.' + f" Error triggered by definitions.variables.{varname} [Line 4 column 4]." + ) assert normalize_doc_link(e_info.value.args[0]) == msg @@ -163,7 +166,7 @@ def test_parsing_variables_unused(manifest_parser_pkg_foo): msg = ( 'The variable "UNUSED" is unused. Either use it or remove it.' - " The variable was declared at definitions.variables.UNUSED." + " The variable was declared at definitions.variables.UNUSED [Line 4 column 4]." ) assert normalize_doc_link(e_info.value.args[0]) == msg @@ -181,7 +184,7 @@ def test_parsing_package_foo_empty(manifest_parser_pkg_foo): manifest_parser_pkg_foo.parse_manifest(fd=content) msg = ( - "The attribute packages.foo must be a non-empty mapping. Please see" + "The attribute packages.foo [Line 3 column 4] must be a non-empty mapping. Please see" " {{DEBPUTY_DOC_ROOT_DIR}}/MANIFEST-FORMAT.md#binary-package-rules for the documentation." ) assert normalize_doc_link(e_info.value.args[0]) == msg @@ -238,8 +241,8 @@ def test_create_symlinks_missing_path(manifest_parser_pkg_foo): manifest_parser_pkg_foo.parse_manifest(fd=content) msg = ( - "The following keys were required but not present at packages.foo.transformations[0].create-symlink: 'path'" - " (Documentation: " + "The following keys were required but not present at packages.foo.transformations[0].create-symlink" + " [Line 5 column 12]: 'path' (Documentation: " "{{DEBPUTY_DOC_ROOT_DIR}}/MANIFEST-FORMAT.md#create-symlinks-transformation-rule-create-symlink)" ) assert normalize_doc_link(e_info.value.args[0]) == msg @@ -263,7 +266,7 @@ def test_create_symlinks_unknown_replacement_rule(manifest_parser_pkg_foo): manifest_parser_pkg_foo.parse_manifest(fd=content) msg = ( - 'The attribute "packages.foo.transformations[0].create-symlink.replacement-rule <Search for: usr/share/foo>"' + 'The attribute "packages.foo.transformations[0].create-symlink.replacement-rule [Line 8 column 32]"' " did not have a valid structure/type: Value (golf) must be one of the following literal values:" ' "error-if-exists", "error-if-directory", "abort-on-non-empty-directory", "discard-existing"' ) @@ -286,8 +289,8 @@ def test_create_symlinks_missing_target(manifest_parser_pkg_foo): manifest_parser_pkg_foo.parse_manifest(fd=content) msg = ( - "The following keys were required but not present at packages.foo.transformations[0].create-symlink: 'target'" - " (Documentation: " + "The following keys were required but not present at packages.foo.transformations[0].create-symlink" + " [Line 5 column 12]: 'target' (Documentation: " "{{DEBPUTY_DOC_ROOT_DIR}}/MANIFEST-FORMAT.md#create-symlinks-transformation-rule-create-symlink)" ) assert normalize_doc_link(e_info.value.args[0]) == msg @@ -310,7 +313,7 @@ def test_create_symlinks_not_normalized_path(manifest_parser_pkg_foo): manifest_parser_pkg_foo.parse_manifest(fd=content) expected = ( - 'The path "../bar" provided in packages.foo.transformations[0].create-symlink.path <Search for: ../bar>' + 'The path "../bar" provided in packages.foo.transformations[0].create-symlink.path [Line 6 column 20]' ' should be relative to the root of the package and not use any ".." or "." segments.' ) assert e_info.value.args[0] == expected @@ -332,7 +335,7 @@ def test_unresolvable_subst_in_source_context(manifest_parser_pkg_foo): expected = ( "The variable {{PACKAGE}} is not available while processing installations[0].install.as" - " <Search for: foo.sh>." + " [Line 5 column 7]." ) assert e_info.value.args[0] == expected @@ -397,7 +400,7 @@ def test_yaml_octal_mode_int(manifest_parser_pkg_foo): manifest_parser_pkg_foo.parse_manifest(fd=content) msg = ( - 'The attribute "packages.foo.transformations[0].path-metadata.mode <Search for: usr/share/bar>" did not' + 'The attribute "packages.foo.transformations[0].path-metadata.mode [Line 7 column 20]" did not' " have a valid structure/type: The attribute must be a FileSystemMode (string)" ) diff --git a/tests/test_style.py b/tests/test_style.py index ef6ddc4..d3cfb14 100644 --- a/tests/test_style.py +++ b/tests/test_style.py @@ -4,9 +4,9 @@ import pytest from debian.deb822 import Deb822 from debputy.yaml.compat import CommentedMap -from debputy.lsp.style_prefs import ( - StylePreferenceTable, - determine_effective_style, +from debputy.lsp.maint_prefs import ( + MaintainerPreferenceTable, + determine_effective_preference, EffectivePreference, _WAS_DEFAULTS, ) @@ -14,7 +14,7 @@ from debputy.packages import SourcePackage def test_load_styles() -> None: - styles = StylePreferenceTable.load_styles() + styles = MaintainerPreferenceTable.load_preferences() assert "niels@thykier.net" in styles.maintainer_preferences nt_style = styles.maintainer_preferences["niels@thykier.net"] # Note this is data dependent; if it fails because the style changes, update the test @@ -32,7 +32,7 @@ def test_load_styles() -> None: def test_load_named_styles() -> None: - styles = StylePreferenceTable.load_styles() + styles = MaintainerPreferenceTable.load_preferences() assert "black" in styles.named_styles black_style = styles.named_styles["black"] # Note this is data dependent; if it fails because the style changes, update the test @@ -48,7 +48,7 @@ def test_load_named_styles() -> None: def test_compat_styles() -> None: - styles = StylePreferenceTable.load_styles() + styles = MaintainerPreferenceTable.load_preferences() # Data dependent; if it breaks, provide a stubbed style preference table assert "niels@thykier.net" in styles.maintainer_preferences @@ -71,30 +71,33 @@ def test_compat_styles() -> None: ) src = SourcePackage(fields) - effective_style, _ = determine_effective_style(styles, src, None) + effective_style, tool, _ = determine_effective_preference(styles, src, None) assert effective_style == nt_pref + assert tool == "debputy reformat" fields["Uploaders"] = ( "Niels Thykier <niels@thykier.net>, Chris Hofstaedtler <zeha@debian.org>" ) src = SourcePackage(fields) - effective_style, _ = determine_effective_style(styles, src, None) + effective_style, tool, _ = determine_effective_preference(styles, src, None) assert effective_style == nt_pref assert effective_style == zeha_pref + assert tool == "debputy reformat" fields["Uploaders"] = ( "Niels Thykier <niels@thykier.net>, Chris Hofstaedtler <zeha@debian.org>, Random Developer <random@example.org>" ) src = SourcePackage(fields) - effective_style, _ = determine_effective_style(styles, src, None) + effective_style, tool, _ = determine_effective_preference(styles, src, None) assert effective_style is None + assert tool is None @pytest.mark.xfail def test_compat_styles_team_maint() -> None: - styles = StylePreferenceTable.load_styles() + styles = MaintainerPreferenceTable.load_preferences() fields = Deb822( { "Package": "foo", @@ -108,12 +111,13 @@ def test_compat_styles_team_maint() -> None: assert "random@example.org" not in styles.maintainer_preferences team_style = styles.maintainer_preferences["team@lists.debian.org"] assert team_style.is_packaging_team - effective_style, _ = determine_effective_style(styles, src, None) + effective_style, tool, _ = determine_effective_preference(styles, src, None) assert effective_style == team_style.as_effective_pref() + assert tool is None def test_x_style() -> None: - styles = StylePreferenceTable.load_styles() + styles = MaintainerPreferenceTable.load_preferences() fields = Deb822( { "Package": "foo", @@ -125,12 +129,13 @@ def test_x_style() -> None: assert "random@example.org" not in styles.maintainer_preferences assert "black" in styles.named_styles black_style = styles.named_styles["black"] - effective_style, _ = determine_effective_style(styles, src, None) + effective_style, tool, _ = determine_effective_preference(styles, src, None) assert effective_style == black_style + assert tool == "debputy reformat" def test_was_from_salsa_ci_style() -> None: - styles = StylePreferenceTable.load_styles() + styles = MaintainerPreferenceTable.load_preferences() fields = Deb822( { "Package": "foo", @@ -139,20 +144,23 @@ def test_was_from_salsa_ci_style() -> None: ) src = SourcePackage(fields) assert "random@example.org" not in styles.maintainer_preferences - effective_style, _ = determine_effective_style(styles, src, None) + effective_style, tool, _ = determine_effective_preference(styles, src, None) assert effective_style is None + assert tool is None salsa_ci = CommentedMap( {"variables": CommentedMap({"SALSA_CI_DISABLE_WRAP_AND_SORT": "yes"})} ) - effective_style, _ = determine_effective_style(styles, src, salsa_ci) + effective_style, tool, _ = determine_effective_preference(styles, src, salsa_ci) assert effective_style is None + assert tool is None salsa_ci = CommentedMap( {"variables": CommentedMap({"SALSA_CI_DISABLE_WRAP_AND_SORT": "no"})} ) - effective_style, _ = determine_effective_style(styles, src, salsa_ci) + effective_style, tool, _ = determine_effective_preference(styles, src, salsa_ci) was_style = EffectivePreference(**_WAS_DEFAULTS) assert effective_style == was_style + assert tool == "wrap-and-sort" @pytest.mark.parametrize( @@ -197,9 +205,10 @@ def test_was_from_salsa_ci_style() -> None: ], ) def test_was_from_salsa_ci_style_args( - was_args: str, style_delta: Optional[Mapping[str, Any]] + was_args: str, + style_delta: Optional[Mapping[str, Any]], ) -> None: - styles = StylePreferenceTable.load_styles() + styles = MaintainerPreferenceTable.load_preferences() fields = Deb822( { "Package": "foo", @@ -218,12 +227,14 @@ def test_was_from_salsa_ci_style_args( ) } ) - effective_style, _ = determine_effective_style(styles, src, salsa_ci) + effective_style, tool, _ = determine_effective_preference(styles, src, salsa_ci) if style_delta is None: assert effective_style is None + assert tool is None else: was_style = EffectivePreference(**_WAS_DEFAULTS).replace( **style_delta, ) assert effective_style == was_style + assert tool == f"wrap-and-sort {was_args}".strip() |