From 603d7e38226d352911eceeab4da1923650e643f6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 4 May 2024 03:05:28 +0200 Subject: Merging upstream version 0.1.30. Signed-off-by: Daniel Baumann --- .gitlab-ci.yml | 17 +- debputy.pod | 58 +- src/debputy/commands/debputy_cmd/__main__.py | 1 - .../commands/debputy_cmd/lint_and_lsp_cmds.py | 70 ++- src/debputy/commands/debputy_cmd/output.py | 8 + src/debputy/deb_packaging_support.py | 2 +- src/debputy/linting/lint_impl.py | 177 ++++++- src/debputy/linting/lint_util.py | 28 +- src/debputy/lsp/debputy_ls.py | 39 +- src/debputy/lsp/diagnostics.py | 8 + src/debputy/lsp/lsp_debian_changelog.py | 27 +- src/debputy/lsp/lsp_debian_control.py | 148 +++--- .../lsp/lsp_debian_control_reference_data.py | 533 ++++++++++++++++--- src/debputy/lsp/lsp_debian_copyright.py | 65 ++- src/debputy/lsp/lsp_debian_debputy_manifest.py | 5 +- src/debputy/lsp/lsp_debian_rules.py | 3 +- src/debputy/lsp/lsp_debian_tests_control.py | 86 ++- src/debputy/lsp/lsp_dispatch.py | 41 +- src/debputy/lsp/lsp_features.py | 17 + src/debputy/lsp/lsp_generic_deb822.py | 81 ++- src/debputy/lsp/lsp_reference_keyword.py | 45 ++ src/debputy/lsp/quickfixes.py | 12 +- src/debputy/lsp/spellchecking.py | 5 +- src/debputy/lsp/style-preferences.yaml | 67 +++ src/debputy/lsp/style_prefs.py | 582 +++++++++++++++++++++ src/debputy/lsp/text_edit.py | 8 +- src/debputy/lsp/text_util.py | 29 +- src/debputy/lsp/vendoring/_deb822_repro/_util.py | 2 +- .../lsp/vendoring/_deb822_repro/formatter.py | 1 + .../lsp/vendoring/_deb822_repro/locatable.py | 76 +-- src/debputy/lsp/vendoring/_deb822_repro/parsing.py | 91 +--- src/debputy/lsp/vendoring/_deb822_repro/tokens.py | 7 +- src/debputy/lsp/vendoring/wrap_and_sort.py | 113 ++++ src/debputy/util.py | 4 +- tests/lint_tests/conftest.py | 1 + tests/lint_tests/lint_tutil.py | 6 + tests/lint_tests/test_lint_dctrl.py | 58 +- tests/lsp_tests/test_debpkg_metadata.py | 2 +- tests/test_style.py | 121 +++++ 39 files changed, 2228 insertions(+), 416 deletions(-) create mode 100644 src/debputy/lsp/diagnostics.py create mode 100644 src/debputy/lsp/lsp_reference_keyword.py create mode 100644 src/debputy/lsp/style-preferences.yaml create mode 100644 src/debputy/lsp/style_prefs.py create mode 100644 src/debputy/lsp/vendoring/wrap_and_sort.py create mode 100644 tests/test_style.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 565d0d9..b3d32b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -143,17 +143,26 @@ pages: - main variables: - SALSA_CI_DISABLE_WRAP_AND_SORT: 0 - SALSA_CI_WRAP_AND_SORT_ARGS: '-abkt' + SALSA_CI_DISABLE_WRAP_AND_SORT: 1 SALSA_CI_AUTOPKGTEST_ALLOWED_EXIT_STATUS: 0 SALSA_CI_DISABLE_APTLY: 0 +debputy-reformat: + stage: ci-test + image: debian:sid-slim + script: + - apt-get update -qq && apt-get -qq build-dep --no-install-recommends --yes . + - ./debputy.sh reformat --linter-exit-code --no-auto-fix + except: + variables: + - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/ + debputy-lint: stage: ci-test image: debian:sid-slim script: - - apt-get update -qq && apt-get -qq install --no-install-recommends --yes dh-debputy python3-pygls - - PERL5LIB=lib debputy lint --spellcheck + - apt-get update -qq && apt-get -qq build-dep --no-install-recommends --yes . + - PERL5LIB=lib ./debputy.sh lint --spellcheck except: variables: - $CI_COMMIT_TAG != null && $SALSA_CI_ENABLE_PIPELINE_ON_TAGS !~ /^(1|yes|true)$/ diff --git a/debputy.pod b/debputy.pod index 933b6d0..0cee740 100644 --- a/debputy.pod +++ b/debputy.pod @@ -14,7 +14,9 @@ B B [B<--debputy-manifest=>I] B B -B B +B B [--auto-fix] + +B B [--no-auto-fix] B B B B @@ -117,6 +119,60 @@ The B<--no-act> is an alias of B<--no-apply-changes> to match the name that B or B >> + +Apply the formatting style on the packaging files. + +This is the same style that B would have applied if requested to reformat the files. + +The B tool reacts to having a B field in F from where you can pick +a named style. The recommended style is B, which is named such to match the B code formatter +for B (which imposes a style that evolves over time). + +For packages that does not have the B field, B will result to looking up the maintainer +(and possibly co-maintainers) for known style preferences in its built-in configuration. If B +can derive a style that all parties would agree too (or the team style for packaging teams), then that +style will be used. + +At the time of introduction, the B style is similar to that of B, since +that was one of the most common style according to L, But the style is +expected to evolve over time and the two styles may diverge over time. + +The command accepts the following options: + +=over 4 + +=item B<--style=black> + +Override the package style and use the B style. Any auto-detection or B setting will +be ignored. + +=item B<--auto-fix>, B<--no-auto-fix> + +Decide whether any changes should be fixed automatically. + +Either way, a difference (B) is displayed to stdout if any changes were detected. + +=item B<--linter-exit-code>, B<--no-linter-exit-code> + +There is a convention among linter tools to return a non-zero exit code for "issues". The +B<--linter-exit-code> will enforce this behaviour while the B<--no-linter-exit-code> will disable +it. + +The B program will use exit code B<2> for "issues" as a "linter exit code" when +linting based exit codes are active. + +Not having a linter based exit code can be useful if you want to run the tool programmatically +to perform the action and you only want the exit code to tell whether there was a problem +providing the results. + +If you rely on the exit code, you are recommended to explicitly pass the relevant variant of the +flag even if the current default matches your wishes. + +=back + =item lint I<< Note: This subcommand needs optional dependencies to work from B or B >> diff --git a/src/debputy/commands/debputy_cmd/__main__.py b/src/debputy/commands/debputy_cmd/__main__.py index 1a7a737..2d6519f 100644 --- a/src/debputy/commands/debputy_cmd/__main__.py +++ b/src/debputy/commands/debputy_cmd/__main__.py @@ -358,7 +358,6 @@ def parse_args() -> argparse.Namespace: ) def _check_manifest(context: CommandContext) -> None: context.parse_manifest() - _info("No errors detected.") def _install_plugin_from_plugin_metadata( diff --git a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py index 2f283e8..ea97665 100644 --- a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py +++ b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py @@ -2,6 +2,7 @@ import textwrap from argparse import BooleanOptionalAction from debputy.commands.debputy_cmd.context import ROOT_COMMAND, CommandContext, add_arg +from debputy.lsp.lsp_reference_keyword import ALL_PUBLIC_NAMED_STYLES from debputy.util import _error @@ -26,9 +27,9 @@ _EDITOR_SNIPPETS = { '(debian-changelog-mode . ("debputy" "lsp" "server" "--ignore-language-ids"))) (add-to-list 'eglot-server-programs '(debian-copyright-mode . ("debputy" "lsp" "server" "--ignore-language-ids"))) - ;; Requires elpa-dpkg-dev-el (>> 37.11) - ;; (add-to-list 'eglot-server-programs - ;; '(debian-autopkgtest-control-mode . ("debputy" "lsp" "server" "--ignore-language-ids"))) + ;; Requires elpa-dpkg-dev-el (>= 37.12) + (add-to-list 'eglot-server-programs + '(debian-autopkgtest-control-mode . ("debputy" "lsp" "server" "--ignore-language-ids"))) ;; The debian/rules file uses the qmake mode. (add-to-list 'eglot-server-programs '(makefile-gmake-mode . ("debputy" "lsp" "server" "--ignore-language-ids"))) @@ -38,13 +39,15 @@ _EDITOR_SNIPPETS = { ;; Auto-start eglot for the relevant modes. (add-hook 'debian-control-mode-hook 'eglot-ensure) - ;; NOTE: changelog disabled by default because for some reason it - ;; this hook causes perceivable delay (several seconds) when - ;; opening the first changelog. It seems to be related to imenu. - ;; (add-hook 'debian-changelog-mode-hook 'eglot-ensure) + ;; Requires elpa-dpkg-dev-el (>= 37.12) + ;; Technically, the `eglot-ensure` works before then, but it causes a + ;; visible and very annoying long delay on opening the first changelog. + ;; It still has a minor delay in 37.12, which may still be too long for + ;; for your preference. In that case, comment it out. + (add-hook 'debian-changelog-mode-hook 'eglot-ensure) (add-hook 'debian-copyright-mode-hook 'eglot-ensure) - ;; Requires elpa-dpkg-dev-el (>> 37.11) - ;; (add-hook 'debian-autopkgtest-control-mode-hook 'eglot-ensure) + ;; Requires elpa-dpkg-dev-el (>= 37.12) + (add-hook 'debian-autopkgtest-control-mode-hook 'eglot-ensure) (add-hook 'makefile-gmake-mode-hook 'eglot-ensure) (add-hook 'yaml-mode-hook 'eglot-ensure) """ @@ -178,6 +181,8 @@ def lsp_server_cmd(context: CommandContext) -> None: debputy_language_server.dctrl_parser = context.dctrl_parser debputy_language_server.trust_language_ids = parsed_args.trust_language_ids + debputy_language_server.finish_initialization() + if parsed_args.tcp and parsed_args.ws: _error("Sorry, --tcp and --ws are mutually exclusive") @@ -254,7 +259,6 @@ def lsp_describe_features(context: CommandContext) -> None: "--auto-fix", dest="auto_fix", action="store_true", - shared=True, help="Automatically fix problems with trivial or obvious corrections.", ), add_arg( @@ -264,6 +268,13 @@ def lsp_describe_features(context: CommandContext) -> None: action=BooleanOptionalAction, help='Enable or disable the "linter" convention of exiting with an error if severe issues were found', ), + add_arg( + "--warn-about-check-manifest", + dest="warn_about_check_manifest", + default=True, + action=BooleanOptionalAction, + help="Warn about limitations that check-manifest would cover if d/debputy.manifest is present", + ), ], ) def lint_cmd(context: CommandContext) -> None: @@ -278,9 +289,48 @@ def lint_cmd(context: CommandContext) -> None: perform_linting(context) +@ROOT_COMMAND.register_subcommand( + "reformat", + argparser=[ + add_arg( + "--style", + dest="named_style", + choices=ALL_PUBLIC_NAMED_STYLES, + default=None, + help="The formatting style to use (overrides packaging style).", + ), + add_arg( + "--auto-fix", + dest="auto_fix", + default=True, + action=BooleanOptionalAction, + help="Whether to automatically apply any style changes.", + ), + add_arg( + "--linter-exit-code", + dest="linter_exit_code", + default=True, + action=BooleanOptionalAction, + help='Enable or disable the "linter" convention of exiting with an error if issues were found', + ), + ], +) +def reformat_cmd(context: CommandContext) -> None: + try: + import lsprotocol + except ImportError: + _error("This feature requires lsprotocol (apt-get install python3-lsprotocol)") + + from debputy.linting.lint_impl import perform_reformat + + context.must_be_called_in_source_root() + perform_reformat(context, named_style=context.parsed_args.named_style) + + def ensure_lint_and_lsp_commands_are_loaded(): # Loading the module does the heavy lifting # However, having this function means that we do not have an "unused" import that some tool # gets tempted to remove assert ROOT_COMMAND.has_command("lsp") assert ROOT_COMMAND.has_command("lint") + assert ROOT_COMMAND.has_command("reformat") diff --git a/src/debputy/commands/debputy_cmd/output.py b/src/debputy/commands/debputy_cmd/output.py index 2e117ba..5159980 100644 --- a/src/debputy/commands/debputy_cmd/output.py +++ b/src/debputy/commands/debputy_cmd/output.py @@ -22,6 +22,14 @@ from debputy.util import assume_not_none try: import colored + + if ( + not hasattr(colored, "Style") + or not hasattr(colored, "Fore") + or not hasattr(colored, "Back") + ): + # Seen with python3-colored v1 (bookworm) + raise ImportError except ImportError: colored = None diff --git a/src/debputy/deb_packaging_support.py b/src/debputy/deb_packaging_support.py index 6de61b4..8ab0484 100644 --- a/src/debputy/deb_packaging_support.py +++ b/src/debputy/deb_packaging_support.py @@ -1216,7 +1216,7 @@ def setup_control_files( return if generated_triggers: - assert not allow_ctrl_file_management + assert allow_ctrl_file_management dest_file = os.path.join(control_output_dir, "triggers") with open(dest_file, "at", encoding="utf-8") as fd: fd.writelines( diff --git a/src/debputy/linting/lint_impl.py b/src/debputy/linting/lint_impl.py index ec13d53..c4e9310 100644 --- a/src/debputy/linting/lint_impl.py +++ b/src/debputy/linting/lint_impl.py @@ -1,6 +1,7 @@ import dataclasses import os import stat +import subprocess import sys from typing import Optional, List, Union, NoReturn, Mapping @@ -23,19 +24,35 @@ from debputy.linting.lint_util import ( LinterImpl, LintReport, LintStateImpl, + FormatterImpl, ) from debputy.lsp.lsp_debian_changelog import _lint_debian_changelog -from debputy.lsp.lsp_debian_control import _lint_debian_control -from debputy.lsp.lsp_debian_copyright import _lint_debian_copyright +from debputy.lsp.lsp_debian_control import ( + _lint_debian_control, + _reformat_debian_control, +) +from debputy.lsp.lsp_debian_copyright import ( + _lint_debian_copyright, + _reformat_debian_copyright, +) from debputy.lsp.lsp_debian_debputy_manifest import _lint_debian_debputy_manifest from debputy.lsp.lsp_debian_rules import _lint_debian_rules_impl -from debputy.lsp.lsp_debian_tests_control import _lint_debian_tests_control +from debputy.lsp.lsp_debian_tests_control import ( + _lint_debian_tests_control, + _reformat_debian_tests_control, +) from debputy.lsp.quickfixes import provide_standard_quickfixes_from_diagnostics from debputy.lsp.spellchecking import disable_spellchecking +from debputy.lsp.style_prefs import ( + StylePreferenceTable, + EffectivePreference, + determine_effective_style, +) from debputy.lsp.text_edit import ( get_well_formatted_edit, merge_sort_text_edits, apply_text_edits, + OverLappingTextEditException, ) from debputy.packages import SourcePackage, BinaryPackage from debputy.plugin.api.feature_set import PluginProvidedFeatureSet @@ -51,24 +68,39 @@ LINTER_FORMATS = { } +REFORMAT_FORMATS = { + "debian/control": _reformat_debian_control, + "debian/copyright": _reformat_debian_copyright, + "debian/tests/control": _reformat_debian_tests_control, +} + + @dataclasses.dataclass(slots=True) class LintContext: plugin_feature_set: PluginProvidedFeatureSet + style_preference_table: StylePreferenceTable source_package: Optional[SourcePackage] = None binary_packages: Optional[Mapping[str, BinaryPackage]] = None + effective_preference: Optional[EffectivePreference] = None - def state_for(self, path, lines) -> LintStateImpl: + def state_for(self, path: str, content: str, lines: List[str]) -> LintStateImpl: return LintStateImpl( self.plugin_feature_set, + self.style_preference_table, path, + content, lines, self.source_package, self.binary_packages, + self.effective_preference, ) def gather_lint_info(context: CommandContext) -> LintContext: - lint_context = LintContext(context.load_plugins()) + lint_context = LintContext( + context.load_plugins(), + StylePreferenceTable.load_styles(), + ) try: with open("debian/control") as fd: source_package, binary_packages = ( @@ -76,6 +108,11 @@ def gather_lint_info(context: CommandContext) -> LintContext: ) lint_context.source_package = source_package lint_context.binary_packages = binary_packages + if source_package is not None: + lint_context.effective_preference = determine_effective_style( + lint_context.style_preference_table, + source_package, + ) except FileNotFoundError: pass @@ -112,7 +149,9 @@ def perform_linting(context: CommandContext) -> None: "Some sub-linters reported issues. Please report the bug and include the output" ) - if os.path.isfile("debian/debputy.manifest"): + if parsed_args.warn_about_check_manifest and os.path.isfile( + "debian/debputy.manifest" + ): _info("Note: Due to a limitation in the linter, debian/debputy.manifest is") _info("only **partially** checked by this command at the time of writing.") _info("Please use `debputy check-manifest` to fully check the manifest.") @@ -121,6 +160,129 @@ def perform_linting(context: CommandContext) -> None: _exit_with_lint_code(lint_report) +def perform_reformat( + context: CommandContext, + *, + named_style: Optional[str] = None, +) -> None: + parsed_args = context.parsed_args + if not parsed_args.spellcheck: + disable_spellchecking() + fo = _output_styling(context.parsed_args, sys.stdout) + lint_context = gather_lint_info(context) + if named_style is not None: + style = lint_context.style_preference_table.named_styles.get(named_style) + if style is None: + styles = ", ".join(lint_context.style_preference_table.named_styles) + _error(f'There is no style named "{style}". Options include: {styles}') + if ( + lint_context.effective_preference is not None + and lint_context.effective_preference != style + ): + _info( + f'Note that the style "{named_style}" does not match the style that `debputy` was configured to use.' + ) + _info("This may be a non-issue (if the configuration is out of date).") + lint_context.effective_preference = style + + if lint_context.effective_preference is None: + _info( + "You can set `X-Style: black` in the source stanza of `debian/control` to pick" + ) + _info("`black` as the preferred style for this package.") + _info("") + _info( + "You can also include your style in `debputy`. This has the advantage that `debputy`" + ) + _info( + "can sometimes automatically derive an agreed style between all maintainers and" + ) + _info( + "co-maintainers for informal teams (provided all agree to the same style), where" + ) + _info( + "no explicit synchronization has been done (that is, there is no `X-Style` field)" + ) + _info("") + _info( + "For packaging teams, you can also have a team style that is applied to all team" + ) + _info("maintained packages by having it recorded in `debputy`.") + _error("Sorry; `debputy` does not know which style to use for this package.") + + changes = False + auto_fix = context.parsed_args.auto_fix + for name_stem in REFORMAT_FORMATS: + formatter = REFORMAT_FORMATS.get(name_stem) + filename = f"./{name_stem}" + if formatter is None or not os.path.isfile(filename): + continue + + reformatted = perform_reformat_of_file( + fo, + lint_context, + filename, + formatter, + auto_fix, + ) + if reformatted: + changes = True + + if changes and parsed_args.linter_exit_code: + sys.exit(2) + + +def perform_reformat_of_file( + fo: OutputStylingBase, + lint_context: LintContext, + filename: str, + formatter: FormatterImpl, + auto_fix: bool, +) -> bool: + with open(filename, "rt", encoding="utf-8") as fd: + text = fd.read() + + lines = text.splitlines(keepends=True) + lint_state = lint_context.state_for( + filename, + text, + lines, + ) + edits = formatter(lint_state) + if not edits: + return False + + try: + replacement = apply_text_edits(text, lines, edits) + except OverLappingTextEditException: + _error( + f"The reformatter for {filename} produced overlapping edits (which is broken and will not work)" + ) + + output_filename = f"{filename}.tmp" + with open(output_filename, "wt", encoding="utf-8") as fd: + fd.write(replacement) + + r = subprocess.run(["diff", "-u", filename, output_filename]).returncode + if r != 0 and r != 1: + _warn(f"diff -u {filename} {output_filename} failed!?") + if auto_fix: + orig_mode = stat.S_IMODE(os.stat(filename).st_mode) + os.chmod(output_filename, orig_mode) + os.rename(output_filename, filename) + print( + fo.colored( + f"Reformatted {filename}.", + fg="green", + style="bold", + ) + ) + else: + os.unlink(output_filename) + + return True + + def _exit_with_lint_code(lint_report: LintReport) -> NoReturn: diagnostics_count = lint_report.diagnostics_count if ( @@ -193,6 +355,7 @@ def _auto_fix_run( lines = text.splitlines(keepends=True) lint_state = lint_context.state_for( filename, + text, lines, ) current_issues = linter(lint_state) @@ -346,7 +509,7 @@ def _diagnostics_run( lint_report: LintReport, ) -> None: lines = text.splitlines(keepends=True) - lint_state = lint_context.state_for(filename, lines) + lint_state = lint_context.state_for(filename, text, lines) issues = linter(lint_state) or [] for diagnostic in issues: actions = provide_standard_quickfixes_from_diagnostics( diff --git a/src/debputy/linting/lint_util.py b/src/debputy/linting/lint_util.py index 8f226fa..e997097 100644 --- a/src/debputy/linting/lint_util.py +++ b/src/debputy/linting/lint_util.py @@ -1,8 +1,8 @@ import dataclasses import os -from typing import List, Optional, Callable, Counter, TYPE_CHECKING, Mapping +from typing import List, Optional, Callable, Counter, TYPE_CHECKING, Mapping, Sequence -from lsprotocol.types import Position, Range, Diagnostic, DiagnosticSeverity +from lsprotocol.types import Position, Range, Diagnostic, DiagnosticSeverity, TextEdit from debputy.commands.debputy_cmd.output import OutputStylingBase from debputy.packages import SourcePackage, BinaryPackage @@ -11,9 +11,14 @@ from debputy.util import _DEFAULT_LOGGER, _warn if TYPE_CHECKING: from debputy.lsp.text_util import LintCapablePositionCodec + from debputy.lsp.style_prefs import ( + StylePreferenceTable, + EffectivePreference, + ) LinterImpl = Callable[["LintState"], Optional[List[Diagnostic]]] +FormatterImpl = Callable[["LintState"], Optional[Sequence[TextEdit]]] class LintState: @@ -30,6 +35,10 @@ class LintState: def path(self) -> str: raise NotImplementedError + @property + def content(self) -> str: + raise NotImplementedError + @property def lines(self) -> List[str]: raise NotImplementedError @@ -46,14 +55,25 @@ class LintState: def binary_packages(self) -> Optional[Mapping[str, BinaryPackage]]: raise NotImplementedError + @property + def style_preference_table(self) -> "StylePreferenceTable": + raise NotImplementedError + + @property + def effective_preference(self) -> Optional["EffectivePreference"]: + raise NotImplementedError + @dataclasses.dataclass(slots=True) class LintStateImpl(LintState): plugin_feature_set: PluginProvidedFeatureSet = dataclasses.field(repr=False) + style_preference_table: "StylePreferenceTable" = dataclasses.field(repr=False) path: str + content: str lines: List[str] - source_package: Optional[SourcePackage] - binary_packages: Optional[Mapping[str, BinaryPackage]] + source_package: Optional[SourcePackage] = None + binary_packages: Optional[Mapping[str, BinaryPackage]] = None + effective_preference: Optional["EffectivePreference"] = None @property def doc_uri(self) -> str: diff --git a/src/debputy/lsp/debputy_ls.py b/src/debputy/lsp/debputy_ls.py index cc3f00e..e475ab4 100644 --- a/src/debputy/lsp/debputy_ls.py +++ b/src/debputy/lsp/debputy_ls.py @@ -13,7 +13,14 @@ from typing import ( from lsprotocol.types import MarkupKind -from debputy.linting.lint_util import LintState +from debputy.linting.lint_util import ( + LintState, +) +from debputy.lsp.style_prefs import ( + StylePreferenceTable, + MaintainerPreference, + determine_effective_style, +) from debputy.lsp.text_util import LintCapablePositionCodec from debputy.packages import ( SourcePackage, @@ -21,6 +28,7 @@ from debputy.packages import ( DctrlParser, ) from debputy.plugin.api.feature_set import PluginProvidedFeatureSet +from debputy.util import _info if TYPE_CHECKING: from pygls.server import LanguageServer @@ -88,6 +96,10 @@ class LSProvidedLintState(LintState): def path(self) -> str: return self._doc.path + @property + def content(self) -> str: + return self._doc.source + @property def lines(self) -> List[str]: return self._lines @@ -146,6 +158,17 @@ class LSProvidedLintState(LintState): dctrl = self._resolve_dctrl() return dctrl.binary_packages if dctrl is not None else None + @property + def effective_preference(self) -> Optional[MaintainerPreference]: + source_package = self.source_package + if source_package is None: + return None + return determine_effective_style(self.style_preference_table, source_package) + + @property + def style_preference_table(self) -> StylePreferenceTable: + return self._ls.style_preferences + def _preference( client_preference: Optional[List[MarkupKind]], @@ -171,6 +194,20 @@ class DebputyLanguageServer(LanguageServer): self._dctrl_parser: Optional[DctrlParser] = None self._plugin_feature_set: Optional[PluginProvidedFeatureSet] = None self._trust_language_ids: Optional[bool] = None + self._finished_initialization = False + self.style_preferences = StylePreferenceTable({}, {}) + + def finish_initialization(self) -> None: + if self._finished_initialization: + return + assert self._dctrl_parser is not None + assert self._plugin_feature_set is not None + assert self._trust_language_ids is not None + self.style_preferences = self.style_preferences.load_styles() + _info( + f"Loaded style preferences: {len(self.style_preferences.maintainer_preferences)} unique maintainer preferences recorded" + ) + self._finished_initialization = True @property def plugin_feature_set(self) -> PluginProvidedFeatureSet: diff --git a/src/debputy/lsp/diagnostics.py b/src/debputy/lsp/diagnostics.py new file mode 100644 index 0000000..6e0b88a --- /dev/null +++ b/src/debputy/lsp/diagnostics.py @@ -0,0 +1,8 @@ +from typing import TypedDict, NotRequired, List, Any, Literal, Optional + +LintSeverity = Literal["style"] + + +class DiagnosticData(TypedDict): + quickfixes: NotRequired[Optional[List[Any]]] + lint_severity: NotRequired[Optional[LintSeverity]] diff --git a/src/debputy/lsp/lsp_debian_changelog.py b/src/debputy/lsp/lsp_debian_changelog.py index ecff192..c6166c5 100644 --- a/src/debputy/lsp/lsp_debian_changelog.py +++ b/src/debputy/lsp/lsp_debian_changelog.py @@ -23,6 +23,7 @@ from lsprotocol.types import ( ) from debputy.linting.lint_util import LintState +from debputy.lsp.diagnostics import DiagnosticData from debputy.lsp.lsp_features import lsp_diagnostics, lsp_standard_handler from debputy.lsp.quickfixes import ( propose_correct_text_quick_fix, @@ -64,23 +65,6 @@ _WEEKDAYS_BY_IDX = [ ] _KNOWN_WEEK_DAYS = frozenset(_WEEKDAYS_BY_IDX) -DOCUMENT_VERSION_TABLE: Dict[str, int] = {} - - -def _handle_close( - ls: "LanguageServer", - params: DidCloseTextDocumentParams, -) -> None: - try: - del DOCUMENT_VERSION_TABLE[params.text_document.uri] - except KeyError: - pass - - -def is_doc_at_version(uri: str, version: int) -> bool: - dv = DOCUMENT_VERSION_TABLE.get(uri) - return dv == version - lsp_standard_handler(_LANGUAGE_IDS, TEXT_DOCUMENT_CODE_ACTION) lsp_standard_handler(_LANGUAGE_IDS, TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL) @@ -212,7 +196,9 @@ def _check_footer_line( f"The date was a {expected_week_day}day.", severity=DiagnosticSeverity.Warning, source="debputy", - data=[propose_correct_text_quick_fix(expected_week_day)], + data=DiagnosticData( + quickfixes=[propose_correct_text_quick_fix(expected_week_day)] + ), ) @@ -232,8 +218,9 @@ def _check_header_line( if ( entry_no == 1 and dctrl_source_pkg is not None - and dctrl_source_pkg.name != source_name + and dctrl_source_pkg.fields.get("Source") != source_name ): + expected_name = dctrl_source_pkg.fields.get("Source") or "(missing)" start_pos, end_pos = m.span(1) range_server_units = Range( Position( @@ -248,7 +235,7 @@ def _check_header_line( yield Diagnostic( position_codec.range_to_client_units(lint_state.lines, range_server_units), f"The first entry must use the same source name as debian/control." - f' Changelog uses: "{source_name}" while d/control uses: "{dctrl_source_pkg.name}"', + f' Changelog uses: "{source_name}" while d/control uses: "{expected_name}"', severity=DiagnosticSeverity.Error, source="debputy", ) diff --git a/src/debputy/lsp/lsp_debian_control.py b/src/debputy/lsp/lsp_debian_control.py index b44e8f9..3a4107b 100644 --- a/src/debputy/lsp/lsp_debian_control.py +++ b/src/debputy/lsp/lsp_debian_control.py @@ -6,43 +6,22 @@ from typing import ( Union, Sequence, Tuple, - Iterator, Optional, - Iterable, Mapping, List, - FrozenSet, Dict, ) -from debputy.lsp.debputy_ls import DebputyLanguageServer -from lsprotocol.types import ( - DiagnosticSeverity, - Range, - Diagnostic, - Position, - FoldingRange, - FoldingRangeParams, - CompletionItem, - CompletionList, - CompletionParams, - TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, - DiagnosticRelatedInformation, - Location, - HoverParams, - Hover, - TEXT_DOCUMENT_CODE_ACTION, - SemanticTokens, - SemanticTokensParams, -) - from debputy.linting.lint_util import LintState +from debputy.lsp.debputy_ls import DebputyLanguageServer +from debputy.lsp.diagnostics import DiagnosticData from debputy.lsp.lsp_debian_control_reference_data import ( DctrlKnownField, BINARY_FIELDS, SOURCE_FIELDS, DctrlFileMetadata, package_name_to_section, + all_package_relationship_fields, ) from debputy.lsp.lsp_features import ( lint_diagnostics, @@ -51,6 +30,8 @@ from debputy.lsp.lsp_features import ( lsp_standard_handler, lsp_folding_ranges, lsp_semantic_tokens_full, + lsp_will_save_wait_until, + lsp_format_document, ) from debputy.lsp.lsp_generic_deb822 import ( deb822_completer, @@ -58,6 +39,7 @@ from debputy.lsp.lsp_generic_deb822 import ( deb822_folding_ranges, deb822_semantic_tokens_full, deb822_token_iter, + deb822_format_file, ) from debputy.lsp.quickfixes import ( propose_remove_line_quick_fix, @@ -81,10 +63,28 @@ from debputy.lsp.vendoring._deb822_repro.parsing import ( Deb822KeyValuePairElement, LIST_SPACE_SEPARATED_INTERPRETATION, ) -from debputy.lsp.vendoring._deb822_repro.tokens import ( - Deb822Token, +from lsprotocol.types import ( + DiagnosticSeverity, + Range, + Diagnostic, + Position, + FoldingRange, + FoldingRangeParams, + CompletionItem, + CompletionList, + CompletionParams, + TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, + DiagnosticRelatedInformation, + Location, + HoverParams, + Hover, + TEXT_DOCUMENT_CODE_ACTION, + SemanticTokens, + SemanticTokensParams, + WillSaveTextDocumentParams, + TextEdit, + DocumentFormattingParams, ) -from debputy.util import _info try: from debputy.lsp.vendoring._deb822_repro.locatable import ( @@ -126,7 +126,7 @@ class SubstvarMetadata: def relationship_substvar_for_field(substvar: str) -> Optional[str]: - relationship_fields = _relationship_fields() + relationship_fields = all_package_relationship_fields() try: col_idx = substvar.rindex(":") except ValueError: @@ -287,28 +287,6 @@ _DCTRL_FILE_METADATA = DctrlFileMetadata() lsp_standard_handler(_LANGUAGE_IDS, TEXT_DOCUMENT_CODE_ACTION) -lsp_standard_handler(_LANGUAGE_IDS, TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL) - - -@lru_cache -def _relationship_fields() -> Mapping[str, str]: - # TODO: Pull from `dpkg-dev` when possible fallback only to the static list. - return { - f.lower(): f - for f in ( - "Pre-Depends", - "Depends", - "Recommends", - "Suggests", - "Enhances", - "Conflicts", - "Breaks", - "Replaces", - "Provides", - "Built-Using", - "Static-Built-Using", - ) - } @lsp_hover(_LANGUAGE_IDS) @@ -539,7 +517,7 @@ def _binary_package_checks( f'The Section should be "{component_prefix}{guessed_section}"{section_diagnostic_rationale}', severity=DiagnosticSeverity.Warning, source="debputy", - data=quickfix_data, + data=DiagnosticData(quickfixes=quickfix_data), ) ) @@ -557,17 +535,12 @@ def _diagnostics_for_paragraph( diagnostics: List[Diagnostic], ) -> None: representation_field = _paragraph_representation_field(stanza) - representation_field_pos = representation_field.position_in_parent().relative_to( + representation_field_range = representation_field.range_in_parent().relative_to( stanza_position ) - representation_field_range_server_units = te_range_to_lsp( - TERange.from_position_and_size( - representation_field_pos, representation_field.size() - ) - ) representation_field_range = position_codec.range_to_client_units( lines, - representation_field_range_server_units, + te_range_to_lsp(representation_field_range), ) for known_field in known_fields.values(): missing_field_severity = known_field.missing_field_severity @@ -606,7 +579,10 @@ def _diagnostics_for_paragraph( normalized_field_name_lc = normalize_dctrl_field_name(field_name_lc) known_field = known_fields.get(normalized_field_name_lc) field_value = stanza[field_name] - field_range_te = kvpair.range_in_parent().relative_to(stanza_position) + kvpair_position = kvpair.position_in_parent().relative_to(stanza_position) + field_range_te = kvpair.field_token.range_in_parent().relative_to( + kvpair_position + ) field_position_te = field_range_te.start_pos field_range_server_units = te_range_to_lsp(field_range_te) field_range = position_codec.range_to_client_units( @@ -646,10 +622,12 @@ def _diagnostics_for_paragraph( f'The "{field_name}" looks like a typo of "{known_field.name}".', severity=DiagnosticSeverity.Warning, source="debputy", - data=[ - propose_correct_text_quick_fix(known_fields[m].name) - for m in candidates - ], + data=DiagnosticData( + quickfixes=[ + propose_correct_text_quick_fix(known_fields[m].name) + for m in candidates + ] + ), ) ) if known_field is None: @@ -682,6 +660,7 @@ def _diagnostics_for_paragraph( kvpair, stanza, stanza_position, + kvpair_position, position_codec, lines, field_name_typo_reported=field_name_typo_detected, @@ -722,9 +701,12 @@ def _diagnostics_for_paragraph( f'Spelling "{word}"', severity=DiagnosticSeverity.Hint, source="debputy", - data=[ - propose_correct_text_quick_fix(c) for c in corrections - ], + data=DiagnosticData( + quickfixes=[ + propose_correct_text_quick_fix(c) + for c in corrections + ] + ), ) ) source_value = source_stanza.get(field_name) @@ -750,7 +732,7 @@ def _diagnostics_for_paragraph( f"The field {field_name} duplicates the value from the Source stanza.", severity=DiagnosticSeverity.Information, source="debputy", - data=fix_data, + data=DiagnosticData(quickfixes=fix_data), ) ) for ( @@ -846,7 +828,11 @@ def _scan_for_syntax_errors_and_token_level_diagnostics( f'Spelling "{word}"', severity=DiagnosticSeverity.Hint, source="debputy", - data=[propose_correct_text_quick_fix(c) for c in corrections], + data=DiagnosticData( + quickfixes=[ + propose_correct_text_quick_fix(c) for c in corrections + ] + ), ) ) return first_error @@ -903,6 +889,32 @@ def _lint_debian_control( return diagnostics +@lsp_will_save_wait_until(_LANGUAGE_IDS) +def _debian_control_on_save_formatting( + ls: "DebputyLanguageServer", + params: WillSaveTextDocumentParams, +) -> Optional[Sequence[TextEdit]]: + doc = ls.workspace.get_text_document(params.text_document.uri) + lint_state = ls.lint_state(doc) + return _reformat_debian_control(lint_state) + + +def _reformat_debian_control( + lint_state: LintState, +) -> Optional[Sequence[TextEdit]]: + return deb822_format_file(lint_state, _DCTRL_FILE_METADATA) + + +@lsp_format_document(_LANGUAGE_IDS) +def _debian_control_on_save_formatting( + ls: "DebputyLanguageServer", + params: DocumentFormattingParams, +) -> Optional[Sequence[TextEdit]]: + doc = ls.workspace.get_text_document(params.text_document.uri) + lint_state = ls.lint_state(doc) + return deb822_format_file(lint_state, _DCTRL_FILE_METADATA) + + @lsp_semantic_tokens_full(_LANGUAGE_IDS) def _debian_control_semantic_tokens_full( ls: "DebputyLanguageServer", diff --git a/src/debputy/lsp/lsp_debian_control_reference_data.py b/src/debputy/lsp/lsp_debian_control_reference_data.py index 898faab..60e47d7 100644 --- a/src/debputy/lsp/lsp_debian_control_reference_data.py +++ b/src/debputy/lsp/lsp_debian_control_reference_data.py @@ -1,6 +1,7 @@ import dataclasses import functools import itertools +import operator import re import sys import textwrap @@ -19,18 +20,39 @@ from typing import ( Callable, Tuple, Any, + Set, + TYPE_CHECKING, ) +from debian._deb822_repro.types import TE from debian.debian_support import DpkgArchTable +from lsprotocol.types import ( + DiagnosticSeverity, + Diagnostic, + DiagnosticTag, + Range, + TextEdit, + Position, +) + +from debputy.lsp.diagnostics import DiagnosticData + +from debputy.lsp.lsp_reference_keyword import ( + ALL_PUBLIC_NAMED_STYLES, + Keyword, + allowed_values, +) from debputy.lsp.quickfixes import ( propose_correct_text_quick_fix, propose_remove_line_quick_fix, ) +from debputy.lsp.text_edit import apply_text_edits from debputy.lsp.text_util import ( normalize_dctrl_field_name, LintCapablePositionCodec, detect_possible_typo, te_range_to_lsp, + trim_end_of_line_whitespace, ) from debputy.lsp.vendoring._deb822_repro.parsing import ( Deb822KeyValuePairElement, @@ -44,6 +66,7 @@ from debputy.lsp.vendoring._deb822_repro.parsing import ( Deb822ParsedValueElement, LIST_UPLOADERS_INTERPRETATION, _parse_whitespace_list_value, + parse_deb822_file, ) from debputy.lsp.vendoring._deb822_repro.tokens import ( Deb822FieldNameToken, @@ -53,8 +76,9 @@ from debputy.lsp.vendoring._deb822_repro.tokens import ( _RE_WHITESPACE_SEPARATED_WORD_LIST, Deb822SpaceSeparatorToken, ) +from debputy.lsp.vendoring._deb822_repro.types import FormatterCallback +from debputy.lsp.vendoring.wrap_and_sort import _sort_packages_key from debputy.util import PKGNAME_REGEX -from lsprotocol.types import DiagnosticSeverity, Diagnostic, DiagnosticTag, Range try: from debputy.lsp.vendoring._deb822_repro.locatable import ( @@ -66,6 +90,10 @@ except ImportError: pass +if TYPE_CHECKING: + from debputy.lsp.style_prefs import EffectivePreference + + F = TypeVar("F", bound="Deb822KnownField") S = TypeVar("S", bound="StanzaMetadata") @@ -103,7 +131,7 @@ LIST_COMMA_OR_SPACE_SEPARATED_INTERPRETATION = ListInterpretation( _parse_whitespace_list_value, Deb822ParsedValueElement, Deb822SpaceSeparatorToken, - Deb822SpaceSeparatorToken, + lambda: Deb822SpaceSeparatorToken(","), _parsed_value_render_factory, ) @@ -121,6 +149,43 @@ CustomFieldCheck = Callable[ ] +@functools.lru_cache +def all_package_relationship_fields() -> Mapping[str, str]: + # TODO: Pull from `dpkg-dev` when possible fallback only to the static list. + return { + f.lower(): f + for f in ( + "Pre-Depends", + "Depends", + "Recommends", + "Suggests", + "Enhances", + "Conflicts", + "Breaks", + "Replaces", + "Provides", + "Built-Using", + "Static-Built-Using", + ) + } + + +@functools.lru_cache +def all_source_relationship_fields() -> Mapping[str, str]: + # TODO: Pull from `dpkg-dev` when possible fallback only to the static list. + return { + f.lower(): f + for f in ( + "Build-Depends", + "Build-Depends-Arch", + "Build-Depends-Indep", + "Build-Conflicts", + "Build-Conflicts-Arch", + "Build-Conflicts-Indep", + ) + } + + ALL_SECTIONS_WITHOUT_COMPONENT = frozenset( [ "admin", @@ -199,28 +264,7 @@ def _fields(*fields: F) -> Mapping[str, F]: return {normalize_dctrl_field_name(f.name.lower()): f for f in fields} -@dataclasses.dataclass(slots=True, frozen=True) -class Keyword: - value: str - hover_text: Optional[str] = None - is_obsolete: bool = False - replaced_by: Optional[str] = None - is_exclusive: bool = False - """For keywords in fields that allow multiple keywords, the `is_exclusive` can be - used for keywords that cannot be used with other keywords. As an example, the `all` - value in `Architecture` of `debian/control` cannot be used with any other architecture. - """ - - -def _allowed_values(*values: Union[str, Keyword]) -> Mapping[str, Keyword]: - as_keywords = [k if isinstance(k, Keyword) else Keyword(k) for k in values] - as_mapping = {k.value: k for k in as_keywords if k.value} - # Simple bug check - assert len(as_keywords) == len(as_mapping) - return as_mapping - - -ALL_SECTIONS = _allowed_values( +ALL_SECTIONS = allowed_values( *[ s if c is None else f"{c}/{s}" for c, s in itertools.product( @@ -230,7 +274,7 @@ ALL_SECTIONS = _allowed_values( ] ) -ALL_PRIORITIES = _allowed_values( +ALL_PRIORITIES = allowed_values( Keyword( "required", hover_text=textwrap.dedent( @@ -762,17 +806,19 @@ class Deb822KnownField: kvpair: Deb822KeyValuePairElement, stanza: Deb822ParagraphElement, stanza_position: "TEPosition", + kvpair_position: "TEPosition", position_codec: "LintCapablePositionCodec", lines: List[str], *, field_name_typo_reported: bool = False, ) -> Iterable[Diagnostic]: field_name_token = kvpair.field_token - field_range_te = kvpair.range_in_parent().relative_to(stanza_position) - field_position_te = field_range_te.start_pos + field_range_te = kvpair.field_token.range_in_parent().relative_to( + kvpair_position + ) yield from self._diagnostics_for_field_name( field_name_token, - field_position_te, + field_range_te, field_name_typo_reported, position_codec, lines, @@ -789,13 +835,16 @@ class Deb822KnownField: ) if not self.spellcheck_value: yield from self._known_value_diagnostics( - kvpair, field_position_te, position_codec, lines + kvpair, + kvpair_position, + position_codec, + lines, ) def _diagnostics_for_field_name( self, token: Deb822FieldNameToken, - token_position: "TEPosition", + token_range: "TERange", typo_detected: bool, position_codec: "LintCapablePositionCodec", lines: List[str], @@ -803,9 +852,7 @@ class Deb822KnownField: field_name = token.text # Defeat the case-insensitivity from python-debian field_name_cased = str(field_name) - token_range_server_units = te_range_to_lsp( - TERange.from_position_and_size(token_position, token.size()) - ) + token_range_server_units = te_range_to_lsp(token_range) token_range = position_codec.range_to_client_units( lines, token_range_server_units, @@ -817,7 +864,7 @@ class Deb822KnownField: severity=DiagnosticSeverity.Warning, source="debputy", tags=[DiagnosticTag.Deprecated], - data=propose_remove_line_quick_fix(), + data=DiagnosticData(quickfixes=[propose_remove_line_quick_fix()]), ) elif self.replaced_by is not None: yield Diagnostic( @@ -826,7 +873,9 @@ class Deb822KnownField: severity=DiagnosticSeverity.Warning, source="debputy", tags=[DiagnosticTag.Deprecated], - data=propose_correct_text_quick_fix(self.replaced_by), + data=DiagnosticData( + quickfixes=[propose_correct_text_quick_fix(self.replaced_by)], + ), ) if not typo_detected and field_name_cased != self.name: @@ -835,25 +884,52 @@ class Deb822KnownField: f"Non-canonical spelling of {self.name}", severity=DiagnosticSeverity.Information, source="debputy", - data=propose_correct_text_quick_fix(self.name), + data=DiagnosticData( + quickfixes=[propose_correct_text_quick_fix(self.name)] + ), ) def _known_value_diagnostics( self, kvpair: Deb822KeyValuePairElement, - field_position_te: "TEPosition", + kvpair_position: "TEPosition", position_codec: "LintCapablePositionCodec", lines: List[str], ) -> Iterable[Diagnostic]: unknown_value_severity = self.unknown_value_diagnostic_severity - allowed_values = self.known_values interpreter = self.field_value_class.interpreter() - if not allowed_values or interpreter is None: + if interpreter is None: return values = kvpair.interpret_as(interpreter) value_off = kvpair.value_element.position_in_parent().relative_to( - field_position_te + kvpair_position ) + + last_token_non_ws_sep_token: Optional[TE] = None + for token in values.iter_parts(): + if token.is_whitespace: + continue + if not token.is_separator: + last_token_non_ws_sep_token = None + continue + if last_token_non_ws_sep_token is not None: + sep_range_te = token.range_in_parent().relative_to(value_off) + value_range = position_codec.range_to_client_units( + lines, + te_range_to_lsp(sep_range_te), + ) + yield Diagnostic( + value_range, + "Duplicate separator", + severity=DiagnosticSeverity.Error, + source="debputy", + ) + last_token_non_ws_sep_token = token + + allowed_values = self.known_values + if not allowed_values: + return + first_value = None first_exclusive_value_ref = None first_exclusive_value = None @@ -866,12 +942,8 @@ class Deb822KnownField: and self.field_value_class == FieldValueClass.SINGLE_VALUE ): value_loc = value_ref.locatable - value_position_te = value_loc.position_in_parent().relative_to( - value_off - ) - value_range_in_server_units = te_range_to_lsp( - TERange.from_position_and_size(value_position_te, value_loc.size()) - ) + range_position_te = value_loc.range_in_parent().relative_to(value_off) + value_range_in_server_units = te_range_to_lsp(range_position_te) value_range = position_codec.range_to_client_units( lines, value_range_in_server_units, @@ -935,7 +1007,7 @@ class Deb822KnownField: "message": unknown_value_message, "severity": unknown_severity, "source": "debputy", - "data": typo_fix_data, + "data": DiagnosticData(quickfixes=typo_fix_data), } ) @@ -958,7 +1030,7 @@ class Deb822KnownField: "message": obsolete_value_message, "severity": obsolete_severity, "source": "debputy", - "data": obsolete_fix_data, + "data": DiagnosticData(quickfixes=obsolete_fix_data), } ) @@ -974,11 +1046,180 @@ class Deb822KnownField: ) yield from (Diagnostic(value_range, **issue_data) for issue_data in issues) + def reformat_field( + self, + effective_preference: "EffectivePreference", + stanza_range: TERange, + kvpair: Deb822KeyValuePairElement, + formatter: FormatterCallback, + position_codec: LintCapablePositionCodec, + lines: List[str], + ) -> Iterable[TextEdit]: + kvpair_range = kvpair.range_in_parent().relative_to(stanza_range.start_pos) + return trim_end_of_line_whitespace( + position_codec, + lines, + line_range=range( + kvpair_range.start_pos.line_position, + kvpair_range.end_pos.line_position, + ), + ) + @dataclasses.dataclass(slots=True, frozen=True) -class DctrlKnownField(Deb822KnownField): +class DctrlLikeKnownField(Deb822KnownField): + + def reformat_field( + self, + effective_preference: "EffectivePreference", + stanza_range: TERange, + kvpair: Deb822KeyValuePairElement, + formatter: FormatterCallback, + position_codec: LintCapablePositionCodec, + lines: List[str], + ) -> Iterable[TextEdit]: + interpretation = self.field_value_class.interpreter() + if ( + not effective_preference.formatting_deb822_normalize_field_content + or interpretation is None + ): + yield from super(DctrlLikeKnownField, self).reformat_field( + effective_preference, + stanza_range, + kvpair, + formatter, + position_codec, + lines, + ) + return + if not self.reformattable_field: + yield from super(DctrlLikeKnownField, self).reformat_field( + effective_preference, + stanza_range, + kvpair, + formatter, + position_codec, + lines, + ) + return + seen: Set[str] = set() + old_kvpair_range = kvpair.range_in_parent() + sort = self.is_sortable_field + + # Avoid the context manager as we do not want to perform the change (it would contaminate future ranges) + field_content = kvpair.interpret_as(interpretation) + old_value = field_content.convert_to_text(with_field_name=True) + for package_ref in field_content.iter_value_references(): + value = package_ref.value + if self.is_relationship_field: + new_value = " | ".join(x.strip() for x in value.split("|")) + else: + new_value = value + if not sort or new_value not in seen: + if new_value != value: + package_ref.value = new_value + seen.add(new_value) + else: + package_ref.remove() + if sort: + field_content.sort(key=_sort_packages_key) + field_content.value_formatter(formatter) + field_content.reformat_when_finished() + + new_value = field_content.convert_to_text(with_field_name=True) + if new_value != old_value: + range_server_units = te_range_to_lsp( + old_kvpair_range.relative_to(stanza_range.start_pos) + ) + yield TextEdit( + position_codec.range_to_client_units(lines, range_server_units), + new_value, + ) + + @property + def reformattable_field(self) -> bool: + return self.is_relationship_field or self.is_sortable_field + + @property + def is_relationship_field(self) -> bool: + return False + + @property + def is_sortable_field(self) -> bool: + return self.is_relationship_field + + +@dataclasses.dataclass(slots=True, frozen=True) +class DTestsCtrlKnownField(DctrlLikeKnownField): + @property + def is_relationship_field(self) -> bool: + return self.name == "Depends" + + @property + def is_sortable_field(self) -> bool: + return self.is_relationship_field or self.name in ( + "Features", + "Restrictions", + "Tests", + ) + + +@dataclasses.dataclass(slots=True, frozen=True) +class DctrlKnownField(DctrlLikeKnownField): inherits_from_source: bool = False + def reformat_field( + self, + effective_preference: "EffectivePreference", + stanza_range: TERange, + kvpair: Deb822KeyValuePairElement, + formatter: FormatterCallback, + position_codec: LintCapablePositionCodec, + lines: List[str], + ) -> Iterable[TextEdit]: + if ( + self.name == "Architecture" + and effective_preference.formatting_deb822_normalize_field_content + ): + interpretation = self.field_value_class.interpreter() + assert interpretation is not None + archs = list(kvpair.interpret_as(interpretation)) + # Sort, with wildcard entries (such as linux-any) first: + archs = sorted(archs, key=lambda x: ("any" not in x, x)) + new_value = f"{kvpair.field_name}: {' '.join(archs)}\n" + if new_value != kvpair.convert_to_text(): + kvpair_range = te_range_to_lsp( + kvpair.range_in_parent().relative_to(stanza_range.start_pos) + ) + return [ + TextEdit( + position_codec.range_to_client_units(lines, kvpair_range), + new_value, + ) + ] + return tuple() + + return super(DctrlKnownField, self).reformat_field( + effective_preference, + stanza_range, + kvpair, + formatter, + position_codec, + lines, + ) + + @property + def is_relationship_field(self) -> bool: + name_lc = self.name.lower() + return ( + name_lc in all_package_relationship_fields() + or name_lc in all_source_relationship_fields() + ) + + @property + def reformattable_field(self) -> bool: + return self.is_relationship_field or self.name == "Uploaders" + SOURCE_FIELDS = _fields( DctrlKnownField( @@ -1216,8 +1457,8 @@ SOURCE_FIELDS = _fields( FieldValueClass.SINGLE_VALUE, deprecated_with_no_replacement=True, default_value="no", - known_values=_allowed_values("yes", "no"), - synopsis_doc="**Obsolete**: Old ACL mechanism for Debian Managers", + known_values=allowed_values("yes", "no"), + synopsis_doc="**Obsolete**: Old ACL mechanism for Debian Maintainers", hover_text=textwrap.dedent( """\ Obsolete field @@ -1355,7 +1596,7 @@ SOURCE_FIELDS = _fields( "Rules-Requires-Root", FieldValueClass.SPACE_SEPARATED_LIST, unknown_value_diagnostic_severity=None, - known_values=_allowed_values( + known_values=allowed_values( Keyword( "no", is_exclusive=True, @@ -1480,7 +1721,7 @@ SOURCE_FIELDS = _fields( DctrlKnownField( "XS-Autobuild", FieldValueClass.SINGLE_VALUE, - known_values=_allowed_values("yes"), + known_values=allowed_values("yes"), synopsis_doc="Whether this non-free is auto-buildable on buildds", hover_text=textwrap.dedent( """\ @@ -1491,6 +1732,28 @@ SOURCE_FIELDS = _fields( """ ), ), + DctrlKnownField( + "X-Style", + FieldValueClass.SINGLE_VALUE, + known_values=ALL_PUBLIC_NAMED_STYLES, + unknown_value_diagnostic_severity=DiagnosticSeverity.Warning, + synopsis_doc="Choose a formatting style", + hover_text=textwrap.dedent( + """\ + This field is read by `debputy` to determine how it should format the files in the package. + + In its absence, `debputy` will attempt to determine the formatting style by looking at + the maintainers and built-in style preferences. + + This value influences commands such as `debputy reformat` and `debputy lsp server`. When this + field is present, it will overrule any built-in style detection that `debputy` would otherwise + have applied. + + Note that unknown styles will cause the styling to be disabled (and trigger a `debputy lint` + warning). + """ + ), + ), DctrlKnownField( "Description", FieldValueClass.FREE_TEXT_FIELD, @@ -1555,7 +1818,7 @@ BINARY_FIELDS = _fields( "Package-Type", FieldValueClass.SINGLE_VALUE, default_value="deb", - known_values=_allowed_values( + known_values=allowed_values( Keyword("deb", hover_text="The package will be built as a regular deb."), Keyword( "udeb", @@ -1579,7 +1842,7 @@ BINARY_FIELDS = _fields( FieldValueClass.SPACE_SEPARATED_LIST, missing_field_severity=DiagnosticSeverity.Error, unknown_value_diagnostic_severity=None, - known_values=_allowed_values(*dpkg_arch_and_wildcards()), + known_values=allowed_values(*dpkg_arch_and_wildcards()), synopsis_doc="Architecture of the package", hover_text=textwrap.dedent( """\ @@ -1614,7 +1877,7 @@ BINARY_FIELDS = _fields( "Essential", FieldValueClass.SINGLE_VALUE, default_value="no", - known_values=_allowed_values( + known_values=allowed_values( Keyword( "yes", hover_text="The package is essential and uninstalling it will completely and utterly break the" @@ -1661,7 +1924,7 @@ BINARY_FIELDS = _fields( replaced_by="Protected", default_value="no", synopsis_doc="**Deprecated**: Use Protected instead", - known_values=_allowed_values( + known_values=allowed_values( Keyword( "yes", hover_text="The package is protected and attempts to uninstall it will cause strong warnings to the" @@ -1691,7 +1954,7 @@ BINARY_FIELDS = _fields( "Protected", FieldValueClass.SINGLE_VALUE, default_value="no", - known_values=_allowed_values( + known_values=allowed_values( Keyword( "yes", hover_text="The package is protected and attempts to uninstall it will cause strong warnings to the" @@ -2066,7 +2329,7 @@ BINARY_FIELDS = _fields( warn_if_default=False, default_value="no", custom_field_check=_dctrl_ma_field_validation, - known_values=_allowed_values( + known_values=allowed_values( Keyword( "no", hover_text=textwrap.dedent( @@ -2269,7 +2532,7 @@ BINARY_FIELDS = _fields( FieldValueClass.SINGLE_VALUE, custom_field_check=_arch_not_all_only_field_validation, default_value="host", - known_values=_allowed_values( + known_values=allowed_values( Keyword( "host", hover_text="The package should be compiled for `DEB_HOST_TARGET` (the default).", @@ -2870,11 +3133,11 @@ _DEP5_LICENSE_FIELDS = _fields( ) _DTESTSCTRL_FIELDS = _fields( - Deb822KnownField( + DTestsCtrlKnownField( "Architecture", FieldValueClass.SPACE_SEPARATED_LIST, unknown_value_diagnostic_severity=None, - known_values=_allowed_values(*dpkg_arch_and_wildcards()), + known_values=allowed_values(*dpkg_arch_and_wildcards()), synopsis_doc="Only run these tests on specific architectures", hover_text=textwrap.dedent( """\ @@ -2890,7 +3153,7 @@ _DTESTSCTRL_FIELDS = _fields( """ ), ), - Deb822KnownField( + DTestsCtrlKnownField( "Classes", FieldValueClass.FREE_TEXT_FIELD, synopsis_doc="Hardware related tagging", @@ -2912,7 +3175,7 @@ _DTESTSCTRL_FIELDS = _fields( """ ), ), - Deb822KnownField( + DTestsCtrlKnownField( "Depends", FieldValueClass.COMMA_SEPARATED_LIST, default_value="@", @@ -2952,7 +3215,7 @@ _DTESTSCTRL_FIELDS = _fields( the source are installed unless explicitly requested. """, ), - Deb822KnownField( + DTestsCtrlKnownField( "Features", FieldValueClass.COMMA_OR_SPACE_SEPARATED_LIST, hover_text=textwrap.dedent( @@ -2965,11 +3228,11 @@ _DTESTSCTRL_FIELDS = _fields( """ ), ), - Deb822KnownField( + DTestsCtrlKnownField( "Restrictions", FieldValueClass.COMMA_OR_SPACE_SEPARATED_LIST, unknown_value_diagnostic_severity=DiagnosticSeverity.Warning, - known_values=_allowed_values( + known_values=allowed_values( Keyword( "allow-stderr", hover_text=textwrap.dedent( @@ -3287,7 +3550,7 @@ _DTESTSCTRL_FIELDS = _fields( """ ), ), - Deb822KnownField( + DTestsCtrlKnownField( "Tests", FieldValueClass.COMMA_OR_SPACE_SEPARATED_LIST, synopsis_doc="List of test scripts to run", @@ -3304,7 +3567,7 @@ _DTESTSCTRL_FIELDS = _fields( """ ), ), - Deb822KnownField( + DTestsCtrlKnownField( "Test-Command", FieldValueClass.FREE_TEXT_FIELD, synopsis_doc="Single test command", @@ -3323,7 +3586,7 @@ _DTESTSCTRL_FIELDS = _fields( """ ), ), - Deb822KnownField( + DTestsCtrlKnownField( "Test-Directory", FieldValueClass.FREE_TEXT_FIELD, # TODO: Single path default_value="debian/tests", @@ -3367,6 +3630,28 @@ class StanzaMetadata(Mapping[str, F], Generic[F], ABC): def __iter__(self): return iter(self.stanza_fields.keys()) + def reformat_stanza( + self, + effective_preference: "EffectivePreference", + stanza: Deb822ParagraphElement, + stanza_range: TERange, + formatter: FormatterCallback, + position_codec: LintCapablePositionCodec, + lines: List[str], + ) -> Iterable[TextEdit]: + for known_field in self.stanza_fields.values(): + kvpair = stanza.get_kvpair_element(known_field.name, use_get=True) + if kvpair is None: + continue + yield from known_field.reformat_field( + effective_preference, + stanza_range, + kvpair, + formatter, + position_codec, + lines, + ) + @dataclasses.dataclass(slots=True, frozen=True) class Dep5StanzaMetadata(StanzaMetadata[Deb822KnownField]): @@ -3390,7 +3675,7 @@ class DctrlStanzaMetadata(StanzaMetadata[DctrlKnownField]): @dataclasses.dataclass(slots=True, frozen=True) -class DTestsCtrlStanzaMetadata(StanzaMetadata[Deb822KnownField]): +class DTestsCtrlStanzaMetadata(StanzaMetadata[DTestsCtrlKnownField]): def stanza_diagnostics( self, @@ -3425,6 +3710,40 @@ class Deb822FileMetadata(Generic[S]): except KeyError: return None + def reformat( + self, + effective_preference: "EffectivePreference", + deb822_file: Deb822FileElement, + formatter: FormatterCallback, + _content: str, + position_codec: LintCapablePositionCodec, + lines: List[str], + ) -> Iterable[TextEdit]: + stanza_idx = -1 + for token_or_element in deb822_file.iter_parts(): + if isinstance(token_or_element, Deb822ParagraphElement): + stanza_range = token_or_element.range_in_parent() + stanza_idx += 1 + stanza_metadata = self.classify_stanza(token_or_element, stanza_idx) + yield from stanza_metadata.reformat_stanza( + effective_preference, + token_or_element, + stanza_range, + formatter, + position_codec, + lines, + ) + else: + token_range = token_or_element.range_in_parent() + yield from trim_end_of_line_whitespace( + position_codec, + lines, + line_range=range( + token_range.start_pos.line_position, + token_range.end_pos.line_position, + ), + ) + _DCTRL_SOURCE_STANZA = DctrlStanzaMetadata( "Source", @@ -3503,6 +3822,78 @@ class DctrlFileMetadata(Deb822FileMetadata[DctrlStanzaMetadata]): return _DCTRL_PACKAGE_STANZA raise KeyError(item) + def reformat( + self, + effective_preference: "EffectivePreference", + deb822_file: Deb822FileElement, + formatter: FormatterCallback, + content: str, + position_codec: LintCapablePositionCodec, + lines: List[str], + ) -> Iterable[TextEdit]: + edits = list( + super().reformat( + effective_preference, + deb822_file, + formatter, + content, + position_codec, + lines, + ) + ) + + if ( + not effective_preference.formatting_deb822_normalize_stanza_order + or deb822_file.find_first_error_element() is not None + ): + return edits + names = [] + for idx, stanza in enumerate(deb822_file): + if idx < 2: + continue + name = stanza.get("Package") + if name is None: + return edits + names.append(name) + + reordered = sorted(names) + if names == reordered: + return edits + + if edits: + content = apply_text_edits(content, lines, edits) + lines = content.splitlines(keepends=True) + deb822_file = parse_deb822_file( + lines, + accept_files_with_duplicated_fields=True, + accept_files_with_error_tokens=True, + ) + + stanzas = list(deb822_file) + reordered_stanza = stanzas[:2] + sorted( + stanzas[2:], key=operator.itemgetter("Package") + ) + bits = [] + stanza_idx = 0 + for token_or_element in deb822_file.iter_parts(): + if isinstance(token_or_element, Deb822ParagraphElement): + bits.append(reordered_stanza[stanza_idx].dump()) + stanza_idx += 1 + else: + bits.append(token_or_element.convert_to_text()) + + new_content = "".join(bits) + + return [ + TextEdit( + Range( + Position(0, 0), + Position(len(lines) + 1, 0), + ), + new_content, + ) + ] + class DTestsCtrlFileMetadata(Deb822FileMetadata[DctrlStanzaMetadata]): def guess_stanza_classification_by_idx(self, stanza_idx: int) -> S: diff --git a/src/debputy/lsp/lsp_debian_copyright.py b/src/debputy/lsp/lsp_debian_copyright.py index b037792..abd5488 100644 --- a/src/debputy/lsp/lsp_debian_copyright.py +++ b/src/debputy/lsp/lsp_debian_copyright.py @@ -30,9 +30,13 @@ from lsprotocol.types import ( SemanticTokensParams, FoldingRangeParams, FoldingRange, + WillSaveTextDocumentParams, + TextEdit, + DocumentFormattingParams, ) from debputy.linting.lint_util import LintState +from debputy.lsp.diagnostics import DiagnosticData from debputy.lsp.lsp_debian_control_reference_data import ( _DEP5_HEADER_FIELDS, _DEP5_FILES_FIELDS, @@ -47,6 +51,8 @@ from debputy.lsp.lsp_features import ( lsp_standard_handler, lsp_folding_ranges, lsp_semantic_tokens_full, + lsp_will_save_wait_until, + lsp_format_document, ) from debputy.lsp.lsp_generic_deb822 import ( deb822_completer, @@ -54,6 +60,7 @@ from debputy.lsp.lsp_generic_deb822 import ( deb822_folding_ranges, deb822_semantic_tokens_full, deb822_token_iter, + deb822_format_file, ) from debputy.lsp.quickfixes import ( propose_correct_text_quick_fix, @@ -103,7 +110,6 @@ _LANGUAGE_IDS = [ _DEP5_FILE_METADATA = Dep5FileMetadata() lsp_standard_handler(_LANGUAGE_IDS, TEXT_DOCUMENT_CODE_ACTION) -lsp_standard_handler(_LANGUAGE_IDS, TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL) @lsp_hover(_LANGUAGE_IDS) @@ -183,7 +189,10 @@ def _diagnostics_for_paragraph( normalized_field_name_lc = normalize_dctrl_field_name(field_name_lc) known_field = known_fields.get(normalized_field_name_lc) field_value = stanza[field_name] - field_range_te = kvpair.range_in_parent().relative_to(stanza_position) + kvpair_position = kvpair.position_in_parent().relative_to(stanza_position) + field_range_te = kvpair.field_token.range_in_parent().relative_to( + kvpair_position + ) field_position_te = field_range_te.start_pos field_range_server_units = te_range_to_lsp(field_range_te) field_range = position_codec.range_to_client_units( @@ -223,10 +232,12 @@ def _diagnostics_for_paragraph( f'The "{field_name}" looks like a typo of "{known_field.name}".', severity=DiagnosticSeverity.Warning, source="debputy", - data=[ - propose_correct_text_quick_fix(known_fields[m].name) - for m in candidates - ], + data=DiagnosticData( + quickfixes=[ + propose_correct_text_quick_fix(known_fields[m].name) + for m in candidates + ] + ), ) ) if known_field is None: @@ -261,6 +272,7 @@ def _diagnostics_for_paragraph( kvpair, stanza, stanza_position, + kvpair_position, position_codec, lines, field_name_typo_reported=field_name_typo_detected, @@ -301,9 +313,12 @@ def _diagnostics_for_paragraph( f'Spelling "{word}"', severity=DiagnosticSeverity.Hint, source="debputy", - data=[ - propose_correct_text_quick_fix(c) for c in corrections - ], + data=DiagnosticData( + quickfixes=[ + propose_correct_text_quick_fix(c) + for c in corrections + ] + ), ) ) if known_field.warn_if_default and field_value == known_field.default_value: @@ -409,7 +424,11 @@ def _scan_for_syntax_errors_and_token_level_diagnostics( f'Spelling "{word}"', severity=DiagnosticSeverity.Hint, source="debputy", - data=[propose_correct_text_quick_fix(c) for c in corrections], + data=DiagnosticData( + quickfixes=[ + propose_correct_text_quick_fix(c) for c in corrections + ] + ), ) ) return first_error @@ -471,6 +490,32 @@ def _lint_debian_copyright( return diagnostics +@lsp_will_save_wait_until(_LANGUAGE_IDS) +def _debian_copyright_on_save_formatting( + ls: "DebputyLanguageServer", + params: WillSaveTextDocumentParams, +) -> Optional[Sequence[TextEdit]]: + doc = ls.workspace.get_text_document(params.text_document.uri) + lint_state = ls.lint_state(doc) + return deb822_format_file(lint_state, _DEP5_FILE_METADATA) + + +def _reformat_debian_copyright( + lint_state: LintState, +) -> Optional[Sequence[TextEdit]]: + return deb822_format_file(lint_state, _DEP5_FILE_METADATA) + + +@lsp_format_document(_LANGUAGE_IDS) +def _debian_copyright_on_save_formatting( + ls: "DebputyLanguageServer", + params: DocumentFormattingParams, +) -> Optional[Sequence[TextEdit]]: + doc = ls.workspace.get_text_document(params.text_document.uri) + lint_state = ls.lint_state(doc) + return deb822_format_file(lint_state, _DEP5_FILE_METADATA) + + @lsp_semantic_tokens_full(_LANGUAGE_IDS) def _debian_copyright_semantic_tokens_full( ls: "DebputyLanguageServer", diff --git a/src/debputy/lsp/lsp_debian_debputy_manifest.py b/src/debputy/lsp/lsp_debian_debputy_manifest.py index 74b5d7b..e75534b 100644 --- a/src/debputy/lsp/lsp_debian_debputy_manifest.py +++ b/src/debputy/lsp/lsp_debian_debputy_manifest.py @@ -30,6 +30,7 @@ from lsprotocol.types import ( ) from debputy.linting.lint_util import LintState +from debputy.lsp.diagnostics import DiagnosticData from debputy.lsp.quickfixes import propose_correct_text_quick_fix from debputy.manifest_parser.base_types import DebputyDispatchableType from debputy.plugin.api.feature_set import PluginProvidedFeatureSet @@ -227,7 +228,9 @@ def _unknown_key( message_format.format(key=key) + extra, DiagnosticSeverity.Error, source="debputy", - data=[propose_correct_text_quick_fix(n) for n in candidates], + data=DiagnosticData( + quickfixes=[propose_correct_text_quick_fix(n) for n in candidates] + ), ) return diagnostic, corrected_key diff --git a/src/debputy/lsp/lsp_debian_rules.py b/src/debputy/lsp/lsp_debian_rules.py index 7f5aef9..ec8d9d6 100644 --- a/src/debputy/lsp/lsp_debian_rules.py +++ b/src/debputy/lsp/lsp_debian_rules.py @@ -29,6 +29,7 @@ from lsprotocol.types import ( from debputy.debhelper_emulation import parse_drules_for_addons from debputy.linting.lint_util import LintState +from debputy.lsp.diagnostics import DiagnosticData from debputy.lsp.lsp_features import ( lint_diagnostics, lsp_standard_handler, @@ -309,7 +310,7 @@ def _lint_debian_rules_impl( r, msg, severity=DiagnosticSeverity.Warning, - data=fixes, + data=DiagnosticData(quickfixes=fixes), source=source, ) ) diff --git a/src/debputy/lsp/lsp_debian_tests_control.py b/src/debputy/lsp/lsp_debian_tests_control.py index cc27579..001dbe2 100644 --- a/src/debputy/lsp/lsp_debian_tests_control.py +++ b/src/debputy/lsp/lsp_debian_tests_control.py @@ -3,16 +3,12 @@ from typing import ( Union, Sequence, Tuple, - Iterator, Optional, - Iterable, Mapping, List, - Set, Dict, ) -from debputy.lsp.debputy_ls import DebputyLanguageServer from lsprotocol.types import ( DiagnosticSeverity, Range, @@ -21,7 +17,6 @@ from lsprotocol.types import ( CompletionItem, CompletionList, CompletionParams, - TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, DiagnosticRelatedInformation, Location, HoverParams, @@ -31,9 +26,14 @@ from lsprotocol.types import ( SemanticTokensParams, FoldingRangeParams, FoldingRange, + WillSaveTextDocumentParams, + TextEdit, + DocumentFormattingParams, ) from debputy.linting.lint_util import LintState +from debputy.lsp.debputy_ls import DebputyLanguageServer +from debputy.lsp.diagnostics import DiagnosticData from debputy.lsp.lsp_debian_control_reference_data import ( Deb822KnownField, DTestsCtrlFileMetadata, @@ -46,6 +46,8 @@ from debputy.lsp.lsp_features import ( lsp_standard_handler, lsp_folding_ranges, lsp_semantic_tokens_full, + lsp_will_save_wait_until, + lsp_format_document, ) from debputy.lsp.lsp_generic_deb822 import ( deb822_completer, @@ -53,6 +55,7 @@ from debputy.lsp.lsp_generic_deb822 import ( deb822_folding_ranges, deb822_semantic_tokens_full, deb822_token_iter, + deb822_format_file, ) from debputy.lsp.quickfixes import ( propose_correct_text_quick_fix, @@ -73,9 +76,6 @@ from debputy.lsp.vendoring._deb822_repro.parsing import ( Deb822KeyValuePairElement, LIST_SPACE_SEPARATED_INTERPRETATION, ) -from debputy.lsp.vendoring._deb822_repro.tokens import ( - Deb822Token, -) try: from debputy.lsp.vendoring._deb822_repro.locatable import ( @@ -99,10 +99,9 @@ _LANGUAGE_IDS = [ "debtestscontrol", ] -_DEP5_FILE_METADATA = DTestsCtrlFileMetadata() +_DTESTS_CTRL_FILE_METADATA = DTestsCtrlFileMetadata() lsp_standard_handler(_LANGUAGE_IDS, TEXT_DOCUMENT_CODE_ACTION) -lsp_standard_handler(_LANGUAGE_IDS, TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL) @lsp_hover(_LANGUAGE_IDS) @@ -110,7 +109,7 @@ def debian_tests_control_hover( ls: "DebputyLanguageServer", params: HoverParams, ) -> Optional[Hover]: - return deb822_hover(ls, params, _DEP5_FILE_METADATA) + return deb822_hover(ls, params, _DTESTS_CTRL_FILE_METADATA) @lsp_completer(_LANGUAGE_IDS) @@ -118,7 +117,7 @@ def debian_tests_control_completions( ls: "DebputyLanguageServer", params: CompletionParams, ) -> Optional[Union[CompletionList, Sequence[CompletionItem]]]: - return deb822_completer(ls, params, _DEP5_FILE_METADATA) + return deb822_completer(ls, params, _DTESTS_CTRL_FILE_METADATA) @lsp_folding_ranges(_LANGUAGE_IDS) @@ -126,7 +125,7 @@ def debian_tests_control_folding_ranges( ls: "DebputyLanguageServer", params: FoldingRangeParams, ) -> Optional[Sequence[FoldingRange]]: - return deb822_folding_ranges(ls, params, _DEP5_FILE_METADATA) + return deb822_folding_ranges(ls, params, _DTESTS_CTRL_FILE_METADATA) def _paragraph_representation_field( @@ -175,7 +174,7 @@ def _diagnostics_for_paragraph( diagnostics.append( Diagnostic( representation_field_range, - f'Stanza must have either a "Tests" or a "Test-Command" field', + 'Stanza must have either a "Tests" or a "Test-Command" field', severity=DiagnosticSeverity.Error, source="debputy", ) @@ -199,7 +198,10 @@ def _diagnostics_for_paragraph( normalized_field_name_lc = normalize_dctrl_field_name(field_name_lc) known_field = known_fields.get(normalized_field_name_lc) field_value = stanza[field_name] - field_range_te = kvpair.range_in_parent().relative_to(stanza_position) + kvpair_position = kvpair.position_in_parent().relative_to(stanza_position) + field_range_te = kvpair.field_token.range_in_parent().relative_to( + kvpair_position + ) field_position_te = field_range_te.start_pos field_range_server_units = te_range_to_lsp(field_range_te) field_range = position_codec.range_to_client_units( @@ -239,10 +241,12 @@ def _diagnostics_for_paragraph( f'The "{field_name}" looks like a typo of "{known_field.name}".', severity=DiagnosticSeverity.Warning, source="debputy", - data=[ - propose_correct_text_quick_fix(known_fields[m].name) - for m in candidates - ], + data=DiagnosticData( + quickfixes=[ + propose_correct_text_quick_fix(known_fields[m].name) + for m in candidates + ] + ), ) ) if field_value.strip() == "": @@ -260,6 +264,7 @@ def _diagnostics_for_paragraph( kvpair, stanza, stanza_position, + kvpair_position, position_codec, lines, field_name_typo_reported=field_name_typo_detected, @@ -300,9 +305,12 @@ def _diagnostics_for_paragraph( f'Spelling "{word}"', severity=DiagnosticSeverity.Hint, source="debputy", - data=[ - propose_correct_text_quick_fix(c) for c in corrections - ], + data=DiagnosticData( + quickfixes=[ + propose_correct_text_quick_fix(c) + for c in corrections + ] + ), ) ) if known_field.warn_if_default and field_value == known_field.default_value: @@ -407,7 +415,11 @@ def _scan_for_syntax_errors_and_token_level_diagnostics( f'Spelling "{word}"', severity=DiagnosticSeverity.Hint, source="debputy", - data=[propose_correct_text_quick_fix(c) for c in corrections], + data=DiagnosticData( + quickfixes=[ + propose_correct_text_quick_fix(c) for c in corrections + ] + ), ) ) return first_error @@ -453,6 +465,32 @@ def _lint_debian_tests_control( return diagnostics +@lsp_will_save_wait_until(_LANGUAGE_IDS) +def _debian_tests_control_on_save_formatting( + ls: "DebputyLanguageServer", + params: WillSaveTextDocumentParams, +) -> Optional[Sequence[TextEdit]]: + doc = ls.workspace.get_text_document(params.text_document.uri) + lint_state = ls.lint_state(doc) + return deb822_format_file(lint_state, _DTESTS_CTRL_FILE_METADATA) + + +def _reformat_debian_tests_control( + lint_state: LintState, +) -> Optional[Sequence[TextEdit]]: + return deb822_format_file(lint_state, _DTESTS_CTRL_FILE_METADATA) + + +@lsp_format_document(_LANGUAGE_IDS) +def _debian_tests_control_on_save_formatting( + ls: "DebputyLanguageServer", + params: DocumentFormattingParams, +) -> Optional[Sequence[TextEdit]]: + doc = ls.workspace.get_text_document(params.text_document.uri) + lint_state = ls.lint_state(doc) + return deb822_format_file(lint_state, _DTESTS_CTRL_FILE_METADATA) + + @lsp_semantic_tokens_full(_LANGUAGE_IDS) def _debian_tests_control_semantic_tokens_full( ls: "DebputyLanguageServer", @@ -461,5 +499,5 @@ def _debian_tests_control_semantic_tokens_full( return deb822_semantic_tokens_full( ls, request, - _DEP5_FILE_METADATA, + _DTESTS_CTRL_FILE_METADATA, ) diff --git a/src/debputy/lsp/lsp_dispatch.py b/src/debputy/lsp/lsp_dispatch.py index 5d09a44..99479dc 100644 --- a/src/debputy/lsp/lsp_dispatch.py +++ b/src/debputy/lsp/lsp_dispatch.py @@ -1,5 +1,4 @@ import asyncio -import os.path from typing import ( Dict, Sequence, @@ -9,8 +8,6 @@ from typing import ( Callable, Mapping, List, - Tuple, - Literal, ) from lsprotocol.types import ( @@ -35,6 +32,10 @@ from lsprotocol.types import ( CodeAction, CodeActionParams, SemanticTokensRegistrationOptions, + TextEdit, + TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, + WillSaveTextDocumentParams, + TEXT_DOCUMENT_FORMATTING, ) from debputy import __version__ @@ -45,6 +46,8 @@ from debputy.lsp.lsp_features import ( SEMANTIC_TOKENS_FULL_HANDLERS, CODE_ACTION_HANDLERS, SEMANTIC_TOKENS_LEGEND, + WILL_SAVE_WAIT_UNTIL_HANDLERS, + FORMAT_FILE_HANDLERS, ) from debputy.util import _info @@ -86,8 +89,8 @@ async def _open_document( @DEBPUTY_LANGUAGE_SERVER.feature(TEXT_DOCUMENT_DID_CHANGE) async def _changed_document( - ls: "DebputyLanguageServer", - params: DidChangeTextDocumentParams, + ls: "DebputyLanguageServer", + params: DidChangeTextDocumentParams, ) -> None: await _open_or_changed_document(ls, params) @@ -207,6 +210,34 @@ def _semantic_tokens_full( ) +@DEBPUTY_LANGUAGE_SERVER.feature(TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL) +def _will_save_wait_until( + ls: "DebputyLanguageServer", + params: WillSaveTextDocumentParams, +) -> Optional[Sequence[TextEdit]]: + return _dispatch_standard_handler( + ls, + params.text_document.uri, + params, + WILL_SAVE_WAIT_UNTIL_HANDLERS, + "On-save formatting", + ) + + +@DEBPUTY_LANGUAGE_SERVER.feature(TEXT_DOCUMENT_FORMATTING) +def _will_save_wait_until( + ls: "DebputyLanguageServer", + params: WillSaveTextDocumentParams, +) -> Optional[Sequence[TextEdit]]: + return _dispatch_standard_handler( + ls, + params.text_document.uri, + params, + FORMAT_FILE_HANDLERS, + "Full document formatting", + ) + + def _dispatch_standard_handler( ls: "DebputyLanguageServer", doc_uri: str, diff --git a/src/debputy/lsp/lsp_features.py b/src/debputy/lsp/lsp_features.py index e7b4445..63e4cd2 100644 --- a/src/debputy/lsp/lsp_features.py +++ b/src/debputy/lsp/lsp_features.py @@ -19,6 +19,7 @@ from lsprotocol.types import ( Diagnostic, DidOpenTextDocumentParams, SemanticTokensLegend, + TEXT_DOCUMENT_FORMATTING, ) from debputy.commands.debputy_cmd.context import CommandContext @@ -61,9 +62,14 @@ CODE_ACTION_HANDLERS = {} FOLDING_RANGE_HANDLERS = {} SEMANTIC_TOKENS_FULL_HANDLERS = {} WILL_SAVE_WAIT_UNTIL_HANDLERS = {} +FORMAT_FILE_HANDLERS = {} _ALIAS_OF = {} _STANDARD_HANDLERS = { + TEXT_DOCUMENT_FORMATTING: ( + FORMAT_FILE_HANDLERS, + on_save_trim_end_of_line_whitespace, + ), TEXT_DOCUMENT_CODE_ACTION: ( CODE_ACTION_HANDLERS, lambda ls, params: provide_standard_quickfixes_from_diagnostics(params), @@ -145,6 +151,16 @@ def lsp_folding_ranges(file_formats: Union[str, Sequence[str]]) -> Callable[[C], return _registering_wrapper(file_formats, FOLDING_RANGE_HANDLERS) +def lsp_will_save_wait_until( + file_formats: Union[str, Sequence[str]] +) -> Callable[[C], C]: + return _registering_wrapper(file_formats, WILL_SAVE_WAIT_UNTIL_HANDLERS) + + +def lsp_format_document(file_formats: Union[str, Sequence[str]]) -> Callable[[C], C]: + return _registering_wrapper(file_formats, FORMAT_FILE_HANDLERS) + + def lsp_semantic_tokens_full( file_formats: Union[str, Sequence[str]] ) -> Callable[[C], C]: @@ -214,6 +230,7 @@ def describe_lsp_features(context: CommandContext) -> None: ("folding ranges", FOLDING_RANGE_HANDLERS), ("semantic tokens", SEMANTIC_TOKENS_FULL_HANDLERS), ("on-save handler", WILL_SAVE_WAIT_UNTIL_HANDLERS), + ("format file handler", FORMAT_FILE_HANDLERS), ] print("LSP language IDs and their features:") all_ids = sorted(set(lid for _, t in feature_list for lid in t)) diff --git a/src/debputy/lsp/lsp_generic_deb822.py b/src/debputy/lsp/lsp_generic_deb822.py index e2124e4..409b27e 100644 --- a/src/debputy/lsp/lsp_generic_deb822.py +++ b/src/debputy/lsp/lsp_generic_deb822.py @@ -14,26 +14,6 @@ from typing import ( Callable, ) -from debputy.lsp.debputy_ls import DebputyLanguageServer -from debputy.lsp.lsp_debian_control_reference_data import ( - Deb822FileMetadata, - Deb822KnownField, - StanzaMetadata, - FieldValueClass, - F, - S, -) -from debputy.lsp.lsp_features import SEMANTIC_TOKEN_TYPES_IDS -from debputy.lsp.text_util import normalize_dctrl_field_name, te_position_to_lsp -from debputy.lsp.vendoring._deb822_repro import parse_deb822_file -from debputy.lsp.vendoring._deb822_repro.parsing import ( - Deb822KeyValuePairElement, - LIST_SPACE_SEPARATED_INTERPRETATION, - Deb822ParagraphElement, - Deb822ValueLineElement, -) -from debputy.lsp.vendoring._deb822_repro.tokens import tokenize_deb822_file, Deb822Token -from debputy.util import _info from lsprotocol.types import ( CompletionParams, CompletionList, @@ -49,7 +29,33 @@ from lsprotocol.types import ( FoldingRangeKind, SemanticTokensParams, SemanticTokens, + WillSaveTextDocumentParams, + TextEdit, + DocumentFormattingParams, +) + +from debputy.linting.lint_util import LintState +from debputy.lsp.debputy_ls import DebputyLanguageServer +from debputy.lsp.lsp_debian_control_reference_data import ( + Deb822FileMetadata, + Deb822KnownField, + StanzaMetadata, + F, + S, +) +from debputy.lsp.lsp_features import SEMANTIC_TOKEN_TYPES_IDS +from debputy.lsp.text_util import ( + normalize_dctrl_field_name, + te_position_to_lsp, + trim_end_of_line_whitespace, +) +from debputy.lsp.vendoring._deb822_repro import parse_deb822_file +from debputy.lsp.vendoring._deb822_repro.parsing import ( + Deb822KeyValuePairElement, + Deb822ParagraphElement, ) +from debputy.lsp.vendoring._deb822_repro.tokens import tokenize_deb822_file, Deb822Token +from debputy.util import _info try: from pygls.server import LanguageServer @@ -345,7 +351,10 @@ def _deb822_paragraph_semantic_tokens_full( stanza_idx=stanza_idx, ) for kvpair in stanza.iter_parts_of_type(Deb822KeyValuePairElement): - field_start = kvpair.key_position_in_stanza().relative_to(stanza_position) + kvpair_position = kvpair.position_in_parent().relative_to(stanza_position) + field_start = kvpair.field_token.position_in_parent().relative_to( + kvpair_position + ) comment = kvpair.comment_element if comment: comment_start_line = field_start.line_position - len(comment) @@ -378,8 +387,8 @@ def _deb822_paragraph_semantic_tokens_full( known_values = frozenset() interpretation = None - value_element_pos = kvpair.value_position_in_stanza().relative_to( - stanza_position + value_element_pos = kvpair.value_element.position_in_parent().relative_to( + kvpair_position ) if interpretation is None: # TODO: Emit tokens for value comments of unknown fields. @@ -410,6 +419,32 @@ def _deb822_paragraph_semantic_tokens_full( ) +def deb822_format_file( + lint_state: LintState, + file_metadata: Deb822FileMetadata[Any], +) -> Optional[Sequence[TextEdit]]: + effective_preference = lint_state.effective_preference + if effective_preference is None: + return trim_end_of_line_whitespace(lint_state.position_codec, lint_state.lines) + formatter = effective_preference.deb822_formatter() + lines = lint_state.lines + deb822_file = parse_deb822_file( + lines, + accept_files_with_duplicated_fields=True, + accept_files_with_error_tokens=True, + ) + return list( + file_metadata.reformat( + effective_preference, + deb822_file, + formatter, + lint_state.content, + lint_state.position_codec, + lines, + ) + ) + + def deb822_semantic_tokens_full( ls: "DebputyLanguageServer", request: SemanticTokensParams, diff --git a/src/debputy/lsp/lsp_reference_keyword.py b/src/debputy/lsp/lsp_reference_keyword.py new file mode 100644 index 0000000..44f43fd --- /dev/null +++ b/src/debputy/lsp/lsp_reference_keyword.py @@ -0,0 +1,45 @@ +import dataclasses +import textwrap +from typing import Optional, Union, Mapping + + +@dataclasses.dataclass(slots=True, frozen=True) +class Keyword: + value: str + hover_text: Optional[str] = None + is_obsolete: bool = False + replaced_by: Optional[str] = None + is_exclusive: bool = False + """For keywords in fields that allow multiple keywords, the `is_exclusive` can be + used for keywords that cannot be used with other keywords. As an example, the `all` + value in `Architecture` of `debian/control` cannot be used with any other architecture. + """ + + +def allowed_values(*values: Union[str, Keyword]) -> Mapping[str, Keyword]: + as_keywords = [k if isinstance(k, Keyword) else Keyword(k) for k in values] + as_mapping = {k.value: k for k in as_keywords if k.value} + # Simple bug check + assert len(as_keywords) == len(as_mapping) + return as_mapping + + +# This is the set of styles that `debputy` explicitly supports, which is more narrow than +# the ones in the config file. +ALL_PUBLIC_NAMED_STYLES = allowed_values( + Keyword( + "black", + hover_text=textwrap.dedent( + """\ + Uncompromising file formatting of Debian packaging files + + By using it, you agree to cede control over minutiae of hand-formatting. In + return, the formatter gives you speed, determinism, and freedom from style + discussions about formatting. + + The `black` style is inspired by the `black` Python code formatter. Like with + `black`, the style will evolve over time. + """ + ), + ), +) diff --git a/src/debputy/lsp/quickfixes.py b/src/debputy/lsp/quickfixes.py index 2d564f4..7a8a904 100644 --- a/src/debputy/lsp/quickfixes.py +++ b/src/debputy/lsp/quickfixes.py @@ -26,6 +26,7 @@ from lsprotocol.types import ( CodeActionKind, ) +from debputy.lsp.diagnostics import DiagnosticData from debputy.util import _warn try: @@ -234,10 +235,13 @@ def provide_standard_quickfixes_from_diagnostics( ) -> Optional[List[Union[Command, CodeAction]]]: actions: List[Union[Command, CodeAction]] = [] for diagnostic in code_action_params.context.diagnostics: - data = diagnostic.data - if not isinstance(data, list): - data = [data] - for action_suggestion in data: + if not isinstance(diagnostic.data, dict): + continue + data: DiagnosticData = cast("DiagnosticData", diagnostic.data) + quickfixes = data.get("quickfixes") + if quickfixes is None: + continue + for action_suggestion in quickfixes: if ( action_suggestion and isinstance(action_suggestion, Mapping) diff --git a/src/debputy/lsp/spellchecking.py b/src/debputy/lsp/spellchecking.py index f9027af..b767802 100644 --- a/src/debputy/lsp/spellchecking.py +++ b/src/debputy/lsp/spellchecking.py @@ -8,6 +8,7 @@ from typing import Iterable, FrozenSet, Tuple, Optional, List from debian.debian_support import Release from lsprotocol.types import Diagnostic, Range, Position, DiagnosticSeverity +from debputy.lsp.diagnostics import DiagnosticData from debputy.lsp.quickfixes import propose_correct_text_quick_fix from debputy.lsp.text_util import LintCapablePositionCodec from debputy.util import _info, _warn @@ -158,7 +159,9 @@ def spellcheck_line( f'Spelling "{word}"', severity=DiagnosticSeverity.Hint, source="debputy", - data=[propose_correct_text_quick_fix(c) for c in corrections], + data=DiagnosticData( + quickfixes=[propose_correct_text_quick_fix(c) for c in corrections] + ), ) diff --git a/src/debputy/lsp/style-preferences.yaml b/src/debputy/lsp/style-preferences.yaml new file mode 100644 index 0000000..6e76d2a --- /dev/null +++ b/src/debputy/lsp/style-preferences.yaml @@ -0,0 +1,67 @@ +formatting: + # Like Python's `black`; enforce a style and let me focus on more important matters + # Based on the numbers in #895570. + # + # Note that like Python's `black`, this is a moving target and it will evolve over time. + black: + deb822: + short-indent: true + always-wrap: true + trailing-separator: true + normalize-field-content: true + max-line-length: 79 + normalize-stanza-order: true + # Not yet implemented: + # normalize-field-order: true + +maintainer-rules: + + niels@thykier.net: + canonical-name: Niels Thykier + formatting: black + + zeha@debian.org: + canonical-name: Chris Hofstaedtler + formatting: black + + elbrus@debian.org: + canonical-name: Paul Gevers + formatting: black + + packages@qa.debian.org: + canonical-name: Debian QA Group + is-packaging-team: true # ish; it is for `debputy` definition + + + # Add ad-hoc single package maintainer teams below here (like foo@packages.debian.org) + # + # For these avoid setting "canonical-name": Since the maintainer is only used in one + # package, there is no value gain in debputy trying to remind other people how it's name + # is spelled (instead, it would just make `debputy` annoying if the name is ever changed) + # + # Note this should ideally just use `X-Style: black` if they use a style that can be + # put in `X-Style: black`. + + util-linux@packages.debian.org: + # Omitting canonical name for single use ad-hoc team maintainer name + formatting: black + is-packaging-team: true + + wtmpdb@packages.debian.org: + # Omitting canonical name for single use ad-hoc team maintainer name + formatting: black + is-packaging-team: true + + pdns@packages.debian.org: + # Omitting canonical name for single use ad-hoc team maintainer name + formatting: black + is-packaging-team: true + + pdns-recursor@packages.debian.org: + # Omitting canonical name for single use ad-hoc team maintainer name + formatting: black + is-packaging-team: true + + dnsdist@packages.debian.org: + # Omitting canonical name for single use ad-hoc team maintainer name + formatting: black diff --git a/src/debputy/lsp/style_prefs.py b/src/debputy/lsp/style_prefs.py new file mode 100644 index 0000000..5dfdac2 --- /dev/null +++ b/src/debputy/lsp/style_prefs.py @@ -0,0 +1,582 @@ +import dataclasses +import functools +import os.path +import re +import textwrap +from typing import ( + Type, + TypeVar, + Generic, + Optional, + List, + Union, + Callable, + Mapping, + Self, + Dict, +) + +from debputy.lsp.lsp_reference_keyword import ALL_PUBLIC_NAMED_STYLES +from debputy.lsp.vendoring._deb822_repro.types import FormatterCallback +from debputy.lsp.vendoring.wrap_and_sort import wrap_and_sort_formatter +from debputy.packages import SourcePackage +from debputy.util import _error +from debputy.yaml import MANIFEST_YAML +from debputy.yaml.compat import CommentedMap + +PT = TypeVar("PT", bool, str, int) + + +BUILTIN_STYLES = os.path.join(os.path.dirname(__file__), "style-preferences.yaml") + +_NORMALISE_FIELD_CONTENT_KEY = ["formatting", "deb822", "normalize-field-content"] +_UPLOADER_SPLIT_RE = re.compile(r"(?<=>)\s*,") + + +@dataclasses.dataclass(slots=True, frozen=True, kw_only=True) +class PreferenceOption(Generic[PT]): + key: Union[str, List[str]] + expected_type: Type[PT] + description: str + default_value: Optional[Union[PT, Callable[[CommentedMap], Optional[PT]]]] = None + + @property + def name(self) -> str: + if isinstance(self.key, str): + return self.key + return ".".join(self.key) + + @property + def attribute_name(self) -> str: + return self.name.replace("-", "_").replace(".", "_") + + def extract_value( + self, + filename: str, + key: str, + data: CommentedMap, + ) -> Optional[PT]: + v = data.mlget(self.key, list_ok=True) + if v is None: + default_value = self.default_value + if callable(default_value): + return default_value(data) + return default_value + if isinstance(v, self.expected_type): + return v + raise ValueError( + f'The value "{self.name}" for key {key} in file "{filename}" should have been a' + f" {self.expected_type} but it was not" + ) + + +def _is_packaging_team_default(m: CommentedMap) -> bool: + v = m.get("canonical-name") + if not isinstance(v, str): + return False + v = v.lower() + return v.endswith((" maintainer", " maintainers", " team")) + + +def _false_when_formatting_content(m: CommentedMap) -> Optional[bool]: + return m.mlget(_NORMALISE_FIELD_CONTENT_KEY, list_ok=True, default=False) is True + + +OPTIONS: List[PreferenceOption] = [ + PreferenceOption( + key="canonical-name", + expected_type=str, + description=textwrap.dedent( + """\ + Canonical spelling/case of the maintainer name. + + The `debputy` linter will emit a diagnostic if the name is not spelled exactly as provided here. + Can be useful to ensure your name is updated after a change of name. + """ + ), + ), + PreferenceOption( + key="is-packaging-team", + expected_type=bool, + default_value=_is_packaging_team_default, + description=textwrap.dedent( + """\ + Whether this entry is for a packaging team + + This affects how styles are applied when multiple maintainers (`Maintainer` + `Uploaders`) are listed + in `debian/control`. For package teams, the team preference prevails when the team is in the `Maintainer` + field. For non-packaging teams, generally the rules do not apply as soon as there are co-maintainers. + + The default is derived from the canonical name. If said name ends with phrases like "Team" or "Maintainer" + then the email is assumed to be for a team by default. + """ + ), + ), + PreferenceOption( + key=["formatting", "deb822", "short-indent"], + expected_type=bool, + description=textwrap.dedent( + """\ + Whether to use "short" indents for relationship fields (such as `Depends`). + + This roughly corresponds to `wrap-and-sort`'s `-s` option. + + **Example**: + + When `true`, the following: + ``` + Depends: foo, + bar + ``` + + would be reformatted as: + + ``` + Depends: + foo, + bar + ``` + + (Assuming `formatting.deb822.short-indent` is `false`) + + Note that defaults to `false` *if* (and only if) other formatting options will trigger reformat of + the field and this option has not been set. Setting this option can trigger reformatting of fields + that span multiple lines. + + Additionally, this only triggers when a field is being reformatted. Generally that requires + another option such as `formatting.deb822.normalize-field-content` for that to happen. + """ + ), + ), + PreferenceOption( + key=["formatting", "deb822", "always-wrap"], + expected_type=bool, + description=textwrap.dedent( + """\ + Whether to always wrap fields (such as `Depends`). + + This roughly corresponds to `wrap-and-sort`'s `-a` option. + + **Example**: + + When `true`, the following: + ``` + Depends: foo, bar + ``` + + would be reformatted as: + + ``` + Depends: foo, + bar + ``` + + (Assuming `formatting.deb822.short-indent` is `false`) + + This option only applies to fields where formatting is a pure style preference. As an + example, `Description` (`debian/control`) or `License` (`debian/copyright`) will not + be affected by this option. + + Note: When `true`, this option overrules `formatting.deb822.max-line-length` when they interact. + Additionally, this only triggers when a field is being reformatted. Generally that requires + another option such as `formatting.deb822.normalize-field-content` for that to happen. + """ + ), + ), + PreferenceOption( + key=["formatting", "deb822", "trailing-separator"], + expected_type=bool, + default_value=False, + description=textwrap.dedent( + """\ + Whether to always end relationship fields (such as `Depends`) with a trailing separator. + + This roughly corresponds to `wrap-and-sort`'s `-t` option. + + **Example**: + + When `true`, the following: + ``` + Depends: foo, + bar + ``` + + would be reformatted as: + + ``` + Depends: foo, + bar, + ``` + + Note: The trailing separator is only applied if the field is reformatted. This means this option + generally requires another option to trigger reformatting (like + `formatting.deb822.normalize-field-content`). + """ + ), + ), + PreferenceOption( + key=["formatting", "deb822", "max-line-length"], + expected_type=int, + default_value=79, + description=textwrap.dedent( + """\ + How long a value line can be before it should be line wrapped. + + This roughly corresponds to `wrap-and-sort`'s `--max-line-length` option. + + This option only applies to fields where formatting is a pure style preference. As an + example, `Description` (`debian/control`) or `License` (`debian/copyright`) will not + be affected by this option. + + Note: When `formatting.deb822.always-wrap` is `true`, then this option will be overruled. + Additionally, this only triggers when a field is being reformatted. Generally that requires + another option such as `formatting.deb822.normalize-field-content` for that to happen. + """ + ), + ), + PreferenceOption( + key=_NORMALISE_FIELD_CONTENT_KEY, + expected_type=bool, + default_value=False, + description=textwrap.dedent( + """\ + Whether to normalize field content. + + This roughly corresponds to the subset of `wrap-and-sort` that normalizes field content + like sorting and normalizing relations or sorting the architecture field. + + **Example**: + + When `true`, the following: + ``` + Depends: foo, + bar|baz + ``` + + would be reformatted as: + + ``` + Depends: bar | baz, + foo, + ``` + + This causes affected fields to always be rewritten and therefore be sure that other options + such as `formatting.deb822.short-indent` or `formatting.deb822.always-wrap` is set according + to taste. + + Note: The field may be rewritten without this being set to `true`. As an example, the `always-wrap` + option can trigger a field rewrite. However, in that case, the values (including any internal whitespace) + are left as-is while the whitespace normalization between the values is still applied. + """ + ), + ), + PreferenceOption( + key=["formatting", "deb822", "normalize-field-order"], + expected_type=bool, + default_value=False, + description=textwrap.dedent( + """\ + Whether to normalize field order in a stanza. + + There is no `wrap-and-sort` feature matching this. + + **Example**: + + When `true`, the following: + ``` + Depends: bar + Package: foo + ``` + + would be reformatted as: + + ``` + Depends: foo + Package: bar + ``` + + The field order is not by field name but by a logic order defined in `debputy` based on existing + conventions. The `deb822` format does not dictate any field order inside stanzas in general, so + reordering of fields is generally safe. + + If a field of the first stanza is known to be a format discriminator such as the `Format' in + `debian/copyright`, then it will be put first. Generally that matches existing convention plus + it maximizes the odds that existing tools will correctly identify the file format. + """ + ), + ), + PreferenceOption( + key=["formatting", "deb822", "normalize-stanza-order"], + expected_type=bool, + default_value=False, + description=textwrap.dedent( + """\ + Whether to normalize stanza order in a file. + + This roughly corresponds to `wrap-and-sort`'s `-kb` feature except this may apply to other deb822 + files. + + **Example**: + + When `true`, the following: + ``` + Source: zzbar + + Package: zzbar + + Package: zzbar-util + + Package: libzzbar-dev + + Package: libzzbar2 + ``` + + would be reformatted as: + + ``` + Source: zzbar + + Package: zzbar + + Package: libzzbar2 + + Package: libzzbar-dev + + Package: zzbar-util + ``` + + Reordering will only performed when: + 1) There is a convention for a normalized order + 2) The normalization can be performed without changing semantics + + Note: This option only guards style/preference related re-ordering. It does not influence + warnings about the order being semantic incorrect (which will still be emitted regardless + of this setting). + """ + ), + ), +] + + +@dataclasses.dataclass(slots=True, frozen=True) +class EffectivePreference: + formatting_deb822_short_indent: Optional[bool] = None + formatting_deb822_always_wrap: Optional[bool] = None + formatting_deb822_trailing_separator: bool = False + formatting_deb822_normalize_field_content: bool = False + formatting_deb822_normalize_field_order: bool = False + formatting_deb822_normalize_stanza_order: bool = False + formatting_deb822_max_line_length: int = 79 + + @classmethod + def from_file( + cls, + filename: str, + key: str, + stylees: CommentedMap, + ) -> Self: + attr = {} + + for option in OPTIONS: + if not hasattr(cls, option.attribute_name): + continue + value = option.extract_value(filename, key, stylees) + attr[option.attribute_name] = value + return cls(**attr) # type: ignore + + @classmethod + def aligned_preference( + cls, + a: Optional["EffectivePreference"], + b: Optional["EffectivePreference"], + ) -> Optional["EffectivePreference"]: + if a is None or b is None: + return None + + for option in OPTIONS: + attr_name = option.attribute_name + if not hasattr(EffectivePreference, attr_name): + continue + a_value = getattr(a, attr_name) + b_value = getattr(b, attr_name) + if a_value != b_value: + print(f"{attr_name} was misaligned") + return None + return a + + def deb822_formatter(self) -> FormatterCallback: + line_length = self.formatting_deb822_max_line_length + return wrap_and_sort_formatter( + 1 if self.formatting_deb822_short_indent else "FIELD_NAME_LENGTH", + trailing_separator=self.formatting_deb822_trailing_separator, + immediate_empty_line=self.formatting_deb822_short_indent or False, + max_line_length_one_liner=( + 0 if self.formatting_deb822_always_wrap else line_length + ), + ) + + +@dataclasses.dataclass(slots=True, frozen=True) +class MaintainerPreference(EffectivePreference): + canonical_name: Optional[str] = None + is_packaging_team: bool = False + + def as_effective_pref(self) -> EffectivePreference: + fields = { + k: v + for k, v in dataclasses.asdict(self).items() + if hasattr(EffectivePreference, k) + } + return EffectivePreference(**fields) + + +class StylePreferenceTable: + + def __init__( + self, + named_styles: Mapping[str, EffectivePreference], + maintainer_preferences: Mapping[str, MaintainerPreference], + ) -> None: + self._named_styles = named_styles + self._maintainer_preferences = maintainer_preferences + + @classmethod + def load_styles(cls) -> Self: + named_styles: Dict[str, EffectivePreference] = {} + maintainer_preferences: Dict[str, MaintainerPreference] = {} + with open(BUILTIN_STYLES) as fd: + parse_file(named_styles, maintainer_preferences, BUILTIN_STYLES, fd) + + missing_keys = set(named_styles.keys()).difference( + ALL_PUBLIC_NAMED_STYLES.keys() + ) + if missing_keys: + missing_styles = ", ".join(sorted(missing_keys)) + _error( + f"The following named styles are public API but not present in the config file: {missing_styles}" + ) + + # TODO: Support fetching styles online to pull them in faster than waiting for a stable release. + return cls(named_styles, maintainer_preferences) + + @property + def named_styles(self) -> Mapping[str, EffectivePreference]: + return self._named_styles + + @property + def maintainer_preferences(self) -> Mapping[str, MaintainerPreference]: + return self._maintainer_preferences + + +def parse_file( + named_styles: Dict[str, EffectivePreference], + maintainer_preferences: Dict[str, MaintainerPreference], + filename: str, + fd, +) -> None: + content = MANIFEST_YAML.load(fd) + if not isinstance(content, CommentedMap): + raise ValueError( + f'The file "{filename}" should be a YAML file with a single mapping at the root' + ) + try: + maintainer_rules = content["maintainer-rules"] + if not isinstance(maintainer_rules, CommentedMap): + raise KeyError("maintainer-rules") from None + except KeyError: + raise ValueError( + f'The file "{filename}" should have a "maintainer-rules" key which must be a mapping.' + ) + named_styles_raw = content.get("formatting") + if named_styles_raw is None or not isinstance(named_styles_raw, CommentedMap): + named_styles_raw = {} + + for style_name, content in named_styles_raw.items(): + wrapped_style = CommentedMap({"formatting": content}) + style = EffectivePreference.from_file( + filename, + style_name, + wrapped_style, + ) + named_styles[style_name] = style + + for maintainer_email, maintainer_styles in maintainer_rules.items(): + if not isinstance(maintainer_styles, CommentedMap): + line_no = maintainer_rules.lc.key(maintainer_email).line + raise ValueError( + f'The value for maintainer "{maintainer_email}" should have been a mapping,' + f' but it is not. The problem entry is at line {line_no} in "{filename}"' + ) + formatting = maintainer_styles.get("formatting") + if isinstance(formatting, str): + try: + style = named_styles_raw[formatting] + except KeyError: + line_no = maintainer_rules.lc.key(maintainer_email).line + raise ValueError( + f'The maintainer "{maintainer_email}" requested the named style "{formatting}",' + f' but said style was not defined {filename}. The problem entry is at line {line_no} in "{filename}"' + ) from None + maintainer_styles["formatting"] = style + maintainer_preferences[maintainer_email] = MaintainerPreference.from_file( + filename, + maintainer_email, + maintainer_styles, + ) + + +@functools.lru_cache(64) +def extract_maint_email(maint: str) -> str: + if not maint.endswith(">"): + return "" + + try: + idx = maint.index("<") + except ValueError: + return "" + return maint[idx + 1 : -1] + + +def determine_effective_style( + style_preference_table: StylePreferenceTable, + source_package: SourcePackage, +) -> Optional[EffectivePreference]: + style = source_package.fields.get("X-Style") + if style is not None: + if style not in ALL_PUBLIC_NAMED_STYLES: + return None + return style_preference_table.named_styles.get(style) + + maint = source_package.fields.get("Maintainer") + if maint is None: + return None + maint_email = extract_maint_email(maint) + maint_style = style_preference_table.maintainer_preferences.get(maint_email) + # Special-case "@packages.debian.org" when missing, since they are likely to be "ad-hoc" + # teams that will not be registered. In that case, we fall back to looking at the uploader + # preferences as-if the maintainer had not been listed at all. + if maint_style is None and not maint_email.endswith("@packages.debian.org"): + return None + if maint_style is not None and maint_style.is_packaging_team: + # When the maintainer is registered as a packaging team, then we assume the packaging + # team's style applies unconditionally. + return maint_style.as_effective_pref() + uploaders = source_package.fields.get("Uploaders") + if uploaders is None: + return maint_style.as_effective_pref() if maint_style is not None else None + all_styles: List[Optional[EffectivePreference]] = [] + if maint_style is not None: + all_styles.append(maint_style) + for uploader in _UPLOADER_SPLIT_RE.split(uploaders): + uploader_email = extract_maint_email(uploader) + uploader_style = style_preference_table.maintainer_preferences.get( + uploader_email + ) + all_styles.append(uploader_style) + + if not all_styles: + return None + r = functools.reduce(EffectivePreference.aligned_preference, all_styles) + if isinstance(r, MaintainerPreference): + return r.as_effective_pref() + return r diff --git a/src/debputy/lsp/text_edit.py b/src/debputy/lsp/text_edit.py index 770a837..4b210b2 100644 --- a/src/debputy/lsp/text_edit.py +++ b/src/debputy/lsp/text_edit.py @@ -4,7 +4,7 @@ # Copyright 2021- Python Language Server Contributors. # License: Expat (MIT/X11) # -from typing import List +from typing import List, Sequence from lsprotocol.types import Range, TextEdit, Position @@ -88,7 +88,11 @@ def offset_at_position(lines: List[str], server_position: Position) -> int: return col + sum(len(line) for line in lines[:row]) -def apply_text_edits(text: str, lines: List[str], text_edits: List[TextEdit]) -> str: +def apply_text_edits( + text: str, + lines: List[str], + text_edits: Sequence[TextEdit], +) -> str: sorted_edits = merge_sort_text_edits( [get_well_formatted_edit(e) for e in text_edits] ) diff --git a/src/debputy/lsp/text_util.py b/src/debputy/lsp/text_util.py index ef4cd0a..e58990f 100644 --- a/src/debputy/lsp/text_util.py +++ b/src/debputy/lsp/text_util.py @@ -5,6 +5,7 @@ from lsprotocol.types import ( Position, Range, WillSaveTextDocumentParams, + DocumentFormattingParams, ) from debputy.linting.lint_util import LinterPositionCodec @@ -73,18 +74,24 @@ def normalize_dctrl_field_name(f: str) -> str: def on_save_trim_end_of_line_whitespace( ls: "LanguageServer", - params: WillSaveTextDocumentParams, + params: Union[WillSaveTextDocumentParams, DocumentFormattingParams], ) -> Optional[Sequence[TextEdit]]: doc = ls.workspace.get_text_document(params.text_document.uri) - return trim_end_of_line_whitespace(doc, doc.lines) + return trim_end_of_line_whitespace(doc.position_codec, doc.lines) def trim_end_of_line_whitespace( - doc: "TextDocument", + position_codec: "LintCapablePositionCodec", lines: List[str], + *, + line_range: Optional[Iterable[int]] = None, + line_relative_line_no: int = 0, ) -> Optional[Sequence[TextEdit]]: edits = [] - for line_no, orig_line in enumerate(lines): + if line_range is None: + line_range = range(0, len(lines)) + for line_no in line_range: + orig_line = lines[line_no] orig_len = len(orig_line) if orig_line.endswith("\n"): orig_len -= 1 @@ -92,16 +99,20 @@ def trim_end_of_line_whitespace( if stripped_len == orig_len: continue - edit_range = doc.position_codec.range_to_client_units( + stripped_len_client_off = position_codec.client_num_units( + orig_line[:stripped_len] + ) + orig_len_client_off = position_codec.client_num_units(orig_line[:orig_len]) + edit_range = position_codec.range_to_client_units( lines, Range( Position( - line_no, - stripped_len, + line_no + line_relative_line_no, + stripped_len_client_off, ), Position( - line_no, - orig_len, + line_no + line_relative_line_no, + orig_len_client_off, ), ), ) diff --git a/src/debputy/lsp/vendoring/_deb822_repro/_util.py b/src/debputy/lsp/vendoring/_deb822_repro/_util.py index a79426d..0e68a03 100644 --- a/src/debputy/lsp/vendoring/_deb822_repro/_util.py +++ b/src/debputy/lsp/vendoring/_deb822_repro/_util.py @@ -279,7 +279,7 @@ def len_check_iterator( Value parser did not fully cover the entire line with tokens ( missing range {covered}..{content_len}). Occurred when parsing "{content}" """ - ).format(covered=covered, content_len=content_len, line=content) + ).format(covered=covered, content_len=content_len, content=content) raise ValueError(msg) msg = textwrap.dedent( """\ diff --git a/src/debputy/lsp/vendoring/_deb822_repro/formatter.py b/src/debputy/lsp/vendoring/_deb822_repro/formatter.py index a2b797b..066fa3f 100644 --- a/src/debputy/lsp/vendoring/_deb822_repro/formatter.py +++ b/src/debputy/lsp/vendoring/_deb822_repro/formatter.py @@ -207,6 +207,7 @@ def one_value_per_line_formatter( indent_len = len(name) + 2 else: indent_len = indentation + assert isinstance(indent_len, int) # hint for PyCharm indent = " " * indent_len emitted_first_line = False diff --git a/src/debputy/lsp/vendoring/_deb822_repro/locatable.py b/src/debputy/lsp/vendoring/_deb822_repro/locatable.py index afebf14..b5da920 100644 --- a/src/debputy/lsp/vendoring/_deb822_repro/locatable.py +++ b/src/debputy/lsp/vendoring/_deb822_repro/locatable.py @@ -52,9 +52,7 @@ class Position: >>> parent: Locatable = ... # doctest: +SKIP >>> children: Iterable[Locatable] = ... # doctest: +SKIP >>> # This will expensive - >>> parent_pos = parent.position_in_file( # doctest: +SKIP - ... skip_leading_comments=False - ... ) + >>> parent_pos = parent.position_in_file() # doctest: +SKIP >>> for child in children: # doctest: +SKIP ... child_pos = child.position_in_parent() ... # Avoid a position_in_file() for each child @@ -181,9 +179,7 @@ class Range: >>> parent: Locatable = ... # doctest: +SKIP >>> children: Iterable[Locatable] = ... # doctest: +SKIP >>> # This will expensive - >>> parent_pos = parent.position_in_file( # doctest: +SKIP - ... skip_leading_comments=False - ... ) + >>> parent_pos = parent.position_in_file() # doctest: +SKIP >>> for child in children: # doctest: +SKIP ... child_range = child.range_in_parent() ... # Avoid a position_in_file() for each child @@ -312,26 +308,12 @@ class Locatable: # type: () -> Optional[Deb822Element] raise NotImplementedError - def position_in_parent(self, *, skip_leading_comments: bool = True) -> Position: + def position_in_parent(self) -> Position: """The start position of this token/element inside its parent This is operation is generally linear to the number of "parts" (elements/tokens) inside the parent. - - :param skip_leading_comments: If True, then if any leading comment that - that can be skipped will be excluded in the position of this locatable. - This is useful if you want the position "semantic" content of a field - without also highlighting a leading comment. Remember to align this - parameter with the `size` call, so the range does not "overshoot" - into the next element (or falls short and only covers part of an - element). Note that this option can only be used to filter out leading - comments when the comments are a subset of the element. It has no - effect on elements that are entirely made of comments. """ - # pylint: disable=unused-argument - # Note: The base class makes no assumptions about what tokens can be skipped, - # therefore, skip_leading_comments is unused here. However, I do not want the - # API to differ between elements and tokens. parent = self.parent_element if parent is None: @@ -343,32 +325,23 @@ class Locatable: ) span = Range.from_position_and_sizes( START_POSITION, - (x.size(skip_leading_comments=False) for x in relevant_parts), + (x.size() for x in relevant_parts), ) return span.end_pos - def range_in_parent(self, *, skip_leading_comments: bool = True) -> Range: + def range_in_parent(self) -> Range: """The range of this token/element inside its parent This is operation is generally linear to the number of "parts" (elements/tokens) inside the parent. - - :param skip_leading_comments: If True, then if any leading comment that - that can be skipped will be excluded in the position of this locatable. - This is useful if you want the position "semantic" content of a field - without also highlighting a leading comment. Remember to align this - parameter with the `size` call, so the range does not "overshoot" - into the next element (or falls short and only covers part of an - element). Note that this option can only be used to filter out leading - comments when the comments are a subset of the element. It has no - effect on elements that are entirely made of comments. """ - pos = self.position_in_parent(skip_leading_comments=skip_leading_comments) + pos = self.position_in_parent() return Range.from_position_and_size( - pos, self.size(skip_leading_comments=skip_leading_comments) + pos, + self.size(), ) - def position_in_file(self, *, skip_leading_comments: bool = True) -> Position: + def position_in_file(self) -> Position: """The start position of this token/element in this file This is an *expensive* operation and in many cases have to traverse @@ -377,37 +350,14 @@ class Locatable: `position_in_parent()` combined with `child_position.relative_to(parent_position)` - :param skip_leading_comments: If True, then if any leading comment that - that can be skipped will be excluded in the position of this locatable. - This is useful if you want the position "semantic" content of a field - without also highlighting a leading comment. Remember to align this - parameter with the `size` call, so the range does not "overshoot" - into the next element (or falls short and only covers part of an - element). Note that this option can only be used to filter out leading - comments when the comments are a subset of the element. It has no - effect on elements that are entirely made of comments. """ - position = self.position_in_parent( - skip_leading_comments=skip_leading_comments, - ) + position = self.position_in_parent() parent = self.parent_element if parent is not None: - parent_position = parent.position_in_file(skip_leading_comments=False) + parent_position = parent.position_in_file() position = position.relative_to(parent_position) return position - def size(self, *, skip_leading_comments: bool = True) -> Range: - """Describe the objects size as a continuous range - - :param skip_leading_comments: If True, then if any leading comment that - that can be skipped will be excluded in the position of this locatable. - This is useful if you want the position "semantic" content of a field - without also highlighting a leading comment. Remember to align this - parameter with the `position_in_file` or `position_in_parent` call, - so the range does not "overshoot" into the next element (or falls - short and only covers part of an element). Note that this option can - only be used to filter out leading comments when the comments are a - subset of the element. It has no effect on elements that are entirely - made of comments. - """ + def size(self) -> Range: + """Describe the objects size as a continuous range""" raise NotImplementedError diff --git a/src/debputy/lsp/vendoring/_deb822_repro/parsing.py b/src/debputy/lsp/vendoring/_deb822_repro/parsing.py index c5753e2..71aa79a 100644 --- a/src/debputy/lsp/vendoring/_deb822_repro/parsing.py +++ b/src/debputy/lsp/vendoring/_deb822_repro/parsing.py @@ -620,8 +620,7 @@ class Deb822ParsedTokenList( # type: () -> str return "".join(t.text for t in self._iter_content_as_tokens()) - def _update_field(self): - # type: () -> None + def _generate_kvpair(self) -> "Deb822KeyValuePairElement": kvpair_element = self._kvpair_element field_name = kvpair_element.field_name token_list = self._token_list @@ -669,7 +668,17 @@ class Deb822ParsedTokenList( assert isinstance(paragraph, Deb822NoDuplicateFieldsParagraphElement) new_kvpair_element = paragraph.get_kvpair_element(field_name) assert new_kvpair_element is not None - kvpair_element.value_element = new_kvpair_element.value_element + return new_kvpair_element + + def convert_to_text(self, *, with_field_name: bool = False) -> str: + kvpair = self._generate_kvpair() + element = kvpair if with_field_name else kvpair.value_element + return element.convert_to_text() + + def _update_field(self): + # type: () -> None + kvpair_element = self._kvpair_element + kvpair_element.value_element = self._generate_kvpair().value_element self._changed = False def sort_elements( @@ -1119,12 +1128,12 @@ class Deb822Element(Locatable): if parent is self.parent_element: self._parent_element = None - def size(self, *, skip_leading_comments: bool = True) -> Range: + def size(self) -> Range: size_cache = self._full_size_cache if size_cache is None: size_cache = Range.from_position_and_sizes( START_POSITION, - (p.size(skip_leading_comments=False) for p in self.iter_parts()), + (p.size() for p in self.iter_parts()), ) self._full_size_cache = size_cache return size_cache @@ -1147,21 +1156,21 @@ class Deb822InterpretationProxyElement(Deb822Element): # type: () -> Iterable[TokenOrElement] return iter(self.parts) - def position_in_parent(self, *, skip_leading_comments: bool = True) -> Position: + def position_in_parent(self) -> Position: parent = self.parent_element if parent is None: raise RuntimeError("parent was garbage collected") return parent.position_in_parent() - def position_in_file(self, *, skip_leading_comments: bool = True) -> Position: + def position_in_file(self) -> Position: parent = self.parent_element if parent is None: raise RuntimeError("parent was garbage collected") return parent.position_in_file() - def size(self, *, skip_leading_comments: bool = True) -> Range: + def size(self) -> Range: # Same as parent except we never use a cache. - sizes = (p.size(skip_leading_comments=False) for p in self.iter_parts()) + sizes = (p.size() for p in self.iter_parts()) return Range.from_position_and_sizes(START_POSITION, sizes) @@ -1294,28 +1303,6 @@ class Deb822ValueLineElement(Deb822Element): if self._newline_token: yield self._newline_token - def size(self, *, skip_leading_comments: bool = True) -> Range: - if skip_leading_comments: - return Range.from_position_and_sizes( - START_POSITION, - ( - p.size(skip_leading_comments=False) - for p in self.iter_parts() - if not p.is_comment - ), - ) - return super().size(skip_leading_comments=skip_leading_comments) - - def position_in_parent(self, *, skip_leading_comments: bool = True) -> Position: - base_pos = super().position_in_parent(skip_leading_comments=False) - if skip_leading_comments: - for p in self.iter_parts(): - if p.is_comment: - continue - non_comment_pos = p.position_in_parent(skip_leading_comments=False) - base_pos = non_comment_pos.relative_to(base_pos) - return base_pos - class Deb822ValueElement(Deb822Element): __slots__ = ("_value_entry_elements",) @@ -1503,43 +1490,9 @@ class Deb822KeyValuePairElement(Deb822Element): yield self._separator_token yield self._value_element - def key_position_in_stanza(self) -> Position: - position = super().position_in_parent(skip_leading_comments=False) - if self._comment_element: - field_pos = self._field_token.position_in_parent() - position = field_pos.relative_to(position) - return position - def value_position_in_stanza(self) -> Position: - position = super().position_in_parent(skip_leading_comments=False) - if self._comment_element: - value_pos = self._value_element.position_in_parent() - position = value_pos.relative_to(position) - return position - - def position_in_parent( - self, - *, - skip_leading_comments: bool = True, - ) -> Position: - position = super().position_in_parent(skip_leading_comments=False) - if skip_leading_comments: - if self._comment_element: - field_pos = self._field_token.position_in_parent() - position = field_pos.relative_to(position) - return position - - def size(self, *, skip_leading_comments: bool = True) -> Range: - if skip_leading_comments: - return Range.from_position_and_sizes( - START_POSITION, - ( - p.size(skip_leading_comments=False) - for p in self.iter_parts() - if not p.is_comment - ), - ) - return super().size(skip_leading_comments=False) + value_pos = self._value_element.position_in_parent() + return value_pos.relative_to(self.position_in_parent()) def _format_comment(c): @@ -3167,11 +3120,11 @@ class Deb822FileElement(Deb822Element): t.parent_element = self return t - def position_in_parent(self, *, skip_leading_comments: bool = True) -> Position: + def position_in_parent(self) -> Position: # Recursive base-case return START_POSITION - def position_in_file(self, *, skip_leading_comments: bool = True) -> Position: + def position_in_file(self) -> Position: # By definition return START_POSITION diff --git a/src/debputy/lsp/vendoring/_deb822_repro/tokens.py b/src/debputy/lsp/vendoring/_deb822_repro/tokens.py index 88d2058..b11b580 100644 --- a/src/debputy/lsp/vendoring/_deb822_repro/tokens.py +++ b/src/debputy/lsp/vendoring/_deb822_repro/tokens.py @@ -174,7 +174,7 @@ class Deb822Token(Locatable): # type: () -> str return self._text - def size(self, *, skip_leading_comments: bool = False) -> Range: + def size(self) -> Range: # As tokens are an atomic unit token_size = self._token_size if token_size is not None: @@ -499,6 +499,11 @@ def _value_line_tokenizer(func): def whitespace_split_tokenizer(v): # type: (str) -> Iterable[Deb822Token] assert "\n" not in v + if not v or v.isspace(): + # Special-case: Empty field/whitespace only field + if v: + yield Deb822SpaceSeparatorToken(sys.intern(v)) + return for match in _RE_WHITESPACE_SEPARATED_WORD_LIST.finditer(v): space_before, word, space_after = match.groups() if space_before: diff --git a/src/debputy/lsp/vendoring/wrap_and_sort.py b/src/debputy/lsp/vendoring/wrap_and_sort.py new file mode 100644 index 0000000..5b25891 --- /dev/null +++ b/src/debputy/lsp/vendoring/wrap_and_sort.py @@ -0,0 +1,113 @@ +# Code extracted from devscripts/wrap-and-sort with typing added +import re +from typing import Tuple, Iterable, Literal, Union + +from debputy.lsp.vendoring._deb822_repro.formatter import ( + one_value_per_line_formatter, + FormatterContentToken, +) +from debputy.lsp.vendoring._deb822_repro.types import FormatterCallback + +PACKAGE_SORT = re.compile("^[a-z0-9]") + + +def _sort_packages_key(package: str) -> Tuple[int, str]: + # Sort dependencies starting with a "real" package name before ones starting + # with a substvar + return 0 if PACKAGE_SORT.search(package) else 1, package + + +def _emit_one_line_value( + value_tokens: Iterable[FormatterContentToken], + sep_token: FormatterContentToken, + trailing_separator: bool, +) -> Iterable[Union[str, FormatterContentToken]]: + first_token = True + yield " " + for token in value_tokens: + if not first_token: + yield sep_token + if not sep_token.is_whitespace: + yield " " + first_token = False + yield token + if trailing_separator and not sep_token.is_whitespace: + yield sep_token + yield "\n" + + +def wrap_and_sort_formatter( + indentation: Union[int, Literal["FIELD_NAME_LENGTH"]], + trailing_separator: bool = True, + immediate_empty_line: bool = False, + max_line_length_one_liner: int = 0, +) -> FormatterCallback: + """Provide a formatter that can handle indentation and trailing separators + + This is a custom wrap-and-sort formatter capable of supporting wrap-and-sort's + needs. Where possible it delegates to python-debian's own formatter. + + :param indentation: Either the literal string "FIELD_NAME_LENGTH" or a positive + integer, which determines the indentation fields. If it is an integer, + then a fixed indentation is used (notably the value 1 ensures the shortest + possible indentation). Otherwise, if it is "FIELD_NAME_LENGTH", then the + indentation is set such that it aligns the values based on the field name. + This parameter only affects values placed on the second line or later lines. + :param trailing_separator: If True, then the last value will have a trailing + separator token (e.g., ",") after it. + :param immediate_empty_line: Whether the value should always start with an + empty line. If True, then the result becomes something like "Field:\n value". + This parameter only applies to the values that will be formatted over more than + one line. + :param max_line_length_one_liner: If greater than zero, then this is the max length + of the value if it is crammed into a "one-liner" value. If the value(s) fit into + one line, this parameter will overrule immediate_empty_line. + + """ + if indentation != "FIELD_NAME_LENGTH" and indentation < 1: + raise ValueError('indentation must be at least 1 (or "FIELD_NAME_LENGTH")') + + # The python-debian library provides support for all cases except cramming + # everything into a single line. So we "only" have to implement the single-line + # case(s) ourselves (which sadly takes plenty of code on its own) + + _chain_formatter = one_value_per_line_formatter( + indentation, + trailing_separator=trailing_separator, + immediate_empty_line=immediate_empty_line, + ) + + if max_line_length_one_liner < 1: + return _chain_formatter + + def _formatter(name, sep_token, formatter_tokens): + # We should have unconditionally delegated to the python-debian formatter + # if max_line_length_one_liner was set to "wrap_always" + assert max_line_length_one_liner > 0 + all_tokens = list(formatter_tokens) + values_and_comments = [x for x in all_tokens if x.is_comment or x.is_value] + # There are special-cases where you could do a one-liner with comments, but + # they are probably a lot more effort than it is worth investing. + # - If you are here because you disagree, patches welcome. :) + if all(x.is_value for x in values_and_comments): + # We use " " (1 char) or ", " (2 chars) as separated depending on the field. + # (at the time of writing, wrap-and-sort only uses this formatted for + # dependency fields meaning this will be "2" - but now it is future proof). + chars_between_values = 1 + (0 if sep_token.is_whitespace else 1) + # Compute the total line length of the field as the sum of all values + total_len = sum(len(x.text) for x in values_and_comments) + # ... plus the separators + total_len += (len(values_and_comments) - 1) * chars_between_values + # plus the field name + the ": " after the field name + total_len += len(name) + 2 + if total_len <= max_line_length_one_liner: + yield from _emit_one_line_value( + values_and_comments, sep_token, trailing_separator + ) + return + # If it does not fit in one line, we fall through + # Chain into the python-debian provided formatter, which will handle this + # formatting for us. + yield from _chain_formatter(name, sep_token, iter(all_tokens)) + + return _formatter diff --git a/src/debputy/util.py b/src/debputy/util.py index d8cfd67..11f6ccd 100644 --- a/src/debputy/util.py +++ b/src/debputy/util.py @@ -763,8 +763,8 @@ def setup_logging( colorlog.ColoredFormatter(color_format, style="{", force_color=True) ) logger = logging.getLogger() - if existing_stdout_handler is not None: - logger.removeHandler(existing_stdout_handler) + if existing_stderr_handler is not None: + logger.removeHandler(existing_stderr_handler) _STDERR_HANDLER = stderr_handler logger.addHandler(stderr_handler) else: diff --git a/tests/lint_tests/conftest.py b/tests/lint_tests/conftest.py index 2c54eb7..bf4b3b0 100644 --- a/tests/lint_tests/conftest.py +++ b/tests/lint_tests/conftest.py @@ -3,6 +3,7 @@ from debian.debian_support import DpkgArchTable from debputy._deb_options_profiles import DebBuildOptionsAndProfiles from debputy.architecture_support import DpkgArchitectureBuildProcessValuesTable +from debputy.lsp.style_prefs import StylePreferenceTable from debputy.packages import DctrlParser from debputy.util import setup_logging diff --git a/tests/lint_tests/lint_tutil.py b/tests/lint_tests/lint_tutil.py index 83b69fd..8290712 100644 --- a/tests/lint_tests/lint_tutil.py +++ b/tests/lint_tests/lint_tutil.py @@ -9,6 +9,7 @@ from debputy.linting.lint_util import ( LintStateImpl, LintState, ) +from debputy.lsp.style_prefs import StylePreferenceTable, EffectivePreference from debputy.packages import DctrlParser from debputy.plugin.api.feature_set import PluginProvidedFeatureSet @@ -43,6 +44,8 @@ class LintWrapper: self.dctrl_lines: Optional[List[str]] = None self.path = path self._dctrl_parser = dctrl_parser + self.lint_style_preference_table = StylePreferenceTable({}, {}) + self.effective_preference: Optional[EffectivePreference] = None def __call__(self, lines: List[str]) -> Optional[List["Diagnostic"]]: source_package = None @@ -56,10 +59,13 @@ class LintWrapper: ) state = LintStateImpl( self._debputy_plugin_feature_set, + self.lint_style_preference_table, self.path, + "".join(dctrl_lines) if dctrl_lines is not None else "", lines, source_package, binary_packages, + self.effective_preference, ) return check_diagnostics(self._handler(state)) diff --git a/tests/lint_tests/test_lint_dctrl.py b/tests/lint_tests/test_lint_dctrl.py index ce34d7c..bcb1613 100644 --- a/tests/lint_tests/test_lint_dctrl.py +++ b/tests/lint_tests/test_lint_dctrl.py @@ -18,6 +18,9 @@ except ImportError: pass +STANDARDS_VERSION = "4.7.0" + + class DctrlLintWrapper(LintWrapper): def __call__(self, lines: List[str]) -> Optional[List["Diagnostic"]]: @@ -93,15 +96,15 @@ def test_dctrl_lint(line_linter: LintWrapper) -> None: msg = 'The value "base" is not supported in Section.' assert second_warn.message == msg - assert f"{second_warn.range}" == "8:9-8:13" + assert f"{second_warn.range}" == "7:9-7:13" @requires_levenshtein def test_dctrl_lint_typos(line_linter: LintWrapper) -> None: lines = textwrap.dedent( - """\ + f"""\ Source: foo - Standards-Version: 4.5.2 + Standards-Version: {STANDARDS_VERSION} Priority: optional Section: devel Maintainer: Jane Developer @@ -132,9 +135,9 @@ def test_dctrl_lint_typos(line_linter: LintWrapper) -> None: @requires_levenshtein def test_dctrl_lint_mx_value_with_typo(line_linter: LintWrapper) -> None: lines = textwrap.dedent( - """\ + f"""\ Source: foo - Standards-Version: 4.5.2 + Standards-Version: {STANDARDS_VERSION} Priority: optional Section: devel Maintainer: Jane Developer @@ -165,15 +168,15 @@ def test_dctrl_lint_mx_value_with_typo(line_linter: LintWrapper) -> None: typo_msg = 'It is possible that the value is a typo of "all".' assert mx_diag.message == mx_msg assert typo_diag.message == typo_msg - assert f"{mx_diag.range}" == "10:24-10:28" - assert f"{typo_diag.range}" == "10:24-10:28" + assert f"{mx_diag.range}" == "9:24-9:28" + assert f"{typo_diag.range}" == "9:24-9:28" def test_dctrl_lint_mx_value(line_linter: LintWrapper) -> None: lines = textwrap.dedent( - """\ + f"""\ Source: foo - Standards-Version: 4.5.2 + Standards-Version: {STANDARDS_VERSION} Priority: optional Section: devel Maintainer: Jane Developer @@ -200,9 +203,9 @@ def test_dctrl_lint_mx_value(line_linter: LintWrapper) -> None: assert f"{diag.range}" == "8:14-8:17" lines = textwrap.dedent( - """\ + f"""\ Source: foo - Standards-Version: 4.5.2 + Standards-Version: {STANDARDS_VERSION} Priority: optional Section: devel Maintainer: Jane Developer @@ -227,3 +230,36 @@ def test_dctrl_lint_mx_value(line_linter: LintWrapper) -> None: assert diag.message == msg assert diag.severity == DiagnosticSeverity.Error assert f"{diag.range}" == "8:24-8:27" + + +def test_dctrl_lint_dup_sep(line_linter: LintWrapper) -> None: + lines = textwrap.dedent( + f"""\ + Source: foo + Section: devel + Priority: optional + Standards-Version: {STANDARDS_VERSION} + Maintainer: Jane Developer + Build-Depends: debhelper-compat (= 13) + + Package: foo + Architecture: all + Depends: foo, + , bar + Description: Some very interesting synopsis + A very interesting description + that spans multiple lines + . + Just so be clear, this is for a test. + """ + ).splitlines(keepends=True) + + diagnostics = line_linter(lines) + print(diagnostics) + assert diagnostics and len(diagnostics) == 1 + error = diagnostics[0] + + msg = "Duplicate separator" + assert error.message == msg + assert f"{error.range}" == "10:1-10:2" + assert error.severity == DiagnosticSeverity.Error diff --git a/tests/lsp_tests/test_debpkg_metadata.py b/tests/lsp_tests/test_debpkg_metadata.py index f784b0a..fdb0df8 100644 --- a/tests/lsp_tests/test_debpkg_metadata.py +++ b/tests/lsp_tests/test_debpkg_metadata.py @@ -19,7 +19,7 @@ from debputy.lsp.lsp_debian_control_reference_data import package_name_to_sectio ("xxx-l10n-bar", "localization"), ("libfoo4", "libs"), ("unknown", None), - ] + ], ) def test_package_name_to_section(name: str, guessed_section: Optional[str]) -> None: assert package_name_to_section(name) == guessed_section diff --git a/tests/test_style.py b/tests/test_style.py new file mode 100644 index 0000000..9ce83bd --- /dev/null +++ b/tests/test_style.py @@ -0,0 +1,121 @@ +import pytest +from debian.deb822 import Deb822 + +from debputy.lsp.style_prefs import StylePreferenceTable, determine_effective_style +from debputy.packages import SourcePackage + + +def test_load_styles() -> None: + styles = StylePreferenceTable.load_styles() + assert "niels@thykier.net" in styles.maintainer_preferences + nt_style = styles.maintainer_preferences["niels@thykier.net"] + # Note this is data dependent; if it fails because the style changes, update the test + assert nt_style.canonical_name == "Niels Thykier" + assert not nt_style.is_packaging_team + assert nt_style.formatting_deb822_normalize_field_content + assert nt_style.formatting_deb822_short_indent + assert nt_style.formatting_deb822_always_wrap + assert nt_style.formatting_deb822_trailing_separator + assert nt_style.formatting_deb822_max_line_length == 79 + assert nt_style.formatting_deb822_normalize_stanza_order + + # TODO: Not implemented yet + assert not nt_style.formatting_deb822_normalize_field_order + + +def test_load_named_styles() -> None: + styles = StylePreferenceTable.load_styles() + assert "black" in styles.named_styles + black_style = styles.named_styles["black"] + # Note this is data dependent; if it fails because the style changes, update the test + assert black_style.formatting_deb822_normalize_field_content + assert black_style.formatting_deb822_short_indent + assert black_style.formatting_deb822_always_wrap + assert black_style.formatting_deb822_trailing_separator + assert black_style.formatting_deb822_max_line_length == 79 + assert black_style.formatting_deb822_normalize_stanza_order + + # TODO: Not implemented yet + assert not black_style.formatting_deb822_normalize_field_order + + +def test_compat_styles() -> None: + styles = StylePreferenceTable.load_styles() + + # Data dependent; if it breaks, provide a stubbed style preference table + assert "niels@thykier.net" in styles.maintainer_preferences + assert "zeha@debian.org" in styles.maintainer_preferences + assert "random-package@packages.debian.org" not in styles.maintainer_preferences + assert "random@example.org" not in styles.maintainer_preferences + + nt_pref = styles.maintainer_preferences["niels@thykier.net"].as_effective_pref() + zeha_pref = styles.maintainer_preferences["zeha@debian.org"].as_effective_pref() + + # Data dependency + assert nt_pref == zeha_pref + + fields = Deb822( + { + "Package": "foo", + "Maintainer": "Foo ", + "Uploaders": "Niels Thykier ", + }, + ) + src = SourcePackage(fields) + + effective_style = determine_effective_style(styles, src) + assert effective_style == nt_pref + + fields["Uploaders"] = ( + "Niels Thykier , Chris Hofstaedtler " + ) + src = SourcePackage(fields) + + effective_style = determine_effective_style(styles, src) + assert effective_style == nt_pref + assert effective_style == zeha_pref + + fields["Uploaders"] = ( + "Niels Thykier , Chris Hofstaedtler , Random Developer " + ) + src = SourcePackage(fields) + + effective_style = determine_effective_style(styles, src) + assert effective_style is None + + +@pytest.mark.xfail +def test_compat_styles_team_maint() -> None: + styles = StylePreferenceTable.load_styles() + fields = Deb822( + { + "Package": "foo", + # Missing a stubbed definition for `team@lists.debian.org` + "Maintainer": "Packaging Team ", + "Uploaders": "Random Developer ", + }, + ) + src = SourcePackage(fields) + assert "team@lists.debian.org" in styles.maintainer_preferences + assert "random@example.org" not in styles.maintainer_preferences + team_style = styles.maintainer_preferences["team@lists.debian.org"] + assert team_style.is_packaging_team + effective_style = determine_effective_style(styles, src) + assert effective_style == team_style.as_effective_pref() + + +def test_x_style() -> None: + styles = StylePreferenceTable.load_styles() + fields = Deb822( + { + "Package": "foo", + "X-Style": "black", + "Maintainer": "Random Developer ", + }, + ) + src = SourcePackage(fields) + assert "random@example.org" not in styles.maintainer_preferences + assert "black" in styles.named_styles + black_style = styles.named_styles["black"] + effective_style = determine_effective_style(styles, src) + assert effective_style == black_style -- cgit v1.2.3