diff options
Diffstat (limited to '')
22 files changed, 1095 insertions, 306 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3d32b2..c0c5ccf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,32 +7,43 @@ include: stages: - ci-test + - ci-os-support - aggregate-coverage - pages + - os-build-tests - provisioning - build - publish - test tests-testing: - stage: ci-test - image: debian:testing + stage: os-build-tests + image: debian:testing-slim script: - apt-get update - apt-get build-dep -y . - dpkg-buildpackage -us -uc -tc tests-unstable: - stage: ci-test - image: debian:unstable + stage: os-build-tests + image: debian:unstable-slim script: - apt-get update - apt-get build-dep -Ppkg.debputy.ci -y . - dpkg-buildpackage -Ppkg.debputy.ci -us -uc -tc +tests-ubuntu-noble: + stage: os-build-tests + image: ubuntu:noble + script: + - apt-get update + - apt-get build-dep -Ppkg.debputy.ci -y . + - dpkg-buildpackage -Ppkg.debputy.ci -us -uc -tc + + code-lint-mypy: stage: ci-test - image: debian:unstable + image: debian:unstable-slim script: - apt-get update - apt-get build-dep -Ppkg.debputy.ci -y . @@ -53,12 +64,11 @@ code-lint-mypy: tests-unstable-coverage-without-optional-bd: stage: ci-test - image: debian:unstable + image: debian:unstable-slim script: - apt-get update - apt-get build-dep -Ppkg.debputy.minimal-tests,pkg.debputy.test-coverage -y . - py.test-3 -v --cov --cov-branch --doctest-modules --junit-xml=xunit-report.xml --cov-report xml:coverage.xml - - dpkg-buildpackage -Ppkg.debputy.minimal-tests -us -uc -tc after_script: - mkdir -p coverage-results/tests-unstable-coverage-without-optional-bd - cp .coverage coverage-results/tests-unstable-coverage-without-optional-bd/coverage @@ -73,7 +83,7 @@ tests-unstable-coverage-without-optional-bd: tests-unstable-coverage: stage: ci-test - image: debian:unstable + image: debian:unstable-slim script: - apt-get update - apt-get build-dep -Ppkg.debputy.test-coverage -y . @@ -109,9 +119,50 @@ tests-unstable-coverage-with-extra-bd: coverage_format: cobertura path: coverage.xml + +tests-ubuntu-noble-coverage-with-extra-bd: + stage: ci-os-support + image: ubuntu:noble + script: + - apt-get update + - apt-get build-dep -Ppkg.debputy.ci,pkg.debputy.test-coverage -y . + - py.test-3 -v --cov --cov-branch --doctest-modules --junit-xml=xunit-report.xml --cov-report xml:coverage.xml + after_script: + - mkdir -p coverage-results/tests-ubuntu-noble-coverage-with-extra-bd + - cp .coverage coverage-results/tests-ubuntu-noble-coverage-with-extra-bd/coverage + artifacts: + paths: + - coverage-results/tests-ubuntu-noble-coverage-with-extra-bd + reports: + junit: xunit-report.xml + coverage_report: + coverage_format: cobertura + path: coverage.xml + + +tests-ubuntu-noble-coverage-without-optional-bd: + stage: ci-os-support + image: ubuntu:noble + script: + - apt-get update + - apt-get build-dep -Ppkg.debputy.minimal-tests,pkg.debputy.test-coverage -y . + - py.test-3 -v --cov --cov-branch --doctest-modules --junit-xml=xunit-report.xml --cov-report xml:coverage.xml + after_script: + - mkdir -p coverage-results/tests-ubuntu-noble-coverage-without-optional-bd + - cp .coverage coverage-results/tests-ubuntu-noble-coverage-without-optional-bd/coverage + artifacts: + paths: + - coverage-results/tests-ubuntu-noble-coverage-without-optional-bd + reports: + junit: xunit-report.xml + coverage_report: + coverage_format: cobertura + path: coverage.xml + + aggregate-coverage: stage: aggregate-coverage - image: debian:unstable + image: debian:unstable-slim coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/' script: - apt-get update -y @@ -126,6 +177,8 @@ aggregate-coverage: - tests-unstable-coverage - tests-unstable-coverage-without-optional-bd - tests-unstable-coverage-with-extra-bd + - tests-ubuntu-noble-coverage-with-extra-bd + - tests-ubuntu-noble-coverage-without-optional-bd pages: stage: pages @@ -149,7 +202,7 @@ variables: debputy-reformat: stage: ci-test - image: debian:sid-slim + image: debian:unstable-slim script: - apt-get update -qq && apt-get -qq build-dep --no-install-recommends --yes . - ./debputy.sh reformat --linter-exit-code --no-auto-fix diff --git a/debputy.pod b/debputy.pod index 0cee740..c53fa23 100644 --- a/debputy.pod +++ b/debputy.pod @@ -171,6 +171,21 @@ providing the results. If you rely on the exit code, you are recommended to explicitly pass the relevant variant of the flag even if the current default matches your wishes. +=item B<--missing-style-is-ok> + +By default, B<debputy reformat> will exit with an error when it cannot determine which style to +use. This is generally what you want for "per package" CI or other lint checks to inform you that +the reformatting will not work. + +However, for "cross package" pipelines, like the default Debian Salsa CI pipeline, having +B<debputy reformat> automatically work when a style is set and doing nothing otherwise is +preferable, since the pipeline can then provide a B<debputy reformat> job for all consumers +without worrying about breaking their pipelines. + +It can also be useful for scripts or automated mass-edits where you want B<debputy> to fixup +the changes you did if there is a known style without being hampered by the packages that +have no known style. + =back =item lint diff --git a/src/debputy/commands/debputy_cmd/context.py b/src/debputy/commands/debputy_cmd/context.py index 4d28408..29bc573 100644 --- a/src/debputy/commands/debputy_cmd/context.py +++ b/src/debputy/commands/debputy_cmd/context.py @@ -262,7 +262,7 @@ class CommandContext: os.path.join(self.debian_dir.fs_path, "control"), ) with debian_control.open() as fd: - source_package, binary_packages = ( + _, source_package, binary_packages = ( self.dctrl_parser.parse_source_debian_control( fd, ) diff --git a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py index ea97665..a5e1142 100644 --- a/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py +++ b/src/debputy/commands/debputy_cmd/lint_and_lsp_cmds.py @@ -313,6 +313,14 @@ def lint_cmd(context: CommandContext) -> None: action=BooleanOptionalAction, help='Enable or disable the "linter" convention of exiting with an error if issues were found', ), + add_arg( + "--missing-style-is-ok", + dest="declared_style_required", + default=True, + action="store_false", + help="Do not exit with an error if no style can be identified. Useful for general pipelines to implement" + ' "reformat if possible"', + ), ], ) def reformat_cmd(context: CommandContext) -> None: diff --git a/src/debputy/linting/lint_impl.py b/src/debputy/linting/lint_impl.py index c4e9310..8ce78c1 100644 --- a/src/debputy/linting/lint_impl.py +++ b/src/debputy/linting/lint_impl.py @@ -3,8 +3,11 @@ import os import stat import subprocess import sys +import textwrap from typing import Optional, List, Union, NoReturn, Mapping +from debputy.lsp.vendoring._deb822_repro import Deb822FileElement +from debputy.yaml import MANIFEST_YAML, YAMLError from lsprotocol.types import ( CodeAction, Command, @@ -57,6 +60,7 @@ from debputy.lsp.text_edit import ( from debputy.packages import SourcePackage, BinaryPackage from debputy.plugin.api.feature_set import PluginProvidedFeatureSet from debputy.util import _warn, _error, _info +from debputy.yaml.compat import CommentedMap, __all__ LINTER_FORMATS = { "debian/changelog": _lint_debian_changelog, @@ -79,9 +83,11 @@ REFORMAT_FORMATS = { class LintContext: plugin_feature_set: PluginProvidedFeatureSet style_preference_table: StylePreferenceTable + parsed_deb822_file_content: Optional[Deb822FileElement] = None source_package: Optional[SourcePackage] = None binary_packages: Optional[Mapping[str, BinaryPackage]] = None effective_preference: Optional[EffectivePreference] = None + salsa_ci: Optional[CommentedMap] = None def state_for(self, path: str, content: str, lines: List[str]) -> LintStateImpl: return LintStateImpl( @@ -103,18 +109,33 @@ def gather_lint_info(context: CommandContext) -> LintContext: ) try: with open("debian/control") as fd: - source_package, binary_packages = ( + deb822_file, source_package, binary_packages = ( context.dctrl_parser.parse_source_debian_control(fd, ignore_errors=True) ) + except FileNotFoundError: + source_package = None + else: + lint_context.parsed_deb822_file_content = deb822_file lint_context.source_package = source_package lint_context.binary_packages = binary_packages - if source_package is not None: - lint_context.effective_preference = determine_effective_style( - lint_context.style_preference_table, - source_package, - ) - except FileNotFoundError: - pass + salsa_ci_map: Optional[CommentedMap] = None + for ci_file in ("debian/salsa-ci.yml", ".gitlab-ci.yml"): + try: + with open(ci_file) as fd: + salsa_ci_map = MANIFEST_YAML.load(fd) + if not isinstance(salsa_ci_map, CommentedMap): + salsa_ci_map = None + break + except FileNotFoundError: + pass + except YAMLError: + break + if source_package is not None or salsa_ci_map is not None: + lint_context.effective_preference = determine_effective_style( + lint_context.style_preference_table, + source_package, + salsa_ci_map, + ) return lint_context @@ -186,29 +207,45 @@ def perform_reformat( lint_context.effective_preference = style if lint_context.effective_preference is None: - _info( - "You can set `X-Style: black` in the source stanza of `debian/control` to pick" - ) - _info("`black` as the preferred style for this package.") - _info("") - _info( - "You can also include your style in `debputy`. This has the advantage that `debputy`" - ) - _info( - "can sometimes automatically derive an agreed style between all maintainers and" - ) - _info( - "co-maintainers for informal teams (provided all agree to the same style), where" - ) - _info( - "no explicit synchronization has been done (that is, there is no `X-Style` field)" + print( + textwrap.dedent( + """\ + You can enable set a style by doing either of: + + * You can set `X-Style: black` in the source stanza of `debian/control` to pick + `black` as the preferred style for this package. + - Note: `black` is an opinionated style that follows the spirit of the `black` code formatter + for Python. + - If you use `pre-commit`, then there is a formatting hook at + https://salsa.debian.org/debian/debputy-pre-commit-hooks + + * If you use the Debian Salsa CI pipeline, then you can set SALSA_CI_DISABLE_WRAP_AND_SORT + to a truth value and `debputy` will pick up the configuration from there. + - Note: The option must be in `.gitlab-ci.yml` or `debian/salsa-ci.yml` to work. The Salsa CI + pipeline will use `wrap-and-sort` while `debputy` uses its own emulation of `wrap-and-sort` + (`debputy` also needs to apply the style via `debputy lsp server`). + + * The `debputy` code also comes with a built-in style database. This may be interesting for + packaging teams, so set a default team style that applies to all packages maintained by + that packaging team. + - Individuals can also add their style, which can useful for ad-hoc packaging teams, where + `debputy` will automatically apply a style if *all* co-maintainers agree to it. + + Note the above list is an ordered list of how `debputy` determines which style to use in case + multiple options are available. + """ + ) ) + if parsed_args.declared_style_required: + _error( + "Sorry; `debputy` does not know which style to use for this package." + ) _info("") _info( - "For packaging teams, you can also have a team style that is applied to all team" + "Doing nothing since no style could be identified as requested." + " See above how to set a style." ) - _info("maintained packages by having it recorded in `debputy`.") - _error("Sorry; `debputy` does not know which style to use for this package.") + sys.exit(0) changes = False auto_fix = context.parsed_args.auto_fix diff --git a/src/debputy/linting/lint_util.py b/src/debputy/linting/lint_util.py index e997097..02b8804 100644 --- a/src/debputy/linting/lint_util.py +++ b/src/debputy/linting/lint_util.py @@ -5,6 +5,7 @@ from typing import List, Optional, Callable, Counter, TYPE_CHECKING, Mapping, Se from lsprotocol.types import Position, Range, Diagnostic, DiagnosticSeverity, TextEdit from debputy.commands.debputy_cmd.output import OutputStylingBase +from debputy.lsp.vendoring._deb822_repro import Deb822FileElement, parse_deb822_file from debputy.packages import SourcePackage, BinaryPackage from debputy.plugin.api.feature_set import PluginProvidedFeatureSet from debputy.util import _DEFAULT_LOGGER, _warn @@ -48,6 +49,10 @@ class LintState: raise NotImplementedError @property + def parsed_deb822_file_content(self) -> Optional[Deb822FileElement]: + raise NotImplementedError + + @property def source_package(self) -> Optional[SourcePackage]: raise NotImplementedError @@ -74,6 +79,7 @@ class LintStateImpl(LintState): source_package: Optional[SourcePackage] = None binary_packages: Optional[Mapping[str, BinaryPackage]] = None effective_preference: Optional["EffectivePreference"] = None + _parsed_cache: Optional[Deb822FileElement] = None @property def doc_uri(self) -> str: @@ -85,6 +91,18 @@ class LintStateImpl(LintState): def position_codec(self) -> "LintCapablePositionCodec": return LINTER_POSITION_CODEC + @property + def parsed_deb822_file_content(self) -> Optional[Deb822FileElement]: + cache = self._parsed_cache + if cache is None: + cache = parse_deb822_file( + self.lines, + accept_files_with_error_tokens=True, + accept_files_with_duplicated_fields=True, + ) + self._parsed_cache = cache + return cache + @dataclasses.dataclass(slots=True) class LintReport: diff --git a/src/debputy/lsp/debputy_ls.py b/src/debputy/lsp/debputy_ls.py index e475ab4..c08fd18 100644 --- a/src/debputy/lsp/debputy_ls.py +++ b/src/debputy/lsp/debputy_ls.py @@ -22,6 +22,7 @@ from debputy.lsp.style_prefs import ( determine_effective_style, ) from debputy.lsp.text_util import LintCapablePositionCodec +from debputy.lsp.vendoring._deb822_repro import Deb822FileElement, parse_deb822_file from debputy.packages import ( SourcePackage, BinaryPackage, @@ -29,6 +30,8 @@ from debputy.packages import ( ) from debputy.plugin.api.feature_set import PluginProvidedFeatureSet from debputy.util import _info +from debputy.yaml import MANIFEST_YAML, YAMLError +from debputy.yaml.compat import CommentedMap if TYPE_CHECKING: from pygls.server import LanguageServer @@ -46,18 +49,117 @@ else: def __init__(self, *args, **kwargs) -> None: """Placeholder to work if pygls is not installed""" # Should not be called + global e raise e # pragma: no cover @dataclasses.dataclass(slots=True) -class DctrlCache: +class FileCache: doc_uri: str path: str - is_open_in_editor: Optional[bool] - last_doc_version: Optional[int] - last_mtime: Optional[float] - source_package: Optional[SourcePackage] - binary_packages: Optional[Mapping[str, BinaryPackage]] + is_open_in_editor: Optional[bool] = None + last_doc_version: Optional[int] = None + last_mtime: Optional[float] = None + is_valid: bool = False + + def _update_cache(self, doc: "TextDocument", source: str) -> None: + raise NotImplementedError + + def _clear_cache(self) -> None: + raise NotImplementedError + + def resolve_cache(self, ls: "DebputyLanguageServer") -> bool: + doc = ls.workspace.text_documents.get(self.doc_uri) + if doc is None: + doc = ls.workspace.get_text_document(self.doc_uri) + is_open = False + else: + is_open = True + new_content: Optional[str] = None + if is_open: + last_doc_version = self.last_doc_version + dctrl_doc_version = doc.version + if ( + not self.is_open_in_editor + or last_doc_version is None + or dctrl_doc_version is None + or last_doc_version < dctrl_doc_version + ): + new_content = doc.source + + self.last_doc_version = doc.version + elif doc.uri.startswith("file://"): + try: + with open(self.path) as fd: + st = os.fstat(fd.fileno()) + current_mtime = st.st_mtime + last_mtime = self.last_mtime or current_mtime - 1 + if self.is_open_in_editor or current_mtime > last_mtime: + new_content = fd.read() + self.last_mtime = current_mtime + except FileNotFoundError: + self._clear_cache() + self.is_valid = False + return False + if new_content is not None: + self._update_cache(doc, new_content) + self.is_valid = True + return True + + +@dataclasses.dataclass(slots=True) +class Deb822FileCache(FileCache): + deb822_file: Optional[Deb822FileElement] = None + + def _update_cache(self, doc: "TextDocument", source: str) -> None: + deb822_file = parse_deb822_file( + source.splitlines(keepends=True), + accept_files_with_error_tokens=True, + accept_files_with_duplicated_fields=True, + ) + self.deb822_file = deb822_file + + def _clear_cache(self) -> None: + self.deb822_file = None + + +@dataclasses.dataclass(slots=True) +class DctrlFileCache(Deb822FileCache): + dctrl_parser: Optional[DctrlParser] = None + source_package: Optional[SourcePackage] = None + binary_packages: Optional[Mapping[str, BinaryPackage]] = None + + def _update_cache(self, doc: "TextDocument", source: str) -> None: + deb822_file, source_package, binary_packages = ( + self.dctrl_parser.parse_source_debian_control( + source.splitlines(keepends=True), + ignore_errors=True, + ) + ) + self.deb822_file = deb822_file + self.source_package = source_package + self.binary_packages = binary_packages + + def _clear_cache(self) -> None: + super()._clear_cache() + self.source_package = None + self.binary_packages = None + + +@dataclasses.dataclass(slots=True) +class SalsaCICache(FileCache): + parsed_content: Optional[CommentedMap] = None + + def _update_cache(self, doc: "TextDocument", source: str) -> None: + try: + value = MANIFEST_YAML.load(source) + if isinstance(value, CommentedMap): + self.parsed_content = value + except YAMLError: + pass + + def _clear_cache(self) -> None: + self.parsed_content = None class LSProvidedLintState(LintState): @@ -72,17 +174,27 @@ class LSProvidedLintState(LintState): self._doc = doc # Cache lines (doc.lines re-splits everytime) self._lines = doc.lines - self._dctrl_parser = dctrl_parser dctrl_file = os.path.join(debian_dir_path, "control") - self._dctrl_cache: DctrlCache = DctrlCache( + self._dctrl_cache: DctrlFileCache = DctrlFileCache( from_fs_path(dctrl_file), dctrl_file, - is_open_in_editor=None, # Unresolved - last_doc_version=None, - last_mtime=None, - source_package=None, - binary_packages=None, + dctrl_parser=dctrl_parser, ) + if dctrl_file != doc.path: + self._deb822_file: Deb822FileCache = Deb822FileCache( + from_fs_path(dctrl_file), + dctrl_file, + ) + else: + self._deb822_file = self._dctrl_cache + + self._salsa_ci_caches = [ + SalsaCICache( + from_fs_path(os.path.join(debian_dir_path, p)), + os.path.join(debian_dir_path, p), + ) + for p in ("salsa-ci.yml", os.path.join("..", ".gitlab-ci.yml")) + ] @property def plugin_feature_set(self) -> PluginProvidedFeatureSet: @@ -108,67 +220,51 @@ class LSProvidedLintState(LintState): def position_codec(self) -> LintCapablePositionCodec: return self._doc.position_codec - def _resolve_dctrl(self) -> Optional[DctrlCache]: + def _resolve_dctrl(self) -> Optional[DctrlFileCache]: dctrl_cache = self._dctrl_cache - doc = self._ls.workspace.text_documents.get(dctrl_cache.doc_uri) - is_open = doc is not None - dctrl_doc = self._ls.workspace.get_text_document(dctrl_cache.doc_uri) - re_parse_lines: Optional[List[str]] = None - if is_open: - last_doc_version = dctrl_cache.last_doc_version - dctrl_doc_version = dctrl_doc.version - if ( - not dctrl_cache.is_open_in_editor - or last_doc_version is None - or dctrl_doc_version is None - or last_doc_version < dctrl_doc_version - ): - re_parse_lines = doc.lines - - dctrl_cache.last_doc_version = dctrl_doc.version - elif self._doc.uri.startswith("file://"): - try: - with open(dctrl_cache.path) as fd: - st = os.fstat(fd.fileno()) - current_mtime = st.st_mtime - last_mtime = dctrl_cache.last_mtime or current_mtime - 1 - if dctrl_cache.is_open_in_editor or current_mtime > last_mtime: - re_parse_lines = list(fd) - dctrl_cache.last_mtime = current_mtime - except FileNotFoundError: - return None - if re_parse_lines is not None: - source_package, binary_packages = ( - self._dctrl_parser.parse_source_debian_control( - re_parse_lines, - ignore_errors=True, - ) - ) - dctrl_cache.source_package = source_package - dctrl_cache.binary_packages = binary_packages + dctrl_cache.resolve_cache(self._ls) return dctrl_cache @property + def parsed_deb822_file_content(self) -> Optional[Deb822FileElement]: + cache = self._deb822_file + cache.resolve_cache(self._ls) + return cache.deb822_file + + @property def source_package(self) -> Optional[SourcePackage]: - dctrl = self._resolve_dctrl() - return dctrl.source_package if dctrl is not None else None + return self._resolve_dctrl().source_package @property def binary_packages(self) -> Optional[Mapping[str, BinaryPackage]]: - dctrl = self._resolve_dctrl() - return dctrl.binary_packages if dctrl is not None else None + return self._resolve_dctrl().binary_packages + + def _resolve_salsa_ci(self) -> Optional[CommentedMap]: + for salsa_ci_cache in self._salsa_ci_caches: + if salsa_ci_cache.resolve_cache(self._ls): + return salsa_ci_cache.parsed_content + return None @property def effective_preference(self) -> Optional[MaintainerPreference]: source_package = self.source_package - if source_package is None: + salsa_ci = self._resolve_salsa_ci() + if source_package is None and salsa_ci is None: return None - return determine_effective_style(self.style_preference_table, source_package) + return determine_effective_style( + self.style_preference_table, + source_package, + salsa_ci, + ) @property def style_preference_table(self) -> StylePreferenceTable: return self._ls.style_preferences + @property + def salsa_ci(self) -> Optional[CommentedMap]: + return None + def _preference( client_preference: Optional[List[MarkupKind]], diff --git a/src/debputy/lsp/lsp_debian_changelog.py b/src/debputy/lsp/lsp_debian_changelog.py index c6166c5..2deff97 100644 --- a/src/debputy/lsp/lsp_debian_changelog.py +++ b/src/debputy/lsp/lsp_debian_changelog.py @@ -4,7 +4,6 @@ from email.utils import parsedate_to_datetime from typing import ( Union, List, - Dict, Iterator, Optional, Iterable, @@ -16,7 +15,6 @@ from lsprotocol.types import ( DidChangeTextDocumentParams, TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, TEXT_DOCUMENT_CODE_ACTION, - DidCloseTextDocumentParams, Range, Position, DiagnosticSeverity, @@ -29,12 +27,12 @@ from debputy.lsp.quickfixes import ( propose_correct_text_quick_fix, ) from debputy.lsp.spellchecking import spellcheck_line -from debputy.lsp.text_util import ( - LintCapablePositionCodec, -) try: - from debian._deb822_repro.locatable import Position as TEPosition, Ranage as TERange + from debputy.lsp.vendoring._deb822_repro.locatable import ( + Position as TEPosition, + Range as TERange, + ) from pygls.server import LanguageServer from pygls.workspace import TextDocument diff --git a/src/debputy/lsp/lsp_debian_control.py b/src/debputy/lsp/lsp_debian_control.py index 3a4107b..699193c 100644 --- a/src/debputy/lsp/lsp_debian_control.py +++ b/src/debputy/lsp/lsp_debian_control.py @@ -1,7 +1,6 @@ import dataclasses import re import textwrap -from functools import lru_cache from typing import ( Union, Sequence, @@ -12,6 +11,28 @@ from typing import ( Dict, ) +from lsprotocol.types import ( + DiagnosticSeverity, + Range, + Diagnostic, + Position, + FoldingRange, + FoldingRangeParams, + CompletionItem, + CompletionList, + CompletionParams, + DiagnosticRelatedInformation, + Location, + HoverParams, + Hover, + TEXT_DOCUMENT_CODE_ACTION, + SemanticTokens, + SemanticTokensParams, + WillSaveTextDocumentParams, + TextEdit, + DocumentFormattingParams, +) + from debputy.linting.lint_util import LintState from debputy.lsp.debputy_ls import DebputyLanguageServer from debputy.lsp.diagnostics import DiagnosticData @@ -55,7 +76,6 @@ from debputy.lsp.text_util import ( te_range_to_lsp, ) from debputy.lsp.vendoring._deb822_repro import ( - parse_deb822_file, Deb822FileElement, Deb822ParagraphElement, ) @@ -63,28 +83,6 @@ from debputy.lsp.vendoring._deb822_repro.parsing import ( Deb822KeyValuePairElement, LIST_SPACE_SEPARATED_INTERPRETATION, ) -from lsprotocol.types import ( - DiagnosticSeverity, - Range, - Diagnostic, - Position, - FoldingRange, - FoldingRangeParams, - CompletionItem, - CompletionList, - CompletionParams, - TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, - DiagnosticRelatedInformation, - Location, - HoverParams, - Hover, - TEXT_DOCUMENT_CODE_ACTION, - SemanticTokens, - SemanticTokensParams, - WillSaveTextDocumentParams, - TextEdit, - DocumentFormattingParams, -) try: from debputy.lsp.vendoring._deb822_repro.locatable import ( @@ -846,11 +844,7 @@ def _lint_debian_control( position_codec = lint_state.position_codec doc_reference = lint_state.doc_uri diagnostics = [] - deb822_file = parse_deb822_file( - lines, - accept_files_with_duplicated_fields=True, - accept_files_with_error_tokens=True, - ) + deb822_file = lint_state.parsed_deb822_file_content first_error = _scan_for_syntax_errors_and_token_level_diagnostics( deb822_file, diff --git a/src/debputy/lsp/lsp_debian_control_reference_data.py b/src/debputy/lsp/lsp_debian_control_reference_data.py index 60e47d7..1d4628c 100644 --- a/src/debputy/lsp/lsp_debian_control_reference_data.py +++ b/src/debputy/lsp/lsp_debian_control_reference_data.py @@ -22,9 +22,10 @@ from typing import ( Any, Set, TYPE_CHECKING, + Sequence, ) -from debian._deb822_repro.types import TE +from debputy.lsp.vendoring._deb822_repro.types import TE from debian.debian_support import DpkgArchTable from lsprotocol.types import ( DiagnosticSeverity, @@ -33,10 +34,13 @@ from lsprotocol.types import ( Range, TextEdit, Position, + CompletionItem, + MarkupContent, + CompletionItemTag, + MarkupKind, ) from debputy.lsp.diagnostics import DiagnosticData - from debputy.lsp.lsp_reference_keyword import ( ALL_PUBLIC_NAMED_STYLES, Keyword, @@ -205,9 +209,11 @@ ALL_SECTIONS_WITHOUT_COMPONENT = frozenset( "gnome", "gnu-r", "gnustep", + "golang", "graphics", "hamradio", "haskell", + "httpd", "interpreters", "introspection", "java", @@ -455,6 +461,29 @@ def _udeb_only_field_validation( ) +def _complete_only_in_arch_dep_pkgs( + stanza_parts: Iterable[Deb822ParagraphElement], +) -> bool: + for stanza in stanza_parts: + arch = stanza.get("Architecture") + if arch is None: + continue + archs = arch.split() + return "all" not in archs + return False + + +def _complete_only_for_udeb_pkgs( + stanza_parts: Iterable[Deb822ParagraphElement], +) -> bool: + for stanza in stanza_parts: + for option in ("Package-Type", "XC-Package-Type"): + pkg_type = stanza.get(option) + if pkg_type is not None: + return pkg_type == "udeb" + return False + + def _arch_not_all_only_field_validation( known_field: "F", _kvpair: Deb822KeyValuePairElement, @@ -800,6 +829,92 @@ class Deb822KnownField: is_stanza_name: bool = False is_single_value_field: bool = True custom_field_check: Optional[CustomFieldCheck] = None + can_complete_field_in_stanza: Optional[ + Callable[[Iterable[Deb822ParagraphElement]], bool] + ] = None + + def _can_complete_field_in_stanza( + self, + stanza_parts: Sequence[Deb822ParagraphElement], + ) -> bool: + return ( + self.can_complete_field_in_stanza is None + or self.can_complete_field_in_stanza(stanza_parts) + ) + + def complete_field( + self, + stanza_parts: Sequence[Deb822ParagraphElement], + markdown_kind: MarkupKind, + ) -> Optional[CompletionItem]: + if not self._can_complete_field_in_stanza(stanza_parts): + return None + name = self.name + complete_as = name + ": " + options = self.value_options_for_completer( + stanza_parts, + is_completion_for_field=True, + ) + if options is not None and len(options) == 1: + value = options[0].insert_text + if value is not None: + complete_as += value + tags = [] + is_deprecated = False + if self.replaced_by or self.deprecated_with_no_replacement: + is_deprecated = True + tags.append(CompletionItemTag.Deprecated) + + doc = self.hover_text + if doc: + doc = MarkupContent( + value=doc, + kind=markdown_kind, + ) + else: + doc = None + + return CompletionItem( + name, + insert_text=complete_as, + deprecated=is_deprecated, + tags=tags, + detail=self.synopsis_doc, + documentation=doc, + ) + + def value_options_for_completer( + self, + stanza_parts: Sequence[Deb822ParagraphElement], + *, + is_completion_for_field: bool = False, + ) -> Optional[Sequence[CompletionItem]]: + known_values = self.known_values + if known_values is None: + return None + if is_completion_for_field and ( + len(known_values) == 1 + or ( + len(known_values) == 2 + and self.warn_if_default + and self.default_value is not None + ) + ): + value = next( + iter(v for v in self.known_values if v != self.default_value), + None, + ) + if value is None: + return None + return [CompletionItem(value, insert_text=value)] + return [ + CompletionItem( + keyword.value, + insert_text=keyword.value, + ) + for keyword in known_values.values() + if keyword.is_keyword_valid_completion_in_stanza(stanza_parts) + ] def field_diagnostics( self, @@ -1723,6 +1838,7 @@ SOURCE_FIELDS = _fields( FieldValueClass.SINGLE_VALUE, known_values=allowed_values("yes"), synopsis_doc="Whether this non-free is auto-buildable on buildds", + # TODO: Needs logic for reading source section: can_complete_field_in_stanza=_complete_only_for_non_free_pkgs, hover_text=textwrap.dedent( """\ Used for non-free packages to denote that they may be auto-build on the Debian build infrastructure @@ -2364,6 +2480,7 @@ BINARY_FIELDS = _fields( ), Keyword( "same", + can_complete_keyword_in_stanza=_complete_only_in_arch_dep_pkgs, hover_text=textwrap.dedent( """\ The same version of the package can be co-installed for multiple architecture. However, @@ -2381,8 +2498,14 @@ BINARY_FIELDS = _fields( "allowed", hover_text=textwrap.dedent( """\ - **Advanced value**. The package is *not* co-installable with itself but can satisfy Multi-Arch - foreign and Multi-Arch same relations at the same. This is useful for implementations of + **Advanced and very rare value**. This value is exceedingly rare to the point that less + than 0.40% of all packages in Debian used in it (May 2024). It is even rarer for for + `Architecture: all` packages, where the number an order of magnitude smaller. Unless + a Multi-Arch expert or the Multi-Arch hinter suggested that you use this value, for + this package, then probably it is not the right choice. + + The value means that the package is *not* co-installable with itself but can satisfy Multi-Arch + `foreign` and Multi-Arch `same` relations at the same time. This is useful for implementations of scripting languages (such as Perl or Python). Here the interpreter contextually need to satisfy some relations as `Multi-Arch: foreign` and others as `Multi-Arch: same`. @@ -2504,6 +2627,7 @@ BINARY_FIELDS = _fields( DctrlKnownField( "XB-Installer-Menu-Item", FieldValueClass.SINGLE_VALUE, + can_complete_field_in_stanza=_complete_only_for_udeb_pkgs, custom_field_check=_combined_custom_field_check( _udeb_only_field_validation, _each_value_match_regex_validation(re.compile(r"^[1-9]\d{3,4}$")), @@ -2531,6 +2655,7 @@ BINARY_FIELDS = _fields( "X-DH-Build-For-Type", FieldValueClass.SINGLE_VALUE, custom_field_check=_arch_not_all_only_field_validation, + can_complete_field_in_stanza=_complete_only_in_arch_dep_pkgs, default_value="host", known_values=allowed_values( Keyword( @@ -2572,7 +2697,11 @@ BINARY_FIELDS = _fields( DctrlKnownField( "X-Time64-Compat", FieldValueClass.SINGLE_VALUE, - custom_field_check=_each_value_match_regex_validation(PKGNAME_REGEX), + can_complete_field_in_stanza=_complete_only_in_arch_dep_pkgs, + custom_field_check=_combined_custom_field_check( + _each_value_match_regex_validation(PKGNAME_REGEX), + _arch_not_all_only_field_validation, + ), synopsis_doc="(Special purpose) Compat name for time64_t transition", hover_text=textwrap.dedent( """\ @@ -2581,6 +2710,8 @@ BINARY_FIELDS = _fields( It is used to inform packaging helpers what the original (non-transitioned) package name was when the auto-detection is inadequate. The non-transitioned package name is then conditionally provided in the `${t64:Provides}` substitution variable. + + The field only works for architecture dependent packages. """ ), ), diff --git a/src/debputy/lsp/lsp_debian_copyright.py b/src/debputy/lsp/lsp_debian_copyright.py index abd5488..843627e 100644 --- a/src/debputy/lsp/lsp_debian_copyright.py +++ b/src/debputy/lsp/lsp_debian_copyright.py @@ -3,15 +3,12 @@ from typing import ( Union, Sequence, Tuple, - Iterator, Optional, - Iterable, Mapping, List, Dict, ) -from debputy.lsp.debputy_ls import DebputyLanguageServer from lsprotocol.types import ( DiagnosticSeverity, Range, @@ -20,7 +17,6 @@ from lsprotocol.types import ( CompletionItem, CompletionList, CompletionParams, - TEXT_DOCUMENT_WILL_SAVE_WAIT_UNTIL, DiagnosticRelatedInformation, Location, HoverParams, @@ -36,6 +32,7 @@ from lsprotocol.types import ( ) from debputy.linting.lint_util import LintState +from debputy.lsp.debputy_ls import DebputyLanguageServer from debputy.lsp.diagnostics import DiagnosticData from debputy.lsp.lsp_debian_control_reference_data import ( _DEP5_HEADER_FIELDS, @@ -73,7 +70,6 @@ from debputy.lsp.text_util import ( te_range_to_lsp, ) from debputy.lsp.vendoring._deb822_repro import ( - parse_deb822_file, Deb822FileElement, Deb822ParagraphElement, ) @@ -81,9 +77,6 @@ from debputy.lsp.vendoring._deb822_repro.parsing import ( Deb822KeyValuePairElement, LIST_SPACE_SEPARATED_INTERPRETATION, ) -from debputy.lsp.vendoring._deb822_repro.tokens import ( - Deb822Token, -) try: from debputy.lsp.vendoring._deb822_repro.locatable import ( @@ -442,11 +435,7 @@ def _lint_debian_copyright( position_codec = lint_state.position_codec doc_reference = lint_state.doc_uri diagnostics: List[Diagnostic] = [] - deb822_file = parse_deb822_file( - lines, - accept_files_with_duplicated_fields=True, - accept_files_with_error_tokens=True, - ) + deb822_file = lint_state.parsed_deb822_file_content first_error = _scan_for_syntax_errors_and_token_level_diagnostics( deb822_file, diff --git a/src/debputy/lsp/lsp_debian_rules.py b/src/debputy/lsp/lsp_debian_rules.py index ec8d9d6..390ddfa 100644 --- a/src/debputy/lsp/lsp_debian_rules.py +++ b/src/debputy/lsp/lsp_debian_rules.py @@ -43,7 +43,7 @@ from debputy.lsp.text_util import ( from debputy.util import _warn try: - from debian._deb822_repro.locatable import ( + from debputy.lsp.vendoring._deb822_repro.locatable import ( Position as TEPosition, Range as TERange, START_POSITION, diff --git a/src/debputy/lsp/lsp_debian_tests_control.py b/src/debputy/lsp/lsp_debian_tests_control.py index 001dbe2..20a198c 100644 --- a/src/debputy/lsp/lsp_debian_tests_control.py +++ b/src/debputy/lsp/lsp_debian_tests_control.py @@ -68,7 +68,6 @@ from debputy.lsp.text_util import ( te_range_to_lsp, ) from debputy.lsp.vendoring._deb822_repro import ( - parse_deb822_file, Deb822FileElement, Deb822ParagraphElement, ) @@ -259,6 +258,8 @@ def _diagnostics_for_paragraph( ) ) continue + if known_field is None: + continue diagnostics.extend( known_field.field_diagnostics( kvpair, @@ -433,11 +434,7 @@ def _lint_debian_tests_control( position_codec = lint_state.position_codec doc_reference = lint_state.doc_uri diagnostics: List[Diagnostic] = [] - deb822_file = parse_deb822_file( - lines, - accept_files_with_duplicated_fields=True, - accept_files_with_error_tokens=True, - ) + deb822_file = lint_state.parsed_deb822_file_content first_error = _scan_for_syntax_errors_and_token_level_diagnostics( deb822_file, diff --git a/src/debputy/lsp/lsp_dispatch.py b/src/debputy/lsp/lsp_dispatch.py index 99479dc..9a54c2b 100644 --- a/src/debputy/lsp/lsp_dispatch.py +++ b/src/debputy/lsp/lsp_dispatch.py @@ -8,6 +8,7 @@ from typing import ( Callable, Mapping, List, + TYPE_CHECKING, ) from lsprotocol.types import ( @@ -53,20 +54,30 @@ from debputy.util import _info _DOCUMENT_VERSION_TABLE: Dict[str, int] = {} -try: - from pygls.server import LanguageServer - from pygls.workspace import TextDocument + +if TYPE_CHECKING: + try: + from pygls.server import LanguageServer + except ImportError: + pass + from debputy.lsp.debputy_ls import DebputyLanguageServer DEBPUTY_LANGUAGE_SERVER = DebputyLanguageServer("debputy", f"v{__version__}") -except ImportError as e: +else: + try: + from pygls.server import LanguageServer + from debputy.lsp.debputy_ls import DebputyLanguageServer + + DEBPUTY_LANGUAGE_SERVER = DebputyLanguageServer("debputy", f"v{__version__}") + except ImportError: - class Mock: + class Mock: - def feature(self, *args, **kwargs): - return lambda x: x + def feature(self, *args, **kwargs): + return lambda x: x - DEBPUTY_LANGUAGE_SERVER = Mock() + DEBPUTY_LANGUAGE_SERVER = Mock() P = TypeVar("P") @@ -225,7 +236,7 @@ def _will_save_wait_until( @DEBPUTY_LANGUAGE_SERVER.feature(TEXT_DOCUMENT_FORMATTING) -def _will_save_wait_until( +def _format_document( ls: "DebputyLanguageServer", params: WillSaveTextDocumentParams, ) -> Optional[Sequence[TextEdit]]: @@ -244,7 +255,7 @@ def _dispatch_standard_handler( params: P, handler_table: Mapping[str, Callable[[L, P], R]], request_type: str, -) -> R: +) -> Optional[R]: doc = ls.workspace.get_text_document(doc_uri) id_source, language_id = ls.determine_language_id(doc) @@ -253,7 +264,7 @@ def _dispatch_standard_handler( _info( f"{request_type} for document: {doc.path} ({language_id}, {id_source}) - no handler" ) - return + return None _info( f"{request_type} for document: {doc.path} ({language_id}, {id_source}) - delegating to handler" ) diff --git a/src/debputy/lsp/lsp_generic_deb822.py b/src/debputy/lsp/lsp_generic_deb822.py index 409b27e..6d44d1a 100644 --- a/src/debputy/lsp/lsp_generic_deb822.py +++ b/src/debputy/lsp/lsp_generic_deb822.py @@ -1,5 +1,6 @@ import dataclasses import re +from itertools import chain from typing import ( Optional, Union, @@ -12,6 +13,7 @@ from typing import ( Iterable, Iterator, Callable, + cast, ) from lsprotocol.types import ( @@ -29,9 +31,8 @@ from lsprotocol.types import ( FoldingRangeKind, SemanticTokensParams, SemanticTokens, - WillSaveTextDocumentParams, TextEdit, - DocumentFormattingParams, + MessageType, ) from debputy.linting.lint_util import LintState @@ -45,17 +46,21 @@ from debputy.lsp.lsp_debian_control_reference_data import ( ) from debputy.lsp.lsp_features import SEMANTIC_TOKEN_TYPES_IDS from debputy.lsp.text_util import ( - normalize_dctrl_field_name, te_position_to_lsp, trim_end_of_line_whitespace, ) -from debputy.lsp.vendoring._deb822_repro import parse_deb822_file +from debputy.lsp.vendoring._deb822_repro.locatable import ( + START_POSITION, + Range as TERange, +) from debputy.lsp.vendoring._deb822_repro.parsing import ( Deb822KeyValuePairElement, Deb822ParagraphElement, + Deb822FileElement, ) from debputy.lsp.vendoring._deb822_repro.tokens import tokenize_deb822_file, Deb822Token -from debputy.util import _info +from debputy.lsp.vendoring._deb822_repro.types import TokenOrElement +from debputy.util import _info, _warn try: from pygls.server import LanguageServer @@ -67,61 +72,113 @@ except ImportError: _CONTAINS_SPACE_OR_COLON = re.compile(r"[\s:]") +def in_range(te_range: TERange, cursor_position: Position) -> bool: + start_pos = te_range.start_pos + end_pos = te_range.end_pos + if start_pos.line_position < cursor_position.line < end_pos.line_position: + return True + if ( + cursor_position.line == start_pos.line_position + and cursor_position.character >= start_pos.cursor_position + ): + return True + return ( + cursor_position.line == end_pos.line_position + and cursor_position.character < end_pos.cursor_position + ) + + +def _field_at_position( + stanza: Deb822ParagraphElement, + stanza_range: TERange, + position: Position, +) -> Tuple[Optional[Deb822KeyValuePairElement], bool]: + te_range = TERange(stanza_range.start_pos, stanza_range.start_pos) + for token_or_element in stanza.iter_parts(): + te_range = token_or_element.size().relative_to(te_range.end_pos) + if not in_range(te_range, position): + continue + if isinstance(token_or_element, Deb822KeyValuePairElement): + value_range = token_or_element.value_element.range_in_parent().relative_to( + te_range.start_pos + ) + in_value = in_range(value_range, position) + return token_or_element, in_value + return None, False + + +def _allow_stanza_continuation( + token_or_element: TokenOrElement, + is_completion: bool, +) -> bool: + if not is_completion: + return False + if token_or_element.is_error or token_or_element.is_comment: + return True + return ( + token_or_element.is_whitespace + and token_or_element.convert_to_text().count("\n") < 2 + ) + + def _at_cursor( + deb822_file: Deb822FileElement, doc: "TextDocument", lines: List[str], client_position: Position, -) -> Tuple[Position, Optional[str], str, bool, int, Set[str]]: - paragraph_no = -1 - paragraph_started = False - seen_fields: Set[str] = set() - last_field_seen: Optional[str] = None - current_field: Optional[str] = None + is_completion: bool = False, +) -> Tuple[Position, Optional[str], str, bool, int, Iterable[Deb822ParagraphElement]]: server_position = doc.position_codec.position_from_client_units( lines, client_position, ) - position_line_no = server_position.line + te_range = TERange( + START_POSITION, + START_POSITION, + ) + paragraph_no = -1 + previous_stanza: Optional[Deb822ParagraphElement] = None + next_stanza: Optional[Deb822ParagraphElement] = None + current_word = doc.word_at_position(client_position) + in_value: bool = False + file_iter = iter(deb822_file.iter_parts()) + matched_token: Optional[TokenOrElement] = None + matched_field: Optional[str] = None + for token_or_element in file_iter: + te_range = token_or_element.size().relative_to(te_range.end_pos) + if isinstance(token_or_element, Deb822ParagraphElement): + previous_stanza = token_or_element + paragraph_no += 1 + elif not _allow_stanza_continuation(token_or_element, is_completion): + previous_stanza = None + if not in_range(te_range, server_position): + continue + matched_token = token_or_element + if isinstance(token_or_element, Deb822ParagraphElement): + kvpair, in_value = _field_at_position( + token_or_element, te_range, server_position + ) + if kvpair is not None: + matched_field = kvpair.field_name + break - line_at_position = lines[position_line_no] - line_start = "" - if server_position.character: - line_start = line_at_position[0 : server_position.character] + if matched_token is not None and _allow_stanza_continuation( + matched_token, + is_completion, + ): + next_te = next(file_iter, None) + if isinstance(next_te, Deb822ParagraphElement): + next_stanza = next_te + + stanza_parts = (p for p in (previous_stanza, next_stanza) if p is not None) - for line_no, line in enumerate(lines): - if not line or line.isspace(): - if line_no == position_line_no: - current_field = last_field_seen - continue - last_field_seen = None - if line_no > position_line_no: - break - paragraph_started = False - elif line and line[0] == "#": - continue - elif line and not line[0].isspace() and ":" in line: - if not paragraph_started: - paragraph_started = True - seen_fields = set() - paragraph_no += 1 - key, _ = line.split(":", 1) - key_lc = key.lower() - last_field_seen = key_lc - if line_no == position_line_no: - current_field = key_lc - seen_fields.add(key_lc) - - in_value = bool(_CONTAINS_SPACE_OR_COLON.search(line_start)) - current_word = doc.word_at_position(client_position) - if current_field is not None: - current_field = normalize_dctrl_field_name(current_field) return ( server_position, - current_field, + matched_field, current_word, in_value, paragraph_no, - seen_fields, + stanza_parts, ) @@ -132,32 +189,44 @@ def deb822_completer( ) -> Optional[Union[CompletionList, Sequence[CompletionItem]]]: doc = ls.workspace.get_text_document(params.text_document.uri) lines = doc.lines + deb822_file = ls.lint_state(doc).parsed_deb822_file_content + if deb822_file is None: + _warn("The deb822 result missing failed!?") + ls.show_message_log( + "Internal error; could not get deb822 content!?", MessageType.Warning + ) + return None - _a, current_field, _b, in_value, paragraph_no, seen_fields = _at_cursor( + _a, current_field, _b, in_value, paragraph_no, matched_stanzas = _at_cursor( + deb822_file, doc, lines, params.position, + is_completion=True, ) stanza_metadata = file_metadata.guess_stanza_classification_by_idx(paragraph_no) + items: Optional[Sequence[CompletionItem]] if in_value: _info(f"Completion for field value {current_field}") if current_field is None: return None - known_field = stanza_metadata.get(current_field) + known_field: Deb822KnownField = stanza_metadata.get(current_field) if known_field is None: return None - items = _complete_field_value(known_field) + items = known_field.value_options_for_completer(list(matched_stanzas)) else: _info("Completing field name") items = _complete_field_name( ls, stanza_metadata, - seen_fields, + matched_stanzas, ) - _info(f"Completion candidates: {items}") + _info( + f"Completion candidates: {[i.label for i in items] if items is not None else 'None'}" + ) return items @@ -184,8 +253,19 @@ def deb822_hover( ) -> Optional[Hover]: doc = ls.workspace.get_text_document(params.text_document.uri) lines = doc.lines + deb822_file = ls.lint_state(doc).parsed_deb822_file_content + if deb822_file is None: + _warn("The deb822 result missing failed!?") + ls.show_message_log( + "Internal error; could not get deb822 content!?", MessageType.Warning + ) + return None + server_pos, current_field, word_at_position, in_value, paragraph_no, _ = _at_cursor( - doc, lines, params.position + deb822_file, + doc, + lines, + params.position, ) stanza_metadata = file_metadata.guess_stanza_classification_by_idx(paragraph_no) @@ -428,11 +508,11 @@ def deb822_format_file( return trim_end_of_line_whitespace(lint_state.position_codec, lint_state.lines) formatter = effective_preference.deb822_formatter() lines = lint_state.lines - deb822_file = parse_deb822_file( - lines, - accept_files_with_duplicated_fields=True, - accept_files_with_error_tokens=True, - ) + deb822_file = lint_state.parsed_deb822_file_content + if deb822_file is None: + _warn("The deb822 result missing failed!?") + return None + return list( file_metadata.reformat( effective_preference, @@ -453,11 +533,14 @@ def deb822_semantic_tokens_full( doc = ls.workspace.get_text_document(request.text_document.uri) position_codec = doc.position_codec lines = doc.lines - deb822_file = parse_deb822_file( - lines, - accept_files_with_duplicated_fields=True, - accept_files_with_error_tokens=True, - ) + deb822_file = ls.lint_state(doc).parsed_deb822_file_content + if deb822_file is None: + _warn("The deb822 result missing failed!?") + ls.show_message_log( + "Internal error; could not get deb822 content!?", MessageType.Warning + ) + return None + tokens: List[int] = [] comment_token_code = SEMANTIC_TOKEN_TYPES_IDS["comment"] sem_token_state = SemanticTokenState( @@ -494,65 +577,33 @@ def deb822_semantic_tokens_full( return SemanticTokens(tokens) -def _should_complete_field_with_value(cand: Deb822KnownField) -> bool: - return cand.known_values is not None and ( - len(cand.known_values) == 1 - or ( - len(cand.known_values) == 2 - and cand.warn_if_default - and cand.default_value is not None - ) - ) - - def _complete_field_name( ls: "DebputyLanguageServer", fields: StanzaMetadata[Any], - seen_fields: Container[str], -) -> Optional[Union[CompletionList, Sequence[CompletionItem]]]: + matched_stanzas: Iterable[Deb822ParagraphElement], +) -> Sequence[CompletionItem]: items = [] markdown_kind = ls.completion_item_document_markup( MarkupKind.Markdown, MarkupKind.PlainText ) + matched_stanzas = list(matched_stanzas) + # TODO: Normalize fields according to file rules (X[BCS]- should be stripped in some files) + seen_fields = set( + f.lower() + for f in chain.from_iterable( + # The typing from python3-debian is not entirely optimal here. The iter always return a + # `str`, but the provided type is `ParagraphKey` (because `__getitem__` supports those) + # and that is not exclusively a `str`. + # + # So, this cast for now + cast("Iterable[str]", s) + for s in matched_stanzas + ) + ) for cand_key, cand in fields.items(): if cand_key.lower() in seen_fields: continue - name = cand.name - complete_as = name + ": " - if _should_complete_field_with_value(cand): - value = next(iter(v for v in cand.known_values if v != cand.default_value)) - complete_as += value - tags = [] - is_deprecated = False - if cand.replaced_by or cand.deprecated_with_no_replacement: - is_deprecated = True - tags.append(CompletionItemTag.Deprecated) - - doc = cand.hover_text - if doc: - doc = MarkupContent( - value=doc, - kind=markdown_kind, - ) - else: - doc = None - - items.append( - CompletionItem( - name, - insert_text=complete_as, - deprecated=is_deprecated, - tags=tags, - detail=cand.synopsis_doc, - documentation=doc, - ) - ) + item = cand.complete_field(matched_stanzas, markdown_kind) + if item is not None: + items.append(item) return items - - -def _complete_field_value( - field: Deb822KnownField, -) -> Optional[Union[CompletionList, Sequence[CompletionItem]]]: - if field.known_values is None: - return None - return [CompletionItem(v) for v in field.known_values] diff --git a/src/debputy/lsp/lsp_reference_keyword.py b/src/debputy/lsp/lsp_reference_keyword.py index 44f43fd..c71352a 100644 --- a/src/debputy/lsp/lsp_reference_keyword.py +++ b/src/debputy/lsp/lsp_reference_keyword.py @@ -1,6 +1,8 @@ import dataclasses import textwrap -from typing import Optional, Union, Mapping +from typing import Optional, Union, Mapping, Sequence, Callable, Iterable + +from debputy.lsp.vendoring._deb822_repro import Deb822ParagraphElement @dataclasses.dataclass(slots=True, frozen=True) @@ -10,11 +12,23 @@ class Keyword: is_obsolete: bool = False replaced_by: Optional[str] = None is_exclusive: bool = False + can_complete_keyword_in_stanza: Optional[ + Callable[[Iterable[Deb822ParagraphElement]], bool] + ] = None """For keywords in fields that allow multiple keywords, the `is_exclusive` can be used for keywords that cannot be used with other keywords. As an example, the `all` value in `Architecture` of `debian/control` cannot be used with any other architecture. """ + def is_keyword_valid_completion_in_stanza( + self, + stanza_parts: Sequence[Deb822ParagraphElement], + ) -> bool: + return ( + self.can_complete_keyword_in_stanza is None + or self.can_complete_keyword_in_stanza(stanza_parts) + ) + def allowed_values(*values: Union[str, Keyword]) -> Mapping[str, Keyword]: as_keywords = [k if isinstance(k, Keyword) else Keyword(k) for k in values] diff --git a/src/debputy/lsp/quickfixes.py b/src/debputy/lsp/quickfixes.py index 7a8a904..a9fcf7b 100644 --- a/src/debputy/lsp/quickfixes.py +++ b/src/debputy/lsp/quickfixes.py @@ -137,7 +137,7 @@ def _correct_value_code_action( @_code_handler_for("insert-text-on-line-after-diagnostic") -def _correct_value_code_action( +def _insert_text_on_line_after_diagnostic_code_action( code_action_data: InsertTextOnLineAfterDiagnosticCodeAction, code_action_params: CodeActionParams, diagnostic: Diagnostic, diff --git a/src/debputy/lsp/style_prefs.py b/src/debputy/lsp/style_prefs.py index 5dfdac2..40d850b 100644 --- a/src/debputy/lsp/style_prefs.py +++ b/src/debputy/lsp/style_prefs.py @@ -14,13 +14,15 @@ from typing import ( Mapping, Self, Dict, + Iterable, + Any, ) from debputy.lsp.lsp_reference_keyword import ALL_PUBLIC_NAMED_STYLES from debputy.lsp.vendoring._deb822_repro.types import FormatterCallback from debputy.lsp.vendoring.wrap_and_sort import wrap_and_sort_formatter from debputy.packages import SourcePackage -from debputy.util import _error +from debputy.util import _error, _info from debputy.yaml import MANIFEST_YAML from debputy.yaml.compat import CommentedMap @@ -32,6 +34,29 @@ BUILTIN_STYLES = os.path.join(os.path.dirname(__file__), "style-preferences.yaml _NORMALISE_FIELD_CONTENT_KEY = ["formatting", "deb822", "normalize-field-content"] _UPLOADER_SPLIT_RE = re.compile(r"(?<=>)\s*,") +_WAS_OPTIONS = { + "-a": ("formatting_deb822_always_wrap", True), + "--always-wrap": ("formatting_deb822_always_wrap", True), + "-s": ("formatting_deb822_short_indent", True), + "--short-indent": ("formatting_deb822_short_indent", True), + "-t": ("formatting_deb822_trailing_separator", True), + "--trailing-separator": ("formatting_deb822_trailing_separator", True), + # Noise option for us; we do not accept `--no-keep-first` though + "-k": (None, True), + "--keep-first": (None, True), + "--no-keep-first": ("DISABLE_NORMALIZE_STANZA_ORDER", True), + "-b": ("formatting_deb822_normalize_stanza_order", True), + "--sort-binary-packages": ("formatting_deb822_normalize_stanza_order", True), +} + +_WAS_DEFAULTS = { + "formatting_deb822_always_wrap": False, + "formatting_deb822_short_indent": False, + "formatting_deb822_trailing_separator": False, + "formatting_deb822_normalize_stanza_order": False, + "formatting_deb822_normalize_field_content": True, +} + @dataclasses.dataclass(slots=True, frozen=True, kw_only=True) class PreferenceOption(Generic[PT]): @@ -400,7 +425,6 @@ class EffectivePreference: a_value = getattr(a, attr_name) b_value = getattr(b, attr_name) if a_value != b_value: - print(f"{attr_name} was misaligned") return None return a @@ -415,6 +439,9 @@ class EffectivePreference: ), ) + def replace(self, /, **changes: Any) -> Self: + return dataclasses.replace(self, **changes) + @dataclasses.dataclass(slots=True, frozen=True) class MaintainerPreference(EffectivePreference): @@ -539,14 +566,45 @@ def extract_maint_email(maint: str) -> str: def determine_effective_style( style_preference_table: StylePreferenceTable, - source_package: SourcePackage, + source_package: Optional[SourcePackage], + salsa_ci: Optional[CommentedMap], ) -> Optional[EffectivePreference]: - style = source_package.fields.get("X-Style") + style = source_package.fields.get("X-Style") if source_package is not None else None if style is not None: if style not in ALL_PUBLIC_NAMED_STYLES: return None return style_preference_table.named_styles.get(style) + if salsa_ci: + disable_wrap_and_sort = salsa_ci.mlget( + ["variables", "SALSA_CI_DISABLE_WRAP_AND_SORT"], + list_ok=True, + default=True, + ) + + if isinstance(disable_wrap_and_sort, str): + disable_wrap_and_sort = disable_wrap_and_sort in ("yes", "1", "true") + elif not isinstance(disable_wrap_and_sort, (int, bool)): + disable_wrap_and_sort = True + else: + disable_wrap_and_sort = ( + disable_wrap_and_sort is True or disable_wrap_and_sort == 1 + ) + if not disable_wrap_and_sort: + wrap_and_sort_options = salsa_ci.mlget( + ["variables", "SALSA_CI_WRAP_AND_SORT_ARGS"], + list_ok=True, + default=None, + ) + if wrap_and_sort_options is None: + wrap_and_sort_options = "" + elif not isinstance(wrap_and_sort_options, str): + return None + return parse_salsa_ci_wrap_and_sort_args(wrap_and_sort_options) + + if source_package is None: + return None + maint = source_package.fields.get("Maintainer") if maint is None: return None @@ -580,3 +638,33 @@ def determine_effective_style( if isinstance(r, MaintainerPreference): return r.as_effective_pref() return r + + +def _split_options(args: Iterable[str]) -> Iterable[str]: + for arg in args: + if arg.startswith("--"): + yield arg + continue + if not arg.startswith("-") or len(arg) < 2: + yield arg + continue + for sarg in arg[1:]: + yield f"-{sarg}" + + +@functools.lru_cache +def parse_salsa_ci_wrap_and_sort_args(args: str) -> Optional[EffectivePreference]: + options = dict(_WAS_DEFAULTS) + for arg in _split_options(args.split()): + v = _WAS_OPTIONS.get(arg) + if v is None: + return None + varname, value = v + if varname is None: + continue + options[varname] = value + if "DISABLE_NORMALIZE_STANZA_ORDER" in options: + del options["DISABLE_NORMALIZE_STANZA_ORDER"] + options["formatting_deb822_normalize_stanza_order"] = False + + return EffectivePreference(**options) # type: ignore diff --git a/src/debputy/packages.py b/src/debputy/packages.py index 4dfdd49..3a6ee16 100644 --- a/src/debputy/packages.py +++ b/src/debputy/packages.py @@ -11,6 +11,7 @@ from typing import ( overload, ) +from debian.deb822 import Deb822 from debian.debian_support import DpkgArchTable from ._deb_options_profiles import DebBuildOptionsAndProfiles @@ -18,7 +19,11 @@ from .architecture_support import ( DpkgArchitectureBuildProcessValuesTable, dpkg_architecture_table, ) -from .lsp.vendoring._deb822_repro import parse_deb822_file, Deb822ParagraphElement +from .lsp.vendoring._deb822_repro import ( + parse_deb822_file, + Deb822ParagraphElement, + Deb822FileElement, +) from .util import DEFAULT_PACKAGE_TYPE, UDEB_PACKAGE_TYPE, _error, active_profiles_match _MANDATORY_BINARY_PACKAGE_FIELD = [ @@ -68,7 +73,7 @@ class DctrlParser: def parse_source_debian_control( self, debian_control_lines: Iterable[str], - ) -> Tuple["SourcePackage", Dict[str, "BinaryPackage"]]: ... + ) -> Tuple[Deb822FileElement, "SourcePackage", Dict[str, "BinaryPackage"]]: ... @overload def parse_source_debian_control( @@ -76,21 +81,28 @@ class DctrlParser: debian_control_lines: Iterable[str], *, ignore_errors: bool = False, - ) -> Tuple[Optional["SourcePackage"], Optional[Dict[str, "BinaryPackage"]]]: ... + ) -> Tuple[ + Deb822FileElement, + Optional["SourcePackage"], + Optional[Dict[str, "BinaryPackage"]], + ]: ... def parse_source_debian_control( self, debian_control_lines: Iterable[str], *, ignore_errors: bool = False, - ) -> Tuple[Optional["SourcePackage"], Optional[Dict[str, "BinaryPackage"]]]: - dctrl_paragraphs = list( - parse_deb822_file( - debian_control_lines, - accept_files_with_error_tokens=ignore_errors, - accept_files_with_duplicated_fields=ignore_errors, - ) + ) -> Tuple[ + Optional[Deb822FileElement], + Optional["SourcePackage"], + Optional[Dict[str, "BinaryPackage"]], + ]: + deb822_file = parse_deb822_file( + debian_control_lines, + accept_files_with_error_tokens=ignore_errors, + accept_files_with_duplicated_fields=ignore_errors, ) + dctrl_paragraphs = list(deb822_file) if len(dctrl_paragraphs) < 2: if not ignore_errors: _error( @@ -99,7 +111,7 @@ class DctrlParser: source_package = ( SourcePackage(dctrl_paragraphs[0]) if dctrl_paragraphs else None ) - return source_package, None + return deb822_file, source_package, None source_package = SourcePackage(dctrl_paragraphs[0]) bin_pkgs = [] @@ -107,9 +119,16 @@ class DctrlParser: if ignore_errors: if "Package" not in p: continue - for f in _MANDATORY_BINARY_PACKAGE_FIELD: - if f not in p: - p[f] = "unknown" + missing_field = any(f not in p for f in _MANDATORY_BINARY_PACKAGE_FIELD) + if missing_field: + # In the LSP context, it is problematic if we "add" fields as it ranges and provides invalid + # results. However, `debputy` also needs the mandatory fields to be there, so we clone the + # stanzas that `debputy` (build) will see to add missing fields. + copy = Deb822(p) + for f in _MANDATORY_BINARY_PACKAGE_FIELD: + if f not in p: + copy[f] = "unknown" + p = copy bin_pkgs.append( _create_binary_package( p, @@ -137,7 +156,7 @@ class DctrlParser: f"The following *excluded* packages (-N) are not listed in debian/control: {sorted(unknown)}" ) - return source_package, bin_pkgs_table + return deb822_file, source_package, bin_pkgs_table def _check_package_sets( diff --git a/tests/lint_tests/lint_tutil.py b/tests/lint_tests/lint_tutil.py index 8290712..08120ee 100644 --- a/tests/lint_tests/lint_tutil.py +++ b/tests/lint_tests/lint_tutil.py @@ -52,7 +52,7 @@ class LintWrapper: binary_packages = None dctrl_lines = self.dctrl_lines if dctrl_lines is not None: - source_package, binary_packages = ( + _, source_package, binary_packages = ( self._dctrl_parser.parse_source_debian_control( dctrl_lines, ignore_errors=True ) diff --git a/tests/lsp_tests/test_lsp_dctrl.py b/tests/lsp_tests/test_lsp_dctrl.py index 2bc90ba..e93ba17 100644 --- a/tests/lsp_tests/test_lsp_dctrl.py +++ b/tests/lsp_tests/test_lsp_dctrl.py @@ -1,4 +1,7 @@ import textwrap +from typing import Optional + +import pytest from debputy.lsp.debputy_ls import DebputyLanguageServer @@ -56,6 +59,165 @@ def test_dctrl_complete_field(ls: "DebputyLanguageServer") -> None: assert "Package" not in keywords assert "Source" not in keywords + cursor_pos = put_doc_with_cursor( + ls, + dctrl_uri, + "debian/control", + textwrap.dedent( + """\ + Source: foo + + Package: foo + <CURSOR> + Architecture: any +""" + ), + ) + + matches = _debian_control_completions( + ls, + CompletionParams(TextDocumentIdentifier(dctrl_uri), cursor_pos), + ) + assert isinstance(matches, list) + keywords = {m.label for m in matches} + assert "Multi-Arch" in keywords + # Should be considered present even though it is parsed as two stanzas with a space + assert "Architecture" not in keywords + # Already present or wrong section + assert "Package" not in keywords + assert "Source" not in keywords + + cursor_pos = put_doc_with_cursor( + ls, + dctrl_uri, + "debian/control", + textwrap.dedent( + """\ + Source: foo + + Package: foo + Sec<CURSOR> + Architecture: any +""" + ), + ) + + matches = _debian_control_completions( + ls, + CompletionParams(TextDocumentIdentifier(dctrl_uri), cursor_pos), + ) + assert isinstance(matches, list) + keywords = {m.label for m in matches} + # Included since we rely on client filtering (some clients let "RRR" match "R(ules-)R(equires-)R(oot), etc). + assert "Multi-Arch" in keywords + # Should be considered present even though it is parsed as two stanzas with an error + assert "Architecture" not in keywords + # Already present or wrong section + assert "Package" not in keywords + assert "Source" not in keywords + + +@pytest.mark.parametrize( + "case,is_arch_all", + [ + ("Architecture: any\n<CURSOR>", False), + ("Architecture: any\nM-A<CURSOR>", False), + ("<CURSOR>\nArchitecture: any", False), + ("M-A<CURSOR>\nArchitecture: any", False), + ("Architecture: all\n<CURSOR>", True), + ("Architecture: all\nM-A<CURSOR>", True), + ("<CURSOR>\nArchitecture: all", True), + ("M-A<CURSOR>\nArchitecture: all", True), + # Does not have architecture + ("M-A<CURSOR>", None), + ], +) +def test_dctrl_complete_field_context( + ls: "DebputyLanguageServer", + case: str, + is_arch_all: Optional[bool], +) -> None: + dctrl_uri = "file:///nowhere/debian/control" + + content = textwrap.dedent( + """\ + Source: foo + + Package: foo + {CASE} +""" + ).format(CASE=case) + cursor_pos = put_doc_with_cursor( + ls, + dctrl_uri, + "debian/control", + content, + ) + + matches = _debian_control_completions( + ls, + CompletionParams(TextDocumentIdentifier(dctrl_uri), cursor_pos), + ) + assert isinstance(matches, list) + keywords = {m.label for m in matches} + # Missing Architecture counts as "arch:all" by the completion logic + if is_arch_all is False: + assert "X-DH-Build-For-Type" in keywords + else: + assert "X-DH-Build-For-Type" not in keywords + + +def test_dctrl_complete_field_value_context(ls: "DebputyLanguageServer") -> None: + dctrl_uri = "file:///nowhere/debian/control" + + content = textwrap.dedent( + """\ + Source: foo + + Package: foo + Architecture: any + Multi-Arch: <CURSOR> +""" + ) + cursor_pos = put_doc_with_cursor( + ls, + dctrl_uri, + "debian/control", + content, + ) + + matches = _debian_control_completions( + ls, + CompletionParams(TextDocumentIdentifier(dctrl_uri), cursor_pos), + ) + assert isinstance(matches, list) + keywords = {m.label for m in matches} + assert keywords == {"no", "same", "foreign", "allowed"} + + content = textwrap.dedent( + """\ + Source: foo + + Package: foo + Architecture: all + Multi-Arch: <CURSOR> +""" + ) + cursor_pos = put_doc_with_cursor( + ls, + dctrl_uri, + "debian/control", + content, + ) + + matches = _debian_control_completions( + ls, + CompletionParams(TextDocumentIdentifier(dctrl_uri), cursor_pos), + ) + assert isinstance(matches, list) + keywords = {m.label for m in matches} + assert keywords == {"no", "foreign", "allowed"} + def test_dctrl_hover_doc_field(ls: "DebputyLanguageServer") -> None: dctrl_uri = "file:///nowhere/debian/control" diff --git a/tests/test_style.py b/tests/test_style.py index 9ce83bd..cae4510 100644 --- a/tests/test_style.py +++ b/tests/test_style.py @@ -1,7 +1,15 @@ +from typing import Mapping, Any, Optional + import pytest from debian.deb822 import Deb822 - -from debputy.lsp.style_prefs import StylePreferenceTable, determine_effective_style +from debputy.yaml.compat import CommentedMap + +from debputy.lsp.style_prefs import ( + StylePreferenceTable, + determine_effective_style, + EffectivePreference, + _WAS_DEFAULTS, +) from debputy.packages import SourcePackage @@ -63,7 +71,7 @@ def test_compat_styles() -> None: ) src = SourcePackage(fields) - effective_style = determine_effective_style(styles, src) + effective_style = determine_effective_style(styles, src, None) assert effective_style == nt_pref fields["Uploaders"] = ( @@ -71,7 +79,7 @@ def test_compat_styles() -> None: ) src = SourcePackage(fields) - effective_style = determine_effective_style(styles, src) + effective_style = determine_effective_style(styles, src, None) assert effective_style == nt_pref assert effective_style == zeha_pref @@ -80,7 +88,7 @@ def test_compat_styles() -> None: ) src = SourcePackage(fields) - effective_style = determine_effective_style(styles, src) + effective_style = determine_effective_style(styles, src, None) assert effective_style is None @@ -100,7 +108,7 @@ def test_compat_styles_team_maint() -> None: assert "random@example.org" not in styles.maintainer_preferences team_style = styles.maintainer_preferences["team@lists.debian.org"] assert team_style.is_packaging_team - effective_style = determine_effective_style(styles, src) + effective_style = determine_effective_style(styles, src, None) assert effective_style == team_style.as_effective_pref() @@ -117,5 +125,105 @@ def test_x_style() -> None: assert "random@example.org" not in styles.maintainer_preferences assert "black" in styles.named_styles black_style = styles.named_styles["black"] - effective_style = determine_effective_style(styles, src) + effective_style = determine_effective_style(styles, src, None) assert effective_style == black_style + + +def test_was_from_salsa_ci_style() -> None: + styles = StylePreferenceTable.load_styles() + fields = Deb822( + { + "Package": "foo", + "Maintainer": "Random Developer <random@example.org>", + }, + ) + src = SourcePackage(fields) + assert "random@example.org" not in styles.maintainer_preferences + effective_style = determine_effective_style(styles, src, None) + assert effective_style is None + salsa_ci = CommentedMap( + {"variables": CommentedMap({"SALSA_CI_DISABLE_WRAP_AND_SORT": "yes"})} + ) + effective_style = determine_effective_style(styles, src, salsa_ci) + assert effective_style is None + + salsa_ci = CommentedMap( + {"variables": CommentedMap({"SALSA_CI_DISABLE_WRAP_AND_SORT": "no"})} + ) + effective_style = determine_effective_style(styles, src, salsa_ci) + was_style = EffectivePreference(**_WAS_DEFAULTS) + assert effective_style == was_style + + +@pytest.mark.parametrize( + "was_args,style_delta", + [ + ( + "-a", + { + "formatting_deb822_always_wrap": True, + }, + ), + ( + "-sa", + { + "formatting_deb822_always_wrap": True, + "formatting_deb822_short_indent": True, + }, + ), + ( + "-sa --keep-first", + { + "formatting_deb822_always_wrap": True, + "formatting_deb822_short_indent": True, + }, + ), + ( + "-sab --keep-first", + { + "formatting_deb822_always_wrap": True, + "formatting_deb822_short_indent": True, + "formatting_deb822_normalize_stanza_order": True, + }, + ), + ( + "-sab --no-keep-first", + { + "formatting_deb822_always_wrap": True, + "formatting_deb822_short_indent": True, + "formatting_deb822_normalize_stanza_order": False, + }, + ), + ], +) +def test_was_from_salsa_ci_style_args( + was_args: str, style_delta: Optional[Mapping[str, Any]] +) -> None: + styles = StylePreferenceTable.load_styles() + fields = Deb822( + { + "Package": "foo", + "Maintainer": "Random Developer <random@example.org>", + }, + ) + src = SourcePackage(fields) + assert "random@example.org" not in styles.maintainer_preferences + salsa_ci = CommentedMap( + { + "variables": CommentedMap( + { + "SALSA_CI_DISABLE_WRAP_AND_SORT": "no", + "SALSA_CI_WRAP_AND_SORT_ARGS": was_args, + } + ) + } + ) + effective_style = determine_effective_style(styles, src, salsa_ci) + if style_delta is None: + assert effective_style is None + else: + was_style = EffectivePreference(**_WAS_DEFAULTS).replace( + **style_delta, + ) + + assert effective_style == was_style |