Merging upstream version 0.1.29.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

4 files changed, 188 insertions, 20 deletions

diff --git a/tests/lsp_tests/lsp_tutil.py b/tests/lsp_tests/lsp_tutil.py
index 0843f79..bc0fa91 100644
--- a/tests/lsp_tests/lsp_tutil.py
+++ b/tests/lsp_tests/lsp_tutil.py
@@ -1,16 +1,52 @@
-from typing import Tuple, Union
+import dataclasses
+from typing import Tuple, Union, FrozenSet, Optional, List
+
+from debputy.lsp.lsp_features import SEMANTIC_TOKENS_LEGEND
+from debputy.util import grouper
 
 try:
-    from pygls.server import LanguageServer
     from lsprotocol.types import (
         TextDocumentItem,
         Position,
+        Range,
+        SemanticTokens,
     )
     from debputy.lsp.debputy_ls import DebputyLanguageServer
 except ImportError:
     pass
 
 
+@dataclasses.dataclass(slots=True, frozen=True)
+class ResolvedSemanticToken:
+    range: "Range"
+    token_name: str
+    modifiers: FrozenSet[str] = frozenset()
+
+
+def resolved_semantic_token(
+    line_no: int,
+    col_start: int,
+    token_len: int,
+    token_type: str,
+    *,
+    token_modifiers: FrozenSet[str] = frozenset(),
+) -> ResolvedSemanticToken:
+    return ResolvedSemanticToken(
+        Range(
+            Position(
+                line_no,
+                col_start,
+            ),
+            Position(
+                line_no,
+                col_start + token_len,
+            ),
+        ),
+        token_type,
+        token_modifiers,
+    )
+
+
 def _locate_cursor(text: str) -> Tuple[str, "Position"]:
     lines = text.splitlines(keepends=True)
     for line_no in range(len(lines)):
@@ -27,12 +63,27 @@ def _locate_cursor(text: str) -> Tuple[str, "Position"]:
 
 
 def put_doc_with_cursor(
-    ls: Union["LanguageServer", "DebputyLanguageServer"],
+    ls: "DebputyLanguageServer",
     uri: str,
     language_id: str,
     content: str,
 ) -> "Position":
     cleaned_content, cursor_pos = _locate_cursor(content)
+    put_doc_no_cursor(
+        ls,
+        uri,
+        language_id,
+        cleaned_content,
+    )
+    return cursor_pos
+
+
+def put_doc_no_cursor(
+    ls: "DebputyLanguageServer",
+    uri: str,
+    language_id: str,
+    content: str,
+) -> None:
     doc_version = 1
     existing = ls.workspace.text_documents.get(uri)
     if existing is not None:
@@ -42,7 +93,38 @@ def put_doc_with_cursor(
             uri,
             language_id,
             doc_version,
-            cleaned_content,
+            content,
         )
     )
-    return cursor_pos
+
+
+def resolve_semantic_tokens(
+    token_result: Optional["SemanticTokens"],
+) -> Optional[List[ResolvedSemanticToken]]:
+    if token_result is None:
+        return None
+    assert (len(token_result.data) % 5) == 0
+    current_line = 0
+    current_col = 0
+    resolved_tokens = []
+    token_types = SEMANTIC_TOKENS_LEGEND.token_types
+    for token_data in grouper(token_result.data, 5, incomplete="strict"):
+        line_delta, col_start_delta, token_len, token_code, modifier_codes = token_data
+        if line_delta:
+            current_col = 0
+        current_line += line_delta
+        current_col += col_start_delta
+        assert (
+            not modifier_codes
+        ), "TODO: Modifiers not supported (no modifiers defined)"
+
+        resolved_tokens.append(
+            resolved_semantic_token(
+                current_line,
+                current_col,
+                token_len,
+                token_types[token_code],
+            ),
+        )
+
+    return resolved_tokens
diff --git a/tests/lsp_tests/test_debpkg_metadata.py b/tests/lsp_tests/test_debpkg_metadata.py
new file mode 100644
index 0000000..f784b0a
--- /dev/null
+++ b/tests/lsp_tests/test_debpkg_metadata.py
@@ -0,0 +1,25 @@
+from typing import Optional
+
+import pytest
+
+from debputy.lsp.lsp_debian_control_reference_data import package_name_to_section
+
+
+@pytest.mark.parametrize(
+    "name,guessed_section",
+    [
+        ("foo-udeb", "debian-installer"),
+        ("python-foo", "python"),
+        ("python-foo-doc", "doc"),
+        ("libfoo-dev", "libdevel"),
+        ("php-foo", "php"),
+        ("libpam-foo", "admin"),
+        ("fonts-foo", "fonts"),
+        ("xxx-l10n", "localization"),
+        ("xxx-l10n-bar", "localization"),
+        ("libfoo4", "libs"),
+        ("unknown", None),
+    ]
+)
+def test_package_name_to_section(name: str, guessed_section: Optional[str]) -> None:
+    assert package_name_to_section(name) == guessed_section
diff --git a/tests/lsp_tests/test_lsp_dctrl.py b/tests/lsp_tests/test_lsp_dctrl.py
index 122b929..2bc90ba 100644
--- a/tests/lsp_tests/test_lsp_dctrl.py
+++ b/tests/lsp_tests/test_lsp_dctrl.py
@@ -1,25 +1,34 @@
 import textwrap
 
+from debputy.lsp.debputy_ls import DebputyLanguageServer
+
 try:
     from lsprotocol.types import (
         CompletionParams,
         TextDocumentIdentifier,
         HoverParams,
         MarkupContent,
+        SemanticTokensParams,
     )
 
     from debputy.lsp.lsp_debian_control import (
         _debian_control_completions,
         _debian_control_hover,
+        _debian_control_semantic_tokens_full,
     )
 
     from pygls.server import LanguageServer
 except ImportError:
     pass
-from lsp_tests.lsp_tutil import put_doc_with_cursor
+from lsp_tests.lsp_tutil import (
+    put_doc_with_cursor,
+    put_doc_no_cursor,
+    resolve_semantic_tokens,
+    resolved_semantic_token,
+)
 
 
-def test_dctrl_complete_field(ls: "LanguageServer") -> None:
+def test_dctrl_complete_field(ls: "DebputyLanguageServer") -> None:
     dctrl_uri = "file:///nowhere/debian/control"
 
     cursor_pos = put_doc_with_cursor(
@@ -48,7 +57,7 @@ def test_dctrl_complete_field(ls: "LanguageServer") -> None:
     assert "Source" not in keywords
 
 
-def test_dctrl_hover_doc_field(ls: "LanguageServer") -> None:
+def test_dctrl_hover_doc_field(ls: "DebputyLanguageServer") -> None:
     dctrl_uri = "file:///nowhere/debian/control"
     cursor_pos = put_doc_with_cursor(
         ls,
@@ -72,7 +81,7 @@ def test_dctrl_hover_doc_field(ls: "LanguageServer") -> None:
     assert "Determines which architecture" in hover_doc.contents.value
 
 
-def test_dctrl_hover_doc_synopsis(ls: "LanguageServer") -> None:
+def test_dctrl_hover_doc_synopsis(ls: "DebputyLanguageServer") -> None:
     dctrl_uri = "file:///nowhere/debian/control"
     cursor_pos = put_doc_with_cursor(
         ls,
@@ -98,7 +107,7 @@ def test_dctrl_hover_doc_synopsis(ls: "LanguageServer") -> None:
     assert "super charged tool with batteries included" in hover_doc.contents.value
 
 
-def test_dctrl_hover_doc_substvars(ls: "LanguageServer") -> None:
+def test_dctrl_hover_doc_substvars(ls: "DebputyLanguageServer") -> None:
     dctrl_uri = "file:///nowhere/debian/control"
     matching_cases = [
         "bar (= <CURSOR>${binary:Version})",
@@ -158,3 +167,54 @@ def test_dctrl_hover_doc_substvars(ls: "LanguageServer") -> None:
         if hover_doc is not None and isinstance(hover_doc.contents, MarkupContent):
             provided_doc = hover_doc.contents.value
         assert not provided_doc.startswith("# Substvar `${binary:Version}`")
+
+
+def test_dctrl_semantic_tokens(ls: "DebputyLanguageServer") -> None:
+    dctrl_uri = "file:///nowhere/debian/control"
+    put_doc_no_cursor(
+        ls,
+        dctrl_uri,
+        "debian/control",
+        textwrap.dedent(
+            """\
+        # Some leading comment
+
+        Source: foo
+
+        # Comment between stanzas
+
+        Package: foo
+        # Comment before Architecture
+        Architecture: any
+        Depends:
+        # Comment about bar
+             bar (>= 1.0),
+             baz [linux-any] <!pkg.foo.bootstrap>
+        Description: super charged tool with batteries included
+        Unknown-Field: Some value
+        # Comment in that field
+          that we do not know about.
+"""
+        ),
+    )
+
+    semantic_tokens = _debian_control_semantic_tokens_full(
+        ls,
+        SemanticTokensParams(TextDocumentIdentifier(dctrl_uri)),
+    )
+    resolved_semantic_tokens = resolve_semantic_tokens(semantic_tokens)
+    assert resolved_semantic_tokens is not None
+    assert resolved_semantic_tokens == [
+        resolved_semantic_token(0, 0, len("# Some leading comment"), "comment"),
+        resolved_semantic_token(2, 0, len("Source"), "keyword"),
+        resolved_semantic_token(4, 0, len("# Comment between stanzas"), "comment"),
+        resolved_semantic_token(6, 0, len("Package"), "keyword"),
+        resolved_semantic_token(7, 0, len("# Comment before Architecture"), "comment"),
+        resolved_semantic_token(8, 0, len("Architecture"), "keyword"),
+        resolved_semantic_token(8, len("Architecture: "), len("any"), "enumMember"),
+        resolved_semantic_token(9, 0, len("Depends"), "keyword"),
+        resolved_semantic_token(10, 0, len("# Comment about bar"), "comment"),
+        resolved_semantic_token(13, 0, len("Description"), "keyword"),
+        resolved_semantic_token(14, 0, len("Unknown-Field"), "keyword"),
+        # TODO: resolved_semantic_token(15, 0, len("# Comment in that field"), "comment"),
+    ]
diff --git a/tests/lsp_tests/test_lsp_debputy_manifest_hover.py b/tests/lsp_tests/test_lsp_debputy_manifest_hover.py
index c66db80..54c6b6a 100644
--- a/tests/lsp_tests/test_lsp_debputy_manifest_hover.py
+++ b/tests/lsp_tests/test_lsp_debputy_manifest_hover.py
@@ -2,6 +2,7 @@ import textwrap
 
 import pytest
 
+from debputy.lsp.debputy_ls import DebputyLanguageServer
 from lsp_tests.lsp_tutil import put_doc_with_cursor
 
 try:
@@ -25,7 +26,7 @@ except ImportError:
     HAS_PYGLS = False
 
 
-def test_basic_debputy_hover_tlk(ls: "LanguageServer") -> None:
+def test_basic_debputy_hover_tlk(ls: "DebputyLanguageServer") -> None:
     debputy_manifest_uri = "file:///nowhere/debian/debputy.manifest"
     cursor_pos = put_doc_with_cursor(
         ls,
@@ -52,7 +53,7 @@ def test_basic_debputy_hover_tlk(ls: "LanguageServer") -> None:
     assert hover_doc.contents.value.startswith("Installations")
 
 
-def test_basic_debputy_hover_install_docs_key(ls: "LanguageServer") -> None:
+def test_basic_debputy_hover_install_docs_key(ls: "DebputyLanguageServer") -> None:
     debputy_manifest_uri = "file:///nowhere/debian/debputy.manifest"
     cursor_pos = put_doc_with_cursor(
         ls,
@@ -79,7 +80,7 @@ def test_basic_debputy_hover_install_docs_key(ls: "LanguageServer") -> None:
     assert hover_doc.contents.value.startswith("Install documentation (`install-docs`)")
 
 
-def test_basic_debputy_hover_install_docs_sources(ls: "LanguageServer") -> None:
+def test_basic_debputy_hover_install_docs_sources(ls: "DebputyLanguageServer") -> None:
     debputy_manifest_uri = "file:///nowhere/debian/debputy.manifest"
     cursor_pos = put_doc_with_cursor(
         ls,
@@ -106,7 +107,7 @@ def test_basic_debputy_hover_install_docs_sources(ls: "LanguageServer") -> None:
     assert hover_doc.contents.value.startswith("# Attribute `sources`")
 
 
-def test_basic_debputy_hover_install_docs_when(ls: "LanguageServer") -> None:
+def test_basic_debputy_hover_install_docs_when(ls: "DebputyLanguageServer") -> None:
     debputy_manifest_uri = "file:///nowhere/debian/debputy.manifest"
     cursor_pos = put_doc_with_cursor(
         ls,
@@ -134,7 +135,7 @@ def test_basic_debputy_hover_install_docs_when(ls: "LanguageServer") -> None:
     assert hover_doc.contents.value.startswith("# Attribute `when`")
 
 
-def test_basic_debputy_hover_install_docs_str_cond(ls: "LanguageServer") -> None:
+def test_basic_debputy_hover_install_docs_str_cond(ls: "DebputyLanguageServer") -> None:
     debputy_manifest_uri = "file:///nowhere/debian/debputy.manifest"
     cursor_pos = put_doc_with_cursor(
         ls,
@@ -165,7 +166,7 @@ def test_basic_debputy_hover_install_docs_str_cond(ls: "LanguageServer") -> None
 
 
 def test_basic_debputy_hover_install_docs_mapping_cond_key(
-    ls: "LanguageServer",
+    ls: "DebputyLanguageServer",
 ) -> None:
     debputy_manifest_uri = "file:///nowhere/debian/debputy.manifest"
     cursor_pos = put_doc_with_cursor(
@@ -197,7 +198,7 @@ def test_basic_debputy_hover_install_docs_mapping_cond_key(
 
 @pytest.mark.xfail
 def test_basic_debputy_hover_install_docs_mapping_cond_str_value(
-    ls: "LanguageServer",
+    ls: "DebputyLanguageServer",
 ) -> None:
     debputy_manifest_uri = "file:///nowhere/debian/debputy.manifest"
     cursor_pos = put_doc_with_cursor(
@@ -230,7 +231,7 @@ def test_basic_debputy_hover_install_docs_mapping_cond_str_value(
     )
 
 
-def test_basic_debputy_hover_binary_version(ls: "LanguageServer") -> None:
+def test_basic_debputy_hover_binary_version(ls: "DebputyLanguageServer") -> None:
     debputy_manifest_uri = "file:///nowhere/debian/debputy.manifest"
     cursor_pos = put_doc_with_cursor(
         ls,
@@ -256,7 +257,7 @@ def test_basic_debputy_hover_binary_version(ls: "LanguageServer") -> None:
     )
 
 
-def test_basic_debputy_hover_services(ls: "LanguageServer") -> None:
+def test_basic_debputy_hover_services(ls: "DebputyLanguageServer") -> None:
     debputy_manifest_uri = "file:///nowhere/debian/debputy.manifest"
     cursor_pos = put_doc_with_cursor(
         ls,
@@ -283,7 +284,7 @@ def test_basic_debputy_hover_services(ls: "LanguageServer") -> None:
     )
 
 
-def test_basic_debputy_hover_services_service(ls: "LanguageServer") -> None:
+def test_basic_debputy_hover_services_service(ls: "DebputyLanguageServer") -> None:
     debputy_manifest_uri = "file:///nowhere/debian/debputy.manifest"
     cursor_pos = put_doc_with_cursor(
         ls,