Adding upstream version 2.25.15.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

scripts/wrap-and-sort

@@ -0,0 +1,542 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2010-2018, Benjamin Drung <bdrung@debian.org>
+#               2010, Stefano Rivera <stefanor@ubuntu.com>
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+# pylint: disable=invalid-name
+# pylint: enable=invalid-name
+
+import argparse
+import glob
+import operator
+import os
+import re
+import sys
+
+from devscripts.control import (
+    HAS_FULL_RTS_FORMATTING,
+    HAS_RTS_PARSER,
+    Control,
+    wrap_and_sort_formatter,
+)
+
+try:
+    from debian._deb822_repro import LIST_COMMA_SEPARATED_INTERPRETATION
+except ImportError:
+    LIST_COMMA_SEPARATED_INTERPRETATION = object()
+
+
+CONTROL_LIST_FIELDS = (
+    "Breaks",
+    "Build-Conflicts",
+    "Build-Conflicts-Arch",
+    "Build-Conflicts-Indep",
+    "Build-Depends",
+    "Build-Depends-Arch",
+    "Build-Depends-Indep",
+    "Built-Using",
+    "Conflicts",
+    "Depends",
+    "Enhances",
+    "Features",
+    "Pre-Depends",
+    "Provides",
+    "Recommends",
+    "Replaces",
+    "Restrictions",
+    "Suggests",
+    "Tests",
+    "Xb-Npp-MimeType",
+)
+
+SUPPORTED_FILES = (
+    "clean",
+    "control",
+    "control*.in",
+    "copyright",
+    "copyright.in",
+    "dirs",
+    "*.dirs",
+    "docs",
+    "*.docs",
+    "examples",
+    "*.examples",
+    "info",
+    "*.info",
+    "install",
+    "*.install",
+    "links",
+    "*.links",
+    "mainscript",
+    "*.maintscript",
+    "manpages",
+    "*.manpages",
+    "tests/control",
+)
+
+BUILD_SYSTEMS = (
+    "cdbs",
+    "debhelper-compat",
+    "debhelper",
+    "debputy",
+    "dh-.*",
+    "dpkg-build-api",
+    "dpkg-dev",
+)
+
+BUILD_SYSTEMS_RE = re.compile("(" + "|".join(BUILD_SYSTEMS) + ")")
+
+
+def erase_and_write(file_ob, data):
+    """When a file is opened via r+ mode, replaces its content with data"""
+    file_ob.seek(0)
+    file_ob.write(data)
+    file_ob.truncate()
+
+
+class WrapAndSortControl(Control):
+    def __init__(self, filename, args):
+        # The Control module supports the RTS parser with python-debian 0.1.43.
+        # However, the `wrap_and_sort_formatter` requires 0.1.44.  The command line
+        # option check handles the compatibility check for now.
+        super().__init__(filename, use_rts_parser=args.rts_parser)
+        self.args = args
+        self._formatter = None
+        if args.rts_parser and HAS_FULL_RTS_FORMATTING:
+            max_line_length = args.max_line_length
+            self._formatter = wrap_and_sort_formatter(
+                1 if args.short_indent else "FIELD_NAME_LENGTH",
+                trailing_separator=args.trailing_comma,
+                immediate_empty_line=args.short_indent,
+                max_line_length_one_liner=0 if args.wrap_always else max_line_length,
+            )
+
+    def wrap_and_sort(self):
+        for paragraph in self.paragraphs:
+            for field in CONTROL_LIST_FIELDS:
+                if field in paragraph:
+                    self._wrap_field(paragraph, field, True)
+            if "Uploaders" in paragraph:
+                self._wrap_field(paragraph, "Uploaders", False)
+            if "Architecture" in paragraph:
+                archs = set(paragraph["Architecture"].split())
+                # Sort, with wildcard entries (such as linux-any) first:
+                archs = sorted(archs, key=lambda x: ("any" not in x, x))
+                paragraph["Architecture"] = " ".join(archs)
+
+        if not self.args.sort_binary_packages or self.filename.endswith(
+            "tests/control"
+        ):
+            return
+
+        if self.had_parse_errors:
+            if not self.args.dry_run:
+                print(
+                    f"Skipping sorting of binary packages in {self.filename}:"
+                    " It had parse errors or used template language and the"
+                    " sorting could risk changing the semantics the file."
+                )
+            return
+
+        first = self.paragraphs[: 1 + int(self.args.keep_first)]
+        sortable = self.paragraphs[1 + int(self.args.keep_first) :]
+        sort_key = operator.itemgetter("Package")
+        self.paragraphs = first + sorted(sortable, key=sort_key)
+
+    def _wrap_field(self, control, entry, sort):
+        if self.is_roundtrip_safe:
+            self._wrap_field_rts(control, entry, sort)
+        else:
+            self._wrap_field_deb822(control, entry, sort)
+
+    def _wrap_field_rts(self, control, entry, sort):
+        view = control.as_interpreted_dict_view(LIST_COMMA_SEPARATED_INTERPRETATION)
+        with view[entry] as field_content:
+            seen = set()
+            for package_ref in field_content.iter_value_references():
+                value = package_ref.value
+                new_value = " | ".join(x.strip() for x in value.split("|"))
+                if not sort or new_value not in seen:
+                    package_ref.value = new_value
+                    seen.add(new_value)
+                else:
+                    package_ref.remove()
+            if sort:
+                field_content.sort(key=_sort_packages_key)
+            if self._formatter:
+                field_content.value_formatter(self._formatter)
+            field_content.reformat_when_finished()
+
+    def _wrap_field_deb822(self, control, entry, sort):
+        # An empty element is not explicitly disallowed by Policy but known to
+        # break QA tools, so remove any
+        packages = [x.strip() for x in control[entry].split(",") if x.strip()]
+
+        # Sanitize alternative packages. E.g. "a|b  |c" -> "a | b | c"
+        packages = [" | ".join(x.strip() for x in p.split("|")) for p in packages]
+
+        if sort:
+            # Remove duplicate entries
+            packages = set(packages)
+            packages = sort_list(packages)
+
+        length = len(entry) + sum(2 + len(package) for package in packages)
+        if self.args.wrap_always or length > self.args.max_line_length:
+            indentation = " "
+            if not self.args.short_indent:
+                indentation *= len(entry) + len(": ")
+            packages_with_indention = [indentation + x for x in packages]
+            packages_with_indention = ",\n".join(packages_with_indention)
+            if self.args.trailing_comma:
+                packages_with_indention += ","
+            if self.args.short_indent:
+                control[entry] = "\n" + packages_with_indention
+            else:
+                control[entry] = packages_with_indention.strip()
+        else:
+            new_value = ", ".join(packages)
+            if self.args.trailing_comma:
+                new_value += ","
+            control[entry] = new_value
+
+    def check_changed(self):
+        """Checks if the content has changed in the control file"""
+        content = self.dump()
+        with open(self.filename, "r", encoding="utf8") as control_file:
+            if content != control_file.read():
+                return True
+        return False
+
+
+class InstallContent:
+    __slots__ = ("content", "comments")
+
+    def __init__(self, content, comments=None):
+        self.content = content
+        self.comments = comments
+
+    def __str__(self):
+        comments = "\n".join(self.comments) + "\n" if self.comments else ""
+        return comments + self.content
+
+    def __eq__(self, other):
+        return self.content == other.content
+
+    def __lt__(self, other):
+        return self.content < other.content
+
+
+class Install:
+    def __init__(self, filename, args):
+        self.content = None
+        self.filename = None
+        self.args = args
+        self.leading_comments = None
+        self.trailing_comments = None
+        self.open(filename)
+
+    def open(self, filename):
+        assert os.path.isfile(filename), f"{filename} does not exist."
+        self.filename = filename
+        comments = []
+        content = []
+        with open(filename, encoding="utf8") as f:
+            # When reading a debhelper file, we want to preserve blocks of comments.
+            #
+            # For the purpose of wrap-and-sort, we generally associate comments with
+            # a line of (non-whitespace) content. Though as special cases, we also
+            # preserve a file starting with a comment (with an empty line before the
+            # first content to distinguish it from a comment to that comment line) as
+            # well as trailing comments (i.e. comments before EOF).
+            for line in f:
+                line = line.strip()
+                if not line:
+                    if comments and not content:
+                        self.leading_comments = InstallContent("", comments=comments)
+                        comments = []
+                    continue
+                if line[0] == "#":
+                    comments.append(line)
+                    continue
+                if comments:
+                    content.append(InstallContent(line, comments=comments))
+                    comments = []
+                else:
+                    content.append(InstallContent(line, comments=None))
+        self.trailing_comments = comments
+        self.content = content
+
+    def save(self):
+        to_write = self._serialize_content()
+
+        with open(self.filename, "r+", encoding="utf8") as install_file:
+            content = install_file.read()
+            if to_write != content:
+                if not self.args.dry_run:
+                    erase_and_write(install_file, to_write)
+                return True
+        return False
+
+    def _serialize_content(self):
+        elements = []
+        if self.leading_comments:
+            elements.append(str(self.leading_comments))
+        elements.extend(str(x) for x in self.content)
+        if self.trailing_comments:
+            # Add a space between the last content and the trailing comments
+            # for readability
+            elements.append("\n" + "\n".join(self.trailing_comments))
+        return "\n".join(elements) + "\n"
+
+    def sort(self):
+        self.content = sorted(self.content)
+
+
+def remove_trailing_whitespaces(filename, args):
+    assert os.path.isfile(filename), f"{filename} does not exist."
+    with open(filename, "br+") as file_object:
+        content = file_object.read()
+        if not content:
+            return True
+        new_content = content.strip() + b"\n"
+        new_content = b"\n".join([line.rstrip() for line in new_content.split(b"\n")])
+        if new_content != content:
+            if not args.dry_run:
+                erase_and_write(file_object, new_content)
+            return True
+    return False
+
+
+def sort_list(unsorted_list):
+    return sorted(unsorted_list, key=_sort_packages_key)
+
+
+def _sort_packages_key(package):
+    if re.match(BUILD_SYSTEMS_RE, package):
+        # Sort build-system related packages first.
+        group_order = -1
+    elif not re.match("[a-z0-9]", package):
+        # Sort dependencies without a "real" package name (e.g. substvars)
+        # last.
+        group_order = 1
+    else:
+        group_order = 0
+
+    return (group_order, package)
+
+
+def wrap_and_sort(args):
+    modified_files = []
+    control_files = [f for f in args.files if re.search("/control[^/]*$", f)]
+    for control_file in control_files:
+        if args.verbose:
+            print(control_file)
+        try:
+            control = WrapAndSortControl(control_file, args)
+        except ValueError as e:
+            print(
+                f"W: Could not parse {control_file} as a Deb822 file: {str(e.args[0])}",
+                file=sys.stderr,
+            )
+            continue
+        if args.cleanup:
+            control.strip_trailing_whitespace_on_save = True
+        control.wrap_and_sort()
+        if control.check_changed():
+            if not args.dry_run:
+                control.save()
+            modified_files.append(control_file)
+
+    copyright_files = [f for f in args.files if re.search("/copyright[^/]*$", f)]
+    for copyright_file in copyright_files:
+        if args.verbose:
+            print(copyright_file)
+        if remove_trailing_whitespaces(copyright_file, args):
+            modified_files.append(copyright_file)
+
+    pattern = "(dirs|docs|examples|info|install|links|maintscript|manpages)$"
+    install_files = [f for f in args.files if re.search(pattern, f)]
+    for install_file in sorted(install_files):
+        if args.verbose:
+            print(install_file)
+        install = Install(install_file, args)
+        install.sort()
+        if install.save():
+            modified_files.append(install_file)
+
+    return modified_files
+
+
+def get_files(debian_directory):
+    """Returns a list of files that should be wrapped and sorted."""
+    files = []
+    for supported_files in SUPPORTED_FILES:
+        file_pattern = os.path.join(debian_directory, supported_files)
+        files.extend(
+            file_name
+            for file_name in glob.glob(file_pattern)
+            if not os.access(file_name, os.X_OK)
+        )
+    return files
+
+
+def main():
+    script_name = os.path.basename(sys.argv[0])
+    epilog = f"See {script_name}(1) for more info."
+    parser = argparse.ArgumentParser(
+        epilog=epilog,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    # Remember to keep doc/wrap-and-sort.1 updated!
+    parser.add_argument(
+        "-a",
+        "--wrap-always",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+        help="wrap lists even if they do not exceed the line length limit",
+    )
+    parser.add_argument(
+        "-s",
+        "--short-indent",
+        dest="short_indent",
+        help="only indent wrapped lines by one space",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+    )
+    parser.add_argument(
+        "-b",
+        "--sort-binary-packages",
+        help="sort binary package paragraphs by name",
+        dest="sort_binary_packages",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+    )
+    parser.add_argument(
+        "-k",
+        "--keep-first",
+        help="when sorting binary packages, leave the first one at the top",
+        dest="keep_first",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+    )
+    parser.add_argument(
+        "-n",
+        "--cleanup",
+        help="remove trailing whitespaces",
+        dest="cleanup",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+    )
+    parser.add_argument(
+        "-t",
+        "--trailing-comma",
+        help="add trailing comma",
+        dest="trailing_comma",
+        action=argparse.BooleanOptionalAction,
+        default=False,
+    )
+    parser.add_argument(
+        "-d",
+        "--debian-directory",
+        dest="debian_directory",
+        help="location of the 'debian' directory",
+        metavar="PATH",
+        default="debian",
+    )
+    parser.add_argument(
+        "-f",
+        "--file",
+        metavar="FILE",
+        dest="files",
+        action="append",
+        default=[],
+        help="wrap and sort only the specified file",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="print all files that are touched",
+        dest="verbose",
+        action="store_true",
+        default=False,
+    )
+    parser.add_argument(
+        "--max-line-length",
+        type=int,
+        default=79,
+        help="set maximum allowed line length before wrapping",
+    )
+    parser.add_argument(
+        "-N",
+        "--dry-run",
+        dest="dry_run",
+        action="store_true",
+        default=False,
+        help="do not modify any file, instead only print the files"
+        " that would be modified",
+    )
+    parser.add_argument(
+        "--experimental-rts-parser",
+        dest="rts_parser",
+        action="store_true",
+        default=False,
+        help=argparse.SUPPRESS,
+    )
+
+    args = parser.parse_args()
+    use_rts_parser = HAS_RTS_PARSER
+    if use_rts_parser and not HAS_FULL_RTS_FORMATTING:
+        # Cases where we might have some but not full support.
+        if args.short_indent or not args.wrap_always or not args.trailing_comma:
+            use_rts_parser = False
+    setattr(args, "rts_parser", use_rts_parser)
+
+    if not os.path.isdir(args.debian_directory):
+        parser.error(
+            f'Debian directory not found, expecting "{args.debian_directory}".'
+        )
+
+    not_found = [f for f in args.files if not os.path.isfile(f)]
+    if not_found:
+        parser.error(f"Specified files not found: {', '.join(not_found)}")
+
+    if not args.files:
+        dctrl = os.path.join(args.debian_directory, "control")
+        control = Control(dctrl) if os.path.isfile(dctrl) else None
+        if control is not None and control.uses_different_style_tool:
+            print(
+                "The package declares it style via `X-Style`."
+                " Please use `debputy reformat` instead."
+            )
+            sys.exit(1)
+        args.files = get_files(args.debian_directory)
+
+    modified_files = wrap_and_sort(args)
+
+    # Only report at the end, to avoid potential clash with --verbose
+    if modified_files and (args.verbose or args.dry_run):
+        if args.dry_run:
+            header = "--- Dry run, these files would be modified ---"
+        else:
+            header = "--- Modified files ---"
+        print(header)
+        print("\n".join(modified_files))
+    elif args.verbose:
+        print("--- No file needs modification ---")
+
+
+if __name__ == "__main__":
+    main()