path: root/src/debputy/plugin/debputy/binary_package_rules.py

import os
import textwrap
from typing import (
    Any,
    List,
    NotRequired,
    Union,
    Literal,
    TypedDict,
    Annotated,
    Optional,
)

from debputy import DEBPUTY_DOC_ROOT_DIR
from debputy.maintscript_snippet import DpkgMaintscriptHelperCommand, MaintscriptSnippet
from debputy.manifest_parser.base_types import (
    DebputyParsedContent,
    FileSystemExactMatchRule,
)
from debputy.manifest_parser.declarative_parser import (
    DebputyParseHint,
    ParserGenerator,
)
from debputy.manifest_parser.exceptions import ManifestParseException
from debputy.manifest_parser.parser_data import ParserContextData
from debputy.manifest_parser.util import AttributePath
from debputy.path_matcher import MatchRule, MATCH_ANYTHING, ExactFileSystemPath
from debputy.plugin.api import reference_documentation
from debputy.plugin.api.impl import DebputyPluginInitializerProvider
from debputy.plugin.api.impl_types import OPARSER_PACKAGES
from debputy.transformation_rules import TransformationRule


ACCEPTABLE_CLEAN_ON_REMOVAL_FOR_GLOBS_AND_EXACT_MATCHES = frozenset(
    [
        "./var/log",
    ]
)


ACCEPTABLE_CLEAN_ON_REMOVAL_IF_EXACT_MATCH_OR_SUBDIR_OF = frozenset(
    [
        "./etc",
        "./run",
        "./var/lib",
        "./var/cache",
        "./var/backups",
        "./var/spool",
        # linux-image uses these paths with some `rm -f`
        "./usr/lib/modules",
        "./lib/modules",
        # udev special case
        "./lib/udev",
        "./usr/lib/udev",
        # pciutils deletes /usr/share/misc/pci.ids.<ext>
        "./usr/share/misc",
    ]
)


def register_binary_package_rules(api: DebputyPluginInitializerProvider) -> None:
    api.pluggable_manifest_rule(
        OPARSER_PACKAGES,
        "binary-version",
        BinaryVersionParsedFormat,
        _parse_binary_version,
        source_format=str,
        inline_reference_documentation=reference_documentation(
            title="Custom binary version (`binary-version`)",
            description=textwrap.dedent(
                """\
                In the *rare* case that you need a binary package to have a custom version, you can use
                the `binary-version:` key to describe the desired package version.  An example being:

                    packages:
                        foo:
                            # The foo package needs a different epoch because we took it over from a different
                            # source package with higher epoch version
                            binary-version: '1:{{DEB_VERSION_UPSTREAM_REVISION}}'

                Use this feature sparingly as it is generally not possible to undo as each version must be
                monotonously higher than the previous one. This feature translates into `-v` option for
                `dpkg-gencontrol`.

                The value for the `binary-version` key is a string that defines the binary version.  Generally,
                you will want it to contain one of the versioned related substitution variables such as 
                `{{DEB_VERSION_UPSTREAM_REVISION}}`.  Otherwise, you will have to remember to bump the version
                manually with each upload as versions cannot be reused and the package would not support binNMUs
                either.
            """
            ),
            reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#custom-binary-version-binary-version",
        ),
    )

    api.pluggable_manifest_rule(
        OPARSER_PACKAGES,
        "transformations",
        ListOfTransformationRulesFormat,
        _unpack_list,
        source_format=List[TransformationRule],
        inline_reference_documentation=reference_documentation(
            title="Transformations (`packages.{{PACKAGE}}.transformations`)",
            description=textwrap.dedent(
                """\
                You can define a `transformations` under the package definition, which is a list a transformation
                rules.  An example:

                    packages:
                        foo:
                            transformations:
                              - remove: 'usr/share/doc/{{PACKAGE}}/INSTALL.md'
                              - move:
                                    source: bar/*
                                    target: foo/


                Transformations are ordered and are applied in the listed order.  A path can be matched by multiple
                transformations; how that plays out depends on which transformations are applied and in which order.
                A quick summary:

                 - Transformations that modify the file system layout affect how path matches in later transformations.
                   As an example, `move` and `remove` transformations affects what globs and path matches expand to in
                   later transformation rules.

                 - For other transformations generally the latter transformation overrules the earlier one, when they
                   overlap or conflict.
            """
            ),
            reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#transformations-packagespackagetransformations",
        ),
    )

    api.pluggable_manifest_rule(
        OPARSER_PACKAGES,
        "conffile-management",
        ListOfDpkgMaintscriptHelperCommandFormat,
        _unpack_list,
        source_format=List[DpkgMaintscriptHelperCommand],
    )

    api.pluggable_manifest_rule(
        OPARSER_PACKAGES,
        "clean-after-removal",
        ListParsedFormat,
        _parse_clean_after_removal,
        source_format=List[Any],
        # FIXME: debputy won't see the attributes for this one :'(
        inline_reference_documentation=reference_documentation(
            title="Remove runtime created paths on purge or post removal (`clean-after-removal`)",
            description=textwrap.dedent(
                """\
        For some packages, it is necessary to clean up some run-time created paths. Typical use cases are
        deleting log files, cache files, or persistent state. This can be done via the `clean-after-removal`.
        An example being:

            packages:
                foo:
                    clean-after-removal:
                    - /var/log/foo/*.log
                    - /var/log/foo/*.log.gz
                    - path: /var/log/foo/
                      ignore-non-empty-dir: true
                    - /etc/non-conffile-configuration.conf
                    - path: /var/cache/foo
                      recursive: true

        The `clean-after-removal` key accepts a list, where each element is either a mapping, a string or a list
        of strings. When an element is a mapping, then the following key/value pairs are applicable:

         * `path` or `paths` (required): A path match (`path`) or a list of path matches (`paths`) defining the
           path(s) that should be removed after clean. The path match(es) can use globs and manifest variables.
           Every path matched will by default be removed via `rm -f` or `rmdir` depending on whether the path
           provided ends with a *literal* `/`. Special-rules for matches:
            - Glob is interpreted by the shell, so shell (`/bin/sh`) rules apply to globs rather than
              `debputy`'s glob rules.  As an example, `foo/*` will **not** match `foo/.hidden-file`.
            - `debputy` cannot evaluate whether these paths/globs will match the desired paths (or anything at
              all). Be sure to test the resulting package.
            - When a symlink is matched, it is not followed.
            - Directory handling depends on the `recursive` attribute and whether the pattern ends with a literal
              "/".
            - `debputy` has restrictions on the globs being used to prevent rules that could cause massive damage
              to the system.

         * `recursive` (optional): When `true`, the removal rule will use `rm -fr` rather than `rm -f` or `rmdir`
            meaning any directory matched will be deleted along with all of its contents.

         * `ignore-non-empty-dir` (optional): When `true`, each path must be or match a directory (and as a
           consequence each path must with a literal `/`). The affected directories will be deleted only if they
           are empty. Non-empty directories will be skipped. This option is mutually exclusive with `recursive`.

         * `delete-on` (optional, defaults to `purge`): This attribute defines when the removal happens. It can
           be set to one of the following values:
           - `purge`: The removal happens with the package is being purged. This is the default. At a technical
             level, the removal occurs at `postrm purge`.
           - `removal`: The removal happens immediately after the package has been removed. At a technical level,
             the removal occurs at `postrm remove`.

        This feature resembles the concept of `rpm`'s `%ghost` files.
            """
            ),
            reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#remove-runtime-created-paths-on-purge-or-post-removal-clean-after-removal",
        ),
    )

    api.pluggable_manifest_rule(
        OPARSER_PACKAGES,
        "installation-search-dirs",
        InstallationSearchDirsParsedFormat,
        _parse_installation_search_dirs,
        source_format=List[FileSystemExactMatchRule],
        inline_reference_documentation=reference_documentation(
            title="Custom installation time search directories (`installation-search-dirs`)",
            description=textwrap.dedent(
                """\
        For source packages that does multiple build, it can be an advantage to provide a custom list of
        installation-time search directories. This can be done via the `installation-search-dirs` key. A common
        example is building  the source twice with different optimization and feature settings where the second
        build is for the `debian-installer` (in the form of a `udeb` package). A sample manifest snippet could
        look something like:

            installations:
            - install:
                # Because of the search order (see below), `foo` installs `debian/tmp/usr/bin/tool`,
                # while `foo-udeb` installs `debian/tmp-udeb/usr/bin/tool` (assuming both paths are
                # available). Note the rule can be split into two with the same effect if that aids
                # readability or understanding.
                source: usr/bin/tool
                into:
                  - foo
                  - foo-udeb
            packages:
                foo-udeb:
                    installation-search-dirs:
                    - debian/tmp-udeb


        The `installation-search-dirs` key accepts a list, where each element is a path (str) relative from the
        source root to the directory that should be used as a search directory (absolute paths are still interpreted
        as relative to the source root).  This list should contain all search directories that should be applicable
        for this package (except the source root itself, which is always appended after the provided list). If the
        key is omitted, then `debputy` will provide a default  search order (In the `dh` integration, the default
        is the directory `debian/tmp`).

        If a non-existing or non-directory path is listed, then it will be skipped (info-level note). If the path
        exists and is a directory, it will also be checked for "not-installed" paths.
            """
            ),
            reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#custom-installation-time-search-directories-installation-search-dirs",
        ),
    )


class BinaryVersionParsedFormat(DebputyParsedContent):
    binary_version: str


class ListParsedFormat(DebputyParsedContent):
    elements: List[Any]


class ListOfTransformationRulesFormat(DebputyParsedContent):
    elements: List[TransformationRule]


class ListOfDpkgMaintscriptHelperCommandFormat(DebputyParsedContent):
    elements: List[DpkgMaintscriptHelperCommand]


class InstallationSearchDirsParsedFormat(DebputyParsedContent):
    installation_search_dirs: List[FileSystemExactMatchRule]


def _parse_binary_version(
    _name: str,
    parsed_data: BinaryVersionParsedFormat,
    _attribute_path: AttributePath,
    _parser_context: ParserContextData,
) -> str:
    return parsed_data["binary_version"]


def _parse_installation_search_dirs(
    _name: str,
    parsed_data: InstallationSearchDirsParsedFormat,
    _attribute_path: AttributePath,
    _parser_context: ParserContextData,
) -> List[FileSystemExactMatchRule]:
    return parsed_data["installation_search_dirs"]


def _unpack_list(
    _name: str,
    parsed_data: ListParsedFormat,
    _attribute_path: AttributePath,
    _parser_context: ParserContextData,
) -> List[Any]:
    return parsed_data["elements"]


class CleanAfterRemovalRuleSourceFormat(TypedDict):
    path: NotRequired[Annotated[str, DebputyParseHint.target_attribute("paths")]]
    paths: NotRequired[List[str]]
    delete_on: NotRequired[Literal["purge", "removal"]]
    recursive: NotRequired[bool]
    ignore_non_empty_dir: NotRequired[bool]


class CleanAfterRemovalRule(DebputyParsedContent):
    paths: List[str]
    delete_on: NotRequired[Literal["purge", "removal"]]
    recursive: NotRequired[bool]
    ignore_non_empty_dir: NotRequired[bool]


# FIXME: Not optimal that we are doing an initialization of ParserGenerator here. But the rule is not depending on any
#  complex types that is regiersted by plugins, so it will work for now.
_CLEAN_AFTER_REMOVAL_RULE_PARSER = ParserGenerator().parser_from_typed_dict(
    CleanAfterRemovalRule,
    source_content=Union[CleanAfterRemovalRuleSourceFormat, str, List[str]],
    inline_reference_documentation=reference_documentation(
        reference_documentation_url=f"{DEBPUTY_DOC_ROOT_DIR}/MANIFEST-FORMAT.md#remove-runtime-created-paths-on-purge-or-post-removal-clean-after-removal",
    ),
)


# Order between clean_on_removal and conffile_management is
# important. We want the dpkg conffile management rules to happen before the
# clean clean_on_removal rules.  Since the latter only affects `postrm`
# and the order is reversed for `postrm` scripts (among other), we need do
# clean_on_removal first to account for the reversing of order.
#
# FIXME: All of this is currently not really possible todo, but it should be.
# (I think it is the correct order by "mistake" rather than by "design", which is
# what this note is about)
def _parse_clean_after_removal(
    _name: str,
    parsed_data: ListParsedFormat,
    attribute_path: AttributePath,
    parser_context: ParserContextData,
) -> None:  # TODO: Return and pass to a maintscript helper
    raw_clean_after_removal = parsed_data["elements"]
    package_state = parser_context.current_binary_package_state

    for no, raw_transformation in enumerate(raw_clean_after_removal):
        definition_source = attribute_path[no]
        clean_after_removal_rules = _CLEAN_AFTER_REMOVAL_RULE_PARSER.parse_input(
            raw_transformation,
            definition_source,
            parser_context=parser_context,
        )
        patterns = clean_after_removal_rules["paths"]
        if patterns:
            definition_source.path_hint = patterns[0]
        delete_on = clean_after_removal_rules.get("delete_on") or "purge"
        recurse = clean_after_removal_rules.get("recursive") or False
        ignore_non_empty_dir = (
            clean_after_removal_rules.get("ignore_non_empty_dir") or False
        )
        if delete_on == "purge":
            condition = '[ "$1" = "purge" ]'
        else:
            condition = '[ "$1" = "remove" ]'

        if ignore_non_empty_dir:
            if recurse:
                raise ManifestParseException(
                    'The "recursive" and "ignore-non-empty-dir" options are mutually exclusive.'
                    f" Both were enabled at the same time in at {definition_source.path}"
                )
            for pattern in patterns:
                if not pattern.endswith("/"):
                    raise ManifestParseException(
                        'When ignore-non-empty-dir is True, then all patterns must end with a literal "/"'
                        f' to ensure they only apply to directories. The pattern "{pattern}" at'
                        f" {definition_source.path} did not."
                    )

        substitution = parser_context.substitution
        match_rules = [
            MatchRule.from_path_or_glob(
                p, definition_source.path, substitution=substitution
            )
            for p in patterns
        ]
        content_lines = [
            f"if {condition}; then\n",
        ]
        for idx, match_rule in enumerate(match_rules):
            original_pattern = patterns[idx]
            if match_rule is MATCH_ANYTHING:
                raise ManifestParseException(
                    f'Using "{original_pattern}" in a clean rule would trash the system.'
                    f" Please restrict this pattern at {definition_source.path} considerably."
                )
            is_subdir_match = False
            matched_directory: Optional[str]
            if isinstance(match_rule, ExactFileSystemPath):
                matched_directory = (
                    os.path.dirname(match_rule.path)
                    if match_rule.path not in ("/", ".", "./")
                    else match_rule.path
                )
                is_subdir_match = True
            else:
                matched_directory = getattr(match_rule, "directory", None)

            if matched_directory is None:
                raise ManifestParseException(
                    f'The pattern "{original_pattern}" defined at {definition_source.path} is not'
                    f" trivially anchored in a specific directory. Cowardly refusing to use it"
                    f" in a clean rule as it may trash the system if the pattern is overreaching."
                    f" Please avoid glob characters in the top level directories."
                )
            assert matched_directory.startswith("./") or matched_directory in (
                ".",
                "./",
                "",
            )
            acceptable_directory = False
            would_have_allowed_direct_match = False
            while matched_directory not in (".", "./", ""):
                # Our acceptable paths set includes "/var/lib" or "/etc".  We require that the
                # pattern is either an exact match, in which case it may match directly inside
                # the acceptable directory OR it is a pattern against a subdirectory of the
                # acceptable path. As an example:
                #
                # /etc/inputrc <-- OK, exact match
                # /etc/foo/*   <-- OK, subdir match
                # /etc/*       <-- ERROR, glob directly in the accepted directory.
                if is_subdir_match and (
                    matched_directory
                    in ACCEPTABLE_CLEAN_ON_REMOVAL_IF_EXACT_MATCH_OR_SUBDIR_OF
                ):
                    acceptable_directory = True
                    break
                if (
                    matched_directory
                    in ACCEPTABLE_CLEAN_ON_REMOVAL_FOR_GLOBS_AND_EXACT_MATCHES
                ):
                    # Special-case: In some directories (such as /var/log), we allow globs directly.
                    # Notably, X11's log files are /var/log/Xorg.*.log
                    acceptable_directory = True
                    break
                if (
                    matched_directory
                    in ACCEPTABLE_CLEAN_ON_REMOVAL_IF_EXACT_MATCH_OR_SUBDIR_OF
                ):
                    would_have_allowed_direct_match = True
                    break
                matched_directory = os.path.dirname(matched_directory)
                is_subdir_match = True

            if would_have_allowed_direct_match and not acceptable_directory:
                raise ManifestParseException(
                    f'The pattern "{original_pattern}" defined at {definition_source.path} seems to'
                    " be overreaching. If it has been a path (and not use a glob), the rule would"
                    " have been permitted."
                )
            elif not acceptable_directory:
                raise ManifestParseException(
                    f'The pattern or path "{original_pattern}" defined at {definition_source.path} seems to'
                    f' be overreaching or not limited to the set of "known acceptable" directories.'
                )

            try:
                shell_escaped_pattern = match_rule.shell_escape_pattern()
            except TypeError:
                raise ManifestParseException(
                    f'Sorry, the pattern "{original_pattern}" defined at {definition_source.path}'
                    f" is unfortunately not supported by `debputy` for clean-after-removal rules."
                    f" If you can rewrite the rule to something like `/var/log/foo/*.log` or"
                    f' similar "trivial" patterns. You may have to rewrite the pattern the rule '
                    f" into multiple patterns to achieve this.  This restriction is to enable "
                    f' `debputy` to ensure the pattern is correctly executed plus catch "obvious'
                    f' system trashing" patterns. Apologies for the inconvenience.'
                )

            if ignore_non_empty_dir:
                cmd = f'    rmdir --ignore-fail-on-non-empty "${{DPKG_ROOT}}"{shell_escaped_pattern}\n'
            elif recurse:
                cmd = f'    rm -fr "${{DPKG_ROOT}}"{shell_escaped_pattern}\n'
            elif original_pattern.endswith("/"):
                cmd = f'    rmdir "${{DPKG_ROOT}}"{shell_escaped_pattern}\n'
            else:
                cmd = f'    rm -f "${{DPKG_ROOT}}"{shell_escaped_pattern}\n'
            content_lines.append(cmd)
        content_lines.append("fi\n")

        snippet = MaintscriptSnippet(definition_source.path, "".join(content_lines))
        package_state.maintscript_snippets["postrm"].append(snippet)