From 129a1fb4dbc375be0fa926964aa1be46a0cdbbef Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 14 Apr 2024 21:54:34 +0200 Subject: Adding upstream version 0.1.21. Signed-off-by: Daniel Baumann --- tests/conftest.py | 167 +++ tests/data/custom-plugin.json.in | 16 + tests/data/custom_plugin.py | 25 + tests/plugin_tests/conftest.py | 20 + tests/plugin_tests/gnome_test.py | 45 + tests/plugin_tests/numpy3_test.data | 4 + tests/plugin_tests/numpy3_test.py | 38 + tests/plugin_tests/perl-openssl_test.py | 33 + tests/plugin_tests/perl-ssl_test.sh | 5 + tests/test_apply_compression.py | 33 + tests/test_architecture.py | 59 ++ tests/test_cross_check_precheck.py | 124 +++ tests/test_deb_packaging_support.py | 218 ++++ tests/test_debputy_plugin.py | 1246 ++++++++++++++++++++++ tests/test_declarative_parser.py | 211 ++++ tests/test_fs_metadata.py | 770 ++++++++++++++ tests/test_install_rules.py | 1059 ++++++++++++++++++ tests/test_interpreter.py | 162 +++ tests/test_migrations.py | 1767 +++++++++++++++++++++++++++++++ tests/test_output_filename.py | 27 + tests/test_packager_provided_files.py | 210 ++++ tests/test_packer_pack.py | 86 ++ tests/test_parser.py | 473 +++++++++ tests/test_path.py | 47 + tests/test_plugin_tester.py | 285 +++++ tests/test_substitute.py | 66 ++ tests/test_symbolic_mode.py | 25 + tests/test_symlink_normalization.py | 35 + tests/tutil.py | 66 ++ 29 files changed, 7322 insertions(+) create mode 100644 tests/conftest.py create mode 100644 tests/data/custom-plugin.json.in create mode 100644 tests/data/custom_plugin.py create mode 100644 tests/plugin_tests/conftest.py create mode 100644 tests/plugin_tests/gnome_test.py create mode 100644 tests/plugin_tests/numpy3_test.data create mode 100644 tests/plugin_tests/numpy3_test.py create mode 100644 tests/plugin_tests/perl-openssl_test.py create mode 100755 tests/plugin_tests/perl-ssl_test.sh create mode 100644 tests/test_apply_compression.py create mode 100644 tests/test_architecture.py create mode 100644 tests/test_cross_check_precheck.py create mode 100644 tests/test_deb_packaging_support.py create mode 100644 tests/test_debputy_plugin.py create mode 100644 tests/test_declarative_parser.py create mode 100644 tests/test_fs_metadata.py create mode 100644 tests/test_install_rules.py create mode 100644 tests/test_interpreter.py create mode 100644 tests/test_migrations.py create mode 100644 tests/test_output_filename.py create mode 100644 tests/test_packager_provided_files.py create mode 100644 tests/test_packer_pack.py create mode 100644 tests/test_parser.py create mode 100644 tests/test_path.py create mode 100644 tests/test_plugin_tester.py create mode 100644 tests/test_substitute.py create mode 100644 tests/test_symbolic_mode.py create mode 100644 tests/test_symlink_normalization.py create mode 100644 tests/tutil.py (limited to 'tests') diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..1f2ed3d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,167 @@ +import os +from typing import Mapping + +import pytest +from debian.deb822 import Deb822 +from debian.debian_support import DpkgArchTable + +from debputy._deb_options_profiles import DebBuildOptionsAndProfiles +from debputy.architecture_support import ( + DpkgArchitectureBuildProcessValuesTable, + faked_arch_table, +) +from debputy.filesystem_scan import FSROOverlay +from debputy.manifest_parser.util import AttributePath +from debputy.packages import BinaryPackage, SourcePackage +from debputy.plugin.debputy.debputy_plugin import initialize_debputy_features +from debputy.plugin.api.impl_types import ( + DebputyPluginMetadata, +) +from debputy.plugin.api.feature_set import PluginProvidedFeatureSet +from debputy.plugin.api.impl import DebputyPluginInitializerProvider +from debputy.substitution import ( + NULL_SUBSTITUTION, + Substitution, + SubstitutionImpl, + VariableContext, +) + +# Disable dpkg's translation layer. It is very slow and disabling it makes it easier to debug +# test-failure reports from systems with translations active. +os.environ["DPKG_NLS"] = "0" + + +@pytest.fixture(scope="session") +def amd64_dpkg_architecture_variables() -> DpkgArchitectureBuildProcessValuesTable: + return faked_arch_table("amd64") + + +@pytest.fixture(scope="session") +def dpkg_arch_query() -> DpkgArchTable: + return DpkgArchTable.load_arch_table() + + +@pytest.fixture() +def source_package() -> SourcePackage: + return SourcePackage( + { + "Source": "foo", + } + ) + + +@pytest.fixture() +def package_single_foo_arch_all_cxt_amd64( + amd64_dpkg_architecture_variables, + dpkg_arch_query, +) -> Mapping[str, BinaryPackage]: + return { + p.name: p + for p in [ + BinaryPackage( + Deb822( + { + "Package": "foo", + "Architecture": "all", + } + ), + amd64_dpkg_architecture_variables, + dpkg_arch_query, + is_main_package=True, + ) + ] + } + + +@pytest.fixture() +def package_foo_w_udeb_arch_any_cxt_amd64( + amd64_dpkg_architecture_variables, + dpkg_arch_query, +) -> Mapping[str, BinaryPackage]: + return { + p.name: p + for p in [ + BinaryPackage( + Deb822( + { + "Package": "foo", + "Architecture": "any", + } + ), + amd64_dpkg_architecture_variables, + dpkg_arch_query, + is_main_package=True, + ), + BinaryPackage( + Deb822( + { + "Package": "foo-udeb", + "Architecture": "any", + "Package-Type": "udeb", + } + ), + amd64_dpkg_architecture_variables, + dpkg_arch_query, + ), + ] + } + + +@pytest.fixture(scope="session") +def null_substitution() -> Substitution: + return NULL_SUBSTITUTION + + +@pytest.fixture(scope="session") +def _empty_debputy_plugin_feature_set() -> PluginProvidedFeatureSet: + return PluginProvidedFeatureSet() + + +@pytest.fixture(scope="session") +def amd64_substitution( + amd64_dpkg_architecture_variables, + _empty_debputy_plugin_feature_set, +) -> Substitution: + debian_dir = FSROOverlay.create_root_dir("debian", "debian") + variable_context = VariableContext( + debian_dir, + ) + return SubstitutionImpl( + plugin_feature_set=_empty_debputy_plugin_feature_set, + dpkg_arch_table=amd64_dpkg_architecture_variables, + static_variables=None, + environment={}, + unresolvable_substitutions=frozenset(["SOURCE_DATE_EPOCH", "PACKAGE"]), + variable_context=variable_context, + ) + + +@pytest.fixture(scope="session") +def no_profiles_or_build_options() -> DebBuildOptionsAndProfiles: + return DebBuildOptionsAndProfiles(environ={}) + + +@pytest.fixture(scope="session") +def debputy_plugin_feature_set( + _empty_debputy_plugin_feature_set, amd64_substitution +) -> PluginProvidedFeatureSet: + plugin_metadata = DebputyPluginMetadata( + plugin_name="debputy", + api_compat_version=1, + plugin_initializer=initialize_debputy_features, + plugin_loader=None, + plugin_path="", + ) + feature_set = _empty_debputy_plugin_feature_set + api = DebputyPluginInitializerProvider( + plugin_metadata, + feature_set, + amd64_substitution, + ) + api.load_plugin() + return feature_set + + +@pytest.fixture +def attribute_path(request) -> AttributePath: + return AttributePath.builtin_path()[request.node.nodeid] diff --git a/tests/data/custom-plugin.json.in b/tests/data/custom-plugin.json.in new file mode 100644 index 0000000..67ca847 --- /dev/null +++ b/tests/data/custom-plugin.json.in @@ -0,0 +1,16 @@ +{ + "plugin-initializer": "custom_plugin_initializer", + "api-compat-version": 1, + "packager-provided-files": [ + { + "stem": "test-file-from-json", + "installed-path": "/usr/share/test-files/{name}.test", + "default-mode": "0644", + "allow-name-segment": true, + "reference-documentation": { + "description": "test of loading PPFs from a JSON", + "format-documentation-uris": ["man:ls(1)"] + } + } + ] +} diff --git a/tests/data/custom_plugin.py b/tests/data/custom_plugin.py new file mode 100644 index 0000000..41d9779 --- /dev/null +++ b/tests/data/custom_plugin.py @@ -0,0 +1,25 @@ +from debputy.plugin.api import ( + DebputyPluginInitializer, + VirtualPath, + BinaryCtrlAccessor, + PackageProcessingContext, +) + + +def _udeb_metadata_detector( + _path: VirtualPath, + ctrl: BinaryCtrlAccessor, + _context: PackageProcessingContext, +) -> None: + ctrl.substvars["Test:Udeb-Metadata-Detector"] = "was-run" + + +def custom_plugin_initializer(api: DebputyPluginInitializer) -> None: + api.packager_provided_file( + "my-file", + "/no-where/this/is/a/test/plugin/{name}.conf", + post_formatting_rewrite=lambda x: x.replace("+", "_"), + ) + api.metadata_or_maintscript_detector( + "udeb-only", _udeb_metadata_detector, package_type="udeb" + ) diff --git a/tests/plugin_tests/conftest.py b/tests/plugin_tests/conftest.py new file mode 100644 index 0000000..f2a8aea --- /dev/null +++ b/tests/plugin_tests/conftest.py @@ -0,0 +1,20 @@ +import os + +import pytest + + +@pytest.fixture(autouse=True) +def workaround_debputys_own_test_suite() -> None: + # This fixture is only required as long as the tests are run inside `debputy`'s + # own test suite. If you copy out a plugin + tests, you should *not* need this + # fixture. + # + # The problem appears because in the debputy source package, these plugins are + # always provided in their "installed" location. + orig = os.environ.get("DEBPUTY_TEST_PLUGIN_LOCATION") + os.environ["DEBPUTY_TEST_PLUGIN_LOCATION"] = "installed" + yield + if orig is None: + del os.environ["DEBPUTY_TEST_PLUGIN_LOCATION"] + else: + os.environ["DEBPUTY_TEST_PLUGIN_LOCATION"] = orig diff --git a/tests/plugin_tests/gnome_test.py b/tests/plugin_tests/gnome_test.py new file mode 100644 index 0000000..ec27a85 --- /dev/null +++ b/tests/plugin_tests/gnome_test.py @@ -0,0 +1,45 @@ +import pytest + +from debputy.plugin.api.test_api import ( + initialize_plugin_under_test, + build_virtual_file_system, + package_metadata_context, +) + + +@pytest.mark.parametrize( + "version,expected_version,expected_next_version", + [ + ( + "1:3.36.1", + "1:3.36", + "1:3.38", + ), + ( + "3.38.2", + "3.38", + "40", + ), + ( + "40.2.0", + "40~", + "41~", + ), + ( + "40", + "40~", + "41~", + ), + ], +) +def test_gnome_plugin( + version: str, + expected_version: str, + expected_next_version: str, +) -> None: + plugin = initialize_plugin_under_test() + fs = build_virtual_file_system([]) + context = package_metadata_context(binary_package_version=version) + metadata = plugin.run_metadata_detector("gnome-versions", fs, context) + assert metadata.substvars["gnome:Version"] == expected_version + assert metadata.substvars["gnome:NextVersion"] == expected_next_version diff --git a/tests/plugin_tests/numpy3_test.data b/tests/plugin_tests/numpy3_test.data new file mode 100644 index 0000000..22b65a2 --- /dev/null +++ b/tests/plugin_tests/numpy3_test.data @@ -0,0 +1,4 @@ +# Values taken from 1:1.24.2-1 +abi 9 +api 16 +api-min-version 1:1.22.0 diff --git a/tests/plugin_tests/numpy3_test.py b/tests/plugin_tests/numpy3_test.py new file mode 100644 index 0000000..9b252fb --- /dev/null +++ b/tests/plugin_tests/numpy3_test.py @@ -0,0 +1,38 @@ +import os + +import pytest + +from debputy.plugin.api.test_api import ( + initialize_plugin_under_test, + build_virtual_file_system, + package_metadata_context, +) + +DATA_FILE = os.path.join(os.path.dirname(__file__), "numpy3_test.data") + + +@pytest.fixture(scope="session") +def numpy3_stub_data_file() -> None: + os.environ["_NUMPY_TEST_PATH"] = DATA_FILE + yield + try: + del os.environ["_NUMPY_TEST_PATH"] + except KeyError: + pass + + +def test_numpy3_plugin_arch_all(numpy3_stub_data_file) -> None: + plugin = initialize_plugin_under_test() + fs = build_virtual_file_system([]) + context = package_metadata_context(package_fields={"Architecture": "all"}) + metadata = plugin.run_metadata_detector("numpy-depends", fs, context) + assert metadata.substvars["python3:Depends"] == "python3-numpy" + + +def test_numpy3_plugin_arch_any(numpy3_stub_data_file) -> None: + plugin = initialize_plugin_under_test() + fs = build_virtual_file_system([]) + context = package_metadata_context(package_fields={"Architecture": "any"}) + metadata = plugin.run_metadata_detector("numpy-depends", fs, context) + expected = "python3-numpy (>= 1:1.22.0), python3-numpy-abi9" + assert metadata.substvars["python3:Depends"] == expected diff --git a/tests/plugin_tests/perl-openssl_test.py b/tests/plugin_tests/perl-openssl_test.py new file mode 100644 index 0000000..37f2ba1 --- /dev/null +++ b/tests/plugin_tests/perl-openssl_test.py @@ -0,0 +1,33 @@ +import stat +import os + +import pytest + +from debputy.plugin.api.test_api import ( + initialize_plugin_under_test, + build_virtual_file_system, + package_metadata_context, +) + +STUB_CMD = os.path.join(os.path.dirname(__file__), "perl-ssl_test.sh") + + +@pytest.fixture(scope="session") +def perl_ssl_stub_cmd() -> None: + os.environ["_PERL_SSL_DEFAULTS_TEST_PATH"] = STUB_CMD + mode = stat.S_IMODE(os.stat(STUB_CMD).st_mode) + if (mode & 0o500) != 0o500: + os.chmod(STUB_CMD, mode | 0o500) + yield + try: + del os.environ["_PERL_SSL_DEFAULTS_TEST_PATH"] + except KeyError: + pass + + +def test_perl_openssl(perl_ssl_stub_cmd) -> None: + plugin = initialize_plugin_under_test() + fs = build_virtual_file_system([]) + context = package_metadata_context(package_fields={"Architecture": "all"}) + metadata = plugin.run_metadata_detector("perl-openssl-abi", fs, context) + assert metadata.substvars["perl:Depends"] == "perl-openssl-abi-3" diff --git a/tests/plugin_tests/perl-ssl_test.sh b/tests/plugin_tests/perl-ssl_test.sh new file mode 100755 index 0000000..5238dc7 --- /dev/null +++ b/tests/plugin_tests/perl-ssl_test.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +# The original script looks up libssl.so and extract part of the SONAME, +# which is all adequately stubbed by: +echo 3 diff --git a/tests/test_apply_compression.py b/tests/test_apply_compression.py new file mode 100644 index 0000000..70817f9 --- /dev/null +++ b/tests/test_apply_compression.py @@ -0,0 +1,33 @@ +from debputy.filesystem_scan import build_virtual_fs +from debputy.plugin.api import virtual_path_def +from debputy.plugin.debputy.package_processors import apply_compression + + +def test_apply_compression(): + # TODO: This test should be a proper plugin test + fs_root = build_virtual_fs( + [ + virtual_path_def( + "./usr/share/man/man1/foo.1", + materialized_content="manpage content", + ), + virtual_path_def("./usr/share/man/man1/bar.1", link_target="foo.1"), + virtual_path_def( + "./usr/share/man/de/man1/bar.1", link_target="../../man1/foo.1" + ), + ], + read_write_fs=True, + ) + apply_compression(fs_root, None, None) + + assert fs_root.lookup("./usr/share/man/man1/foo.1") is None + assert fs_root.lookup("./usr/share/man/man1/foo.1.gz") is not None + assert fs_root.lookup("./usr/share/man/man1/bar.1") is None + bar_symlink = fs_root.lookup("./usr/share/man/man1/bar.1.gz") + assert bar_symlink is not None + assert bar_symlink.readlink() == "foo.1.gz" + + assert fs_root.lookup("./usr/share/man/de/man1/bar.1") is None + de_bar_symlink = fs_root.lookup("./usr/share/man/de/man1/bar.1.gz") + assert de_bar_symlink is not None + assert de_bar_symlink.readlink() == "../../man1/foo.1.gz" diff --git a/tests/test_architecture.py b/tests/test_architecture.py new file mode 100644 index 0000000..b0431bc --- /dev/null +++ b/tests/test_architecture.py @@ -0,0 +1,59 @@ +from debputy.architecture_support import faked_arch_table + + +# Ensure our mocks seem to be working reasonably +def test_mock_arch_table(): + amd64_native_table = faked_arch_table("amd64") + amd64_cross_table = faked_arch_table("amd64", build_arch="i386") + amd64_cross_target_table = faked_arch_table("amd64", target_arch="arm64") + all_differ_table = faked_arch_table("amd64", build_arch="i386", target_arch="arm64") + + for var_stem in ["ARCH", "MULTIARCH"]: + host_var = f"DEB_HOST_{var_stem}" + build_var = f"DEB_BUILD_{var_stem}" + target_var = f"DEB_TARGET_{var_stem}" + + assert ( + amd64_cross_table.current_host_arch == amd64_native_table.current_host_arch + ) + assert amd64_native_table[host_var] == amd64_native_table[build_var] + assert amd64_native_table[host_var] == amd64_native_table[target_var] + + # HOST_ARCH differ in a cross build, but the rest remain the same + assert amd64_cross_table[host_var] == amd64_native_table[host_var] + assert amd64_cross_table[target_var] == amd64_native_table[target_var] + assert amd64_cross_table[build_var] != amd64_native_table[build_var] + assert amd64_cross_table[target_var] == amd64_native_table[target_var] + assert ( + amd64_cross_table.current_host_multiarch + == amd64_native_table.current_host_multiarch + ) + + # TARGET_ARCH differ in a cross-compiler build, but the rest remain the same + assert amd64_cross_target_table[host_var] == amd64_native_table[host_var] + assert amd64_cross_target_table[target_var] != amd64_native_table[target_var] + assert amd64_cross_target_table[build_var] == amd64_native_table[build_var] + assert ( + amd64_cross_target_table.current_host_multiarch + == amd64_native_table.current_host_multiarch + ) + + # TARGET_ARCH differ in a cross-compiler build, but the rest remain the same + assert all_differ_table[host_var] == amd64_native_table[host_var] + assert all_differ_table[target_var] != amd64_native_table[target_var] + assert all_differ_table[build_var] != amd64_native_table[build_var] + assert all_differ_table[build_var] == amd64_cross_table[build_var] + assert all_differ_table[target_var] == amd64_cross_target_table[target_var] + assert ( + all_differ_table.current_host_arch == amd64_native_table.current_host_arch + ) + assert ( + all_differ_table.current_host_multiarch + == amd64_native_table.current_host_multiarch + ) + + # Finally, check is_cross_compiling + assert not amd64_native_table.is_cross_compiling + assert amd64_cross_table.is_cross_compiling + assert not amd64_cross_target_table.is_cross_compiling + assert all_differ_table.is_cross_compiling diff --git a/tests/test_cross_check_precheck.py b/tests/test_cross_check_precheck.py new file mode 100644 index 0000000..41bf01c --- /dev/null +++ b/tests/test_cross_check_precheck.py @@ -0,0 +1,124 @@ +from typing import Optional + +import pytest + +from debputy.plugin.api.test_api import package_metadata_context +from debputy.util import package_cross_check_precheck + + +@pytest.mark.parametrize( + "a_arch,b_arch,a_bp,b_bp,act_on_a,act_on_b,ex_res_a2b,ex_res_b2a", + [ + # Both way OK + ("any", "any", None, None, True, True, True, True), + ("all", "all", None, None, True, True, True, True), + ("any", "any", "", "", True, True, True, True), + # OK as well. Same BPs just reordered + ( + "any", + "any", + "", + "", + True, + True, + True, + True, + ), + ( + "any", + "any", + " ", + " ", + True, + True, + True, + True, + ), + # One way OK + ("any", "any", None, "", True, True, True, False), + ("any", "any", None, "", True, True, True, False), + # One way OK - BP is clearly a subset of the other + ( + "any", + "any", + "", + " ", + True, + True, + True, + False, + ), + ( + "any", + "any", + "", + " ", + True, + True, + True, + False, + ), + # Currently fails but should probably allow one way + ( + "any", + "any", + "", + " ", + True, + True, + False, + False, + ), + ( + "any", + "any", + "", + " ", + True, + True, + False, + False, + ), + # Negative tests + ("any", "all", None, None, True, True, False, False), + ("all", "all", None, None, True, False, False, False), + ("any", "any", None, None, False, True, False, False), + ("i386", "amd64", None, None, True, True, False, False), + ], +) +def test_generate_deb_filename( + a_arch: str, + b_arch: str, + a_bp: Optional[str], + b_bp: Optional[str], + act_on_a: bool, + act_on_b: bool, + ex_res_a2b: bool, + ex_res_b2a: bool, +): + pkg_a_fields = { + "Package": "pkg-a", + "Architecture": a_arch, + } + if a_bp is not None: + pkg_a_fields["Build-Profiles"] = a_bp + + pkg_b_fields = { + "Package": "pkg-b", + "Architecture": b_arch, + } + if b_bp is not None: + pkg_b_fields["Build-Profiles"] = b_bp + + pkg_a = package_metadata_context( + package_fields=pkg_a_fields, + should_be_acted_on=act_on_a, + ).binary_package + pkg_b = package_metadata_context( + package_fields=pkg_b_fields, + should_be_acted_on=act_on_b, + ).binary_package + + assert package_cross_check_precheck(pkg_a, pkg_b) == (ex_res_a2b, ex_res_b2a) + # Inverted should functionally give the same answer + assert package_cross_check_precheck(pkg_b, pkg_a) == (ex_res_b2a, ex_res_a2b) diff --git a/tests/test_deb_packaging_support.py b/tests/test_deb_packaging_support.py new file mode 100644 index 0000000..d47526d --- /dev/null +++ b/tests/test_deb_packaging_support.py @@ -0,0 +1,218 @@ +import pytest + +from debputy.deb_packaging_support import install_upstream_changelog +from debputy.filesystem_scan import build_virtual_fs +from debputy.plugin.api import virtual_path_def + + +@pytest.mark.parametrize( + "upstream_changelog_name,other_files", + [ + ( + "changelog.txt", + [ + "changelog.md", + "CHANGELOG.rst", + "random-file", + ], + ), + ( + "CHANGELOG.rst", + [ + "doc/CHANGELOG.txt", + "docs/CHANGELOG.md", + ], + ), + ( + "docs/CHANGELOG.rst", + [ + "docs/history.md", + ], + ), + ( + "changelog", + [], + ), + ], +) +def test_upstream_changelog_from_source( + package_single_foo_arch_all_cxt_amd64, + upstream_changelog_name, + other_files, +) -> None: + upstream_changelog_content = "Some upstream changelog" + dctrl = package_single_foo_arch_all_cxt_amd64["foo"] + data_fs_root = build_virtual_fs([], read_write_fs=True) + upstream_fs_contents = [ + virtual_path_def("CHANGELOG", materialized_content="Some upstream changelog") + ] + upstream_fs_contents.extend( + virtual_path_def(x, materialized_content="Wrong file!") for x in other_files + ) + source_fs_root = build_virtual_fs(upstream_fs_contents) + + install_upstream_changelog(dctrl, data_fs_root, source_fs_root) + + upstream_changelog = data_fs_root.lookup(f"usr/share/doc/{dctrl.name}/changelog") + assert upstream_changelog is not None + assert upstream_changelog.is_file + with upstream_changelog.open() as fd: + content = fd.read() + assert upstream_changelog_content == content + + +@pytest.mark.parametrize( + "upstream_changelog_basename,other_data_files,other_source_files", + [ + ( + "CHANGELOG", + [ + "history.txt", + "changes.md", + ], + [ + "changelog", + "doc/CHANGELOG.txt", + "docs/CHANGELOG.md", + ], + ), + ( + "changelog", + [ + "history.txt", + "changes.md", + ], + [ + "changelog", + "doc/CHANGELOG.txt", + "docs/CHANGELOG.md", + ], + ), + ( + "changes.md", + [ + "changelog.rst", + ], + ["changelog"], + ), + ], +) +def test_upstream_changelog_from_data_fs( + package_single_foo_arch_all_cxt_amd64, + upstream_changelog_basename, + other_data_files, + other_source_files, +) -> None: + upstream_changelog_content = "Some upstream changelog" + dctrl = package_single_foo_arch_all_cxt_amd64["foo"] + doc_dir = f"./usr/share/doc/{dctrl.name}" + data_fs_contents = [ + virtual_path_def( + f"{doc_dir}/{upstream_changelog_basename}", + materialized_content="Some upstream changelog", + ) + ] + data_fs_contents.extend( + virtual_path_def( + f"{doc_dir}/{x}", + materialized_content="Wrong file!", + ) + for x in other_data_files + ) + data_fs_root = build_virtual_fs(data_fs_contents, read_write_fs=True) + source_fs_root = build_virtual_fs( + [ + virtual_path_def( + x, + materialized_content="Wrong file!", + ) + for x in other_source_files + ] + ) + + install_upstream_changelog(dctrl, data_fs_root, source_fs_root) + + upstream_changelog = data_fs_root.lookup(f"usr/share/doc/{dctrl.name}/changelog") + assert upstream_changelog is not None + assert upstream_changelog.is_file + with upstream_changelog.open() as fd: + content = fd.read() + assert upstream_changelog_content == content + + +def test_upstream_changelog_pre_installed_compressed( + package_single_foo_arch_all_cxt_amd64, +) -> None: + dctrl = package_single_foo_arch_all_cxt_amd64["foo"] + changelog = f"./usr/share/doc/{dctrl.name}/changelog.gz" + data_fs_root = build_virtual_fs( + [virtual_path_def(changelog, fs_path="/nowhere/should/not/be/resolved")], + read_write_fs=True, + ) + source_fs_root = build_virtual_fs( + [virtual_path_def("changelog", materialized_content="Wrong file!")] + ) + + install_upstream_changelog(dctrl, data_fs_root, source_fs_root) + + upstream_ch_compressed = data_fs_root.lookup( + f"usr/share/doc/{dctrl.name}/changelog.gz" + ) + assert upstream_ch_compressed is not None + assert upstream_ch_compressed.is_file + upstream_ch_uncompressed = data_fs_root.lookup( + f"usr/share/doc/{dctrl.name}/changelog" + ) + assert upstream_ch_uncompressed is None + + +def test_upstream_changelog_no_matches( + package_single_foo_arch_all_cxt_amd64, +) -> None: + dctrl = package_single_foo_arch_all_cxt_amd64["foo"] + doc_dir = f"./usr/share/doc/{dctrl.name}" + data_fs_root = build_virtual_fs( + [ + virtual_path_def( + f"{doc_dir}/random-file", materialized_content="Wrong file!" + ), + virtual_path_def( + f"{doc_dir}/changelog.Debian", materialized_content="Wrong file!" + ), + ], + read_write_fs=True, + ) + source_fs_root = build_virtual_fs( + [virtual_path_def("some-random-file", materialized_content="Wrong file!")] + ) + + install_upstream_changelog(dctrl, data_fs_root, source_fs_root) + + upstream_ch_compressed = data_fs_root.lookup( + f"usr/share/doc/{dctrl.name}/changelog.gz" + ) + assert upstream_ch_compressed is None + upstream_ch_uncompressed = data_fs_root.lookup( + f"usr/share/doc/{dctrl.name}/changelog" + ) + assert upstream_ch_uncompressed is None + + +def test_upstream_changelog_salsa_issue_49( + package_single_foo_arch_all_cxt_amd64, +) -> None: + # https://salsa.debian.org/debian/debputy/-/issues/49 + dctrl = package_single_foo_arch_all_cxt_amd64["foo"] + doc_dir = f"./usr/share/doc/{dctrl.name}" + data_fs_root = build_virtual_fs( + [virtual_path_def(f"{doc_dir}", link_target="foo-data")], read_write_fs=True + ) + source_fs_root = build_virtual_fs( + [virtual_path_def("changelog", materialized_content="Wrong file!")] + ) + + install_upstream_changelog(dctrl, data_fs_root, source_fs_root) + + doc_dir = data_fs_root.lookup(f"usr/share/doc/{dctrl.name}") + assert doc_dir is not None + assert doc_dir.is_symlink diff --git a/tests/test_debputy_plugin.py b/tests/test_debputy_plugin.py new file mode 100644 index 0000000..a5d7758 --- /dev/null +++ b/tests/test_debputy_plugin.py @@ -0,0 +1,1246 @@ +import os +import textwrap +from typing import Sequence + +import pytest + +from debputy.exceptions import ( + DebputyManifestVariableRequiresDebianDirError, + DebputySubstitutionError, +) +from debputy.manifest_parser.base_types import SymbolicMode +from debputy.manifest_parser.util import AttributePath +from debputy.plugin.api import virtual_path_def +from debputy.plugin.api.spec import DSD +from debputy.plugin.api.test_api import ( + build_virtual_file_system, + package_metadata_context, +) +from debputy.plugin.api.test_api import manifest_variable_resolution_context +from debputy.plugin.api.test_api.test_impl import initialize_plugin_under_test_preloaded +from debputy.plugin.api.test_api.test_spec import DetectedService +from debputy.plugin.debputy.debputy_plugin import initialize_debputy_features +from debputy.plugin.debputy.private_api import load_libcap +from debputy.plugin.debputy.service_management import SystemdServiceContext +from debputy.plugin.debputy.types import DebputyCapability + + +def test_debputy_packager_provided_files(): + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + ppf_by_stem = plugin.packager_provided_files_by_stem() + # Verify that all the files are loaded + assert set(ppf_by_stem.keys()) == { + "tmpfiles", + "sysusers", + "bash-completion", + "pam", + "ppp.ip-up", + "ppp.ip-down", + "logrotate", + "logcheck.cracking", + "logcheck.violations", + "logcheck.violations.ignore", + "logcheck.ignore.workstation", + "logcheck.ignore.server", + "logcheck.ignore.paranoid", + "mime", + "sharedmimeinfo", + "if-pre-up", + "if-up", + "if-down", + "if-post-down", + "cron.hourly", + "cron.daily", + "cron.weekly", + "cron.monthly", + "cron.yearly", + "cron.d", + "initramfs-hook", + "modprobe", + "gsettings-override", + "lintian-overrides", + "bug-script", + "bug-control", + "bug-presubj", + "changelog", + "NEWS", + "copyright", + "README.Debian", + "TODO", + "doc-base", + "shlibs", + "symbols", + "alternatives", + "init", + "default", + "templates", + # dh_installsytemd + "mount", + "path", + "service", + "socket", + "target", + "timer", + "@path", + "@service", + "@socket", + "@target", + "@timer", + } + # Verify the post_rewrite_hook + assert ( + ppf_by_stem["logcheck.ignore.paranoid"].compute_dest("foo.bar")[1] == "foo_bar" + ) + # Verify custom formats work + assert ppf_by_stem["tmpfiles"].compute_dest("foo.bar")[1] == "foo.bar.conf" + assert ppf_by_stem["sharedmimeinfo"].compute_dest("foo.bar")[1] == "foo.bar.xml" + assert ppf_by_stem["modprobe"].compute_dest("foo.bar")[1] == "foo.bar.conf" + assert ( + ppf_by_stem["gsettings-override"].compute_dest("foo.bar", assigned_priority=20)[ + 1 + ] + == "20_foo.bar.gschema.override" + ) + + +def test_debputy_docbase_naming() -> None: + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + doc_base_pff = plugin.packager_provided_files_by_stem()["doc-base"] + fs_root = build_virtual_file_system( + [virtual_path_def("foo.doc-base", content="Document: bar")] + ) + _, basename = doc_base_pff.compute_dest("foo", path=fs_root["foo.doc-base"]) + assert basename == "foo.bar" + + +def test_debputy_adr_examples() -> None: + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + issues = plugin.automatic_discard_rules_examples_with_issues() + assert not issues + + +def test_debputy_metadata_detector_gsettings_dependencies(): + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + # By default, the plugin will not add a substvars + fs_root = build_virtual_file_system(["./bin/ls"]) + metadata = plugin.run_metadata_detector("gsettings-dependencies", fs_root) + assert "misc:Depends" not in metadata.substvars + + # It will not react if there is only directories or non-files + fs_root = build_virtual_file_system(["./usr/share/glib-2.0/schemas/some-dir/"]) + metadata = plugin.run_metadata_detector("gsettings-dependencies", fs_root) + assert "misc:Depends" not in metadata.substvars + + # However, it will if there is a file beneath the schemas dir + fs_root = build_virtual_file_system(["./usr/share/glib-2.0/schemas/foo.xml"]) + metadata = plugin.run_metadata_detector("gsettings-dependencies", fs_root) + assert ( + metadata.substvars["misc:Depends"] + == "dconf-gsettings-backend | gsettings-backend" + ) + + +def test_debputy_metadata_detector_initramfs_hooks(): + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + metadata_detector_id = "initramfs-hooks" + + # By default, the plugin will not add a trigger + fs_root = build_virtual_file_system(["./bin/ls"]) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.triggers == [] + + # It will not react if the directory is empty + fs_root = build_virtual_file_system( + [ + # Use an absolute path to verify that also work (it should and third-party plugin are likely + # use absolute paths) + "/usr/share/initramfs-tools/hooks/" + ] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.triggers == [] + + # However, it will if there is a file beneath the schemas dir + fs_root = build_virtual_file_system(["./usr/share/initramfs-tools/hooks/some-hook"]) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + result = [t.serialized_format() for t in metadata.triggers] + assert result == ["activate-noawait update-initramfs"] + + +def test_debputy_metadata_detector_systemd_tmpfiles(): + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + metadata_detector_id = "systemd-tmpfiles" + + # By default, the plugin will not add anything + fs_root = build_virtual_file_system(["./bin/ls"]) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.maintscripts() == [] + + # It only reacts to ".conf" files + fs_root = build_virtual_file_system(["./usr/lib/tmpfiles.d/foo"]) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.maintscripts() == [] + + fs_root = build_virtual_file_system( + [ + "./usr/lib/tmpfiles.d/foo.conf", + "./etc/tmpfiles.d/foo.conf", + ] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + snippets = metadata.maintscripts() + assert len(snippets) == 1 + snippet = snippets[0] + assert snippet.maintscript == "postinst" + assert snippet.registration_method == "on_configure" + # The snippet should use "systemd-tmpfiles [...] --create foo.conf ..." + assert "--create foo.conf" in snippet.plugin_provided_script + # The "foo.conf" should only be listed once + assert snippet.plugin_provided_script.count("foo.conf") == 1 + + +def test_debputy_metadata_detector_systemd_sysusers(): + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + metadata_detector_id = "systemd-sysusers" + + # By default, the plugin will not add anything + fs_root = build_virtual_file_system(["./bin/ls"]) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.maintscripts() == [] + + # It only reacts to ".conf" files + fs_root = build_virtual_file_system(["./usr/lib/sysusers.d/foo"]) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.maintscripts() == [] + + fs_root = build_virtual_file_system(["./usr/lib/sysusers.d/foo.conf"]) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + snippets = metadata.maintscripts() + assert len(snippets) == 1 + snippet = snippets[0] + assert snippet.maintscript == "postinst" + assert snippet.registration_method == "on_configure" + # The snippet should use "systemd-sysusers [...] foo.conf ..." + assert "systemd-sysusers" in snippet.plugin_provided_script + assert "foo.conf" in snippet.plugin_provided_script + # The "foo.conf" should only be listed once + assert snippet.plugin_provided_script.count("foo.conf") == 1 + + +def test_debputy_metadata_detector_xfonts(): + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + metadata_detector_id = "xfonts" + + # By default, the plugin will not add anything + fs_root = build_virtual_file_system(["./bin/ls"]) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.maintscripts() == [] + assert "misc:Depends" not in metadata.substvars + + # It ignores files in the X11 dir and directories starting with ".". + fs_root = build_virtual_file_system( + ["./usr/share/fonts/X11/foo", "./usr/share/fonts/X11/.a/"] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.maintscripts() == [] + assert "misc:Depends" not in metadata.substvars + + fs_root = build_virtual_file_system( + [ + "./usr/share/fonts/X11/some-font-dir/", + ] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + snippets = metadata.maintscripts() + assert metadata.substvars["misc:Depends"] == "xfonts-utils" + assert len(snippets) == 2 + assert set(s.maintscript for s in snippets) == {"postinst", "postrm"} + postinst_snippet = metadata.maintscripts(maintscript="postinst")[0] + postrm_snippet = metadata.maintscripts(maintscript="postrm")[0] + + assert postinst_snippet.maintscript == "postinst" + assert postinst_snippet.registration_method == "unconditionally_in_script" + assert ( + "update-fonts-scale some-font-dir" + not in postinst_snippet.plugin_provided_script + ) + assert "--x11r7-layout some-font-dir" in postinst_snippet.plugin_provided_script + assert ( + f"update-fonts-alias --include" not in postinst_snippet.plugin_provided_script + ) + + assert postrm_snippet.maintscript == "postrm" + assert postrm_snippet.registration_method == "unconditionally_in_script" + assert ( + "update-fonts-scale some-font-dir" not in postrm_snippet.plugin_provided_script + ) + assert "--x11r7-layout some-font-dir" in postrm_snippet.plugin_provided_script + assert f"update-fonts-alias --exclude" not in postrm_snippet.plugin_provided_script + + +def test_debputy_metadata_detector_xfonts_scale_and_alias(): + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + + metadata_detector_id = "xfonts" + package_name = "bar" + fs_root = build_virtual_file_system( + [ + "./usr/share/fonts/X11/some-font-dir/", + f"./etc/X11/xfonts/some-font-dir/{package_name}.scale", + f"./etc/X11/xfonts/some-font-dir/{package_name}.alias", + ] + ) + metadata = plugin.run_metadata_detector( + metadata_detector_id, + fs_root, + package_metadata_context( + package_fields={ + "Package": package_name, + } + ), + ) + snippets = metadata.maintscripts() + assert metadata.substvars["misc:Depends"] == "xfonts-utils" + assert len(snippets) == 2 + assert set(s.maintscript for s in snippets) == {"postinst", "postrm"} + postinst_snippet = metadata.maintscripts(maintscript="postinst")[0] + postrm_snippet = metadata.maintscripts(maintscript="postrm")[0] + + assert postinst_snippet.maintscript == "postinst" + assert postinst_snippet.registration_method == "unconditionally_in_script" + assert "update-fonts-scale some-font-dir" in postinst_snippet.plugin_provided_script + assert "--x11r7-layout some-font-dir" in postinst_snippet.plugin_provided_script + assert ( + f"update-fonts-alias --include /etc/X11/xfonts/some-font-dir/{package_name}.alias some-font-dir" + in postinst_snippet.plugin_provided_script + ) + + assert postrm_snippet.maintscript == "postrm" + assert postrm_snippet.registration_method == "unconditionally_in_script" + assert "update-fonts-scale some-font-dir" in postrm_snippet.plugin_provided_script + assert "--x11r7-layout some-font-dir" in postrm_snippet.plugin_provided_script + assert ( + f"update-fonts-alias --exclude /etc/X11/xfonts/some-font-dir/{package_name}.alias some-font-dir" + in postrm_snippet.plugin_provided_script + ) + + +def test_debputy_metadata_detector_icon_cache(): + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + metadata_detector_id = "icon-cache" + icon_dir = "usr/share/icons" + + # By default, the plugin will not add anything + fs_root = build_virtual_file_system(["./bin/ls"]) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.maintscripts() == [] + + fs_root = build_virtual_file_system( + [ + # Ignored subdirs (dh_icons ignores them too) + f"./{icon_dir}/gnome/foo.png", + f"./{icon_dir}/hicolor/foo.png", + # Unknown image format, so it does not trigger the update-icon-caches call + f"./{icon_dir}/subdir-a/unknown-image-format.img", + ] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.maintscripts() == [] + + fs_root = build_virtual_file_system( + [ + f"./{icon_dir}/subdir-a/foo.png", + f"./{icon_dir}/subdir-b/subsubdir/bar.svg", + ] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + snippets = metadata.maintscripts() + assert len(snippets) == 2 + assert set(s.maintscript for s in snippets) == {"postinst", "postrm"} + postinst_snippet = metadata.maintscripts(maintscript="postinst")[0] + postrm_snippet = metadata.maintscripts(maintscript="postrm")[0] + + assert postinst_snippet.registration_method == "on_configure" + assert postrm_snippet.registration_method == "unconditionally_in_script" + + # Directory order is stable according to the BinaryPackagePath API. + assert ( + f"update-icon-caches /{icon_dir}/subdir-a /{icon_dir}/subdir-b" + in postinst_snippet.plugin_provided_script + ) + assert ( + f"update-icon-caches /{icon_dir}/subdir-a /{icon_dir}/subdir-b" + in postrm_snippet.plugin_provided_script + ) + + +def test_debputy_metadata_detector_kernel_modules(): + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + metadata_detector_id = "kernel-modules" + module_dir = "lib/modules" + + # By default, the plugin will not add anything + fs_root = build_virtual_file_system(["./bin/ls"]) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.maintscripts() == [] + + fs_root = build_virtual_file_system( + [ + # Ignore files directly in the path or with wrong extension + f"./{module_dir}/README", + f"./{module_dir}/3.11/ignored-file.txt", + ] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.maintscripts() == [] + + fs_root = build_virtual_file_system( + [ + f"./{module_dir}/3.11/foo.ko", + f"./usr/{module_dir}/3.12/bar.ko.xz", + ] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + snippets = metadata.maintscripts() + assert len(snippets) == 4 # Two for each version + assert set(s.maintscript for s in snippets) == {"postinst", "postrm"} + postinst_snippets = metadata.maintscripts(maintscript="postinst") + postrm_snippets = metadata.maintscripts(maintscript="postrm") + + assert len(postinst_snippets) == 2 + assert len(postrm_snippets) == 2 + assert {s.registration_method for s in postinst_snippets} == {"on_configure"} + assert {s.registration_method for s in postrm_snippets} == { + "unconditionally_in_script" + } + + assert ( + "depmod -a -F /boot/System.map-3.11 3.11" + in postinst_snippets[0].plugin_provided_script + ) + assert ( + "depmod -a -F /boot/System.map-3.12 3.12" + in postinst_snippets[1].plugin_provided_script + ) + + assert ( + "depmod -a -F /boot/System.map-3.11 3.11" + in postrm_snippets[0].plugin_provided_script + ) + assert ( + "depmod -a -F /boot/System.map-3.12 3.12" + in postrm_snippets[1].plugin_provided_script + ) + + +def test_debputy_metadata_detector_dpkg_shlibdeps(): + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + metadata_detector_id = "dpkg-shlibdeps" + skip_root_dir = "usr/lib/debug/" + + # By default, the plugin will not add anything + fs_root = build_virtual_file_system( + [ + "./usr/share/doc/foo/copyright", + virtual_path_def("./usr/lib/debputy/test.py", fs_path=__file__), + ] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert "shlibs:Depends" not in metadata.substvars + + fs_root = build_virtual_file_system( + [ + # Verify that certain directories are skipped as promised + virtual_path_def(f"./{skip_root_dir}/bin/ls", fs_path="/bin/ls") + ] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert "shlibs:Depends" not in metadata.substvars + + # But we detect ELF binaries elsewhere + fs_root = build_virtual_file_system( + [virtual_path_def(f"./bin/ls", fs_path="/bin/ls")] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + # Do not make assertions about the content of `shlibs:Depends` as + # package name and versions change over time. + assert "shlibs:Depends" in metadata.substvars + + # Re-run to verify it runs for udebs as well + metadata = plugin.run_metadata_detector( + metadata_detector_id, + fs_root, + context=package_metadata_context( + package_fields={"Package-Type": "udeb"}, + ), + ) + assert "shlibs:Depends" in metadata.substvars + + +def test_debputy_metadata_detector_pycompile_files(): + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + metadata_detector_id = "pycompile-files" + module_dir = "usr/lib/python3/dist-packages" + + # By default, the plugin will not add anything + fs_root = build_virtual_file_system(["./bin/ls"]) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.maintscripts() == [] + + fs_root = build_virtual_file_system( + [ + # Ignore files in unknown directories by default + "./random-dir/foo.py", + # Must be in "dist-packages" to count + "./usr/lib/python3/foo.py", + ] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + assert metadata.maintscripts() == [] + + fs_root = build_virtual_file_system( + [ + f"./{module_dir}/debputy/foo.py", + ] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + snippets = metadata.maintscripts() + assert len(snippets) == 2 + assert set(s.maintscript for s in snippets) == {"postinst", "prerm"} + postinst_snippets = metadata.maintscripts(maintscript="postinst") + prerm_snippets = metadata.maintscripts(maintscript="prerm") + + assert len(postinst_snippets) == 1 + assert len(prerm_snippets) == 1 + assert {s.registration_method for s in postinst_snippets} == {"on_configure"} + assert {s.registration_method for s in prerm_snippets} == { + "unconditionally_in_script" + } + + assert "py3compile -p foo" in postinst_snippets[0].plugin_provided_script + + assert "py3clean -p foo" in prerm_snippets[0].plugin_provided_script + + +def test_debputy_metadata_detector_pycompile_files_private_package_dir(): + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + metadata_detector_id = "pycompile-files" + module_dir = "usr/share/foo" + + fs_root = build_virtual_file_system( + [ + f"./{module_dir}/debputy/foo.py", + ] + ) + metadata = plugin.run_metadata_detector(metadata_detector_id, fs_root) + snippets = metadata.maintscripts() + assert len(snippets) == 2 + assert set(s.maintscript for s in snippets) == {"postinst", "prerm"} + postinst_snippets = metadata.maintscripts(maintscript="postinst") + prerm_snippets = metadata.maintscripts(maintscript="prerm") + + assert len(postinst_snippets) == 1 + assert len(prerm_snippets) == 1 + assert {s.registration_method for s in postinst_snippets} == {"on_configure"} + assert {s.registration_method for s in prerm_snippets} == { + "unconditionally_in_script" + } + + assert ( + f"py3compile -p foo /{module_dir}" + in postinst_snippets[0].plugin_provided_script + ) + + assert "py3clean -p foo" in prerm_snippets[0].plugin_provided_script + + +def _extract_service( + services: Sequence[DetectedService[DSD]], name: str +) -> DetectedService[DSD]: + v = [s for s in services if name in s.names] + assert len(v) == 1 + return v[0] + + +def test_system_service_detection() -> None: + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + systemd_service_root_dir = "usr/lib/systemd" + systemd_service_system_dir = f"{systemd_service_root_dir}/system" + systemd_service_user_dir = f"{systemd_service_root_dir}/user" + + services, _ = plugin.run_service_detection_and_integrations( + "systemd", build_virtual_file_system([]) + ) + assert not services + + services, _ = plugin.run_service_detection_and_integrations( + "systemd", + build_virtual_file_system( + [f"{systemd_service_system_dir}/", f"{systemd_service_user_dir}/"] + ), + ) + assert not services + + fs_root = build_virtual_file_system( + [ + virtual_path_def( + f"{systemd_service_system_dir}/foo.service", + content=textwrap.dedent( + """\ + Alias="myname.service" + [Install] + """ + ), + ), + virtual_path_def( + f"{systemd_service_system_dir}/foo@.service", + content=textwrap.dedent( + """\ + # dh_installsystemd ignores template services - we do for now as well. + Alias="ignored.service" + [Install] + """ + ), + ), + virtual_path_def( + f"{systemd_service_system_dir}/alias.service", link_target="foo.service" + ), + virtual_path_def(f"{systemd_service_system_dir}/bar.timer", content=""), + ] + ) + services, metadata = plugin.run_service_detection_and_integrations( + "systemd", + fs_root, + service_context_type_hint=SystemdServiceContext, + ) + assert len(services) == 2 + assert {s.names[0] for s in services} == {"foo.service", "bar.timer"} + foo_service = _extract_service(services, "foo.service") + assert set(foo_service.names) == { + "foo.service", + "foo", + "alias", + "alias.service", + "myname.service", + "myname", + } + assert foo_service.type_of_service == "service" + assert foo_service.service_scope == "system" + assert foo_service.enable_by_default + assert foo_service.start_by_default + assert foo_service.default_upgrade_rule == "restart" + assert foo_service.service_context.had_install_section + + bar_timer = _extract_service(services, "bar.timer") + assert set(bar_timer.names) == {"bar.timer"} + assert bar_timer.type_of_service == "timer" + assert bar_timer.service_scope == "system" + assert not bar_timer.enable_by_default + assert bar_timer.start_by_default + assert bar_timer.default_upgrade_rule == "restart" + assert not bar_timer.service_context.had_install_section + + snippets = metadata.maintscripts() + assert len(snippets) == 4 + postinsts = metadata.maintscripts(maintscript="postinst") + assert len(postinsts) == 2 + enable_postinst, start_postinst = postinsts + assert ( + "deb-systemd-helper debian-installed foo.service" + in enable_postinst.plugin_provided_script + ) + assert ( + "deb-systemd-invoke start foo.service" in start_postinst.plugin_provided_script + ) + assert ( + "deb-systemd-invoke restart foo.service" + in start_postinst.plugin_provided_script + ) + + +def test_sysv_service_detection() -> None: + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + init_dir = "etc/init.d" + + services, _ = plugin.run_service_detection_and_integrations( + "sysvinit", build_virtual_file_system([]) + ) + assert not services + + services, _ = plugin.run_service_detection_and_integrations( + "sysvinit", + build_virtual_file_system( + [ + f"{init_dir}/", + ] + ), + ) + assert not services + + services, _ = plugin.run_service_detection_and_integrations( + "sysvinit", + build_virtual_file_system( + [ + virtual_path_def( + f"{init_dir}/README", + mode=0o644, + ), + ] + ), + ) + assert not services + + fs_root = build_virtual_file_system( + [ + virtual_path_def( + f"{init_dir}/foo", + mode=0o755, + ), + ] + ) + services, metadata = plugin.run_service_detection_and_integrations( + "sysvinit", fs_root + ) + assert len(services) == 1 + assert {s.names[0] for s in services} == {"foo"} + foo_service = _extract_service(services, "foo") + assert set(foo_service.names) == {"foo"} + assert foo_service.type_of_service == "service" + assert foo_service.service_scope == "system" + assert foo_service.enable_by_default + assert foo_service.start_by_default + assert foo_service.default_upgrade_rule == "restart" + + snippets = metadata.maintscripts() + assert len(snippets) == 4 + postinsts = metadata.maintscripts(maintscript="postinst") + assert len(postinsts) == 1 + postinst = postinsts[0] + assert postinst.registration_method == "on_configure" + assert "" in postinst.plugin_provided_script + assert "update-rc.d foo defaults" in postinst.plugin_provided_script + assert ( + "invoke-rc.d --skip-systemd-native foo start" in postinst.plugin_provided_script + ) + assert ( + "invoke-rc.d --skip-systemd-native foo restart" + in postinst.plugin_provided_script + ) + + +def test_debputy_manifest_variables() -> None: + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + manifest_variables_no_dch = plugin.manifest_variables() + assert manifest_variables_no_dch.keys() == { + "DEB_BUILD_ARCH", + "DEB_BUILD_ARCH_ABI", + "DEB_BUILD_ARCH_BITS", + "DEB_BUILD_ARCH_CPU", + "DEB_BUILD_ARCH_ENDIAN", + "DEB_BUILD_ARCH_LIBC", + "DEB_BUILD_ARCH_OS", + "DEB_BUILD_GNU_CPU", + "DEB_BUILD_GNU_SYSTEM", + "DEB_BUILD_GNU_TYPE", + "DEB_BUILD_MULTIARCH", + "DEB_HOST_ARCH", + "DEB_HOST_ARCH_ABI", + "DEB_HOST_ARCH_BITS", + "DEB_HOST_ARCH_CPU", + "DEB_HOST_ARCH_ENDIAN", + "DEB_HOST_ARCH_LIBC", + "DEB_HOST_ARCH_OS", + "DEB_HOST_GNU_CPU", + "DEB_HOST_GNU_SYSTEM", + "DEB_HOST_GNU_TYPE", + "DEB_HOST_MULTIARCH", + "DEB_SOURCE", + "DEB_TARGET_ARCH", + "DEB_TARGET_ARCH_ABI", + "DEB_TARGET_ARCH_BITS", + "DEB_TARGET_ARCH_CPU", + "DEB_TARGET_ARCH_ENDIAN", + "DEB_TARGET_ARCH_LIBC", + "DEB_TARGET_ARCH_OS", + "DEB_TARGET_GNU_CPU", + "DEB_TARGET_GNU_SYSTEM", + "DEB_TARGET_GNU_TYPE", + "DEB_TARGET_MULTIARCH", + "DEB_VERSION", + "DEB_VERSION_EPOCH_UPSTREAM", + "DEB_VERSION_UPSTREAM", + "DEB_VERSION_UPSTREAM_REVISION", + "PACKAGE", + "SOURCE_DATE_EPOCH", + "_DEBPUTY_INTERNAL_NON_BINNMU_SOURCE", + "_DEBPUTY_SND_SOURCE_DATE_EPOCH", + "path:BASH_COMPLETION_DIR", + "path:GNU_INFO_DIR", + "token:CLOSE_CURLY_BRACE", + "token:DOUBLE_CLOSE_CURLY_BRACE", + "token:DOUBLE_OPEN_CURLY_BRACE", + "token:NEWLINE", + "token:NL", + "token:OPEN_CURLY_BRACE", + "token:TAB", + } + + for v in [ + "DEB_SOURCE", + "DEB_VERSION", + "DEB_VERSION_EPOCH_UPSTREAM", + "DEB_VERSION_UPSTREAM", + "DEB_VERSION_UPSTREAM_REVISION", + "SOURCE_DATE_EPOCH", + "_DEBPUTY_INTERNAL_NON_BINNMU_SOURCE", + "_DEBPUTY_SND_SOURCE_DATE_EPOCH", + ]: + with pytest.raises(DebputyManifestVariableRequiresDebianDirError): + manifest_variables_no_dch[v] + + with pytest.raises(DebputySubstitutionError): + manifest_variables_no_dch["PACKAGE"] + + dch_content = textwrap.dedent( + """\ + mscgen (1:0.20-15) unstable; urgency=medium + + * Irrelevant stuff here... + * Also, some details have been tweaked for better testing + + -- Niels Thykier Mon, 09 Oct 2023 14:50:06 +0000 + """ + ) + + debian_dir = build_virtual_file_system( + [virtual_path_def("changelog", content=dch_content)] + ) + resolution_context = manifest_variable_resolution_context(debian_dir=debian_dir) + manifest_variables = plugin.manifest_variables( + resolution_context=resolution_context + ) + + assert manifest_variables["DEB_SOURCE"] == "mscgen" + assert manifest_variables["DEB_VERSION"] == "1:0.20-15" + assert manifest_variables["_DEBPUTY_INTERNAL_NON_BINNMU_SOURCE"] == "1:0.20-15" + + assert manifest_variables["DEB_VERSION_EPOCH_UPSTREAM"] == "1:0.20" + assert manifest_variables["DEB_VERSION_UPSTREAM"] == "0.20" + assert manifest_variables["DEB_VERSION_UPSTREAM_REVISION"] == "0.20-15" + assert manifest_variables["SOURCE_DATE_EPOCH"] == "1696863006" + assert manifest_variables["_DEBPUTY_SND_SOURCE_DATE_EPOCH"] == "1696863006" + + # This one remains unresolvable + with pytest.raises(DebputySubstitutionError): + manifest_variables["PACKAGE"] + + static_values = { + "path:BASH_COMPLETION_DIR": "/usr/share/bash-completion/completions", + "path:GNU_INFO_DIR": "/usr/share/info", + } + + for k, v in static_values.items(): + assert manifest_variables[k] == v + + dch_content_bin_nmu = textwrap.dedent( + """\ + mscgen (1:0.20-15+b4) unstable; urgency=medium, binary-only=yes + + * Some binNMU entry here + + -- Niels Thykier Mon, 10 Nov 2023 16:01:17 +0000 + + mscgen (1:0.20-15) unstable; urgency=medium + + * Irrelevant stuff here... + * Also, some details have been tweaked for better testing + + -- Niels Thykier Mon, 09 Oct 2023 14:50:06 +0000 + """ + ) + + debian_dir_bin_nmu = build_virtual_file_system( + [virtual_path_def("changelog", content=dch_content_bin_nmu)] + ) + resolution_context_bin_nmu = manifest_variable_resolution_context( + debian_dir=debian_dir_bin_nmu + ) + manifest_variables_bin_nmu = plugin.manifest_variables( + resolution_context=resolution_context_bin_nmu + ) + + assert manifest_variables_bin_nmu["DEB_SOURCE"] == "mscgen" + assert manifest_variables_bin_nmu["DEB_VERSION"] == "1:0.20-15+b4" + assert ( + manifest_variables_bin_nmu["_DEBPUTY_INTERNAL_NON_BINNMU_SOURCE"] == "1:0.20-15" + ) + + assert manifest_variables_bin_nmu["DEB_VERSION_EPOCH_UPSTREAM"] == "1:0.20" + assert manifest_variables_bin_nmu["DEB_VERSION_UPSTREAM"] == "0.20" + assert manifest_variables_bin_nmu["DEB_VERSION_UPSTREAM_REVISION"] == "0.20-15+b4" + assert manifest_variables_bin_nmu["SOURCE_DATE_EPOCH"] == "1699632077" + assert manifest_variables_bin_nmu["_DEBPUTY_SND_SOURCE_DATE_EPOCH"] == "1696863006" + + +def test_cap_validator() -> None: + has_libcap, _, is_valid_cap = load_libcap() + + if not has_libcap: + if os.environ.get("DEBPUTY_REQUIRE_LIBCAP", "") != "": + pytest.fail("Could not load libcap, but DEBPUTY_REQUIRE_CAP was non-empty") + pytest.skip("Could not load libcap.so") + assert not is_valid_cap("foo") + assert is_valid_cap("cap_dac_override,cap_bpf,cap_net_admin=ep") + + +def test_clean_la_files() -> None: + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + fs_root = build_virtual_file_system( + [virtual_path_def("usr/bin/foo", content="#!/bin/sh\n")] + ) + # Does nothing by default + plugin.run_package_processor( + "clean-la-files", + fs_root, + ) + + la_file_content = textwrap.dedent( + """\ + dependency_libs = 'foo bar' + another_line = 'foo bar' + """ + ) + expected_content = textwrap.dedent( + """\ + dependency_libs = '' + another_line = 'foo bar' + """ + ) + la_file_content_no_change = expected_content + expected_content = textwrap.dedent( + """\ + dependency_libs = '' + another_line = 'foo bar' + """ + ) + + fs_root = build_virtual_file_system( + [ + virtual_path_def("usr/lib/libfoo.la", materialized_content=la_file_content), + virtual_path_def( + "usr/lib/libfoo-unchanged.la", + content=la_file_content_no_change, + ), + ] + ) + + plugin.run_package_processor( + "clean-la-files", + fs_root, + ) + for basename in ("libfoo.la", "libfoo-unchanged.la"): + la_file = fs_root.lookup(f"usr/lib/{basename}") + assert la_file is not None and la_file.is_file + if basename == "libfoo-unchanged.la": + # it should never have been rewritten + assert not la_file.has_fs_path + else: + assert la_file.has_fs_path + with la_file.open() as fd: + rewritten_content = fd.read() + assert rewritten_content == expected_content + + +def test_strip_nondeterminism() -> None: + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + + fs_root = build_virtual_file_system( + [ + # Note, we are only testing a negative example as a positive example crashes + # because we do not have a SOURCE_DATE_EPOCH value/substitution + virtual_path_def("test/not-really-a-png.png", content="Not a PNG") + ] + ) + + plugin.run_package_processor( + "strip-nondeterminism", + fs_root, + ) + + +def test_translate_capabilities() -> None: + attribute_path = AttributePath.test_path() + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + + fs_root = build_virtual_file_system([virtual_path_def("usr/bin/foo", mode=0o4755)]) + + foo = fs_root.lookup("usr/bin/foo") + assert foo is not None + assert foo.is_file + assert foo.is_read_write + + metadata_no_cap = plugin.run_metadata_detector( + "translate-capabilities", + fs_root, + ) + + assert not metadata_no_cap.maintscripts(maintscript="postinst") + + cap = foo.metadata(DebputyCapability) + assert not cap.is_present + assert cap.can_write + cap.value = DebputyCapability( + capabilities="cap_net_raw+ep", + capability_mode=SymbolicMode.parse_filesystem_mode( + "u-s", + attribute_path["cap_mode"], + ), + definition_source="test", + ) + metadata_w_cap = plugin.run_metadata_detector( + "translate-capabilities", + fs_root, + ) + + postinsts = metadata_w_cap.maintscripts(maintscript="postinst") + assert len(postinsts) == 1 + postinst = postinsts[0] + assert postinst.registration_method == "on_configure" + assert "setcap cap_net_raw+ep " in postinst.plugin_provided_script + assert "chmod u-s " in postinst.plugin_provided_script + + +def test_pam_auth_update() -> None: + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + + fs_root = build_virtual_file_system(["usr/bin/foo"]) + + empty_metadata = plugin.run_metadata_detector("pam-auth-update", fs_root) + assert not empty_metadata.maintscripts() + + fs_root = build_virtual_file_system(["/usr/share/pam-configs/foo-pam"]) + + pam_metadata = plugin.run_metadata_detector("pam-auth-update", fs_root) + postinsts = pam_metadata.maintscripts(maintscript="postinst") + assert len(postinsts) == 1 + prerms = pam_metadata.maintscripts(maintscript="prerm") + assert len(prerms) == 1 + + postinst = postinsts[0] + assert postinst.registration_method == "on_configure" + assert "pam-auth-update --package" in postinst.plugin_provided_script + + prerms = prerms[0] + assert prerms.registration_method == "on_before_removal" + assert "pam-auth-update --package --remove foo-pam" in prerms.plugin_provided_script + + +def test_auto_depends_solink() -> None: + plugin = initialize_plugin_under_test_preloaded( + 1, + initialize_debputy_features, + plugin_name="debputy", + load_debputy_plugin=False, + ) + + fs_root = build_virtual_file_system(["usr/bin/foo"]) + + empty_metadata = plugin.run_metadata_detector( + "auto-depends-arch-any-solink", + fs_root, + ) + assert "misc:Depends" not in empty_metadata.substvars + fs_root = build_virtual_file_system( + [ + "usr/lib/x86_64-linux-gnu/libfoo.la", + virtual_path_def( + "usr/lib/x86_64-linux-gnu/libfoo.so", link_target="libfoo.so.1" + ), + ] + ) + + still_empty_metadata = plugin.run_metadata_detector( + "auto-depends-arch-any-solink", + fs_root, + ) + assert "misc:Depends" not in still_empty_metadata.substvars + + libfoo1_fs_root = build_virtual_file_system( + [ + virtual_path_def( + "usr/lib/x86_64-linux-gnu/libfoo.so.1", link_target="libfoo.so.1.0.0" + ), + ] + ) + + context_correct = package_metadata_context( + package_fields={ + "Package": "libfoo-dev", + }, + accessible_package_roots=[ + ( + { + "Package": "libfoo1", + "Architecture": "any", + }, + libfoo1_fs_root, + ) + ], + ) + sodep_metadata = plugin.run_metadata_detector( + "auto-depends-arch-any-solink", + fs_root, + context=context_correct, + ) + assert "misc:Depends" in sodep_metadata.substvars + assert sodep_metadata.substvars["misc:Depends"] == "libfoo1 (= ${binary:Version})" + + context_incorrect = package_metadata_context( + package_fields={"Package": "libfoo-dev", "Architecture": "all"}, + accessible_package_roots=[ + ( + { + "Package": "foo", + "Architecture": "all", + }, + build_virtual_file_system([]), + ) + ], + ) + sodep_metadata = plugin.run_metadata_detector( + "auto-depends-arch-any-solink", + fs_root, + context=context_incorrect, + ) + assert "misc:Depends" not in sodep_metadata.substvars + + context_too_many_matches = package_metadata_context( + package_fields={"Package": "libfoo-dev"}, + accessible_package_roots=[ + ( + { + "Package": "libfoo1-a", + "Architecture": "any", + }, + libfoo1_fs_root, + ), + ( + { + "Package": "libfoo1-b", + "Architecture": "any", + }, + libfoo1_fs_root, + ), + ], + ) + sodep_metadata = plugin.run_metadata_detector( + "auto-depends-arch-any-solink", + fs_root, + context=context_too_many_matches, + ) + assert "misc:Depends" not in sodep_metadata.substvars diff --git a/tests/test_declarative_parser.py b/tests/test_declarative_parser.py new file mode 100644 index 0000000..a5061cb --- /dev/null +++ b/tests/test_declarative_parser.py @@ -0,0 +1,211 @@ +from typing import List, TypedDict, NotRequired, Annotated, Union, Mapping + +import pytest + +from debputy.highlevel_manifest import PackageTransformationDefinition +from debputy.manifest_parser.base_types import DebputyParsedContent, TypeMapping +from debputy.manifest_parser.declarative_parser import ( + DebputyParseHint, + ParserGenerator, +) +from debputy.manifest_parser.mapper_code import type_mapper_str2package +from debputy.manifest_parser.parser_data import ParserContextData +from debputy.manifest_parser.util import AttributePath +from debputy.packages import BinaryPackage +from debputy.substitution import NULL_SUBSTITUTION +from tutil import faked_binary_package + + +class TFinalEntity(DebputyParsedContent): + sources: List[str] + install_as: NotRequired[str] + into: NotRequired[List[BinaryPackage]] + recursive: NotRequired[bool] + + +class TSourceEntity(TypedDict): + sources: NotRequired[List[str]] + source: Annotated[NotRequired[str], DebputyParseHint.target_attribute("sources")] + as_: NotRequired[ + Annotated[ + str, + DebputyParseHint.target_attribute("install_as"), + DebputyParseHint.conflicts_with_source_attributes("sources"), + ] + ] + into: NotRequired[Union[BinaryPackage, List[BinaryPackage]]] + recursive: NotRequired[bool] + + +TSourceEntityAltFormat = Union[TSourceEntity, List[str], str] + + +foo_package = faked_binary_package("foo") +context_packages = { + foo_package.name: foo_package, +} +context_package_states = { + p.name: PackageTransformationDefinition( + p, + NULL_SUBSTITUTION, + False, + ) + for p in context_packages.values() +} + + +class TestParserContextData(ParserContextData): + @property + def _package_states(self) -> Mapping[str, PackageTransformationDefinition]: + return context_package_states + + @property + def binary_packages(self) -> Mapping[str, BinaryPackage]: + return context_packages + + +@pytest.fixture +def parser_context(): + return TestParserContextData() + + +@pytest.mark.parametrize( + "source_payload,expected_data,expected_attribute_path,parse_content,source_content", + [ + ( + {"sources": ["foo", "bar"]}, + {"sources": ["foo", "bar"]}, + { + "sources": "sources", + }, + TFinalEntity, + None, + ), + ( + {"sources": ["foo", "bar"], "install-as": "as-value"}, + {"sources": ["foo", "bar"], "install_as": "as-value"}, + {"sources": "sources", "install_as": "install-as"}, + TFinalEntity, + None, + ), + ( + {"sources": ["foo", "bar"], "install-as": "as-value", "into": ["foo"]}, + { + "sources": ["foo", "bar"], + "install_as": "as-value", + "into": [foo_package], + }, + {"sources": "sources", "install_as": "install-as", "into": "into"}, + TFinalEntity, + None, + ), + ( + {"source": "foo", "as": "as-value", "into": ["foo"]}, + { + "sources": ["foo"], + "install_as": "as-value", + "into": [foo_package], + }, + {"sources": "source", "install_as": "as", "into": "into"}, + TFinalEntity, + TSourceEntity, + ), + ( + {"source": "foo", "as": "as-value", "into": ["foo"]}, + { + "sources": ["foo"], + "install_as": "as-value", + "into": [foo_package], + }, + {"sources": "source", "install_as": "as", "into": "into"}, + TFinalEntity, + TSourceEntityAltFormat, + ), + ( + ["foo", "bar"], + { + "sources": ["foo", "bar"], + }, + {"sources": "parse-root"}, + TFinalEntity, + TSourceEntityAltFormat, + ), + ( + "foo", + { + "sources": ["foo"], + }, + {"sources": "parse-root"}, + TFinalEntity, + TSourceEntityAltFormat, + ), + ( + "foo", + { + "sources": ["foo"], + }, + {"sources": "parse-root"}, + TFinalEntity, + str, + ), + ( + ["foo", "bar"], + { + "sources": ["foo", "bar"], + }, + {"sources": "parse-root"}, + TFinalEntity, + List[str], + ), + ( + "foo", + { + "sources": ["foo"], + }, + {"sources": "parse-root"}, + TFinalEntity, + Union[str, List[str]], + ), + ( + ["foo", "bar"], + { + "sources": ["foo", "bar"], + }, + {"sources": "parse-root"}, + TFinalEntity, + Union[str, List[str]], + ), + ( + {"source": "foo", "recursive": True}, + { + "sources": ["foo"], + "recursive": True, + }, + {"sources": "source", "recursive": "recursive"}, + TFinalEntity, + TSourceEntityAltFormat, + ), + ], +) +def test_declarative_parser_ok( + attribute_path: AttributePath, + parser_context: ParserContextData, + source_payload, + expected_data, + expected_attribute_path, + parse_content, + source_content, +): + pg = ParserGenerator() + pg.register_mapped_type(TypeMapping(BinaryPackage, str, type_mapper_str2package)) + parser = pg.parser_from_typed_dict( + parse_content, + source_content=source_content, + ) + data_path = attribute_path["parse-root"] + parsed_data = parser.parse_input( + source_payload, data_path, parser_context=parser_context + ) + assert expected_data == parsed_data + attributes = {k: data_path[k].name for k in expected_attribute_path} + assert attributes == expected_attribute_path diff --git a/tests/test_fs_metadata.py b/tests/test_fs_metadata.py new file mode 100644 index 0000000..14a397f --- /dev/null +++ b/tests/test_fs_metadata.py @@ -0,0 +1,770 @@ +import dataclasses +import textwrap +from typing import Tuple, List, Optional, Union + +import pytest + +from debputy.filesystem_scan import PathDef, build_virtual_fs +from debputy.highlevel_manifest_parser import YAMLManifestParser +from debputy.intermediate_manifest import PathType, IntermediateManifest, TarMember +from debputy.plugin.api import virtual_path_def +from debputy.plugin.api.test_api import build_virtual_file_system +from debputy.transformation_rules import TransformationRuntimeError + + +@pytest.fixture() +def manifest_parser_pkg_foo( + amd64_dpkg_architecture_variables, + dpkg_arch_query, + source_package, + package_single_foo_arch_all_cxt_amd64, + amd64_substitution, + no_profiles_or_build_options, + debputy_plugin_feature_set, +) -> YAMLManifestParser: + # We need an empty directory to avoid triggering packager provided files. + debian_dir = build_virtual_file_system([]) + return YAMLManifestParser( + "debian/test-debputy.manifest", + source_package, + package_single_foo_arch_all_cxt_amd64, + amd64_substitution, + amd64_dpkg_architecture_variables, + dpkg_arch_query, + no_profiles_or_build_options, + debputy_plugin_feature_set, + debian_dir=debian_dir, + ) + + +@dataclasses.dataclass(slots=True, kw_only=True) +class Expected: + mtime: Optional[float] + mode: Optional[int] = None + link_target: Optional[str] = None + owner: str = "root" + group: str = "root" + has_fs_path: bool = True + + +def _show_name_on_error(_: str) -> bool: + return False + + +def _has_fs_path(tm: TarMember) -> bool: + return tm.fs_path is not None + + +def verify_paths( + intermediate_manifest: IntermediateManifest, + expected_results: List[Tuple[Union[str, PathDef], Expected]], +) -> None: + result = {tm.member_path: tm for tm in intermediate_manifest} + expected_table = { + f"./{p}" if isinstance(p, str) else f"./{p.path_name}": e + for p, e in expected_results + } + + for path_name, expected in expected_table.items(): + tm = result[path_name] + if tm.path_type == PathType.SYMLINK: + assert tm.link_target == expected.link_target or _show_name_on_error( + path_name + ) + else: + assert tm.link_target == "" or _show_name_on_error(path_name) + if expected.mode is not None: + assert oct(tm.mode) == oct(expected.mode) or _show_name_on_error(path_name) + if expected.mtime is not None: + assert tm.mtime == expected.mtime or _show_name_on_error(path_name) + assert tm.owner == expected.owner or _show_name_on_error(path_name) + assert tm.group == expected.group or _show_name_on_error(path_name) + assert _has_fs_path(tm) == expected.has_fs_path or _show_name_on_error( + path_name + ) + + del result["./"] + if len(result) != len(expected_results): + for tm in result.values(): + assert tm.member_path in expected_table + + +def test_mtime_clamp_and_builtin_dir_mode(manifest_parser_pkg_foo): + manifest = manifest_parser_pkg_foo.build_manifest() + claim_mtime_to = 255 + path_defs: List[Tuple[PathDef, Expected]] = [ + ( + virtual_path_def("usr/", mode=0o700, mtime=10, fs_path="/nowhere/usr/"), + Expected(mode=0o755, mtime=10), + ), + ( + virtual_path_def( + "usr/bin/", mode=0o2534, mtime=5000, fs_path="/nowhere/usr/bin/" + ), + Expected(mode=0o755, mtime=claim_mtime_to), + ), + ( + virtual_path_def( + "usr/bin/my-exec", + mtime=5000, + fs_path="/nowhere/usr/bin/my-exec", + link_target="../../some/where/else", + ), + # Implementation detail; symlinks do not refer to their FS path in the intermediate manifest. + Expected( + mtime=claim_mtime_to, link_target="/some/where/else", has_fs_path=False + ), + ), + ] + + fs_root = build_virtual_fs([d[0] for d in path_defs], read_write_fs=True) + + assert [p.name for p in manifest.all_packages] == ["foo"] + + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, claim_mtime_to + ) + verify_paths(intermediate_manifest, path_defs) + + +def test_transformations_create_symlink(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - create-symlink: + path: 'usr/bin/my-exec' + target: '../../some/where/else' + - create-symlink: + path: 'usr/bin/{{PACKAGE}}' + target: '/usr/lib/{{DEB_HOST_MULTIARCH}}/{{PACKAGE}}/tool' + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + claim_mtime_to = 255 + fs_root = build_virtual_fs(["./"], read_write_fs=True) + expected_results = [ + ("usr/", Expected(mode=0o755, mtime=claim_mtime_to, has_fs_path=False)), + ("usr/bin/", Expected(mode=0o755, mtime=claim_mtime_to, has_fs_path=False)), + ( + "usr/bin/my-exec", + Expected( + mtime=claim_mtime_to, link_target="/some/where/else", has_fs_path=False + ), + ), + ( + "usr/bin/foo", + Expected( + mtime=claim_mtime_to, + # Test is using a "static" dpkg-architecture, so it will always be `x86_64-linux-gnu` + link_target="../lib/x86_64-linux-gnu/foo/tool", + has_fs_path=False, + ), + ), + ] + + assert [p.name for p in manifest.all_packages] == ["foo"] + + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, claim_mtime_to + ) + verify_paths(intermediate_manifest, expected_results) + + +def test_transformations_create_symlink_replace_success(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - create-symlink: + path: 'usr/bin/my-exec' + target: '../../some/where/else' + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + clamp_mtime_to = 255 + fs_root = build_virtual_fs(["./usr/bin/my-exec"], read_write_fs=True) + expected_results = [ + ("usr/", Expected(mode=0o755, mtime=clamp_mtime_to, has_fs_path=False)), + ("usr/bin/", Expected(mode=0o755, mtime=clamp_mtime_to, has_fs_path=False)), + ( + "usr/bin/my-exec", + Expected( + mtime=clamp_mtime_to, link_target="/some/where/else", has_fs_path=False + ), + ), + ] + + assert [p.name for p in manifest.all_packages] == ["foo"] + + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, clamp_mtime_to + ) + verify_paths(intermediate_manifest, expected_results) + + +@pytest.mark.parametrize( + "replacement_rule, reason", + [ + ( + "abort-on-non-empty-directory", + "the path is a non-empty directory", + ), + ( + "error-if-directory", + "the path is a directory", + ), + ( + "error-if-exists", + "the path exists", + ), + ], +) +def test_transformations_create_symlink_replace_failure( + manifest_parser_pkg_foo, replacement_rule, reason +): + content = textwrap.dedent( + f"""\ + manifest-version: '0.1' + packages: + foo: + transformations: + - create-symlink: + path: 'usr/share/foo' + target: 'somewhere-else' + replacement-rule: {replacement_rule} + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + clamp_mtime_to = 255 + fs_root = build_virtual_fs(["./usr/share/foo/bar"], read_write_fs=True) + + assert [p.name for p in manifest.all_packages] == ["foo"] + + with pytest.raises(TransformationRuntimeError) as e_info: + manifest.apply_to_binary_staging_directory("foo", fs_root, clamp_mtime_to) + + msg = ( + f"Refusing to replace ./usr/share/foo with a symlink; {reason} and the active" + f" replacement-rule was {replacement_rule}. You can set the replacement-rule to" + ' "discard-existing", if you are not interested in the contents of ./usr/share/foo. This error' + " was triggered by packages.foo.transformations[0].create-symlink ." + ) + assert e_info.value.args[0] == msg + + +def test_transformations_create_symlink_replace_with_explicit_remove( + manifest_parser_pkg_foo, +): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - remove: usr/share/foo + - create-symlink: + path: 'usr/share/foo' + target: 'somewhere-else' + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + clamp_mtime_to = 255 + fs_root = build_virtual_fs(["./usr/share/foo/bar"], read_write_fs=True) + expected_results = [ + ("usr/", Expected(mode=0o755, mtime=clamp_mtime_to, has_fs_path=False)), + ("usr/share/", Expected(mode=0o755, mtime=clamp_mtime_to, has_fs_path=False)), + ( + "usr/share/foo", + Expected( + mtime=clamp_mtime_to, link_target="somewhere-else", has_fs_path=False + ), + ), + ] + + assert [p.name for p in manifest.all_packages] == ["foo"] + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, clamp_mtime_to + ) + verify_paths(intermediate_manifest, expected_results) + + +def test_transformations_create_symlink_replace_with_replacement_rule( + manifest_parser_pkg_foo, +): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - remove: usr/share/foo + - create-symlink: + path: 'usr/share/foo' + target: 'somewhere-else' + replacement-rule: 'discard-existing' + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + clamp_mtime_to = 255 + fs_root = build_virtual_fs(["./usr/share/foo/bar"], read_write_fs=True) + expected_results = [ + ("usr/", Expected(mode=0o755, mtime=clamp_mtime_to, has_fs_path=False)), + ("usr/share/", Expected(mode=0o755, mtime=clamp_mtime_to, has_fs_path=False)), + ( + "usr/share/foo", + Expected( + mtime=clamp_mtime_to, link_target="somewhere-else", has_fs_path=False + ), + ), + ] + + assert [p.name for p in manifest.all_packages] == ["foo"] + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, clamp_mtime_to + ) + verify_paths(intermediate_manifest, expected_results) + + +def test_transformations_path_metadata(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - path-metadata: + path: 'usr/bin/my-exec' + mode: "-x" + owner: "bin" + group: 2 + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + claim_mtime_to = 255 + fs_root = build_virtual_fs( + [ + virtual_path_def( + "./usr/bin/my-exec", fs_path="/no-where", mode=0o755, mtime=10 + ), + ], + read_write_fs=True, + ) + expected_results = [ + ("usr/", Expected(mode=0o755, mtime=claim_mtime_to, has_fs_path=False)), + ("usr/bin/", Expected(mode=0o755, mtime=claim_mtime_to, has_fs_path=False)), + ( + "usr/bin/my-exec", + Expected( + mtime=10, + has_fs_path=True, + mode=0o644, + owner="bin", + group="bin", + ), + ), + ] + + assert [p.name for p in manifest.all_packages] == ["foo"] + + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, claim_mtime_to + ) + verify_paths(intermediate_manifest, expected_results) + + +def test_transformations_directories(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - create-directories: + path: some/empty/directory + mode: "0700" + - create-directories: another/empty/directory + - create-directories: + path: a/third-empty/directory + owner: www-data + group: www-data + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + claim_mtime_to = 255 + paths = [ + virtual_path_def("some/", mtime=10, fs_path="/nowhere/some"), + virtual_path_def("some/empty/", mtime=10, fs_path="/nowhere/some/empty"), + virtual_path_def( + "some/empty/directory/", + mode=0o755, + mtime=10, + fs_path="/nowhere/some/empty/directory", + ), + ] + fs_root = build_virtual_fs(paths, read_write_fs=True) + expected_results = [ + ("some/", Expected(mode=0o755, mtime=10, has_fs_path=True)), + ("some/empty/", Expected(mode=0o755, mtime=10, has_fs_path=True)), + ( + "some/empty/directory/", + Expected(mode=0o700, mtime=10, has_fs_path=True), + ), + ("another/", Expected(mode=0o755, mtime=claim_mtime_to, has_fs_path=False)), + ( + "another/empty/", + Expected(mode=0o755, mtime=claim_mtime_to, has_fs_path=False), + ), + ( + "another/empty/directory/", + Expected(mode=0o755, mtime=claim_mtime_to, has_fs_path=False), + ), + ("a/", Expected(mode=0o755, mtime=claim_mtime_to, has_fs_path=False)), + ( + "a/third-empty/", + Expected(mode=0o755, mtime=claim_mtime_to, has_fs_path=False), + ), + ( + "a/third-empty/directory/", + Expected( + mode=0o755, + mtime=claim_mtime_to, + owner="www-data", + group="www-data", + has_fs_path=False, + ), + ), + ] + + assert [p.name for p in manifest.all_packages] == ["foo"] + + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, claim_mtime_to + ) + verify_paths(intermediate_manifest, expected_results) + + +def test_transformation_remove(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - remove: some/empty + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + claim_mtime_to = 255 + paths = [ + virtual_path_def("some/", mode=0o700, mtime=10, fs_path="/nowhere/some"), + virtual_path_def( + "some/empty/", mode=0o700, mtime=10, fs_path="/nowhere/some/empty" + ), + virtual_path_def( + "some/empty/directory/", + mode=0o755, + mtime=10, + fs_path="/nowhere/some/empty/directory", + ), + ] + fs_root = build_virtual_fs(paths, read_write_fs=True) + expected_results = [] + + assert [p.name for p in manifest.all_packages] == ["foo"] + + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, claim_mtime_to + ) + + verify_paths(intermediate_manifest, expected_results) + + +def test_transformation_remove_keep_empty(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - remove: + path: some/empty + keep-empty-parent-dirs: true + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + claim_mtime_to = 255 + paths = [ + virtual_path_def("some/", mode=0o700, mtime=10, fs_path="/nowhere/some"), + virtual_path_def( + "some/empty/", mode=0o700, mtime=10, fs_path="/nowhere/some/empty" + ), + virtual_path_def( + "some/empty/directory/", + mode=0o755, + mtime=10, + fs_path="/nowhere/some/empty/directory", + ), + ] + fs_root = build_virtual_fs(paths, read_write_fs=True) + expected_results = [ + ("some/", Expected(mode=0o755, mtime=10)), + ] + + assert [p.name for p in manifest.all_packages] == ["foo"] + + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, claim_mtime_to + ) + + verify_paths(intermediate_manifest, expected_results) + + +def test_transformation_remove_glob(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - remove: some/*.json + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + claim_mtime_to = 255 + paths = [ + virtual_path_def("some/", mode=0o700, mtime=10, fs_path="/nowhere/some"), + virtual_path_def( + "some/foo.json", + mode=0o600, + mtime=10, + ), + virtual_path_def( + "some/bar.json", + mode=0o600, + mtime=10, + ), + virtual_path_def( + "some/empty/", mode=0o700, mtime=10, fs_path="/nowhere/some/empty" + ), + virtual_path_def( + "some/bar.txt", mode=0o600, mtime=10, fs_path="/nowhere/some/bar.txt" + ), + virtual_path_def( + "some/bar.JSON", mode=0o600, mtime=10, fs_path="/nowhere/some/bar.JSON" + ), + ] + fs_root = build_virtual_fs(paths, read_write_fs=True) + expected_results = [ + ("some/", Expected(mode=0o755, mtime=10)), + ("some/empty/", Expected(mode=0o755, mtime=10)), + ("some/bar.txt", Expected(mode=0o644, mtime=10)), + # Survives because pattern is case-sensitive + ("some/bar.JSON", Expected(mode=0o644, mtime=10)), + ] + + assert [p.name for p in manifest.all_packages] == ["foo"] + + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, claim_mtime_to + ) + + verify_paths(intermediate_manifest, expected_results) + + +def test_transformation_remove_no_match(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - remove: some/non-existing-path + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + claim_mtime_to = 255 + paths = [ + virtual_path_def("some/", mode=0o700, mtime=10, fs_path="/nowhere/some"), + virtual_path_def( + "some/empty/", mode=0o700, mtime=10, fs_path="/nowhere/some/empty" + ), + virtual_path_def( + "some/empty/directory/", + mode=0o755, + mtime=10, + fs_path="/nowhere/some/empty/directory", + ), + ] + fs_root = build_virtual_fs(paths, read_write_fs=True) + assert [p.name for p in manifest.all_packages] == ["foo"] + + with pytest.raises(TransformationRuntimeError) as e_info: + manifest.apply_to_binary_staging_directory("foo", fs_root, claim_mtime_to) + expected = ( + 'The match rule "./some/non-existing-path" in transformation' + ' "packages.foo.transformations[0].remove " did not match any paths. Either' + " the definition is redundant (and can be omitted) or the match rule is incorrect." + ) + assert expected == e_info.value.args[0] + + +def test_transformation_move_basic(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - move: + source: some/dir + target: new/dir/where-else + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + claim_mtime_to = 255 + paths = [ + virtual_path_def("some/", mode=0o700, mtime=10, fs_path="/nowhere/some"), + virtual_path_def( + "some/dir/", mode=0o700, mtime=10, fs_path="/nowhere/some/empty" + ), + virtual_path_def( + "some/dir/some-dir-symlink1", mtime=10, link_target="/abs/some-target1" + ), + virtual_path_def( + "some/dir/some-dir-symlink2", mtime=10, link_target="../some-target2" + ), + virtual_path_def( + "some/dir/some-dir-symlink3", + mtime=10, + link_target="/new/dir/where-else/some-target3", + ), + ] + fs_root = build_virtual_fs(paths, read_write_fs=True) + assert [p.name for p in manifest.all_packages] == ["foo"] + + expected_results = [ + ("some/", Expected(mode=0o755, mtime=10)), + ("new/", Expected(mode=0o755, mtime=claim_mtime_to, has_fs_path=False)), + ("new/dir/", Expected(mode=0o755, mtime=claim_mtime_to, has_fs_path=False)), + ("new/dir/where-else/", Expected(mode=0o755, mtime=10)), + # FIXME: should be 10 + ( + "new/dir/where-else/some-dir-symlink1", + Expected(mtime=None, link_target="/abs/some-target1", has_fs_path=False), + ), + ( + "new/dir/where-else/some-dir-symlink2", + Expected(mtime=None, link_target="../some-target2", has_fs_path=False), + ), + ( + "new/dir/where-else/some-dir-symlink3", + Expected(mtime=None, link_target="some-target3", has_fs_path=False), + ), + ] + assert [p.name for p in manifest.all_packages] == ["foo"] + + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, claim_mtime_to + ) + + print(intermediate_manifest) + + verify_paths(intermediate_manifest, expected_results) + + +def test_transformation_move_no_match(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - move: + source: some/non-existing-path + target: some/where-else + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + claim_mtime_to = 255 + paths = [ + virtual_path_def("some/", mode=0o700, mtime=10, fs_path="/nowhere/some"), + virtual_path_def( + "some/empty/", mode=0o700, mtime=10, fs_path="/nowhere/some/empty" + ), + virtual_path_def( + "some/empty/directory/", + mode=0o755, + mtime=10, + fs_path="/nowhere/some/empty/directory", + ), + ] + fs_root = build_virtual_fs(paths, read_write_fs=True) + assert [p.name for p in manifest.all_packages] == ["foo"] + + with pytest.raises(TransformationRuntimeError) as e_info: + manifest.apply_to_binary_staging_directory("foo", fs_root, claim_mtime_to) + expected = ( + 'The match rule "./some/non-existing-path" in transformation' + ' "packages.foo.transformations[0].move " did not match any paths. Either' + " the definition is redundant (and can be omitted) or the match rule is incorrect." + ) + assert expected == e_info.value.args[0] + + +def test_builtin_mode_normalization(manifest_parser_pkg_foo): + manifest = manifest_parser_pkg_foo.build_manifest() + claim_mtime_to = 255 + sh_script_content = "#!/bin/sh" + python_script_content = "#! /usr/bin/python" + unrelated_content = "... random stuff ..." + paths = [ + virtual_path_def("some/", mode=0o700, mtime=10, fs_path="/nowhere/some"), + virtual_path_def( + "some/dir/", mode=0o700, mtime=10, fs_path="/nowhere/some/empty" + ), + virtual_path_def( + "some/dir/script.sh", + mode=0o600, + mtime=10, + fs_path="/nowhere/script.sh", + content=sh_script_content, + ), + virtual_path_def( + "some/dir/script.py", + mode=0o600, + mtime=10, + fs_path="/nowhere/script.py", + content=python_script_content, + ), + virtual_path_def( + "some/dir/non-script-file", + mode=0o600, + mtime=10, + fs_path="/nowhere/non-script-file", + content=unrelated_content, + ), + ] + fs_root = build_virtual_fs(paths, read_write_fs=True) + assert [p.name for p in manifest.all_packages] == ["foo"] + + expected_results = [ + ("some/", Expected(mode=0o755, mtime=10)), + ("some/dir/", Expected(mode=0o755, mtime=10)), + ("some/dir/script.sh", Expected(mode=0o755, mtime=10)), + ("some/dir/script.py", Expected(mode=0o755, mtime=10)), + ("some/dir/non-script-file", Expected(mode=0o644, mtime=10)), + ] + assert [p.name for p in manifest.all_packages] == ["foo"] + + intermediate_manifest = manifest.apply_to_binary_staging_directory( + "foo", fs_root, claim_mtime_to + ) + + print(intermediate_manifest) + + verify_paths(intermediate_manifest, expected_results) diff --git a/tests/test_install_rules.py b/tests/test_install_rules.py new file mode 100644 index 0000000..c8ffb84 --- /dev/null +++ b/tests/test_install_rules.py @@ -0,0 +1,1059 @@ +import textwrap + +import pytest + +from debputy.highlevel_manifest_parser import YAMLManifestParser +from debputy.installations import ( + InstallSearchDirContext, + NoMatchForInstallPatternError, + SearchDir, +) +from debputy.plugin.api import virtual_path_def +from debputy.plugin.api.test_api import build_virtual_file_system + + +@pytest.fixture() +def manifest_parser_pkg_foo( + amd64_dpkg_architecture_variables, + dpkg_arch_query, + source_package, + package_single_foo_arch_all_cxt_amd64, + amd64_substitution, + no_profiles_or_build_options, + debputy_plugin_feature_set, +) -> YAMLManifestParser: + # We need an empty directory to avoid triggering packager provided files. + debian_dir = build_virtual_file_system([]) + return YAMLManifestParser( + "debian/test-debputy.manifest", + source_package, + package_single_foo_arch_all_cxt_amd64, + amd64_substitution, + amd64_dpkg_architecture_variables, + dpkg_arch_query, + no_profiles_or_build_options, + debputy_plugin_feature_set, + debian_dir=debian_dir, + ) + + +@pytest.fixture() +def manifest_parser_pkg_foo_w_udeb( + amd64_dpkg_architecture_variables, + dpkg_arch_query, + source_package, + package_foo_w_udeb_arch_any_cxt_amd64, + amd64_substitution, + no_profiles_or_build_options, + debputy_plugin_feature_set, +) -> YAMLManifestParser: + # We need an empty directory to avoid triggering packager provided files. + debian_dir = build_virtual_file_system([]) + return YAMLManifestParser( + "debian/test-debputy.manifest", + source_package, + package_foo_w_udeb_arch_any_cxt_amd64, + amd64_substitution, + amd64_dpkg_architecture_variables, + dpkg_arch_query, + no_profiles_or_build_options, + debputy_plugin_feature_set, + debian_dir=debian_dir, + ) + + +def test_install_rules(manifest_parser_pkg_foo) -> None: + debian_source_root_dir = build_virtual_file_system( + [virtual_path_def(".", fs_path="/nowhere")] + ) + debian_tmp_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere/debian/tmp"), + virtual_path_def("usr/", fs_path="/nowhere/debian/tmp/usr"), + virtual_path_def("usr/bin/", fs_path="/nowhere/debian/tmp/usr/bin"), + virtual_path_def( + "usr/bin/foo", + fs_path="/nowhere/debian/tmp/usr/bin/foo", + content="#!/bin/sh\n", + mtime=10, + ), + virtual_path_def( + "usr/bin/foo-util", + fs_path="/nowhere/debian/tmp/usr/bin/foo-util", + content="#!/bin/sh\n", + mtime=10, + ), + virtual_path_def( + "usr/bin/tool.sh", + fs_path="/nowhere/debian/tmp/usr/bin/tool.sh", + link_target="./foo", + ), + virtual_path_def("usr/share/", fs_path="/nowhere/debian/tmp/usr/share"), + virtual_path_def( + "usr/share/foo/", fs_path="/nowhere/debian/tmp/usr/share/foo" + ), + virtual_path_def( + "usr/share/foo/foo.txt", + fs_path="/nowhere/debian/tmp/usr/share/foo/foo.txt", + content="A text file", + ), + ] + ) + manifest_content = textwrap.dedent( + """\ + manifest-version: "0.1" + installations: + - install: + - /usr/share/foo + - /usr/bin/foo + - /usr/bin/foo-util + - install: + source: usr/bin/tool.sh + as: usr/bin/tool + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=manifest_content) + all_pkgs = frozenset(manifest.all_packages) + + result = manifest.perform_installations( + install_request_context=InstallSearchDirContext( + [ + SearchDir(debian_tmp_dir, all_pkgs), + SearchDir(debian_source_root_dir, all_pkgs), + ], + [debian_tmp_dir], + ) + ) + assert "foo" in result + foo_fs_root = result["foo"].fs_root + + ub_dir = foo_fs_root.lookup("/usr/bin") + assert ub_dir is not None + assert ub_dir.is_dir + assert not ub_dir.has_fs_path # This will be "generated" + + tool = ub_dir.get("tool") + assert tool is not None + assert tool.is_symlink + assert tool.readlink() == "./foo" + + assert {"foo", "foo-util", "tool"} == {p.name for p in ub_dir.iterdir} + for n in ["foo", "foo-util"]: + assert ub_dir[n].mtime == 10 + usf_dir = foo_fs_root.lookup("/usr/share/foo") + assert usf_dir is not None + assert usf_dir.is_dir + # Here we are installing an actual directory, so it should be present too + assert usf_dir.has_fs_path + assert usf_dir.fs_path == "/nowhere/debian/tmp/usr/share/foo" + assert {"foo.txt"} == {p.name for p in usf_dir.iterdir} + + +def test_multi_dest_install_rules(manifest_parser_pkg_foo) -> None: + debian_source_root_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere"), + virtual_path_def("source/", fs_path="/nowhere/source"), + virtual_path_def("source/foo/", fs_path="/nowhere/foo"), + virtual_path_def( + "source/foo/foo-a.data", + fs_path="/nowhere/foo/foo-a.data", + content="data file", + ), + virtual_path_def( + "source/foo/foo-b.data", + fs_path="/nowhere/foo/foo-b.data", + link_target="./foo-a.data", + ), + virtual_path_def("source/bar/", fs_path="/nowhere/bar"), + virtual_path_def( + "source/bar/bar-a.data", + fs_path="/nowhere/bar/bar-a.data", + content="data file", + ), + virtual_path_def( + "source/bar/bar-b.data", + fs_path="/nowhere/bar/bar-b.data", + content="data file", + ), + virtual_path_def( + "source/tool.sh", + fs_path="/nowhere/source/tool.sh", + content="#!/bin/sh\n# run some command ...", + ), + ] + ) + manifest_content = textwrap.dedent( + """\ + manifest-version: "0.1" + installations: + - multi-dest-install: + sources: + - source/foo/* + - source/bar + dest-dirs: + - usr/share/foo + - usr/share/foo2 + - multi-dest-install: + source: source/tool.sh + as: + - usr/share/foo/tool + - usr/share/foo2/tool + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=manifest_content) + all_pkgs = frozenset(manifest.all_packages) + + result = manifest.perform_installations( + install_request_context=InstallSearchDirContext( + [ + SearchDir(debian_source_root_dir, all_pkgs), + ], + [], + ) + ) + assert "foo" in result + foo_fs_root = result["foo"].fs_root + + for stem in ["foo", "foo2"]: + foo_dir = foo_fs_root.lookup(f"/usr/share/{stem}") + assert foo_dir is not None + assert foo_dir.is_dir + + assert {"foo-a.data", "foo-b.data", "bar", "tool"} == { + p.name for p in foo_dir.iterdir + } + + tool = foo_dir["tool"] + assert tool.is_file + with tool.open() as fd: + content = fd.read() + assert content.startswith("#!/bin/sh") + foo_a = foo_dir["foo-a.data"] + assert foo_a.is_file + assert foo_a.fs_path == "/nowhere/foo/foo-a.data" + with foo_a.open() as fd: + content = fd.read() + assert "data" in content + foo_b = foo_dir["foo-b.data"] + assert foo_b.is_symlink + assert foo_b.readlink() == "./foo-a.data" + + bar = foo_dir["bar"] + assert bar.is_dir + assert {"bar-a.data", "bar-b.data"} == {p.name for p in bar.iterdir} + assert {"/nowhere/bar/bar-a.data", "/nowhere/bar/bar-b.data"} == { + p.fs_path for p in bar.iterdir + } + + +def test_install_rules_with_glob(manifest_parser_pkg_foo) -> None: + debian_source_root_dir = build_virtual_file_system( + [virtual_path_def(".", fs_path="/nowhere")] + ) + debian_tmp_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere/debian/tmp"), + virtual_path_def("usr/", fs_path="/nowhere/debian/tmp/usr"), + virtual_path_def("usr/bin/", fs_path="/nowhere/debian/tmp/usr/bin"), + virtual_path_def( + "usr/bin/foo", + fs_path="/nowhere/debian/tmp/usr/bin/foo", + content="#!/bin/sh\n", + ), + virtual_path_def( + "usr/bin/foo-util", + fs_path="/nowhere/debian/tmp/usr/bin/foo-util", + content="#!/bin/sh\n", + ), + virtual_path_def( + "usr/bin/tool.sh", + fs_path="/nowhere/debian/tmp/usr/bin/tool.sh", + link_target="./foo", + ), + virtual_path_def("usr/share/", fs_path="/nowhere/debian/tmp/usr/share"), + virtual_path_def( + "usr/share/foo/", fs_path="/nowhere/debian/tmp/usr/share/foo" + ), + virtual_path_def( + "usr/share/foo/foo.txt", + fs_path="/nowhere/debian/tmp/usr/share/foo/foo.txt", + content="A text file", + ), + ] + ) + manifest_content = textwrap.dedent( + """\ + manifest-version: "0.1" + installations: + - install: + source: usr/bin/tool.sh + as: usr/bin/tool + - install: + - /usr/share/foo + - /usr/bin/foo* + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=manifest_content) + all_pkgs = frozenset(manifest.all_packages) + + result = manifest.perform_installations( + install_request_context=InstallSearchDirContext( + [ + SearchDir(debian_tmp_dir, all_pkgs), + SearchDir(debian_source_root_dir, all_pkgs), + ], + [debian_tmp_dir], + ) + ) + assert "foo" in result + foo_fs_root = result["foo"].fs_root + + ub_dir = foo_fs_root.lookup("/usr/bin") + assert ub_dir is not None + assert ub_dir.is_dir + assert not ub_dir.has_fs_path # This will be "generated" + + tool = ub_dir.get("tool") + assert tool is not None + assert tool.is_symlink + assert tool.readlink() == "./foo" + + assert {"foo", "foo-util", "tool"} == {p.name for p in ub_dir.iterdir} + usf_dir = foo_fs_root.lookup("/usr/share/foo") + assert usf_dir is not None + assert usf_dir.is_dir + # Here we are installing an actual directory, so it should be present too + assert usf_dir.has_fs_path + assert usf_dir.fs_path == "/nowhere/debian/tmp/usr/share/foo" + assert {"foo.txt"} == {p.name for p in usf_dir.iterdir} + + +def test_install_rules_auto_discard_rules_dir(manifest_parser_pkg_foo) -> None: + debian_source_root_dir = build_virtual_file_system( + [virtual_path_def(".", fs_path="/nowhere")] + ) + debian_tmp_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere/debian/tmp"), + virtual_path_def("usr/", fs_path="/nowhere/debian/tmp/usr"), + virtual_path_def("usr/lib/", fs_path="/nowhere/debian/tmp/usr/lib"), + virtual_path_def( + "usr/lib/libfoo.so.1.0.0", + fs_path="/nowhere/debian/tmp/usr/lib/libfoo.so.1.0.0", + content="Not really an ELF FILE", + ), + virtual_path_def( + "usr/lib/libfoo.la", + fs_path="/nowhere/debian/tmp/usr/lib/libfoo.la", + content="Not really a LA FILE", + ), + ] + ) + manifest_content = textwrap.dedent( + """\ + manifest-version: "0.1" + installations: + - install: + - /usr/lib + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=manifest_content) + + all_pkgs = frozenset(manifest.all_packages) + + result = manifest.perform_installations( + install_request_context=InstallSearchDirContext( + [ + SearchDir(debian_tmp_dir, all_pkgs), + SearchDir(debian_source_root_dir, all_pkgs), + ], + [debian_tmp_dir], + ) + ) + assert "foo" in result + foo_fs_root = result["foo"].fs_root + + lib_dir = foo_fs_root.lookup("/usr/lib") + assert lib_dir is not None + assert lib_dir.is_dir + assert lib_dir.has_fs_path + assert lib_dir.fs_path == "/nowhere/debian/tmp/usr/lib" + + so_file = lib_dir.get("libfoo.so.1.0.0") + assert so_file is not None + assert so_file.is_file + assert so_file.has_fs_path + assert so_file.fs_path == "/nowhere/debian/tmp/usr/lib/libfoo.so.1.0.0" + + assert {"libfoo.so.1.0.0"} == {p.name for p in lib_dir.iterdir} + + +def test_install_rules_auto_discard_rules_glob(manifest_parser_pkg_foo) -> None: + debian_source_root_dir = build_virtual_file_system( + [virtual_path_def(".", fs_path="/nowhere")] + ) + debian_tmp_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere/debian/tmp"), + virtual_path_def("usr/", fs_path="/nowhere/debian/tmp/usr"), + virtual_path_def("usr/lib/", fs_path="/nowhere/debian/tmp/usr/lib"), + virtual_path_def( + "usr/lib/libfoo.so.1.0.0", + fs_path="/nowhere/debian/tmp/usr/lib/libfoo.so.1.0.0", + content="Not really an ELF FILE", + ), + virtual_path_def( + "usr/lib/libfoo.la", + fs_path="/nowhere/debian/tmp/usr/lib/libfoo.la", + content="Not really an ELF FILE", + ), + ] + ) + manifest_content = textwrap.dedent( + """\ + manifest-version: "0.1" + installations: + - install: + - /usr/lib/* + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=manifest_content) + all_pkgs = frozenset(manifest.all_packages) + + result = manifest.perform_installations( + install_request_context=InstallSearchDirContext( + [ + SearchDir(debian_tmp_dir, all_pkgs), + SearchDir(debian_source_root_dir, all_pkgs), + ], + [debian_tmp_dir], + ) + ) + assert "foo" in result + foo_fs_root = result["foo"].fs_root + + lib_dir = foo_fs_root.lookup("/usr/lib") + assert lib_dir is not None + assert lib_dir.is_dir + assert not lib_dir.has_fs_path + + so_file = lib_dir.get("libfoo.so.1.0.0") + assert so_file is not None + assert so_file.is_file + assert so_file.has_fs_path + assert so_file.fs_path == "/nowhere/debian/tmp/usr/lib/libfoo.so.1.0.0" + + assert {"libfoo.so.1.0.0"} == {p.name for p in lib_dir.iterdir} + + +def test_install_rules_auto_discard_rules_overruled_by_explicit_install_rule( + manifest_parser_pkg_foo, +) -> None: + debian_source_root_dir = build_virtual_file_system( + [virtual_path_def(".", fs_path="/nowhere")] + ) + debian_tmp_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere/debian/tmp"), + virtual_path_def("usr/", fs_path="/nowhere/debian/tmp/usr"), + virtual_path_def("usr/lib/", fs_path="/nowhere/debian/tmp/usr/lib"), + virtual_path_def( + "usr/lib/libfoo.so.1.0.0", + fs_path="/nowhere/debian/tmp/usr/lib/libfoo.so.1.0.0", + content="Not really an ELF FILE", + ), + virtual_path_def( + "usr/lib/libfoo.la", + fs_path="/nowhere/debian/tmp/usr/lib/libfoo.la", + content="Not really an ELF FILE", + ), + virtual_path_def( + "usr/lib/libfoo.so", + fs_path="/nowhere/debian/tmp/usr/lib/libfoo.so", + link_target="libfoo.so.1.0.0", + ), + ] + ) + manifest_content = textwrap.dedent( + """\ + manifest-version: "0.1" + installations: + - install: + - /usr/lib + - /usr/lib/libfoo.la + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=manifest_content) + all_pkgs = frozenset(manifest.all_packages) + + result = manifest.perform_installations( + install_request_context=InstallSearchDirContext( + [ + SearchDir(debian_tmp_dir, all_pkgs), + SearchDir(debian_source_root_dir, all_pkgs), + ], + [debian_tmp_dir], + ) + ) + assert "foo" in result + foo_fs_root = result["foo"].fs_root + + lib_dir = foo_fs_root.lookup("/usr/lib") + assert lib_dir is not None + assert lib_dir.is_dir + assert lib_dir.has_fs_path + assert lib_dir.fs_path == "/nowhere/debian/tmp/usr/lib" + + so_file = lib_dir.get("libfoo.so.1.0.0") + assert so_file is not None + assert so_file.is_file + assert so_file.has_fs_path + assert so_file.fs_path == "/nowhere/debian/tmp/usr/lib/libfoo.so.1.0.0" + + la_file = lib_dir.get("libfoo.la") + assert la_file is not None + assert la_file.is_file + assert la_file.has_fs_path + assert la_file.fs_path == "/nowhere/debian/tmp/usr/lib/libfoo.la" + + so_link = lib_dir.get("libfoo.so") + assert so_link is not None + assert so_link.is_symlink + assert so_link.readlink() == "libfoo.so.1.0.0" + + assert {"libfoo.so.1.0.0", "libfoo.so", "libfoo.la"} == { + p.name for p in lib_dir.iterdir + } + + +def test_install_rules_install_as_with_var(manifest_parser_pkg_foo) -> None: + debian_source_root_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere"), + virtual_path_def("build/", fs_path="/nowhere/build"), + virtual_path_def( + "build/private-arch-tool.sh", + fs_path="/nowhere/build/private-arch-tool.sh", + content="#!/bin/sh\n", + ), + ] + ) + debian_tmp_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere/debian/tmp"), + ] + ) + manifest_content = textwrap.dedent( + """\ + manifest-version: "0.1" + installations: + - install: + source: build/private-arch-tool.sh + as: /usr/lib/{{DEB_HOST_MULTIARCH}}/foo/private-arch-tool + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=manifest_content) + all_pkgs = frozenset(manifest.all_packages) + + result = manifest.perform_installations( + install_request_context=InstallSearchDirContext( + [ + SearchDir(debian_tmp_dir, all_pkgs), + SearchDir(debian_source_root_dir, all_pkgs), + ], + [debian_tmp_dir], + ) + ) + assert "foo" in result + foo_fs_root = result["foo"].fs_root + + # The variable is always resolved in amd64 context, so we can hard code the resolved + # variable + tool = foo_fs_root.lookup("/usr/lib/x86_64-linux-gnu/foo/private-arch-tool") + assert tool is not None + assert tool.is_file + assert tool.fs_path == "/nowhere/build/private-arch-tool.sh" + + +def test_install_rules_no_matches(manifest_parser_pkg_foo) -> None: + debian_source_root_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere"), + virtual_path_def("build/", fs_path="/nowhere/build"), + virtual_path_def( + "build/private-arch-tool.sh", + fs_path="/nowhere/build/private-arch-tool.sh", + content="#!/bin/sh\n", + ), + ] + ) + debian_tmp_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere/debian/tmp"), + ] + ) + manifest_content = textwrap.dedent( + """\ + manifest-version: "0.1" + installations: + - install: + # Typo: the path should have ended with ".sh" + source: build/private-arch-tool + as: /usr/lib/foo/private-arch-tool + """ + ) + manifest = manifest_parser_pkg_foo.parse_manifest(fd=manifest_content) + all_pkgs = frozenset(manifest.all_packages) + + with pytest.raises(NoMatchForInstallPatternError) as e_info: + manifest.perform_installations( + install_request_context=InstallSearchDirContext( + [ + SearchDir(debian_tmp_dir, all_pkgs), + SearchDir(debian_source_root_dir, all_pkgs), + ], + [debian_tmp_dir], + ) + ) + expected_msg = ( + "There were no matches for build/private-arch-tool in /nowhere/debian/tmp, /nowhere" + " (definition: installations[0].install )." + " Match rule: ./build/private-arch-tool (the exact path / no globbing)" + ) + assert e_info.value.message == expected_msg + + +def test_install_rules_per_package_search_dirs(manifest_parser_pkg_foo_w_udeb) -> None: + debian_source_root_dir = build_virtual_file_system( + [virtual_path_def(".", fs_path="/nowhere")] + ) + debian_tmp_deb_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere/debian/tmp-deb"), + virtual_path_def("usr/", fs_path="/nowhere/debian/tmp-deb/usr"), + virtual_path_def("usr/bin/", fs_path="/nowhere/debian/tmp-deb/usr/bin"), + virtual_path_def( + "usr/bin/foo", + fs_path="/nowhere/debian/tmp-deb/usr/bin/foo", + content="#!/bin/sh\ndeb", + ), + virtual_path_def( + "usr/bin/foo-util", + fs_path="/nowhere/debian/tmp-deb/usr/bin/foo-util", + content="#!/bin/sh\ndeb", + ), + virtual_path_def( + "usr/bin/tool.sh", + fs_path="/nowhere/debian/tmp-deb/usr/bin/tool.sh", + link_target="./foo", + ), + virtual_path_def("usr/share/", fs_path="/nowhere/debian/tmp-deb/usr/share"), + virtual_path_def( + "usr/share/foo/", fs_path="/nowhere/debian/tmp-deb/usr/share/foo" + ), + virtual_path_def( + "usr/share/foo/foo.txt", + fs_path="/nowhere/debian/tmp-deb/usr/share/foo/foo.txt", + content="A deb text file", + ), + ] + ) + debian_tmp_udeb_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere/debian/tmp-udeb"), + virtual_path_def("usr/", fs_path="/nowhere/debian/tmp-udeb/usr"), + virtual_path_def("usr/bin/", fs_path="/nowhere/debian/tmp-udeb/usr/bin"), + virtual_path_def( + "usr/bin/foo", + fs_path="/nowhere/debian/tmp-udeb/usr/bin/foo", + content="#!/bin/sh\nudeb", + ), + virtual_path_def( + "usr/bin/foo-util", + fs_path="/nowhere/debian/tmp-udeb/usr/bin/foo-util", + content="#!/bin/sh\nudeb", + ), + virtual_path_def( + "usr/bin/tool.sh", + fs_path="/nowhere/debian/tmp-udeb/usr/bin/tool.sh", + link_target="./foo", + ), + virtual_path_def( + "usr/share/", fs_path="/nowhere/debian/tmp-udeb/usr/share" + ), + virtual_path_def( + "usr/share/foo/", fs_path="/nowhere/debian/tmp-udeb/usr/share/foo" + ), + virtual_path_def( + "usr/share/foo/foo.txt", + fs_path="/nowhere/debian/tmp-udeb/usr/share/foo/foo.txt", + content="A udeb text file", + ), + ] + ) + manifest_content = textwrap.dedent( + """\ + manifest-version: "0.1" + installations: + - install: + source: usr/bin/tool.sh + as: usr/bin/tool + into: + - foo + - foo-udeb + - install: + sources: + - /usr/share/foo + - /usr/bin/foo* + into: + - foo + - foo-udeb + """ + ) + manifest = manifest_parser_pkg_foo_w_udeb.parse_manifest(fd=manifest_content) + all_pkgs = frozenset(manifest.all_packages) + all_deb_pkgs = frozenset({p for p in all_pkgs if not p.is_udeb}) + all_udeb_pkgs = frozenset({p for p in all_pkgs if p.is_udeb}) + + result = manifest.perform_installations( + install_request_context=InstallSearchDirContext( + [ + SearchDir(debian_tmp_deb_dir, all_deb_pkgs), + SearchDir(debian_tmp_udeb_dir, all_udeb_pkgs), + SearchDir(debian_source_root_dir, all_pkgs), + ], + [debian_tmp_deb_dir], + ) + ) + for pkg, ptype in [ + ("foo", "deb"), + ("foo-udeb", "udeb"), + ]: + assert pkg in result + fs_root = result[pkg].fs_root + + ub_dir = fs_root.lookup("/usr/bin") + assert ub_dir is not None + assert ub_dir.is_dir + assert not ub_dir.has_fs_path # This will be "generated" + + tool = ub_dir.get("tool") + assert tool is not None + assert tool.is_symlink + assert tool.readlink() == "./foo" + + assert {"foo", "foo-util", "tool"} == {p.name for p in ub_dir.iterdir} + + for p in ub_dir.iterdir: + assert p.has_fs_path + assert f"/nowhere/debian/tmp-{ptype}/usr/bin" in p.fs_path + + usf_dir = fs_root.lookup("/usr/share/foo") + assert usf_dir is not None + assert usf_dir.is_dir + # Here we are installing an actual directory, so it should be present too + assert usf_dir.has_fs_path + assert usf_dir.fs_path == f"/nowhere/debian/tmp-{ptype}/usr/share/foo" + assert {"foo.txt"} == {p.name for p in usf_dir.iterdir} + foo_txt = usf_dir["foo.txt"] + assert foo_txt.fs_path == f"/nowhere/debian/tmp-{ptype}/usr/share/foo/foo.txt" + + +def test_install_rules_multi_into(manifest_parser_pkg_foo_w_udeb) -> None: + debian_source_root_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere"), + virtual_path_def("source/", fs_path="/nowhere/source"), + virtual_path_def("source/foo/", fs_path="/nowhere/foo"), + virtual_path_def( + "source/foo/foo-a.data", + fs_path="/nowhere/foo/foo-a.data", + content="data file", + ), + virtual_path_def( + "source/foo/foo-b.data", + fs_path="/nowhere/foo/foo-b.data", + link_target="./foo-a.data", + ), + virtual_path_def("source/bar/", fs_path="/nowhere/bar"), + virtual_path_def( + "source/bar/bar-a.data", + fs_path="/nowhere/bar/bar-a.data", + content="data file", + ), + virtual_path_def( + "source/bar/bar-b.data", + fs_path="/nowhere/bar/bar-b.data", + content="data file", + ), + virtual_path_def( + "source/tool.sh", + fs_path="/nowhere/source/tool.sh", + content="#!/bin/sh\n# run some command ...", + ), + ] + ) + manifest_content = textwrap.dedent( + """\ + manifest-version: "0.1" + installations: + - install: + sources: + - source/foo/* + - source/bar + dest-dir: usr/share/foo + into: + - foo + - foo-udeb + - install: + source: source/tool.sh + as: usr/share/foo/tool + into: + - foo + - foo-udeb + """ + ) + manifest = manifest_parser_pkg_foo_w_udeb.parse_manifest(fd=manifest_content) + all_pkgs = frozenset(manifest.all_packages) + + result = manifest.perform_installations( + install_request_context=InstallSearchDirContext( + [ + SearchDir(debian_source_root_dir, all_pkgs), + ], + [], + ) + ) + for pkg in ["foo", "foo-udeb"]: + assert pkg in result + foo_fs_root = result[pkg].fs_root + + foo_dir = foo_fs_root.lookup("/usr/share/foo") + assert foo_dir is not None + assert foo_dir.is_dir + + assert {"foo-a.data", "foo-b.data", "bar", "tool"} == { + p.name for p in foo_dir.iterdir + } + + tool = foo_dir["tool"] + assert tool.is_file + with tool.open() as fd: + content = fd.read() + assert content.startswith("#!/bin/sh") + foo_a = foo_dir["foo-a.data"] + assert foo_a.is_file + assert foo_a.fs_path == "/nowhere/foo/foo-a.data" + with foo_a.open() as fd: + content = fd.read() + assert "data" in content + foo_b = foo_dir["foo-b.data"] + assert foo_b.is_symlink + assert foo_b.readlink() == "./foo-a.data" + + bar = foo_dir["bar"] + assert bar.is_dir + assert {"bar-a.data", "bar-b.data"} == {p.name for p in bar.iterdir} + assert {"/nowhere/bar/bar-a.data", "/nowhere/bar/bar-b.data"} == { + p.fs_path for p in bar.iterdir + } + + +def test_auto_install_d_pkg(manifest_parser_pkg_foo_w_udeb) -> None: + debian_source_root_dir = build_virtual_file_system( + [virtual_path_def(".", fs_path="/nowhere")] + ) + debian_foo_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere/debian/foo"), + virtual_path_def("usr/", fs_path="/nowhere/debian/foo/usr"), + virtual_path_def("usr/bin/", fs_path="/nowhere/debian/foo/usr/bin"), + virtual_path_def( + "usr/bin/foo", + fs_path="/nowhere/debian/foo/usr/bin/foo", + content="#!/bin/sh\ndeb", + ), + virtual_path_def( + "usr/bin/foo-util", + fs_path="/nowhere/debian/foo/usr/bin/foo-util", + content="#!/bin/sh\ndeb", + ), + virtual_path_def( + "usr/bin/tool", + fs_path="/nowhere/debian/foo/usr/bin/tool", + link_target="./foo", + ), + virtual_path_def("usr/share/", fs_path="/nowhere/debian/foo/usr/share"), + virtual_path_def( + "usr/share/foo/", fs_path="/nowhere/debian/foo/usr/share/foo" + ), + virtual_path_def( + "usr/share/foo/foo.txt", + fs_path="/nowhere/debian/foo/usr/share/foo/foo.txt", + content="A deb text file", + ), + ] + ) + debian_foo_udeb_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere/debian/foo-udeb"), + virtual_path_def("usr/", fs_path="/nowhere/debian/foo-udeb/usr"), + virtual_path_def("usr/bin/", fs_path="/nowhere/debian/foo-udeb/usr/bin"), + virtual_path_def( + "usr/bin/foo", + fs_path="/nowhere/debian/foo-udeb/usr/bin/foo", + content="#!/bin/sh\nudeb", + ), + virtual_path_def( + "usr/bin/foo-util", + fs_path="/nowhere/debian/foo-udeb/usr/bin/foo-util", + content="#!/bin/sh\nudeb", + ), + virtual_path_def( + "usr/bin/tool", + fs_path="/nowhere/debian/foo-udeb/usr/bin/tool", + link_target="./foo", + ), + virtual_path_def( + "usr/share/", fs_path="/nowhere/debian/foo-udeb/usr/share" + ), + virtual_path_def( + "usr/share/foo/", fs_path="/nowhere/debian/foo-udeb/usr/share/foo" + ), + virtual_path_def( + "usr/share/foo/foo.txt", + fs_path="/nowhere/debian/foo-udeb/usr/share/foo/foo.txt", + content="A udeb text file", + ), + ] + ) + manifest_content = textwrap.dedent( + """\ + manifest-version: "0.1" + """ + ) + manifest = manifest_parser_pkg_foo_w_udeb.parse_manifest(fd=manifest_content) + all_pkgs = frozenset(manifest.all_packages) + + result = manifest.perform_installations( + install_request_context=InstallSearchDirContext( + [ + SearchDir(debian_source_root_dir, all_pkgs), + ], + [debian_foo_dir], + { + "foo": debian_foo_dir, + "foo-udeb": debian_foo_udeb_dir, + }, + ) + ) + for pkg in ["foo", "foo-udeb"]: + assert pkg in result + fs_root = result[pkg].fs_root + ub_dir = fs_root.lookup("/usr/bin") + assert ub_dir is not None + assert ub_dir.is_dir + assert ub_dir.has_fs_path + assert ub_dir.fs_path == f"/nowhere/debian/{pkg}/usr/bin" + + assert {"foo", "foo-util", "tool"} == {p.name for p in ub_dir.iterdir} + + tool = ub_dir.get("tool") + assert tool is not None + assert tool.is_symlink + assert tool.readlink() == "./foo" + + for p in ub_dir.iterdir: + assert p.has_fs_path + assert f"/nowhere/debian/{pkg}/usr/bin" in p.fs_path + + usf_dir = fs_root.lookup("/usr/share/foo") + assert usf_dir is not None + assert usf_dir.is_dir + # Here we are installing an actual directory, so it should be present too + assert usf_dir.has_fs_path + assert usf_dir.fs_path == f"/nowhere/debian/{pkg}/usr/share/foo" + assert {"foo.txt"} == {p.name for p in usf_dir.iterdir} + foo_txt = usf_dir["foo.txt"] + assert foo_txt.fs_path == f"/nowhere/debian/{pkg}/usr/share/foo/foo.txt" + + +def test_install_doc_rules_ignore_udeb(manifest_parser_pkg_foo_w_udeb) -> None: + debian_source_root_dir = build_virtual_file_system( + [ + virtual_path_def(".", fs_path="/nowhere"), + virtual_path_def("source/", fs_path="/nowhere/source"), + virtual_path_def("source/foo/", fs_path="/nowhere/foo"), + virtual_path_def( + "source/foo/foo-a.txt", + fs_path="/nowhere/foo/foo-a.txt", + content="data file", + ), + virtual_path_def( + "source/foo/foo-b.txt", + fs_path="/nowhere/foo/foo-b.txt", + link_target="./foo-a.txt", + ), + virtual_path_def("source/html/", fs_path="/nowhere/html"), + virtual_path_def( + "source/html/bar-a.html", + fs_path="/nowhere/html/bar-a.html", + content="data file", + ), + virtual_path_def( + "source/html/bar-b.html", + fs_path="/nowhere/html/bar-b.html", + content="data file", + ), + ] + ) + manifest_content = textwrap.dedent( + """\ + manifest-version: "0.1" + installations: + - install-doc: + sources: + - source/foo/* + - source/html + """ + ) + manifest = manifest_parser_pkg_foo_w_udeb.parse_manifest(fd=manifest_content) + all_pkgs = frozenset(manifest.all_packages) + + result = manifest.perform_installations( + install_request_context=InstallSearchDirContext( + [ + SearchDir(debian_source_root_dir, all_pkgs), + ], + [], + ) + ) + assert "foo" in result + foo_fs_root = result["foo"].fs_root + + foo_dir = foo_fs_root.lookup("/usr/share/doc/foo") + assert foo_dir is not None + assert foo_dir.is_dir + + assert {"foo-a.txt", "foo-b.txt", "html"} == {p.name for p in foo_dir.iterdir} + + foo_a = foo_dir["foo-a.txt"] + assert foo_a.is_file + assert foo_a.fs_path == "/nowhere/foo/foo-a.txt" + foo_b = foo_dir["foo-b.txt"] + assert foo_b.is_symlink + assert foo_b.readlink() == "./foo-a.txt" + + html_dir = foo_dir["html"] + assert html_dir.is_dir + assert {"bar-a.html", "bar-b.html"} == {p.name for p in html_dir.iterdir} + assert {"/nowhere/html/bar-a.html", "/nowhere/html/bar-b.html"} == { + p.fs_path for p in html_dir.iterdir + } + + assert "foo-udeb" in result + foo_udeb_fs_root = result["foo-udeb"].fs_root + + udeb_doc_dir = foo_udeb_fs_root.lookup("/usr/share/doc") + assert udeb_doc_dir is None diff --git a/tests/test_interpreter.py b/tests/test_interpreter.py new file mode 100644 index 0000000..6cbfd44 --- /dev/null +++ b/tests/test_interpreter.py @@ -0,0 +1,162 @@ +import textwrap +from typing import Optional + +import pytest + +from debputy.highlevel_manifest import HighLevelManifest +from debputy.highlevel_manifest_parser import YAMLManifestParser +from debputy.interpreter import extract_shebang_interpreter +from debputy.plugin.api import virtual_path_def +from debputy.plugin.api.test_api import build_virtual_file_system +from debputy.transformation_rules import NormalizeShebangLineTransformation + + +@pytest.mark.parametrize( + "raw_shebang,original_command,command_full_basename,command_stem,correct_command,corrected_shebang_line", + [ + ( + b"#! /usr/bin/false\r\n", + "/usr/bin/false", + "false", + "false", + None, + None, + ), + ( + b"#!/usr/bin/python3 -b", + "/usr/bin/python3", + "python3", + "python", + "/usr/bin/python3", + None, + ), + ( + b"#!/usr/bin/env python3 -b", + "/usr/bin/env python3", + "python3", + "python", + "/usr/bin/python3", + "#! /usr/bin/python3 -b", + ), + ( + b"#! /bin/env python3.12-dbg -b", + "/bin/env python3.12-dbg", + "python3.12-dbg", + "python", + "/usr/bin/python3.12-dbg", + "#! /usr/bin/python3.12-dbg -b", + ), + ( + b"#! /usr/bin/bash", + "/usr/bin/bash", + "bash", + "bash", + "/bin/bash", + "#! /bin/bash", + ), + ( + b"#! /usr/bin/env sh", + "/usr/bin/env sh", + "sh", + "sh", + "/bin/sh", + "#! /bin/sh", + ), + ( + b"#! /usr/local/bin/perl", + "/usr/local/bin/perl", + "perl", + "perl", + "/usr/bin/perl", + "#! /usr/bin/perl", + ), + ], +) +def test_interpreter_detection( + raw_shebang: bytes, + original_command: str, + command_full_basename: str, + command_stem: str, + correct_command: Optional[str], + corrected_shebang_line: Optional[str], +) -> None: + interpreter = extract_shebang_interpreter(raw_shebang) + # The `and ...` part is just to get the raw line in the error message + assert interpreter is not None or raw_shebang == b"" + + assert interpreter.original_command == original_command + assert interpreter.command_full_basename == command_full_basename + assert interpreter.command_stem == command_stem + assert interpreter.correct_command == correct_command + assert interpreter.corrected_shebang_line == corrected_shebang_line + assert interpreter.fixup_needed == (corrected_shebang_line is not None) + + +@pytest.mark.parametrize( + "raw_data", + [ + b"#!#!#! /usr/bin/false", + b"#!perl", # Used in files as an editor hint + b"\x7FELF/usr/bin/perl", + b"\x00\01\x02\x03/usr/bin/perl", + b"PK\x03\x03/usr/bin/perl", + ], +) +def test_interpreter_negative(raw_data: bytes) -> None: + assert extract_shebang_interpreter(raw_data) is None + + +@pytest.fixture +def empty_manifest( + amd64_dpkg_architecture_variables, + dpkg_arch_query, + source_package, + package_single_foo_arch_all_cxt_amd64, + amd64_substitution, + no_profiles_or_build_options, + debputy_plugin_feature_set, +) -> HighLevelManifest: + # We need an empty directory to avoid triggering packager provided files. + debian_dir = build_virtual_file_system([]) + return YAMLManifestParser( + "debian/test-debputy.manifest", + source_package, + package_single_foo_arch_all_cxt_amd64, + amd64_substitution, + amd64_dpkg_architecture_variables, + dpkg_arch_query, + no_profiles_or_build_options, + debputy_plugin_feature_set, + debian_dir=debian_dir, + ).build_manifest() + + +def test_interpreter_rewrite(empty_manifest: HighLevelManifest) -> None: + condition_context = empty_manifest.condition_context("foo") + fs_root = build_virtual_file_system( + [ + virtual_path_def("usr/bin/foo", content="random data"), + virtual_path_def( + "usr/bin/foo.sh", + materialized_content="#!/usr/bin/sh\nset -e\n", + ), + ] + ) + interpreter_normalization = NormalizeShebangLineTransformation() + interpreter_normalization.transform_file_system(fs_root, condition_context) + foo = fs_root.lookup("usr/bin/foo") + foo_sh = fs_root.lookup("usr/bin/foo.sh") + + assert foo.is_file + with foo.open() as fd: + assert fd.read() == "random data" + + assert foo_sh.is_file + with foo_sh.open() as fd: + expected = textwrap.dedent( + """\ + #! /bin/sh + set -e + """ + ) + assert fd.read() == expected diff --git a/tests/test_migrations.py b/tests/test_migrations.py new file mode 100644 index 0000000..dc07d4c --- /dev/null +++ b/tests/test_migrations.py @@ -0,0 +1,1767 @@ +import io +import textwrap +from typing import Iterable, Callable, Optional, List, Tuple, Sequence + +import pytest + +from debputy.dh_migration.migrators import Migrator +from debputy.dh_migration.migrators_impl import ( + migrate_tmpfile, + migrate_lintian_overrides_files, + detect_pam_files, + migrate_doc_base_files, + migrate_installexamples_file, + migrate_installdocs_file, + migrate_install_file, + migrate_maintscript, + migrate_links_files, + detect_dh_addons, + migrate_not_installed_file, + migrate_installman_file, + migrate_bash_completion, + migrate_installinfo_file, + migrate_dh_installsystemd_files, + detect_obsolete_substvars, + MIGRATION_TARGET_DH_DEBPUTY, + MIGRATION_TARGET_DH_DEBPUTY_RRR, + detect_dh_addons_zz_debputy_rrr, +) +from debputy.dh_migration.models import ( + FeatureMigration, + AcceptableMigrationIssues, + UnsupportedFeature, +) +from debputy.highlevel_manifest import HighLevelManifest +from debputy.highlevel_manifest_parser import YAMLManifestParser +from debputy.plugin.api import virtual_path_def, VirtualPath +from debputy.plugin.api.test_api import ( + build_virtual_file_system, +) + + +DEBIAN_DIR_ENTRY = virtual_path_def(".", fs_path="/nowhere/debian") + + +@pytest.fixture() +def manifest_parser_pkg_foo_factory( + amd64_dpkg_architecture_variables, + dpkg_arch_query, + source_package, + package_single_foo_arch_all_cxt_amd64, + amd64_substitution, + no_profiles_or_build_options, + debputy_plugin_feature_set, +) -> Callable[[], YAMLManifestParser]: + # We need an empty directory to avoid triggering packager provided files. + debian_dir = build_virtual_file_system([]) + + def _factory(): + return YAMLManifestParser( + "debian/test-debputy.manifest", + source_package, + package_single_foo_arch_all_cxt_amd64, + amd64_substitution, + amd64_dpkg_architecture_variables, + dpkg_arch_query, + no_profiles_or_build_options, + debputy_plugin_feature_set, + debian_dir=debian_dir, + ) + + return _factory + + +@pytest.fixture(scope="session") +def accept_no_migration_issues() -> AcceptableMigrationIssues: + return AcceptableMigrationIssues(frozenset()) + + +@pytest.fixture(scope="session") +def accept_any_migration_issues() -> AcceptableMigrationIssues: + return AcceptableMigrationIssues(frozenset(["ALL"])) + + +@pytest.fixture +def empty_manifest_pkg_foo( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], +) -> HighLevelManifest: + return manifest_parser_pkg_foo_factory().build_manifest() + + +def run_migrator( + migrator: Migrator, + path: VirtualPath, + manifest: HighLevelManifest, + acceptable_migration_issues: AcceptableMigrationIssues, + *, + migration_target=MIGRATION_TARGET_DH_DEBPUTY, +) -> FeatureMigration: + feature_migration = FeatureMigration(migrator.__name__) + migrator( + path, + manifest, + acceptable_migration_issues, + feature_migration, + migration_target, + ) + return feature_migration + + +def _assert_unsupported_feature( + migrator: Migrator, + path: VirtualPath, + manifest: HighLevelManifest, + acceptable_migration_issues: AcceptableMigrationIssues, +): + with pytest.raises(UnsupportedFeature) as e: + run_migrator(migrator, path, manifest, acceptable_migration_issues) + return e + + +def _write_manifest(manifest: HighLevelManifest) -> str: + with io.StringIO() as fd: + manifest.mutable_manifest.write_to(fd) + return fd.getvalue() + + +def _verify_migrator_generates_parsable_manifest( + migrator: Migrator, + parser_factory: Callable[[], YAMLManifestParser], + acceptable_migration_issues: AcceptableMigrationIssues, + dh_config_name: str, + dh_config_content: str, + expected_manifest_content: str, + expected_warnings: Optional[List[str]] = None, + expected_renamed_paths: Optional[List[Tuple[str, str]]] = None, + expected_removals: Optional[List[str]] = None, + required_plugins: Optional[Sequence[str]] = tuple(), + dh_config_mode: Optional[int] = None, +) -> None: + # No file, no changes + empty_fs = build_virtual_file_system([DEBIAN_DIR_ENTRY]) + migration = run_migrator( + migrator, + empty_fs, + parser_factory().build_manifest(), + acceptable_migration_issues, + ) + + assert not migration.anything_to_do + assert not migration.warnings + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + assert not migration.required_plugins + + if dh_config_mode is None: + if dh_config_content.startswith(("#!/usr/bin/dh-exec", "#! /usr/bin/dh-exec")): + dh_config_mode = 0o755 + else: + dh_config_mode = 0o644 + + # Test with a dh_config file now + fs_w_dh_config = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + dh_config_name, + fs_path=f"/nowhere/debian/{dh_config_name}", + content=dh_config_content, + mode=dh_config_mode, + ), + ] + ) + manifest = parser_factory().build_manifest() + + migration = run_migrator( + migrator, + fs_w_dh_config, + manifest, + acceptable_migration_issues, + ) + + assert migration.anything_to_do + if expected_warnings is not None: + assert migration.warnings == expected_warnings + else: + assert not migration.warnings + assert migration.remove_paths_on_success == [f"/nowhere/debian/{dh_config_name}"] + if expected_removals is None: + assert migration.remove_paths_on_success == [ + f"/nowhere/debian/{dh_config_name}" + ] + else: + assert migration.remove_paths_on_success == expected_removals + if expected_renamed_paths is not None: + assert migration.rename_paths_on_success == expected_renamed_paths + else: + assert not migration.rename_paths_on_success + assert tuple(migration.required_plugins) == tuple(required_plugins) + actual_manifest = _write_manifest(manifest) + assert actual_manifest == expected_manifest_content + + # Verify that the output is actually parsable + parser_factory().parse_manifest(fd=actual_manifest) + + +def test_migrate_tmpfile( + empty_manifest_pkg_foo: HighLevelManifest, + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + migrator = migrate_tmpfile + empty_debian_dir = build_virtual_file_system([DEBIAN_DIR_ENTRY]) + migration = run_migrator( + migrator, + empty_debian_dir, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert not migration.anything_to_do + assert not migration.warnings + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + + tmpfile_debian_dir = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def("tmpfile", fs_path="/nowhere/debian/tmpfile"), + ] + ) + + migration = run_migrator( + migrator, + tmpfile_debian_dir, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert migration.anything_to_do + assert not migration.warnings + assert not migration.remove_paths_on_success + assert migration.rename_paths_on_success == [ + ("/nowhere/debian/tmpfile", "/nowhere/debian/tmpfiles") + ] + + tmpfile_debian_dir = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + # Use real paths here to make `cmp -s` discover that they are the same + virtual_path_def("tmpfile", fs_path="debian/control"), + virtual_path_def("tmpfiles", fs_path="debian/control"), + ] + ) + + migration = run_migrator( + migrator, + tmpfile_debian_dir, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + assert migration.anything_to_do + assert not migration.warnings + assert migration.remove_paths_on_success == ["debian/control"] + assert not migration.rename_paths_on_success + + conflict_tmpfile_debian_dir = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + # Use real paths here to make `cmp -s` discover a difference + virtual_path_def("tmpfile", fs_path="debian/control"), + virtual_path_def("tmpfiles", fs_path="debian/changelog"), + ] + ) + + migration = run_migrator( + migrator, + conflict_tmpfile_debian_dir, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert len(migration.warnings) == 1 + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + + conflict_tmpfile_debian_dir = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def("tmpfile", fs_path="/nowhere/debian/tmpfile"), + virtual_path_def("tmpfiles/", fs_path="/nowhere/debian/tmpfiles"), + ] + ) + + migration = run_migrator( + migrator, + conflict_tmpfile_debian_dir, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert len(migration.warnings) == 1 + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + + conflict_tmpfile_debian_dir = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "tmpfile", + link_target="/nowhere/debian/tmpfiles", + fs_path="/nowhere/debian/tmpfile", + ), + ] + ) + + migration = run_migrator( + migrator, + conflict_tmpfile_debian_dir, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert len(migration.warnings) == 1 + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + + +def test_migrate_lintian_overrides_files( + empty_manifest_pkg_foo: HighLevelManifest, + accept_no_migration_issues: AcceptableMigrationIssues, + accept_any_migration_issues: AcceptableMigrationIssues, +) -> None: + migrator = migrate_lintian_overrides_files + no_override_fs = build_virtual_file_system([DEBIAN_DIR_ENTRY]) + single_noexec_override_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "foo.lintian-overrides", + fs_path="/nowhere/no-exec/debian/foo.lintian-overrides", + ), + ] + ) + single_exec_override_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "foo.lintian-overrides", + fs_path="/nowhere/exec/debian/foo.lintian-overrides", + mode=0o755, + ), + ] + ) + for no_issue_fs in [no_override_fs, single_noexec_override_fs]: + migration = run_migrator( + migrator, + no_issue_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert not migration.anything_to_do + assert not migration.warnings + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + + _assert_unsupported_feature( + migrator, + single_exec_override_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + migration = run_migrator( + migrator, + single_exec_override_fs, + empty_manifest_pkg_foo, + accept_any_migration_issues, + ) + + assert migration.anything_to_do + assert len(migration.warnings) == 1 + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + + +def test_detect_pam_files( + empty_manifest_pkg_foo: HighLevelManifest, + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + migrator = detect_pam_files + empty_fs = build_virtual_file_system([DEBIAN_DIR_ENTRY]) + pam_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def("pam", fs_path="/nowhere/debian/foo.pam"), + ] + ) + + migration = run_migrator( + migrator, + empty_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert not migration.anything_to_do + assert not migration.warnings + assert migration.assumed_compat is None + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + + migration = run_migrator( + migrator, + pam_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert not migration.anything_to_do + assert not migration.warnings + assert migration.assumed_compat == 14 + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + + +def test_migrate_doc_base_files( + empty_manifest_pkg_foo: HighLevelManifest, + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + migrator = migrate_doc_base_files + empty_fs = build_virtual_file_system([DEBIAN_DIR_ENTRY]) + doc_base_ok_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def("doc-base", fs_path="/nowhere/debian/doc-base"), + virtual_path_def( + "foo.doc-base.EX", fs_path="/nowhere/debian/foo.doc-base.EX" + ), + ] + ) + doc_base_migration_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "foo.doc-base.bar", fs_path="/nowhere/debian/foo.doc-base.bar" + ), + ] + ) + + for no_change_fs in [empty_fs, doc_base_ok_fs]: + migration = run_migrator( + migrator, + no_change_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert not migration.anything_to_do + assert not migration.warnings + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + + migration = run_migrator( + migrator, + doc_base_migration_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert migration.anything_to_do + assert not migration.warnings + assert not migration.remove_paths_on_success + assert migration.rename_paths_on_success == [ + ("/nowhere/debian/foo.doc-base.bar", "/nowhere/debian/foo.bar.doc-base") + ] + + +def test_migrate_dh_installsystemd_files( + empty_manifest_pkg_foo: HighLevelManifest, + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + migrator = migrate_dh_installsystemd_files + empty_fs = build_virtual_file_system([DEBIAN_DIR_ENTRY]) + files_ok_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def("@service", fs_path="/nowhere/debian/@service"), + virtual_path_def("foo.@service", fs_path="/nowhere/debian/foo.@service"), + ] + ) + migration_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def("foo@.service", fs_path="/nowhere/debian/foo@.service"), + ] + ) + + for no_change_fs in [empty_fs, files_ok_fs]: + migration = run_migrator( + migrator, + no_change_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert not migration.anything_to_do + assert not migration.warnings + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + + migration = run_migrator( + migrator, + migration_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert migration.anything_to_do + assert not migration.warnings + assert not migration.remove_paths_on_success + assert migration.rename_paths_on_success == [ + ("/nowhere/debian/foo@.service", "/nowhere/debian/foo.@service") + ] + + +def test_migrate_installexamples_file( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + foo/* + bar + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install-examples: + sources: + - foo/* + - bar + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_installexamples_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "examples", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_installinfo_file( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + foo/* + bar + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install-docs: + sources: + - foo/* + - bar + dest-dir: '{{path:GNU_INFO_DIR}}' + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_installinfo_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "info", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_installinfo_file_conditionals( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + #!/usr/bin/dh-exec + foo/* + bar + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install-docs: + sources: + - foo/* + - bar + dest-dir: '{{path:GNU_INFO_DIR}}' + when: + build-profiles-matches: + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_installinfo_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "info", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_installexamples_file_single_source( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + foo/* + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install-examples: + source: foo/* + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_installexamples_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "examples", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_installdocs_file( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + foo/* + bar + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install-docs: + sources: + - foo/* + - bar + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_installdocs_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "docs", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_installdocs_file_single_source( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + foo/* + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install-docs: + source: foo/* + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_installdocs_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "docs", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_install_file( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + bar usr/bin + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install: + source: bar + dest-dir: usr/bin + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_install_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "install", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_install_file_conditionals_simple_arch( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + #!/usr/bin/dh-exec + bar usr/bin [linux-any] + foo usr/bin [linux-any] + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install: + sources: + - bar + - foo + dest-dir: usr/bin + when: + arch-matches: linux-any + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_install_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "install", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_install_file_util_linux_locales( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + # Parts of the `d/util-linux-locales.install` file. It uses d/tmp for most of its paths + # and that breaks the default dest-dir (dh_install always strips --sourcedir, `debputy` + # currently does not) + dh_config_content = textwrap.dedent( + """\ + #!/usr/bin/dh-exec + usr/share/locale/*/*/util-linux.mo + + # bsdextrautils + debian/tmp/usr/share/man/*/man1/col.1 + + debian/tmp/usr/share/man/*/man3/libblkid.3 + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install: + sources: + - usr/share/man/*/man1/col.1 + - usr/share/man/*/man3/libblkid.3 + when: + build-profiles-matches: + - install: + source: usr/share/locale/*/*/util-linux.mo + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_install_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "install", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_install_file_conditionals_simple_combined_cond( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + for cond in [" [linux-any]", "[linux-any] "]: + dh_config_content = textwrap.dedent( + """\ + #!/usr/bin/dh-exec + bar usr/bin {CONDITION} + foo usr/bin {CONDITION} + """ + ).format(CONDITION=cond) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install: + sources: + - bar + - foo + dest-dir: usr/bin + when: + all-of: + - arch-matches: linux-any + - build-profiles-matches: + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_install_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "install", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_install_file_conditionals_unknown_subst( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_any_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + #!/usr/bin/dh-exec + bar ${unknown_substvar} + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + definitions: + variables: + unknown_substvar: 'TODO: Provide variable value for unknown_substvar' + installations: + - install: + source: bar + dest-dir: '{{unknown_substvar}}' + """ + ) + expected_warning = ( + "TODO: MANUAL MIGRATION of unresolved substitution variable {{unknown_substvar}}" + ' from ./install line 2 token "${unknown_substvar}"' + ) + _verify_migrator_generates_parsable_manifest( + migrate_install_file, + manifest_parser_pkg_foo_factory, + accept_any_migration_issues, + "install", + dh_config_content, + expected_manifest_content, + expected_warnings=[expected_warning], + ) + + +def test_migrate_install_file_multidest( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + # Issue #66 + # - observed in kafs-client / the original install file copied in here. + + src/aklog-kafs usr/bin + src/kafs-check-config usr/bin + # + src/kafs-preload usr/sbin + # + src/kafs-dns usr/libexec + # + conf/cellservdb.conf usr/share/kafs-client + conf/client.conf etc/kafs + # + conf/kafs_dns.conf etc/request-key.d + # + conf/cellservdb.conf usr/share/kafs + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install: + sources: + - src/aklog-kafs + - src/kafs-check-config + dest-dir: usr/bin + - install: + source: src/kafs-preload + dest-dir: usr/sbin + - install: + source: src/kafs-dns + dest-dir: usr/libexec + - install: + source: conf/client.conf + dest-dir: etc/kafs + - install: + source: conf/kafs_dns.conf + dest-dir: etc/request-key.d + - multi-dest-install: + source: conf/cellservdb.conf + dest-dirs: + - usr/share/kafs-client + - usr/share/kafs + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_install_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "install", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_install_file_multidest_default_dest( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + # Relaed to issue #66 - testing corner case not present in the original install file + + src/aklog-kafs usr/bin + src/kafs-check-config usr/bin + # + src/kafs-preload usr/sbin + # + src/kafs-dns usr/libexec + # + usr/share/kafs-client/cellservdb.conf + conf/client.conf etc/kafs + # + conf/kafs_dns.conf etc/request-key.d + # + usr/share/kafs-client/cellservdb.conf usr/share/kafs + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install: + sources: + - src/aklog-kafs + - src/kafs-check-config + dest-dir: usr/bin + - install: + source: src/kafs-preload + dest-dir: usr/sbin + - install: + source: src/kafs-dns + dest-dir: usr/libexec + - install: + source: conf/client.conf + dest-dir: etc/kafs + - install: + source: conf/kafs_dns.conf + dest-dir: etc/request-key.d + - multi-dest-install: + source: usr/share/kafs-client/cellservdb.conf + dest-dirs: + - usr/share/kafs + - usr/share/kafs-client + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_install_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "install", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_install_file_multidest_default_dest_warning( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + # Relaed to issue #66 - testing corner case not present in the original install file + + src/aklog-kafs usr/bin + src/kafs-check-config usr/bin + # + src/kafs-preload usr/sbin + # + src/kafs-dns usr/libexec + # + usr/share/kafs-*/cellservdb.conf + conf/client.conf etc/kafs + # + conf/kafs_dns.conf etc/request-key.d + # + usr/share/kafs-*/cellservdb.conf usr/share/kafs + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install: + sources: + - src/aklog-kafs + - src/kafs-check-config + dest-dir: usr/bin + - install: + source: src/kafs-preload + dest-dir: usr/sbin + - install: + source: src/kafs-dns + dest-dir: usr/libexec + - install: + source: conf/client.conf + dest-dir: etc/kafs + - install: + source: conf/kafs_dns.conf + dest-dir: etc/request-key.d + - multi-dest-install: + source: usr/share/kafs-*/cellservdb.conf + dest-dirs: + - usr/share/kafs + - 'FIXME: usr/share/kafs-* (could not reliably compute the dest dir)' + """ + ) + warnings = [ + "TODO: FIXME left in dest-dir(s) of some installation rules." + " Please review these and remove the FIXME (plus correct as necessary)" + ] + _verify_migrator_generates_parsable_manifest( + migrate_install_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "install", + dh_config_content, + expected_manifest_content, + expected_warnings=warnings, + ) + + +def test_migrate_installman_file( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + man/foo.1 man/bar.1 + man2/*.2 + man3/bar.3 man3/bar.de.3 + man/de/man3/bar.pl.3 + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install-man: + sources: + - man/foo.1 + - man/bar.1 + - man2/*.2 + - man/de/man3/bar.pl.3 + - install-man: + sources: + - man3/bar.3 + - man3/bar.de.3 + language: derive-from-basename + """ + ) + expected_warnings = [ + 'Detected manpages that might rely on "derive-from-basename" logic. Please double check' + " that the generated `install-man` rules are correct" + ] + _verify_migrator_generates_parsable_manifest( + migrate_installman_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "manpages", + dh_config_content, + expected_manifest_content, + expected_warnings=expected_warnings, + ) + + +def test_migrate_install_dh_exec_file( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + #!/usr/bin/dh-exec + + foo/script.sh => usr/bin/script + => usr/bin/bar + usr/bin/* usr/share/foo/extra/* usr/share/foo/extra + another-util usr/share/foo/extra + # This will not be merged with `=> usr/bin/bar` + usr/share/foo/features + usr/share/foo/bugs + some-file.txt usr/share/foo/online-doc + # TODO: Support migration of these + # pathA pathB conditional/arch [linux-anx] + # another-path conditional/profile + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install: + source: usr/bin/bar + - install: + source: foo/script.sh + as: usr/bin/script + - install: + sources: + - usr/bin/* + - usr/share/foo/extra/* + - another-util + dest-dir: usr/share/foo/extra + - install: + source: some-file.txt + dest-dir: usr/share/foo/online-doc + - install: + sources: + - usr/share/foo/features + - usr/share/foo/bugs + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_install_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "install", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_maintscript( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + rm_conffile /etc/foo.conf + mv_conffile /etc/bar.conf /etc/new-foo.conf 1.0~ bar + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + conffile-management: + - remove: + path: /etc/foo.conf + - rename: + source: /etc/bar.conf + target: /etc/new-foo.conf + prior-to-version: 1.0~ + owning-package: bar + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_maintscript, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "maintscript", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_not_installed_file( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + foo/*.txt bar/${DEB_HOST_MULTIARCH}/*.so* + baz/script.sh + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - discard: + - foo/*.txt + - bar/{{DEB_HOST_MULTIARCH}}/*.so* + - baz/script.sh + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_not_installed_file, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "not-installed", + dh_config_content, + expected_manifest_content, + ) + + +def test_migrate_links_files( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + usr/share/target usr/bin/symlink + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - create-symlink: + path: usr/bin/symlink + target: /usr/share/target + """ + ) + _verify_migrator_generates_parsable_manifest( + migrate_links_files, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "links", + dh_config_content, + expected_manifest_content, + ) + + +def test_detect_obsolete_substvars( + empty_manifest_pkg_foo: HighLevelManifest, + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + migrator = detect_obsolete_substvars + + dctrl_content = textwrap.dedent( + """\ + Source: foo + Build-Depends: debhelper-compat (= 13), + dh-sequence-debputy, + dh-sequence-foo, + + Package: foo + Architecture: any + Description: ... + Depends: ${misc:Depends}, + ${shlibs:Depends}, + bar (>= 1.0~) | baz, ${so:Depends}, + """ + ) + dctrl_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "control", + fs_path="/nowhere/debian/control", + content=dctrl_content, + ), + ] + ) + + migration = run_migrator( + migrator, + dctrl_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + msg = ( + "The following relationship substitution variables can be removed from foo:" + " ${misc:Depends}, ${shlibs:Depends}, ${so:Depends}" + ) + assert migration.anything_to_do + assert migration.warnings == [msg] + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + assert not migration.required_plugins + + +def test_detect_obsolete_substvars_remove_field( + empty_manifest_pkg_foo: HighLevelManifest, + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + migrator = detect_obsolete_substvars + + dctrl_content = textwrap.dedent( + """\ + Source: foo + Build-Depends: debhelper-compat (= 13), + dh-sequence-debputy, + dh-sequence-foo, + + Package: foo + Architecture: any + Description: ... + Pre-Depends: ${misc:Pre-Depends} + Depends: bar (>= 1.0~) | baz + """ + ) + dctrl_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "control", + fs_path="/nowhere/debian/control", + content=dctrl_content, + ), + ] + ) + + migration = run_migrator( + migrator, + dctrl_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + msg = ( + "The following relationship fields can be removed from foo: Pre-Depends." + " (The content in them would be applied automatically.)" + ) + assert migration.anything_to_do + assert migration.warnings == [msg] + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + assert not migration.required_plugins + + +def test_detect_obsolete_substvars_remove_field_essential( + empty_manifest_pkg_foo: HighLevelManifest, + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + migrator = detect_obsolete_substvars + + dctrl_content = textwrap.dedent( + """\ + Source: foo + Build-Depends: debhelper-compat (= 13), + dh-sequence-debputy, + dh-sequence-foo, + + Package: foo + Architecture: any + Description: ... + Essential: yes + # Obsolete because the package is essential + Pre-Depends: ${shlibs:Depends} + Depends: bar (>= 1.0~) | baz + """ + ) + dctrl_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "control", + fs_path="/nowhere/debian/control", + content=dctrl_content, + ), + ] + ) + + migration = run_migrator( + migrator, + dctrl_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + msg = ( + "The following relationship fields can be removed from foo: Pre-Depends." + " (The content in them would be applied automatically.)" + ) + assert migration.anything_to_do + assert migration.warnings == [msg] + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + assert not migration.required_plugins + + +def test_detect_obsolete_substvars_remove_field_non_essential( + empty_manifest_pkg_foo: HighLevelManifest, + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + migrator = detect_obsolete_substvars + + dctrl_content = textwrap.dedent( + """\ + Source: foo + Build-Depends: debhelper-compat (= 13), + dh-sequence-debputy, + dh-sequence-foo, + + Package: foo + Architecture: any + Description: ... + # This is not obsolete since the package is not essential + Pre-Depends: ${shlibs:Depends} + Depends: bar (>= 1.0~) | baz + """ + ) + dctrl_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "control", + fs_path="/nowhere/debian/control", + content=dctrl_content, + ), + ] + ) + + migration = run_migrator( + migrator, + dctrl_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + assert not migration.anything_to_do + assert not migration.warnings + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + assert not migration.required_plugins + + +def test_detect_dh_addons( + empty_manifest_pkg_foo: HighLevelManifest, + accept_no_migration_issues: AcceptableMigrationIssues, + accept_any_migration_issues: AcceptableMigrationIssues, +) -> None: + migrator = detect_dh_addons + empty_fs = build_virtual_file_system([DEBIAN_DIR_ENTRY]) + + dctrl_no_addons_content = textwrap.dedent( + """\ + Source: foo + Build-Depends: debhelper-compat (= 13) + + Package: foo + Architecture: all + Description: ... + """ + ) + + dctrl_w_addons_content = textwrap.dedent( + """\ + Source: foo + Build-Depends: debhelper-compat (= 13), + dh-sequence-debputy, + dh-sequence-foo, + + Package: foo + Architecture: all + Description: ... + """ + ) + + dctrl_w_migrateable_addons_content = textwrap.dedent( + """\ + Source: foo + Build-Depends: debhelper-compat (= 13), + dh-sequence-debputy, + dh-sequence-numpy3, + + Package: foo + Architecture: all + Description: ... + """ + ) + + dctrl_no_addons_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "control", + fs_path="/nowhere/debian/control-without-addons", + content=dctrl_no_addons_content, + ), + ] + ) + dctrl_w_addons_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "control", + fs_path="/nowhere/debian/control-with-addons", + content=dctrl_w_addons_content, + ), + ] + ) + dctrl_w_migrateable_addons_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "control", + fs_path="/nowhere/debian/control-with-migrateable-addons", + content=dctrl_w_migrateable_addons_content, + ), + ] + ) + no_ctrl_msg = ( + "Cannot find debian/control. Detection of unsupported/missing dh-sequence addon" + " could not be performed. Please ensure the package will Build-Depend on dh-sequence-zz-debputy" + " and not rely on any other debhelper sequence addons except those debputy explicitly supports." + ) + missing_debputy_bd_msg = "Missing Build-Depends on dh-sequence-zz-debputy" + unsupported_sequence_msg = ( + 'The dh addon "foo" is not known to work with dh-debputy and might malfunction' + ) + + migration = run_migrator( + migrator, + empty_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + assert migration.anything_to_do + assert migration.warnings == [no_ctrl_msg] + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + assert not migration.required_plugins + + migration = run_migrator( + migrator, + dctrl_no_addons_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + assert migration.anything_to_do + assert migration.warnings == [missing_debputy_bd_msg] + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + assert not migration.required_plugins + + _assert_unsupported_feature( + migrator, + dctrl_w_addons_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + migration = run_migrator( + migrator, + dctrl_w_addons_fs, + empty_manifest_pkg_foo, + accept_any_migration_issues, + ) + + assert migration.anything_to_do + assert migration.warnings == [unsupported_sequence_msg] + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + assert not migration.required_plugins + + migration = run_migrator( + migrator, + dctrl_w_migrateable_addons_fs, + empty_manifest_pkg_foo, + accept_any_migration_issues, + ) + assert not migration.anything_to_do + assert not migration.warnings + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + assert migration.required_plugins == ["numpy3"] + + +def test_detect_dh_addons_rrr( + empty_manifest_pkg_foo: HighLevelManifest, + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + migrator = detect_dh_addons_zz_debputy_rrr + empty_fs = build_virtual_file_system([DEBIAN_DIR_ENTRY]) + + dctrl_no_addons_content = textwrap.dedent( + """\ + Source: foo + Build-Depends: debhelper-compat (= 13) + + Package: foo + Architecture: all + Description: ... + """ + ) + + dctrl_w_addons_content = textwrap.dedent( + """\ + Source: foo + Build-Depends: debhelper-compat (= 13), + dh-sequence-zz-debputy-rrr, + dh-sequence-foo, + + Package: foo + Architecture: all + Description: ... + """ + ) + + dctrl_no_addons_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "control", + fs_path="/nowhere/debian/control-without-addons", + content=dctrl_no_addons_content, + ), + ] + ) + dctrl_w_addons_fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + "control", + fs_path="/nowhere/debian/control-with-addons", + content=dctrl_w_addons_content, + ), + ] + ) + no_ctrl_msg = ( + "Cannot find debian/control. Detection of unsupported/missing dh-sequence addon" + " could not be performed. Please ensure the package will Build-Depend on dh-sequence-zz-debputy-rrr." + ) + missing_debputy_bd_msg = "Missing Build-Depends on dh-sequence-zz-debputy-rrr" + + migration = run_migrator( + migrator, + empty_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + migration_target=MIGRATION_TARGET_DH_DEBPUTY_RRR, + ) + assert migration.anything_to_do + assert migration.warnings == [no_ctrl_msg] + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + assert not migration.required_plugins + + migration = run_migrator( + migrator, + dctrl_no_addons_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + assert migration.anything_to_do + assert migration.warnings == [missing_debputy_bd_msg] + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + assert not migration.required_plugins + + migration = run_migrator( + migrator, + dctrl_w_addons_fs, + empty_manifest_pkg_foo, + accept_no_migration_issues, + ) + + assert not migration.anything_to_do + assert not migration.warnings + assert not migration.remove_paths_on_success + assert not migration.rename_paths_on_success + assert not migration.required_plugins + + +def test_migrate_bash_completion_file_no_changes( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + compgen -A + """ + ) + dh_config_name = "bash-completion" + fs = build_virtual_file_system( + [ + DEBIAN_DIR_ENTRY, + virtual_path_def( + dh_config_name, + fs_path=f"/nowhere/debian/{dh_config_name}", + content=dh_config_content, + ), + ] + ) + migration = run_migrator( + migrate_bash_completion, + fs, + manifest_parser_pkg_foo_factory().build_manifest(), + accept_no_migration_issues, + ) + assert not migration.rename_paths_on_success + assert not migration.remove_paths_on_success + assert not migration.warnings + assert not migration.required_plugins + + +def test_migrate_bash_completion_file( + manifest_parser_pkg_foo_factory: Callable[[], YAMLManifestParser], + accept_no_migration_issues: AcceptableMigrationIssues, +) -> None: + dh_config_content = textwrap.dedent( + """\ + foo/* + bar baz + debian/bar-completion bar + debian/foo-completion foo + debian/*.cmpl + """ + ) + expected_manifest_content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install: + sources: + - foo/* + - debian/*.cmpl + dest-dir: '{{path:BASH_COMPLETION_DIR}}' + - install: + source: bar + as: '{{path:BASH_COMPLETION_DIR}}/baz' + """ + ) + expected_renames = [ + ("debian/bar-completion", "debian/foo.bar.bash-completion"), + ("debian/foo-completion", "debian/foo.bash-completion"), + ] + _verify_migrator_generates_parsable_manifest( + migrate_bash_completion, + manifest_parser_pkg_foo_factory, + accept_no_migration_issues, + "bash-completion", + dh_config_content, + expected_manifest_content, + expected_renamed_paths=expected_renames, + ) diff --git a/tests/test_output_filename.py b/tests/test_output_filename.py new file mode 100644 index 0000000..fab0e98 --- /dev/null +++ b/tests/test_output_filename.py @@ -0,0 +1,27 @@ +from pathlib import Path + +import pytest + +from debputy.util import compute_output_filename + + +def write_unpacked_deb(root: Path, package: str, version: str, arch: str): + (root / "control").write_text( + f"Package: {package}\nVersion: {version}\nArchitecture: {arch}\n" + ) + + +@pytest.mark.parametrize( + "package,version,arch,is_udeb,expected", + [ + ("fake", "1.0", "amd64", False, "fake_1.0_amd64.deb"), + ("fake", "1.0", "amd64", True, "fake_1.0_amd64.udeb"), + ("fake", "2:1.0", "amd64", False, "fake_1.0_amd64.deb"), + ("fake", "2:1.0", "amd64", True, "fake_1.0_amd64.udeb"), + ("fake", "3.0", "all", False, "fake_3.0_all.deb"), + ("fake", "3.0", "all", True, "fake_3.0_all.udeb"), + ], +) +def test_generate_deb_filename(tmp_path, package, version, arch, is_udeb, expected): + write_unpacked_deb(tmp_path, package, version, arch) + assert compute_output_filename(str(tmp_path), is_udeb) == expected diff --git a/tests/test_packager_provided_files.py b/tests/test_packager_provided_files.py new file mode 100644 index 0000000..149564d --- /dev/null +++ b/tests/test_packager_provided_files.py @@ -0,0 +1,210 @@ +import random +from typing import cast + +import pytest + +from debputy.packager_provided_files import detect_all_packager_provided_files +from debputy.plugin.api import DebputyPluginInitializer +from debputy.plugin.api.feature_set import PluginProvidedFeatureSet +from debputy.plugin.api.test_api import ( + InitializedPluginUnderTest, + build_virtual_file_system, +) +from debputy.plugin.api.test_api.test_impl import initialize_plugin_under_test_preloaded +from tutil import faked_binary_package, binary_package_table + + +def ppf_test_plugin(api: DebputyPluginInitializer) -> None: + api.packager_provided_file( + "arch-specific-dash", + "/some/test-directory/{name}.conf", + allow_architecture_segment=True, + ) + api.packager_provided_file( + "arch.specific.dot", + "/some/test.directory/{name}", + allow_architecture_segment=True, + ) + + api.packager_provided_file( + "arch.specific.with.priority", + "/some/test.priority.directory/{priority:02}-{name}", + allow_architecture_segment=True, + default_priority=60, + ) + + api.packager_provided_file( + "packageless-fallback", + "/some/test-plfb/{name}", + packageless_is_fallback_for_all_packages=True, + ) + api.packager_provided_file( + "packageless.fallback", + "/some/test.plfb/{name}", + packageless_is_fallback_for_all_packages=True, + ) + + +@pytest.mark.parametrize( + "package_name,basename,install_target,is_main_binary", + [ + ("foo", "foo.arch-specific-dash", "./some/test-directory/foo.conf", True), + # main package match + ("foo", "arch-specific-dash", "./some/test-directory/foo.conf", True), + # arch match + ("foo", "foo.arch-specific-dash.amd64", "./some/test-directory/foo.conf", True), + # Matches with periods in both package name and in the file type + ("foo.bar", "foo.bar.arch.specific.dot", "./some/test.directory/foo.bar", True), + ("foo.bar", "arch.specific.dot", "./some/test.directory/foo.bar", True), + ( + "foo.bar", + "foo.bar.arch.specific.dot.amd64", + "./some/test.directory/foo.bar", + True, + ), + # Priority + ( + "foo.bar", + "foo.bar.arch.specific.with.priority", + "./some/test.priority.directory/60-foo.bar", + True, + ), + ( + "foo.bar", + "arch.specific.with.priority", + "./some/test.priority.directory/60-foo.bar", + True, + ), + ( + "foo.bar", + "foo.bar.arch.specific.with.priority.amd64", + "./some/test.priority.directory/60-foo.bar", + True, + ), + # Name + ( + "foo.bar", + "foo.bar.special.name.arch.specific.with.priority", + "./some/test.priority.directory/60-special.name", + True, + ), + ( + "foo.bar", + "foo.bar.special.name.arch.specific.with.priority.amd64", + "./some/test.priority.directory/60-special.name", + True, + ), + ( + "foo.bar", + "packageless-fallback", + "./some/test-plfb/foo.bar", + False, + ), + ( + "foo.bar", + "packageless.fallback", + "./some/test.plfb/foo.bar", + False, + ), + ], +) +def test_packager_provided_files( + package_name: str, basename: str, install_target: str, is_main_binary: bool +) -> None: + # Inject our custom files + plugin = initialize_plugin_under_test_preloaded( + 1, + ppf_test_plugin, + plugin_name="pff-test-plugin", + ) + debputy_plugin_feature_set = _fetch_debputy_plugin_feature_set(plugin) + + debian_dir = build_virtual_file_system([basename]) + binary_under_test = faked_binary_package( + package_name, is_main_package=is_main_binary + ) + main_package = ( + binary_under_test if is_main_binary else faked_binary_package("main-pkg") + ) + binaries = [main_package] + if not is_main_binary: + binaries.append(binary_under_test) + binary_packages = binary_package_table(*binaries) + + ppf = detect_all_packager_provided_files( + debputy_plugin_feature_set.packager_provided_files, + debian_dir, + binary_packages, + ) + assert package_name in ppf + all_matched = ppf[package_name].auto_installable + assert len(all_matched) == 1 + matched = all_matched[0] + assert basename == matched.path.name + actual_install_target = "/".join(matched.compute_dest()) + assert actual_install_target == install_target + + +@pytest.mark.parametrize( + "package_name,expected_basename,non_matched", + [ + ("foo", "foo.arch-specific-dash", ["arch-specific-dash"]), + ( + "foo", + "foo.arch-specific-dash.amd64", + [ + "foo.arch-specific-dash", + "arch-specific-dash", + "foo.arch-specific-dash.i386", + ], + ), + ( + "foo", + "foo.arch-specific-dash", + ["arch-specific-dash", "foo.arch-specific-dash.i386"], + ), + ], +) +def test_packager_provided_files_priority( + package_name, expected_basename, non_matched +) -> None: + assert len(non_matched) > 0 + # Inject our custom files + plugin = initialize_plugin_under_test_preloaded( + 1, + ppf_test_plugin, + plugin_name="pff-test-plugin", + ) + debputy_plugin_feature_set = _fetch_debputy_plugin_feature_set(plugin) + binary_packages = binary_package_table(faked_binary_package(package_name)) + all_entries_base = [x for x in non_matched] + + for order in (0, len(all_entries_base), None): + all_entries = all_entries_base.copy() + print(f"Order: {order}") + if order is not None: + all_entries.insert(order, expected_basename) + else: + all_entries.append(expected_basename) + # Ensure there are no order dependencies in the test by randomizing it. + random.shuffle(all_entries) + + debian_dir = build_virtual_file_system(all_entries) + ppf = detect_all_packager_provided_files( + debputy_plugin_feature_set.packager_provided_files, + debian_dir, + binary_packages, + ) + assert package_name in ppf + all_matched = ppf[package_name].auto_installable + assert len(all_matched) == 1 + matched = all_matched[0] + assert expected_basename == matched.path.name + + +def _fetch_debputy_plugin_feature_set( + plugin: InitializedPluginUnderTest, +) -> PluginProvidedFeatureSet: + # Very much not public API, but we need it to avoid testing on production data (also, it is hard to find + # relevant debputy files for all the cases we want to test). + return cast("InitializedPluginUnderTestImpl", plugin)._feature_set diff --git a/tests/test_packer_pack.py b/tests/test_packer_pack.py new file mode 100644 index 0000000..17529ef --- /dev/null +++ b/tests/test_packer_pack.py @@ -0,0 +1,86 @@ +import argparse +import json +from pathlib import Path + +from debputy.commands import deb_packer +from debputy.intermediate_manifest import TarMember, PathType + + +def write_unpacked_deb(root: Path, package: str, version: str, arch: str): + debian = root / "DEBIAN" + debian.mkdir(mode=0o755) + (debian / "control").write_text( + f"Package: {package}\nVersion: {version}\nArchitecture: {arch}\n" + ) + + +def test_pack_smoke(tmp_path): + mtime = 1668973695 + + root_dir = tmp_path / "root" + root_dir.mkdir() + write_unpacked_deb(root_dir, "fake", "1.0", "amd64") + output_path = tmp_path / "out" + output_path.mkdir() + deb_file = Path(output_path) / "output.deb" + + parsed_args = argparse.Namespace( + is_udeb=False, compression_level=None, compression_strategy=None + ) + + data_compression = deb_packer.COMPRESSIONS["xz"] + data_compression_cmd = data_compression.as_cmdline(parsed_args) + ctrl_compression = data_compression + ctrl_compression_cmd = data_compression_cmd + + package_manifest = tmp_path / "temporary-manifest.json" + package_manifest.write_text( + json.dumps( + [ + TarMember.virtual_path( + "./", PathType.DIRECTORY, mode=0o755, mtime=1668973695 + ).to_manifest() + ] + ) + ) + + deb_packer.pack( + str(deb_file), + ctrl_compression, + data_compression, + str(root_dir), + str(package_manifest), + mtime, + ctrl_compression_cmd, + data_compression_cmd, + prefer_raw_exceptions=True, + ) + + binary = deb_file.read_bytes() + + assert binary == ( + b"!\n" + b"debian-binary 1668973695 0 0 100644 4 `\n" + b"2.0\n" + b"control.tar.xz 1668973695 0 0 100644 244 `\n" + b"\xfd7zXZ\x00\x00\x04\xe6\xd6\xb4F\x04\xc0\xb4\x01\x80P!\x01\x16\x00\x00\x00" + b"\x00\x00\x00\x00\x19\x87 E\xe0'\xff\x00\xac]\x00\x17\x0b\xbc\x1c}" + b"\x01\x95\xc0\x1dJ>y\x15\xc2\xcc&\xa3^\x11\xb5\x81\xa6\x8cI\xd2\xf0m\xdd\x04" + b"M\xb2|Tdy\xf5\x00H\xab\xa6B\x11\x8d2\x0e\x1d\xf8F\x9e\x9a\xb0\xb8_]\xa3;M" + b"t\x90\x9a\xe3)\xeb\xadF\xfet'b\x05\x85\xd5\x04g\x7f\x89\xeb=(\xfd\xf6" + b'"p\xc3\x91\xf2\xd3\xd2\xb3\xed%i\x9a\xfa\\\xde7\xd5\x01\x18I\x14D\x10E' + b"\xba\xdf\xfb\x12{\x84\xc4\x10\x08,\xbc\x9e\xac+w\x07\r`|\xcfFL#\xbb" + b"S\x91\xb4\\\x9b\x80&\x1d\x9ej\x13\xe3\x13\x02=\xe9\xd5\xcf\xb0\xdf?L\xf1\x96" + b"\xd2\xc6bh\x19|?\xc2j\xe58If\xb7Y\xb9\x18:\x00\x00|\xfb\xcf\x82e/\xd05" + b"\x00\x01\xd0\x01\x80P\x00\x00\xc9y\xeem\xb1\xc4g\xfb\x02\x00\x00\x00" + b"\x00\x04YZ" + b"data.tar.xz 1668973695 0 0 100644 168 `\n" + b"\xfd7zXZ\x00\x00\x04\xe6\xd6\xb4F\x04\xc0h\x80P!\x01\x16\x00\x00\x00\x00" + b"\x00\x00\x00\x00\xc2bi\xe8\xe0'\xff\x00`]\x00\x17\x0b\xbc\x1c}" + b"\x01\x95\xc0\x1dJ>y\x15\xc2\xcc&\xa3^\x11\xb5\x81\xa6\x8cI\xd2\xf0m\xdd\x04" + b"M\xb2|Tdy\xf5\x00H\xab\xa6B\x11\x8d2\x0e\x1d\xf8F\x9e\x9a\xb0\xb8_]\xa4W%" + b"\xa2\x14N\xb9\xe7\xbd\xf3a\x16\xe5\xb7\xe6\x80\x95\xcc\xe6+\xe1;I" + b"\xf2\x1f\xed\x08\xac\xd7UZ\xc0P\x0b\xfb\nK\xef~\xcb\x8f\x80\x00\x9b\x19\xf8A" + b"Q_\xe7\xeb\x00\x01\x84\x01\x80P\x00\x00(3\xf1\xfa\xb1\xc4g\xfb" + b"\x02\x00\x00\x00\x00\x04YZ" + ) diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..bc041fc --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,473 @@ +import textwrap + +import pytest + +from debputy import DEBPUTY_DOC_ROOT_DIR +from debputy.exceptions import DebputySubstitutionError +from debputy.highlevel_manifest_parser import YAMLManifestParser +from debputy.manifest_parser.exceptions import ManifestParseException +from debputy.plugin.api.test_api import build_virtual_file_system + + +def normalize_doc_link(message) -> str: + return message.replace(DEBPUTY_DOC_ROOT_DIR, "{{DEBPUTY_DOC_ROOT_DIR}}") + + +@pytest.fixture() +def manifest_parser_pkg_foo( + amd64_dpkg_architecture_variables, + dpkg_arch_query, + source_package, + package_single_foo_arch_all_cxt_amd64, + amd64_substitution, + no_profiles_or_build_options, + debputy_plugin_feature_set, +) -> YAMLManifestParser: + # We need an empty directory to avoid triggering packager provided files. + debian_dir = build_virtual_file_system([]) + return YAMLManifestParser( + "debian/test-debputy.manifest", + source_package, + package_single_foo_arch_all_cxt_amd64, + amd64_substitution, + amd64_dpkg_architecture_variables, + dpkg_arch_query, + no_profiles_or_build_options, + debputy_plugin_feature_set, + debian_dir=debian_dir, + ) + + +def test_parsing_no_manifest(manifest_parser_pkg_foo): + manifest = manifest_parser_pkg_foo.build_manifest() + + assert [p.name for p in manifest.all_packages] == ["foo"] + assert [p.name for p in manifest.active_packages] == ["foo"] + + +def test_parsing_version_only(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + """ + ) + + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + + assert [p.name for p in manifest.all_packages] == ["foo"] + assert [p.name for p in manifest.active_packages] == ["foo"] + + +def test_parsing_variables(manifest_parser_pkg_foo): + # https://salsa.debian.org/debian/debputy/-/issues/58 + content = textwrap.dedent( + """\ + manifest-version: '0.1' + definitions: + variables: + LIBPATH: "/usr/lib/{{DEB_HOST_MULTIARCH}}" + SONAME: "1" + LETTER_O: "o" + installations: + - install: + source: build/libfoo.so.{{SONAME}} + dest-dir: "{{LIBPATH}}" + into: f{{LETTER_O}}{{LETTER_O}} + packages: + f{{LETTER_O}}{{LETTER_O}}: + transformations: + - create-symlink: + path: "{{LIBPATH}}/libfoo.so.{{SONAME}}.0.0" + target: "{{LIBPATH}}/libfoo.so.{{SONAME}}" + """ + ) + manifest_parser_pkg_foo.parse_manifest(fd=content) + # TODO: Verify that the substitution is applied correctly throughout + # (currently, the test just verifies that we do not reject the manifest) + + +@pytest.mark.parametrize( + "varname", + [ + "PACKAGE", + "DEB_HOST_ARCH", + "DEB_BLAH_ARCH", + "env:UNEXISTING", + "token:TAB", + ], +) +def test_parsing_variables_reserved(manifest_parser_pkg_foo, varname): + content = textwrap.dedent( + f"""\ + manifest-version: '0.1' + definitions: + variables: + '{varname}': "test" + """ + ) + + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + msg = f'The variable "{varname}" is already reserved/defined. Error triggered by definitions.variables.{varname}.' + assert normalize_doc_link(e_info.value.args[0]) == msg + + +def test_parsing_variables_interdependent_ok(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + definitions: + variables: + DOC_PATH: "/usr/share/doc/foo" + EXAMPLE_PATH: "{{DOC_PATH}}/examples" + installations: + - install: + source: foo.example + dest-dir: "{{EXAMPLE_PATH}}" + """ + ) + + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + resolved = manifest.substitution.substitute("{{EXAMPLE_PATH}}", "test") + assert resolved == "/usr/share/doc/foo/examples" + + +def test_parsing_variables_unused(manifest_parser_pkg_foo): + content = textwrap.dedent( + f"""\ + manifest-version: '0.1' + definitions: + variables: + UNUSED: "test" + """ + ) + + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + msg = ( + 'The variable "UNUSED" is unused. Either use it or remove it.' + " The variable was declared at definitions.variables.UNUSED." + ) + assert normalize_doc_link(e_info.value.args[0]) == msg + + +def test_parsing_package_foo_empty(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + """ + ) + + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + msg = ( + "The attribute packages.foo must be a non-empty mapping. Please see" + " {{DEBPUTY_DOC_ROOT_DIR}}/MANIFEST-FORMAT.md#binary-package-rules for the documentation." + ) + assert normalize_doc_link(e_info.value.args[0]) == msg + + +def test_parsing_package_bar_empty(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + bar: + """ + ) + + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + assert 'package "bar" is not present' in e_info.value.args[0] + + +def test_transformations_no_list(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + create-symlinks: + path: a + target: b + """ + ) + + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + assert "packages.foo.transformations" in e_info.value.args[0] + assert "must be a list" in e_info.value.args[0] + + +def test_create_symlinks_missing_path(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - create-symlink: + target: b + """ + ) + + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + msg = ( + "The following keys were required but not present at packages.foo.transformations[0].create-symlink: 'path'" + " (Documentation: " + "{{DEBPUTY_DOC_ROOT_DIR}}/MANIFEST-FORMAT.md#create-symlinks-transformation-rule-create-symlink)" + ) + assert normalize_doc_link(e_info.value.args[0]) == msg + + +def test_create_symlinks_unknown_replacement_rule(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - create-symlink: + path: usr/share/foo + target: /usr/share/bar + replacement-rule: golf + """ + ) + + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + msg = ( + 'The attribute "packages.foo.transformations[0].create-symlink.replacement-rule "' + " did not have a valid structure/type: Value (golf) must be one of the following literal values:" + ' "error-if-exists", "error-if-directory", "abort-on-non-empty-directory", "discard-existing"' + ) + assert normalize_doc_link(e_info.value.args[0]) == msg + + +def test_create_symlinks_missing_target(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - create-symlink: + path: a + """ + ) + + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + msg = ( + "The following keys were required but not present at packages.foo.transformations[0].create-symlink: 'target'" + " (Documentation: " + "{{DEBPUTY_DOC_ROOT_DIR}}/MANIFEST-FORMAT.md#create-symlinks-transformation-rule-create-symlink)" + ) + assert normalize_doc_link(e_info.value.args[0]) == msg + + +def test_create_symlinks_not_normalized_path(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - create-symlink: + path: ../bar + target: b + """ + ) + + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + expected = ( + 'The path "../bar" provided in packages.foo.transformations[0].create-symlink.path ' + ' should be relative to the root of the package and not use any ".." or "." segments.' + ) + assert e_info.value.args[0] == expected + + +def test_unresolvable_subst_in_source_context(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + installations: + - install: + source: "foo.sh" + as: "usr/bin/{{PACKAGE}}" + """ + ) + + with pytest.raises(DebputySubstitutionError) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + expected = ( + "The variable {{PACKAGE}} is not available while processing installations[0].install.as" + " ." + ) + + assert e_info.value.args[0] == expected + + +def test_yaml_error_duplicate_key(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - create-symlink: + path: ../bar + target: b + # Duplicate key error + path: ../foo + """ + ) + + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + assert "duplicate key" in e_info.value.args[0] + + +def test_yaml_error_tab_start(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - create-symlink: + path: ../bar + target: b + # Tab is not allowed here in this case. + \ta + """ + ) + + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + assert "'\\t' that cannot start any token" in e_info.value.args[0] + + +def test_yaml_octal_mode_int(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + transformations: + - path-metadata: + path: usr/share/bar + mode: 0755 + """ + ) + + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) + + msg = ( + 'The attribute "packages.foo.transformations[0].path-metadata.mode " did not' + " have a valid structure/type: The attribute must be a FileSystemMode (string)" + ) + + assert e_info.value.args[0] == msg + + +def test_yaml_clean_after_removal(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + clean-after-removal: + - /var/log/foo/*.log + - /var/log/foo/*.log.gz + - path: /var/log/foo/ + ignore-non-empty-dir: true + - /etc/non-conffile-configuration.conf + - path: /var/cache/foo + recursive: true + + """ + ) + + manifest_parser_pkg_foo.parse_manifest(fd=content) + + +def test_binary_version(manifest_parser_pkg_foo): + content = textwrap.dedent( + """\ + manifest-version: '0.1' + packages: + foo: + binary-version: 1:2.3 + + """ + ) + + manifest = manifest_parser_pkg_foo.parse_manifest(fd=content) + assert manifest.package_state_for("foo").binary_version == "1:2.3" + + +@pytest.mark.parametrize( + "path,is_accepted", + [ + ("usr/bin/foo", False), + ("var/cache/foo*", False), + ("var/cache/foo", True), + ("var/cache/foo/", True), + ("var/cache/foo/*", True), + ("var/cache/foo/*.*", True), + ("var/cache/foo/*.txt", True), + ("var/cache/foo/cache.*", True), + ("etc/foo*", False), + ("etc/foo/*", True), + ("etc/foo/", True), + # /var/log is special-cased + ("/var/log/foo*", True), + ("/var/log/foo/*.*", True), + ("/var/log/foo/", True), + # Unsupported pattern at the time of writing + ("/var/log/foo/*.*.*", False), + # Questionable rules + ("*", False), + ("*.la", False), + ("*/foo/*", False), + ], +) +def test_yaml_clean_after_removal_unsafe_path( + manifest_parser_pkg_foo, + path: str, + is_accepted: bool, +) -> None: + content = textwrap.dedent( + f"""\ + manifest-version: '0.1' + packages: + foo: + clean-after-removal: + - {path} + """ + ) + + if is_accepted: + manifest_parser_pkg_foo.parse_manifest(fd=content) + else: + with pytest.raises(ManifestParseException) as e_info: + manifest_parser_pkg_foo.parse_manifest(fd=content) diff --git a/tests/test_path.py b/tests/test_path.py new file mode 100644 index 0000000..a9e826e --- /dev/null +++ b/tests/test_path.py @@ -0,0 +1,47 @@ +from typing import cast + +import pytest + +from debputy.exceptions import SymlinkLoopError +from debputy.filesystem_scan import VirtualPathBase +from debputy.plugin.api import virtual_path_def +from debputy.plugin.api.test_api import build_virtual_file_system + + +def test_symlink_lookup() -> None: + fs = build_virtual_file_system( + [ + virtual_path_def("./usr/share/doc/bar", link_target="foo"), + "./usr/share/doc/foo/copyright", + virtual_path_def("./usr/share/bar/data", link_target="../foo/data"), + "./usr/share/foo/data/foo.dat", + virtual_path_def("./usr/share/baz/data", link_target="/usr/share/foo/data"), + virtual_path_def( + "./usr/share/test/loop-a", link_target="/usr/share/test/loop-b" + ), + virtual_path_def("./usr/share/test/loop-b", link_target="./loop-c"), + virtual_path_def("./usr/share/test/loop-c", link_target="../test/loop-a"), + ] + ) + assert fs.lookup("/usr/share/doc/bar/copyright") is not None + assert fs.lookup("/usr/share/bar/data/foo.dat") is not None + assert fs.lookup("/usr/share/baz/data/foo.dat") is not None + + vp_fs: VirtualPathBase = cast("VirtualPathBase", fs) + p, missing = vp_fs.attempt_lookup("/usr/share/doc/foo/non-existent") + assert p.path == "./usr/share/doc/foo" + assert missing == ["non-existent"] + + p, missing = vp_fs.attempt_lookup("/usr/share/bar/data/non-existent") + assert p.path == "./usr/share/foo/data" + assert missing == ["non-existent"] + + p, missing = vp_fs.attempt_lookup("/usr/share/baz/data/non-existent") + assert p.path == "./usr/share/foo/data" + assert missing == ["non-existent"] + + # The symlink can be looked up + assert fs.lookup("./usr/share/test/loop-a") is not None + with pytest.raises(SymlinkLoopError): + # But resolving it will cause issues + fs.lookup("./usr/share/test/loop-a/") diff --git a/tests/test_plugin_tester.py b/tests/test_plugin_tester.py new file mode 100644 index 0000000..8078a02 --- /dev/null +++ b/tests/test_plugin_tester.py @@ -0,0 +1,285 @@ +import json +import os.path +from typing import List, Tuple, Type, cast + +import pytest + +from debputy.exceptions import DebputyFSIsROError +from debputy.plugin.api import ( + DebputyPluginInitializer, + BinaryCtrlAccessor, + PackageProcessingContext, + VirtualPath, + virtual_path_def, +) +from debputy.exceptions import PluginConflictError, PluginAPIViolationError +from debputy.plugin.api.impl import DebputyPluginInitializerProvider +from debputy.plugin.api.impl_types import automatic_discard_rule_example +from debputy.plugin.api.test_api import ( + build_virtual_file_system, + package_metadata_context, + initialize_plugin_under_test, +) +from debputy.plugin.api.test_api.test_impl import ( + initialize_plugin_under_test_preloaded, + initialize_plugin_under_test_from_inline_json, +) + +CUSTOM_PLUGIN_JSON_FILE = os.path.join( + os.path.dirname(__file__), "data", "custom-plugin.json.in" +) + + +def bad_metadata_detector_fs_rw( + path: VirtualPath, + _ctrl: BinaryCtrlAccessor, + _context: PackageProcessingContext, +) -> None: + del path["foo"] + + +def conflicting_plugin(api: DebputyPluginInitializer) -> None: + api.packager_provided_file( + "logrotate", + "/some/where/that/is/not/etc/logrotate.d/{name}", + ) + + +def bad_plugin(api: DebputyPluginInitializer) -> None: + api.metadata_or_maintscript_detector("fs_rw", bad_metadata_detector_fs_rw) + + +def adr_inconsistent_example_plugin(api: DebputyPluginInitializerProvider) -> None: + api.automatic_discard_rule( + "adr-example-test", + lambda p: p.name == "discard-me", + examples=automatic_discard_rule_example( + "foo/discard-me", + ("bar/discard-me", False), + "baz/something", + ("discard-me/foo", False), + ), + ) + + +def test_conflict_with_debputy(): + with pytest.raises(PluginConflictError) as e_info: + initialize_plugin_under_test_preloaded( + 1, + conflicting_plugin, + plugin_name="conflicting-plugin", + ) + message = ( + 'The stem "logrotate" is registered twice for packager provided files.' + " Once by debputy and once by conflicting-plugin" + ) + assert message == e_info.value.args[0] + + +def test_metadata_read_only(): + plugin = initialize_plugin_under_test_preloaded( + 1, + bad_plugin, + plugin_name="bad-plugin", + ) + fs = build_virtual_file_system(["./foo"]) + with pytest.raises(PluginAPIViolationError) as e_info: + plugin.run_metadata_detector("fs_rw", fs) + + assert isinstance(e_info.value.__cause__, DebputyFSIsROError) + + +def test_packager_provided_files(): + plugin = initialize_plugin_under_test(plugin_desc_file=CUSTOM_PLUGIN_JSON_FILE) + assert plugin.packager_provided_files_by_stem().keys() == { + "my-file", + "test-file-from-json", + } + my_file = [p for p in plugin.packager_provided_files() if p.stem == "my-file"][0] + + assert my_file.stem == "my-file" + assert my_file.compute_dest("g++-3.1")[1] == "g__-3.1.conf" + + +def test_path_checks(): + symlink_path = "./foo" + with pytest.raises(ValueError) as e_info: + virtual_path_def(symlink_path, link_target="/bar", mode=0o0755) + assert ( + e_info.value.args[0] + == f'Please do not provide mode for symlinks. Triggered by "{symlink_path}"' + ) + with pytest.raises(ValueError) as e_info: + virtual_path_def(symlink_path + "/", link_target="/bar") + msg = ( + "Path name looks like a directory, but a symlink target was also provided." + f' Please remove the trailing slash OR the symlink_target. Triggered by "{symlink_path}/"' + ) + assert e_info.value.args[0] == msg + + +def test_metadata_detector_applies_to_check(): + plugin_name = "custom-plugin" + metadata_detector_id = "udeb-only" + plugin = initialize_plugin_under_test(plugin_desc_file=CUSTOM_PLUGIN_JSON_FILE) + with pytest.raises(ValueError) as e_info: + plugin.run_metadata_detector( + metadata_detector_id, + build_virtual_file_system(["./usr/share/doc/foo/copyright"]), + ) + msg = f'The detector "{metadata_detector_id}" from {plugin_name} does not apply to the given package.' + assert e_info.value.args[0].startswith(msg) + + metadata = plugin.run_metadata_detector( + metadata_detector_id, + build_virtual_file_system(["./usr/share/doc/foo/copyright"]), + context=package_metadata_context(package_fields={"Package-Type": "udeb"}), + ) + assert metadata.substvars["Test:Udeb-Metadata-Detector"] == "was-run" + + +@pytest.mark.parametrize( + "variables,exec_type", + [ + ( + [("DEBPUTY_VAR", "RESERVED")], + ValueError, + ), + ( + [("_DEBPUTY_VAR", "RESERVED")], + ValueError, + ), + ( + [("_FOO", "RESERVED")], + ValueError, + ), + ( + [("path:_var", "RESERVED")], + ValueError, + ), + ( + [("path:DEBPUTY_VAR", "RESERVED")], + ValueError, + ), + ( + [("DEB_VAR", "RESERVED")], + ValueError, + ), + ( + [("DPKG_VAR", "RESERVED")], + ValueError, + ), + ( + [("PACKAGE", "RESERVED")], + ValueError, + ), + ( + [("foo:var", "RESERVED")], + ValueError, + ), + ( + [("env:var", "RESERVED")], + ValueError, + ), + ( + [("SOURCE_DATE_EPOCH", "RESERVED")], + ValueError, + ), + ( + [("!MY_VAR", "INVALID_NAME")], + ValueError, + ), + ( + [("VALUE_DEPENDS_ON_VAR", "{{UNKNOWN_VAR}}")], + ValueError, + ), + ( + [("VALUE_DEPENDS_ON_VAR", "{{DEB_HOST_ARCH}}")], + ValueError, + ), + ( + [("DEFINED_TWICE", "ONCE"), ("DEFINED_TWICE", "TWICE")], + PluginConflictError, + ), + ], +) +def test_invalid_manifest_variables( + variables: List[Tuple[str, str]], + exec_type: Type[Exception], +) -> None: + def _initializer(api: DebputyPluginInitializer): + with pytest.raises(exec_type): + for varname, value in variables: + api.manifest_variable(varname, value) + + initialize_plugin_under_test_preloaded( + 1, + _initializer, + plugin_name="test-plugin", + load_debputy_plugin=False, + ) + + +def test_valid_manifest_variables() -> None: + variables = { + "PLUGIN_VAR": "TEST VALUE", + "ANOTHER_PLUGIN_VAR": "ANOTHER VALUE", + "path:SOMEWHERE_DIR": "/usr/share/some-where", + } + + def _initializer(api: DebputyPluginInitializer): + for k, v in variables.items(): + api.manifest_variable(k, v) + + plugin = initialize_plugin_under_test_preloaded( + 1, + _initializer, + plugin_name="test-plugin", + load_debputy_plugin=False, + ) + + assert plugin.declared_manifest_variables == variables.keys() + + +def test_valid_manifest_variables_json() -> None: + variables = { + "PLUGIN_VAR": "TEST VALUE", + "ANOTHER_PLUGIN_VAR": "ANOTHER VALUE", + "path:SOMEWHERE_DIR": "/usr/share/some-where", + } + content = { + "api-compat-version": 1, + "manifest-variables": [ + { + "name": k, + "value": v, + } + for k, v in variables.items() + ], + } + plugin = initialize_plugin_under_test_from_inline_json( + "test-plugin", + json.dumps(content), + ) + assert plugin.declared_manifest_variables == variables.keys() + + +def test_automatic_discard_rules_example() -> None: + plugin = initialize_plugin_under_test_preloaded( + 1, + # Internal API used + cast("PluginInitializationEntryPoint", adr_inconsistent_example_plugin), + # API is restricted + plugin_name="debputy", + load_debputy_plugin=False, + ) + issues = plugin.automatic_discard_rules_examples_with_issues() + assert len(issues) == 1 + issue = issues[0] + assert issue.name == "adr-example-test" + assert issue.example_index == 0 + assert set(issue.inconsistent_paths) == { + "/discard-me/foo", + "/bar/discard-me", + "/baz/something", + } diff --git a/tests/test_substitute.py b/tests/test_substitute.py new file mode 100644 index 0000000..a83cc7f --- /dev/null +++ b/tests/test_substitute.py @@ -0,0 +1,66 @@ +import pytest + +from debputy.architecture_support import faked_arch_table +from debputy.dh_migration.models import ( + DHMigrationSubstitution, + AcceptableMigrationIssues, + FeatureMigration, +) +from debputy.filesystem_scan import FSROOverlay +from debputy.highlevel_manifest import MutableYAMLManifest +from debputy.substitution import SubstitutionImpl, VariableContext + +MOCK_ENV = { + # This conflicts with the dpkg arch table intentionally (to ensure we can tell which one is being resolved) + "DEB_HOST_ARCHITECTURE": "i386", +} +MOCK_DPKG_ARCH_TABLE = faked_arch_table("amd64", build_arch="i386") +MOCK_VARIABLE_CONTEXT = VariableContext(FSROOverlay.create_root_dir("debian", "debian")) + + +@pytest.mark.parametrize( + "value,expected", + [ + ( + "unchanged", + "unchanged", + ), + ( + "unchanged\\{{\n}}", + "unchanged\\{{\n}}", + ), # Newline is not an allowed part of a substitution + ( + "{{token:DOUBLE_OPEN_CURLY_BRACE}}{{token:NL}}{{token:DOUBLE_CLOSE_CURLY_BRACE}}", + "{{\n}}", + ), + ( + "{{token:DOUBLE_OPEN_CURLY_BRACE}}token:TAB}}{{token:TAB{{token:DOUBLE_CLOSE_CURLY_BRACE}}", + "{{token:TAB}}{{token:TAB}}", + ), + ( + "/usr/lib/{{DEB_HOST_MULTIARCH}}", + f'/usr/lib/{MOCK_DPKG_ARCH_TABLE["DEB_HOST_MULTIARCH"]}', + ), + ], +) +def test_substitution_match(debputy_plugin_feature_set, value, expected) -> None: + subst = SubstitutionImpl( + plugin_feature_set=debputy_plugin_feature_set, + dpkg_arch_table=MOCK_DPKG_ARCH_TABLE, + environment=MOCK_ENV, + variable_context=MOCK_VARIABLE_CONTEXT, + ) + replacement = subst.substitute(value, "test def") + assert replacement == expected + + +def test_migrate_substitution() -> None: + feature_migration = FeatureMigration("test migration") + subst = DHMigrationSubstitution( + MOCK_DPKG_ARCH_TABLE, + AcceptableMigrationIssues(frozenset()), + feature_migration, + MutableYAMLManifest({}), + ) + replacement = subst.substitute("usr/lib/${DEB_HOST_MULTIARCH}/foo", "test def") + assert replacement == "usr/lib/{{DEB_HOST_MULTIARCH}}/foo" diff --git a/tests/test_symbolic_mode.py b/tests/test_symbolic_mode.py new file mode 100644 index 0000000..a0ad81d --- /dev/null +++ b/tests/test_symbolic_mode.py @@ -0,0 +1,25 @@ +import pytest + +from debputy.manifest_parser.base_types import SymbolicMode + + +@pytest.mark.parametrize( + "base_mode,is_dir,symbolic_mode,expected", + [ + (0o0644, False, "u+rwX,og=rX", 0o0644), + (0o0000, False, "u+rwX,og=rX", 0o0644), + (0o0400, True, "u+rwX,og=rX", 0o0755), + (0o0000, True, "u+rwX,og=rX", 0o0755), + (0o2400, False, "u+rwxs,og=rx", 0o04755), + (0o7400, False, "u=rwX,og=rX", 0o0644), + (0o0641, False, "u=rwX,og=rX", 0o0755), + (0o4755, False, "a-x", 0o04644), + ], +) +def test_generate_deb_filename( + attribute_path, base_mode, is_dir, symbolic_mode, expected +): + print(attribute_path.path) + parsed_mode = SymbolicMode.parse_filesystem_mode(symbolic_mode, attribute_path) + actual = parsed_mode.compute_mode(base_mode, is_dir) + assert oct(actual)[2:] == oct(expected)[2:] diff --git a/tests/test_symlink_normalization.py b/tests/test_symlink_normalization.py new file mode 100644 index 0000000..0b11dc8 --- /dev/null +++ b/tests/test_symlink_normalization.py @@ -0,0 +1,35 @@ +import pytest + +from debputy.util import debian_policy_normalize_symlink_target + + +@pytest.mark.parametrize( + "link_path,link_target,expected", + [ + ("usr/share/doc/pkg/my-symlink", "/etc/foo.conf", "/etc/foo.conf"), + ("usr/share/doc/pkg/my-symlink", "/usr/share/doc/pkg", "."), + ("usr/share/doc/pkg/my-symlink", "/usr/share/doc/pkg/.", "."), + ("usr/share/doc/pkg/my-symlink", "/usr/share/bar/../doc/pkg/.", "."), + ( + "usr/share/doc/pkg/my-symlink", + "/usr/share/bar/../doc/pkg/../other-pkg", + "../other-pkg", + ), + ("usr/share/doc/pkg/my-symlink", "/usr/share/doc/other-pkg/.", "../other-pkg"), + ("usr/share/doc/pkg/my-symlink", "../other-pkg/.", "../other-pkg"), + ("usr/share/doc/pkg/my-symlink", "/usr/share/doc/other-pkg", "../other-pkg"), + ("usr/share/doc/pkg/my-symlink", "../other-pkg", "../other-pkg"), + ( + "usr/share/doc/pkg/my-symlink", + "/usr/share/doc/pkg/../../../../etc/foo.conf", + "/etc/foo.conf", + ), + ], +) +def test_symlink_normalization(link_path: str, link_target: str, expected: str) -> None: + actual = debian_policy_normalize_symlink_target( + link_path, + link_target, + normalize_link_path=True, + ) + assert actual == expected diff --git a/tests/tutil.py b/tests/tutil.py new file mode 100644 index 0000000..9b622b9 --- /dev/null +++ b/tests/tutil.py @@ -0,0 +1,66 @@ +from typing import Tuple, Mapping + +from debian.deb822 import Deb822 +from debian.debian_support import DpkgArchTable + +from debputy.architecture_support import ( + faked_arch_table, + DpkgArchitectureBuildProcessValuesTable, +) +from debputy.packages import BinaryPackage + +_DPKG_ARCHITECTURE_TABLE_NATIVE_AMD64 = None +_DPKG_ARCH_QUERY_TABLE = None + + +def faked_binary_package( + package, architecture="any", section="misc", is_main_package=False, **fields +) -> BinaryPackage: + _arch_data_tables_loaded() + + dpkg_arch_table, dpkg_arch_query = _arch_data_tables_loaded() + return BinaryPackage( + Deb822( + { + "Package": package, + "Architecture": architecture, + "Section": section, + **fields, + } + ), + dpkg_arch_table, + dpkg_arch_query, + is_main_package=is_main_package, + ) + + +def binary_package_table(*args: BinaryPackage) -> Mapping[str, BinaryPackage]: + packages = list(args) + if not any(p.is_main_package for p in args): + p = args[0] + np = faked_binary_package( + p.name, + architecture=p.declared_architecture, + section=p.archive_section, + is_main_package=True, + **{ + k: v + for k, v in p.fields.items() + if k.lower() not in ("package", "architecture", "section") + }, + ) + packages[0] = np + return {p.name: p for p in packages} + + +def _arch_data_tables_loaded() -> ( + Tuple[DpkgArchitectureBuildProcessValuesTable, DpkgArchTable] +): + global _DPKG_ARCHITECTURE_TABLE_NATIVE_AMD64 + global _DPKG_ARCH_QUERY_TABLE + if _DPKG_ARCHITECTURE_TABLE_NATIVE_AMD64 is None: + _DPKG_ARCHITECTURE_TABLE_NATIVE_AMD64 = faked_arch_table("amd64") + if _DPKG_ARCH_QUERY_TABLE is None: + # TODO: Make a faked table instead, so we do not have data dependencies in the test. + _DPKG_ARCH_QUERY_TABLE = DpkgArchTable.load_arch_table() + return _DPKG_ARCHITECTURE_TABLE_NATIVE_AMD64, _DPKG_ARCH_QUERY_TABLE -- cgit v1.2.3